microsoft--graphrag/packages/graphrag-llm/notebooks/02_encoding_decoding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "578551ee",
   "metadata": {},
   "source": [
    "# Encoding/Decoding\n",
    "\n",
    "`LLMCompletion` and `LLMEmbedding` expose a `Tokenizer` property corresponding to the underlying model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "986a0bad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded tokens: [9906, 11, 1917, 0]\n",
      "Number of tokens: 4\n",
      "Number of tokens: 4\n",
      "Decoded text: Hello, world!\n"
     ]
    }
   ],
   "source": [
    "# Copyright (c) 2024 Microsoft Corporation.\n",
    "# Licensed under the MIT License\n",
    "\n",
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "from graphrag_llm.completion import LLMCompletion, create_completion\n",
    "from graphrag_llm.config import AuthMethod, ModelConfig\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "api_key = os.getenv(\"GRAPHRAG_API_KEY\")\n",
    "model_config = ModelConfig(\n",
    "    model_provider=\"azure\",\n",
    "    model=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
    "    azure_deployment_name=os.getenv(\"GRAPHRAG_MODEL\", \"gpt-4o\"),\n",
    "    api_base=os.getenv(\"GRAPHRAG_API_BASE\"),\n",
    "    api_version=os.getenv(\"GRAPHRAG_API_VERSION\", \"2025-04-01-preview\"),\n",
    "    api_key=api_key,\n",
    "    auth_method=AuthMethod.AzureManagedIdentity if not api_key else AuthMethod.ApiKey,\n",
    ")\n",
    "llm_completion: LLMCompletion = create_completion(model_config)\n",
    "\n",
    "encoded = llm_completion.tokenizer.encode(\"Hello, world!\")\n",
    "print(f\"Encoded tokens: {encoded}\")\n",
    "print(f\"Number of tokens: {len(encoded)}\")\n",
    "# OR\n",
    "print(f\"Number of tokens: {llm_completion.tokenizer.num_tokens('Hello, world!')}\")\n",
    "decoded = llm_completion.tokenizer.decode(encoded)\n",
    "print(f\"Decoded text: {decoded}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e4a7515",
   "metadata": {},
   "source": [
    "## Standalone Tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5920cf74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded tokens: [9906, 11, 1917, 0]\n",
      "Number of tokens: 4\n",
      "Decoded text: Hello, world!\n"
     ]
    }
   ],
   "source": [
    "from graphrag_llm.config import TokenizerConfig, TokenizerType\n",
    "from graphrag_llm.tokenizer import create_tokenizer\n",
    "\n",
    "tokenizer = create_tokenizer(\n",
    "    TokenizerConfig(\n",
    "        type=TokenizerType.LiteLLM,\n",
    "        model_id=\"openai/text-embedding-3-small\",\n",
    "    )\n",
    ")\n",
    "\n",
    "encoded = tokenizer.encode(\"Hello, world!\")\n",
    "print(f\"Encoded tokens: {encoded}\")\n",
    "print(f\"Number of tokens: {len(encoded)}\")\n",
    "decoded = tokenizer.decode(encoded)\n",
    "print(f\"Decoded text: {decoded}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "115f63b9",
   "metadata": {},
   "source": [
    "## Tiktoken\n",
    "\n",
    "By default, `LLMCompletion` and `LLMEmbedding` use a litellm based tokenizer that supports the 100+ models that litellm supports but you may use a tiktoken based tokenizer by specifying a tokenizer type of `TokenizerType.Tiktoken` and providing an `encoding_name` to the config.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "abeb9753",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded tokens: [13225, 11, 2375, 0]\n",
      "Encoded tokens: [13225, 11, 2375, 0]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = create_tokenizer(\n",
    "    TokenizerConfig(\n",
    "        type=TokenizerType.Tiktoken,\n",
    "        encoding_name=\"o200k_base\",\n",
    "    )\n",
    ")\n",
    "encoded = tokenizer.encode(\"Hello, world!\")\n",
    "print(f\"Encoded tokens: {encoded}\")\n",
    "\n",
    "# Using with LLMCompletion\n",
    "llm_completion: LLMCompletion = create_completion(model_config, tokenizer=tokenizer)\n",
    "\n",
    "encoded = llm_completion.tokenizer.encode(\"Hello, world!\")\n",
    "print(f\"Encoded tokens: {encoded}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}