microsoft--graphrag/packages/graphrag-chunking/example_notebooks/token_chunking_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41bf35b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (c) 2026 Microsoft Corporation.\n",
    "# Licensed under the MIT License."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c59c030",
   "metadata": {},
   "source": [
    "## Token chunking example\n",
    "\n",
    "The TokenChunker splits text into fixed-size chunks based on token count rather than sentence boundaries. It uses a tokenizer to encode text into tokens, then creates chunks of a specified size with configurable overlap between chunks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd653d39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TextChunk(original='This is a', text='This is a', index=0, start_char=0, end_char=8, token_count=3), TextChunk(original=' random test fragment', text=' random test fragment', index=1, start_char=9, end_char=29, token_count=3), TextChunk(original=' of some text', text=' of some text', index=2, start_char=30, end_char=42, token_count=3)]\n"
     ]
    }
   ],
   "source": [
    "import tiktoken\n",
    "from graphrag_chunking.token_chunker import TokenChunker\n",
    "\n",
    "tokenizer = tiktoken.get_encoding(\"o200k_base\")\n",
    "chunker = TokenChunker(\n",
    "    size=3, overlap=0, encode=tokenizer.encode, decode=tokenizer.decode\n",
    ")\n",
    "chunks = chunker.chunk(\"This is a random test fragment of some text\")\n",
    "print(chunks)  # [\"This is a\", \" random test fragment\", \" of some text\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}