fix(decode): prevent false array header detection for unquoted keys c… (#314)

* fix(decode): prevent false array header detection for unquoted keys containing brackets and colons

* test(decode): pin stream parity for an unquoted bracket-colon scalar

* chore(toon): bump `@toon-format/spec` to ^3.3.2

---------

Co-authored-by: Johann Schopplich <johann@schopplich.com>

.github/workflows/ci.yml

@@ -4,32 +4,55 @@ on:
   push:
     branches:
       - main
-
   pull_request:
     branches:
       - main
 
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 24
-          cache: pnpm
-      - run: pnpm install
-      - run: pnpm run lint
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
-  typecheck:
-    runs-on: ubuntu-latest
+permissions:
+  contents: read
+
+jobs:
+  ci:
+    runs-on: ubuntu-slim
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v4
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
           node-version: 24
-          cache: pnpm
-      - run: pnpm install
-      - run: pnpm run test:types
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm dependencies
+        uses: actions/cache@v5
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Lint
+        run: pnpm run lint
+
+      - name: Typecheck
+        run: pnpm run test:types
+
+      - name: Test
+        run: pnpm run test

.github/workflows/deploy.yml

@@ -0,0 +1,58 @@
+name: Deploy Docs
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - automd.config.ts
+      - package.json
+      - eslint.config.mjs
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  deploy:
+    name: Deploy Docs
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm dependencies
+        uses: actions/cache@v5
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build docs
+        run: pnpm run docs:build
+
+      - name: Deploy to Cloudflare
+        run: cd docs && npx wrangler deploy
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}

.github/workflows/pr-title.yml

@@ -0,0 +1,36 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    types: [opened, edited]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-pr-title:
+    name: Lint PR title
+    runs-on: ubuntu-slim
+
+    if: ${{ (github.event.action == 'opened' || github.event.changes.title != null) && github.actor != 'renovate[bot]' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          # Only fetch the config file from the repository
+          sparse-checkout-cone-mode: false
+          sparse-checkout: commitlint.config.ts
+
+      - name: Install dependencies
+        run: npm install -D @commitlint/cli @commitlint/config-conventional
+
+      - name: Validate PR title with commitlint
+        run: echo "$PR_TITLE" | npx commitlint
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}

.github/workflows/release.yml

@@ -1,37 +1,52 @@
-name: Release
+name: Release + Publish
 
 on:
   push:
     tags:
       - 'v*'
 
-permissions:
-  id-token: write
-  contents: write
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    name: Release
+    runs-on: ubuntu-slim
+
+    permissions:
+      id-token: write
+      contents: write
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v6
         with:
-          fetch-depth: 0
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v4
+          fetch-depth: 0 # Required for fetching tags and generating release notes
+          persist-credentials: true
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
           node-version: 24
           registry-url: https://registry.npmjs.org/
           cache: pnpm
 
-      - name: Publish changelog
+      - name: Generate changelog and create GitHub release
         run: npx changelogithub
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - run: pnpm install
-      - run: pnpm run build
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
 
-      - name: Publish to npm
-        run: npm install -g npm@latest && npm publish --access public
+      - name: Build packages
+        run: pnpm run build
+
+      - name: Publish packages to npm
+        run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -2,3 +2,7 @@ dist
 node_modules
 .DS_Store
 .env
+docs/.vitepress/dist
+docs/.vitepress/cache
+packages/toon/test/fixtures/*.json
+packages/toon/test/fixtures/*.toon

.npmrc

@@ -1 +0,0 @@
-shamefully-hoist=true

.vscode/settings.json

@@ -1,7 +1,4 @@
 {
-  // Enable the ESLint flat config support
-  "eslint.useFlatConfig": true,
-
   // Disable the default formatter, use ESLint instead
   "prettier.enable": false,
   "editor.formatOnSave": false,
@@ -37,6 +34,17 @@
     "markdown",
     "json",
     "jsonc",
-    "yaml"
+    "yaml",
+    "toml",
+    "xml",
+    "gql",
+    "graphql",
+    "astro",
+    "svelte",
+    "css",
+    "less",
+    "scss",
+    "pcss",
+    "postcss"
   ]
 }

README.md

@@ -1,914 +0,0 @@
-![TOON logo with step‑by‑step guide](./.github/og.png)
-
-# Token-Oriented Object Notation (TOON)
-
-**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
-
-TOON's sweet spot is **uniform arrays of objects** – multiple fields per row, same structure across items. It borrows YAML's indentation-based structure for nested objects and CSV's tabular format for uniform data rows, then optimizes both for token efficiency in LLM contexts. For deeply nested or non-uniform data, JSON may be more efficient.
-
-## Why TOON?
-
-AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – and standard JSON is verbose and token-expensive:
-
-```json
-{
-  "users": [
-    { "id": 1, "name": "Alice", "role": "admin" },
-    { "id": 2, "name": "Bob", "role": "user" }
-  ]
-}
-```
-
-TOON conveys the same information with **fewer tokens**:
-
-```
-users[2]{id,name,role}:
-  1,Alice,admin
-  2,Bob,user
-```
-
-<details>
-<summary>Another reason</summary>
-
-[![xkcd: Standards](https://imgs.xkcd.com/comics/standards_2x.png)](https://xkcd.com/927/)
-
-</details>
-
-## Key Features
-
-- 💸 **Token-efficient:** typically 30–60% fewer tokens than JSON
-- 🤿 **LLM-friendly guardrails:** explicit lengths and field lists help models validate output
-- 🍱 **Minimal syntax:** removes redundant punctuation (braces, brackets, most quotes)
-- 📐 **Indentation-based structure:** replaces braces with whitespace for better readability
-- 🧺 **Tabular arrays:** declare keys once, then stream rows without repetition
-
-## Benchmarks
-
-The benchmarks test datasets that favor TOON's strengths (uniform tabular data). Real-world performance depends heavily on your data structure.
-
-<!-- automd:file src="./benchmarks/results/token-efficiency.md" -->
-
-### Token Efficiency
-
-```
-⭐ GitHub Repositories       ██████████████░░░░░░░░░░░   8,745 tokens
-                             vs JSON: 15,145  (-42.3%)
-                             vs YAML: 13,129  (-33.4%)
-                             vs XML:  17,095  (-48.8%)
-
-📈 Daily Analytics           ██████████░░░░░░░░░░░░░░░   4,507 tokens
-                             vs JSON: 10,977  (-58.9%)
-                             vs YAML:  8,810  (-48.8%)
-                             vs XML:  13,128  (-65.7%)
-
-🛒 E-Commerce Order          ████████████████░░░░░░░░░     166 tokens
-                             vs JSON:    257  (-35.4%)
-                             vs YAML:    197  (-15.7%)
-                             vs XML:     271  (-38.7%)
-
-─────────────────────────────────────────────────────────────────────
-Total                        █████████████░░░░░░░░░░░░  13,418 tokens
-                             vs JSON: 26,379  (-49.1%)
-                             vs YAML: 22,136  (-39.4%)
-                             vs XML:  30,494  (-56.0%)
-```
-
-<details>
-<summary><strong>View detailed examples</strong></summary>
-
-#### ⭐ GitHub Repositories
-
-**Configuration:** Top 100 GitHub repositories with stars, forks, and metadata
-
-**Savings:** 6,400 tokens (42.3% reduction vs JSON)
-
-**JSON** (15,145 tokens):
-
-```json
-{
-  "repositories": [
-    {
-      "id": 28457823,
-      "name": "freeCodeCamp",
-      "repo": "freeCodeCamp/freeCodeCamp",
-      "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
-      "createdAt": "2014-12-24T17:49:19Z",
-      "updatedAt": "2025-10-28T11:58:08Z",
-      "pushedAt": "2025-10-28T10:17:16Z",
-      "stars": 430886,
-      "watchers": 8583,
-      "forks": 42146,
-      "defaultBranch": "main"
-    },
-    {
-      "id": 132750724,
-      "name": "build-your-own-x",
-      "repo": "codecrafters-io/build-your-own-x",
-      "description": "Master programming by recreating your favorite technologies from scratch.",
-      "createdAt": "2018-05-09T12:03:18Z",
-      "updatedAt": "2025-10-28T12:37:11Z",
-      "pushedAt": "2025-10-10T18:45:01Z",
-      "stars": 430877,
-      "watchers": 6332,
-      "forks": 40453,
-      "defaultBranch": "master"
-    },
-    {
-      "id": 21737465,
-      "name": "awesome",
-      "repo": "sindresorhus/awesome",
-      "description": "😎 Awesome lists about all kinds of interesting topics",
-      "createdAt": "2014-07-11T13:42:37Z",
-      "updatedAt": "2025-10-28T12:40:21Z",
-      "pushedAt": "2025-10-27T17:57:31Z",
-      "stars": 410052,
-      "watchers": 8017,
-      "forks": 32029,
-      "defaultBranch": "main"
-    }
-  ]
-}
-```
-
-**TOON** (8,745 tokens):
-
-```
-repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
-  28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
-  132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
-  21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
-```
-
----
-
-#### 📈 Daily Analytics
-
-**Configuration:** 180 days of web metrics (views, clicks, conversions, revenue)
-
-**Savings:** 6,470 tokens (58.9% reduction vs JSON)
-
-**JSON** (10,977 tokens):
-
-```json
-{
-  "metrics": [
-    {
-      "date": "2025-01-01",
-      "views": 6890,
-      "clicks": 401,
-      "conversions": 23,
-      "revenue": 6015.59,
-      "bounceRate": 0.63
-    },
-    {
-      "date": "2025-01-02",
-      "views": 6940,
-      "clicks": 323,
-      "conversions": 37,
-      "revenue": 9086.44,
-      "bounceRate": 0.36
-    },
-    {
-      "date": "2025-01-03",
-      "views": 4390,
-      "clicks": 346,
-      "conversions": 26,
-      "revenue": 6360.75,
-      "bounceRate": 0.48
-    },
-    {
-      "date": "2025-01-04",
-      "views": 3429,
-      "clicks": 231,
-      "conversions": 13,
-      "revenue": 2360.96,
-      "bounceRate": 0.65
-    },
-    {
-      "date": "2025-01-05",
-      "views": 5804,
-      "clicks": 186,
-      "conversions": 22,
-      "revenue": 2535.96,
-      "bounceRate": 0.37
-    }
-  ]
-}
-```
-
-**TOON** (4,507 tokens):
-
-```
-metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
-  2025-01-01,6890,401,23,6015.59,0.63
-  2025-01-02,6940,323,37,9086.44,0.36
-  2025-01-03,4390,346,26,6360.75,0.48
-  2025-01-04,3429,231,13,2360.96,0.65
-  2025-01-05,5804,186,22,2535.96,0.37
-```
-
-</details>
-
-<!-- /automd -->
-
-> [!NOTE]
-> Measured with [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer) using `o200k_base` encoding (used by GPT-5 and other modern models). Savings will vary across models and tokenizers.
-
-<!-- automd:file src="./benchmarks/results/retrieval-accuracy.md" -->
-
-### Retrieval Accuracy
-
-Accuracy across **4 LLMs** on 154 data retrieval questions:
-
-```
-gpt-5-nano
-→ toon         ███████████████████░  96.1% (148/154)
-  csv          ██████████████████░░  90.3% (139/154)
-  yaml         ██████████████████░░  89.0% (137/154)
-  json         ██████████████████░░  87.7% (135/154)
-  xml          █████████████████░░░  83.8% (129/154)
-
-claude-haiku-4-5-20251001
-  yaml         ██████████░░░░░░░░░░  49.4% (76/154)
-→ toon         ██████████░░░░░░░░░░  48.1% (74/154)
-  csv          ██████████░░░░░░░░░░  48.1% (74/154)
-  json         █████████░░░░░░░░░░░  47.4% (73/154)
-  xml          █████████░░░░░░░░░░░  46.8% (72/154)
-
-gemini-2.5-flash
-  csv          ██████████████████░░  87.7% (135/154)
-  xml          █████████████████░░░  85.1% (131/154)
-→ toon         █████████████████░░░  83.8% (129/154)
-  json         ████████████████░░░░  78.6% (121/154)
-  yaml         ███████████████░░░░░  76.6% (118/154)
-
-grok-4-fast-non-reasoning
-→ toon         ██████████░░░░░░░░░░  48.7% (75/154)
-  json         ██████████░░░░░░░░░░  48.1% (74/154)
-  xml          █████████░░░░░░░░░░░  47.4% (73/154)
-  yaml         █████████░░░░░░░░░░░  46.8% (72/154)
-  csv          █████████░░░░░░░░░░░  45.5% (70/154)
-```
-
-**Key tradeoff:** TOON achieves **69.2% accuracy** (vs JSON's 65.4%) while using **46.3% fewer tokens** on these datasets.
-
-<details>
-<summary><strong>Performance by dataset and model</strong></summary>
-
-#### Performance by Dataset
-
-##### Uniform employee records (TOON optimal format)
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 67.0% | 2,337 | 134/200 |
-| `toon` | 66.5% | 2,483 | 133/200 |
-| `yaml` | 65.5% | 4,969 | 131/200 |
-| `json` | 63.5% | 6,347 | 127/200 |
-| `xml` | 66.5% | 7,314 | 133/200 |
-
-##### E-commerce orders with nested structures
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `toon` | 78.8% | 5,967 | 126/160 |
-| `csv` | 71.9% | 6,735 | 115/160 |
-| `yaml` | 71.9% | 7,328 | 115/160 |
-| `json` | 73.1% | 9,694 | 117/160 |
-| `xml` | 73.8% | 10,992 | 118/160 |
-
-##### Time-series analytics data
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 67.6% | 1,393 | 92/136 |
-| `toon` | 67.6% | 1,515 | 92/136 |
-| `yaml` | 64.7% | 2,938 | 88/136 |
-| `json` | 68.4% | 3,665 | 93/136 |
-| `xml` | 66.2% | 4,376 | 90/136 |
-
-##### Top 100 GitHub repositories
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 64.2% | 8,513 | 77/120 |
-| `toon` | 62.5% | 8,745 | 75/120 |
-| `yaml` | 57.5% | 13,129 | 69/120 |
-| `json` | 55.0% | 15,145 | 66/120 |
-| `xml` | 53.3% | 17,095 | 64/120 |
-
-#### Performance by Model
-
-##### gpt-5-nano
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 96.1% | 148/154 |
-| `csv` | 90.3% | 139/154 |
-| `yaml` | 89.0% | 137/154 |
-| `json` | 87.7% | 135/154 |
-| `xml` | 83.8% | 129/154 |
-
-##### claude-haiku-4-5-20251001
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `yaml` | 49.4% | 76/154 |
-| `toon` | 48.1% | 74/154 |
-| `csv` | 48.1% | 74/154 |
-| `json` | 47.4% | 73/154 |
-| `xml` | 46.8% | 72/154 |
-
-##### gemini-2.5-flash
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `csv` | 87.7% | 135/154 |
-| `xml` | 85.1% | 131/154 |
-| `toon` | 83.8% | 129/154 |
-| `json` | 78.6% | 121/154 |
-| `yaml` | 76.6% | 118/154 |
-
-##### grok-4-fast-non-reasoning
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 48.7% | 75/154 |
-| `json` | 48.1% | 74/154 |
-| `xml` | 47.4% | 73/154 |
-| `yaml` | 46.8% | 72/154 |
-| `csv` | 45.5% | 70/154 |
-
-</details>
-
-<details>
-<summary><strong>How the benchmark works</strong></summary>
-
-#### What's Being Measured
-
-This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it (this does **not** test model's ability to generate TOON output).
-
-#### Datasets Tested
-
-Four datasets designed to test different structural patterns (all contain arrays of uniform objects, TOON's optimal format):
-
-1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
-2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
-3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
-4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
-
-#### Question Types
-
-154 questions are generated dynamically across three categories:
-
-- **Field retrieval (40%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
-  - Example: "What is Alice's salary?" → `75000`
-  - Example: "How many items are in order ORD-0042?" → `3`
-  - Example: "What is the customer name for order ORD-0042?" → `John Doe`
-
-- **Aggregation (32%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
-  - Example: "How many employees work in Engineering?" → `17`
-  - Example: "What is the total revenue across all orders?" → `45123.50`
-  - Example: "How many employees have salary > 80000?" → `23`
-
-- **Filtering (28%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
-  - Example: "How many employees in Sales have salary > 80000?" → `5`
-  - Example: "How many active employees have more than 10 years of experience?" → `8`
-
-#### Evaluation Process
-
-1. **Format conversion**: Each dataset is converted to all 5 formats (TOON, CSV, XML, JSON, YAML).
-2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
-3. **Validate with LLM-as-judge**: `gpt-5-nano` validates if the answer is semantically correct (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`).
-
-#### Models & Configuration
-
-- **Models tested**: `gpt-5-nano`, `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `grok-4-fast-non-reasoning`
-- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
-- **Temperature**: Not set (models use their defaults)
-- **Total evaluations**: 154 questions × 5 formats × 4 models = 3,080 LLM calls
-
-</details>
-
-<!-- /automd -->
-
-## Installation
-
-```bash
-# npm
-npm install @byjohann/toon
-
-# pnpm
-pnpm add @byjohann/toon
-
-# yarn
-yarn add @byjohann/toon
-```
-
-## Quick Start
-
-```ts
-import { encode } from '@byjohann/toon'
-
-const data = {
-  user: {
-    id: 123,
-    name: 'Ada',
-    tags: ['reading', 'gaming'],
-    active: true,
-    preferences: []
-  }
-}
-
-console.log(encode(data))
-```
-
-Output:
-
-```
-user:
-  id: 123
-  name: Ada
-  tags[2]: reading,gaming
-  active: true
-  preferences[0]:
-```
-
-## Canonical Formatting Rules
-
-TOON formatting is deterministic and minimal:
-
-- **Indentation**: 2 spaces per nesting level.
-- **Lines**:
-  - `key: value` for primitives (single space after colon).
-  - `key:` for nested/empty objects (no trailing space on that line).
-- **Arrays**:
-  - Delimiter encoding: Comma delimiters are implicit in array headers (e.g., `tags[3]:`, `items[2]{id,name}:`). Tab and pipe delimiters are explicitly shown in array headers (e.g., `tags[3|]:`, `items[2	]{id	name}:`).
-  - Primitive arrays inline: `key[N]: v1,v2` (comma) or `key[N<delim>]: v1<delim>v2` (tab/pipe).
-  - Tabular arrays: `key[N]{f1,f2}: …` (comma) or `key[N<delim>]{f1<delim>f2}: …` (tab/pipe).
-  - List items: two spaces, hyphen, space (`"  - …"`).
-- **Whitespace invariants**:
-  - No trailing spaces at end of any line.
-  - No trailing newline at end of output.
-
-## Format Overview
-
-### Objects
-
-Simple objects with primitive values:
-
-```ts
-encode({
-  id: 123,
-  name: 'Ada',
-  active: true
-})
-```
-
-```
-id: 123
-name: Ada
-active: true
-```
-
-Nested objects:
-
-```ts
-encode({
-  user: {
-    id: 123,
-    name: 'Ada'
-  }
-})
-```
-
-```
-user:
-  id: 123
-  name: Ada
-```
-
-### Arrays
-
-> [!TIP]
-> TOON includes the array length in brackets (e.g., `items[3]`). When using comma delimiters (default), the delimiter is implicit. When using tab or pipe delimiters, the delimiter is explicitly shown in the header (e.g., `tags[2|]` or `[2	]`). This encoding helps LLMs identify the delimiter and track the number of elements, reducing errors when generating or validating structured output.
-
-#### Primitive Arrays (Inline)
-
-```ts
-encode({
-  tags: ['admin', 'ops', 'dev']
-})
-```
-
-```
-tags[3]: admin,ops,dev
-```
-
-#### Arrays of Objects (Tabular)
-
-When all objects share the same primitive fields, TOON uses an efficient **tabular format**:
-
-```ts
-encode({
-  items: [
-    { sku: 'A1', qty: 2, price: 9.99 },
-    { sku: 'B2', qty: 1, price: 14.5 }
-  ]
-})
-```
-
-```
-items[2]{sku,qty,price}:
-  A1,2,9.99
-  B2,1,14.5
-```
-
-**Tabular formatting applies recursively:** nested arrays of objects (whether as object properties or inside list items) also use tabular format if they meet the same requirements.
-
-```ts
-encode({
-  items: [
-    {
-      users: [
-        { id: 1, name: 'Ada' },
-        { id: 2, name: 'Bob' }
-      ],
-      status: 'active'
-    }
-  ]
-})
-```
-
-```
-items[1]:
-  - users[2]{id,name}:
-    1,Ada
-    2,Bob
-    status: active
-```
-
-#### Mixed and Non-Uniform Arrays
-
-Arrays that don't meet the tabular requirements use list format:
-
-```
-items[3]:
-  - 1
-  - a: 1
-  - text
-```
-
-When objects appear in list format, the first field is placed on the hyphen line:
-
-```
-items[2]:
-  - id: 1
-    name: First
-  - id: 2
-    name: Second
-    extra: true
-```
-
-> [!NOTE]
-> **Nested array indentation:** When the first field of a list item is an array (primitive, tabular, or nested), its contents are indented two spaces under the header line, and subsequent fields of the same object appear at that same indentation level. This remains unambiguous because list items begin with `"- "`, tabular arrays declare a fixed row count in their header, and object fields contain `":"`.
-
-#### Arrays of Arrays
-
-When you have arrays containing primitive inner arrays:
-
-```ts
-encode({
-  pairs: [
-    [1, 2],
-    [3, 4]
-  ]
-})
-```
-
-```
-pairs[2]:
-  - [2]: 1,2
-  - [2]: 3,4
-```
-
-#### Empty Arrays and Objects
-
-Empty containers have special representations:
-
-```ts
-encode({ items: [] }) // items[0]:
-encode([]) // [0]:
-encode({}) // (empty output)
-encode({ config: {} }) // config:
-```
-
-### Quoting Rules
-
-TOON quotes strings **only when necessary** to maximize token efficiency:
-
-- Inner spaces are allowed; leading or trailing spaces force quotes.
-- Unicode and emoji are safe unquoted.
-- Quotes and control characters are escaped with backslash.
-
-> [!NOTE]
-> When using alternative delimiters (tab or pipe), the quoting rules adapt automatically. Strings containing the active delimiter will be quoted, while other delimiters remain safe.
-
-#### Object Keys and Field Names
-
-Keys are unquoted if they match the identifier pattern: start with a letter or underscore, followed by letters, digits, underscores, or dots (e.g., `id`, `userName`, `user_name`, `user.name`, `_private`). All other keys must be quoted (e.g., `"user name"`, `"order-id"`, `"123"`, `"order:id"`, `""`).
-
-#### String Values
-
-String values are quoted when any of the following is true:
-
-| Condition | Examples |
-|---|---|
-| Empty string | `""` |
-| Leading or trailing spaces | `" padded "`, `"  "` |
-| Contains active delimiter, colon, quote, backslash, or control chars | `"a,b"` (comma), `"a\tb"` (tab), `"a\|b"` (pipe), `"a:b"`, `"say \"hi\""`, `"C:\\Users"`, `"line1\\nline2"` |
-| Looks like boolean/number/null | `"true"`, `"false"`, `"null"`, `"42"`, `"-3.14"`, `"1e-6"`, `"05"` |
-| Starts with `"- "` (list-like) | `"- item"` |
-| Looks like structural token | `"[5]"`, `"{key}"`, `"[3]: x,y"` |
-
-**Examples of unquoted strings:** Unicode and emoji are safe (`hello 👋 world`), as are strings with inner spaces (`hello world`).
-
-> [!IMPORTANT]
-> **Delimiter-aware quoting:** Unquoted strings never contain `:` or the active delimiter. This makes TOON reliably parseable with simple heuristics: split key/value on first `: `, and split array values on the delimiter declared in the array header. When using tab or pipe delimiters, commas don't need quoting – only the active delimiter triggers quoting for both array values and object values.
-
-## Type Conversions
-
-Some non-JSON types are automatically normalized for LLM-safe output:
-
-| Input | Output |
-|---|---|
-| Number (finite) | Decimal form, no scientific notation (e.g., `-0` → `0`, `1e6` → `1000000`) |
-| Number (`NaN`, `±Infinity`) | `null` |
-| `BigInt` | If within safe integer range: converted to number. Otherwise: quoted decimal string (e.g., `"9007199254740993"`) |
-| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
-| `undefined` | `null` |
-| `function` | `null` |
-| `symbol` | `null` |
-
-## API
-
-### `encode(value: unknown, options?: EncodeOptions): string`
-
-Converts any JSON-serializable value to TOON format.
-
-**Parameters:**
-
-- `value` – Any JSON-serializable value (object, array, primitive, or nested structure). Non-JSON-serializable values (functions, symbols, undefined, non-finite numbers) are converted to `null`. Dates are converted to ISO strings, and BigInts are emitted as decimal integers (no quotes).
-- `options` – Optional encoding options:
-  - `indent?: number` – Number of spaces per indentation level (default: `2`)
-  - `delimiter?: ',' | '\t' | '|'` – Delimiter for array values and tabular rows (default: `','`)
-  - `lengthMarker?: '#' | false` – Optional marker to prefix array lengths (default: `false`)
-
-**Returns:**
-
-A TOON-formatted string with no trailing newline or spaces.
-
-**Example:**
-
-```ts
-import { encode } from '@byjohann/toon'
-
-const items = [
-  { sku: 'A1', qty: 2, price: 9.99 },
-  { sku: 'B2', qty: 1, price: 14.5 }
-]
-
-encode({ items })
-```
-
-**Output:**
-
-```
-items[2]{sku,qty,price}:
-  A1,2,9.99
-  B2,1,14.5
-```
-
-#### Delimiter Options
-
-The `delimiter` option allows you to choose between comma (default), tab, or pipe delimiters for array values and tabular rows. Alternative delimiters can provide additional token savings in specific contexts.
-
-##### Tab Delimiter (`\t`)
-
-Using tab delimiters instead of commas can reduce token count further, especially for tabular data:
-
-```ts
-const data = {
-  items: [
-    { sku: 'A1', name: 'Widget', qty: 2, price: 9.99 },
-    { sku: 'B2', name: 'Gadget', qty: 1, price: 14.5 }
-  ]
-}
-
-encode(data, { delimiter: '\t' })
-```
-
-**Output:**
-
-```
-items[2	]{sku	name	qty	price}:
-  A1	Widget	2	9.99
-  B2	Gadget	1	14.5
-```
-
-**Benefits:**
-
-- Tabs are single characters and often tokenize more efficiently than commas.
-- Tabs rarely appear in natural text, reducing the need for quote-escaping.
-- The delimiter is explicitly encoded in the array header, making it self-descriptive.
-
-**Considerations:**
-
-- Some terminals and editors may collapse or expand tabs visually.
-- String values containing tabs will still require quoting.
-
-##### Pipe Delimiter (`|`)
-
-Pipe delimiters offer a middle ground between commas and tabs:
-
-```ts
-encode(data, { delimiter: '|' })
-```
-
-**Output:**
-
-```
-items[2|]{sku|name|qty|price}:
-  A1|Widget|2|9.99
-  B2|Gadget|1|14.5
-```
-
-#### Length Marker Option
-
-The `lengthMarker` option adds an optional hash (`#`) prefix to array lengths to emphasize that the bracketed value represents a count, not an index:
-
-```ts
-const data = {
-  tags: ['reading', 'gaming', 'coding'],
-  items: [
-    { sku: 'A1', qty: 2, price: 9.99 },
-    { sku: 'B2', qty: 1, price: 14.5 },
-  ],
-}
-
-encode(data, { lengthMarker: '#' })
-// tags[#3]: reading,gaming,coding
-// items[#2]{sku,qty,price}:
-//   A1,2,9.99
-//   B2,1,14.5
-
-// Works with custom delimiters
-encode(data, { lengthMarker: '#', delimiter: '|' })
-// tags[#3|]: reading|gaming|coding
-// items[#2|]{sku|qty|price}:
-//   A1|2|9.99
-//   B2|1|14.5
-```
-
-### `decode(input: string, options?: DecodeOptions): JsonValue`
-
-Converts a TOON-formatted string back to JavaScript values.
-
-**Parameters:**
-
-- `input` – A TOON-formatted string to parse
-- `options` – Optional decoding options:
-  - `indent?: number` – Expected number of spaces per indentation level (default: `2`)
-  - `strict?: boolean` – Enable strict validation (default: `true`)
-
-**Returns:**
-
-A JavaScript value (object, array, or primitive) representing the parsed TOON data.
-
-**Example:**
-
-```ts
-import { decode } from '@byjohann/toon'
-
-const toon = `items[2]{sku,qty,price}:
-  A1,2,9.99
-  B2,1,14.5`
-
-const data = decode(toon)
-// {
-//   items: [
-//     { sku: 'A1', qty: 2, price: 9.99 },
-//     { sku: 'B2', qty: 1, price: 14.5 }
-//   ]
-// }
-```
-
-**Strict Mode:**
-
-By default, the decoder validates input strictly:
-
-- **Invalid escape sequences** – Throws on `"\x"`, unterminated strings
-- **Syntax errors** – Throws on missing colons, malformed headers
-- **Array length mismatches** – Throws when declared length doesn't match actual count
-- **Delimiter mismatches** – Throws when row delimiters don't match header
-
-## Notes and Limitations
-
-- Format familiarity and structure matter as much as token count. TOON's tabular format requires arrays of objects with identical keys and primitive values only. When this doesn't hold (due to mixed types, non-uniform objects, or nested structures), TOON switches to list format where JSON can be more efficient at scale.
-  - **TOON excels at:** Uniform arrays of objects (same fields, primitive values), especially large datasets with consistent structure.
-  - **JSON is better for:** Non-uniform data, deeply nested structures, and objects with varying field sets.
-- **Token counts vary by tokenizer and model.** Benchmarks use a GPT-style tokenizer (cl100k/o200k); actual savings will differ with other models (e.g., [SentencePiece](https://github.com/google/sentencepiece)).
-- **TOON is designed for LLM input** where human readability and token efficiency matter. It's **not** a drop-in replacement for JSON in APIs or storage.
-
-## Using TOON in LLM Prompts
-
-TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern.
-
-### Sending TOON to LLMs (Input)
-
-Wrap your encoded data in a fenced code block (label it \`\`\`toon for clarity). The indentation and headers are usually enough – models treat it like familiar YAML or CSV. The explicit length markers (`[N]`) and field headers (`{field1,field2}`) help the model track structure, especially for large tables.
-
-### Generating TOON from LLMs (Output)
-
-For output, be more explicit. When you want the model to **generate** TOON:
-
-- **Show the expected header** (`users[N]{id,name,role}:`). The model fills rows instead of repeating keys, reducing generation errors.
-- **State the rules:** 2-space indent, no trailing spaces, `[N]` matches row count.
-
-Here's a prompt that works for both reading and generating:
-
-```
-Data is in TOON format (2-space indent, arrays show length and fields).
-
-\`\`\`toon
-users[3]{id,name,role,lastLogin}:
-  1,Alice,admin,2025-01-15T10:30:00Z
-  2,Bob,user,2025-01-14T15:22:00Z
-  3,Charlie,user,2025-01-13T09:45:00Z
-\`\`\`
-
-Task: Return only users with role "user" as TOON. Use the same header. Set [N] to match the row count. Output only the code block.
-```
-
-> [!TIP]
-> For large uniform tables, use `encode(data, { delimiter: '\t' })` and tell the model "fields are tab-separated." Tabs often tokenize better than commas and reduce the need for quote-escaping.
-
-## Quick Reference
-
-```
-// Object
-{ id: 1, name: 'Ada' }          → id: 1
-                                  name: Ada
-
-// Nested object
-{ user: { id: 1 } }             → user:
-                                    id: 1
-
-// Primitive array (inline)
-{ tags: ['foo', 'bar'] }        → tags[2]: foo,bar
-
-// Tabular array (uniform objects)
-{ items: [                      → items[2]{id,qty}:
-  { id: 1, qty: 5 },                1,5
-  { id: 2, qty: 3 }                 2,3
-]}
-
-// Mixed / non-uniform (list)
-{ items: [1, { a: 1 }, 'x'] }   → items[3]:
-                                    - 1
-                                    - a: 1
-                                    - x
-
-// Array of arrays
-{ pairs: [[1, 2], [3, 4]] }     → pairs[2]:
-                                    - [2]: 1,2
-                                    - [2]: 3,4
-
-// Root array
-['x', 'y']                      → [2]: x,y
-
-// Empty containers
-{}                              → (empty output)
-{ items: [] }                   → items[0]:
-
-// Special quoting
-{ note: 'hello, world' }        → note: "hello, world"
-{ items: ['true', true] }       → items[2]: "true",true
-```
-
-## Ports in Other Languages
-
-- **Elixir:** [toon_ex](https://github.com/kentaro/toon_ex)
-- **PHP:** [toon-php](https://github.com/HelgeSverre/toon-php)
-- **Python:** [pytoon](https://github.com/bpradana/pytoon)
-  - [python-toon](https://github.com/xaviviro/python-toon)
-  - [toon-python](https://gitlab.com/KanTakahiro/toon-python)
-- **Ruby:** [toon-ruby](https://github.com/andrepcg/toon-ruby)
-- **Java:** [JToon](https://github.com/felipestanzani/JToon)
-- **.NET:** [toon.NET](https://github.com/ghost1face/toon.NET)
-- **Swift:** [TOONEncoder](https://github.com/mattt/TOONEncoder)
-- **Go:** [gotoon](https://github.com/alpkeskin/gotoon)
-
-## License
-
-[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

README.md

@@ -0,0 +1 @@
+packages/toon/README.md

SPEC.md

@@ -1,689 +1,28 @@
-# TOON Specification (v1.2)
+# TOON Specification
 
-Status: Draft, normative where indicated. This version specifies both encoding (producer behavior) and decoding (parser behavior).
+The TOON specification has moved to a dedicated repository: [github.com/toon-format/spec](https://github.com/toon-format/spec)
 
-- Normative statements use RFC 2119/8174 keywords: MUST, MUST NOT, SHOULD, SHOULD NOT, MAY.
-- Audience: implementers of encoders/decoders/validators; tool authors; practitioners embedding TOON in LLM prompts.
+## Current Version
 
-Changelog:
-- v1.2:
-  - Centralized decoding rules (primitives, keys) and strict-mode checklist.
-  - Made header grammar normative and clarified delimiter scoping.
-  - Tightened strict-mode indentation (exact multiples; tabs error).
-  - Defined blank-line and trailing-newline decoding behavior with explicit skipping rules outside arrays.
-  - Clarified hyphen-based quoting: "-" or any string starting with "-" MUST be quoted.
-  - Clarified BigInt normalization (quoted string when out of safe range).
-  - Unified root-form detection and row/key disambiguation language; disambiguation uses first unquoted delimiter vs colon.
-  - Introduced "document delimiter" vs "active delimiter" terminology.
-- v1.1: Made decoding behavior normative; added strict-mode rules, delimiter-aware parsing, and reference algorithms; decoder options (indent, strict).
-- v1: Initial encoding, normalization, and conformance rules.
+**Version 3.3** (2026-05-21)
 
-Scope:
-- Defines the data model, encoding normalization (reference JS/TS), concrete syntax, decoding semantics, and conformance requirements for producing and consuming TOON.
+## Quick Links
 
-## 1. Terminology and Conventions
+- **[Full Specification](https://github.com/toon-format/spec/blob/main/SPEC.md)** - Complete technical specification
+- **[Changelog](https://github.com/toon-format/spec/blob/main/CHANGELOG.md)** - Version history
+- **[Examples](https://github.com/toon-format/spec/tree/main/examples)** - Example TOON files
+- **[Conformance Tests](https://github.com/toon-format/spec/tree/main/tests)** - Language-agnostic test fixtures for implementations
+- **[Contributing](https://github.com/toon-format/spec/blob/main/CONTRIBUTING.md)** - How to propose spec changes
 
-- TOON document: A sequence of UTF-8 text lines formatted according to this spec.
-- Line: A sequence of non-newline characters terminated by LF (U+000A) in serialized form. Encoders MUST use LF.
-- Indentation level (depth): Leading indentation measured in fixed-size space units (indentSize). Depth 0 has no indentation.
-- Indentation unit (indentSize): A fixed number of spaces per level (default 2). Tabs MUST NOT be used for indentation.
-- Header: The bracketed declaration for arrays, optionally followed by a field list, and terminating with a colon; e.g., key[3]: or items[2]{a,b}:.
-- Field list: Brace-enclosed, delimiter-separated list of field names for tabular arrays: {f1<delim>f2}.
-- List item: A line beginning with "- " at a given depth representing an element in an expanded array.
-- Delimiter: The character used to separate array/tabular values: comma (default), tab (HTAB, U+0009), or pipe ("|").
-- Document delimiter: The encoder-selected delimiter used for quoting decisions outside any array scope (default comma).
-- Active delimiter: The delimiter declared by the closest array header in scope, used to split inline primitive arrays and tabular rows under that header; it also governs quoting decisions for values within that array’s scope.
-- Length marker: Optional "#" prefix for array lengths in headers, e.g., [#3]. Decoders MUST accept and ignore it semantically.
-- Primitive: string, number, boolean, or null.
-- Object: Mapping from string keys to JsonValue.
-- Array: Ordered sequence of JsonValue.
-- JsonValue: Primitive | Object | Array.
-- Strict mode: Decoder mode that enforces counts, indentation, and delimiter consistency; also rejects invalid escapes and missing colons (default: true).
+## Why a Separate Repo?
 
-Notation:
-- Regular expressions appear in slash-delimited form.
-- ABNF snippets follow RFC 5234; HTAB means the U+0009 character.
-- Examples are informative unless stated otherwise.
+The specification has been moved to `toon-format/spec` to:
 
-## 2. Data Model
+- Provide a canonical, language-agnostic source of truth
+- Enable independent versioning of spec and implementations
+- Support the growing community of TOON implementations across multiple languages
+- Facilitate collaboration on spec evolution through a dedicated RFC process
 
-- TOON models data as:
-  - JsonPrimitive: string | number | boolean | null
-  - JsonObject: { [string]: JsonValue }
-  - JsonArray: JsonValue[]
-- Ordering:
-  - Array order MUST be preserved.
-  - Object key order MUST be preserved as encountered by the encoder.
-- Numbers (encoding):
-  - -0 MUST be normalized to 0.
-  - Finite numbers MUST be rendered without scientific notation (e.g., 1e6 → 1000000; 1e-6 → 0.000001).
-- Null: Represented as the literal null.
+## This Repository
 
-## 3. Encoding Normalization (Reference Encoder)
-
-The reference encoder normalizes non-JSON values to the data model:
-
-- Number:
-  - Finite → number (non-exponential). -0 → 0.
-  - NaN, +Infinity, -Infinity → null.
-  - Implementations MUST ensure decimal rendering does not use exponent notation.
-- BigInt (JavaScript):
-  - If within Number.MIN_SAFE_INTEGER..Number.MAX_SAFE_INTEGER → converted to number.
-  - Otherwise → converted to a decimal string (e.g., "9007199254740993") and encoded as a string (quoted because it is numeric-like).
-- Date → ISO string (e.g., "2025-01-01T00:00:00.000Z").
-- Set → array by iterating entries and normalizing each element.
-- Map → object using String(key) for keys and normalizing values.
-- Plain object → own enumerable string keys in encounter order; values normalized recursively.
-- Function, symbol, undefined, or unrecognized types → null.
-
-Note: Other language ports SHOULD apply analogous normalization consistent with this spec’s data model and encoding rules.
-
-## 4. Decoding Interpretation (Reference Decoder)
-
-Decoders map text tokens to host values:
-
-- Quoted tokens (strings and keys):
-  - MUST be unescaped per Section 7.1 (only \\, \", \n, \r, \t are valid). Any other escape or an unterminated string MUST error.
-  - Quoted primitives remain strings even if they look like numbers/booleans/null.
-- Unquoted value tokens:
-  - true, false, null → booleans/null.
-  - Numeric parsing:
-    - MUST accept standard decimal and exponent forms (e.g., 42, -3.14, 1e-6, -1E+9).
-    - MUST treat tokens with forbidden leading zeros (e.g., "05", "0001") as strings (not numbers).
-    - Only finite numbers are expected from conforming encoders.
-  - Otherwise → string.
-- Keys:
-  - Decoded as strings (quoted keys MUST be unescaped per Section 7.1).
-  - A colon MUST follow a key; missing colon MUST error.
-
-## 5. Concrete Syntax and Root Form
-
-TOON is a deterministic, line-oriented, indentation-based notation.
-
-- Objects:
-  - key: value for primitives.
-  - key: alone for nested or empty objects; nested fields appear at depth +1.
-- Arrays:
-  - Primitive arrays are inline: key[N<delim?>]: v1<delim>v2….
-  - Arrays of arrays (primitives): expanded list items under a header: key[N<delim?>]: then "- [M<delim?>]: …".
-  - Arrays of objects:
-    - Tabular form when uniform and primitive-only: key[N<delim?>]{f1<delim>f2}: then one row per line.
-    - Otherwise: expanded list items: key[N<delim?>]: with "- …" items (see Sections 9.4 and 10).
-- Root form discovery:
-  - If the first non-empty depth-0 line is a valid root array header per Section 6 (must include a colon), decode a root array.
-  - Else if the document has exactly one non-empty line and it is neither a valid array header nor a key-value line (quoted or unquoted key), decode a single primitive.
-  - Otherwise, decode an object.
-  - In strict mode, multiple non-key/value non-header lines at depth 0 is invalid.
-
-## 6. Header Syntax (Normative)
-
-Array headers declare length and active delimiter, and optionally field names.
-
-General forms:
-- Root header (no key): [<marker?>N<delim?>]:
-- With key: key[<marker?>N<delim?>]:
-- Tabular fields: key[<marker?>N<delim?>]{field1<delim>field2<delim>…}:
-
-Where:
-- N is the non-negative integer length.
-- <marker?> is optional "#"; decoders MUST accept and ignore it semantically.
-- <delim?> is:
-  - absent for comma (","),
-  - HTAB (U+0009) for tab,
-  - "|" for pipe.
-- Field names in braces are separated by the same active delimiter and encoded as keys (Section 7.3).
-
-Spacing and delimiters:
-- Every header line MUST end with a colon.
-- When inline values follow a header on the same line (non-empty primitive arrays), there MUST be exactly one space after the colon before the first value.
-- The active delimiter declared by the bracket segment applies to:
-  - splitting inline primitive arrays on that header line,
-  - splitting tabular field names in "{…}",
-  - splitting all rows/items within the header’s scope,
-  - unless a nested header changes it.
-- The same delimiter symbol declared in the bracket MUST be used in the fields segment and in all row/value splits in that scope.
-- Absence of a delimiter symbol in a bracket segment ALWAYS means comma, regardless of any parent header.
-
-Normative header grammar (ABNF):
-```
-bracket-seg   = "[" [ "#" ] 1*DIGIT [ delimsym ] "]"
-delimsym      = HTAB / "|"
-; Field names are keys (quoted/unquoted) separated by the active delimiter
-fields-seg    = "{" fieldname *( delim fieldname ) "}"
-delim         = delimsym / ","
-fieldname     = key
-
-header        = [ key ] bracket-seg [ fields-seg ] ":"
-key           = unquoted-key / quoted-key
-
-; Unquoted keys must match identifier pattern
-unquoted-key  = ( ALPHA / "_" ) *( ALPHA / DIGIT / "_" / "." )
-
-; Quoted keys use only escapes from Section 7.1
-; (Exact escaped-char repertoire is defined in Section 7.1)
-; quoted-key   = DQUOTE *(escaped-char / safe-char) DQUOTE
-```
-
-Decoding requirements:
-- The bracket segment MUST parse as a non-negative integer length N.
-- If a trailing tab or pipe appears inside the brackets, it selects the active delimiter; otherwise comma is active.
-- If a fields segment occurs between the bracket and the colon, parse field names using the active delimiter; quoted names MUST be unescaped per Section 7.1.
-- A colon MUST follow the bracket and optional fields; missing colon MUST error.
-
-## 7. Strings and Keys
-
-### 7.1 Escaping (Encoding and Decoding)
-
-In quoted strings and keys, the following characters MUST be escaped:
-- "\\" → "\\\\"
-- "\"" → "\\\""
-- U+000A newline → "\\n"
-- U+000D carriage return → "\\r"
-- U+0009 tab → "\\t"
-
-Decoders MUST reject any other escape sequence and unterminated strings.
-
-Tabs are allowed inside quoted strings and as a declared delimiter; they MUST NOT be used for indentation (Section 12).
-
-### 7.2 Quoting Rules for String Values (Encoding)
-
-A string value MUST be quoted if any of the following is true:
-- It is empty ("").
-- It has leading or trailing whitespace.
-- It equals true, false, or null (case-sensitive).
-- It is numeric-like:
-  - Matches /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i (e.g., "42", "-3.14", "1e-6").
-  - Or matches /^0\d+$/ (leading-zero decimals such as "05").
-- It contains a colon (:), double quote ("), or backslash (\).
-- It contains brackets or braces ([, ], {, }).
-- It contains control characters: newline, carriage return, or tab.
-- It contains the relevant delimiter:
-  - Inside array scope: the active delimiter (Section 1).
-  - Outside array scope: the document delimiter (Section 1).
-- It equals "-" or starts with "-" (any hyphen at position 0).
-
-Otherwise, the string MAY be emitted without quotes. Unicode, emoji, and strings with internal (non-leading/trailing) spaces are safe unquoted provided they do not violate the conditions.
-
-### 7.3 Key Encoding (Encoding)
-
-Object keys and tabular field names:
-- MAY be unquoted only if they match: ^[A-Za-z_][\w.]*$.
-- Otherwise, they MUST be quoted and escaped per Section 7.1.
-
-### 7.4 Decoding Rules for Strings and Keys (Decoding)
-
-- Quoted strings and keys MUST be unescaped per Section 7.1; any other escape MUST error. Quoted primitives remain strings.
-- Unquoted values:
-  - true/false/null → boolean/null
-  - Numeric tokens → numbers (with the leading-zero rule in Section 4)
-  - Otherwise → strings
-- Keys (quoted or unquoted) MUST be followed by ":"; missing colon MUST error.
-
-## 8. Objects
-
-- Encoding:
-  - Primitive fields: key: value (single space after colon).
-  - Nested or empty objects: key: on its own line. If non-empty, nested fields appear at depth +1.
-  - Key order: Implementations MUST preserve encounter order when emitting fields.
-  - An empty object at the root yields an empty document (no lines).
-- Decoding:
-  - A line "key:" with nothing after the colon at depth d opens an object; subsequent lines at depth > d belong to that object until the depth decreases to ≤ d.
-  - Lines "key: value" at the same depth are sibling fields.
-  - Missing colon after a key MUST error.
-
-## 9. Arrays
-
-### 9.1 Primitive Arrays (Inline)
-
-- Encoding:
-  - Non-empty arrays: key[N<delim?>]: v1<delim>v2<delim>… where each vi is encoded as a primitive (Section 7) with delimiter-aware quoting.
-  - Empty arrays: key[0<delim?>]: (no values following).
-  - Root arrays: [N<delim?>]: v1<delim>…
-- Decoding:
-  - Split using the active delimiter declared by the header; non-active delimiters MUST NOT split values.
-  - In strict mode, the number of decoded values MUST equal N; otherwise MUST error.
-
-### 9.2 Arrays of Arrays (Primitives Only) — Expanded List
-
-- Encoding:
-  - Parent header: key[N<delim?>]: on its own line.
-  - Each inner primitive array is a list item:
-    - - [M<delim?>]: v1<delim>v2<delim>…
-    - Empty inner arrays: - [0<delim?>]:
-- Decoding:
-  - Items appear at depth +1, each starting with "- " and an inner array header "[M<delim?>]: …".
-  - Inner arrays are split using their own active delimiter; in strict mode, counts MUST match M.
-  - In strict mode, the number of list items MUST equal outer N.
-
-### 9.3 Arrays of Objects — Tabular Form
-
-Tabular detection (encoding; MUST hold for all elements):
-- Every element is an object.
-- All objects have the same set of keys (order per object MAY vary).
-- All values across these keys are primitives (no nested arrays/objects).
-
-When satisfied (encoding):
-- Header: key[N<delim?>]{f1<delim>f2<delim>…}: where field order is the first object’s key encounter order.
-- Field names encoded per Section 7.3.
-- Rows: one line per object at depth +1 under the header; values are encoded primitives (Section 7) and joined by the active delimiter.
-- Root tabular arrays omit the key: [N<delim?>]{…}: followed by rows.
-
-Decoding:
-- A tabular header declares the active delimiter and ordered field list.
-- Rows appear at depth +1 as delimiter-separated value lines.
-- Strict mode MUST enforce:
-  - Each row’s value count equals the field count.
-  - The number of rows equals N.
-- Disambiguation at row depth (unquoted tokens):
-  - Compute the first unquoted occurrence of the active delimiter and the first unquoted colon.
-  - If a same-depth line has no unquoted colon → row.
-  - If both appear, compare first-unquoted positions:
-    - Delimiter before colon → row.
-    - Colon before delimiter → key-value line (end of rows).
-  - If a line has an unquoted colon but no unquoted active delimiter → key-value line (end of rows).
-
-### 9.4 Mixed / Non-Uniform Arrays — Expanded List
-
-When tabular requirements are not met (encoding):
-- Header: key[N<delim?>]:
-- Each element is rendered as a list item at depth +1 under the header:
-  - Primitive: - <primitive>
-  - Primitive array: - [M<delim?>]: v1<delim>…
-  - Object: formatted per Section 10 (objects as list items).
-  - Complex arrays: - key'[M<delim?>]: followed by nested items as appropriate.
-
-Decoding:
-- Header declares list length N and the active delimiter for any nested inline arrays.
-- Each list item starts with "- " at depth +1 and is parsed as:
-  - Primitive (no colon and no array header),
-  - Inline primitive array (- [M<delim?>]: …),
-  - Object with first field on the hyphen line (- key: … or - key[N…]{…}: …),
-  - Or nested arrays via nested headers.
-- In strict mode, the number of list items MUST equal N.
-
-## 10. Objects as List Items
-
-For an object appearing as a list item:
-
-- Empty object list item: a single "-" at the list-item indentation level.
-- First field on the hyphen line:
-  - Primitive: - key: value
-  - Primitive array: - key[M<delim?>]: v1<delim>…
-  - Tabular array: - key[N<delim?>]{fields}:
-    - Followed by tabular rows at depth +1 (relative to the hyphen line).
-  - Non-uniform array: - key[N<delim?>]:
-    - Followed by list items at depth +1.
-  - Object: - key:
-    - Nested object fields appear at depth +2 (i.e., one deeper than subsequent sibling fields of the same list item).
-- Remaining fields of the same object appear at depth +1 under the hyphen line in encounter order, using normal object field rules.
-
-Decoding:
-- The first field is parsed from the hyphen line. If it is a nested object (- key:), nested fields are at +2 relative to the hyphen line; subsequent fields of the same list item are at +1.
-- If the first field is a tabular header on the hyphen line, its rows are at +1; subsequent sibling fields continue at +1 after the rows.
-
-## 11. Delimiters
-
-- Supported delimiters:
-  - Comma (default): header omits the delimiter symbol.
-  - Tab: header includes HTAB inside brackets and braces (e.g., [N<TAB>], {a<TAB>b}); rows/inline arrays use tabs.
-  - Pipe: header includes "|" inside brackets and braces; rows/inline arrays use "|".
-- Document vs Active delimiter:
-  - Encoders select a document delimiter (option) that influences quoting in contexts not governed by an array header (e.g., object values).
-  - Inside an array header’s scope, the active delimiter governs splitting and quoting of inline arrays and tabular rows for that array.
-  - Absence of a delimiter symbol in a header ALWAYS means comma for that array’s scope; it does not inherit from any parent.
-- Delimiter-aware quoting (encoding):
-  - Within an array’s scope, strings containing the active delimiter MUST be quoted to avoid splitting.
-  - Outside any array scope, encoders SHOULD use the document delimiter to decide delimiter-aware quoting for values.
-  - Strings containing non-active delimiters do not require quoting unless another quoting condition applies (Section 7.2).
-- Delimiter-aware parsing (decoding):
-  - Inline arrays and tabular rows MUST be split only on the active delimiter declared by the nearest array header.
-  - Strings containing the active delimiter MUST be quoted to avoid splitting; non-active delimiters MUST NOT cause splits.
-  - Nested headers may change the active delimiter; decoding MUST use the delimiter declared by the nearest header.
-  - If the bracket declares tab or pipe, the same symbol MUST be used in the fields segment and for splitting all rows/values in that scope.
-
-## 12. Indentation and Whitespace
-
-- Encoding:
-  - Encoders MUST use a consistent number of spaces per level (default 2; configurable).
-  - Tabs MUST NOT be used for indentation.
-  - Exactly one space after ": " in key: value lines.
-  - Exactly one space after array headers when followed by inline values.
-  - No trailing spaces at the end of any line.
-  - No trailing newline at the end of the document.
-- Decoding:
-  - Strict mode:
-    - The number of leading spaces on a line MUST be an exact multiple of indentSize; otherwise MUST error.
-    - Tabs used as indentation MUST error. Tabs are allowed in quoted strings and as the HTAB delimiter.
-  - Non-strict mode:
-    - Depth MAY be computed as floor(indentSpaces / indentSize).
-    - Tabs in indentation are non-conforming and MAY be accepted or rejected.
-  - Surrounding whitespace around tokens SHOULD be tolerated; internal semantics follow quoting rules.
-  - Blank lines:
-    - Outside arrays/tabular rows: decoders SHOULD ignore completely blank lines (do not create/close structures).
-    - Inside arrays/tabular rows: in strict mode, MUST error; in non-strict mode, MAY be ignored and not counted as a row/item.
-  - Trailing newline at end-of-file: decoders SHOULD accept; validators MAY warn.
-
-Recommended blank-line handling (normative where stated):
-- Before decoding, or during scanning:
-  - Track blank lines with depth.
-  - For strict mode: if a blank line occurs between the first and last row/item line in an array/tabular block, this MUST error.
-  - Otherwise (outside arrays/tabular rows), blank lines SHOULD be skipped and not contribute to root-form detection.
-- Empty input means: after ignoring trailing newlines and ignorable blank lines outside arrays/tabular rows, there are no non-empty lines.
-
-## 13. Conformance and Options
-
-Conformance classes:
-
-- Encoder:
-  - MUST produce output adhering to all normative rules in Sections 2–12 and 15.
-  - MUST be deterministic regarding:
-    - Object field order (encounter order).
-    - Tabular detection (uniform vs non-uniform).
-    - Quoting decisions given values and delimiter context (document delimiter or active delimiter in array scope).
-
-- Decoder:
-  - MUST implement tokenization, escaping, and type interpretation per Sections 4 and 7.4.
-  - MUST parse array headers per Section 6 and apply the declared active delimiter to inline arrays and tabular rows.
-  - MUST implement structure and depth rules per Sections 8–11, including objects-as-list-items placement.
-  - MUST enforce strict-mode rules in Section 14 when strict = true.
-
-- Validator:
-  - SHOULD verify structural conformance (headers, indentation, list markers).
-  - SHOULD verify whitespace invariants.
-  - SHOULD verify delimiter consistency between headers and rows.
-  - SHOULD verify length counts vs declared [N].
-
-Options:
-- Encoder options:
-  - indent (default: 2 spaces)
-  - delimiter (document delimiter; default: comma; alternatives: tab, pipe)
-  - lengthMarker (default: disabled)
-- Decoder options:
-  - indent (default: 2 spaces)
-  - strict (default: true)
-
-Note: Section 14 is authoritative for strict-mode errors; validators MAY add informative diagnostics for style and encoding invariants.
-
-## 14. Strict Mode Errors and Diagnostics (Authoritative Checklist)
-
-When strict mode is enabled (default), decoders MUST error on:
-
-- Array count mismatches:
-  - Inline primitive arrays: decoded value count ≠ declared N.
-  - List arrays: number of list items ≠ declared N.
-  - Tabular arrays: number of rows ≠ declared N.
-- Tabular row width mismatches:
-  - Any row’s value count ≠ field count.
-- Missing colon in key context.
-- Invalid escape sequences or unterminated strings in quoted tokens.
-- Indentation errors:
-  - Leading spaces not a multiple of indentSize.
-  - Any tab used in indentation.
-- Delimiter mismatch (e.g., rows joined by a different delimiter than declared), detected via width/count checks and header scope.
-- Blank lines inside arrays/tabular rows.
-- Empty input (document with no non-empty lines after ignoring trailing newline(s) and ignorable blank lines outside arrays/tabular rows).
-
-Validators SHOULD additionally report:
-- Trailing spaces, trailing newlines (encoding invariants).
-- Headers missing delimiter marks when non-comma delimiter is in use.
-- Values violating delimiter-aware quoting rules.
-
-Recommended error messages (informative):
-- Missing colon after key
-- Unterminated string: missing closing quote
-- Invalid escape sequence: \x
-- Indentation must be an exact multiple of N spaces
-- Tabs are not allowed in indentation
-- Expected N tabular rows, but got M
-- Expected N list array items, but got M
-- Expected K values in row, but got L
-
-## 15. Security Considerations
-
-- Injection and ambiguity are mitigated by quoting rules:
-  - Strings with colon, the relevant delimiter (document or active), hyphen marker cases ("-" or strings starting with "-"), control characters, or brackets/braces MUST be quoted.
-- Strict-mode checks (Section 14) detect malformed strings, truncation, or injected rows/items via length and width mismatches.
-- Encoders SHOULD avoid excessive memory on large inputs; implement streaming/tabular row emission where feasible.
-- Unicode:
-  - Encoders SHOULD avoid altering Unicode beyond required escaping; decoders SHOULD accept valid UTF-8 in quoted strings/keys (with only the five escapes).
-
-## 16. Internationalization
-
-- Full Unicode is supported in keys and values, subject to quoting and escaping rules.
-- Encoders MUST NOT apply locale-dependent formatting for numbers or booleans (e.g., no thousands separators).
-- ISO 8601 strings SHOULD be used for Date normalization.
-
-## 17. Interoperability and Mappings (Informative)
-
-- JSON:
-  - TOON deterministically encodes JSON-compatible data (after normalization).
-  - Arrays of uniform objects map to CSV-like rows; other structures map to YAML-like nested forms.
-- CSV:
-  - TOON tabular sections generalize CSV with explicit lengths, field lists, and flexible delimiter choice.
-- YAML:
-  - TOON borrows indentation and list-item patterns but uses fewer quotes and explicit array headers.
-
-## 18. Media Type and File Extensions (Provisional)
-
-- Suggested media type: text/toon
-- Suggested file extension: .toon
-- Encoding: UTF-8
-- Line endings: LF (U+000A)
-
-## 19. Examples (Informative)
-
-Objects:
-```
-id: 123
-name: Ada
-active: true
-```
-
-Nested objects:
-```
-user:
-  id: 123
-  name: Ada
-```
-
-Primitive arrays:
-```
-tags[3]: admin,ops,dev
-```
-
-Arrays of arrays (primitives):
-```
-pairs[2]:
-  - [2]: 1,2
-  - [2]: 3,4
-```
-
-Tabular arrays:
-```
-items[2]{sku,qty,price}:
-  A1,2,9.99
-  B2,1,14.5
-```
-
-Mixed arrays:
-```
-items[3]:
-  - 1
-  - a: 1
-  - text
-```
-
-Objects as list items (first field on hyphen line):
-```
-items[2]:
-  - id: 1
-    name: First
-  - id: 2
-    name: Second
-    extra: true
-```
-
-Nested tabular inside a list item:
-```
-items[1]:
-  - users[2]{id,name}:
-    1,Ada
-    2,Bob
-    status: active
-```
-
-Delimiter variations:
-```
-# Tab delimiter
-items[2	]{sku	name	qty	price}:
-  A1	Widget	2	9.99
-  B2	Gadget	1	14.5
-
-# Pipe delimiter
-tags[3|]: reading|gaming|coding
-```
-
-Length marker:
-```
-tags[#3]: reading,gaming,coding
-pairs[#2]:
-  - [#2]: a,b
-  - [#2]: c,d
-```
-
-Quoted colons and disambiguation (rows continue; colon is inside quotes):
-```
-links[2]{id,url}:
-  1,"http://a:b"
-  2,"https://example.com?q=a:b"
-```
-
-## 20. Parsing Helpers (Informative)
-
-These sketches illustrate structure and common decoding helpers. They are informative; normative behavior is defined in Sections 4–12 and 14.
-
-### 20.1 Decoding Overview
-
-- Split input into lines; compute depth from leading spaces and indent size (Section 12).
-- Skip ignorable blank lines outside arrays/tabular rows (Section 12).
-- Decide root form per Section 5.
-- For objects at depth d: process lines at depth d; for arrays at depth d: read rows/list items at depth d+1.
-
-### 20.2 Array Header Parsing
-
-- Locate the first "[ … ]" segment on the line; parse:
-  - Optional leading "#" marker (ignored semantically).
-  - Length N as decimal integer.
-  - Optional delimiter symbol at the end: HTAB or pipe (comma otherwise).
-- If a "{ … }" fields segment occurs between the "]" and the ":", parse field names using the active delimiter; unescape quoted names.
-- Require a colon ":" after the bracket/fields segment.
-- Return the header (key?, length, delimiter, fields?, hasLengthMarker) and any inline values after the colon.
-- Absence of a delimiter symbol in the bracket segment ALWAYS means comma for that header (no inheritance).
-
-### 20.3 parseDelimitedValues
-
-- Iterate characters left-to-right while maintaining a current token and an inQuotes flag.
-- On a double quote, toggle inQuotes.
-- While inQuotes, treat backslash + next char as a literal pair (string parser validates later).
-- Only split on the active delimiter when not in quotes (unquoted occurrences).
-- Trim surrounding spaces around each token. Empty tokens decode to empty string.
-
-### 20.4 Primitive Token Parsing
-
-- If token starts with a quote, it MUST be a properly quoted string (no trailing characters after the closing quote). Unescape using only the five escapes; otherwise MUST error.
-- Else if token is true/false/null → boolean/null.
-- Else if token is numeric without forbidden leading zeros and finite → number.
-- Else → string.
-
-### 20.5 Object and List Item Parsing
-
-- Key-value line: parse a key up to the first colon; missing colon → MUST error. The remainder of the line is the primitive value (if present).
-- Nested object: "key:" with nothing after colon opens a nested object. If this is:
-  - A field inside a regular object: nested fields are at depth +1 relative to that line.
-  - The first field on a list-item hyphen line: nested fields at depth +2 relative to the hyphen line; subsequent fields at +1.
-- List items:
-  - Lines start with "- " at one deeper depth than the parent array header.
-  - After "- ":
-    - If "[ … ]:" appears → inline array item; decode with its own header and active delimiter.
-    - Else if a colon appears → object with first field on hyphen line.
-    - Else → primitive token.
-
-### 20.6 Blank-Line Handling
-
-- Track blank lines during scanning with line numbers and depth.
-- For arrays/tabular rows:
-  - In strict mode, any blank line between the first and last item/row line MUST error.
-  - In non-strict mode, blank lines MAY be ignored and not counted as items/rows.
-- Outside arrays/tabular rows:
-  - Blank lines SHOULD be ignored (do not affect root-form detection or object boundaries).
-
-## 21. Test Suite and Compliance (Informative)
-
-Implementations are encouraged to validate against a comprehensive test suite covering:
-- Primitive encoding/decoding, quoting, control-character escaping.
-- Object key encoding/decoding and order preservation.
-- Primitive arrays (inline), empty arrays.
-- Arrays of arrays (expanded), mixed-length and empty inner arrays.
-- Tabular detection and formatting, including delimiter variations.
-- Mixed arrays and objects-as-list-items behavior, including nested arrays and objects.
-- Whitespace invariants (no trailing spaces/newline).
-- Normalization (BigInt, Date, undefined, NaN/Infinity, functions, symbols).
-- Decoder strict-mode errors: count mismatches, invalid escapes, missing colon, delimiter mismatches, indentation errors, blank-line handling.
-
-## 22. TOON Core Profile (Normative Subset)
-
-This profile captures the most common, memory-friendly rules.
-
-- Character set: UTF-8; LF line endings.
-- Indentation: 2 spaces per level (configurable indentSize).
-  - Strict mode: leading spaces MUST be a multiple of indentSize; tabs in indentation MUST error.
-- Keys:
-  - Unquoted if they match ^[A-Za-z_][\w.]*$; otherwise quoted.
-  - A colon MUST follow a key.
-- Strings:
-  - Only these escapes allowed in quotes: \\, \", \n, \r, \t.
-  - Quote if empty; leading/trailing whitespace; equals true/false/null; numeric-like; contains colon/backslash/quote/brackets/braces/control char; contains the relevant delimiter (active inside arrays, document otherwise); equals "-" or starts with "-".
-- Numbers:
-  - Encoder emits non-exponential decimal; -0 → 0.
-  - Decoder accepts decimal and exponent forms; tokens with forbidden leading zeros decode as strings.
-- Arrays and headers:
-  - Header: [#?N[delim?]] where delim is absent (comma), HTAB (tab), or "|" (pipe).
-  - Keyed header: key[#?N[delim?]]:. Optional fields: {f1<delim>f2}.
-  - Primitive arrays inline: key[N]: v1<delim>v2. Empty arrays: key[0]: (no values).
-  - Tabular arrays: key[N]{fields}: then N rows at depth +1.
-  - Otherwise list form: key[N]: then N items, each starting with "- ".
-- Delimiters:
-  - Only split on the active delimiter from the nearest header. Non-active delimiters never split.
-- Objects as list items:
-  - "- value" (primitive), "- [M]: …" (inline array), or "- key: …" (object).
-  - If first field is "- key:" with nested object: nested fields at +2; subsequent sibling fields at +1.
-- Root form:
-  - Root array if the first depth-0 line is a header (per Section 6).
-  - Root primitive if exactly one non-empty line and it is not a header or key-value.
-  - Otherwise object.
-- Strict mode checks:
-  - All count/width checks; missing colon; invalid escapes; indentation multiple-of-indentSize; delimiter mismatches via count checks; blank lines inside arrays/tabular rows; empty input.
-
-## 23. Versioning and Extensibility
-
-- Backward-compatible evolutions SHOULD preserve current headers, quoting rules, and indentation semantics.
-- Reserved/structural characters (colon, brackets, braces, hyphen) MUST retain current meanings.
-- Future work (non-normative): schemas, comments/annotations, additional delimiter profiles, optional \uXXXX escapes (if added, must be precisely defined).
-
-## 24. Acknowledgments and License
-
-- Credits: Author and contributors; ports in other languages (Elixir, PHP, Python, Ruby, Java, .NET, Swift, Go).
-- License: MIT (see repository for details).
-
----
-
-Appendix: Cross-check With Reference Behavior (Informative)
-
-- The reference encoder/decoder test suites implement:
-  - Safe-unquoted string rules and delimiter-aware quoting (document vs active delimiter).
-  - Header formation and delimiter-aware parsing with active delimiter scoping.
-  - Length marker propagation (encoding) and acceptance (decoding).
-  - Tabular detection requiring uniform keys and primitive-only values.
-  - Objects-as-list-items parsing (+2 nested object rule; +1 siblings).
-  - Whitespace invariants for encoding and strict-mode indentation enforcement for decoding.
-  - Blank-line handling and trailing-newline acceptance.
+This repository (`toon-format/toon`) remains the **reference implementation** in TypeScript/JavaScript. For specification discussions, issues, and contributions, please use the spec repository.

automd.config.ts

@@ -0,0 +1,7 @@
+import type { Config } from 'automd'
+
+const config: Config = {
+  input: ['docs/guide/benchmarks.md'],
+}
+
+export default config

benchmarks/README.md

@@ -3,13 +3,13 @@
 Benchmarks measuring TOON's **token efficiency** and **retrieval accuracy** compared to JSON, XML, YAML, and CSV.
 
 > [!NOTE]
-> Results are automatically embedded in the [main README](../README.md#benchmarks). This guide focuses on running the benchmarks locally.
+> Results are automatically embedded in the [main README](https://github.com/toon-format/toon/#benchmarks). This guide focuses on running the benchmarks locally.
 
 ## Quick Start
 
 ```bash
 # Run token efficiency benchmark
-pnpm benchmark:token-efficiency
+pnpm benchmark:tokens
 
 # Run retrieval accuracy benchmark (requires API keys)
 pnpm benchmark:accuracy
@@ -25,30 +25,30 @@ Measures token count reduction across JSON, XML, YAML, CSV, and TOON:
 4. Calculate savings and generate report
 
 ```bash
-pnpm benchmark:token-efficiency
+pnpm benchmark:tokens
 ```
 
 Results are saved to `results/token-efficiency.md`.
 
 ## Retrieval Accuracy Benchmark
 
-Tests how well LLMs can answer questions about data in different formats (TOON, JSON, XML, YAML, CSV):
+Tests how well LLMs can answer questions about data in different formats (TOON, JSON, JSON compact, XML, YAML, CSV):
 
-1. Generate 154 questions across 4 datasets
-2. Convert each dataset to all 5 formats
+1. Generate 209 questions across 11 datasets (6 primary + 5 structural validation; CSV only included for datasets with flat/tabular structure)
+2. Convert each dataset to all supported formats
 3. Query each LLM with formatted data + question
-4. Validate answers using `gpt-5-nano` as judge
+4. Validate answers deterministically using type-aware comparison (no LLM judge needed)
 5. Aggregate metrics and generate report
 
 ### Setup
 
-1. Edit [`src/evaluate.ts`](./src/evaluate.ts) and add models to the `models` array:
+1. Edit [`src/evaluate.ts`](./src/evaluate.ts) and add models to the exported `models` array:
    ```ts
-   export const models: LanguageModelV2[] = [
+   export const models: LanguageModelV3[] = [
      openai('gpt-5-nano'),
      anthropic('claude-haiku-4-5-20251001'),
-     google('gemini-2.5-flash'),
-     xai('grok-4-fast-non-reasoning'),
+     google('gemini-3-flash-preview'),
+     xai('grok-4-1-fast-non-reasoning'),
      // Add your models here
    ]
    ```
@@ -95,10 +95,22 @@ src/
 ├── datasets.ts                   # Test data generators
 ├── evaluate.ts                   # LLM evaluation
 ├── formatters.ts                 # Format converters
-├── questions.ts                  # Question generation
+├── normalize.ts                  # Answer normalization
 ├── report.ts                     # Markdown reports
 ├── storage.ts                    # Result caching
-└── utils.ts                      # Helpers
+├── types.ts                      # Type definitions
+├── utils.ts                      # Helpers
+└── questions/                    # Question generators
+    ├── analytics.ts
+    ├── event-logs.ts
+    ├── github.ts
+    ├── index.ts
+    ├── nested-config.ts
+    ├── nested.ts
+    ├── structural-validation.ts
+    ├── structure.ts
+    ├── tabular.ts
+    └── utils.ts
 data/
 └── github-repos.json             # Top 100 GitHub repos
 results/

benchmarks/package.json

@@ -3,27 +3,26 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "benchmark:token-efficiency": "tsx scripts/token-efficiency-benchmark.ts",
-    "benchmark:accuracy": "tsx --env-file=.env scripts/accuracy-benchmark.ts",
-    "fetch:github-repos": "tsx scripts/fetch-github-repos.ts"
+    "benchmark:tokens": "node scripts/token-efficiency-benchmark.ts",
+    "benchmark:accuracy": "node --env-file=.env scripts/accuracy-benchmark.ts",
+    "fetch:github-repos": "node scripts/fetch-github-repos.ts"
   },
   "devDependencies": {
-    "@ai-sdk/anthropic": "^2.0.37",
-    "@ai-sdk/google": "^2.0.23",
-    "@ai-sdk/openai": "^2.0.53",
-    "@ai-sdk/provider": "^2.0.0",
-    "@ai-sdk/xai": "^2.0.28",
-    "@antfu/eslint-config": "^6.1.0",
-    "@clack/prompts": "^0.11.0",
-    "@faker-js/faker": "^10.1.0",
-    "ai": "^5.0.80",
-    "csv-stringify": "^6.6.0",
-    "fast-xml-parser": "^5.3.0",
-    "gpt-tokenizer": "^3.2.0",
-    "ofetch": "^1.4.1",
-    "p-map": "^7.0.3",
-    "p-queue": "^9.0.0",
-    "unstorage": "^1.17.1",
-    "yaml": "^2.8.1"
+    "@ai-sdk/anthropic": "^3.0.97",
+    "@ai-sdk/google": "^3.0.94",
+    "@ai-sdk/openai": "^3.0.85",
+    "@ai-sdk/provider": "^3.0.14",
+    "@ai-sdk/xai": "^3.0.108",
+    "@clack/prompts": "^1.7.0",
+    "@faker-js/faker": "^10.5.0",
+    "ai": "^6.0.228",
+    "csv-stringify": "^6.8.1",
+    "fast-xml-parser": "^5.10.0",
+    "gpt-tokenizer": "^3.4.0",
+    "ofetch": "^1.5.1",
+    "p-map": "^7.0.5",
+    "p-queue": "^9.3.1",
+    "unstorage": "^1.17.5",
+    "yaml": "^2.9.0"
   }
 }

benchmarks/results/retrieval-accuracy.md

@@ -1,173 +1,339 @@
-### Retrieval Accuracy
-
-Accuracy across **4 LLMs** on 154 data retrieval questions:
-
-```
-gpt-5-nano
-→ toon         ███████████████████░  96.1% (148/154)
-  csv          ██████████████████░░  90.3% (139/154)
-  yaml         ██████████████████░░  89.0% (137/154)
-  json         ██████████████████░░  87.7% (135/154)
-  xml          █████████████████░░░  83.8% (129/154)
-
-claude-haiku-4-5-20251001
-  yaml         ██████████░░░░░░░░░░  49.4% (76/154)
-→ toon         ██████████░░░░░░░░░░  48.1% (74/154)
-  csv          ██████████░░░░░░░░░░  48.1% (74/154)
-  json         █████████░░░░░░░░░░░  47.4% (73/154)
-  xml          █████████░░░░░░░░░░░  46.8% (72/154)
-
-gemini-2.5-flash
-  csv          ██████████████████░░  87.7% (135/154)
-  xml          █████████████████░░░  85.1% (131/154)
-→ toon         █████████████████░░░  83.8% (129/154)
-  json         ████████████████░░░░  78.6% (121/154)
-  yaml         ███████████████░░░░░  76.6% (118/154)
-
-grok-4-fast-non-reasoning
-→ toon         ██████████░░░░░░░░░░  48.7% (75/154)
-  json         ██████████░░░░░░░░░░  48.1% (74/154)
-  xml          █████████░░░░░░░░░░░  47.4% (73/154)
-  yaml         █████████░░░░░░░░░░░  46.8% (72/154)
-  csv          █████████░░░░░░░░░░░  45.5% (70/154)
-```
-
-**Key tradeoff:** TOON achieves **69.2% accuracy** (vs JSON's 65.4%) while using **46.3% fewer tokens** on these datasets.
+Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
 
 <details>
-<summary><strong>Performance by dataset and model</strong></summary>
+<summary><strong>Show Dataset Catalog</strong></summary>
+
+#### Dataset Catalog
+
+| Dataset | Rows | Structure | CSV Support | Eligibility |
+| ------- | ---- | --------- | ----------- | ----------- |
+| Uniform employee records | 100 | uniform | ✓ | 100% |
+| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
+| Time-series analytics data | 60 | uniform | ✓ | 100% |
+| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
+| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
+| Deeply nested configuration | 11 | deep | ✗ | 0% |
+| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
+| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
+| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
+| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
+| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
+
+**Structure classes:**
+- **uniform**: All objects have identical fields with primitive values
+- **semi-uniform**: Mix of uniform and non-uniform structures
+- **nested**: Objects with nested structures (nested objects or arrays)
+- **deep**: Highly nested with minimal tabular eligibility
+
+**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
+
+**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
+
+</details>
+
+#### Efficiency Ranking (Accuracy per 1K Tokens)
+
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
+
+```
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
+```
+
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
+
+**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
+
+#### Per-Model Accuracy
+
+Accuracy across 4 LLMs on 209 data retrieval questions:
+
+```
+claude-haiku-4-5-20251001
+→ TOON           ████████████░░░░░░░░    59.8% (125/209)
+  JSON           ███████████░░░░░░░░░    57.4% (120/209)
+  YAML           ███████████░░░░░░░░░    56.0% (117/209)
+  XML            ███████████░░░░░░░░░    55.5% (116/209)
+  JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
+  CSV            ██████████░░░░░░░░░░    50.5% (55/109)
+
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
+
+gpt-5-nano
+→ TOON           ██████████████████░░    90.9% (190/209)
+  JSON compact   ██████████████████░░    90.9% (190/209)
+  JSON           ██████████████████░░    89.0% (186/209)
+  CSV            ██████████████████░░    89.0% (97/109)
+  YAML           █████████████████░░░    87.1% (182/209)
+  XML            ████████████████░░░░    80.9% (169/209)
+
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
+```
+
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
+
+<details>
+<summary><strong>Performance by dataset, model, and question type</strong></summary>
+
+#### Performance by Question Type
+
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
+| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
 
 #### Performance by Dataset
 
-##### Uniform employee records (TOON optimal format)
+##### Uniform employee records
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 67.0% | 2,337 | 134/200 |
-| `toon` | 66.5% | 2,483 | 133/200 |
-| `yaml` | 65.5% | 4,969 | 131/200 |
-| `json` | 63.5% | 6,347 | 127/200 |
-| `xml` | 66.5% | 7,314 | 133/200 |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
 
 ##### E-commerce orders with nested structures
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 78.8% | 5,967 | 126/160 |
-| `csv` | 71.9% | 6,735 | 115/160 |
-| `yaml` | 71.9% | 7,328 | 115/160 |
-| `json` | 73.1% | 9,694 | 117/160 |
-| `xml` | 73.8% | 10,992 | 118/160 |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
 
 ##### Time-series analytics data
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 67.6% | 1,393 | 92/136 |
-| `toon` | 67.6% | 1,515 | 92/136 |
-| `yaml` | 64.7% | 2,938 | 88/136 |
-| `json` | 68.4% | 3,665 | 93/136 |
-| `xml` | 66.2% | 4,376 | 90/136 |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
 
 ##### Top 100 GitHub repositories
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 64.2% | 8,513 | 77/120 |
-| `toon` | 62.5% | 8,745 | 75/120 |
-| `yaml` | 57.5% | 13,129 | 69/120 |
-| `json` | 55.0% | 15,145 | 66/120 |
-| `xml` | 53.3% | 17,095 | 64/120 |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
+
+##### Semi-uniform event logs
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
+
+##### Deeply nested configuration
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
+
+##### Valid complete dataset (control)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
+
+##### Array truncated: 3 rows removed from end
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
+
+##### Extra rows added beyond declared length
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
+
+##### Inconsistent field count (missing salary in row 10)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
+
+##### Missing required fields (no email in multiple rows)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
 
 #### Performance by Model
 
-##### gpt-5-nano
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 96.1% | 148/154 |
-| `csv` | 90.3% | 139/154 |
-| `yaml` | 89.0% | 137/154 |
-| `json` | 87.7% | 135/154 |
-| `xml` | 83.8% | 129/154 |
-
 ##### claude-haiku-4-5-20251001
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `yaml` | 49.4% | 76/154 |
-| `toon` | 48.1% | 74/154 |
-| `csv` | 48.1% | 74/154 |
-| `json` | 47.4% | 73/154 |
-| `xml` | 46.8% | 72/154 |
+| `toon` | 59.8% | 125/209 |
+| `json-pretty` | 57.4% | 120/209 |
+| `yaml` | 56.0% | 117/209 |
+| `xml` | 55.5% | 116/209 |
+| `json-compact` | 55.0% | 115/209 |
+| `csv` | 50.5% | 55/109 |
 
-##### gemini-2.5-flash
+##### gemini-3-flash-preview
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `csv` | 87.7% | 135/154 |
-| `xml` | 85.1% | 131/154 |
-| `toon` | 83.8% | 129/154 |
-| `json` | 78.6% | 121/154 |
-| `yaml` | 76.6% | 118/154 |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
 
-##### grok-4-fast-non-reasoning
+##### gpt-5-nano
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `toon` | 48.7% | 75/154 |
-| `json` | 48.1% | 74/154 |
-| `xml` | 47.4% | 73/154 |
-| `yaml` | 46.8% | 72/154 |
-| `csv` | 45.5% | 70/154 |
+| `toon` | 90.9% | 190/209 |
+| `json-compact` | 90.9% | 190/209 |
+| `json-pretty` | 89.0% | 186/209 |
+| `csv` | 89.0% | 97/109 |
+| `yaml` | 87.1% | 182/209 |
+| `xml` | 80.9% | 169/209 |
+
+##### grok-4-1-fast-non-reasoning
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
 
 </details>
 
-<details>
-<summary><strong>How the benchmark works</strong></summary>
-
 #### What's Being Measured
 
-This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it (this does **not** test model's ability to generate TOON output).
+This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
 
 #### Datasets Tested
 
-Four datasets designed to test different structural patterns (all contain arrays of uniform objects, TOON's optimal format):
+Eleven datasets designed to test different structural patterns and validation capabilities:
+
+**Primary datasets:**
 
 1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
 2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
 3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
 4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
+5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
+6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
+
+**Structural validation datasets:**
+
+7. **Control**: Valid complete dataset (baseline for validation)
+8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
+9. **Extra rows**: Array with 3 additional rows beyond declared length
+10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
+11. **Missing fields**: Systematic field omissions (no email in multiple rows)
 
 #### Question Types
 
-154 questions are generated dynamically across three categories:
+209 questions are generated dynamically across five categories:
 
-- **Field retrieval (40%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
+- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
   - Example: "What is Alice's salary?" → `75000`
   - Example: "How many items are in order ORD-0042?" → `3`
   - Example: "What is the customer name for order ORD-0042?" → `John Doe`
 
-- **Aggregation (32%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
+- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
   - Example: "How many employees work in Engineering?" → `17`
   - Example: "What is the total revenue across all orders?" → `45123.50`
   - Example: "How many employees have salary > 80000?" → `23`
 
-- **Filtering (28%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
+- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
   - Example: "How many employees in Sales have salary > 80000?" → `5`
   - Example: "How many active employees have more than 10 years of experience?" → `8`
 
+- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
+  - Example: "How many employees are in the dataset?" → `100`
+  - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
+  - Example: "What is the department of the last employee?" → `Sales`
+
+- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
+  - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
+  - Tests TOON's `[N]` length validation and `{fields}` consistency checking
+  - Demonstrates CSV's lack of structural validation capabilities
+
 #### Evaluation Process
 
-1. **Format conversion**: Each dataset is converted to all 5 formats (TOON, CSV, XML, JSON, YAML).
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
 2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
-3. **Validate with LLM-as-judge**: `gpt-5-nano` validates if the answer is semantically correct (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`).
+3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
 
 #### Models & Configuration
 
-- **Models tested**: `gpt-5-nano`, `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `grok-4-fast-non-reasoning`
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
 - **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
 - **Temperature**: Not set (models use their defaults)
-- **Total evaluations**: 154 questions × 5 formats × 4 models = 3,080 LLM calls
-
-</details>
+- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls

benchmarks/results/token-efficiency.md

@@ -1,38 +1,155 @@
-### Token Efficiency
+#### Mixed-Structure Track
+
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
 
 ```
-⭐ GitHub Repositories       ██████████████░░░░░░░░░░░   8,745 tokens
-                             vs JSON: 15,145  (-42.3%)
-                             vs YAML: 13,129  (-33.4%)
-                             vs XML:  17,095  (-48.8%)
+🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
+   │
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
 
-📈 Daily Analytics           ██████████░░░░░░░░░░░░░░░   4,507 tokens
-                             vs JSON: 10,977  (-58.9%)
-                             vs YAML:  8,810  (-48.8%)
-                             vs XML:  13,128  (-65.7%)
+🧾 Semi-uniform event logs  ┊  Tabular: 50%
+   │
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
 
-🛒 E-Commerce Order          ████████████████░░░░░░░░░     166 tokens
-                             vs JSON:    257  (-35.4%)
-                             vs YAML:    197  (-15.7%)
-                             vs XML:     271  (-38.7%)
+🧩 Deeply nested configuration  ┊  Tabular: 0%
+   │
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
 
-─────────────────────────────────────────────────────────────────────
-Total                        █████████████░░░░░░░░░░░░  13,418 tokens
-                             vs JSON: 26,379  (-49.1%)
-                             vs YAML: 22,136  (-39.4%)
-                             vs XML:  30,494  (-56.0%)
+──────────────────────────────────── Total ────────────────────────────────────
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
+```
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+```
+👥 Uniform employee records  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
+
+📈 Time-series analytics data  ┊  Tabular: 100%
+   │
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
+
+⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
 ```
 
 <details>
-<summary><strong>View detailed examples</strong></summary>
+<summary><strong>Show detailed examples</strong></summary>
 
-#### ⭐ GitHub Repositories
+#### 📈 Time-series analytics data
 
-**Configuration:** Top 100 GitHub repositories with stars, forks, and metadata
+**Savings:** 13,130 tokens (59.0% reduction vs JSON)
+
+**JSON** (22,245 tokens):
+
+```json
+{
+  "metrics": [
+    {
+      "date": "2025-01-01",
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
+    },
+    {
+      "date": "2025-01-02",
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
+    },
+    {
+      "date": "2025-01-03",
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
+    },
+    {
+      "date": "2025-01-04",
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
+    },
+    {
+      "date": "2025-01-05",
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
+    }
+  ]
+}
+```
+
+**TOON** (9,115 tokens):
+
+```
+metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
+```
+
+---
+
+#### ⭐ Top 100 GitHub repositories
 
 **Savings:** 6,400 tokens (42.3% reduction vs JSON)
 
-**JSON** (15,145 tokens):
+**JSON** (15,144 tokens):
 
 ```json
 {
@@ -80,7 +197,7 @@ Total                        █████████████░░░░
 }
 ```
 
-**TOON** (8,745 tokens):
+**TOON** (8,744 tokens):
 
 ```
 repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
@@ -89,72 +206,4 @@ repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watc
   21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
 ```
 
----
-
-#### 📈 Daily Analytics
-
-**Configuration:** 180 days of web metrics (views, clicks, conversions, revenue)
-
-**Savings:** 6,470 tokens (58.9% reduction vs JSON)
-
-**JSON** (10,977 tokens):
-
-```json
-{
-  "metrics": [
-    {
-      "date": "2025-01-01",
-      "views": 6890,
-      "clicks": 401,
-      "conversions": 23,
-      "revenue": 6015.59,
-      "bounceRate": 0.63
-    },
-    {
-      "date": "2025-01-02",
-      "views": 6940,
-      "clicks": 323,
-      "conversions": 37,
-      "revenue": 9086.44,
-      "bounceRate": 0.36
-    },
-    {
-      "date": "2025-01-03",
-      "views": 4390,
-      "clicks": 346,
-      "conversions": 26,
-      "revenue": 6360.75,
-      "bounceRate": 0.48
-    },
-    {
-      "date": "2025-01-04",
-      "views": 3429,
-      "clicks": 231,
-      "conversions": 13,
-      "revenue": 2360.96,
-      "bounceRate": 0.65
-    },
-    {
-      "date": "2025-01-05",
-      "views": 5804,
-      "clicks": 186,
-      "conversions": 22,
-      "revenue": 2535.96,
-      "bounceRate": 0.37
-    }
-  ]
-}
-```
-
-**TOON** (4,507 tokens):
-
-```
-metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
-  2025-01-01,6890,401,23,6015.59,0.63
-  2025-01-02,6940,323,37,9086.44,0.36
-  2025-01-03,4390,346,26,6360.75,0.48
-  2025-01-04,3429,231,13,2360.96,0.65
-  2025-01-05,5804,186,22,2535.96,0.37
-```
-
 </details>

benchmarks/scripts/accuracy-benchmark.ts

@@ -1,18 +1,87 @@
-import type { Question } from '../src/types'
+import type { Question } from '../src/types.ts'
+import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import process from 'node:process'
 import * as prompts from '@clack/prompts'
 import PQueue from 'p-queue'
-import { DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants'
-import { datasets } from '../src/datasets'
-import { evaluateQuestion, models } from '../src/evaluate'
-import { formatters } from '../src/formatters'
-import { generateQuestions } from '../src/questions'
-import { calculateFormatResults, calculateTokenCounts, saveResults } from '../src/report'
-import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage'
+import { BENCHMARKS_DIR, DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants.ts'
+import { ACCURACY_DATASETS } from '../src/datasets.ts'
+import { evaluateQuestion, models } from '../src/evaluate.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { generateQuestions } from '../src/questions/index.ts'
+import { calculateFormatResults, calculateTokenCounts, generateAccuracyReport } from '../src/report.ts'
+import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage.ts'
+import { ensureDir } from '../src/utils.ts'
+
+// Constants
+const PROGRESS_UPDATE_INTERVAL = 10
+const RATE_LIMIT_INTERVAL_MS = 60_000
 
 prompts.intro('Retrieval Accuracy Benchmark')
 
+/**
+ * Generate evaluation tasks for a model
+ */
+function generateEvaluationTasks(questions: Question[]): { question: Question, formatName: string }[] {
+  const tasks: { question: Question, formatName: string }[] = []
+
+  for (const question of questions) {
+    for (const [formatName] of Object.entries(formatters)) {
+      // Skip CSV for datasets that don't support it
+      const dataset = ACCURACY_DATASETS.find(d => d.name === question.dataset)
+      if (formatName === 'csv' && dataset && !supportsCSV(dataset))
+        continue
+
+      tasks.push({ question, formatName })
+    }
+  }
+
+  return tasks
+}
+
+/**
+ * Check which models already have saved results
+ */
+async function checkExistingResults(activeModels: typeof models) {
+  const existingModelResults: Record<string, boolean> = {}
+
+  for (const model of activeModels) {
+    const existingResult = await hasModelResults(model.modelId)
+    if (existingResult)
+      existingModelResults[model.modelId] = existingResult
+  }
+
+  return existingModelResults
+}
+
+/**
+ * Create a progress updater function
+ */
+function createProgressUpdater(spinner: ReturnType<typeof prompts.spinner>, total: number) {
+  let completed = 0
+
+  return () => {
+    completed++
+    if (completed % PROGRESS_UPDATE_INTERVAL === 0 || completed === total) {
+      const percent = ((completed / total) * 100).toFixed(1)
+      spinner.message(`Progress: ${completed}/${total} (${percent}%)`)
+    }
+  }
+}
+
+/**
+ * Create a rate-limited queue for model evaluation
+ */
+function createEvaluationQueue(modelId: string) {
+  const rpmLimit = MODEL_RPM_LIMITS[modelId]
+
+  return new PQueue({
+    concurrency: DEFAULT_CONCURRENCY,
+    intervalCap: rpmLimit ?? Infinity,
+    interval: rpmLimit ? RATE_LIMIT_INTERVAL_MS : 0,
+  })
+}
+
 // Prompt user to select which models to benchmark
 const modelChoices = models.map(({ modelId }) => ({
   value: modelId,
@@ -35,15 +104,10 @@ const activeModels = models.filter(m => selectedModels.includes(m.modelId))
 prompts.log.info(`Selected ${activeModels.length} model(s): ${activeModels.map(m => m.modelId).join(', ')}`)
 
 // Check which models already have results
-const existingModelResults: Record<string, boolean> = {}
-for (const model of activeModels) {
-  const existingResult = await hasModelResults(model.modelId)
-  if (existingResult)
-    existingModelResults[model.modelId] = existingResult
-}
+const existingModelResults = await checkExistingResults(activeModels)
 
 if (Object.keys(existingModelResults).length > 0) {
-  prompts.log.info(`Found existing results for ${Object.values(existingModelResults).length} model(s)`)
+  prompts.log.info(`Found existing results for ${Object.keys(existingModelResults).length} model(s)`)
 }
 
 if (DRY_RUN) {
@@ -73,31 +137,22 @@ for (const model of activeModels) {
   prompts.log.step(`Running benchmark for ${modelId}`)
 
   // Generate evaluation tasks for this model
-  const tasks: { question: Question, formatName: string }[] = []
-  for (const question of questions) {
-    for (const [formatName] of Object.entries(formatters)) {
-      tasks.push({ question, formatName })
-    }
-  }
+  const tasks = generateEvaluationTasks(questions)
 
   const total = tasks.length
   const rpmLimit = MODEL_RPM_LIMITS[modelId]
-  const queue = new PQueue({
-    concurrency: DEFAULT_CONCURRENCY,
-    intervalCap: rpmLimit ?? Infinity,
-    interval: rpmLimit ? 60_000 : 0,
-  })
+  const queue = createEvaluationQueue(modelId)
 
   const evalSpinner = prompts.spinner()
   evalSpinner.start(`Running ${total} evaluations (concurrency: ${DEFAULT_CONCURRENCY}, RPM limit: ${rpmLimit ?? 'unlimited'})`)
 
-  let completed = 0
+  const updateProgress = createProgressUpdater(evalSpinner, total)
 
   // Queue all tasks
   const modelResultPromises = tasks.map(task =>
     queue.add(async () => {
       // Format data on-demand
-      const dataset = datasets.find(d => d.name === task.question.dataset)!
+      const dataset = ACCURACY_DATASETS.find(d => d.name === task.question.dataset)!
       const formatter = formatters[task.formatName]!
       const formattedData = formatter(dataset.data)
 
@@ -109,11 +164,7 @@ for (const model of activeModels) {
       })
 
       // Progress update after task completes
-      completed++
-      if (completed % 10 === 0 || completed === total) {
-        const percent = ((completed / total) * 100).toFixed(1)
-        evalSpinner.message(`Progress: ${completed}/${total} (${percent}%)`)
-      }
+      updateProgress()
 
       return result
     }),
@@ -142,13 +193,15 @@ if (allResults.length === 0) {
   process.exit(0)
 }
 
-// Calculate token counts freshly (deterministic, no need to persist)
 const tokenCounts = calculateTokenCounts(formatters)
-
-// Calculate format statistics and save report
 const formatResults = calculateFormatResults(allResults, tokenCounts)
-const resultsDir = await saveResults(allResults, formatResults, questions, tokenCounts)
+const accuracyReport = generateAccuracyReport(allResults, formatResults, tokenCounts)
+
+const resultsDir = path.join(BENCHMARKS_DIR, 'results')
+await ensureDir(resultsDir)
+
+const outputFilePath = path.join(resultsDir, 'retrieval-accuracy.md')
+await fsp.writeFile(outputFilePath, accuracyReport)
 
-const reportPath = path.join(resultsDir, 'retrieval-accuracy.md')
-prompts.log.info(`Report saved to: \`${path.relative(ROOT_DIR, reportPath)}\``)
 reportSpinner.stop('Report generation complete!')
+prompts.log.info(`Report saved to: \`${path.relative(ROOT_DIR, outputFilePath)}\``)

benchmarks/scripts/fetch-github-repos.ts

@@ -1,10 +1,11 @@
+import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import process from 'node:process'
 import * as prompts from '@clack/prompts'
 import { ofetch } from 'ofetch'
 import pMap from 'p-map'
-import { BENCHMARKS_DIR } from '../src/constants'
-import { ensureDir, saveJsonFile } from '../src/utils'
+import { BENCHMARKS_DIR } from '../src/constants.ts'
+import { ensureDir } from '../src/utils.ts'
 
 prompts.intro('GitHub Repositories Fetcher')
 
@@ -79,7 +80,8 @@ async function saveRepos(repos: Record<string, any>[]): Promise<void> {
   const outputFile = path.join(outputDir, 'github-repos.json')
 
   await ensureDir(outputDir)
-  await saveJsonFile(outputFile, repos)
+  const jsonOutput = JSON.stringify(repos, undefined, 2)
+  await fsp.writeFile(outputFile, `${jsonOutput}\n`, 'utf-8')
 
   const relativePath = path.relative(BENCHMARKS_DIR, outputFile)
   prompts.log.info(`Result saved to \`${relativePath}\``)

benchmarks/scripts/token-efficiency-benchmark.ts

@@ -1,198 +1,314 @@
+import type { Dataset } from '../src/types.ts'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import * as prompts from '@clack/prompts'
-import { encode } from '../../src/index'
-import githubRepos from '../data/github-repos.json' with { type: 'json' }
-import { BENCHMARKS_DIR, ROOT_DIR } from '../src/constants'
-import { generateAnalyticsData, generateOrderData } from '../src/datasets'
-import { formatters } from '../src/formatters'
-import { createProgressBar, ensureDir, tokenize } from '../src/utils'
+import { encode } from '../../packages/toon/src/index.ts'
+import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants.ts'
+import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { createProgressBar, ensureDir, tokenize } from '../src/utils.ts'
 
 interface FormatMetrics {
   name: string
   tokens: number
   savings: number
-  savingsPercent: string
+  savingsPercent: number
 }
 
 interface BenchmarkResult {
-  name: string
-  emoji: string
-  description: string
-  data: Record<string, any>
+  dataset: Dataset
   formats: FormatMetrics[]
-  showDetailed: boolean
 }
 
-const BENCHMARK_EXAMPLES = [
-  {
-    name: 'GitHub Repositories',
-    emoji: '⭐',
-    description: 'Top 100 GitHub repositories with stars, forks, and metadata',
-    getData: () => ({ repositories: githubRepos }),
-    showDetailed: true,
-  },
-  {
-    name: 'Daily Analytics',
-    emoji: '📈',
-    description: '180 days of web metrics (views, clicks, conversions, revenue)',
-    getData: () => generateAnalyticsData(180),
-    showDetailed: true,
-  },
-  {
-    name: 'E-Commerce Order',
-    emoji: '🛒',
-    description: 'Single nested order with customer and items',
-    getData: generateOrderData,
-    showDetailed: false,
-  },
-] as const
+// Constants
+const DATASET_ICONS: Record<string, string> = {
+  'tabular': '👥',
+  'nested': '🛒',
+  'analytics': '📈',
+  'github': '⭐',
+  'event-logs': '🧾',
+  'nested-config': '🧩',
+}
+
+const COMPARISON_FORMAT_ORDER = ['json-pretty', 'json-compact', 'yaml', 'xml'] as const
+
+const PROGRESS_BAR_WIDTH = 20
+const TOKEN_PADDING = 7
+
+const DEFAULT_DATASET_ICON = '📊'
+
+const DETAILED_EXAMPLE_DATASETS = ['github', 'analytics'] as const
+const GITHUB_REPO_LIMIT = 3
+const GITHUB_DESC_LIMIT = 80
+const ANALYTICS_METRICS_LIMIT = 5
 
 prompts.intro('Token Efficiency Benchmark')
 
-// Calculate total savings
-let totalJsonTokens = 0
-let totalToonTokens = 0
-let totalXmlTokens = 0
-let totalYamlTokens = 0
+/**
+ * Format a comparison line showing savings vs TOON
+ */
+function formatComparisonLine(format: FormatMetrics, isLast: boolean = false): string {
+  const label = FORMATTER_DISPLAY_NAMES[format.name] || format.name.toUpperCase()
+  const signedPercent = format.savingsPercent >= 0
+    ? `−${format.savingsPercent.toFixed(1)}%`
+    : `+${Math.abs(format.savingsPercent).toFixed(1)}%`
+  const connector = isLast ? '└─' : '├─'
+  const tokenStr = format.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+  return `${connector} vs ${label.padEnd(13)} ${`(${signedPercent})`.padEnd(20)}   ${tokenStr} tokens`
+}
+
+/**
+ * Calculate total tokens and savings for a set of datasets
+ */
+function calculateTotalMetrics(datasets: BenchmarkResult[], formatNames: readonly string[]) {
+  const totalToonTokens = datasets.reduce((sum, r) => {
+    const toon = r.formats.find(f => f.name === 'toon')!
+    return sum + toon.tokens
+  }, 0)
+
+  const totals = formatNames.map((formatName) => {
+    const totalTokens = datasets.reduce((sum, r) => {
+      const format = r.formats.find(f => f.name === formatName)
+      return sum + (format?.tokens || 0)
+    }, 0)
+    const savings = totalTokens - totalToonTokens
+    const savingsPercent = (savings / totalTokens) * 100
+    return { name: formatName, tokens: totalTokens, savingsPercent }
+  })
+
+  return { totalToonTokens, totals }
+}
+
+/**
+ * Generate total lines for a track
+ */
+function generateTotalLines(
+  totalToonTokens: number,
+  totals: { name: string, tokens: number, savingsPercent: number }[],
+  baselineFormat?: { name: string, tokens: number },
+) {
+  const separatorHalf = '─'.repeat(36)
+  const lines = [`${separatorHalf} Total ${separatorHalf}`]
+
+  if (baselineFormat) {
+    // Flat-only track with CSV baseline
+    const csvPercentage = Math.min(100, (baselineFormat.tokens / totalToonTokens) * 100)
+    const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
+    const csvStr = baselineFormat.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   CSV                 ${csvBar}   ${csvStr} tokens`)
+
+    const overheadPercent = ((totalToonTokens - baselineFormat.tokens) / baselineFormat.tokens) * 100
+    const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
+    const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   TOON                ${toonBar}   ${toonStr} tokens   (+${overheadPercent.toFixed(1)}% vs CSV)`)
+  }
+  else {
+    // Mixed-structure track
+    const totalPercentage = Math.min(100, (totalToonTokens / totals[0]!.tokens) * 100)
+    const totalBar = createProgressBar(totalPercentage, 100, PROGRESS_BAR_WIDTH)
+    const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   TOON                ${totalBar}   ${toonStr} tokens`)
+  }
+
+  // Add comparison lines
+  for (let i = 0; i < totals.length; i++) {
+    const format = totals[i]!
+    const isLast = i === totals.length - 1
+    lines.push(`   ${formatComparisonLine({
+      name: format.name,
+      tokens: format.tokens,
+      savings: 0, // Not used in this context
+      savingsPercent: format.savingsPercent,
+    }, isLast)}`)
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Generate bar chart for a dataset
+ */
+function generateDatasetChart(result: BenchmarkResult): string {
+  const { dataset, formats } = result
+  const toon = formats.find(f => f.name === 'toon')!
+  const jsonPretty = formats.find(f => f.name === 'json-pretty')!
+
+  const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
+  const eligibility = dataset.metadata.tabularEligibility
+  const name = dataset.description
+
+  const percentage = Math.min(100, 100 - jsonPretty.savingsPercent)
+  const bar = createProgressBar(percentage, 100, PROGRESS_BAR_WIDTH)
+  const toonStr = toon.tokens.toLocaleString('en-US')
+
+  const line1 = `${emoji} ${name}  ┊  Tabular: ${eligibility}%`
+  const line2 = `   │`
+  const line3 = `   TOON                ${bar}   ${toonStr.padStart(TOKEN_PADDING)} tokens`
+
+  const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
+    const format = formats.find(f => f.name === formatName)
+    if (!format)
+      return undefined
+
+    return `   ${formatComparisonLine(format, index === array.length - 1)}`
+  }).filter(Boolean)
+
+  return [line1, line2, line3, ...comparisonLines].join('\n')
+}
 
 const results: BenchmarkResult[] = []
 
-for (const example of BENCHMARK_EXAMPLES) {
-  const data = example.getData()
+// Calculate token counts for all datasets
+for (const dataset of TOKEN_EFFICIENCY_DATASETS) {
+  const formatMetrics: FormatMetrics[] = []
+  const tokensByFormat: Record<string, number> = {}
 
-  const jsonString = JSON.stringify(data, undefined, 2)
-  const toonString = encode(data)
-  const xmlString = formatters.xml!(data)
-  const yamlString = formatters.yaml!(data)
+  // Calculate tokens for each format
+  for (const [formatName, formatter] of Object.entries(formatters)) {
+    // Skip CSV for datasets that don't support it
+    if (formatName === 'csv' && !supportsCSV(dataset))
+      continue
 
-  const jsonTokens = tokenize(jsonString)
-  const toonTokens = tokenize(toonString)
-  const xmlTokens = tokenize(xmlString)
-  const yamlTokens = tokenize(yamlString)
+    const formattedData = formatter(dataset.data)
+    const tokens = tokenize(formattedData)
+    tokensByFormat[formatName] = tokens
+  }
 
-  const jsonSavings = jsonTokens - toonTokens
-  const xmlSavings = xmlTokens - toonTokens
-  const yamlSavings = yamlTokens - toonTokens
-
-  totalJsonTokens += jsonTokens
-  totalToonTokens += toonTokens
-  totalXmlTokens += xmlTokens
-  totalYamlTokens += yamlTokens
+  // Calculate savings vs TOON
+  const toonTokens = tokensByFormat.toon!
+  for (const [formatName, tokens] of Object.entries(tokensByFormat)) {
+    const savings = tokens - toonTokens
+    formatMetrics.push({
+      name: formatName,
+      tokens,
+      savings,
+      savingsPercent: formatName === 'toon' ? 0 : (savings / tokens) * 100,
+    })
+  }
 
   results.push({
-    name: example.name,
-    emoji: example.emoji,
-    description: example.description,
-    data,
-    formats: [
-      {
-        name: 'toon',
-        tokens: toonTokens,
-        savings: 0,
-        savingsPercent: '0.0',
-      },
-      {
-        name: 'json',
-        tokens: jsonTokens,
-        savings: jsonSavings,
-        savingsPercent: ((jsonSavings / jsonTokens) * 100).toFixed(1),
-      },
-      {
-        name: 'xml',
-        tokens: xmlTokens,
-        savings: xmlSavings,
-        savingsPercent: ((xmlSavings / xmlTokens) * 100).toFixed(1),
-      },
-      {
-        name: 'yaml',
-        tokens: yamlTokens,
-        savings: yamlSavings,
-        savingsPercent: ((yamlSavings / yamlTokens) * 100).toFixed(1),
-      },
-    ],
-    showDetailed: example.showDetailed,
+    dataset,
+    formats: formatMetrics,
   })
 }
 
-const totalJsonSavings = totalJsonTokens - totalToonTokens
-const totalJsonSavingsPercent = ((totalJsonSavings / totalJsonTokens) * 100).toFixed(1)
+// Separate datasets by CSV support
+const mixedStructureDatasets = results.filter(r => !supportsCSV(r.dataset))
+const flatOnlyDatasets = results.filter(r => supportsCSV(r.dataset))
 
-const totalXmlSavings = totalXmlTokens - totalToonTokens
-const totalXmlSavingsPercent = ((totalXmlSavings / totalXmlTokens) * 100).toFixed(1)
+// Mixed-Structure Track (no CSV)
+const mixedCharts = mixedStructureDatasets
+  .map(result => generateDatasetChart(result))
+  .join('\n\n')
 
-const totalYamlSavings = totalYamlTokens - totalToonTokens
-const totalYamlSavingsPercent = ((totalYamlSavings / totalYamlTokens) * 100).toFixed(1)
-
-// Generate ASCII bar chart visualization (stacked compact format)
-const datasetRows = results
+// Flat-Only Track (with CSV)
+const flatCharts = flatOnlyDatasets
   .map((result) => {
+    const csv = result.formats.find(f => f.name === 'csv')
     const toon = result.formats.find(f => f.name === 'toon')!
-    const json = result.formats.find(f => f.name === 'json')!
-    const xml = result.formats.find(f => f.name === 'xml')!
-    const yaml = result.formats.find(f => f.name === 'yaml')!
 
-    const percentage = Number.parseFloat(json.savingsPercent)
-    const bar = createProgressBar(100 - percentage, 100) // Invert to show TOON tokens
+    if (!csv)
+      return generateDatasetChart(result)
+
+    // Special handling to show CSV first with TOON overhead
+    const { dataset } = result
+    const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
+    const eligibility = dataset.metadata.tabularEligibility
+    const name = dataset.description
+
+    // CSV line
+    const csvPercentage = Math.min(100, (csv.tokens / toon.tokens) * 100)
+    const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
+    const csvStr = csv.tokens.toLocaleString('en-US')
+
+    const line1 = `${emoji} ${name}  ┊  Tabular: ${eligibility}%`
+    const line2 = `   │`
+    const line3 = `   CSV                 ${csvBar}   ${csvStr.padStart(TOKEN_PADDING)} tokens`
+
+    const toonOverhead = toon.tokens - csv.tokens
+    const toonOverheadPercent = (toonOverhead / csv.tokens) * 100
+    const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
     const toonStr = toon.tokens.toLocaleString('en-US')
-    const jsonStr = json.tokens.toLocaleString('en-US')
-    const xmlStr = xml.tokens.toLocaleString('en-US')
-    const yamlStr = yaml.tokens.toLocaleString('en-US')
+    const toonVsCSV = toonOverheadPercent >= 0
+      ? `(+${toonOverheadPercent.toFixed(1)}% vs CSV)`
+      : `(${toonOverheadPercent.toFixed(1)}% vs CSV)`
+    const toonLine = `   TOON                ${toonBar}   ${toonStr.padStart(TOKEN_PADDING)} tokens   ${toonVsCSV}`
 
-    const line1 = `${result.emoji} ${result.name.padEnd(25)} ${bar}  ${toonStr.padStart(6)} tokens`
-    const line2 = `                             vs JSON: ${jsonStr.padStart(6)}  (-${json.savingsPercent}%)`
-    const line3 = `                             vs YAML: ${yamlStr.padStart(6)}  (-${yaml.savingsPercent}%)`
-    const line4 = `                             vs XML:  ${xmlStr.padStart(6)}  (-${xml.savingsPercent}%)`
+    // Other format comparisons (vs TOON)
+    const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
+      const format = result.formats.find(f => f.name === formatName)
+      if (!format)
+        return undefined
 
-    return `${line1}\n${line2}\n${line3}\n${line4}`
+      return `   ${formatComparisonLine(format, index === array.length - 1)}`
+    }).filter(Boolean)
+
+    return [line1, line2, line3, toonLine, ...comparisonLines].join('\n')
   })
   .join('\n\n')
 
-// Add separator and totals row
-const separator = '─────────────────────────────────────────────────────────────────────'
+// Calculate totals for mixed structure
+const { totalToonTokens: totalToonTokensMixed, totals: mixedTotals } = calculateTotalMetrics(mixedStructureDatasets, COMPARISON_FORMAT_ORDER)
+const mixedTotalLines = generateTotalLines(totalToonTokensMixed, mixedTotals)
 
-// Calculate bar for totals (TOON vs average of JSON+YAML+XML)
-const averageComparisonTokens = (totalJsonTokens + totalYamlTokens + totalXmlTokens) / 3
-const totalPercentage = (totalToonTokens / averageComparisonTokens) * 100
-const totalBar = createProgressBar(totalPercentage, 100)
+// Calculate totals for flat-only
+const { totalToonTokens: totalToonTokensFlat, totals: flatTotals } = calculateTotalMetrics(flatOnlyDatasets, COMPARISON_FORMAT_ORDER)
+const totalCSVTokensFlat = flatOnlyDatasets.reduce((sum, r) => {
+  const csv = r.formats.find(f => f.name === 'csv')
+  return sum + (csv?.tokens || 0)
+}, 0)
+const flatTotalLines = generateTotalLines(totalToonTokensFlat, flatTotals, { name: 'csv', tokens: totalCSVTokensFlat })
 
-const totalLine1 = `Total                        ${totalBar}  ${totalToonTokens.toLocaleString('en-US').padStart(6)} tokens`
-const totalLine2 = `                             vs JSON: ${totalJsonTokens.toLocaleString('en-US').padStart(6)}  (-${totalJsonSavingsPercent}%)`
-const totalLine3 = `                             vs YAML: ${totalYamlTokens.toLocaleString('en-US').padStart(6)}  (-${totalYamlSavingsPercent}%)`
-const totalLine4 = `                             vs XML:  ${totalXmlTokens.toLocaleString('en-US').padStart(6)}  (-${totalXmlSavingsPercent}%)`
+const barChartSection = `
+#### Mixed-Structure Track
 
-const barChartSection = `${datasetRows}\n\n${separator}\n${totalLine1}\n${totalLine2}\n${totalLine3}\n${totalLine4}`
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
 
-// Generate detailed examples (only for selected examples)
-// Note: Large datasets are truncated for display readability in the report.
-// Token counts are calculated from the full datasets, not the truncated versions.
+\`\`\`
+${mixedCharts}
+
+${mixedTotalLines}
+\`\`\`
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+\`\`\`
+${flatCharts}
+
+${flatTotalLines}
+\`\`\`
+`.trim()
+
+// Generate detailed examples (optional: show a few examples)
 const detailedExamples = results
-  .filter(result => result.showDetailed)
+  .filter(r => DETAILED_EXAMPLE_DATASETS.includes(r.dataset.name as any))
   .map((result, i, filtered) => {
-    // Truncate large datasets for display
-    let displayData = result.data
-    if (result.name === 'GitHub Repositories') {
+    let displayData = result.dataset.data
+
+    // Truncate for display
+    if (result.dataset.name === 'github') {
       displayData = {
-        repositories: result.data.repositories.slice(0, 3).map((repo: Record<string, any>) => ({
+        repositories: displayData.repositories.slice(0, GITHUB_REPO_LIMIT).map((repo: Record<string, any>) => ({
           ...repo,
-          description: repo.description?.slice(0, 80) + (repo.description?.length > 80 ? '…' : ''),
+          description: repo.description?.slice(0, GITHUB_DESC_LIMIT) + (repo.description?.length > GITHUB_DESC_LIMIT ? '…' : ''),
         })),
       }
     }
-    else if (result.name === 'Daily Analytics') {
-      displayData = { metrics: result.data.metrics.slice(0, 5) }
+    else if (result.dataset.name === 'analytics') {
+      displayData = { metrics: displayData.metrics.slice(0, ANALYTICS_METRICS_LIMIT) }
     }
 
-    const separator = i < filtered.length - 1 ? '\n\n---' : ''
-
-    const json = result.formats.find(f => f.name === 'json')!
+    const emoji = DATASET_ICONS[result.dataset.name] || DEFAULT_DATASET_ICON
+    const json = result.formats.find(f => f.name === 'json-pretty')!
     const toon = result.formats.find(f => f.name === 'toon')!
+    const separator = i < filtered.length - 1 ? '---' : ''
 
-    return `#### ${result.emoji} ${result.name}
+    return `
+#### ${emoji} ${result.dataset.description}
 
-**Configuration:** ${result.description}
-
-**Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent}% reduction vs JSON)
+**Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent.toFixed(1)}% reduction vs JSON)
 
 **JSON** (${json.tokens.toLocaleString('en-US')} tokens):
 
@@ -204,25 +320,25 @@ ${JSON.stringify(displayData, undefined, 2)}
 
 \`\`\`
 ${encode(displayData)}
-\`\`\`${separator}`
+\`\`\`
+
+${separator}
+`.trim()
   })
   .join('\n\n')
 
-const markdown = `### Token Efficiency
-
-\`\`\`
+const markdown = `
 ${barChartSection}
-\`\`\`
 
 <details>
-<summary><strong>View detailed examples</strong></summary>
+<summary><strong>Show detailed examples</strong></summary>
 
 ${detailedExamples}
 
 </details>
 `.trimStart()
 
-prompts.log.message(`${barChartSection}\n`)
+prompts.log.message(barChartSection)
 
 const resultsDir = path.join(BENCHMARKS_DIR, 'results')
 await ensureDir(resultsDir)
@@ -230,4 +346,4 @@ await ensureDir(resultsDir)
 const outputFilePath = path.join(resultsDir, 'token-efficiency.md')
 await fsp.writeFile(outputFilePath, markdown, 'utf-8')
 
-prompts.log.success(`Result saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)
+prompts.log.success(`Report saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)

benchmarks/src/constants.ts

@@ -4,40 +4,16 @@ import * as url from 'node:url'
 export const ROOT_DIR: string = url.fileURLToPath(new URL('../../', import.meta.url))
 export const BENCHMARKS_DIR: string = url.fileURLToPath(new URL('../', import.meta.url))
 
-/**
- * Model-specific RPM (requests per minute) limits to handle API quotas
- *
- * @remarks
- * Set `undefined` for models without specific limits
- */
-/// keep-sorted
-export const MODEL_RPM_LIMITS: Record<string, number | undefined> = {
-  'claude-haiku-4-5-20251001': 50,
-  'gemini-2.5-flash': 25,
-  'gpt-5-nano': undefined,
-  'grok-4-fast-non-reasoning': 50,
-}
-
 /**
  * Default concurrency for parallel evaluations to prevent bursting
  */
 export const DEFAULT_CONCURRENCY = 10
 
-/**
- * Progress bar configuration
- */
-export const PROGRESS_BAR = {
-  /** Default width for progress bars */
-  defaultWidth: 25,
-  /** Compact width for inline displays */
-  compactWidth: 20,
-} as const
-
 /**
  * Enable dry run mode for quick testing with limited AI requests
  *
  * @remarks
- * Set via environment variable: `DRY_RUN=true`
+ * Set via environment variable: `DRY_RUN=true`.
  */
 export const DRY_RUN: boolean = process.env.DRY_RUN === 'true'
 
@@ -49,12 +25,90 @@ export const DRY_RUN_LIMITS = {
   maxQuestions: 10,
 }
 
+/**
+ * Model-specific RPM (requests per minute) limits to handle API quotas
+ *
+ * @remarks
+ * Set `undefined` for models without specific limits.
+ */
+/// keep-sorted
+export const MODEL_RPM_LIMITS: Record<string, number | undefined> = {
+  'claude-haiku-4-5-20251001': 50,
+  'gemini-3-flash-preview': 25,
+  'gpt-5-nano': 50,
+  'grok-4-1-fast-non-reasoning': 25,
+}
+
+/**
+ * Display names for data format types
+ */
+export const FORMATTER_DISPLAY_NAMES: Record<string, string> = {
+  'json-pretty': 'JSON',
+  'json-compact': 'JSON compact',
+  'toon': 'TOON',
+  'csv': 'CSV',
+  'xml': 'XML',
+  'yaml': 'YAML',
+} as const
+
+/**
+ * Question type identifiers
+ */
+export const QUESTION_TYPES = [
+  'field-retrieval',
+  'retrieval',
+  'aggregation',
+  'filtering',
+  'structure-awareness',
+  'structural-validation',
+] as const
+
+/**
+ * Display names for question types
+ */
+export const QUESTION_TYPE_LABELS = {
+  'field-retrieval': 'Field Retrieval',
+  'retrieval': 'Retrieval',
+  'aggregation': 'Aggregation',
+  'filtering': 'Filtering',
+  'structure-awareness': 'Structure Awareness',
+  'structural-validation': 'Structural Validation',
+} as const
+
+/**
+ * Dataset identifiers
+ */
+export const DATASET_NAMES = [
+  'tabular',
+  'nested',
+  'analytics',
+  'github',
+  'event-logs',
+  'nested-config',
+  'large-uniform',
+  'structural-validation-control',
+  'structural-validation-truncated',
+  'structural-validation-extra-rows',
+  'structural-validation-width-mismatch',
+  'structural-validation-missing-fields',
+] as const
+
+/**
+ * Structure class identifiers
+ */
+export const STRUCTURE_CLASSES = [
+  'uniform',
+  'semi-uniform',
+  'nested',
+  'deep',
+] as const
+
 /**
  * Threshold values for filtering and aggregation questions
  */
 export const QUESTION_THRESHOLDS = {
   tabular: {
-    salaryRanges: [60000, 80000, 100000, 120000],
+    salaryRanges: [60000, 80000, 100000],
     experienceYears: [5, 10, 15, 20],
     departmentSalaryThreshold: 80000,
     departmentExperienceThreshold: 10,
@@ -66,11 +120,11 @@ export const QUESTION_THRESHOLDS = {
     totalThresholdsForItems: [300, 500],
   },
   analytics: {
-    views: [5000, 7000],
-    conversions: [10, 30],
+    views: [6000],
+    conversions: [20],
     viewsForFiltering: [6000, 7000],
     conversionsForFiltering: 15,
-    revenueThresholds: [500, 1000, 1500, 2000, 2500],
+    revenueThresholds: [1000, 1500, 2000],
     viewsThresholdForRevenue: 6000,
     clicksForFiltering: [250, 400],
     conversionsForClickFiltering: 15,
@@ -79,8 +133,8 @@ export const QUESTION_THRESHOLDS = {
   },
   github: {
     stars: [100000, 150000, 200000],
-    forks: [20000, 35000, 50000],
-    watchers: [5000, 8000],
+    forks: [20000, 35000],
+    watchers: [8000],
     starForkCombinations: [
       { stars: 75000, forks: 15000 },
       { stars: 100000, forks: 20000 },
@@ -99,26 +153,37 @@ export const QUESTION_THRESHOLDS = {
  */
 export const QUESTION_LIMITS = {
   tabular: {
-    fieldRetrieval: 20,
-    aggregationDepartments: 6,
-    filteringMultiConditionDepartments: 6,
-    filteringExperience: 4,
+    fieldRetrieval: 12,
+    aggregationDepartments: 3,
+    filteringMultiConditionDepartments: 5,
+    filteringExperience: 3,
     filteringDepartmentExp: 3,
-    filteringDepartmentActive: 3,
+    filteringDepartmentActive: 2,
   },
   nested: {
     fieldRetrievalOrders: 8,
-    fieldRetrievalCustomers: 10,
-    aggregationStatuses: 5,
-    filteringStatusAndValue: 5,
+    fieldRetrievalCustomers: 8,
+    aggregationStatuses: 3,
+    filteringStatusAndValue: 4,
     filteringStatusAndItems: 3,
   },
   analytics: {
-    fieldRetrievalDates: 13,
+    fieldRetrievalDates: 9,
   },
   github: {
     fieldRetrievalRepos: 11,
     aggregationBranches: 2,
-    filteringStarsAndForks: 8,
+    filteringStarsAndForks: 3,
+  },
+  eventLogs: {
+    fieldRetrieval: 10,
+    aggregationEndpoints: 2,
+    filteringLevelAndStatus: 3,
+    filteringEndpointAndStatus: 3,
+    filteringEndpointRetryable: 2,
+  },
+  nestedConfig: {
+    fieldRetrieval: 10,
+    filteringComplex: 5,
   },
 } as const

benchmarks/src/datasets.ts

@@ -1,4 +1,4 @@
-import type { Dataset } from './types'
+import type { Dataset } from './types.ts'
 import { faker } from '@faker-js/faker'
 import githubRepos from '../data/github-repos.json' with { type: 'json' }
 
@@ -61,7 +61,6 @@ export interface AnalyticsMetric {
 export interface Repository {
   id: number
   name: string
-  owner: string
   repo: string
   description: string
   stars: number
@@ -73,6 +72,102 @@ export interface Repository {
   pushedAt: string
 }
 
+/**
+ * Event log structure for semi-uniform dataset
+ */
+export interface EventLog {
+  timestamp: string
+  level: 'info' | 'warn' | 'error'
+  endpoint: string
+  statusCode: number
+  responseTime: number
+  userId: number
+  error?: {
+    message: string
+    stack: string
+    retryable: boolean
+  }
+}
+
+/**
+ * Nested configuration structure for deeply nested dataset
+ */
+export interface NestedConfig {
+  environment: string
+  version: string
+  database: {
+    host: string
+    port: number
+    name: string
+    pool: {
+      min: number
+      max: number
+      idleTimeout: number
+    }
+    replicas: {
+      host: string
+      port: number
+      priority: number
+    }[]
+  }
+  features: Record<string, {
+    enabled: boolean
+    rollout: number
+    variants: {
+      name: string
+      weight: number
+      config: Record<string, any>
+    }[]
+  }>
+  authentication: {
+    providers: {
+      name: string
+      clientId: string
+      scopes: string[]
+      config: Record<string, any>
+    }[]
+    session: {
+      secret: string
+      duration: number
+      refreshThreshold: number
+    }
+  }
+  permissions: {
+    roles: Record<string, {
+      permissions: string[]
+      inherits: string[]
+    }>
+    groups: Record<string, {
+      members: string[]
+      roles: string[]
+    }>
+  }
+}
+
+/**
+ * Product structure for large uniform arrays
+ */
+export interface Product {
+  sku: string
+  name: string
+  category: string
+  price: number
+  qty: number
+  lastUpdated: string
+}
+
+/**
+ * Internal types for structural validation pattern generation
+ */
+type StructuralValidationType = 'truncated' | 'extra-rows' | 'width-mismatch' | 'missing-fields'
+
+interface StructuralValidationFixture {
+  type: StructuralValidationType
+  description: string
+  data: Record<string, unknown>
+  isValid: boolean
+}
+
 /**
  * Generate analytics time-series data
  */
@@ -108,17 +203,13 @@ export function generateAnalyticsData(days: number, startDate = '2025-01-01'): {
 }
 
 /**
- * Tabular dataset: 100 uniform employee records
- *
- * @remarks
- * Tests TOON's tabular array format
+ * Generate employee data (uniform tabular structure)
  */
-const departments: readonly string[] = ['Engineering', 'Sales', 'Marketing', 'HR', 'Operations', 'Finance'] as const
-const tabularDataset: Dataset = {
-  name: 'tabular',
-  description: 'Uniform employee records (TOON optimal format)',
-  data: {
-    employees: Array.from({ length: 100 }, (_, i): Employee => {
+const departments = ['Engineering', 'Sales', 'Marketing', 'HR', 'Operations', 'Finance'] as const
+
+function generateEmployees(count: number): { employees: Employee[] } {
+  return {
+    employees: Array.from({ length: count }, (_, i): Employee => {
       const yearsExp = faker.number.int({ min: 1, max: 25 })
       return {
         id: i + 1,
@@ -130,38 +221,56 @@ const tabularDataset: Dataset = {
         active: faker.datatype.boolean(0.8), // 80% active
       }
     }),
+  }
+}
+
+/**
+ * Tabular dataset: Uniform employee records
+ *
+ * @remarks
+ * Tests TOON's tabular array format.
+ */
+const tabularDataset: Dataset = {
+  name: 'tabular',
+  description: 'Uniform employee records',
+  data: generateEmployees(100),
+  metadata: {
+    supportsCSV: true,
+    structureClass: 'uniform',
+    tabularEligibility: 100, // All arrays contain uniform objects with primitive values only
   },
 }
 
 /**
- * Nested dataset: 50 e-commerce orders with nested structures
- *
- * @remarks
- * Tests TOON's handling of complex nested objects
+ * Generate e-commerce orders (nested structure)
  */
-const productNames: readonly string[] = ['Wireless Mouse', 'USB Cable', 'Laptop Stand', 'Keyboard', 'Webcam', 'Headphones', 'Monitor', 'Desk Lamp'] as const
-const statuses: readonly string[] = ['pending', 'processing', 'shipped', 'delivered', 'cancelled'] as const
+const PRODUCT_NAMES = ['Wireless Mouse', 'USB Cable', 'Laptop Stand', 'Keyboard', 'Webcam', 'Headphones', 'Monitor', 'Desk Lamp'] as const
+const ORDER_STATUSES = ['pending', 'processing', 'shipped', 'delivered', 'cancelled'] as const
 
-const nestedDataset: Dataset = {
-  name: 'nested',
-  description: 'E-commerce orders with nested structures',
-  data: {
-    orders: Array.from({ length: 50 }, (_, i) => {
-      const customerId = (i % 20) + 1
-      const itemCount = faker.number.int({ min: 1, max: 4 })
+function generateOrders(count: number): { orders: Order[] } {
+  return {
+    orders: Array.from({ length: count }, (_, i) => {
+      const customerId = (i % 20) + 1 // Rotate through 20 customers
+      const itemCount = faker.number.int({ min: 1, max: 4 }) // 1-4 items per order
 
       const items = Array.from({ length: itemCount }, (_, j) => {
-        const price = faker.number.float({ min: 9.99, max: 199.99, fractionDigits: 2 })
+        const price = faker.number.float({
+          min: 9.99,
+          max: 199.99,
+          fractionDigits: 2,
+        })
         const quantity = faker.number.int({ min: 1, max: 5 })
         return {
           sku: `SKU-${faker.string.alphanumeric({ length: 6 }).toUpperCase()}`,
-          name: productNames[j % productNames.length]!,
+          name: PRODUCT_NAMES[j % PRODUCT_NAMES.length]!,
           quantity,
           price,
         }
       })
 
-      const total = Number(items.reduce((sum, item) => sum + (item.price * item.quantity), 0).toFixed(2))
+      const subtotal = Number(items.reduce((sum, item) => sum + (item.price * item.quantity), 0).toFixed(2))
+      const tax = Number((subtotal * 0.08).toFixed(2)) // 8% tax rate
+      const total = Number((subtotal + tax).toFixed(2))
 
       return {
         orderId: `ORD-${String(i + 1).padStart(4, '0')}`,
@@ -169,33 +278,58 @@ const nestedDataset: Dataset = {
           id: customerId,
           name: faker.person.fullName(),
           email: faker.internet.email().toLowerCase(),
+          phone: faker.phone.number(),
         },
         items,
+        subtotal,
+        tax,
         total,
-        status: statuses[i % statuses.length]!,
+        status: ORDER_STATUSES[i % ORDER_STATUSES.length]!,
         orderDate: faker.date.recent({ days: 90 }).toISOString().split('T')[0],
       }
     }),
+  }
+}
+
+/**
+ * Nested dataset: E-commerce orders with nested structures
+ *
+ * @remarks
+ * Tests TOON's handling of complex nested objects.
+ */
+const nestedDataset: Dataset = {
+  name: 'nested',
+  description: 'E-commerce orders with nested structures',
+  data: generateOrders(50),
+  metadata: {
+    supportsCSV: false,
+    structureClass: 'nested',
+    tabularEligibility: 33, // Top-level orders array has nested objects (not tabular), but nested items arrays are tabular
   },
 }
 
 /**
- * Analytics dataset: 60 days of time-series metrics
+ * Analytics dataset: Time-series metrics
  *
  * @remarks
- * Tests TOON's handling of numeric data and date fields
+ * Tests TOON's handling of numeric data and date fields.
  */
 const analyticsDataset: Dataset = {
   name: 'analytics',
   description: 'Time-series analytics data',
   data: generateAnalyticsData(60),
+  metadata: {
+    supportsCSV: true,
+    structureClass: 'uniform',
+    tabularEligibility: 100, // Uniform time-series records with consistent primitive fields
+  },
 }
 
 /**
  * Real-world dataset: Top 100 starred GitHub repositories
  *
  * @remarks
- * Tests TOON's tabular format
+ * Tests TOON's tabular format with real data.
  */
 const githubDataset: Dataset = {
   name: 'github',
@@ -203,13 +337,18 @@ const githubDataset: Dataset = {
   data: {
     repositories: githubRepos,
   },
+  metadata: {
+    supportsCSV: true,
+    structureClass: 'uniform',
+    tabularEligibility: 100, // Repository array contains uniform objects with primitive values
+  },
 }
 
 /**
  * Generate a single e-commerce order with nested structure
  *
  * @remarks
- * Used for token efficiency benchmarks
+ * Used for token efficiency benchmarks.
  */
 export function generateOrderData(): Order {
   return {
@@ -235,11 +374,380 @@ export function generateOrderData(): Order {
 }
 
 /**
- * All datasets used in the benchmark
+ * Generate event logs (semi-uniform structure)
+ *
+ * @remarks
+ * Approximately 50% of logs include nested error objects, 50% are flat.
+ * This creates ~45% tabular eligibility.
  */
-export const datasets: Dataset[] = [
-  tabularDataset,
-  nestedDataset,
-  analyticsDataset,
-  githubDataset,
+export function generateEventLogs(count: number): { logs: EventLog[] } {
+  const endpoints = ['/api/users', '/api/orders', '/api/products', '/api/auth', '/api/payments']
+  const levels = ['info', 'warn', 'error'] as const
+
+  return {
+    logs: Array.from({ length: count }, () => {
+      const level = faker.helpers.arrayElement(levels)
+      const hasError = level === 'error' || (level === 'warn' && faker.datatype.boolean(0.3))
+
+      const log: EventLog = {
+        timestamp: faker.date.recent({ days: 7 }).toISOString(),
+        level,
+        endpoint: faker.helpers.arrayElement(endpoints),
+        statusCode: hasError
+          ? faker.number.int({ min: 400, max: 599 })
+          : faker.number.int({ min: 200, max: 299 }),
+        responseTime: faker.number.int({ min: 10, max: 5000 }),
+        userId: faker.number.int({ min: 1000, max: 9999 }),
+      }
+
+      if (hasError) {
+        log.error = {
+          message: faker.helpers.arrayElement([
+            'Database connection timeout',
+            'Invalid authentication token',
+            'Resource not found',
+            'Internal server error',
+            'Rate limit exceeded',
+          ]),
+          stack: `Error: ${faker.lorem.sentence()}\n  at ${faker.lorem.word()}\n  at ${faker.lorem.word()}`,
+          retryable: faker.datatype.boolean(0.6),
+        }
+      }
+
+      return log
+    }),
+  }
+}
+
+/**
+ * Generate deeply nested configuration
+ *
+ * @remarks
+ * Creates a complex nested structure with minimal tabular eligibility (~0%).
+ */
+export function generateNestedConfig(): NestedConfig {
+  return {
+    environment: faker.helpers.arrayElement(['production', 'staging', 'development']),
+    version: faker.system.semver(),
+    database: {
+      host: faker.internet.domainName(),
+      port: 5432,
+      name: faker.database.type(),
+      pool: {
+        min: 2,
+        max: faker.number.int({ min: 10, max: 50 }),
+        idleTimeout: 30000,
+      },
+      replicas: Array.from({ length: 3 }, (_, i) => ({
+        host: `replica-${i + 1}.${faker.internet.domainName()}`,
+        port: 5432,
+        priority: i + 1,
+      })),
+    },
+    features: {
+      darkMode: {
+        enabled: faker.datatype.boolean(),
+        rollout: faker.number.int({ min: 0, max: 100 }),
+        variants: [
+          {
+            name: 'default',
+            weight: 70,
+            config: { theme: 'dark', animations: true },
+          },
+          {
+            name: 'minimal',
+            weight: 30,
+            config: { theme: 'dark', animations: false },
+          },
+        ],
+      },
+      analytics: {
+        enabled: faker.datatype.boolean(),
+        rollout: faker.number.int({ min: 0, max: 100 }),
+        variants: [
+          {
+            name: 'full',
+            weight: 100,
+            config: { tracking: 'all', sampling: 1.0 },
+          },
+        ],
+      },
+    },
+    authentication: {
+      providers: [
+        {
+          name: 'oauth2',
+          clientId: faker.string.uuid(),
+          scopes: ['read', 'write', 'admin'],
+          config: {
+            authUrl: faker.internet.url(),
+            tokenUrl: faker.internet.url(),
+          },
+        },
+        {
+          name: 'saml',
+          clientId: faker.string.uuid(),
+          scopes: ['read'],
+          config: {
+            entryPoint: faker.internet.url(),
+            cert: faker.string.alphanumeric({ length: 64 }),
+          },
+        },
+      ],
+      session: {
+        secret: faker.string.alphanumeric({ length: 32 }),
+        duration: 86400,
+        refreshThreshold: 3600,
+      },
+    },
+    permissions: {
+      roles: {
+        admin: {
+          permissions: ['read', 'write', 'delete', 'manage_users', 'manage_roles'],
+          inherits: [],
+        },
+        editor: {
+          permissions: ['read', 'write'],
+          inherits: ['viewer'],
+        },
+        viewer: {
+          permissions: ['read'],
+          inherits: [],
+        },
+      },
+      groups: {
+        engineering: {
+          members: Array.from({ length: 5 }, () => faker.internet.email()),
+          roles: ['admin', 'editor'],
+        },
+        support: {
+          members: Array.from({ length: 3 }, () => faker.internet.email()),
+          roles: ['viewer'],
+        },
+      },
+    },
+  }
+}
+
+/**
+ * Generate large uniform product array (5000+ rows)
+ *
+ * @remarks
+ * Tests TOON's token efficiency and structural reliability at scale.
+ */
+export function generateProducts(count: number): { products: Product[] } {
+  const categories = ['Electronics', 'Clothing', 'Home & Garden', 'Sports', 'Books', 'Toys'] as const
+
+  return {
+    products: Array.from({ length: count }, (_, i): Product => ({
+      sku: `SKU-${String(i + 1).padStart(6, '0')}`,
+      name: faker.commerce.productName(),
+      category: categories[i % categories.length]!,
+      price: Number(faker.commerce.price({ min: 5, max: 500 })),
+      qty: faker.number.int({ min: 0, max: 1000 }),
+      lastUpdated: faker.date.recent({ days: 30 }).toISOString().split('T')[0]!,
+    })),
+  }
+}
+
+/**
+ * Generate structural validation fixtures from employee data
+ *
+ * @remarks
+ * Creates deliberately corrupted datasets to test TOON's structural validation
+ * capabilities via [N] length declarations and {fields} headers.
+ * Internal function used to generate structural validation datasets.
+ */
+function generateStructuralValidationFixtures(): StructuralValidationFixture[] {
+  const baseData = generateEmployees(20)
+
+  return [
+    // Valid baseline
+    {
+      type: 'truncated' as const,
+      description: 'Valid complete dataset (control)',
+      data: { employees: baseData.employees },
+      isValid: true,
+    },
+    // Truncated array (missing last 3 rows)
+    {
+      type: 'truncated' as const,
+      description: 'Array truncated: 3 rows removed from end',
+      data: { employees: baseData.employees.slice(0, -3) },
+      isValid: false, // [N] won't match actual row count in TOON
+    },
+    // Extra rows (3 more than original)
+    {
+      type: 'extra-rows' as const,
+      description: 'Extra rows added beyond declared length',
+      data: {
+        employees: [
+          ...baseData.employees,
+          ...generateEmployees(3).employees,
+        ],
+      },
+      isValid: false, // [N] won't match actual row count in TOON
+    },
+    // Width mismatch (inconsistent field count)
+    {
+      type: 'width-mismatch' as const,
+      description: 'Inconsistent field count (missing salary in row 10)',
+      data: {
+        employees: baseData.employees.map((emp, i) => {
+          if (i === 9) {
+            // Row 10, missing salary field
+            const { salary, ...rest } = emp
+            return rest
+          }
+          return emp
+        }),
+      },
+      isValid: false, // Not all objects have same fields (tabular requirement)
+    },
+    // Missing required fields
+    {
+      type: 'missing-fields' as const,
+      description: 'Missing required fields (no email in multiple rows)',
+      data: {
+        employees: baseData.employees.map((emp, i) => {
+          if (i % 5 === 0) {
+            // Every 5th row, missing email
+            const { email, ...rest } = emp
+            return rest
+          }
+          return emp
+        }),
+      },
+      isValid: false, // Not all objects have same fields (tabular requirement)
+    },
+  ]
+}
+
+/**
+ * Event logs dataset: Semi-uniform structure
+ *
+ * @remarks
+ * Tests TOON with semi-uniform data (~50% flat, ~50% with nested errors).
+ */
+const eventLogsDataset: Dataset = {
+  name: 'event-logs',
+  description: 'Semi-uniform event logs',
+  data: generateEventLogs(75),
+  metadata: {
+    supportsCSV: false,
+    structureClass: 'semi-uniform',
+    tabularEligibility: 50, // Top-level logs array is tabular, but ~50% have nested optional error objects
+  },
+}
+
+/**
+ * Nested config dataset: Deeply nested structure
+ *
+ * @remarks
+ * Tests TOON's worst-case scenario with deeply nested configuration.
+ */
+const nestedConfigDataset: Dataset = {
+  name: 'nested-config',
+  description: 'Deeply nested configuration',
+  data: generateNestedConfig(),
+  metadata: {
+    supportsCSV: false,
+    structureClass: 'deep',
+    tabularEligibility: 0, // Deeply nested configuration with no tabular arrays
+  },
+}
+
+/**
+ * Structural validation datasets: Tests ability to detect incomplete, truncated, or corrupted data
+ *
+ * @remarks
+ * These datasets test TOON's structural validation advantages via [N] length declarations
+ * and {fields} headers. CSV is included to demonstrate its lack of structural metadata.
+ */
+const structuralValidationDatasets: Dataset[] = generateStructuralValidationFixtures().map((fixture, index) => {
+  const datasetNames = [
+    'structural-validation-control',
+    'structural-validation-truncated',
+    'structural-validation-extra-rows',
+    'structural-validation-width-mismatch',
+    'structural-validation-missing-fields',
+  ] as const
+
+  return {
+    name: datasetNames[index]!,
+    description: fixture.description,
+    data: fixture.data,
+    metadata: {
+      supportsCSV: true, // Include CSV to show it can't validate structure
+      structureClass: 'uniform',
+      tabularEligibility: 100,
+    },
+  }
+})
+
+/**
+ * Datasets for accuracy benchmarks (smaller sizes for faster evaluation)
+ */
+export const ACCURACY_DATASETS: Dataset[] = [
+  tabularDataset, // 100 employees
+  nestedDataset, // 50 orders
+  analyticsDataset, // 60 days
+  githubDataset, // 100 repos
+  eventLogsDataset, // 75 logs
+  nestedConfigDataset, // 1 config
+  ...structuralValidationDatasets, // 5 validation fixtures
+]
+
+/**
+ * Datasets for token efficiency benchmarks (larger sizes to amplify token differences)
+ */
+export const TOKEN_EFFICIENCY_DATASETS: Dataset[] = [
+  // Tabular: 2000 employees
+  {
+    name: 'tabular',
+    description: 'Uniform employee records',
+    data: generateEmployees(2000),
+    metadata: {
+      supportsCSV: true,
+      structureClass: 'uniform',
+      tabularEligibility: 100, // All arrays contain uniform objects with primitive values only
+    },
+  },
+  // Nested: 500 orders
+  {
+    name: 'nested',
+    description: 'E-commerce orders with nested structures',
+    data: generateOrders(500),
+    metadata: {
+      supportsCSV: false,
+      structureClass: 'nested',
+      tabularEligibility: 33, // Top-level orders array has nested objects (not tabular), but nested items arrays are tabular
+    },
+  },
+  // Analytics: 365 days
+  {
+    name: 'analytics',
+    description: 'Time-series analytics data',
+    data: generateAnalyticsData(365),
+    metadata: {
+      supportsCSV: true,
+      structureClass: 'uniform',
+      tabularEligibility: 100, // Uniform time-series records with consistent primitive fields
+    },
+  },
+  // GitHub: 100 repos (same as accuracy)
+  githubDataset,
+  // Event logs: 2000 logs
+  {
+    name: 'event-logs',
+    description: 'Semi-uniform event logs',
+    data: generateEventLogs(2000),
+    metadata: {
+      supportsCSV: false,
+      structureClass: 'semi-uniform',
+      tabularEligibility: 50, // Top-level logs array is tabular, but ~50% have nested optional error objects
+    },
+  },
+  // Nested config: 1 config (same as accuracy)
+  nestedConfigDataset,
 ]

benchmarks/src/evaluate.ts

@@ -1,22 +1,49 @@
-import type { LanguageModelV2 } from '@ai-sdk/provider'
-import type { EvaluationResult, Question } from './types'
+import type { LanguageModelV3 } from '@ai-sdk/provider'
+import type { EvaluationResult, Question } from './types.ts'
 import { anthropic } from '@ai-sdk/anthropic'
 import { google } from '@ai-sdk/google'
 import { openai } from '@ai-sdk/openai'
 import { xai } from '@ai-sdk/xai'
-import * as prompts from '@clack/prompts'
 import { generateText } from 'ai'
+import { compareAnswers } from './normalize.ts'
 
 /**
  * Models used for evaluation
  */
-export const models: LanguageModelV2[] = [
-  openai('gpt-5-nano'),
+export const models: LanguageModelV3[] = [
   anthropic('claude-haiku-4-5-20251001'),
-  google('gemini-2.5-flash'),
-  xai('grok-4-fast-non-reasoning'),
+  google('gemini-3-flash-preview'),
+  openai('gpt-5-nano'),
+  xai('grok-4-1-fast-non-reasoning'),
 ]
 
+/**
+ * Format primers
+ *
+ * @remarks
+ * Neutral descriptions to help models parse each format.
+ */
+export const PRIMERS: Record<string, string> = {
+  'toon': 'TOON: Indentation-based. Arrays declare length and fields (e.g., items[N]{f1,f2}:). Rows use single delimiter. Values may be quoted.',
+  'json-pretty': 'JSON: Strict JSON objects/arrays with repeated keys per row.',
+  'json-compact': 'JSON (compact): Strict JSON without extra whitespace.',
+  'yaml': 'YAML: Indentation-based key/value and lists (- items).',
+  'xml': 'XML: Tag-based tree structure with nested elements.',
+  'csv': 'CSV: Header row, comma-separated values. First row contains field names.',
+}
+
+/**
+ * Code fence language tags for proper syntax highlighting
+ */
+export const FENCE: Record<string, string> = {
+  'toon': 'toon',
+  'json-pretty': 'json',
+  'json-compact': 'json',
+  'yaml': 'yaml',
+  'xml': 'xml',
+  'csv': 'csv',
+}
+
 /**
  * Evaluate a single question with a specific format and model
  */
@@ -31,19 +58,30 @@ export async function evaluateQuestion(
     question: Question
     formatName: string
     formattedData: string
-    model: LanguageModelV2
+    model: LanguageModelV3
   },
 ): Promise<EvaluationResult> {
+  const primer = PRIMERS[formatName] ?? ''
+  const fence = FENCE[formatName] ?? ''
+
   const prompt = `
+${primer}
+
 Given the following data in ${formatName} format:
 
-\`\`\`
+\`\`\`${fence}
 ${formattedData}
 \`\`\`
 
 Question: ${question.prompt}
 
-Provide only the direct answer, without any additional explanation or formatting.
+Answer format requirements:
+- Provide only the value itself, no explanation
+- For numbers: output digits only (no commas, currency symbols, or units)
+- For dates/field names: use the exact string from the data
+- For lists: output comma-separated values with no spaces
+
+Answer:
 `.trim()
 
   const startTime = performance.now()
@@ -52,11 +90,13 @@ Provide only the direct answer, without any additional explanation or formatting
   const actual = text.trim()
   const latencyMs = performance.now() - startTime
 
-  const isCorrect = await validateAnswer({
+  const comparisonResult = compareAnswers(
     actual,
-    expected: question.groundTruth,
-    question: question.prompt,
-  })
+    question.groundTruth,
+    question.answerType ?? 'string',
+    question.normalizationOptions,
+  )
+  const isCorrect = comparisonResult.match
 
   return {
     questionId: question.id,
@@ -70,49 +110,3 @@ Provide only the direct answer, without any additional explanation or formatting
     latencyMs,
   }
 }
-
-/**
- * Validate an answer using LLM-as-judge approach
- */
-async function validateAnswer(
-  {
-    actual,
-    expected,
-    question,
-  }:
-  {
-    actual: string
-    expected: string
-    question: string
-  },
-): Promise<boolean> {
-  const prompt = `
-You are validating answers to questions about structured data.
-
-Question: ${question}
-Expected answer: ${expected}
-Actual answer: ${actual}
-
-Is the actual answer correct? Consider:
-- Exact matches are correct
-- Semantically equivalent answers are correct (e.g., "50000" vs "$50,000" vs "50000 dollars")
-- Minor formatting differences are acceptable
-- Case-insensitive comparison for text
-
-Respond with only "YES" or "NO".
-`.trim()
-
-  try {
-    const { text } = await generateText({
-      model: models.find(m => m.modelId === 'gpt-5-nano')!,
-      prompt,
-    })
-
-    return text.trim().toUpperCase() === 'YES'
-  }
-  catch (error) {
-    prompts.log.error(`Validation error: ${error}`)
-    // Fallback to simple string comparison
-    return actual.toLowerCase().trim() === expected.toLowerCase().trim()
-  }
-}

benchmarks/src/formatters.ts

@@ -1,7 +1,8 @@
+import type { Dataset } from './types.ts'
 import { stringify as stringifyCSV } from 'csv-stringify/sync'
 import { XMLBuilder } from 'fast-xml-parser'
 import { stringify as stringifyYAML } from 'yaml'
-import { encode as encodeToon } from '../../src/index'
+import { encode as encodeToon } from '../../packages/toon/src/index.ts'
 
 /**
  * Format converters registry
@@ -12,11 +13,12 @@ import { encode as encodeToon } from '../../src/index'
  * CSV has inherent limitations with nested structures (see `toCSV` docs).
  */
 export const formatters: Record<string, (data: unknown) => string> = {
-  json: data => JSON.stringify(data, undefined, 2),
-  toon: data => encodeToon(data),
-  csv: data => toCSV(data),
-  xml: data => toXML(data),
-  yaml: data => stringifyYAML(data),
+  'json-pretty': data => JSON.stringify(data, undefined, 2),
+  'json-compact': data => JSON.stringify(data),
+  'toon': data => encodeToon(data),
+  'csv': data => toCSV(data),
+  'xml': data => toXML(data),
+  'yaml': data => stringifyYAML(data),
 }
 
 /**
@@ -74,3 +76,14 @@ function toXML(data: unknown): string {
 
   return builder.build(data)
 }
+
+/**
+ * Check if a dataset supports CSV format
+ *
+ * @remarks
+ * CSV is only suitable for flat tabular data. Datasets with nested structures
+ * should not be compared using CSV as it cannot properly represent the data.
+ */
+export function supportsCSV(dataset: Dataset): boolean {
+  return dataset.metadata.supportsCSV
+}

benchmarks/src/normalize.ts

@@ -0,0 +1,386 @@
+/**
+ * Type of expected answer for deterministic comparison
+ */
+export type AnswerType
+  = | 'integer'
+    | 'number'
+    | 'boolean'
+    | 'date'
+    | 'string'
+    | 'csv-list-ordered'
+    | 'csv-list-unordered'
+
+/**
+ * Options for answer normalization and comparison
+ */
+export interface NormalizationOptions {
+  /**
+   * Tolerance for floating-point number comparison (e.g., 1e-6).
+   * @default 1e-6
+   */
+  tolerance?: number
+
+  /**
+   * Whether string comparison should be case-sensitive.
+   * @default false
+   */
+  caseSensitive?: boolean
+
+  /**
+   * Allow currency symbols ($, €, etc.) in number extraction.
+   * @default true
+   */
+  allowCurrency?: boolean
+
+  /**
+   * Allow percent signs (%) in number extraction (will divide by 100).
+   * @default true
+   */
+  allowPercent?: boolean
+
+  /**
+   * Number of decimal places to round to for number comparison.
+   * If specified, overrides tolerance-based comparison.
+   */
+  decimalPlaces?: number
+}
+
+interface NormalizedResult {
+  success: boolean
+  value?: unknown
+  error?: string
+}
+
+/**
+ * Default normalization options
+ */
+const DEFAULT_OPTIONS: Required<NormalizationOptions> = {
+  tolerance: 1e-6,
+  caseSensitive: false,
+  allowCurrency: true,
+  allowPercent: true,
+  decimalPlaces: undefined!,
+}
+
+// Regex pattern constants
+const INTEGER_PATTERN_WITH_CURRENCY = /[$€£¥]?\s*-?\d[\d,]*/
+const INTEGER_PATTERN = /-?\d[\d,]*/
+const NUMBER_PATTERN_WITH_CURRENCY = /[$€£¥]?\s*-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?%?/i
+const NUMBER_PATTERN = /-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?%?/i
+const WRAPPING_QUOTES_PATTERN = /^["']|["']$/g
+const CODE_FENCE_PATTERN = /^```[\s\S]*?```$/g
+const LANGUAGE_IDENTIFIER_PATTERN = /^\w+\n/
+const CURRENCY_AND_FORMATTING_CHARS = /[$€£¥,\s]/g
+const NUMBER_CLEANUP_CHARS = /[$€£¥,%\s]/g
+
+// Boolean value constants
+const TRUE_VALUES = new Set(['true', 'yes', 'y', '1'])
+const FALSE_VALUES = new Set(['false', 'no', 'n', '0'])
+
+// Numeric constants
+const PERCENTAGE_DIVISOR = 100
+const DECIMAL_BASE = 10
+const MONTH_OFFSET = 1 // JavaScript months are 0-indexed
+const DATE_COMPONENT_WIDTH = 2
+const DATE_PAD_CHAR = '0'
+
+// String constants
+const CSV_DELIMITER = ','
+
+/**
+ * Strip wrapping quotes from a string
+ */
+function stripWrappingQuotes(text: string): string {
+  return text.trim().replace(WRAPPING_QUOTES_PATTERN, '')
+}
+
+/**
+ * Extract and normalize an integer from a string
+ *
+ * @remarks
+ * Handles: "42", "1,234", "$5,678", "  -99  ", "The answer is 42."
+ */
+function normalizeInteger(text: string, options: Required<NormalizationOptions>): NormalizedResult {
+  // Strip common formatting, extract first integer-like token
+  const pattern = options.allowCurrency
+    ? INTEGER_PATTERN_WITH_CURRENCY
+    : INTEGER_PATTERN
+
+  const match = text.match(pattern)
+  if (!match)
+    return { success: false, error: `No integer found in: "${text}"` }
+
+  // Remove currency symbols, spaces, and thousand separators
+  const normalizedValue = match[0].replace(CURRENCY_AND_FORMATTING_CHARS, '')
+  const parsedNumber = Number.parseInt(normalizedValue, DECIMAL_BASE)
+
+  if (Number.isNaN(parsedNumber))
+    return { success: false, error: `Failed to parse integer: "${match[0]}"` }
+
+  return { success: true, value: parsedNumber }
+}
+
+/**
+ * Extract and normalize a floating-point number from a string
+ *
+ * @remarks
+ * Handles: "3.14", "1,234.56", "$5,678.90", "42%", "1.5e-3", "Price: $99.99"
+ */
+function normalizeNumber(text: string, options: Required<NormalizationOptions>): NormalizedResult {
+  // Extract first number-like token (supports scientific notation)
+  const pattern = options.allowCurrency
+    ? NUMBER_PATTERN_WITH_CURRENCY
+    : NUMBER_PATTERN
+
+  const match = text.match(pattern)
+  if (!match)
+    return { success: false, error: `No number found in: "${text}"` }
+
+  const token = match[0]
+  const hasPercentSign = options.allowPercent && token.endsWith('%')
+
+  // Remove currency, commas, spaces, and percent sign
+  const normalizedToken = token.replace(NUMBER_CLEANUP_CHARS, '')
+  let parsedNumber = Number.parseFloat(normalizedToken)
+
+  if (Number.isNaN(parsedNumber))
+    return { success: false, error: `Failed to parse number: "${token}"` }
+
+  // Convert percentage to decimal if present
+  if (hasPercentSign)
+    parsedNumber = parsedNumber / PERCENTAGE_DIVISOR
+
+  // Round to specified decimal places if requested
+  if (options.decimalPlaces !== undefined) {
+    const factor = DECIMAL_BASE ** options.decimalPlaces
+    parsedNumber = Math.round(parsedNumber * factor) / factor
+  }
+
+  return { success: true, value: parsedNumber }
+}
+
+/**
+ * Normalize a boolean/yes-no answer
+ *
+ * @remarks
+ * Handles: "true", "false", "yes", "no", "y", "n", "1", "0" (case-insensitive)
+ */
+function normalizeBoolean(text: string): NormalizedResult {
+  const normalizedValue = text.trim().toLowerCase()
+
+  if (TRUE_VALUES.has(normalizedValue))
+    return { success: true, value: true }
+
+  if (FALSE_VALUES.has(normalizedValue))
+    return { success: true, value: false }
+
+  return { success: false, error: `Not a boolean: "${text}"` }
+}
+
+/**
+ * Normalize a date string to YYYY-MM-DD format
+ *
+ * @remarks
+ * Handles: ISO dates, "Nov 1, 2025", "2025-11-01", RFC 2822, etc.
+ */
+function normalizeDate(text: string): NormalizedResult {
+  const cleaned = stripWrappingQuotes(text)
+
+  // Try parsing as date
+  const parsedDate = new Date(cleaned)
+  if (Number.isNaN(parsedDate.getTime()))
+    return { success: false, error: `Invalid date: "${text}"` }
+
+  // Normalize to YYYY-MM-DD (UTC)
+  const year = parsedDate.getUTCFullYear()
+  const monthPadded = String(parsedDate.getUTCMonth() + MONTH_OFFSET).padStart(DATE_COMPONENT_WIDTH, DATE_PAD_CHAR)
+  const dayPadded = String(parsedDate.getUTCDate()).padStart(DATE_COMPONENT_WIDTH, DATE_PAD_CHAR)
+  const normalized = `${year}-${monthPadded}-${dayPadded}`
+
+  return { success: true, value: normalized }
+}
+
+/**
+ * Normalize a string (trim, optionally case-insensitive)
+ *
+ * @remarks
+ * Handles wrapping quotes and code fences.
+ */
+function normalizeString(text: string, options: Required<NormalizationOptions>): NormalizedResult {
+  let trimmedText = text.trim()
+
+  // Strip wrapping quotes
+  trimmedText = trimmedText.replace(WRAPPING_QUOTES_PATTERN, '')
+
+  // Strip code fences (```...```)
+  trimmedText = trimmedText.replace(CODE_FENCE_PATTERN, (match) => {
+    const inner = match.slice(3, -3).trim()
+    // Remove language identifier if present (e.g., ```json)
+    return inner.replace(LANGUAGE_IDENTIFIER_PATTERN, '')
+  })
+
+  trimmedText = trimmedText.trim()
+
+  const value = options.caseSensitive ? trimmedText : trimmedText.toLowerCase()
+  return { success: true, value }
+}
+
+/**
+ * Normalize a comma-separated list (ordered)
+ *
+ * @remarks
+ * Handles: "a,b,c", "a, b, c", " a , b , c "
+ */
+function normalizeCsvListOrdered(text: string, options: Required<NormalizationOptions>): NormalizedResult {
+  const strippedText = stripWrappingQuotes(text)
+  const items = strippedText
+    .split(CSV_DELIMITER)
+    .map(item => item.trim())
+    .filter(item => item.length > 0)
+
+  const normalizedItems = items.map(item =>
+    options.caseSensitive ? item : item.toLowerCase(),
+  )
+
+  return { success: true, value: normalizedItems }
+}
+
+/**
+ * Normalize a comma-separated list (unordered, compare as sets)
+ *
+ * @remarks
+ * Handles: "c,a,b" equals "a,b,c"
+ */
+function normalizeCsvListUnordered(text: string, options: Required<NormalizationOptions>): NormalizedResult {
+  const result = normalizeCsvListOrdered(text, options)
+  if (!result.success)
+    return result
+
+  // Type guard: ensure result.value is an array
+  if (!Array.isArray(result.value))
+    return { success: false, error: 'Expected array result from normalizeCsvListOrdered' }
+
+  // Sort for deterministic comparison
+  const sorted = [...result.value].sort()
+  return { success: true, value: sorted }
+}
+
+/**
+ * Normalize a value based on its expected kind
+ */
+export function normalizeAnswer(
+  text: string,
+  kind: AnswerType,
+  options: Partial<NormalizationOptions> = {},
+): NormalizedResult {
+  const resolvedOptions: Required<NormalizationOptions> = { ...DEFAULT_OPTIONS, ...options }
+
+  switch (kind) {
+    case 'integer':
+      return normalizeInteger(text, resolvedOptions)
+    case 'number':
+      return normalizeNumber(text, resolvedOptions)
+    case 'boolean':
+      return normalizeBoolean(text)
+    case 'date':
+      return normalizeDate(text)
+    case 'string':
+      return normalizeString(text, resolvedOptions)
+    case 'csv-list-ordered':
+      return normalizeCsvListOrdered(text, resolvedOptions)
+    case 'csv-list-unordered':
+      return normalizeCsvListUnordered(text, resolvedOptions)
+    default:
+      return { success: false, error: `Unknown answer kind: ${kind}` }
+  }
+}
+
+/**
+ * Compare two normalized values based on answer kind
+ */
+function compareValues(
+  actual: unknown,
+  expected: unknown,
+  kind: AnswerType,
+  options: Required<NormalizationOptions>,
+): boolean {
+  switch (kind) {
+    case 'integer':
+    case 'boolean':
+    case 'date':
+    case 'string':
+      return actual === expected
+
+    case 'number':
+      if (typeof actual !== 'number' || typeof expected !== 'number')
+        return false
+
+      if (options.decimalPlaces !== undefined) {
+        // Already rounded during normalization
+        return actual === expected
+      }
+      return Math.abs(actual - expected) <= options.tolerance
+
+    case 'csv-list-ordered':
+      if (!Array.isArray(actual) || !Array.isArray(expected))
+        return false
+      if (actual.length !== expected.length)
+        return false
+      return actual.every((item, i) => item === expected[i])
+
+    case 'csv-list-unordered':
+      if (!Array.isArray(actual) || !Array.isArray(expected))
+        return false
+      if (actual.length !== expected.length)
+        return false
+      // Already sorted during normalization
+      return actual.every((item, i) => item === expected[i])
+
+    default:
+      return false
+  }
+}
+
+/**
+ * Compare actual and expected answers with deterministic, type-aware normalization
+ *
+ * @remarks
+ * Returns true if answers match within the specified tolerance/rules.
+ */
+export function compareAnswers(
+  actual: string,
+  expected: string,
+  kind: AnswerType,
+  options: Partial<NormalizationOptions> = {},
+): { match: boolean, details?: string } {
+  const resolvedOptions: Required<NormalizationOptions> = { ...DEFAULT_OPTIONS, ...options }
+
+  // Normalize both answers
+  const actualResult = normalizeAnswer(actual, kind, resolvedOptions)
+  const expectedResult = normalizeAnswer(expected, kind, resolvedOptions)
+
+  // If either normalization failed, return false with details
+  if (!actualResult.success) {
+    return {
+      match: false,
+      details: `Failed to normalize actual answer: ${actualResult.error}`,
+    }
+  }
+
+  if (!expectedResult.success) {
+    return {
+      match: false,
+      details: `Failed to normalize expected answer: ${expectedResult.error}`,
+    }
+  }
+
+  // Compare normalized values
+  const match = compareValues(actualResult.value, expectedResult.value, kind, resolvedOptions)
+
+  return {
+    match,
+    details: match
+      ? undefined
+      : `Mismatch: actual="${actualResult.value}" vs expected="${expectedResult.value}"`,
+  }
+}

benchmarks/src/questions.ts

@@ -1,711 +0,0 @@
-/**
- * Question generation for TOON benchmarks
- *
- * Generates ~150-160 questions across different question types and datasets:
- * - Field Retrieval: Direct field access with no computation
- *   Examples: "What is X's salary?", "What is the status of order Y?"
- * - Aggregation: Counts, sums, averages, min/max operations (including single-condition filters)
- *   Examples: "How many X?", "What is the total/average?", "How many X > threshold?"
- * - Filtering: Multi-condition queries requiring complex logical operations
- *   Examples: "How many X WHERE condition1 AND condition2?"
- */
-
-import type { AnalyticsMetric, Employee, Order, Repository } from './datasets'
-import type { Question } from './types'
-import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from './constants'
-import { datasets } from './datasets'
-
-/**
- * Generate all questions from datasets
- */
-export function generateQuestions(): Question[] {
-  const questions: Question[] = []
-  let idCounter = 1
-
-  // Get datasets with proper typing
-  const tabular = (datasets.find(d => d.name === 'tabular')?.data.employees as Employee[]) ?? []
-  const nested = (datasets.find(d => d.name === 'nested')?.data.orders as Order[]) ?? []
-  const analytics = (datasets.find(d => d.name === 'analytics')?.data.metrics as AnalyticsMetric[]) ?? []
-  const github = (datasets.find(d => d.name === 'github')?.data.repositories as Repository[]) ?? []
-
-  if (tabular.length > 0) {
-    // Field retrieval: specific employees
-    for (let i = 0; i < Math.min(QUESTION_LIMITS.tabular.fieldRetrieval, tabular.length); i++) {
-      const emp = tabular[i * 2] || tabular[i]
-      if (!emp)
-        continue
-
-      // Rotate through all field types
-      if (i % 5 === 0) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the salary of ${emp.name}?`,
-          groundTruth: String(emp.salary),
-          type: 'field-retrieval',
-          dataset: 'tabular',
-        })
-      }
-      else if (i % 5 === 1) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What department does ${emp.name} work in?`,
-          groundTruth: emp.department,
-          type: 'field-retrieval',
-          dataset: 'tabular',
-        })
-      }
-      else if (i % 5 === 2) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the email address of ${emp.name}?`,
-          groundTruth: emp.email,
-          type: 'field-retrieval',
-          dataset: 'tabular',
-        })
-      }
-      else if (i % 5 === 3) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many years of experience does ${emp.name} have?`,
-          groundTruth: String(emp.yearsExperience),
-          type: 'field-retrieval',
-          dataset: 'tabular',
-        })
-      }
-      else {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `Is ${emp.name} an active employee?`,
-          groundTruth: emp.active ? 'yes' : 'no',
-          type: 'field-retrieval',
-          dataset: 'tabular',
-        })
-      }
-    }
-
-    // Aggregation: count by department
-    const departments = [...new Set(tabular.map(e => e.department))]
-    for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.aggregationDepartments)) {
-      const count = tabular.filter(e => e.department === dept).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many employees work in ${dept}?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'tabular',
-      })
-    }
-
-    // Aggregation: salary ranges (single-condition filters)
-    for (const threshold of QUESTION_THRESHOLDS.tabular.salaryRanges) {
-      const count = tabular.filter(e => e.salary > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many employees have a salary greater than ${threshold}?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'tabular',
-      })
-    }
-
-    // Aggregation: totals and averages
-    const totalEmployees = tabular.length
-    const avgSalary = Math.round(tabular.reduce((sum, e) => sum + e.salary, 0) / totalEmployees)
-    const activeCount = tabular.filter(e => e.active).length
-    const inactiveCount = tabular.filter(e => !e.active).length
-
-    questions.push(
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many employees are in the dataset?',
-        groundTruth: String(totalEmployees),
-        type: 'aggregation',
-        dataset: 'tabular',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average salary across all employees?',
-        groundTruth: String(avgSalary),
-        type: 'aggregation',
-        dataset: 'tabular',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many employees are active?',
-        groundTruth: String(activeCount),
-        type: 'aggregation',
-        dataset: 'tabular',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many employees are inactive?',
-        groundTruth: String(inactiveCount),
-        type: 'aggregation',
-        dataset: 'tabular',
-      },
-    )
-
-    // Filtering: count by department with salary filter (multi-condition)
-    for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringMultiConditionDepartments)) {
-      const count = tabular.filter(e => e.department === dept && e.salary > QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many employees in ${dept} have a salary greater than ${QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold}?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'tabular',
-      })
-    }
-
-    // Filtering: active employees by experience (multi-condition)
-    for (const exp of QUESTION_THRESHOLDS.tabular.experienceYears.slice(0, QUESTION_LIMITS.tabular.filteringExperience)) {
-      const count = tabular.filter(e => e.yearsExperience > exp && e.active).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many active employees have more than ${exp} years of experience?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'tabular',
-      })
-    }
-
-    // Filtering: department by experience (multi-condition)
-    for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentExp)) {
-      const count = tabular.filter(e => e.department === dept && e.yearsExperience > QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many employees in ${dept} have more than ${QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold} years of experience?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'tabular',
-      })
-    }
-
-    // Filtering: department by active status (multi-condition)
-    for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentActive)) {
-      const count = tabular.filter(e => e.department === dept && e.active).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many active employees work in ${dept}?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'tabular',
-      })
-    }
-  }
-
-  if (nested.length > 0) {
-    // Field retrieval: order totals and statuses
-    for (let i = 0; i < Math.min(QUESTION_LIMITS.nested.fieldRetrievalOrders, nested.length); i++) {
-      const order = nested[i * 2] || nested[i]
-      if (!order)
-        continue
-
-      if (i % 2 === 0) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the total for order ${order.orderId}?`,
-          groundTruth: String(order.total),
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-      else {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the status of order ${order.orderId}?`,
-          groundTruth: order.status,
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-    }
-
-    // Field retrieval: customer info and order dates (expanded)
-    for (let i = 0; i < Math.min(QUESTION_LIMITS.nested.fieldRetrievalCustomers, nested.length); i++) {
-      const order = nested[i * 2 + 1] || nested[i]
-      if (!order)
-        continue
-
-      if (i % 4 === 0) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the customer name for order ${order.orderId}?`,
-          groundTruth: order.customer.name,
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-      else if (i % 4 === 1) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the customer email for order ${order.orderId}?`,
-          groundTruth: order.customer.email,
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-      else if (i % 4 === 2) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the order date for order ${order.orderId}?`,
-          groundTruth: order.orderDate || '',
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-      else {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many items are in order ${order.orderId}?`,
-          groundTruth: String(order.items.length),
-          type: 'field-retrieval',
-          dataset: 'nested',
-        })
-      }
-    }
-
-    // Aggregation: totals and averages
-    const totalRevenue = nested.reduce((sum, o) => sum + o.total, 0)
-    const avgOrderValue = totalRevenue / nested.length
-    const totalOrders = nested.length
-    const maxOrderValue = Math.max(...nested.map(o => o.total))
-
-    // Count by status
-    const statuses = [...new Set(nested.map(o => o.status))]
-    for (const status of statuses.slice(0, QUESTION_LIMITS.nested.aggregationStatuses)) {
-      const count = nested.filter(o => o.status === status).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many orders have status "${status}"?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'nested',
-      })
-    }
-
-    questions.push(
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the total revenue across all orders?',
-        groundTruth: String(totalRevenue.toFixed(2)),
-        type: 'aggregation',
-        dataset: 'nested',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average order value?',
-        groundTruth: String(avgOrderValue.toFixed(2)),
-        type: 'aggregation',
-        dataset: 'nested',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many orders are in the dataset?',
-        groundTruth: String(totalOrders),
-        type: 'aggregation',
-        dataset: 'nested',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the highest order total?',
-        groundTruth: String(maxOrderValue.toFixed(2)),
-        type: 'aggregation',
-        dataset: 'nested',
-      },
-    )
-
-    // Aggregation: high-value orders (single-condition filter)
-    for (const threshold of QUESTION_THRESHOLDS.nested.highValueOrders) {
-      const count = nested.filter(o => o.total > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many orders have a total greater than ${threshold}?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'nested',
-      })
-    }
-
-    // Filtering: multi-condition queries (status AND value)
-    const orderStatuses = [...new Set(nested.map(o => o.status))]
-    for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndValue)) {
-      const count = nested.filter(o => o.status === status && o.total > QUESTION_THRESHOLDS.nested.statusValueThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many orders have status "${status}" and total greater than ${QUESTION_THRESHOLDS.nested.statusValueThreshold}?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'nested',
-      })
-    }
-
-    // Filtering: status AND items count (multi-condition)
-    for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndItems)) {
-      const count = nested.filter(o => o.status === status && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many orders have status "${status}" and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'nested',
-      })
-    }
-
-    // Filtering: total AND items count (multi-condition)
-    for (const threshold of QUESTION_THRESHOLDS.nested.totalThresholdsForItems) {
-      const count = nested.filter(o => o.total > threshold && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many orders have a total greater than ${threshold} and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'nested',
-      })
-    }
-  }
-
-  if (analytics.length > 0) {
-    // Field retrieval: specific dates (expanded with all metrics)
-    for (let i = 0; i < Math.min(QUESTION_LIMITS.analytics.fieldRetrievalDates, analytics.length); i++) {
-      const metric = analytics[i * 3] || analytics[i]
-      if (!metric)
-        continue
-
-      if (i % 5 === 0) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many views were recorded on ${metric.date}?`,
-          groundTruth: String(metric.views),
-          type: 'field-retrieval',
-          dataset: 'analytics',
-        })
-      }
-      else if (i % 5 === 1) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What was the revenue on ${metric.date}?`,
-          groundTruth: String(metric.revenue),
-          type: 'field-retrieval',
-          dataset: 'analytics',
-        })
-      }
-      else if (i % 5 === 2) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What was the conversion count on ${metric.date}?`,
-          groundTruth: String(metric.conversions),
-          type: 'field-retrieval',
-          dataset: 'analytics',
-        })
-      }
-      else if (i % 5 === 3) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many clicks were recorded on ${metric.date}?`,
-          groundTruth: String(metric.clicks),
-          type: 'field-retrieval',
-          dataset: 'analytics',
-        })
-      }
-      else {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What was the bounce rate on ${metric.date}?`,
-          groundTruth: String(metric.bounceRate),
-          type: 'field-retrieval',
-          dataset: 'analytics',
-        })
-      }
-    }
-
-    // Aggregation: totals and averages
-    const totalViews = analytics.reduce((sum, m) => sum + m.views, 0)
-    const totalRevenue = analytics.reduce((sum, m) => sum + m.revenue, 0)
-    const totalConversions = analytics.reduce((sum, m) => sum + m.conversions, 0)
-    const avgViews = Math.round(totalViews / analytics.length)
-    const avgRevenue = totalRevenue / analytics.length
-    const avgConversions = Math.round(totalConversions / analytics.length)
-
-    questions.push(
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the total number of views across all dates?',
-        groundTruth: String(totalViews),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the total revenue across all dates?',
-        groundTruth: String(totalRevenue.toFixed(2)),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the total number of conversions across all dates?',
-        groundTruth: String(totalConversions),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average number of views per day?',
-        groundTruth: String(avgViews),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average revenue per day?',
-        groundTruth: String(avgRevenue.toFixed(2)),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average number of conversions per day?',
-        groundTruth: String(avgConversions),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many days are included in the analytics data?',
-        groundTruth: String(analytics.length),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the highest number of views recorded in a single day?',
-        groundTruth: String(Math.max(...analytics.map(m => m.views))),
-        type: 'aggregation',
-        dataset: 'analytics',
-      },
-    )
-
-    // Aggregation: high-performing days (single-condition filters)
-    for (const threshold of QUESTION_THRESHOLDS.analytics.views) {
-      const count = analytics.filter(m => m.views > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many days had more than ${threshold} views?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'analytics',
-      })
-    }
-
-    // Filtering: multi-condition queries (views AND conversions)
-    for (const viewThreshold of QUESTION_THRESHOLDS.analytics.viewsForFiltering) {
-      const count = analytics.filter(m => m.views > viewThreshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForFiltering).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many days had more than ${viewThreshold} views and more than ${QUESTION_THRESHOLDS.analytics.conversionsForFiltering} conversions?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'analytics',
-      })
-    }
-
-    // Filtering: views AND revenue (expanded)
-    for (const revenueThreshold of QUESTION_THRESHOLDS.analytics.revenueThresholds.slice(0, 5)) {
-      const count = analytics.filter(m => m.views > QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue && m.revenue > revenueThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many days had more than ${QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue} views and revenue greater than ${revenueThreshold}?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'analytics',
-      })
-    }
-
-    // Filtering: clicks AND conversions (multi-condition)
-    for (const clickThreshold of QUESTION_THRESHOLDS.analytics.clicksForFiltering) {
-      const count = analytics.filter(m => m.clicks > clickThreshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many days had more than ${clickThreshold} clicks and more than ${QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering} conversions?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'analytics',
-      })
-    }
-
-    // Filtering: revenue AND bounce rate (multi-condition)
-    for (const revenueThreshold of QUESTION_THRESHOLDS.analytics.revenueForBounceRate) {
-      const count = analytics.filter(m => m.revenue > revenueThreshold && m.bounceRate < QUESTION_THRESHOLDS.analytics.bounceRateThreshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many days had revenue greater than ${revenueThreshold} and bounce rate less than ${QUESTION_THRESHOLDS.analytics.bounceRateThreshold}?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'analytics',
-      })
-    }
-  }
-
-  if (github.length > 0) {
-    // Helper to extract owner from repo field
-    const getOwner = (repoFullName: string) => repoFullName.split('/')[0]!
-
-    // Field retrieval: specific repos (diverse fields)
-    for (let i = 0; i < Math.min(QUESTION_LIMITS.github.fieldRetrievalRepos, github.length); i++) {
-      const repo = github[i * 7]
-      if (!repo)
-        continue
-
-      if (i % 5 === 0) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many stars does ${repo.repo} have?`,
-          groundTruth: String(repo.stars),
-          type: 'field-retrieval',
-          dataset: 'github',
-        })
-      }
-      else if (i % 5 === 1) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many forks does ${repo.repo} have?`,
-          groundTruth: String(repo.forks),
-          type: 'field-retrieval',
-          dataset: 'github',
-        })
-      }
-      else if (i % 5 === 2) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `Who is the owner of ${repo.repo}?`,
-          groundTruth: getOwner(repo.repo),
-          type: 'field-retrieval',
-          dataset: 'github',
-        })
-      }
-      else if (i % 5 === 3) {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `What is the default branch of ${repo.repo}?`,
-          groundTruth: repo.defaultBranch,
-          type: 'field-retrieval',
-          dataset: 'github',
-        })
-      }
-      else {
-        questions.push({
-          id: `q${idCounter++}`,
-          prompt: `How many watchers does ${repo.repo} have?`,
-          groundTruth: String(repo.watchers),
-          type: 'field-retrieval',
-          dataset: 'github',
-        })
-      }
-    }
-
-    // Aggregation: popular repositories
-    const totalStars = github.reduce((sum, r) => sum + r.stars, 0)
-    const totalRepos = github.length
-    const avgStars = Math.round(totalStars / totalRepos)
-
-    questions.push(
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the total number of stars across all repositories?',
-        groundTruth: String(totalStars),
-        type: 'aggregation',
-        dataset: 'github',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'How many repositories are in the dataset?',
-        groundTruth: String(totalRepos),
-        type: 'aggregation',
-        dataset: 'github',
-      },
-      {
-        id: `q${idCounter++}`,
-        prompt: 'What is the average number of stars per repository?',
-        groundTruth: String(avgStars),
-        type: 'aggregation',
-        dataset: 'github',
-      },
-    )
-
-    // Aggregation: star thresholds (single-condition filters)
-    for (const threshold of QUESTION_THRESHOLDS.github.stars) {
-      const count = github.filter(r => r.stars > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories have more than ${threshold} stars?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'github',
-      })
-    }
-
-    // Aggregation: fork thresholds (single-condition filters)
-    for (const threshold of QUESTION_THRESHOLDS.github.forks) {
-      const count = github.filter(r => r.forks > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories have more than ${threshold} forks?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'github',
-      })
-    }
-
-    // Aggregation: watcher thresholds (single-condition filters)
-    for (const threshold of QUESTION_THRESHOLDS.github.watchers) {
-      const count = github.filter(r => r.watchers > threshold).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories have more than ${threshold} watchers?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'github',
-      })
-    }
-
-    // Aggregation: default branch counts
-    const branches = [...new Set(github.map(r => r.defaultBranch))]
-    for (const branch of branches.slice(0, QUESTION_LIMITS.github.aggregationBranches)) {
-      const count = github.filter(r => r.defaultBranch === branch).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories use "${branch}" as their default branch?`,
-        groundTruth: String(count),
-        type: 'aggregation',
-        dataset: 'github',
-      })
-    }
-
-    // Filtering: multi-condition queries (stars AND forks)
-    for (const combo of QUESTION_THRESHOLDS.github.starForkCombinations.slice(0, QUESTION_LIMITS.github.filteringStarsAndForks)) {
-      const count = github.filter(r => r.stars > combo.stars && r.forks > combo.forks).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories have more than ${combo.stars} stars and more than ${combo.forks} forks?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'github',
-      })
-    }
-
-    // Filtering: stars AND watchers (multi-condition)
-    for (const combo of QUESTION_THRESHOLDS.github.starWatcherCombinations) {
-      const count = github.filter(r => r.stars > combo.stars && r.watchers > combo.watchers).length
-      questions.push({
-        id: `q${idCounter++}`,
-        prompt: `How many repositories have more than ${combo.stars} stars and more than ${combo.watchers} watchers?`,
-        groundTruth: String(count),
-        type: 'filtering',
-        dataset: 'github',
-      })
-    }
-  }
-
-  return questions
-}

benchmarks/src/questions/analytics.ts

@@ -0,0 +1,208 @@
+import type { AnalyticsMetric } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
+
+/**
+ * Generate analytics (website metrics) questions
+ */
+export function generateAnalyticsQuestions(metrics: AnalyticsMetric[], getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  // Field retrieval: date-based metrics
+  const metricFieldGenerators: Array<(metric: AnalyticsMetric, getId: () => string) => Question> = [
+    (metric, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What are the views for ${metric.date}?`)
+      .groundTruth(String(metric.views))
+      .type('field-retrieval')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+    (metric, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the revenue for ${metric.date}?`)
+      .groundTruth(String(metric.revenue))
+      .type('field-retrieval')
+      .dataset('analytics')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    (metric, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the bounce rate for ${metric.date}?`)
+      .groundTruth(String(metric.bounceRate))
+      .type('field-retrieval')
+      .dataset('analytics')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    (metric, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many conversions were there on ${metric.date}?`)
+      .groundTruth(String(metric.conversions))
+      .type('field-retrieval')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+  ]
+
+  questions.push(...rotateQuestions(
+    metrics,
+    metricFieldGenerators,
+    QUESTION_LIMITS.analytics.fieldRetrievalDates,
+    SAMPLE_STRIDES.ANALYTICS_FIELD,
+    getId,
+  ))
+
+  // Aggregation: basic statistics
+  const totalDays = metrics.length
+  const totalViews = metrics.reduce((sum, m) => sum + m.views, 0)
+  const totalConversions = metrics.reduce((sum, m) => sum + m.conversions, 0)
+  const totalRevenue = metrics.reduce((sum, m) => sum + m.revenue, 0)
+  const avgBounceRate = metrics.reduce((sum, m) => sum + m.bounceRate, 0) / metrics.length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many days of data are in the dataset?')
+      .groundTruth(String(totalDays))
+      .type('aggregation')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of views across all dates?')
+      .groundTruth(String(totalViews))
+      .type('aggregation')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of conversions across all dates?')
+      .groundTruth(String(totalConversions))
+      .type('aggregation')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total revenue across all dates?')
+      .groundTruth(String(totalRevenue.toFixed(2)))
+      .type('aggregation')
+      .dataset('analytics')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the average bounce rate?')
+      .groundTruth(String(avgBounceRate.toFixed(2)))
+      .type('aggregation')
+      .dataset('analytics')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+  )
+
+  // Aggregation: high views/conversions
+  for (const threshold of QUESTION_THRESHOLDS.analytics.views) {
+    const count = metrics.filter(m => m.views > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had more than ${threshold} views?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  for (const threshold of QUESTION_THRESHOLDS.analytics.conversions) {
+    const count = metrics.filter(m => m.conversions > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had more than ${threshold} conversions?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: multi-condition (views AND revenue)
+  for (const threshold of QUESTION_THRESHOLDS.analytics.viewsForFiltering) {
+    const count = metrics.filter(
+      m => m.views > threshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForFiltering,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had more than ${threshold} views and more than ${QUESTION_THRESHOLDS.analytics.conversionsForFiltering} conversions?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: revenue thresholds
+  for (const threshold of QUESTION_THRESHOLDS.analytics.revenueThresholds) {
+    const count = metrics.filter(
+      m => m.revenue > threshold && m.views > QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had revenue greater than ${threshold} with views above ${QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue}?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: clicks and conversions
+  for (const threshold of QUESTION_THRESHOLDS.analytics.clicksForFiltering) {
+    const count = metrics.filter(
+      m => m.clicks > threshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had more than ${threshold} clicks and more than ${QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering} conversions?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: revenue and bounce rate
+  for (const threshold of QUESTION_THRESHOLDS.analytics.revenueForBounceRate) {
+    const count = metrics.filter(
+      m => m.revenue > threshold && m.bounceRate < QUESTION_THRESHOLDS.analytics.bounceRateThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many days had revenue greater than ${threshold} with bounce rate below ${QUESTION_THRESHOLDS.analytics.bounceRateThreshold}?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('analytics')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/event-logs.ts

@@ -0,0 +1,198 @@
+import type { EventLog } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
+
+/**
+ * Generate event log questions
+ */
+export function generateEventLogsQuestions(logs: EventLog[], getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  // Field retrieval: log metadata
+  const logFieldGenerators: Array<(log: EventLog, getId: () => string) => Question> = [
+    (log, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the level of the log at ${log.timestamp}?`)
+      .groundTruth(log.level)
+      .type('field-retrieval')
+      .dataset('event-logs')
+      .answerType('string')
+      .build(),
+    (log, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the endpoint for the log at ${log.timestamp}?`)
+      .groundTruth(log.endpoint)
+      .type('field-retrieval')
+      .dataset('event-logs')
+      .answerType('string')
+      .build(),
+    (log, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the status code for the log at ${log.timestamp}?`)
+      .groundTruth(String(log.statusCode))
+      .type('field-retrieval')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+    (log, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the response time for the log at ${log.timestamp}?`)
+      .groundTruth(String(log.responseTime))
+      .type('field-retrieval')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+  ]
+
+  questions.push(...rotateQuestions(
+    logs,
+    logFieldGenerators,
+    QUESTION_LIMITS.eventLogs.fieldRetrieval,
+    SAMPLE_STRIDES.EVENT_LOG_FIELD,
+    getId,
+  ))
+
+  // Aggregation: basic statistics
+  const totalLogs = logs.length
+  const avgResponseTime = logs.reduce((sum, l) => sum + l.responseTime, 0) / logs.length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many log entries are in the dataset?')
+      .groundTruth(String(totalLogs))
+      .type('aggregation')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the average response time across all logs?')
+      .groundTruth(String(avgResponseTime.toFixed(2)))
+      .type('aggregation')
+      .dataset('event-logs')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+  )
+
+  // Aggregation: by level
+  const levels = [...new Set(logs.map(l => l.level))]
+  for (const level of levels) {
+    const count = logs.filter(l => l.level === level).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many log entries have level "${level}"?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('event-logs')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: by endpoint
+  const endpoints = [...new Set(logs.map(l => l.endpoint))]
+  for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.aggregationEndpoints)) {
+    const count = logs.filter(l => l.endpoint === endpoint).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many log entries are for endpoint "${endpoint}"?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('event-logs')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: by status code range
+  const errorCount = logs.filter(l => l.statusCode >= 400).length
+  const successCount = logs.filter(l => l.statusCode >= 200 && l.statusCode < 300).length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many log entries have a status code indicating an error (>= 400)?')
+      .groundTruth(String(errorCount))
+      .type('aggregation')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many log entries have a successful status code (200-299)?')
+      .groundTruth(String(successCount))
+      .type('aggregation')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: retryable errors
+  const retryableErrorCount = logs.filter(l => l.error?.retryable === true).length
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many log entries have a retryable error?')
+      .groundTruth(String(retryableErrorCount))
+      .type('aggregation')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Filtering: multi-condition (level AND status)
+  for (const level of levels.slice(0, QUESTION_LIMITS.eventLogs.filteringLevelAndStatus)) {
+    // Skip `info` level as it never has status >= 400 by design
+    if (level === 'info')
+      continue
+
+    const count = logs.filter(l => l.level === level && l.statusCode >= 400).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many log entries have level "${level}" and status code >= 400?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('event-logs')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: endpoint AND status
+  for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.filteringEndpointAndStatus)) {
+    const count = logs.filter(l => l.endpoint === endpoint && l.statusCode >= 500).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many log entries are for endpoint "${endpoint}" with status code >= 500?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('event-logs')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: endpoint AND retryable error
+  for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.filteringEndpointRetryable)) {
+    const count = logs.filter(l => l.endpoint === endpoint && l.error?.retryable === true).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many log entries for endpoint "${endpoint}" have a retryable error?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('event-logs')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/github.ts

@@ -0,0 +1,194 @@
+import type { Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
+
+/**
+ * Generate GitHub repository questions
+ */
+export function generateGithubQuestions(repos: Repository[], getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  // Field retrieval: repository metadata
+  const repoFieldGenerators: Array<(repo: Repository, getId: () => string) => Question> = [
+    (repo, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many stars does ${repo.repo} have?`)
+      .groundTruth(String(repo.stars))
+      .type('field-retrieval')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    (repo, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many forks does ${repo.repo} have?`)
+      .groundTruth(String(repo.forks))
+      .type('field-retrieval')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    (repo, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many watchers does ${repo.repo} have?`)
+      .groundTruth(String(repo.watchers))
+      .type('field-retrieval')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    (repo, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the main branch of ${repo.repo}?`)
+      .groundTruth(repo.defaultBranch)
+      .type('field-retrieval')
+      .dataset('github')
+      .answerType('string')
+      .normalize({ caseSensitive: true })
+      .build(),
+  ]
+
+  questions.push(...rotateQuestions(
+    repos,
+    repoFieldGenerators,
+    QUESTION_LIMITS.github.fieldRetrievalRepos,
+    SAMPLE_STRIDES.REPO_FIELD,
+    getId,
+  ))
+
+  // Aggregation: basic statistics
+  const totalRepos = repos.length
+  const totalStars = repos.reduce((sum, r) => sum + r.stars, 0)
+  const totalForks = repos.reduce((sum, r) => sum + r.forks, 0)
+  const avgStars = totalStars / totalRepos
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many repositories are in the dataset?')
+      .groundTruth(String(totalRepos))
+      .type('aggregation')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of stars across all repositories?')
+      .groundTruth(String(totalStars))
+      .type('aggregation')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of forks across all repositories?')
+      .groundTruth(String(totalForks))
+      .type('aggregation')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the average number of stars per repository?')
+      .groundTruth(String(Math.round(avgStars)))
+      .type('aggregation')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: by default branch
+  const branches = [...new Set(repos.map(r => r.defaultBranch))]
+  for (const branch of branches.slice(0, QUESTION_LIMITS.github.aggregationBranches)) {
+    const count = repos.filter(r => r.defaultBranch === branch).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories use "${branch}" as their default branch?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: high star counts
+  for (const threshold of QUESTION_THRESHOLDS.github.stars) {
+    const count = repos.filter(r => r.stars > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories have more than ${threshold} stars?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: high fork counts
+  for (const threshold of QUESTION_THRESHOLDS.github.forks) {
+    const count = repos.filter(r => r.forks > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories have more than ${threshold} forks?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: high watcher counts
+  for (const threshold of QUESTION_THRESHOLDS.github.watchers) {
+    const count = repos.filter(r => r.watchers > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories have more than ${threshold} watchers?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: multi-condition (stars AND forks)
+  for (const combo of QUESTION_THRESHOLDS.github.starForkCombinations.slice(0, QUESTION_LIMITS.github.filteringStarsAndForks)) {
+    const count = repos.filter(
+      r => r.stars > combo.stars && r.forks > combo.forks,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories have more than ${combo.stars} stars and more than ${combo.forks} forks?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: stars AND watchers
+  for (const combo of QUESTION_THRESHOLDS.github.starWatcherCombinations) {
+    const count = repos.filter(
+      r => r.stars > combo.stars && r.watchers > combo.watchers,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many repositories have more than ${combo.stars} stars and more than ${combo.watchers} watchers?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('github')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/index.ts

@@ -0,0 +1,55 @@
+import type { AnalyticsMetric, Employee, EventLog, NestedConfig, Order, Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { ACCURACY_DATASETS } from '../datasets.ts'
+import { generateAnalyticsQuestions } from './analytics.ts'
+import { generateEventLogsQuestions } from './event-logs.ts'
+import { generateGithubQuestions } from './github.ts'
+import { generateNestedConfigQuestions } from './nested-config.ts'
+import { generateNestedQuestions } from './nested.ts'
+import { generateStructuralValidationQuestions } from './structural-validation.ts'
+import { generateStructureQuestions } from './structure.ts'
+import { generateTabularQuestions } from './tabular.ts'
+import { createIdGenerator } from './utils.ts'
+
+/**
+ * Generate questions from all datasets
+ *
+ * @remarks
+ * - Field Retrieval: Direct field access with no computation
+ *   Examples: "What is X's salary?", "What is the status of order Y?"
+ * - Aggregation: Counts, sums, averages, min/max operations (including single-condition filters)
+ *   Examples: "How many X?", "What is the total/average?", "How many X > threshold?"
+ * - Filtering: Multi-condition queries requiring complex logical operations
+ *   Examples: "How many X WHERE condition1 AND condition2?"
+ * - Structure Awareness: Tests format-native structural affordances (TOON's [N] and {fields}, CSV's header)
+ *   Examples: "How many records?", "List the field names", "What is the last record's field?"
+ */
+export function generateQuestions(): Question[] {
+  const questions: Question[] = []
+  const idGen = createIdGenerator()
+  const getId = () => idGen.next().value
+
+  // Get datasets with proper typing
+  const tabular = (ACCURACY_DATASETS.find(d => d.name === 'tabular')?.data.employees as Employee[]) ?? []
+  const nested = (ACCURACY_DATASETS.find(d => d.name === 'nested')?.data.orders as Order[]) ?? []
+  const analytics = (ACCURACY_DATASETS.find(d => d.name === 'analytics')?.data.metrics as AnalyticsMetric[]) ?? []
+  const github = (ACCURACY_DATASETS.find(d => d.name === 'github')?.data.repositories as Repository[]) ?? []
+  const eventLogs = (ACCURACY_DATASETS.find(d => d.name === 'event-logs')?.data.logs as EventLog[]) ?? []
+  const nestedConfig = ACCURACY_DATASETS.find(d => d.name === 'nested-config')?.data as NestedConfig | undefined
+
+  // Generate questions for each dataset
+  questions.push(...generateTabularQuestions(tabular, getId))
+  questions.push(...generateNestedQuestions(nested, getId))
+  questions.push(...generateAnalyticsQuestions(analytics, getId))
+  questions.push(...generateGithubQuestions(github, getId))
+  questions.push(...generateEventLogsQuestions(eventLogs, getId))
+  questions.push(...generateNestedConfigQuestions(nestedConfig, getId))
+
+  // Generate structure-awareness questions (tests format-native affordances)
+  questions.push(...generateStructureQuestions(tabular, nested, analytics, github, eventLogs, getId))
+
+  // Generate structural-validation questions (tests ability to detect corrupted data)
+  questions.push(...generateStructuralValidationQuestions(getId))
+
+  return questions
+}

benchmarks/src/questions/nested-config.ts

@@ -0,0 +1,283 @@
+import type { NestedConfig } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS } from '../constants.ts'
+import { QuestionBuilder } from './utils.ts'
+
+/**
+ * Generate nested configuration questions
+ */
+export function generateNestedConfigQuestions(config: NestedConfig | undefined, getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  if (!config)
+    return questions
+
+  // Field retrieval: top-level config values
+  const fieldRetrievalQuestions = [
+    {
+      prompt: 'What is the environment in the configuration?',
+      groundTruth: config.environment,
+      answerType: 'string' as const,
+    },
+    {
+      prompt: 'What is the database host?',
+      groundTruth: config.database.host,
+      answerType: 'string' as const,
+    },
+    {
+      prompt: 'What is the database port?',
+      groundTruth: String(config.database.port),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the maximum connection pool size?',
+      groundTruth: String(config.database.pool.max),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the session duration?',
+      groundTruth: String(config.authentication.session.duration),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the minimum connection pool size?',
+      groundTruth: String(config.database.pool.min),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the connection pool idle timeout?',
+      groundTruth: String(config.database.pool.idleTimeout),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the database name?',
+      groundTruth: config.database.name,
+      answerType: 'string' as const,
+    },
+    {
+      prompt: 'What is the session refresh threshold?',
+      groundTruth: String(config.authentication.session.refreshThreshold),
+      answerType: 'integer' as const,
+    },
+    {
+      prompt: 'What is the version in the configuration?',
+      groundTruth: config.version,
+      answerType: 'string' as const,
+    },
+  ]
+
+  for (const q of fieldRetrievalQuestions.slice(0, QUESTION_LIMITS.nestedConfig.fieldRetrieval)) {
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(q.prompt)
+        .groundTruth(q.groundTruth)
+        .type('field-retrieval')
+        .dataset('nested-config')
+        .answerType(q.answerType)
+        .build(),
+    )
+  }
+
+  // Aggregation: counts of nested structures
+  const roleCount = Object.keys(config.permissions.roles).length
+  const groupCount = Object.keys(config.permissions.groups).length
+  const providerCount = config.authentication.providers.length
+  const featureCount = Object.keys(config.features).length
+  const replicaCount = config.database.replicas.length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many roles are defined in permissions?')
+      .groundTruth(String(roleCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many groups are defined in permissions?')
+      .groundTruth(String(groupCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many authentication providers are configured?')
+      .groundTruth(String(providerCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many feature flags are defined?')
+      .groundTruth(String(featureCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many database replicas are configured?')
+      .groundTruth(String(replicaCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: providers with admin scope
+  const adminScopeProviderCount = config.authentication.providers.filter(p => p.scopes.includes('admin')).length
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many authentication providers include the "admin" scope?')
+      .groundTruth(String(adminScopeProviderCount))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: feature flag details
+  const enabledFeatures = Object.entries(config.features).filter(([_, f]) => f.enabled).length
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many feature flags are enabled?')
+      .groundTruth(String(enabledFeatures))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: role permissions
+  const adminPermissions = config.permissions.roles.admin?.permissions.length ?? 0
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many permissions does the admin role have?')
+      .groundTruth(String(adminPermissions))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Aggregation: additional nested counts
+  const totalPermissions = Object.values(config.permissions.roles).reduce((sum, role) => sum + role.permissions.length, 0)
+  const distinctPermissions = new Set(Object.values(config.permissions.roles).flatMap(r => r.permissions)).size
+  const totalVariants = Object.values(config.features).reduce((sum, f) => sum + f.variants.length, 0)
+  const highPriorityReplicas = config.database.replicas.filter(r => r.priority > 2).length
+  const featuresWithHighRollout = Object.values(config.features).filter(f => f.rollout > 50).length
+  const groupsWithMultipleRoles = Object.values(config.permissions.groups).filter(g => g.roles.length > 1).length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of permissions across all roles?')
+      .groundTruth(String(totalPermissions))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many distinct permissions are defined across all roles?')
+      .groundTruth(String(distinctPermissions))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total number of variants across all feature flags?')
+      .groundTruth(String(totalVariants))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many database replicas have a priority greater than 2?')
+      .groundTruth(String(highPriorityReplicas))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many feature flags have a rollout percentage greater than 50?')
+      .groundTruth(String(featuresWithHighRollout))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many groups have more than one role assigned?')
+      .groundTruth(String(groupsWithMultipleRoles))
+      .type('aggregation')
+      .dataset('nested-config')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Filtering: complex multi-condition queries
+  const filteringQuestions = [
+    {
+      prompt: 'How many feature flags are enabled with rollout greater than 50%?',
+      groundTruth: String(Object.entries(config.features)
+        .filter(([_, f]) => f.enabled && f.rollout > 50).length),
+    },
+    {
+      prompt: 'How many groups have the admin role?',
+      groundTruth: String(Object.entries(config.permissions.groups)
+        .filter(([_, g]) => g.roles.includes('admin')).length),
+    },
+    {
+      prompt: 'How many database replicas have priority greater than 2 and port 5432?',
+      groundTruth: String(config.database.replicas
+        .filter(r => r.priority > 2 && r.port === 5432).length),
+    },
+    {
+      prompt: 'How many authentication providers have more than 2 scopes?',
+      groundTruth: String(config.authentication.providers
+        .filter(p => p.scopes.length > 2).length),
+    },
+    {
+      prompt: 'How many roles have at least 5 permissions?',
+      groundTruth: String(Object.values(config.permissions.roles)
+        .filter(r => r.permissions.length >= 5).length),
+    },
+    {
+      prompt: 'How many feature flags are disabled with rollout less than 25%?',
+      groundTruth: String(Object.values(config.features)
+        .filter(f => !f.enabled && f.rollout < 25).length),
+    },
+    {
+      prompt: 'How many enabled features have at least 2 variants?',
+      groundTruth: String(Object.values(config.features)
+        .filter(f => f.enabled && f.variants.length >= 2).length),
+    },
+  ]
+
+  for (const q of filteringQuestions.slice(0, QUESTION_LIMITS.nestedConfig.filteringComplex)) {
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(q.prompt)
+        .groundTruth(q.groundTruth)
+        .type('filtering')
+        .dataset('nested-config')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/nested.ts

@@ -0,0 +1,215 @@
+import type { Order } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
+
+/**
+ * Generate nested (orders) questions
+ */
+export function generateNestedQuestions(orders: Order[], getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  // Field retrieval: order totals and statuses
+  const orderFieldGenerators: Array<(order: Order, getId: () => string) => Question> = [
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the total for order ${order.orderId}?`)
+      .groundTruth(String(order.total))
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the status of order ${order.orderId}?`)
+      .groundTruth(order.status)
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('string')
+      .build(),
+  ]
+
+  questions.push(...rotateQuestions(
+    orders,
+    orderFieldGenerators,
+    QUESTION_LIMITS.nested.fieldRetrievalOrders,
+    SAMPLE_STRIDES.ORDER_FIELD,
+    getId,
+  ))
+
+  // Field retrieval: customer info and order dates
+  const customerFieldGenerators: Array<(order: Order, getId: () => string) => Question> = [
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the customer name for order ${order.orderId}?`)
+      .groundTruth(order.customer.name)
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('string')
+      .build(),
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the customer email for order ${order.orderId}?`)
+      .groundTruth(order.customer.email)
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('string')
+      .build(),
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the order date for order ${order.orderId}?`)
+      .groundTruth(order.orderDate || '')
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('string')
+      .build(),
+    (order, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many items are in order ${order.orderId}?`)
+      .groundTruth(String(order.items.length))
+      .type('field-retrieval')
+      .dataset('nested')
+      .answerType('integer')
+      .build(),
+  ]
+
+  // Use stride + 1 for customer fields to offset from order fields
+  const customerOrders = orders.map((_, i) => orders[i * SAMPLE_STRIDES.CUSTOMER_FIELD + 1] || orders[i]).filter(Boolean) as Order[]
+  questions.push(...rotateQuestions(
+    customerOrders,
+    customerFieldGenerators,
+    QUESTION_LIMITS.nested.fieldRetrievalCustomers,
+    1,
+    getId,
+  ))
+
+  // Aggregation: totals and averages
+  const totalRevenue = orders.reduce((sum, o) => sum + o.total, 0)
+  const avgOrderValue = totalRevenue / orders.length
+  const totalOrders = orders.length
+  const maxOrderValue = Math.max(...orders.map(o => o.total))
+
+  // Count by status
+  const statuses = [...new Set(orders.map(o => o.status))]
+  for (const status of statuses.slice(0, QUESTION_LIMITS.nested.aggregationStatuses)) {
+    const count = orders.filter(o => o.status === status).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many orders have status "${status}"?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('nested')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the total revenue across all orders?')
+      .groundTruth(String(totalRevenue.toFixed(2)))
+      .type('aggregation')
+      .dataset('nested')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the average order value?')
+      .groundTruth(String(avgOrderValue.toFixed(2)))
+      .type('aggregation')
+      .dataset('nested')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many orders are in the dataset?')
+      .groundTruth(String(totalOrders))
+      .type('aggregation')
+      .dataset('nested')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the highest order total?')
+      .groundTruth(String(maxOrderValue.toFixed(2)))
+      .type('aggregation')
+      .dataset('nested')
+      .answerType('number')
+      .normalize({ decimalPlaces: 2 })
+      .build(),
+  )
+
+  // Aggregation: high-value orders (single-condition filter)
+  for (const threshold of QUESTION_THRESHOLDS.nested.highValueOrders) {
+    const count = orders.filter(o => o.total > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many orders have a total greater than ${threshold}?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('nested')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: multi-condition queries (status AND value)
+  const orderStatuses = [...new Set(orders.map(o => o.status))]
+  for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndValue)) {
+    const count = orders.filter(
+      o => o.status === status && o.total > QUESTION_THRESHOLDS.nested.statusValueThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many orders have status "${status}" and total greater than ${QUESTION_THRESHOLDS.nested.statusValueThreshold}?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('nested')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: status AND items count (multi-condition)
+  for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndItems)) {
+    const count = orders.filter(
+      o => o.status === status && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many orders have status "${status}" and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('nested')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: total AND items count (multi-condition)
+  for (const threshold of QUESTION_THRESHOLDS.nested.totalThresholdsForItems) {
+    const count = orders.filter(
+      o => o.total > threshold && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many orders have a total greater than ${threshold} and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('nested')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/structural-validation.ts

@@ -0,0 +1,44 @@
+import type { Question } from '../types.ts'
+import { QuestionBuilder } from './utils.ts'
+
+/**
+ * Generate structural validation questions for all incompleteness fixtures
+ *
+ * These questions test the ability to detect incomplete, truncated, or corrupted data
+ * by validating structural metadata (TOON's [N] length declarations and {fields} headers).
+ *
+ * @remarks
+ * - TOON's advantage: Explicit [N] and {fields} enable validation
+ * - CSV disadvantage: No structural metadata to validate against
+ * - JSON/YAML disadvantage: Require manual counting and schema inference
+ */
+export function generateStructuralValidationQuestions(
+  getId: () => string,
+): Question[] {
+  const questions: Question[] = []
+
+  // Dataset names and their expected validity
+  const validationFixtures = [
+    { dataset: 'structural-validation-control', isValid: true, description: 'Valid complete dataset (control)' },
+    { dataset: 'structural-validation-truncated', isValid: false, description: 'Array truncated: 3 rows removed from end' },
+    { dataset: 'structural-validation-extra-rows', isValid: false, description: 'Extra rows added beyond declared length' },
+    { dataset: 'structural-validation-width-mismatch', isValid: false, description: 'Inconsistent field count (missing salary in row 10)' },
+    { dataset: 'structural-validation-missing-fields', isValid: false, description: 'Missing required fields (no email in multiple rows)' },
+  ] as const
+
+  // Generate one validation question per fixture
+  for (const fixture of validationFixtures) {
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt('Is this data complete and valid? Answer only YES or NO.')
+        .groundTruth(fixture.isValid ? 'YES' : 'NO')
+        .type('structural-validation')
+        .dataset(fixture.dataset)
+        .answerType('boolean')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/structure.ts

@@ -0,0 +1,349 @@
+import type { AnalyticsMetric, Employee, EventLog, Order, Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QuestionBuilder } from './utils.ts'
+
+/**
+ * Generate structure-awareness questions across all datasets
+ *
+ * These questions test format-native structural affordances:
+ * - TOON's explicit array length [N] and field declarations {fields}
+ * - CSV's header row (but no explicit length)
+ * - JSON/YAML have neither unless the model counts manually
+ */
+export function generateStructureQuestions(
+  employees: Employee[],
+  orders: Order[],
+  metrics: AnalyticsMetric[],
+  repos: Repository[],
+  logs: EventLog[],
+  getId: () => string,
+): Question[] {
+  const questions: Question[] = []
+
+  // ========== TABULAR DATASET (Employees) ==========
+
+  // Count: Total employees (tests array length awareness)
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many employees are in the dataset?')
+      .groundTruth(String(employees.length))
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Field list: Employee fields (tests field name awareness)
+  const employeeFields = 'id,name,email,department,salary,yearsExperience,active'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('List the field names for employees (comma-separated, in order).')
+      .groundTruth(employeeFields)
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // Positional: Third field name for employees (tests TOON {fields} syntax)
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the 3rd field name for employees?')
+      .groundTruth('email')
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('string')
+      .build(),
+  )
+
+  // Last row: Last employee's department (tests ability to find last row using length)
+  const lastEmployee = employees.at(-1)!
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the department of the last employee in the dataset?')
+      .groundTruth(lastEmployee.department)
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('string')
+      .build(),
+  )
+
+  // Last row: Last employee's name
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the name of the last employee in the dataset?')
+      .groundTruth(lastEmployee.name)
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('string')
+      .build(),
+  )
+
+  // Field count: How many fields per employee (tests schema awareness)
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many fields does each employee record have?')
+      .groundTruth('7')
+      .type('structure-awareness')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+  )
+
+  // ========== NESTED DATASET (Orders) ==========
+
+  // Count: Total orders
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many orders are in the dataset?')
+      .groundTruth(String(orders.length))
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Field list: Order fields
+  const orderFields = 'orderId,customer,items,subtotal,tax,total,status,orderDate'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('List the top-level field names for orders (comma-separated, in order).')
+      .groundTruth(orderFields)
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // Nested count: Items in specific order
+  const orderWithManyItems = orders.reduce((max, order) =>
+    order.items.length > max.items.length ? order : max,
+  )
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many items are in order ${orderWithManyItems.orderId}?`)
+      .groundTruth(String(orderWithManyItems.items.length))
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Nested field list: Item fields
+  const itemFields = 'sku,name,quantity,price'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What are the field names for items within orders (comma-separated, in order)?')
+      .groundTruth(itemFields)
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // Last row: Last order's status
+  const lastOrder = orders.at(-1)!
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the status of the last order in the dataset?')
+      .groundTruth(lastOrder.status)
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('string')
+      .build(),
+  )
+
+  // Customer field list
+  const customerFields = 'id,name,email,phone'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What are the field names for customer objects within orders (comma-separated, in order)?')
+      .groundTruth(customerFields)
+      .type('structure-awareness')
+      .dataset('nested')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // ========== ANALYTICS DATASET (Metrics) ==========
+
+  // Count: Total metrics
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many metric records are in the dataset?')
+      .groundTruth(String(metrics.length))
+      .type('structure-awareness')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Field list: Metric fields
+  const metricFields = 'date,views,clicks,conversions,revenue,bounceRate'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('List the field names for metrics (comma-separated, in order).')
+      .groundTruth(metricFields)
+      .type('structure-awareness')
+      .dataset('analytics')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // Positional: Fifth field name for metrics (tests TOON {fields} syntax)
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the 5th field name for analytics metrics?')
+      .groundTruth('revenue')
+      .type('structure-awareness')
+      .dataset('analytics')
+      .answerType('string')
+      .build(),
+  )
+
+  // Last row: Last metric's date
+  const lastMetric = metrics.at(-1)!
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the date of the last metric record in the dataset?')
+      .groundTruth(lastMetric.date)
+      .type('structure-awareness')
+      .dataset('analytics')
+      .answerType('string')
+      .build(),
+  )
+
+  // Field count: How many fields per metric
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many fields does each metric record have?')
+      .groundTruth('6')
+      .type('structure-awareness')
+      .dataset('analytics')
+      .answerType('integer')
+      .build(),
+  )
+
+  // ========== GITHUB DATASET (Repositories) ==========
+
+  // Count: Total repositories
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many repositories are in the dataset?')
+      .groundTruth(String(repos.length))
+      .type('structure-awareness')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Field list: Repository fields
+  const repoFields = 'id,name,repo,description,stars,watchers,forks,defaultBranch,createdAt,updatedAt,pushedAt'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('List the field names for repositories (comma-separated, in order).')
+      .groundTruth(repoFields)
+      .type('structure-awareness')
+      .dataset('github')
+      .answerType('csv-list-ordered')
+      .build(),
+  )
+
+  // Positional: Seventh field name for repos (tests TOON {fields} syntax)
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the 7th field name for GitHub repositories?')
+      .groundTruth('forks')
+      .type('structure-awareness')
+      .dataset('github')
+      .answerType('string')
+      .build(),
+  )
+
+  // Last row: Last repo's name
+  const lastRepo = repos.at(-1)!
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the name of the last repository in the dataset?')
+      .groundTruth(lastRepo.name)
+      .type('structure-awareness')
+      .dataset('github')
+      .answerType('string')
+      .build(),
+  )
+
+  // Field count: How many fields per repository
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many fields does each repository record have?')
+      .groundTruth('11')
+      .type('structure-awareness')
+      .dataset('github')
+      .answerType('integer')
+      .build(),
+  )
+
+  // ========== EVENT LOGS DATASET ==========
+
+  // Count: Total logs
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many log entries are in the dataset?')
+      .groundTruth(String(logs.length))
+      .type('structure-awareness')
+      .dataset('event-logs')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Field list: Base log fields (including optional error)
+  const logFields = 'timestamp,level,endpoint,statusCode,responseTime,userId,error'
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('List the field names for log entries (comma-separated, any order, including optional fields).')
+      .groundTruth(logFields)
+      .type('structure-awareness')
+      .dataset('event-logs')
+      .answerType('csv-list-unordered')
+      .build(),
+  )
+
+  // Last row: Last log's level
+  const lastLog = logs.at(-1)!
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the level of the last log entry in the dataset?')
+      .groundTruth(lastLog.level)
+      .type('structure-awareness')
+      .dataset('event-logs')
+      .answerType('string')
+      .build(),
+  )
+
+  return questions
+}

benchmarks/src/questions/tabular.ts

@@ -0,0 +1,201 @@
+import type { Employee } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
+
+/**
+ * Generate tabular (employee) questions
+ */
+export function generateTabularQuestions(employees: Employee[], getId: () => string): Question[] {
+  const questions: Question[] = []
+
+  // Field retrieval: specific employees
+  const fieldGenerators: Array<(emp: Employee, getId: () => string) => Question> = [
+    (emp, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the salary of ${emp.name}?`)
+      .groundTruth(String(emp.salary))
+      .type('field-retrieval')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+    (emp, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What department does ${emp.name} work in?`)
+      .groundTruth(emp.department)
+      .type('field-retrieval')
+      .dataset('tabular')
+      .answerType('string')
+      .build(),
+    (emp, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`What is the email address of ${emp.name}?`)
+      .groundTruth(emp.email)
+      .type('field-retrieval')
+      .dataset('tabular')
+      .answerType('string')
+      .build(),
+    (emp, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`How many years of experience does ${emp.name} have?`)
+      .groundTruth(String(emp.yearsExperience))
+      .type('field-retrieval')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+    (emp, getId) => new QuestionBuilder()
+      .id(getId())
+      .prompt(`Is ${emp.name} an active employee?`)
+      .groundTruth(emp.active ? 'yes' : 'no')
+      .type('field-retrieval')
+      .dataset('tabular')
+      .answerType('boolean')
+      .build(),
+  ]
+
+  questions.push(...rotateQuestions(
+    employees,
+    fieldGenerators,
+    QUESTION_LIMITS.tabular.fieldRetrieval,
+    SAMPLE_STRIDES.EMPLOYEE_FIELD,
+    getId,
+  ))
+
+  // Aggregation: count by department
+  const departments = [...new Set(employees.map(e => e.department))]
+  for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.aggregationDepartments)) {
+    const count = employees.filter(e => e.department === dept).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many employees work in ${dept}?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: salary ranges (single-condition filters)
+  for (const threshold of QUESTION_THRESHOLDS.tabular.salaryRanges) {
+    const count = employees.filter(e => e.salary > threshold).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many employees have a salary greater than ${threshold}?`)
+        .groundTruth(String(count))
+        .type('aggregation')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Aggregation: totals and averages
+  const totalEmployees = employees.length
+  const avgSalary = Math.round(employees.reduce((sum, e) => sum + e.salary, 0) / totalEmployees)
+  const activeCount = employees.filter(e => e.active).length
+  const inactiveCount = employees.filter(e => !e.active).length
+
+  questions.push(
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many employees are in the dataset?')
+      .groundTruth(String(totalEmployees))
+      .type('aggregation')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('What is the average salary across all employees?')
+      .groundTruth(String(avgSalary))
+      .type('aggregation')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many employees are active?')
+      .groundTruth(String(activeCount))
+      .type('aggregation')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+    new QuestionBuilder()
+      .id(getId())
+      .prompt('How many employees are inactive?')
+      .groundTruth(String(inactiveCount))
+      .type('aggregation')
+      .dataset('tabular')
+      .answerType('integer')
+      .build(),
+  )
+
+  // Filtering: count by department with salary filter (multi-condition)
+  for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringMultiConditionDepartments)) {
+    const count = employees.filter(
+      e => e.department === dept && e.salary > QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many employees in ${dept} have a salary greater than ${QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold}?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: active employees by experience (multi-condition)
+  for (const exp of QUESTION_THRESHOLDS.tabular.experienceYears.slice(0, QUESTION_LIMITS.tabular.filteringExperience)) {
+    const count = employees.filter(e => e.yearsExperience > exp && e.active).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many active employees have more than ${exp} years of experience?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: department by experience (multi-condition)
+  for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentExp)) {
+    const count = employees.filter(
+      e => e.department === dept && e.yearsExperience > QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold,
+    ).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many employees in ${dept} have more than ${QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold} years of experience?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  // Filtering: department by active status (multi-condition)
+  for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentActive)) {
+    const count = employees.filter(e => e.department === dept && e.active).length
+    questions.push(
+      new QuestionBuilder()
+        .id(getId())
+        .prompt(`How many active employees work in ${dept}?`)
+        .groundTruth(String(count))
+        .type('filtering')
+        .dataset('tabular')
+        .answerType('integer')
+        .build(),
+    )
+  }
+
+  return questions
+}

benchmarks/src/questions/utils.ts

@@ -0,0 +1,100 @@
+import type { AnswerType, NormalizationOptions } from '../normalize.ts'
+import type { Question } from '../types.ts'
+
+// Constants for sampling strides
+export const SAMPLE_STRIDES = {
+  EMPLOYEE_FIELD: 2,
+  ORDER_FIELD: 2,
+  CUSTOMER_FIELD: 2,
+  ANALYTICS_FIELD: 3,
+  METRIC_FIELD: 3,
+  REPO_FIELD: 7,
+  EVENT_LOG_FIELD: 5,
+} as const
+
+/**
+ * ID Generator
+ */
+export function* createIdGenerator(): Generator<string, never, never> {
+  let id = 1
+  while (true) {
+    yield `q${id++}`
+  }
+}
+
+/**
+ * Question Builder class for fluent question creation
+ */
+export class QuestionBuilder {
+  private question: Partial<Question> = {}
+
+  id(id: string): this {
+    this.question.id = id
+    return this
+  }
+
+  prompt(prompt: string): this {
+    this.question.prompt = prompt
+    return this
+  }
+
+  groundTruth(groundTruth: string): this {
+    this.question.groundTruth = groundTruth
+    return this
+  }
+
+  type(type: Question['type']): this {
+    this.question.type = type
+    return this
+  }
+
+  dataset(dataset: Question['dataset']): this {
+    this.question.dataset = dataset
+    return this
+  }
+
+  answerType(kind: AnswerType): this {
+    this.question.answerType = kind
+    return this
+  }
+
+  normalize(options: Partial<NormalizationOptions>): this {
+    this.question.normalizationOptions = options
+    return this
+  }
+
+  build(): Question {
+    if (!this.question.id || !this.question.prompt || !this.question.groundTruth || !this.question.type || !this.question.dataset) {
+      throw new Error('Incomplete question')
+    }
+
+    return this.question as Question
+  }
+}
+
+/**
+ * Rotate through question generators
+ */
+export function rotateQuestions<T>(
+  items: T[],
+  generators: ((item: T, getId: () => string) => Question)[],
+  limit: number,
+  stride: number,
+  getId: () => string,
+): Question[] {
+  const questions: Question[] = []
+
+  for (let i = 0; i < Math.min(limit, items.length); i++) {
+    const item = items[i * stride] || items[i]
+    if (!item)
+      continue
+
+    const generatorIndex = i % generators.length
+    const generator = generators[generatorIndex]
+    if (generator) {
+      questions.push(generator(item, getId))
+    }
+  }
+
+  return questions
+}

benchmarks/src/report.ts

@@ -1,10 +1,41 @@
-import type { EvaluationResult, FormatResult, Question } from './types'
-import * as fsp from 'node:fs/promises'
-import * as path from 'node:path'
-import { BENCHMARKS_DIR } from './constants'
-import { datasets } from './datasets'
-import { models } from './evaluate'
-import { createProgressBar, ensureDir, tokenize } from './utils'
+import type { Dataset, EfficiencyRanking, EvaluationResult, FormatResult, Question } from './types.ts'
+import { FORMATTER_DISPLAY_NAMES, QUESTION_TYPE_LABELS, QUESTION_TYPES } from './constants.ts'
+import { ACCURACY_DATASETS } from './datasets.ts'
+import { models, PRIMERS } from './evaluate.ts'
+import { supportsCSV } from './formatters.ts'
+import { generateQuestions } from './questions/index.ts'
+import { createProgressBar, tokenize } from './utils.ts'
+
+const EFFICIENCY_CHART_STYLE: 'vertical' | 'horizontal' = 'horizontal'
+
+/**
+ * Calculate token counts for all format+dataset combinations
+ *
+ * @remarks
+ * Includes primer tokens for fairer comparison across formats
+ */
+export function calculateTokenCounts(
+  formatters: Record<string, (data: unknown) => string>,
+): Record<string, number> {
+  const tokenCounts: Record<string, number> = {}
+
+  for (const [formatName, formatter] of Object.entries(formatters)) {
+    for (const dataset of ACCURACY_DATASETS) {
+      // Skip CSV for datasets that don't support it
+      if (formatName === 'csv' && !supportsCSV(dataset))
+        continue
+
+      const formattedData = formatter(dataset.data)
+      const primer = PRIMERS[formatName] ?? ''
+      // Include primer in token count for fair comparison
+      const fullPrompt = primer ? `${primer}\n\n${formattedData}` : formattedData
+      const key = `${formatName}-${dataset.name}`
+      tokenCounts[key] = tokenize(fullPrompt)
+    }
+  }
+
+  return tokenCounts
+}
 
 /**
  * Calculate per-format statistics from evaluation results
@@ -22,9 +53,9 @@ export function calculateFormatResults(
     const accuracy = correctCount / totalCount
 
     // Calculate average tokens across all datasets for this format
-    const avgTokens = Object.entries(tokenCounts)
+    const formatTokenEntries = Object.entries(tokenCounts)
       .filter(([key]) => key.startsWith(`${formatName}-`))
-      .reduce((sum, [, tokens]) => sum + tokens, 0) / datasets.length
+    const avgTokens = formatTokenEntries.reduce((sum, [, tokens]) => sum + tokens, 0) / formatTokenEntries.length
 
     const averageLatency = formatResults.reduce((sum, r) => sum + r.latencyMs, 0) / totalCount
 
@@ -40,21 +71,297 @@ export function calculateFormatResults(
 }
 
 /**
- * Generate embeddable markdown report from results
+ * Generate consolidated retrieval accuracy report
  */
-export function generateMarkdownReport(
+export function generateAccuracyReport(
+  results: EvaluationResult[],
+  formatResults: FormatResult[],
+  tokenCounts: Record<string, number>,
+): string {
+  const questions = generateQuestions()
+  const totalQuestions = [...new Set(results.map(r => r.questionId))].length
+  const modelIds = models.map(m => m.modelId)
+  const modelNames = modelIds.filter(id => results.some(r => r.model === id))
+
+  return `
+Benchmarks test LLM comprehension across different input formats using ${totalQuestions} data retrieval questions on ${modelNames.length} ${modelNames.length === 1 ? 'model' : 'models'}.
+
+<details>
+<summary><strong>Show Dataset Catalog</strong></summary>
+
+${generateDatasetCatalog(ACCURACY_DATASETS)}
+
+</details>
+
+#### Efficiency Ranking (Accuracy per 1K Tokens)
+
+${generateEfficiencyRankingReport(formatResults, totalQuestions, modelNames.length)}
+
+#### Per-Model Accuracy
+
+${generateDetailedAccuracyReport(formatResults, results, questions, tokenCounts)}
+`.trimStart()
+}
+
+/**
+ * Generate dataset catalog section
+ */
+function generateDatasetCatalog(datasets: Dataset[]): string {
+  const rows = datasets.map((dataset) => {
+    const csvSupport = supportsCSV(dataset) ? '✓' : '✗'
+    const rowCount = Object.values(dataset.data)[0]?.length ?? 1
+    const structure = dataset.metadata.structureClass
+    const eligibility = `${dataset.metadata.tabularEligibility}%`
+
+    return `| ${dataset.description} | ${rowCount} | ${structure} | ${csvSupport} | ${eligibility} |`
+  }).join('\n')
+
+  return `
+#### Dataset Catalog
+
+| Dataset | Rows | Structure | CSV Support | Eligibility |
+| ------- | ---- | --------- | ----------- | ----------- |
+${rows}
+
+**Structure classes:**
+- **uniform**: All objects have identical fields with primitive values
+- **semi-uniform**: Mix of uniform and non-uniform structures
+- **nested**: Objects with nested structures (nested objects or arrays)
+- **deep**: Highly nested with minimal tabular eligibility
+
+**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
+
+**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
+`.trim()
+}
+
+/**
+ * Generate efficiency ranking report
+ */
+function generateEfficiencyRankingReport(
+  formatResults: FormatResult[],
+  totalQuestions: number,
+  modelCount: number,
+): string {
+  const toon = formatResults.find(r => r.format === 'toon')
+  const json = formatResults.find(r => r.format === 'json-pretty')
+  const csv = formatResults.find(r => r.format === 'csv')
+
+  // Build efficiency ranking (accuracy per 1k tokens)
+  const efficiencyRanking = formatResults
+    // Exclude CSV since it only supports a subset of datasets (~half the questions)
+    .filter(fr => fr.format !== 'csv')
+    .map((fr) => {
+      const efficiency = (fr.accuracy * 100) / (fr.totalTokens / 1000)
+      return {
+        format: fr.format,
+        efficiency,
+        accuracy: fr.accuracy,
+        tokens: fr.totalTokens,
+      }
+    })
+    .sort((a, b) => b.efficiency - a.efficiency)
+
+  const efficiencyChart = EFFICIENCY_CHART_STYLE === 'vertical'
+    ? generateVerticalEfficiencyChart(efficiencyRanking)
+    : generateHorizontalEfficiencyChart(efficiencyRanking)
+
+  // Build summary text
+  let summary = ''
+  if (toon && json) {
+    const toonVsJson = `**${(toon.accuracy * 100).toFixed(1)}%** accuracy (vs JSON's ${(json.accuracy * 100).toFixed(1)}%)`
+    const tokenSavings = `**${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens**`
+    summary = `TOON achieves ${toonVsJson} while using ${tokenSavings}.`
+  }
+
+  // Add CSV note if available
+  let csvNote = ''
+  if (csv) {
+    // CSV totalCount is evaluations (questions × models), so divide by number of models to get question count
+    const csvQuestionCount = csv.totalCount / modelCount
+    csvNote = `**Note on CSV:** Excluded from ranking as it only supports ${csvQuestionCount} of ${totalQuestions} questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.`
+  }
+
+  return `
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
+
+\`\`\`
+${efficiencyChart}
+\`\`\`
+
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> ${summary}
+
+${csvNote}
+`.trim()
+}
+
+/**
+ * Generate detailed accuracy report with breakdowns and methodology
+ */
+function generateDetailedAccuracyReport(
   formatResults: FormatResult[],
   results: EvaluationResult[],
   questions: Question[],
   tokenCounts: Record<string, number>,
 ): string {
   const toon = formatResults.find(r => r.format === 'toon')
-  const json = formatResults.find(r => r.format === 'json')
+  const json = formatResults.find(r => r.format === 'json-pretty')
 
   const modelIds = models.map(m => m.modelId)
   const modelNames = modelIds.filter(id => results.some(r => r.model === id))
 
-  const modelBreakdown = modelNames.map((modelName, i) => {
+  // Generate model breakdown section
+  const modelBreakdown = generateModelBreakdown(formatResults, results, modelNames)
+
+  // Generate summary comparison
+  const summaryComparison = generateSummaryComparison(toon, json)
+
+  // Generate performance by dataset
+  const datasetBreakdown = generateDatasetBreakdown(formatResults, results, questions, tokenCounts)
+
+  // Generate performance by model
+  const modelPerformance = generateModelPerformanceTable(formatResults, results, modelNames)
+
+  // Generate question type breakdown
+  const questionTypeBreakdown = generateQuestionTypeBreakdown(formatResults, results, questions)
+  const totalQuestions = [...new Set(results.map(r => r.questionId))].length
+
+  // Calculate question type distribution
+  const fieldRetrievalCount = questions.filter(q => q.type === 'field-retrieval').length
+  const aggregationCount = questions.filter(q => q.type === 'aggregation').length
+  const filteringCount = questions.filter(q => q.type === 'filtering').length
+  const structureAwarenessCount = questions.filter(q => q.type === 'structure-awareness').length
+  const structuralValidationCount = questions.filter(q => q.type === 'structural-validation').length
+
+  const fieldRetrievalPercent = ((fieldRetrievalCount / totalQuestions) * 100).toFixed(0)
+  const aggregationPercent = ((aggregationCount / totalQuestions) * 100).toFixed(0)
+  const filteringPercent = ((filteringCount / totalQuestions) * 100).toFixed(0)
+  const structureAwarenessPercent = ((structureAwarenessCount / totalQuestions) * 100).toFixed(0)
+  const structuralValidationPercent = ((structuralValidationCount / totalQuestions) * 100).toFixed(0)
+
+  // Calculate dataset sizes
+  const tabularSize = ACCURACY_DATASETS.find(d => d.name === 'tabular')?.data.employees?.length || 0
+  const nestedSize = ACCURACY_DATASETS.find(d => d.name === 'nested')?.data.orders?.length || 0
+  const analyticsSize = ACCURACY_DATASETS.find(d => d.name === 'analytics')?.data.metrics?.length || 0
+  const githubSize = ACCURACY_DATASETS.find(d => d.name === 'github')?.data.repositories?.length || 0
+  const eventLogsSize = ACCURACY_DATASETS.find(d => d.name === 'event-logs')?.data.logs?.length || 0
+  const nestedConfigSize = 1 // Single config object
+
+  // Calculate number of formats and evaluations
+  const formatCount = formatResults.length
+  const totalEvaluations = totalQuestions * formatCount * modelNames.length
+
+  return `
+Accuracy across ${modelNames.length} ${modelNames.length === 1 ? 'LLM' : 'LLMs'} on ${totalQuestions} data retrieval questions:
+
+\`\`\`
+${modelBreakdown}
+\`\`\`
+
+${summaryComparison}
+
+<details>
+<summary><strong>Performance by dataset, model, and question type</strong></summary>
+
+#### Performance by Question Type
+
+${questionTypeBreakdown}
+
+#### Performance by Dataset
+
+${datasetBreakdown}
+
+#### Performance by Model
+
+${modelPerformance}
+
+</details>
+
+#### What's Being Measured
+
+This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
+
+#### Datasets Tested
+
+Eleven datasets designed to test different structural patterns and validation capabilities:
+
+**Primary datasets:**
+
+1. **Tabular** (${tabularSize} employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
+2. **Nested** (${nestedSize} e-commerce orders): Complex structures with nested customer objects and item arrays.
+3. **Analytics** (${analyticsSize} days of metrics): Time-series data with dates and numeric values.
+4. **GitHub** (${githubSize} repositories): Real-world data from top GitHub repos by stars.
+5. **Event Logs** (${eventLogsSize} logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
+6. **Nested Config** (${nestedConfigSize} configuration): Deeply nested configuration with minimal tabular eligibility.
+
+**Structural validation datasets:**
+
+7. **Control**: Valid complete dataset (baseline for validation)
+8. **Truncated**: Array with 3 rows removed from end (tests \`[N]\` length detection)
+9. **Extra rows**: Array with 3 additional rows beyond declared length
+10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
+11. **Missing fields**: Systematic field omissions (no email in multiple rows)
+
+#### Question Types
+
+${totalQuestions} questions are generated dynamically across five categories:
+
+- **Field retrieval (${fieldRetrievalPercent}%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
+  - Example: "What is Alice's salary?" → \`75000\`
+  - Example: "How many items are in order ORD-0042?" → \`3\`
+  - Example: "What is the customer name for order ORD-0042?" → \`John Doe\`
+
+- **Aggregation (${aggregationPercent}%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
+  - Example: "How many employees work in Engineering?" → \`17\`
+  - Example: "What is the total revenue across all orders?" → \`45123.50\`
+  - Example: "How many employees have salary > 80000?" → \`23\`
+
+- **Filtering (${filteringPercent}%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
+  - Example: "How many employees in Sales have salary > 80000?" → \`5\`
+  - Example: "How many active employees have more than 10 years of experience?" → \`8\`
+
+- **Structure awareness (${structureAwarenessPercent}%)**: Tests format-native structural affordances (TOON's \`[N]\` count and \`{fields}\`, CSV's header row)
+  - Example: "How many employees are in the dataset?" → \`100\`
+  - Example: "List the field names for employees" → \`id, name, email, department, salary, yearsExperience, active\`
+  - Example: "What is the department of the last employee?" → \`Sales\`
+
+- **Structural validation (${structuralValidationPercent}%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
+  - Example: "Is this data complete and valid?" → \`YES\` (control dataset) or \`NO\` (corrupted datasets)
+  - Tests TOON's \`[N]\` length validation and \`{fields}\` consistency checking
+  - Demonstrates CSV's lack of structural validation capabilities
+
+#### Evaluation Process
+
+1. **Format conversion**: Each dataset is converted to all ${formatCount} formats (${formatResults.map(f => FORMATTER_DISPLAY_NAMES[f.format] || f.format).join(', ')}).
+2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
+3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., \`50000\` = \`$50,000\`, \`Engineering\` = \`engineering\`, \`2025-01-01\` = \`January 1, 2025\`) without requiring an LLM judge.
+
+#### Models & Configuration
+
+- **Models tested**: ${modelNames.map(m => `\`${m}\``).join(', ')}
+- **Token counting**: Using \`gpt-tokenizer\` with \`o200k_base\` encoding (GPT-5 tokenizer)
+- **Temperature**: Not set (models use their defaults)
+- **Total evaluations**: ${totalQuestions} questions × ${formatCount} formats × ${modelNames.length} models = ${totalEvaluations.toLocaleString('en-US')} LLM calls
+`.trim()
+}
+
+/**
+ * Generate ASCII bar chart showing per-model accuracy across formats
+ */
+function generateModelBreakdown(
+  formatResults: FormatResult[],
+  results: EvaluationResult[],
+  modelNames: string[],
+): string {
+  const maxDisplayNameWidth = Math.max(
+    ...Object.values(FORMATTER_DISPLAY_NAMES).map(name => name.length),
+  )
+  const progressBarWidth = 20
+
+  return modelNames.map((modelName, i) => {
     const modelResults = formatResults.map((fr) => {
       const modelFormatResults = results.filter(r => r.model === modelName && r.format === fr.format)
       const correctCount = modelFormatResults.filter(r => r.isCorrect).length
@@ -70,26 +377,50 @@ export function generateMarkdownReport(
     }).sort((a, b) => b.accuracy - a.accuracy)
 
     const formatLines = modelResults.map((result) => {
-      const bar = createProgressBar(result.accuracy, 1, 20)
-      const accuracyStr = `${(result.accuracy * 100).toFixed(1)}%`.padStart(6)
-      const countStr = `(${result.correctCount}/${result.totalCount})`
+      const bar = createProgressBar(result.accuracy, 1, progressBarWidth)
+      const accuracyString = `${(result.accuracy * 100).toFixed(1)}%`.padStart(6)
+      const countString = `(${result.correctCount}/${result.totalCount})`
       const prefix = result.format === 'toon' ? '→ ' : '  '
-      return `${prefix}${result.format.padEnd(12)} ${bar} ${accuracyStr} ${countStr}`
+      const displayName = FORMATTER_DISPLAY_NAMES[result.format] || result.format
+      return `${prefix}${displayName.padEnd(maxDisplayNameWidth)}   ${bar}   ${accuracyString} ${countString}`
     }).join('\n')
 
     // Add blank line before model name, except for first model
     return `${i > 0 ? '\n' : ''}${modelName}\n${formatLines}`
   }).join('\n')
+}
 
-  // Build summary comparison
-  const summaryComparison = toon && json
-    ? `**Key tradeoff:** TOON achieves **${(toon.accuracy * 100).toFixed(1)}% accuracy** (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using **${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens** on these datasets.`
-    : ''
+/**
+ * Generate summary comparison between TOON and JSON formats
+ */
+function generateSummaryComparison(
+  toon: FormatResult | undefined,
+  json: FormatResult | undefined,
+): string {
+  if (!toon || !json)
+    return ''
 
-  // Build performance by dataset
-  const datasetBreakdown = datasets.map((dataset) => {
+  return `
+> [!TIP]
+> TOON achieves **${(toon.accuracy * 100).toFixed(1)}% accuracy** (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using **${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens** on these datasets.
+`.trim()
+}
+
+/**
+ * Generate per-dataset performance breakdown tables
+ */
+function generateDatasetBreakdown(
+  formatResults: FormatResult[],
+  results: EvaluationResult[],
+  questions: Question[],
+  tokenCounts: Record<string, number>,
+): string {
+  // Build question ID to dataset mapping for O(1) lookups
+  const questionDatasetMap = new Map(questions.map(q => [q.id, q.dataset]))
+
+  return ACCURACY_DATASETS.map((dataset) => {
     const datasetResults = formatResults.map((fr) => {
-      const datasetFormatResults = results.filter(r => r.questionId.includes(dataset.name) || questions.find(q => q.id === r.questionId)?.dataset === dataset.name)
+      const datasetFormatResults = results.filter(r => questionDatasetMap.get(r.questionId) === dataset.name)
       if (datasetFormatResults.length === 0)
         return undefined
 
@@ -136,9 +467,59 @@ export function generateMarkdownReport(
 ${tableRows}
 `.trimStart()
   }).filter(Boolean).join('\n').trim()
+}
 
-  // Build performance by model
-  const modelPerformance = modelNames.map((modelName) => {
+/**
+ * Generate question type breakdown table
+ */
+function generateQuestionTypeBreakdown(
+  formatResults: FormatResult[],
+  results: EvaluationResult[],
+  questions: Question[],
+): string {
+  // Build header
+  const formatNames = formatResults.map(fr => FORMATTER_DISPLAY_NAMES[fr.format] || fr.format)
+  const header = `| Question Type | ${formatNames.join(' | ')} |`
+  const separator = `| ------------- | ${formatNames.map(() => '----').join(' | ')} |`
+
+  // Build rows
+  const rows = QUESTION_TYPES.map((type) => {
+    const questionIds = questions.filter(q => q.type === type).map(q => q.id)
+    const typeResults = results.filter(r => questionIds.includes(r.questionId))
+
+    if (typeResults.length === 0)
+      return undefined
+
+    const accuracies = formatResults.map((fr) => {
+      const formatTypeResults = typeResults.filter(r => r.format === fr.format)
+      if (formatTypeResults.length === 0)
+        return 'N/A'
+
+      const correctCount = formatTypeResults.filter(r => r.isCorrect).length
+      const totalCount = formatTypeResults.length
+      const accuracy = totalCount > 0 ? correctCount / totalCount : 0
+      return `${(accuracy * 100).toFixed(1)}%`
+    })
+
+    return `| ${QUESTION_TYPE_LABELS[type]} | ${accuracies.join(' | ')} |`
+  }).filter(Boolean)
+
+  return `
+${header}
+${separator}
+${rows.join('\n')}
+`.trim()
+}
+
+/**
+ * Generate per-model performance comparison tables
+ */
+function generateModelPerformanceTable(
+  formatResults: FormatResult[],
+  results: EvaluationResult[],
+  modelNames: string[],
+): string {
+  return modelNames.map((modelName) => {
     const modelResults = formatResults.map((fr) => {
       const modelFormatResults = results.filter(r => r.model === modelName && r.format === fr.format)
       const correctCount = modelFormatResults.filter(r => r.isCorrect).length
@@ -165,142 +546,96 @@ ${tableRows}
 ${tableRows}
 `.trimStart()
   }).join('\n').trim()
-
-  // Calculate total unique questions
-  const totalQuestions = [...new Set(results.map(r => r.questionId))].length
-
-  // Calculate question type distribution
-  const fieldRetrievalCount = questions.filter(q => q.type === 'field-retrieval').length
-  const aggregationCount = questions.filter(q => q.type === 'aggregation').length
-  const filteringCount = questions.filter(q => q.type === 'filtering').length
-
-  const fieldRetrievalPercent = ((fieldRetrievalCount / totalQuestions) * 100).toFixed(0)
-  const aggregationPercent = ((aggregationCount / totalQuestions) * 100).toFixed(0)
-  const filteringPercent = ((filteringCount / totalQuestions) * 100).toFixed(0)
-
-  // Calculate dataset sizes
-  const tabularSize = datasets.find(d => d.name === 'tabular')?.data.employees?.length || 0
-  const nestedSize = datasets.find(d => d.name === 'nested')?.data.orders?.length || 0
-  const analyticsSize = datasets.find(d => d.name === 'analytics')?.data.metrics?.length || 0
-  const githubSize = datasets.find(d => d.name === 'github')?.data.repositories?.length || 0
-
-  // Calculate number of formats and evaluations
-  const formatCount = formatResults.length
-  const totalEvaluations = totalQuestions * formatCount * modelNames.length
-
-  return `
-### Retrieval Accuracy
-
-Accuracy across **${modelNames.length} ${modelNames.length === 1 ? 'LLM' : 'LLMs'}** on ${totalQuestions} data retrieval questions:
-
-\`\`\`
-${modelBreakdown}
-\`\`\`
-
-${summaryComparison}
-
-<details>
-<summary><strong>Performance by dataset and model</strong></summary>
-
-#### Performance by Dataset
-
-${datasetBreakdown}
-
-#### Performance by Model
-
-${modelPerformance}
-
-</details>
-
-<details>
-<summary><strong>How the benchmark works</strong></summary>
-
-#### What's Being Measured
-
-This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it (this does **not** test model's ability to generate TOON output).
-
-#### Datasets Tested
-
-Four datasets designed to test different structural patterns (all contain arrays of uniform objects, TOON's optimal format):
-
-1. **Tabular** (${tabularSize} employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
-2. **Nested** (${nestedSize} e-commerce orders): Complex structures with nested customer objects and item arrays.
-3. **Analytics** (${analyticsSize} days of metrics): Time-series data with dates and numeric values.
-4. **GitHub** (${githubSize} repositories): Real-world data from top GitHub repos by stars.
-
-#### Question Types
-
-${totalQuestions} questions are generated dynamically across three categories:
-
-\- **Field retrieval (${fieldRetrievalPercent}%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
-  - Example: "What is Alice's salary?" → \`75000\`
-  - Example: "How many items are in order ORD-0042?" → \`3\`
-  - Example: "What is the customer name for order ORD-0042?" → \`John Doe\`
-
-- **Aggregation (${aggregationPercent}%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
-  - Example: "How many employees work in Engineering?" → \`17\`
-  - Example: "What is the total revenue across all orders?" → \`45123.50\`
-  - Example: "How many employees have salary > 80000?" → \`23\`
-
-- **Filtering (${filteringPercent}%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
-  - Example: "How many employees in Sales have salary > 80000?" → \`5\`
-  - Example: "How many active employees have more than 10 years of experience?" → \`8\`
-
-#### Evaluation Process
-
-1. **Format conversion**: Each dataset is converted to all ${formatCount} formats (${formatResults.map(f => f.format.toUpperCase()).join(', ')}).
-2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
-3. **Validate with LLM-as-judge**: \`gpt-5-nano\` validates if the answer is semantically correct (e.g., \`50000\` = \`$50,000\`, \`Engineering\` = \`engineering\`, \`2025-01-01\` = \`January 1, 2025\`).
-
-#### Models & Configuration
-
-- **Models tested**: ${modelNames.map(m => `\`${m}\``).join(', ')}
-- **Token counting**: Using \`gpt-tokenizer\` with \`o200k_base\` encoding (GPT-5 tokenizer)
-- **Temperature**: Not set (models use their defaults)
-- **Total evaluations**: ${totalQuestions} questions × ${formatCount} formats × ${modelNames.length} models = ${totalEvaluations.toLocaleString('en-US')} LLM calls
-
-</details>
-`.trimStart()
 }
 
 /**
- * Calculate token counts for all format+dataset combinations
+ * Generate horizontal bar chart for efficiency ranking
  */
-export function calculateTokenCounts(
-  formatters: Record<string, (data: unknown) => string>,
-): Record<string, number> {
-  const tokenCounts: Record<string, number> = {}
+function generateHorizontalEfficiencyChart(
+  ranking: EfficiencyRanking[],
+): string {
+  const barWidth = 20
+  const maxEfficiency = Math.max(...ranking.map(r => r.efficiency))
+  const maxFormatWidth = Math.max(...ranking.map((r) => {
+    const displayName = FORMATTER_DISPLAY_NAMES[r.format] || r.format
+    return displayName.length
+  }))
 
-  for (const [formatName, formatter] of Object.entries(formatters)) {
-    for (const dataset of datasets) {
-      const formatted = formatter(dataset.data)
-      const key = `${formatName}-${dataset.name}`
-      tokenCounts[key] = tokenize(formatted)
-    }
+  return ranking
+    .map((r) => {
+      const normalizedValue = r.efficiency / maxEfficiency
+      const bar = createProgressBar(normalizedValue, 1, barWidth)
+      const displayName = FORMATTER_DISPLAY_NAMES[r.format] || r.format
+      const formatName = displayName.padEnd(maxFormatWidth)
+      const efficiency = r.efficiency.toFixed(1).padStart(4)
+      const accuracy = `${(r.accuracy * 100).toFixed(1)}%`.padStart(5)
+      const tokens = r.tokens.toLocaleString('en-US').padStart(5)
+
+      return `${formatName}   ${bar}   ${efficiency} acc%/1K tok  │  ${accuracy} acc  │  ${tokens} tokens`
+    })
+    .join('\n')
+}
+
+/**
+ * Generate vertical bar chart for efficiency ranking
+ */
+function generateVerticalEfficiencyChart(
+  ranking: EfficiencyRanking[],
+): string {
+  const maxEfficiency = Math.max(...ranking.map(r => r.efficiency))
+  const chartHeight = 8
+
+  // Generate rows from top to bottom
+  const rows: string[] = []
+
+  // Y-axis and bars
+  for (let i = chartHeight; i >= 0; i--) {
+    const threshold = (i / chartHeight) * maxEfficiency
+    const yLabel = i === chartHeight || i === Math.floor(chartHeight / 2) || i === 0
+      ? Math.round(threshold).toString().padStart(4)
+      : '    '
+
+    const bars = ranking
+      .map((r) => {
+        const barHeight = (r.efficiency / maxEfficiency) * chartHeight
+        let char = ' '
+        if (barHeight >= i) {
+          // Use different characters for visual distinction
+          if (ranking.indexOf(r) === 0)
+            char = '▓' // Top format
+          else if (ranking.indexOf(r) <= 2)
+            char = '▒' // Top 3
+          else
+            char = '░' // Rest
+        }
+        return char
+      })
+      .join('    ')
+
+    rows.push(`${yLabel}│  ${bars}`)
   }
 
-  return tokenCounts
-}
-
-/**
- * Save results to disk
- *
- * @remarks
- * Per-model results are managed separately via storage.ts
- * This function only generates the aggregated markdown report
- */
-export async function saveResults(
-  results: EvaluationResult[],
-  formatResults: FormatResult[],
-  questions: Question[],
-  tokenCounts: Record<string, number>,
-): Promise<string> {
-  const resultsDir = path.join(BENCHMARKS_DIR, 'results')
-  await ensureDir(resultsDir)
-
-  // Generate markdown report from all available model results
-  const report = generateMarkdownReport(formatResults, results, questions, tokenCounts)
-  await fsp.writeFile(path.join(resultsDir, 'retrieval-accuracy.md'), report)
-
-  return resultsDir
+  // X-axis
+  const axis = `    └──${ranking.map(() => '┴').join('────')}──`
+  rows.push(axis)
+
+  // Format labels (split long names into multiple rows)
+  const formatRow1 = ranking
+    .map((r) => {
+      const parts = r.format.split('-')
+      return (parts[0] || '').padEnd(5).substring(0, 5)
+    })
+    .join('')
+  rows.push(`      ${formatRow1}`)
+
+  const formatRow2 = ranking
+    .map((r) => {
+      const parts = r.format.split('-')
+      return (parts[1] || '').padEnd(5).substring(0, 5)
+    })
+    .join('')
+  if (formatRow2.trim())
+    rows.push(`      ${formatRow2}`)
+
+  return rows.join('\n')
 }

benchmarks/src/storage.ts

@@ -1,9 +1,9 @@
 import type { Storage, StorageValue } from 'unstorage'
-import type { EvaluationResult } from './types'
+import type { EvaluationResult } from './types.ts'
 import * as path from 'node:path'
 import { createStorage } from 'unstorage'
 import fsDriver from 'unstorage/drivers/fs'
-import { BENCHMARKS_DIR } from './constants'
+import { BENCHMARKS_DIR } from './constants.ts'
 
 /**
  * Storage instance for model results

benchmarks/src/types.ts

@@ -1,15 +1,38 @@
+import type { DATASET_NAMES, QUESTION_TYPES, STRUCTURE_CLASSES } from './constants.ts'
+import type { AnswerType, NormalizationOptions } from './normalize.ts'
+
+export type QuestionType = typeof QUESTION_TYPES[number]
+export type DatasetName = typeof DATASET_NAMES[number]
+export type StructureClass = typeof STRUCTURE_CLASSES[number]
+
+export interface DatasetMetadata {
+  supportsCSV: boolean
+  structureClass: StructureClass
+  tabularEligibility: number
+}
+
 export interface Dataset {
-  name: string
+  name: DatasetName
   description: string
   data: Record<string, any>
+  metadata: DatasetMetadata
 }
 
 export interface Question {
   id: string
   prompt: string
   groundTruth: string
-  type: 'field-retrieval' | 'aggregation' | 'filtering' | 'comparison'
-  dataset: string
+  type: QuestionType
+  dataset: DatasetName
+  /**
+   * Expected answer kind for deterministic comparison.
+   * @default 'string'
+   */
+  answerType?: AnswerType
+  /**
+   * Options for answer normalization and comparison.
+   */
+  normalizationOptions?: Partial<NormalizationOptions>
 }
 
 export interface EvaluationResult {
@@ -32,3 +55,10 @@ export interface FormatResult {
   correctCount: number
   totalCount: number
 }
+
+export interface EfficiencyRanking {
+  format: string
+  efficiency: number
+  accuracy: number
+  tokens: number
+}

benchmarks/src/utils.ts

@@ -7,16 +7,25 @@ import { encode } from 'gpt-tokenizer'
  * @param value - Current value
  * @param max - Maximum value
  * @param width - Width of the bar in characters (default: 25)
- * @returns ASCII progress bar string (`█` for filled, `░` for empty)
+ * @param chars - Characters to use for filled and empty sections
+ * @param chars.filled - Character for filled portion (default: '█')
+ * @param chars.empty - Character for empty portion (default: '░')
+ * @returns ASCII progress bar string
  *
  * @example
  * createProgressBar(75, 100, 20) // "███████████████░░░░░"
  * createProgressBar(0.5, 1, 10)  // "█████░░░░░"
+ * createProgressBar(0.75, 1, 20, { filled: '▓', empty: '░' }) // "▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░"
  */
-export function createProgressBar(value: number, max: number, width = 25): string {
+export function createProgressBar(
+  value: number,
+  max: number,
+  width = 25,
+  chars: { filled: string, empty: string } = { filled: '█', empty: '░' },
+): string {
   const filled = Math.round((value / max) * width)
   const empty = width - filled
-  return '█'.repeat(filled) + '░'.repeat(empty)
+  return chars.filled.repeat(filled) + chars.empty.repeat(empty)
 }
 
 /**
@@ -40,19 +49,3 @@ export function tokenize(text: string): number {
 export async function ensureDir(dirPath: string): Promise<void> {
   await fsp.mkdir(dirPath, { recursive: true })
 }
-
-/**
- * Save data as formatted JSON file
- *
- * @param filePath - Path to save the file
- * @param data - Data to serialize as JSON
- * @param indent - Indentation spaces (default: 2)
- */
-export async function saveJsonFile(
-  filePath: string,
-  data: unknown,
-  indent = 2,
-): Promise<void> {
-  const json = JSON.stringify(data, undefined, indent)
-  await fsp.writeFile(filePath, `${json}\n`, 'utf-8')
-}

commitlint.config.ts

@@ -0,0 +1,38 @@
+import type { Rule, UserConfig } from '@commitlint/types'
+import { RuleConfigSeverity } from '@commitlint/types'
+
+// #region Rules
+
+/**
+ * Rule to ensure the first letter of the commit subject is lowercase.
+ *
+ * @param parsed - Parsed commit object containing commit message parts.
+ * @returns A tuple where the first element is a boolean indicating
+ * if the rule passed, and the second is an optional error message.
+ */
+const subjectLowercaseFirst: Rule = async (parsed) => {
+  const firstChar = parsed.subject!.match(/[a-z]/i)?.[0]
+  if (firstChar && firstChar === firstChar.toUpperCase()) {
+    return [false, 'Subject must start with a lowercase letter']
+  }
+  return [true]
+}
+
+// #endregion
+
+const Configuration: UserConfig = {
+  extends: ['@commitlint/config-conventional'],
+  rules: {
+    'subject-case': [RuleConfigSeverity.Disabled],
+    'subject-lowercase-first': [RuleConfigSeverity.Error, 'always'],
+  },
+  plugins: [
+    {
+      rules: {
+        'subject-lowercase-first': subjectLowercaseFirst,
+      },
+    },
+  ],
+}
+
+export default Configuration

docs/.vitepress/config.ts

@@ -0,0 +1,145 @@
+import type { DefaultTheme } from 'vitepress'
+import UnoCSS from 'unocss/vite'
+import { defineConfig } from 'vitepress'
+import llmstxt, { copyOrDownloadAsMarkdownButtons } from 'vitepress-plugin-llms'
+import { description, github, name, ogImage, ogUrl, releases, twitterImage, version } from './meta'
+
+export default defineConfig({
+  title: name,
+  description,
+  head: [
+    ['link', { rel: 'icon', href: '/favicon.svg', type: 'image/svg+xml' }],
+    ['meta', { name: 'author', content: 'Johann Schopplich' }],
+    ['meta', { property: 'og:type', content: 'website' }],
+    ['meta', { property: 'og:url', content: ogUrl }],
+    ['meta', { property: 'og:title', content: name }],
+    ['meta', { property: 'og:description', content: description }],
+    ['meta', { property: 'og:image', content: ogImage }],
+    ['meta', { name: 'twitter:title', content: name }],
+    ['meta', { name: 'twitter:description', content: description }],
+    ['meta', { name: 'twitter:image', content: twitterImage }],
+    ['meta', { name: 'twitter:site', content: '@jschopplich' }],
+    ['meta', { name: 'twitter:creator', content: '@jschopplich' }],
+    ['meta', { name: 'twitter:card', content: 'summary_large_image' }],
+  ],
+
+  vite: {
+    // @ts-expect-error – UnoCSS types are not compatible with Vite yet
+    plugins: [UnoCSS(), llmstxt()],
+  },
+
+  themeConfig: {
+    logo: '/favicon.svg',
+
+    nav: [
+      {
+        text: 'Playground',
+        link: '/playground',
+      },
+      {
+        text: 'Guide',
+        activeMatch: '^/guide/',
+        items: [
+          { text: 'Getting Started', link: '/guide/getting-started' },
+          { text: 'Format Overview', link: '/guide/format-overview' },
+          { text: 'Using TOON with LLMs', link: '/guide/llm-prompts' },
+          { text: 'Benchmarks', link: '/guide/benchmarks' },
+        ],
+      },
+      {
+        text: 'CLI',
+        link: '/cli/',
+      },
+      {
+        text: 'Reference',
+        activeMatch: '^/reference/',
+        items: [
+          { text: 'API', link: '/reference/api' },
+          { text: 'Syntax Cheatsheet', link: '/reference/syntax-cheatsheet' },
+          { text: 'Specification', link: '/reference/spec' },
+          { text: 'Efficiency Formalization', link: '/reference/efficiency-formalization' },
+        ],
+      },
+      {
+        text: 'Ecosystem',
+        activeMatch: '^/ecosystem/',
+        items: [
+          { text: 'Tools & Playgrounds', link: '/ecosystem/tools-and-playgrounds' },
+          { text: 'Implementations', link: '/ecosystem/implementations' },
+        ],
+      },
+      {
+        text: `v${version}`,
+        items: [
+          {
+            text: 'Release Notes',
+            link: releases,
+          },
+        ],
+      },
+    ],
+
+    sidebar: {
+      '/guide/': sidebarPrimary(),
+      '/cli/': sidebarPrimary(),
+      '/reference/': sidebarPrimary(),
+      '/ecosystem/': sidebarPrimary(),
+    },
+
+    socialLinks: [
+      { icon: 'github', link: github },
+    ],
+
+    footer: {
+      message: 'Released under the <a href="https://opensource.org/licenses/MIT" target="_blank">MIT License</a>.',
+      copyright: 'Copyright © 2025-PRESENT <a href="https://johannschopplich.com" target="_blank">Johann Schopplich</a>',
+    },
+
+    search: {
+      provider: 'local',
+    },
+  },
+  markdown: {
+    config(md) {
+      md.use(copyOrDownloadAsMarkdownButtons)
+    },
+    math: true,
+  },
+})
+
+function sidebarPrimary(): DefaultTheme.SidebarItem[] {
+  return [
+    {
+      text: 'Guide',
+      items: [
+        { text: 'Getting Started', link: '/guide/getting-started' },
+        { text: 'Format Overview', link: '/guide/format-overview' },
+        { text: 'Using TOON with LLMs', link: '/guide/llm-prompts' },
+        { text: 'Benchmarks', link: '/guide/benchmarks' },
+      ],
+    },
+    {
+      text: 'Tooling',
+      items: [
+        { text: 'Playground', link: '/playground' },
+        { text: 'CLI Reference', link: '/cli/' },
+      ],
+    },
+    {
+      text: 'Ecosystem',
+      items: [
+        { text: 'Tools & Playgrounds', link: '/ecosystem/tools-and-playgrounds' },
+        { text: 'Implementations', link: '/ecosystem/implementations' },
+      ],
+    },
+    {
+      text: 'Reference',
+      items: [
+        { text: 'API (TypeScript)', link: '/reference/api' },
+        { text: 'Syntax Cheatsheet', link: '/reference/syntax-cheatsheet' },
+        { text: 'Specification', link: '/reference/spec' },
+        { text: 'Efficiency Formalization', link: '/reference/efficiency-formalization' },
+      ],
+    },
+  ]
+}

docs/.vitepress/meta.ts

@@ -0,0 +1,12 @@
+export { description, version } from '../../packages/toon/package.json'
+
+/* VitePress head */
+export const name = 'TOON'
+export const ogUrl = 'https://toonformat.dev/'
+export const ogImage = `${ogUrl}og.png`
+export const twitterImage = `${ogUrl}twitter.png`
+
+/* GitHub and social links */
+export const github = 'https://github.com/toon-format/toon'
+export const releases = 'https://github.com/toon-format/toon/releases'
+export const twitter = 'https://twitter.com/jschopplich'

docs/.vitepress/theme/components/PlaygroundLayout.vue

@@ -0,0 +1,764 @@
+<script setup lang="ts">
+import type { Delimiter, EncodeOptions } from '../../../../packages/toon/src'
+import { useClipboard, useDebounceFn } from '@vueuse/core'
+import { unzlibSync, zlibSync } from 'fflate'
+import { base64ToUint8Array, stringToUint8Array, uint8ArrayToBase64, uint8ArrayToString } from 'uint8array-extras'
+import { computed, onMounted, ref, shallowRef, watch } from 'vue'
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
+import { DEFAULT_DELIMITER, encode } from '../../../../packages/toon/src'
+import VPInput from './VPInput.vue'
+
+type InputFormat = 'json' | 'yaml'
+type JsonFormat = 'pretty-2' | 'pretty-4' | 'pretty-tab' | 'compact'
+
+interface PlaygroundState extends Required<Pick<EncodeOptions, 'delimiter' | 'indent' | 'keyFolding' | 'flattenDepth'>> {
+  input: string
+  inputFormat: InputFormat
+  jsonFormat: JsonFormat
+  /** Pre-YAML share URLs stored input under `json`. Read-only fallback. */
+  json?: string
+}
+
+function parseInput(text: string, format: InputFormat): unknown {
+  return format === 'yaml' ? parseYaml(text) : JSON.parse(text)
+}
+
+function stringifyInputYaml(value: unknown): string {
+  return stringifyYaml(value, { lineWidth: 0 })
+}
+
+const PRESETS = {
+  hikes: {
+    context: {
+      task: 'Our favorite hikes together',
+      location: 'Boulder',
+      season: 'spring_2025',
+    },
+    friends: ['ana', 'luis', 'sam'],
+    hikes: [
+      { id: 1, name: 'Blue Lake Trail', distanceKm: 7.5, elevationGain: 320, companion: 'ana', wasSunny: true },
+      { id: 2, name: 'Ridge Overlook', distanceKm: 9.2, elevationGain: 540, companion: 'luis', wasSunny: false },
+      { id: 3, name: 'Wildflower Loop', distanceKm: 5.1, elevationGain: 180, companion: 'sam', wasSunny: true },
+    ],
+  },
+  orders: {
+    orders: [
+      {
+        orderId: 'ORD-001',
+        customer: { name: 'Alice Chen', email: 'alice@example.com' },
+        items: [
+          { sku: 'WIDGET-A', quantity: 2, price: 29.99 },
+          { sku: 'GADGET-B', quantity: 1, price: 49.99 },
+        ],
+        total: 109.97,
+        status: 'shipped',
+      },
+      {
+        orderId: 'ORD-002',
+        customer: { name: 'Bob Smith', email: 'bob@example.com' },
+        items: [
+          { sku: 'THING-C', quantity: 3, price: 15.00 },
+        ],
+        total: 45.00,
+        status: 'delivered',
+      },
+    ],
+  },
+  metrics: {
+    metrics: [
+      { date: '2025-01-01', views: 5200, clicks: 180, conversions: 24, revenue: 2890.50 },
+      { date: '2025-01-02', views: 6100, clicks: 220, conversions: 31, revenue: 3450.00 },
+      { date: '2025-01-03', views: 4800, clicks: 165, conversions: 19, revenue: 2100.25 },
+      { date: '2025-01-04', views: 5900, clicks: 205, conversions: 28, revenue: 3200.00 },
+    ],
+  },
+  events: {
+    logs: [
+      { timestamp: '2025-01-15T10:23:45Z', level: 'info', endpoint: '/api/users', statusCode: 200, responseTime: 45 },
+      { timestamp: '2025-01-15T10:24:12Z', level: 'error', endpoint: '/api/orders', statusCode: 500, responseTime: 120, error: { message: 'Database timeout', retryable: true } },
+      { timestamp: '2025-01-15T10:25:03Z', level: 'info', endpoint: '/api/products', statusCode: 200, responseTime: 32 },
+      { timestamp: '2025-01-15T10:26:47Z', level: 'warn', endpoint: '/api/payment', statusCode: 429, responseTime: 5, error: { message: 'Rate limit exceeded', retryable: true } },
+    ],
+  },
+} as const
+const DELIMITER_OPTIONS: { value: Delimiter, label: string }[] = [
+  { value: ',', label: 'Comma (,)' },
+  { value: '\t', label: 'Tab (\\t)' },
+  { value: '|', label: 'Pipe (|)' },
+]
+const JSON_FORMAT_OPTIONS: { value: JsonFormat, label: string, indent: string | number | undefined }[] = [
+  { value: 'pretty-2', label: 'Pretty (2 spaces)', indent: 2 },
+  { value: 'pretty-4', label: 'Pretty (4 spaces)', indent: 4 },
+  { value: 'pretty-tab', label: 'Pretty (tabs)', indent: '\t' },
+  { value: 'compact', label: 'Compact', indent: undefined },
+]
+const DEFAULT_JSON = JSON.stringify(PRESETS.hikes, undefined, 2)
+const SHARE_URL_LIMIT = 8 * 1024
+
+// Input state
+const inputText = ref(DEFAULT_JSON)
+const inputFormat = ref<InputFormat>('json')
+const jsonFormat = ref<JsonFormat>('pretty-2')
+const currentFormatIndent = computed(() =>
+  JSON_FORMAT_OPTIONS.find(opt => opt.value === jsonFormat.value)?.indent,
+)
+const formattedInput = computed(() => {
+  try {
+    const data = parseInput(inputText.value, inputFormat.value)
+    return inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+  }
+  catch {
+    return inputText.value
+  }
+})
+
+// Encoder options
+const delimiter = ref<Delimiter>(DEFAULT_DELIMITER)
+const indent = ref(2)
+const keyFolding = ref<'off' | 'safe'>('safe')
+const flattenDepth = ref(2)
+
+// Encoding output
+const encodingResult = computed(() => {
+  try {
+    const parsedInput = parseInput(inputText.value, inputFormat.value)
+    return {
+      output: encode(parsedInput, {
+        indent: indent.value,
+        delimiter: delimiter.value,
+        keyFolding: keyFolding.value,
+        flattenDepth: flattenDepth.value,
+      }),
+      error: undefined,
+    }
+  }
+  catch (error) {
+    const fallback = inputFormat.value === 'yaml' ? 'Invalid YAML' : 'Invalid JSON'
+    return {
+      output: '',
+      error: error instanceof Error ? error.message : fallback,
+    }
+  }
+})
+const toonOutput = computed(() => encodingResult.value.output)
+const error = computed(() => encodingResult.value.error)
+
+// Token analysis
+const tokenizer = shallowRef<typeof import('gpt-tokenizer') | undefined>()
+const inputTokens = computed(() =>
+  tokenizer.value?.encode(formattedInput.value).length,
+)
+const toonTokens = computed(() =>
+  tokenizer.value && toonOutput.value ? tokenizer.value.encode(toonOutput.value).length : undefined,
+)
+const tokenSavings = computed(() => {
+  if (!inputTokens.value || !toonTokens.value)
+    return
+
+  const diff = inputTokens.value - toonTokens.value
+  const percent = Math.abs((diff / inputTokens.value) * 100).toFixed(1)
+  const sign = diff > 0 ? '−' : '+'
+
+  return { diff, percent, sign, isSavings: diff > 0 }
+})
+
+// UI state
+const canShareState = ref(true)
+const hasCopiedUrl = ref(false)
+
+const { copy, copied } = useClipboard({ source: toonOutput })
+const updateUrl = useDebounceFn(() => {
+  const hash = encodeState()
+  const baseUrl = `${window.location.origin}${window.location.pathname}${window.location.search}`
+  const targetUrl = `${baseUrl}#${hash}`
+
+  if (targetUrl.length > SHARE_URL_LIMIT) {
+    canShareState.value = false
+    return
+  }
+
+  canShareState.value = true
+  window.history.replaceState(null, '', `#${hash}`)
+}, 300)
+
+watch([inputText, delimiter, indent, keyFolding, flattenDepth, jsonFormat, inputFormat], () => {
+  updateUrl()
+})
+
+watch(jsonFormat, () => {
+  if (inputFormat.value !== 'json')
+    return
+  try {
+    inputText.value = formatJson(JSON.parse(inputText.value))
+  }
+  catch {}
+})
+
+watch(inputFormat, (next, prev) => {
+  if (prev === next)
+    return
+  try {
+    const data = parseInput(inputText.value, prev)
+    inputText.value = next === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+  }
+  catch {}
+})
+
+onMounted(() => {
+  loadTokenizer()
+
+  const hash = window.location.hash.slice(1)
+  if (!hash)
+    return
+
+  const state = decodeState(hash)
+  if (state) {
+    inputText.value = state.input ?? state.json
+    delimiter.value = state.delimiter
+    indent.value = state.indent
+    keyFolding.value = state.keyFolding ?? 'safe'
+    flattenDepth.value = state.flattenDepth ?? 2
+    jsonFormat.value = state.jsonFormat ?? 'pretty-2'
+    inputFormat.value = state.inputFormat ?? 'json'
+  }
+})
+
+function formatJson(value: unknown) {
+  return JSON.stringify(value, undefined, currentFormatIndent.value)
+}
+
+function encodeState() {
+  const state: PlaygroundState = {
+    input: inputText.value,
+    inputFormat: inputFormat.value,
+    delimiter: delimiter.value,
+    indent: indent.value,
+    keyFolding: keyFolding.value,
+    flattenDepth: flattenDepth.value,
+    jsonFormat: jsonFormat.value,
+  }
+
+  const compressedData = zlibSync(stringToUint8Array(JSON.stringify(state)))
+  return uint8ArrayToBase64(compressedData, { urlSafe: true })
+}
+
+function decodeState(hash: string) {
+  try {
+    const bytes = base64ToUint8Array(hash)
+    const decompressedData = unzlibSync(bytes)
+    const decodedData = uint8ArrayToString(decompressedData)
+    if (decodedData)
+      return JSON.parse(decodedData) as PlaygroundState
+  }
+  catch {}
+}
+
+function loadPreset(name: keyof typeof PRESETS) {
+  const data = PRESETS[name]
+  inputText.value = inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+}
+
+async function copyShareUrl() {
+  if (!canShareState.value)
+    return
+
+  await navigator.clipboard.writeText(window.location.href)
+  hasCopiedUrl.value = true
+  setTimeout(() => (hasCopiedUrl.value = false), 2000)
+}
+
+async function loadTokenizer() {
+  tokenizer.value ??= await import('gpt-tokenizer')
+}
+</script>
+
+<template>
+  <div class="playground">
+    <div class="playground-container">
+      <!-- Header -->
+      <header class="playground-header">
+        <h1>Playground</h1>
+        <p>Convert JSON or YAML to TOON in real time.</p>
+      </header>
+
+      <!-- Options Bar -->
+      <div class="options-bar">
+        <VPInput id="inputFormat" label="Input format">
+          <select id="inputFormat" v-model="inputFormat">
+            <option value="json">
+              JSON
+            </option>
+            <option value="yaml">
+              YAML
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="delimiter" label="Delimiter">
+          <select id="delimiter" v-model="delimiter">
+            <option v-for="opt in DELIMITER_OPTIONS" :key="opt.value" :value="opt.value">
+              {{ opt.label }}
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="indent" label="Indent">
+          <input
+            id="indent"
+            v-model.number="indent"
+            type="number"
+            min="0"
+            max="8"
+          >
+        </VPInput>
+
+        <VPInput id="keyFolding" label="Key Folding">
+          <select id="keyFolding" v-model="keyFolding">
+            <option value="off">
+              Off
+            </option>
+            <option value="safe">
+              Safe
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="flattenDepth" label="Flatten Depth">
+          <input
+            id="flattenDepth"
+            v-model.number="flattenDepth"
+            type="number"
+            min="1"
+            max="10"
+            :disabled="keyFolding === 'off'"
+          >
+        </VPInput>
+
+        <VPInput id="preset" label="Preset">
+          <select id="preset" @change="(e) => loadPreset((e.target as HTMLSelectElement).value as keyof typeof PRESETS)">
+            <option value="" disabled selected>
+              Load example…
+            </option>
+            <option value="hikes">
+              Hikes (mixed structure)
+            </option>
+            <option value="orders">
+              Orders (nested objects)
+            </option>
+            <option value="metrics">
+              Metrics (tabular data)
+            </option>
+            <option value="events">
+              Events (semi-uniform)
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput v-if="inputFormat === 'json'" id="jsonFormat" label="JSON Baseline">
+          <select id="jsonFormat" v-model="jsonFormat">
+            <option v-for="opt in JSON_FORMAT_OPTIONS" :key="opt.value" :value="opt.value">
+              {{ opt.label }}
+            </option>
+          </select>
+        </VPInput>
+
+        <button
+          class="share-button"
+          :class="[hasCopiedUrl && 'copied']"
+          :aria-label="
+            !canShareState
+              ? 'State too large to share via URL'
+              : hasCopiedUrl
+                ? 'Link copied!'
+                : 'Copy shareable URL'
+          "
+          :title="!canShareState ? 'State too large to share via URL' : undefined"
+          :disabled="!canShareState"
+          :aria-disabled="!canShareState"
+          @click="copyShareUrl"
+        >
+          <span class="vpi-link" :class="[hasCopiedUrl && 'check']" aria-hidden="true" />
+          <template v-if="!canShareState">
+            Too large to share
+          </template>
+          <template v-else>
+            {{ hasCopiedUrl ? 'Copied!' : 'Share' }}
+          </template>
+        </button>
+      </div>
+
+      <!-- Editor Container -->
+      <div class="editor-container">
+        <!-- Input -->
+        <div class="editor-pane">
+          <div class="pane-header">
+            <span class="pane-title">{{ inputFormat === 'yaml' ? 'YAML Input' : 'JSON Input' }}</span>
+            <span class="pane-stats">
+              <span class="stat-primary" title="Token count of the formatted input">{{ inputTokens ?? '…' }} tokens</span>
+              <span class="stat-secondary">{{ formattedInput.length }} chars</span>
+            </span>
+          </div>
+          <textarea
+            id="input"
+            v-model="inputText"
+            class="editor-textarea"
+            spellcheck="false"
+            :aria-label="inputFormat === 'yaml' ? 'YAML input' : 'JSON input'"
+            :aria-describedby="error ? 'parse-error' : undefined"
+            :aria-invalid="!!error"
+            :placeholder="inputFormat === 'yaml' ? 'Enter YAML here…' : 'Enter JSON here…'"
+          />
+        </div>
+
+        <!-- TOON Output -->
+        <div class="editor-pane">
+          <div class="pane-header">
+            <span class="pane-title">
+              TOON Output
+              <span v-if="tokenSavings" class="savings-badge" :class="[!tokenSavings.isSavings && 'increase']">
+                {{ tokenSavings.sign }}{{ tokenSavings.percent }}%
+              </span>
+            </span>
+            <span class="pane-stats">
+              <span class="stat-primary">{{ toonTokens ?? '…' }} tokens</span>
+              <span class="stat-secondary">{{ toonOutput.length }} chars</span>
+            </span>
+          </div>
+          <div class="editor-output">
+            <button
+              v-if="!error"
+              class="copy-button"
+              :class="[copied && 'copied']"
+              :aria-label="copied ? 'Copied to clipboard' : 'Copy to clipboard'"
+              :aria-pressed="copied"
+              @click="copy()"
+            />
+            <pre v-if="!error"><code>{{ toonOutput }}</code></pre>
+            <div v-else id="parse-error" role="alert" class="error-message">
+              {{ error }}
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.playground {
+  padding: 32px 24px 32px;
+}
+
+@media (min-width: 768px) {
+  .playground {
+    padding: 48px 32px 48px;
+  }
+}
+
+@media (min-width: 960px) {
+  .playground {
+    padding: 48px 32px 48px;
+  }
+}
+
+.playground-container {
+  max-width: 1400px;
+  margin: 0 auto;
+}
+
+.playground-header {
+  margin-bottom: 24px;
+}
+
+.playground-header h1 {
+  font-size: 28px;
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  line-height: 40px;
+  color: var(--vp-c-text-1);
+  margin: 0 0 8px;
+}
+
+@media (min-width: 768px) {
+  .playground-header h1 {
+    font-size: 32px;
+  }
+}
+
+.playground-header p {
+  font-size: 16px;
+  line-height: 28px;
+  color: var(--vp-c-text-2);
+}
+
+.options-bar {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 12px;
+  align-items: flex-end;
+  margin-bottom: 16px;
+  padding: 12px 16px;
+  background: var(--vp-c-bg-soft);
+  border-radius: 8px;
+  border: 1px solid var(--vp-c-divider);
+}
+
+@media (max-width: 768px) {
+  .options-bar {
+    gap: 8px;
+  }
+}
+
+.vpi-link {
+  --icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' viewBox='0 0 24 24'%3E%3Cpath d='M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71'/%3E%3Cpath d='M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71'/%3E%3C/svg%3E");
+  display: inline-block;
+  width: 1em;
+  height: 1em;
+  -webkit-mask: var(--icon) no-repeat;
+  mask: var(--icon) no-repeat;
+  -webkit-mask-size: 100% 100%;
+  mask-size: 100% 100%;
+  background-color: currentColor;
+}
+
+.vpi-link.check {
+  --icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' viewBox='0 0 24 24'%3E%3Cpath d='M20 6 9 17l-5-5'/%3E%3C/svg%3E");
+}
+
+.share-button {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 0 12px;
+  height: 32px;
+  font-size: 13px;
+  font-weight: 500;
+  color: var(--vp-c-text-1);
+  background: var(--vp-c-bg);
+  border: 1px solid var(--vp-c-border);
+  border-radius: 6px;
+  transition: border-color 0.25s, color 0.25s;
+  margin-left: auto;
+}
+
+.share-button:hover {
+  border-color: var(--vp-c-brand-1);
+  color: var(--vp-c-brand-1);
+}
+
+.share-button:focus-visible {
+  outline: 2px solid var(--vp-c-brand-1);
+  outline-offset: 2px;
+}
+
+.share-button.copied {
+  border-color: var(--vp-c-green-1);
+  color: var(--vp-c-green-1);
+}
+
+.share-button:disabled {
+  color: var(--vp-c-text-3);
+  border-color: var(--vp-c-divider);
+  background: var(--vp-c-bg-soft);
+  cursor: not-allowed;
+}
+
+.editor-container {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 16px;
+}
+
+@media (max-width: 768px) {
+  .editor-container {
+    grid-template-columns: 1fr;
+  }
+}
+
+.editor-pane {
+  display: flex;
+  flex-direction: column;
+  min-height: 500px;
+  border: 1px solid var(--vp-c-divider);
+  border-radius: 8px;
+  overflow: hidden;
+  background: var(--vp-c-bg-soft);
+  transition: border-color 0.25s;
+}
+
+@media (max-width: 768px) {
+  .editor-pane {
+    min-height: 400px;
+  }
+}
+
+.editor-pane:focus-within {
+  border-color: var(--vp-c-brand-1);
+}
+
+.pane-header {
+  display: flex;
+  align-items: center;
+  gap: 12px;
+  padding: 12px 16px;
+  background: var(--vp-c-bg-alt);
+  border-bottom: 1px solid var(--vp-c-divider);
+}
+
+.pane-title {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-size: 0.75rem;
+  font-weight: 600;
+  color: var(--vp-c-text-2);
+  text-transform: uppercase;
+  letter-spacing: 0.05em;
+  line-height: 1.5;
+}
+
+.pane-stats {
+  display: flex;
+  gap: 12px;
+  margin-left: auto;
+  font-size: 0.75rem;
+  font-weight: 400;
+  color: var(--vp-c-text-2);
+  text-transform: none;
+  letter-spacing: normal;
+}
+
+.stat-primary {
+  font-weight: 600;
+  color: var(--vp-c-text-1);
+}
+
+.stat-secondary {
+  color: var(--vp-c-text-3);
+}
+
+.savings-badge {
+  display: inline-flex;
+  padding: 2px 6px;
+  font-size: 0.625rem;
+  font-weight: 600;
+  color: var(--vp-c-green-1);
+  background: var(--vp-c-green-soft);
+  border-radius: 4px;
+  text-transform: none;
+  letter-spacing: normal;
+}
+
+.savings-badge.increase {
+  color: var(--vp-c-yellow-1);
+  background: var(--vp-c-yellow-soft);
+}
+
+.copy-button {
+  position: absolute;
+  top: 12px;
+  right: 12px;
+  z-index: 3;
+  border: 1px solid var(--vp-code-copy-code-border-color);
+  border-radius: 4px;
+  width: 40px;
+  height: 40px;
+  background-color: var(--vp-code-copy-code-bg);
+  opacity: 0;
+  cursor: pointer;
+  background-image: var(--vp-icon-copy);
+  background-position: 50%;
+  background-size: 20px;
+  background-repeat: no-repeat;
+  transition: border-color 0.25s, background-color 0.25s, opacity 0.25s;
+}
+
+.editor-output:hover .copy-button,
+.copy-button:focus {
+  opacity: 1;
+}
+
+.copy-button:hover:not(:disabled),
+.copy-button.copied {
+  border-color: var(--vp-code-copy-code-hover-border-color);
+  background-color: var(--vp-code-copy-code-hover-bg);
+}
+
+.copy-button:focus-visible {
+  outline: 2px solid var(--vp-c-brand-1);
+  outline-offset: 2px;
+}
+
+.copy-button:disabled {
+  opacity: 0.3;
+  cursor: not-allowed;
+}
+
+.copy-button.copied,
+.copy-button:hover.copied {
+  border-radius: 0 4px 4px 0;
+  background-image: var(--vp-icon-copied);
+}
+
+.copy-button.copied::before,
+.copy-button:hover.copied::before {
+  position: relative;
+  top: -1px;
+  transform: translateX(calc(-100% - 1px));
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 1px solid var(--vp-code-copy-code-hover-border-color);
+  border-right: 0;
+  border-radius: 4px 0 0 4px;
+  padding: 0 10px;
+  width: fit-content;
+  height: 40px;
+  text-align: center;
+  font-size: 12px;
+  font-weight: 500;
+  color: var(--vp-code-copy-code-active-text);
+  background-color: var(--vp-code-copy-code-hover-bg);
+  white-space: nowrap;
+  content: var(--vp-code-copy-copied-text-content);
+}
+
+.copy-button[aria-pressed="true"] {
+  opacity: 1;
+}
+
+.editor-textarea,
+.editor-output {
+  flex: 1;
+  padding: 16px;
+  font-family: var(--vp-font-family-mono);
+  font-size: 0.875rem;
+  line-height: 1.7;
+}
+
+.editor-textarea {
+  resize: none;
+  color: var(--vp-c-text-1);
+  background: var(--vp-c-bg);
+}
+
+.editor-output {
+  position: relative;
+  overflow: auto;
+  background: var(--vp-code-block-bg);
+}
+
+.editor-output pre {
+  margin: 0;
+  white-space: pre;
+}
+
+.error-message {
+  color: var(--vp-c-danger-1);
+  padding: 8px 12px;
+  background: var(--vp-c-danger-soft);
+  border-radius: 4px;
+  font-size: 0.875rem;
+  font-family: var(--vp-font-family-base);
+}
+</style>

docs/.vitepress/theme/components/VPInput.vue

@@ -0,0 +1,68 @@
+<script setup lang="ts">
+defineProps<{
+  label: string
+  id: string
+}>()
+</script>
+
+<template>
+  <div class="VPInput">
+    <label :for="id" class="label">{{ label }}</label>
+    <div class="input-wrapper">
+      <slot />
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.VPInput {
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+}
+
+.label {
+  font-size: 11px;
+  font-weight: 500;
+  color: var(--vp-c-text-2);
+}
+
+.input-wrapper :deep(select),
+.input-wrapper :deep(input) {
+  padding: 0 10px;
+  height: 32px;
+  font-size: 13px;
+  font-weight: 500;
+  color: var(--vp-c-text-1);
+  background-color: var(--vp-c-bg);
+  border: 1px solid var(--vp-c-border);
+  border-radius: 6px;
+  transition: border-color 0.25s;
+}
+
+.input-wrapper :deep(select):hover,
+.input-wrapper :deep(input):hover,
+.input-wrapper :deep(select):focus,
+.input-wrapper :deep(input):focus {
+  border-color: var(--vp-c-brand-1);
+}
+
+.input-wrapper :deep(select:disabled),
+.input-wrapper :deep(input:disabled) {
+  color: var(--vp-c-text-3);
+  background-color: var(--vp-c-bg-soft);
+  border-color: var(--vp-c-divider);
+  cursor: not-allowed;
+}
+
+.input-wrapper :deep(select:disabled):hover,
+.input-wrapper :deep(input:disabled):hover,
+.input-wrapper :deep(select:disabled):focus,
+.input-wrapper :deep(input:disabled):focus {
+  border-color: var(--vp-c-divider);
+}
+
+.input-wrapper :deep(input[type="number"]) {
+  width: 70px;
+}
+</style>

docs/.vitepress/theme/index.ts

@@ -0,0 +1,23 @@
+import type { Theme } from 'vitepress'
+import CopyOrDownloadAsMarkdownButtons from 'vitepress-plugin-llms/vitepress-components/CopyOrDownloadAsMarkdownButtons.vue'
+import DefaultTheme from 'vitepress/theme'
+import PlaygroundLayout from './components/PlaygroundLayout.vue'
+import VPInput from './components/VPInput.vue'
+
+import './vars.css'
+import './overrides.css'
+import 'uno.css'
+
+const config: Theme = {
+  extends: DefaultTheme,
+  enhanceApp({ app }) {
+    app.config.globalProperties.$spec = {
+      version: '3.3',
+    }
+    app.component('CopyOrDownloadAsMarkdownButtons', CopyOrDownloadAsMarkdownButtons)
+    app.component('PlaygroundLayout', PlaygroundLayout)
+    app.component('VPInput', VPInput)
+  },
+}
+
+export default config

docs/.vitepress/theme/overrides.css

@@ -0,0 +1,34 @@
+.dark [img-light] {
+  display: none;
+}
+
+html:not(.dark) [img-dark] {
+  display: none;
+}
+
+details summary {
+  cursor: pointer;
+}
+
+.vp-doc [class*="language-"] code {
+  color: var(--vp-c-text-1)
+}
+
+.VPHomeHero .image-src {
+  max-width: 112px;
+  max-height: 112px;
+}
+
+@media (min-width: 640px) {
+  .VPHomeHero .image-src {
+    max-width: 144px;
+    max-height: 144px;
+  }
+}
+
+@media (min-width:960px) {
+  .VPHomeHero .image-src {
+    max-width: 176px;
+    max-height: 176px;
+  }
+}

docs/.vitepress/theme/vars.css

@@ -0,0 +1,41 @@
+/**
+ * Colors Theme
+ * -------------------------------------------------------------------------- */
+
+ :root {
+  --vp-c-brand-1: #d97c06;
+  --vp-c-brand-2: #C57105;
+  --vp-c-brand-3: #B16505;
+  --vp-nav-logo-height: 20px;
+}
+
+/**
+ * Component: Home
+ * -------------------------------------------------------------------------- */
+
+ :root {
+  --vp-home-hero-name-color: transparent;
+  --vp-home-hero-name-background: -webkit-linear-gradient(
+    120deg,
+    #fde98a 15%,
+    #d97c06
+  );
+  --vp-home-hero-image-background-image: linear-gradient(
+    -45deg,
+    #d97c0660 30%,
+    #fde98a60
+  );
+  --vp-home-hero-image-filter: blur(30px);
+}
+
+@media (min-width: 640px) {
+  :root {
+    --vp-home-hero-image-filter: blur(56px);
+  }
+}
+
+@media (min-width: 960px) {
+  :root {
+    --vp-home-hero-image-filter: blur(72px);
+  }
+}

docs/cli/index.md

@@ -0,0 +1,355 @@
+---
+description: Convert JSON to TOON and back from the command line, with token statistics, streaming, and delimiter options.
+---
+
+# Command Line Interface
+
+The `@toon-format/cli` package converts JSON to TOON and TOON to JSON. Use it to measure token savings before integrating TOON into your application, or to pipe JSON through TOON in shell workflows alongside tools like `curl` and `jq`. The CLI supports stdin/stdout, token statistics, streaming for large datasets, and every encoding option in the library.
+
+The CLI is built on the `@toon-format/toon` TypeScript implementation and follows the [latest specification](/reference/spec).
+
+## Usage
+
+### Without Installation
+
+Use `npx` to run the CLI without installing:
+
+::: code-group
+
+```bash [Encode]
+npx @toon-format/cli input.json -o output.toon
+```
+
+```bash [Decode]
+npx @toon-format/cli data.toon -o output.json
+```
+
+```bash [Stdin]
+echo '{"name": "Ada"}' | npx @toon-format/cli
+```
+
+:::
+
+### Global Installation
+
+Or install globally for repeated use:
+
+::: code-group
+
+```bash [npm]
+npm install -g @toon-format/cli
+```
+
+```bash [pnpm]
+pnpm add -g @toon-format/cli
+```
+
+```bash [yarn]
+yarn global add @toon-format/cli
+```
+
+:::
+
+After global installation, use the `toon` command:
+
+```bash
+toon input.json -o output.toon
+```
+
+## Basic Usage
+
+### Auto-Detection
+
+The CLI automatically detects the operation based on file extension:
+- `.json` files → encode (JSON to TOON)
+- `.toon` files → decode (TOON to JSON)
+
+When reading from stdin, use `--encode` or `--decode` flags to specify the operation (defaults to encode).
+
+::: code-group
+
+```bash [Encode JSON to TOON]
+toon input.json -o output.toon
+```
+
+```bash [Decode TOON to JSON]
+toon data.toon -o output.json
+```
+
+```bash [Output to stdout]
+toon input.json
+```
+
+```bash [Pipe from stdin]
+cat data.json | toon
+echo '{"name": "Ada"}' | toon
+```
+
+```bash [Decode from stdin]
+cat data.toon | toon --decode
+```
+
+:::
+
+By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` (see [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations)).
+
+### Standard Input
+
+Omit the input argument or use `-` to read from stdin. This enables piping data directly from other commands:
+
+```bash
+# No argument needed
+cat data.json | toon
+
+# Explicit stdin with hyphen (equivalent)
+cat data.json | toon -
+
+# Decode from stdin
+cat data.toon | toon --decode
+```
+
+## Performance
+
+### Streaming Output
+
+Both encoding and decoding operations use streaming output, writing incrementally without building the full output string in memory. This makes the CLI efficient for large datasets without requiring additional configuration.
+
+**JSON → TOON (Encode)**:
+
+- Streams TOON lines to output.
+- No full TOON string in memory.
+
+**TOON → JSON (Decode)**:
+
+- Uses the same event-based streaming decoder as the `decodeStream` API in `@toon-format/toon`.
+- Streams JSON tokens to output.
+- No full JSON string in memory.
+- When `--expandPaths safe` is enabled, falls back to non-streaming decode internally to apply deep-merge expansion before writing JSON.
+
+Process large files with minimal memory usage:
+
+```bash
+# Encode large JSON file
+toon huge-dataset.json -o output.toon
+
+# Decode large TOON file
+toon huge-dataset.toon -o output.json
+
+# Process millions of records efficiently via stdin
+cat million-records.json | toon > output.toon
+cat million-records.toon | toon --decode > output.json
+```
+
+Peak memory usage scales with data depth, not total size. This allows processing arbitrarily large files as long as individual nested structures fit in memory.
+
+::: tip Token Statistics
+When using the `--stats` flag with encode, the CLI builds the full TOON string once to compute accurate token counts. For maximum memory efficiency on very large files, omit `--stats`.
+:::
+
+## Options
+
+| Option | Description |
+| ------ | ----------- |
+| `-o, --output <file>` | Output file path (prints to stdout if omitted) |
+| `-e, --encode` | Force encode mode (overrides auto-detection) |
+| `-d, --decode` | Force decode mode (overrides auto-detection) |
+| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
+| `--indent <number>` | Indentation size (default: `2`) |
+| `--stats` | Show token count estimates and savings (encode only) |
+| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
+| `--keyFolding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
+| `--flattenDepth <number>` | Maximum segments to fold (default: `Infinity`) – requires `--keyFolding safe` |
+| `--expandPaths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
+| `--verbose` | Show full stack traces and cause chains for errors (default: `false`) |
+
+## Advanced Examples
+
+### Token Statistics
+
+Show token savings when encoding:
+
+```bash
+toon data.json --stats -o output.toon
+```
+
+This helps you estimate token cost savings before sending data to LLMs.
+
+Example output:
+
+```
+✔ Encoded data.json → output.toon
+
+ℹ Token estimates: ~15,145 (JSON) → ~8,745 (TOON)
+✔ Saved ~6,400 tokens (-42.3%)
+```
+
+### Alternative Delimiters
+
+TOON supports three delimiters: comma (default), tab, and pipe. Alternative delimiters can save additional tokens depending on the data.
+
+::: code-group
+
+```bash [Tab-separated (bash/zsh)]
+toon data.json --delimiter $'\t' -o output.toon
+```
+
+```bash [Pipe-separated]
+toon data.json --delimiter "|" -o output.toon
+```
+
+:::
+
+The `--delimiter` value must be the actual delimiter character. In bash/zsh, use `$'\t'` to pass a real tab; literal `"\t"` is rejected as an invalid delimiter.
+
+**Tab delimiter example:**
+
+::: code-group
+
+```yaml [Tab]
+items[2	]{id	name	qty	price}:
+  A1	Widget	2	9.99
+  B2	Gadget	1	14.5
+```
+
+```yaml [Comma (default)]
+items[2]{id,name,qty,price}:
+  A1,Widget,2,9.99
+  B2,Gadget,1,14.5
+```
+
+:::
+
+::: tip
+Tab delimiters often tokenize more efficiently than commas and reduce the need for quote-escaping. Use `--delimiter $'\t'` (bash/zsh) for maximum token savings on large tabular data. See [Delimiter Strategies](/reference/api#delimiter-strategies) for full guidance.
+:::
+
+### Lenient Decoding
+
+Skip validation for faster, more forgiving decoding:
+
+```bash
+toon data.toon --no-strict -o output.json
+```
+
+With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
+
+### Decode Error Output
+
+When a TOON document fails to parse, the CLI renders the offending line with a caret pointing at the first non-whitespace character. Tabs are shown as `→` so the caret column reflects what the decoder actually saw.
+
+For an input file that uses a tab to indent the second line (rendered here with `→`):
+
+```
+a:
+→b: 1
+```
+
+The CLI prints:
+
+```
+ ERROR  Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode
+
+  2 | →b: 1
+      ^
+```
+
+The exit code is `1` on any error. Stack traces are suppressed by default. Pass `--verbose` to include the full stack and the underlying cause chain – useful when filing a bug report or diagnosing an unexpected error path:
+
+```bash
+cat broken.toon | toon --decode --verbose
+```
+
+::: tip Programmatic Access
+Decode errors are thrown as `ToonDecodeError` instances by the library. The CLI's caret rendering is built on the structured `line` and `source` fields exposed on that class. See the [Error Handling](/reference/api#error-handling) section of the API reference if you want the same diagnostic detail in your own code.
+:::
+
+### Stdin Workflows
+
+The CLI integrates seamlessly with Unix pipes and other command-line tools:
+
+```bash
+# Convert API response to TOON
+curl https://api.example.com/data | toon --stats
+
+# Process large dataset
+cat large-dataset.json | toon --delimiter $'\t' > output.toon
+
+# Chain with jq
+jq '.results' data.json | toon > filtered.toon
+```
+
+### Key Folding
+
+Collapse nested wrapper chains to reduce tokens (since spec v1.5):
+
+::: code-group
+
+```bash [Basic key folding]
+toon input.json --keyFolding safe -o output.toon
+```
+
+```bash [Limit folding depth]
+toon input.json --keyFolding safe --flattenDepth 2 -o output.toon
+```
+
+:::
+
+**Example:**
+
+For data like:
+
+```json
+{
+  "data": {
+    "metadata": {
+      "items": ["a", "b"]
+    }
+  }
+}
+```
+
+With `--keyFolding safe`, output becomes:
+
+```yaml
+data.metadata.items[2]: a,b
+```
+
+Instead of:
+
+```yaml
+data:
+  metadata:
+    items[2]: a,b
+```
+
+### Path Expansion
+
+Reconstruct nested structure from folded keys when decoding:
+
+```bash
+toon data.toon --expandPaths safe -o output.json
+```
+
+This pairs with `--keyFolding safe` for lossless round-trips.
+
+### Round-Trip Workflow
+
+```bash
+# Encode with folding
+toon input.json --keyFolding safe -o compressed.toon
+
+# Decode with expansion (restores original structure)
+toon compressed.toon --expandPaths safe -o output.json
+
+# Verify round-trip
+diff input.json output.json
+```
+
+### Combined Options
+
+Combine multiple options for maximum efficiency:
+
+```bash
+# Key folding + tab delimiter + stats
+toon data.json --keyFolding safe --delimiter $'\t' --stats -o output.toon
+```

docs/ecosystem/implementations.md

@@ -0,0 +1,68 @@
+---
+description: Official and community TOON implementations across languages, plus contribution pointers.
+---
+
+# Implementations
+
+TOON has official and community implementations across multiple programming languages. All implementations are intended to conform to the same [Specification](https://github.com/toon-format/spec) to ensure compatibility and interoperability.
+
+The code examples throughout this documentation site use the TypeScript implementation by default, but the format and concepts apply equally to all languages.
+
+> [!NOTE]
+> When implementing TOON in other languages, please follow the [spec](https://github.com/toon-format/spec/blob/main/SPEC.md) to ensure compatibility across implementations. The [conformance tests](https://github.com/toon-format/spec/tree/main/tests) provide language-agnostic test fixtures that validate your implementation.
+
+## Official Implementations
+
+These implementations are actively being developed by dedicated teams. Contributions are welcome! Join the effort by opening issues, submitting PRs, or discussing implementation details in the respective repositories.
+
+| Language | Repository | Status | Maintainer |
+|----------|------------|--------|------------|
+| **.NET** | [toon-dotnet](https://github.com/toon-format/toon-dotnet) | In Development | Official Team |
+| **Dart** | [toon-dart](https://github.com/toon-format/toon-dart) | In Development | Official Team |
+| **Go** | [toon-go](https://github.com/toon-format/toon-go) | In Development | Official Team |
+| **Java** | [toon-java](https://github.com/toon-format/toon-java) | ✅ Stable | Official Team |
+| **Julia** | [ToonFormat.jl](https://github.com/toon-format/ToonFormat.jl) | ✅ Stable | Official Team |
+| **Python** | [toon-python](https://github.com/toon-format/toon-python) | ✅ Stable | Official Team |
+| **Rust** | [toon-rust](https://github.com/toon-format/toon-rust) | ✅ Stable | Official Team |
+| **Swift** | [toon-swift](https://github.com/toon-format/toon-swift) | ✅ Stable | Official Team |
+| **TypeScript/JavaScript** | [toon](https://github.com/toon-format/toon/tree/main/packages/toon) | ✅ Stable | Official Team |
+
+## Community Implementations
+
+Community members have created implementations in additional languages:
+
+| Language | Repository | Maintainer |
+|----------|------------|------------|
+| **Apex** | [ApexToon](https://github.com/Eacaw/ApexToon) | [@Eacaw](https://github.com/Eacaw) |
+| **C** | [TOONc](https://github.com/UsboKirishima/TOONc) | [@UsboKirishima](https://github.com/UsboKirishima) |
+| **C++** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
+| **C#** | [ToonEncoder](https://github.com/Cysharp/ToonEncoder) | [@Cysharp](https://github.com/Cysharp/ToonEncoder) |
+| **Clojure** | [toon](https://github.com/vadelabs/toon) | [@vadelabs](https://github.com/vadelabs) |
+| **Crystal** | [toon-crystal](https://github.com/mamantoha/toon-crystal) | [@mamantoha](https://github.com/mamantoha) |
+| **Delphi** | [delphi-toon](https://github.com/ernestoalconada/delphi-toon) | [@ernestoalconada](https://github.com/ernestoalconada) |
+| **Elixir** | [toon_ex](https://github.com/kentaro/toon_ex) | [@kentaro](https://github.com/kentaro) |
+| **Gleam** | [toon_codec](https://github.com/axelbellec/toon_codec) | [@axelbellec](https://github.com/axelbellec) |
+| **Go** | [gotoon](https://github.com/alpkeskin/gotoon) | [@alpkeskin](https://github.com/alpkeskin) |
+| **Java** | [json-io](https://github.com/jdereg/json-io) | [@jdereg](https://github.com/jdereg) |
+| **Kotlin** | [ktoon](https://github.com/lukelast/ktoon)| [@lukelast](https://github.com/lukelast) |
+| **Laravel Framework** | [laravel-toon](https://github.com/mischasigtermans/laravel-toon) | [@mischasigtermans](https://github.com/mischasigtermans) |
+| **Lua/Neovim** | [toon.nvim](https://github.com/thalesgelinger/toon.nvim) | [@thalesgelinger](https://github.com/thalesgelinger) |
+| **Matlab** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
+| **OCaml** | [ocaml-toon](https://github.com/davesnx/ocaml-toon) | [@davesnx](https://github.com/davesnx) |
+| **Perl** | [Data::TOON](https://github.com/ytnobody/p5-Data-TOON) | [@ytnobody](https://github.com/ytnobody) |
+| **PHP** | [toon-php](https://github.com/HelgeSverre/toon-php) | [@HelgeSverre](https://github.com/HelgeSverre) |
+| **Python** (C++ backend) | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
+| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
+| **R** | [toon](https://github.com/laresbernardo/toon) | [@laresbernardo](https://github.com/laresbernardo) |
+| **Ruby** | [toon-ruby](https://github.com/andrepcg/toon-ruby) | [@andrepcg](https://github.com/andrepcg) |
+| **Scala** | [toon4s](https://github.com/vim89/toon4s) | [@vim89](https://github.com/vim89) |
+| **Zig** | [toon-zig](https://github.com/LatentEvals/toon-zig) | [@montanaflynn](https://github.com/montanaflynn) |
+
+## Contributing an Implementation
+
+Building a TOON implementation for a new language? Great! Here are some steps to get started:
+
+1. **Follow the spec**: Implement the [latest specification](https://github.com/toon-format/spec/blob/main/SPEC.md).
+2. **Add tests**: Run the [reference test suite](https://github.com/toon-format/spec/tree/main/tests).
+3. **Document usage**: Provide a clear README with installation and usage examples.
+4. **Share it**: Open a PR to add your implementation to the README at [github.com/toon-format/toon](https://github.com/toon-format/toon).

docs/ecosystem/tools-and-playgrounds.md

@@ -0,0 +1,82 @@
+---
+description: TOON playgrounds, CLI, editor support, and ecosystem tools.
+---
+
+# Tools and Playgrounds
+
+Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.
+
+## Playgrounds
+
+### Official Playground
+
+The [TOON Playground](/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
+
+### Community Playgrounds
+
+- [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration)
+- [TOON Tools](https://toontools.vercel.app/)
+
+## CLI Tool
+
+The official TOON CLI provides command-line conversion, token statistics, and all encoding/decoding features. See the [CLI reference](/cli/) for full documentation.
+
+```bash
+npx @toon-format/cli input.json --stats -o output.toon
+```
+
+## Editor Support
+
+### VS Code
+
+[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) – Syntax highlighting, validation, conversion, and token analysis.
+
+Install from the [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) or via command line:
+
+```bash
+code --install-extension vishalraut.vscode-toon
+```
+
+### Tree-sitter Grammar
+
+[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) – Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
+
+### Neovim
+
+[toon.nvim](https://github.com/thalesgelinger/toon.nvim) – Lua-based plugin for Neovim.
+
+### Other Editors
+
+Use YAML syntax highlighting as a close approximation. Most editors allow associating `.toon` files with YAML language mode.
+
+## Databases
+
+### ToonStore
+
+[ToonStore](https://github.com/Kalama-Tech/toonstoredb) – Redis-compatible embedded database (Rust) that stores data in TOON format.
+
+## ORMs
+
+### TORM
+
+[TORM](https://github.com/Kalama-Tech/torm) – ORM that works with the ToonStore database, with SDKs for Node.js, Python, Go, and PHP.
+
+## Web APIs
+
+If you're building web applications that work with TOON, you can use the TypeScript library in the browser:
+
+```ts
+import { decode, encode } from '@toon-format/toon'
+
+// Works in browsers, Node.js, Deno, and Bun
+const toon = encode(data)
+const data = decode(toon)
+```
+
+See the [API Reference](/reference/api) for details.
+
+## MCP
+
+### Tooner
+
+[Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON.

docs/guide/benchmarks.md

@@ -0,0 +1,586 @@
+---
+description: Retrieval accuracy and token efficiency results for TOON across mixed-structure and flat-only tracks.
+---
+
+# Benchmarks
+
+The benchmarks on this page measure TOON's performance across two key dimensions:
+
+- **Retrieval Accuracy**: How well LLMs understand and extract information from different input formats.
+- **Token Efficiency**: How many tokens each format requires to represent the same data.
+
+Benchmarks are organized into two tracks to ensure fair comparisons:
+
+- **Mixed-Structure Track**: Datasets with nested or semi-uniform structures (TOON vs JSON, YAML, XML). CSV excluded as it cannot properly represent these structures.
+- **Flat-Only Track**: Datasets with flat tabular structures where CSV is applicable (CSV vs TOON vs JSON, YAML, XML).
+
+## Retrieval Accuracy
+
+<!-- automd:file src="../../benchmarks/results/retrieval-accuracy.md" -->
+
+Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
+
+<details>
+<summary><strong>Show Dataset Catalog</strong></summary>
+
+#### Dataset Catalog
+
+| Dataset | Rows | Structure | CSV Support | Eligibility |
+| ------- | ---- | --------- | ----------- | ----------- |
+| Uniform employee records | 100 | uniform | ✓ | 100% |
+| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
+| Time-series analytics data | 60 | uniform | ✓ | 100% |
+| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
+| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
+| Deeply nested configuration | 11 | deep | ✗ | 0% |
+| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
+| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
+| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
+| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
+| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
+
+**Structure classes:**
+- **uniform**: All objects have identical fields with primitive values
+- **semi-uniform**: Mix of uniform and non-uniform structures
+- **nested**: Objects with nested structures (nested objects or arrays)
+- **deep**: Highly nested with minimal tabular eligibility
+
+**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
+
+**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
+
+</details>
+
+#### Efficiency Ranking (Accuracy per 1K Tokens)
+
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
+
+```
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
+```
+
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
+
+**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
+
+#### Per-Model Accuracy
+
+Accuracy across 4 LLMs on 209 data retrieval questions:
+
+```
+claude-haiku-4-5-20251001
+→ TOON           ████████████░░░░░░░░    59.8% (125/209)
+  JSON           ███████████░░░░░░░░░    57.4% (120/209)
+  YAML           ███████████░░░░░░░░░    56.0% (117/209)
+  XML            ███████████░░░░░░░░░    55.5% (116/209)
+  JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
+  CSV            ██████████░░░░░░░░░░    50.5% (55/109)
+
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
+
+gpt-5-nano
+→ TOON           ██████████████████░░    90.9% (190/209)
+  JSON compact   ██████████████████░░    90.9% (190/209)
+  JSON           ██████████████████░░    89.0% (186/209)
+  CSV            ██████████████████░░    89.0% (97/109)
+  YAML           █████████████████░░░    87.1% (182/209)
+  XML            ████████████████░░░░    80.9% (169/209)
+
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
+```
+
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
+
+<details>
+<summary><strong>Performance by dataset, model, and question type</strong></summary>
+
+#### Performance by Question Type
+
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
+| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
+
+#### Performance by Dataset
+
+##### Uniform employee records
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
+
+##### E-commerce orders with nested structures
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
+
+##### Time-series analytics data
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
+
+##### Top 100 GitHub repositories
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
+
+##### Semi-uniform event logs
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
+
+##### Deeply nested configuration
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
+
+##### Valid complete dataset (control)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
+
+##### Array truncated: 3 rows removed from end
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
+
+##### Extra rows added beyond declared length
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
+
+##### Inconsistent field count (missing salary in row 10)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
+
+##### Missing required fields (no email in multiple rows)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
+
+#### Performance by Model
+
+##### claude-haiku-4-5-20251001
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 59.8% | 125/209 |
+| `json-pretty` | 57.4% | 120/209 |
+| `yaml` | 56.0% | 117/209 |
+| `xml` | 55.5% | 116/209 |
+| `json-compact` | 55.0% | 115/209 |
+| `csv` | 50.5% | 55/109 |
+
+##### gemini-3-flash-preview
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
+
+##### gpt-5-nano
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 90.9% | 190/209 |
+| `json-compact` | 90.9% | 190/209 |
+| `json-pretty` | 89.0% | 186/209 |
+| `csv` | 89.0% | 97/109 |
+| `yaml` | 87.1% | 182/209 |
+| `xml` | 80.9% | 169/209 |
+
+##### grok-4-1-fast-non-reasoning
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
+
+</details>
+
+#### What's Being Measured
+
+This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
+
+#### Datasets Tested
+
+Eleven datasets designed to test different structural patterns and validation capabilities:
+
+**Primary datasets:**
+
+1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
+2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
+3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
+4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
+5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
+6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
+
+**Structural validation datasets:**
+
+7. **Control**: Valid complete dataset (baseline for validation)
+8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
+9. **Extra rows**: Array with 3 additional rows beyond declared length
+10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
+11. **Missing fields**: Systematic field omissions (no email in multiple rows)
+
+#### Question Types
+
+209 questions are generated dynamically across five categories:
+
+- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
+  - Example: "What is Alice's salary?" → `75000`
+  - Example: "How many items are in order ORD-0042?" → `3`
+  - Example: "What is the customer name for order ORD-0042?" → `John Doe`
+
+- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
+  - Example: "How many employees work in Engineering?" → `17`
+  - Example: "What is the total revenue across all orders?" → `45123.50`
+  - Example: "How many employees have salary > 80000?" → `23`
+
+- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
+  - Example: "How many employees in Sales have salary > 80000?" → `5`
+  - Example: "How many active employees have more than 10 years of experience?" → `8`
+
+- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
+  - Example: "How many employees are in the dataset?" → `100`
+  - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
+  - Example: "What is the department of the last employee?" → `Sales`
+
+- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
+  - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
+  - Tests TOON's `[N]` length validation and `{fields}` consistency checking
+  - Demonstrates CSV's lack of structural validation capabilities
+
+#### Evaluation Process
+
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
+2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
+3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
+
+#### Models & Configuration
+
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
+- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
+- **Temperature**: Not set (models use their defaults)
+- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
+
+<!-- /automd -->
+
+## Token Efficiency
+
+Token counts are measured using the GPT-5 `o200k_base` tokenizer via [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer). Savings are calculated against formatted JSON (2-space indentation) as the primary baseline, with additional comparisons to compact JSON (minified), YAML, and XML. Actual savings vary by model and tokenizer.
+
+The benchmarks test datasets across different structural patterns (uniform, semi-uniform, nested, deeply nested) to show where TOON excels and where other formats may be better.
+
+<!-- automd:file src="../../benchmarks/results/token-efficiency.md" -->
+
+#### Mixed-Structure Track
+
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
+
+```
+🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
+   │
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
+
+🧾 Semi-uniform event logs  ┊  Tabular: 50%
+   │
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
+
+🧩 Deeply nested configuration  ┊  Tabular: 0%
+   │
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
+```
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+```
+👥 Uniform employee records  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
+
+📈 Time-series analytics data  ┊  Tabular: 100%
+   │
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
+
+⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
+```
+
+<details>
+<summary><strong>Show detailed examples</strong></summary>
+
+#### 📈 Time-series analytics data
+
+**Savings:** 13,130 tokens (59.0% reduction vs JSON)
+
+**JSON** (22,245 tokens):
+
+```json
+{
+  "metrics": [
+    {
+      "date": "2025-01-01",
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
+    },
+    {
+      "date": "2025-01-02",
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
+    },
+    {
+      "date": "2025-01-03",
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
+    },
+    {
+      "date": "2025-01-04",
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
+    },
+    {
+      "date": "2025-01-05",
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
+    }
+  ]
+}
+```
+
+**TOON** (9,115 tokens):
+
+```
+metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
+```
+
+---
+
+#### ⭐ Top 100 GitHub repositories
+
+**Savings:** 6,400 tokens (42.3% reduction vs JSON)
+
+**JSON** (15,144 tokens):
+
+```json
+{
+  "repositories": [
+    {
+      "id": 28457823,
+      "name": "freeCodeCamp",
+      "repo": "freeCodeCamp/freeCodeCamp",
+      "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
+      "createdAt": "2014-12-24T17:49:19Z",
+      "updatedAt": "2025-10-28T11:58:08Z",
+      "pushedAt": "2025-10-28T10:17:16Z",
+      "stars": 430886,
+      "watchers": 8583,
+      "forks": 42146,
+      "defaultBranch": "main"
+    },
+    {
+      "id": 132750724,
+      "name": "build-your-own-x",
+      "repo": "codecrafters-io/build-your-own-x",
+      "description": "Master programming by recreating your favorite technologies from scratch.",
+      "createdAt": "2018-05-09T12:03:18Z",
+      "updatedAt": "2025-10-28T12:37:11Z",
+      "pushedAt": "2025-10-10T18:45:01Z",
+      "stars": 430877,
+      "watchers": 6332,
+      "forks": 40453,
+      "defaultBranch": "master"
+    },
+    {
+      "id": 21737465,
+      "name": "awesome",
+      "repo": "sindresorhus/awesome",
+      "description": "😎 Awesome lists about all kinds of interesting topics",
+      "createdAt": "2014-07-11T13:42:37Z",
+      "updatedAt": "2025-10-28T12:40:21Z",
+      "pushedAt": "2025-10-27T17:57:31Z",
+      "stars": 410052,
+      "watchers": 8017,
+      "forks": 32029,
+      "defaultBranch": "main"
+    }
+  ]
+}
+```
+
+**TOON** (8,744 tokens):
+
+```
+repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
+  28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
+  132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
+  21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
+```
+
+</details>
+
+<!-- /automd -->
+
+## Related Resources
+
+- [Formal Byte-Level Model](/reference/efficiency-formalization) – Mathematical analysis of byte efficiency compared to JSON
+- [Specification](/reference/spec) – Formal TOON specification

docs/guide/format-overview.md

@@ -0,0 +1,365 @@
+---
+description: TOON syntax with concrete examples – objects, arrays, headers, key folding, and quoting rules.
+---
+
+# Format Overview
+
+TOON syntax reference with concrete examples. See [Getting Started](/guide/getting-started) for an introduction.
+
+## Data Model
+
+TOON models data the same way as JSON:
+
+- **Primitives**: strings, numbers, booleans, and `null`
+- **Objects**: mappings from string keys to values
+- **Arrays**: ordered sequences of values
+
+### Root Forms
+
+A TOON document can represent different root forms:
+
+- **Root object** (most common): Fields appear at depth 0 with no parent key
+- **Root array**: Begins with `[N]:` or `[N]{fields}:` at depth 0
+- **Root primitive**: A single primitive value (string, number, boolean, or null)
+
+Most examples in these docs use root objects, but the format supports all three forms equally ([spec §5](https://github.com/toon-format/spec/blob/main/SPEC.md#5-concrete-syntax-and-root-form)).
+
+## Objects
+
+### Simple Objects
+
+Objects with primitive values use `key: value` syntax, with one field per line:
+
+```yaml
+id: 123
+name: Ada
+active: true
+```
+
+Indentation replaces braces. One space follows the colon.
+
+### Nested Objects
+
+Nested objects add one indentation level (default: 2 spaces):
+
+```yaml
+user:
+  id: 123
+  name: Ada
+```
+
+When a key ends with `:` and has no value on the same line, it opens a nested object. All lines at the next indentation level belong to that object.
+
+### Empty Objects
+
+An empty object at the root yields an empty document (no lines). A nested empty object is `key:` alone, with no children.
+
+## Arrays
+
+TOON detects array structure and chooses the most efficient representation. Arrays always declare their length in brackets: `[N]`.
+
+### Primitive Arrays (Inline)
+
+Arrays of primitives (strings, numbers, booleans, null) are rendered inline:
+
+```yaml
+tags[3]: admin,ops,dev
+```
+
+The delimiter (comma by default) separates values. Strings containing the active delimiter must be quoted.
+
+### Arrays of Objects (Tabular)
+
+When all objects in an array share the same set of primitive-valued keys, TOON uses tabular format:
+
+::: code-group
+
+```yaml [Basic Tabular]
+items[2]{sku,qty,price}:
+  A1,2,9.99
+  B2,1,14.5
+```
+
+```yaml [With Spaces in Values]
+users[2]{id,name,role}:
+  1,Alice Admin,admin
+  2,"Bob Smith",user
+```
+
+:::
+
+The header `items[2]{sku,qty,price}:` declares:
+- **Array length**: `[2]` means 2 rows
+- **Field names**: `{sku,qty,price}` defines the columns
+- **Active delimiter**: comma (default)
+
+Each row contains values in the same order as the field list. Values are encoded as primitives (strings, numbers, booleans, null) and separated by the delimiter.
+
+> [!NOTE]
+> Tabular format requires identical field sets across all objects (same keys, order per object may vary), primitive values only (no nested arrays/objects), and at least one key per object – arrays that contain an empty `{}` element fall back to the expanded list form below.
+
+### Mixed and Non-Uniform Arrays
+
+Arrays that don't meet the tabular requirements use list format with hyphen markers:
+
+```yaml
+items[3]:
+  - 1
+  - a: 1
+  - text
+```
+
+Each element starts with `- ` at one indentation level deeper than the parent array header.
+
+### Objects as List Items
+
+When an array element is an object, it appears as a list item:
+
+```yaml
+items[2]:
+  - id: 1
+    name: First
+  - id: 2
+    name: Second
+    extra: true
+```
+
+When a tabular array is the first field of a list-item object, the tabular header appears on the hyphen line, with rows indented two levels deeper and other fields indented one level deeper:
+
+```yaml
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+    status: active
+```
+
+When the object has only a single tabular field, the same pattern applies:
+
+```yaml
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+```
+
+This is the canonical encoding for list-item objects whose first field is a tabular array.
+
+### Arrays of Arrays
+
+When you have arrays containing primitive inner arrays:
+
+```yaml
+pairs[2]:
+  - [2]: 1,2
+  - [2]: 3,4
+```
+
+Each inner array gets its own header on the list-item line.
+
+When the inner arrays are themselves arrays of objects or non-uniform arrays, the same `- [N]:` header appears on the hyphen line and the nested items follow one indent deeper:
+
+```yaml
+items[3]:
+  - summary
+  - id: 1
+    name: Ada
+  - [2]:
+    - id: 2
+    - status: draft
+```
+
+### Empty Arrays
+
+Empty arrays render as `key: []` for fields and `[]` at the root:
+
+```yaml
+items: []
+```
+
+The legacy `items[0]:` form is still decoded for backward compatibility.
+
+## Array Headers
+
+### Header Syntax
+
+Array headers follow this pattern:
+
+```
+key[N<delimiter?>]<{fields}>:
+```
+
+Where:
+- **N** is the non-negative integer length
+- **delimiter** (optional) explicitly declares the active delimiter:
+  - Absent → comma (`,`)
+  - `\t` (tab character) → tab delimiter
+  - `|` → pipe delimiter
+- **fields** (optional) for tabular arrays: `{field1,field2,field3}`
+
+> [!NOTE]
+> The array length `[N]` helps LLMs validate structure. If you ask a model to generate TOON output, explicit lengths let you detect truncation or malformed data.
+
+### Delimiter Options
+
+TOON supports three delimiters: comma (default), tab, and pipe. The delimiter is scoped to the array header that declares it.
+
+::: code-group
+
+```yaml [Comma (default)]
+items[2]{sku,name,qty,price}:
+  A1,Widget,2,9.99
+  B2,Gadget,1,14.5
+```
+
+```yaml [Tab]
+items[2	]{sku	name	qty	price}:
+  A1	Widget	2	9.99
+  B2	Gadget	1	14.5
+```
+
+```yaml [Pipe]
+items[2|]{sku|name|qty|price}:
+  A1|Widget|2|9.99
+  B2|Gadget|1|14.5
+```
+
+:::
+
+Tab and pipe delimiters are explicitly encoded in the header brackets and field braces. Inside an array scope, only the active delimiter triggers quoting – the others are literal data. Object field values (`key: value`) follow the document delimiter (§11.1) regardless of any surrounding array's active delimiter.
+
+> [!TIP]
+> Tab delimiters often tokenize more efficiently than commas, especially for data with few quoted strings. Use `encode(data, { delimiter: '\t' })` for additional token savings.
+
+## Key Folding (Optional)
+
+Key folding is an optional encoder feature (since spec v1.5) that collapses chains of single-key objects into dotted paths, reducing tokens for deeply nested data.
+
+### Basic Folding
+
+Standard nesting:
+
+```yaml
+data:
+  metadata:
+    items[2]: a,b
+```
+
+With key folding (`keyFolding: 'safe'`):
+
+```yaml
+data.metadata.items[2]: a,b
+```
+
+The three nested objects collapse into a single dotted key `data.metadata.items`.
+
+### When Folding Applies
+
+A chain of objects is foldable when:
+- Each object in the chain has exactly one key (leading to the next object or a leaf value)
+- The leaf value is a primitive, array, or empty object
+- All segments are valid identifier segments (letters, digits, underscores only; no dots within segments)
+- The resulting folded key doesn't collide with existing keys
+
+::: details Advanced Folding Rules
+**Segment Requirements (safe mode):**
+- All folded segments must match `^[A-Za-z_][A-Za-z0-9_]*$` (no dots, hyphens, or other special characters)
+- No segment may require quoting per §7.3 of the spec
+- The resulting folded key must not equal any existing sibling literal key at the same depth (collision avoidance)
+
+**Depth Limit:**
+- The `flattenDepth` option (default: `Infinity`) controls how many segments to fold
+- `flattenDepth: 2` folds only two-segment chains: `{a: {b: val}}` → `a.b: val`
+- Values less than 2 have no practical effect
+
+**Round-Trip with Path Expansion:**
+To reconstruct the original structure when decoding, use `expandPaths: 'safe'`. This splits dotted keys back into nested objects using the same safety rules ([spec §13.4](https://github.com/toon-format/spec/blob/main/SPEC.md#134-key-folding-and-path-expansion)).
+:::
+
+### Round-Trip with Path Expansion
+
+When decoding TOON that used key folding, enable path expansion to restore the nested structure:
+
+```ts
+import { decode, encode } from '@toon-format/toon'
+
+const original = { data: { metadata: { items: ['a', 'b'] } } }
+
+// Encode with folding
+const toon = encode(original, { keyFolding: 'safe' })
+// → "data.metadata.items[2]: a,b"
+
+// Decode with expansion
+const restored = decode(toon, { expandPaths: 'safe' })
+// → { data: { metadata: { items: ['a', 'b'] } } }
+```
+
+Path expansion is off by default, so dotted keys are treated as literal keys unless explicitly enabled.
+
+## Quoting and Types
+
+### When Strings Need Quotes
+
+TOON quotes strings **only when necessary** to maximize token efficiency. A string must be quoted if:
+
+- It's empty (`""`)
+- It has leading or trailing whitespace
+- It equals `true`, `false`, or `null` (case-sensitive)
+- It looks like a number (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
+- It contains special characters: colon (`:`), quote (`"`), backslash (`\`), brackets, braces, or any control character in U+0000–U+001F
+- It contains the relevant delimiter (the active delimiter inside an array scope, or the document delimiter elsewhere)
+- It equals `"-"` or starts with `"-"` followed by any character
+
+Otherwise, strings can be unquoted. Unicode, emoji, and strings with internal (non-leading/trailing) spaces are safe unquoted:
+
+```yaml
+message: Hello 世界 👋
+note: This has inner spaces
+```
+
+### Escape Sequences
+
+In quoted strings and keys, six escape sequences are valid:
+
+| Character | Escape |
+|-----------|--------|
+| Backslash (`\`) | `\\` |
+| Double quote (`"`) | `\"` |
+| Newline (U+000A) | `\n` |
+| Carriage return (U+000D) | `\r` |
+| Tab (U+0009) | `\t` |
+| Any other U+0000–U+001F control character | `\uXXXX` |
+
+Other escapes (e.g., `\x`, `\0`, `\b`) are always rejected, as are lone-surrogate `\uXXXX` values (U+D800–U+DFFF).
+
+### Type Conversions
+
+Numbers are emitted in canonical decimal form for values in the §2 carve-out range; exponent notation is permitted outside. Non-JSON types (`NaN`, `Infinity`, `BigInt`, `Date`, `Set`, `Map`, `undefined`, etc.) are normalized before encoding – see [API Reference – Type Normalization](/reference/api#type-normalization) for the full mapping.
+
+Decoders accept both decimal and exponent forms on input (e.g., `42`, `-3.14`, `1e-6`), and treat tokens with forbidden leading zeros (e.g., `"05"`) as strings, not numbers.
+
+### Custom Serialization with toJSON
+
+Objects with a `toJSON()` method are serialized by calling the method and normalizing its result before encoding, similar to `JSON.stringify`:
+
+```ts
+const obj = {
+  data: 'example',
+  toJSON() {
+    return { info: this.data }
+  }
+}
+
+encode(obj)
+// info: example
+```
+
+The `toJSON()` method:
+
+- Takes precedence over built-in normalization (Date, Array, Set, Map)
+- Results are recursively normalized
+- Is called for objects with `toJSON` in their prototype chain
+
+---
+
+For complete rules on quoting, escaping, type conversions, and strict-mode decoding, see [spec §2–4 (data model), §7 (strings and keys), and §14 (strict mode)](https://github.com/toon-format/spec/blob/main/SPEC.md).

docs/guide/getting-started.md

@@ -0,0 +1,248 @@
+---
+description: What TOON is, when to use it, and a first encode/decode example with the TypeScript library.
+---
+
+# Getting Started
+
+## What Is TOON?
+
+**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It is intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
+
+TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably.
+
+Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input.
+
+### Why TOON?
+
+Standard JSON is verbose and token-expensive. For uniform arrays of objects, JSON repeats every field name for every record:
+
+```json
+{
+  "users": [
+    { "id": 1, "name": "Alice", "role": "admin" },
+    { "id": 2, "name": "Bob", "role": "user" }
+  ]
+}
+```
+
+YAML already reduces some redundancy with indentation instead of braces:
+
+```yaml
+users:
+  - id: 1
+    name: Alice
+    role: admin
+  - id: 2
+    name: Bob
+    role: user
+```
+
+TOON goes further by declaring fields once and streaming data as rows:
+
+```yaml
+users[2]{id,name,role}:
+  1,Alice,admin
+  2,Bob,user
+```
+
+The `[2]` declares the array length, letting LLMs answer dataset-size questions and detect truncation. The `{id,name,role}` declares the field names. Each row is a compact, comma-separated list of values. The pattern is the same throughout TOON: declare structure once, stream data compactly. The result lands close to CSV density with explicit structure preserved.
+
+For a more realistic example, here's how TOON handles a dataset with both nested objects and tabular arrays:
+
+::: code-group
+
+```json [JSON (235 tokens)]
+{
+  "context": {
+    "task": "Our favorite hikes together",
+    "location": "Boulder",
+    "season": "spring_2025"
+  },
+  "friends": ["ana", "luis", "sam"],
+  "hikes": [
+    {
+      "id": 1,
+      "name": "Blue Lake Trail",
+      "distanceKm": 7.5,
+      "elevationGain": 320,
+      "companion": "ana",
+      "wasSunny": true
+    },
+    {
+      "id": 2,
+      "name": "Ridge Overlook",
+      "distanceKm": 9.2,
+      "elevationGain": 540,
+      "companion": "luis",
+      "wasSunny": false
+    },
+    {
+      "id": 3,
+      "name": "Wildflower Loop",
+      "distanceKm": 5.1,
+      "elevationGain": 180,
+      "companion": "sam",
+      "wasSunny": true
+    }
+  ]
+}
+```
+
+```yaml [TOON (106 tokens)]
+context:
+  task: Our favorite hikes together
+  location: Boulder
+  season: spring_2025
+friends[3]: ana,luis,sam
+hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
+  1,Blue Lake Trail,7.5,320,ana,true
+  2,Ridge Overlook,9.2,540,luis,false
+  3,Wildflower Loop,5.1,180,sam,true
+```
+
+:::
+
+Notice how TOON combines YAML's indentation for the `context` object with inline format for the primitive `friends` array and tabular format for the structured `hikes` array. Each format is chosen automatically based on the data structure.
+
+### Design Goals
+
+TOON is optimized for specific use cases. It aims to:
+
+- Make uniform arrays of objects as compact as possible by declaring structure once and streaming data.
+- Stay fully lossless and deterministic – round-trips preserve all data and structure.
+- Keep parsing simple and robust for both LLMs and humans through explicit structure markers.
+- Provide validation guardrails (array lengths, field counts) that help detect truncation and malformed output.
+
+## When to Use TOON
+
+TOON excels with uniform arrays of objects – data with the same structure across items. For LLM prompts, the format produces deterministic, minimally quoted text with built-in validation. Explicit array lengths (`[N]`) and field headers (`{fields}`) help detect truncation and malformed data, while the tabular structure declares fields once rather than repeating them in every row.
+
+::: tip
+The TOON format is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to the [spec](https://github.com/toon-format/spec) or sharing feedback.
+:::
+
+## When Not to Use TOON
+
+TOON is not always the best choice. Consider alternatives when:
+
+- **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels.
+- **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it.
+- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5–10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
+- **Latency-critical applications**: Benchmark on your exact setup. Some deployments (especially local/quantized models) may process compact JSON faster despite TOON's lower token count.
+
+::: info
+For data-driven comparisons across different structures, see [Benchmarks](/guide/benchmarks). When optimizing for latency, measure TTFT, tokens/sec, and total time for both TOON and JSON-compact, and use whichever is faster in your specific environment.
+:::
+
+## Installation
+
+### TypeScript Library
+
+Install the library via your preferred package manager:
+
+::: code-group
+
+```bash [npm]
+npm install @toon-format/toon
+```
+
+```bash [pnpm]
+pnpm add @toon-format/toon
+```
+
+```bash [yarn]
+yarn add @toon-format/toon
+```
+
+:::
+
+### CLI
+
+The CLI can be used without installation via `npx`, or installed globally:
+
+::: code-group
+
+```bash [npx (no install)]
+npx @toon-format/cli input.json -o output.toon
+```
+
+```bash [npm]
+npm install -g @toon-format/cli
+```
+
+```bash [pnpm]
+pnpm add -g @toon-format/cli
+```
+
+```bash [yarn]
+yarn global add @toon-format/cli
+```
+
+:::
+
+For full CLI documentation, see the [CLI reference](/cli/).
+
+## Media Type & File Extension
+
+TOON files conventionally use the `.toon` extension. For HTTP transmission, the provisional media type is `text/toon`, always with UTF-8 encoding. While you may specify `charset=utf-8` explicitly, it's optional – UTF-8 is the default assumption. This follows the registration process outlined in [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations).
+
+## Your First Example
+
+The examples below use the TypeScript library for demonstration, but the same operations work in any language with a TOON implementation.
+
+Let's encode a simple dataset with the TypeScript library:
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const data = {
+  users: [
+    { id: 1, name: 'Alice', role: 'admin' },
+    { id: 2, name: 'Bob', role: 'user' }
+  ]
+}
+
+console.log(encode(data))
+```
+
+**Output:**
+
+```yaml
+users[2]{id,name,role}:
+  1,Alice,admin
+  2,Bob,user
+```
+
+### Decoding Back to JSON
+
+Decoding is just as simple:
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const toon = `
+users[2]{id,name,role}:
+  1,Alice,admin
+  2,Bob,user
+`
+
+const data = decode(toon)
+console.log(JSON.stringify(data, null, 2))
+```
+
+**Output:**
+
+```json
+{
+  "users": [
+    { "id": 1, "name": "Alice", "role": "admin" },
+    { "id": 2, "name": "Bob", "role": "user" }
+  ]
+}
+```
+
+Round-tripping is lossless: `decode(encode(x))` always equals `x` (after normalization of non-JSON types like `Date`, `NaN`, etc.).
+
+## Where to Go Next
+
+Now that you've seen your first TOON document, read the [Format Overview](/guide/format-overview) for complete syntax details (objects, arrays, quoting rules, key folding), then explore [Using TOON with LLMs](/guide/llm-prompts) to see how to use it effectively in prompts. For implementation details, check the [API Reference](/reference/api) (TypeScript) or the [Specification](/reference/spec) (language-agnostic normative rules).

docs/guide/llm-prompts.md

@@ -0,0 +1,191 @@
+---
+description: Prompting strategies for sending TOON to LLMs and validating TOON they generate, with examples.
+---
+
+# Using TOON with LLMs
+
+TOON is designed for passing structured data to Large Language Models with reduced token costs and improved reliability. This guide shows how to use TOON effectively in prompts, both for input (sending data to models) and output (getting models to generate TOON).
+
+This guide is about the TOON format itself. Code examples use the TypeScript library for demonstration, but the same patterns and techniques apply regardless of which programming language you're using.
+
+## Why TOON for LLMs
+
+LLM tokens cost money, and JSON is verbose – repeating every field name for every record in an array. TOON minimizes tokens especially for uniform arrays by declaring fields once and streaming data as rows, typically saving 30–60% compared to formatted JSON.
+
+TOON adds structure guardrails: explicit `[N]` lengths and `{fields}` headers make it easier for models to track rows and for you to validate output. Strict mode helps detect truncation and malformed TOON when decoding model responses.
+
+## Sending TOON as Input
+
+TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern.
+
+Wrap your encoded data in a fenced code block (label it ` ```toon` for clarity):
+
+````md
+Data is in TOON format (2-space indent, arrays show length and fields).
+
+```toon
+users[3]{id,name,role,lastLogin}:
+  1,Alice,admin,"2025-01-15T10:30:00Z"
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
+```
+
+Task: Summarize the user roles and their last activity.
+````
+
+The indentation and headers are usually enough – models treat TOON like familiar YAML or CSV. The explicit array lengths (`[N]`) and field headers (`{fields}`) help the model track structure, especially for large tables.
+
+> [!NOTE]
+> Most models don't have built-in TOON syntax highlighting, so ` ```toon` or ` ```yaml` both work fine. The structure is what matters.
+
+## Generating TOON from LLMs
+
+For output, be more explicit. When you want the model to **generate** TOON:
+
+- **Show the expected header** (e.g., `users[N]{id,name,role}:`). The model fills rows instead of repeating keys, reducing generation errors.
+- **State the rules**: 2-space indent, no trailing spaces, `[N]` matches row count.
+
+Here's a prompt that works for both reading and generating:
+
+````md
+Data is in TOON format (2-space indent, arrays show length and fields).
+
+```toon
+users[3]{id,name,role,lastLogin}:
+  1,Alice,admin,"2025-01-15T10:30:00Z"
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
+```
+
+Task: Return only users with role "user" as TOON. Use the same header format. Set [N] to match the row count. Output only the code block.
+````
+
+**Expected output:**
+
+```toon
+users[2]{id,name,role,lastLogin}:
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
+```
+
+The model adjusts `[N]` to `2` and generates two rows.
+
+### Validation with Strict Mode
+
+When decoding model-generated TOON, use strict mode (default) to catch errors:
+
+```ts
+import { decode } from '@toon-format/toon'
+
+try {
+  const data = decode(modelOutput, { strict: true })
+  // Success – data is valid
+}
+catch (error) {
+  // Model output was malformed (count mismatch, invalid escapes, etc.)
+  console.error('Validation failed:', error.message)
+}
+```
+
+Strict mode checks counts, indentation, and escaping so you can detect truncation or malformed TOON. For complete details, see the [API Reference](/reference/api#decode-input-options).
+
+## Delimiter Choices for Token Efficiency
+
+Use `delimiter: '\t'` for tab-separated tables if you want even fewer tokens. Tabs are single characters, often tokenize more efficiently than commas, and rarely appear in natural text (reducing quote-escaping).
+
+```ts
+const toon = encode(data, { delimiter: '\t' })
+```
+
+Tell the model "fields are tab-separated" when using tabs. For more on delimiters, see the [Format Overview](/guide/format-overview#delimiter-options).
+
+## Streaming Large Outputs
+
+When working with large datasets (thousands of records or deeply nested structures), use `encodeLines()` to stream TOON output line-by-line instead of building the full string in memory.
+
+```ts
+import { encodeLines } from '@toon-format/toon'
+
+const largeData = await fetchThousandsOfRecords()
+
+// Stream large dataset without loading full string in memory
+for (const line of encodeLines(largeData, { delimiter: '\t' })) {
+  process.stdout.write(`${line}\n`)
+}
+```
+
+The CLI also supports streaming for memory-efficient JSON-to-TOON conversion:
+
+```bash
+toon large-dataset.json -o output.toon
+```
+
+This streaming approach prevents out-of-memory errors when preparing large context windows for LLMs. For complete details on `encodeLines()`, see the [API Reference](/reference/api#encodelines-input-options).
+
+**Consuming streaming LLM outputs:** If your LLM client exposes streaming text and you buffer by lines, you can decode TOON incrementally:
+
+```ts
+import { decodeFromLines } from '@toon-format/toon'
+
+// Buffer streaming response into lines
+const lines: string[] = []
+let buffer = ''
+
+for await (const chunk of modelStream) {
+  buffer += chunk
+  let index: number
+
+  while ((index = buffer.indexOf('\n')) !== -1) {
+    lines.push(buffer.slice(0, index))
+    buffer = buffer.slice(index + 1)
+  }
+}
+
+// Decode buffered lines
+const data = decodeFromLines(lines)
+```
+
+For streaming decode APIs, see [`decodeFromLines()`](/reference/api#decodefromlines-lines-options) and [`decodeStream()`](/reference/api#decodestream-source-options).
+
+## Tips and Pitfalls
+
+**Show, don't describe.** Don't explain TOON syntax in detail – just show an example. Models learn the pattern from context. A simple code block with 2–5 rows is more effective than paragraphs of explanation.
+
+**Keep examples small.** Use 2–5 rows in your examples, not hundreds. The model generalizes from the pattern. Large examples waste tokens without improving accuracy.
+
+**Always validate output.** Decode generated TOON with `strict: true` (default) to catch errors early. Don't assume model output is valid TOON without checking.
+
+## Real-World Example
+
+Here's a complete workflow: send data to a model and validate its TOON response.
+
+**Prompt with TOON input:**
+
+````md
+System logs in TOON format (tab-separated):
+
+```toon
+events[4	]{id	level	message	timestamp}:
+  1	error	Connection timeout	"2025-01-15T10:00:00Z"
+  2	warn	Slow query	"2025-01-15T10:05:00Z"
+  3	info	User login	"2025-01-15T10:10:00Z"
+  4	error	Database error	"2025-01-15T10:15:00Z"
+```
+
+Task: Return only error-level events as TOON. Use the same format.
+````
+
+**Validate the response:**
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const modelResponse = `
+events[2	]{id	level	message	timestamp}:
+  1	error	Connection timeout	"2025-01-15T10:00:00Z"
+  4	error	Database error	"2025-01-15T10:15:00Z"
+`
+
+const filtered = decode(modelResponse, { strict: true })
+// ✓ Validated – model correctly filtered and adjusted [N] to 2
+```

docs/index.md

@@ -0,0 +1,51 @@
+---
+layout: home
+titleTemplate: Token-Oriented Object Notation
+hero:
+  name: TOON
+  text: Token-Oriented Object Notation
+  tagline: A compact, human-readable encoding of the JSON data model for LLM prompts.
+  image:
+    dark: /logo-index-dark.svg
+    light: /logo-index-light.svg
+    alt: TOON Logo
+  actions:
+    - theme: brand
+      text: What is TOON?
+      link: /guide/getting-started
+    - theme: alt
+      text: Benchmarks
+      link: /guide/benchmarks
+    - theme: alt
+      text: Playground
+      link: /playground
+    - theme: alt
+      text: CLI
+      link: /cli/
+
+features:
+  - title: Token-Efficient & Accurate
+    icon: 📊
+    details: TOON reaches 76.4% accuracy (vs JSON's 75.0%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
+    link: /guide/benchmarks
+  - title: JSON Data Model
+    icon: 🔁
+    details: Encodes the same objects, arrays, and primitives as JSON with deterministic, lossless round-trips.
+    link: /guide/format-overview
+  - title: LLM-Friendly Guardrails
+    icon: 🛤️
+    details: Explicit [N] lengths and {fields} headers give models a clear schema to follow, improving parsing reliability.
+    link: /guide/format-overview#arrays
+  - title: Minimal Syntax
+    icon: 📐
+    details: Uses indentation instead of braces and minimizes quoting, giving YAML-like readability with CSV-style compactness.
+    link: /guide/format-overview#arrays
+  - title: Tabular Arrays
+    icon: 🧺
+    details: Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
+    link: /guide/format-overview#arrays
+  - title: Multi-Language Ecosystem
+    icon: 🌐
+    details: Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
+    link: /ecosystem/implementations
+---

docs/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@toon-format/docs",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "dev": "vitepress dev",
+    "build": "vitepress build",
+    "preview": "vitepress preview"
+  },
+  "devDependencies": {
+    "@vueuse/core": "^14.3.0",
+    "fflate": "^0.8.3",
+    "gpt-tokenizer": "^3.4.0",
+    "markdown-it-mathjax3": "^4.3.2",
+    "uint8array-extras": "^1.5.0",
+    "unocss": "^66.7.5",
+    "vitepress": "^1.6.4",
+    "vitepress-plugin-llms": "^1.13.3",
+    "yaml": "^2.9.0"
+  }
+}

docs/playground.md

@@ -0,0 +1,4 @@
+---
+layout: PlaygroundLayout
+title: Playground
+---

docs/public/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 180 180"><g clip-path="url(#a)"><path fill="#fef3c0" d="M0 59.76C0 10.548 10.548 0 59.76 0h60.48C169.452 0 180 10.548 180 59.76v60.48c0 49.212-10.548 59.76-59.76 59.76H59.76C10.548 180 0 169.452 0 120.24z"/><path fill="#fff" d="M120 40h20v20h-20z"/><path fill="#1b1b1f" d="M160 80h-60V20h60zm-40-20h20V40h-20zM140 100v20h-20v40h-20v-60zm20 60h-20v-40h20z"/><path fill="#fff" d="M40 120h20v20H40z"/><path fill="#1b1b1f" d="M80 160H20v-60h60zm-40-20h20v-20H40zM60 80H40V40H20V20h60v20H60z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h180v180H0z"/></clipPath></defs></svg>

docs/public/logo-index-dark.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 180 180"><g fill="#fff" clip-path="url(#a)"><path d="M180 77.143h-77.143V0H180zm-51.429-25.714h25.715V25.714h-25.715zM154.286 102.857v25.714h-25.715V180h-25.714v-77.143zM180 180h-25.714v-51.429H180zM77.143 180H0v-77.143h77.143zm-51.429-25.714h25.715v-25.715H25.714zM51.429 77.143H25.714V25.714H0V0h77.143v25.714H51.429z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h180v180H0z"/></clipPath></defs></svg>

docs/public/logo-index-light.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 180 180"><g fill="#1b1b1f" clip-path="url(#a)"><path d="M180 77.143h-77.143V0H180zm-51.429-25.714h25.715V25.714h-25.715zM154.286 102.857v25.714h-25.715V180h-25.714v-77.143zM180 180h-25.714v-51.429H180zM77.143 180H0v-77.143h77.143zm-51.429-25.714h25.715v-25.715H25.714zM51.429 77.143H25.714V25.714H0V0h77.143v25.714H51.429z"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h180v180H0z"/></clipPath></defs></svg>

docs/public/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 180 180"><path fill="#fef3c0" d="M0 0h180v180H0z"/><path fill="#fff" d="M120 40h20v20h-20z"/><path fill="#1b1b1f" d="M160 80h-60V20h60zm-40-20h20V40h-20zM140 100v20h-20v40h-20v-60zm20 60h-20v-40h20z"/><path fill="#fff" d="M40 120h20v20H40z"/><path fill="#1b1b1f" d="M80 160H20v-60h60zm-40-20h20v-20H40zM60 80H40V40H20V20h60v20H60z"/></svg>

docs/reference/api.md

@@ -0,0 +1,766 @@
+---
+description: TypeScript and JavaScript encode and decode functions, options, error types, and streaming decoders for @toon-format/toon.
+---
+
+# API Reference
+
+TypeScript/JavaScript API documentation for the `@toon-format/toon` package. For format rules, see the [Format Overview](/guide/format-overview) or the [Specification](/reference/spec). For other languages, see [Implementations](/ecosystem/implementations).
+
+## Installation
+
+::: code-group
+
+```bash [npm]
+npm install @toon-format/toon
+```
+
+```bash [pnpm]
+pnpm add @toon-format/toon
+```
+
+```bash [yarn]
+yarn add @toon-format/toon
+```
+
+:::
+
+## Encoding Functions
+
+### `encode(input, options?)`
+
+Converts any JSON-serializable value to TOON format.
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const toon = encode(data, {
+  indent: 2,
+  delimiter: ',',
+  keyFolding: 'off',
+  flattenDepth: Infinity
+})
+```
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `input` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
+| `options` | `EncodeOptions?` | Optional encoding options (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns a TOON-formatted string with no trailing newline or spaces.
+
+#### Type Normalization
+
+Non-JSON-serializable values are normalized before encoding:
+
+| Input | Output |
+|-------|--------|
+| `Object` with `toJSON()` method | Result of calling `toJSON()`, recursively normalized |
+| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal (e.g., `1e6` → `1000000`, `-0` → `0`) |
+| Finite number outside that range | Exponent form permitted (e.g., `1e-7`, `1e+21`) |
+| `NaN`, `Infinity`, `-Infinity` | `null` |
+| `BigInt` (within safe range) | Number |
+| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
+| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
+| `Set` | Array of normalized values |
+| `Map` | Object with `String(key)` keys |
+| `undefined`, `function`, `symbol` | `null` |
+
+::: info
+TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
+:::
+
+#### Example
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const items = [
+  { sku: 'A1', qty: 2, price: 9.99 },
+  { sku: 'B2', qty: 1, price: 14.5 }
+]
+
+console.log(encode({ items }))
+```
+
+**Output:**
+
+```yaml
+items[2]{sku,qty,price}:
+  A1,2,9.99
+  B2,1,14.5
+```
+
+### `encodeLines(input, options?)`
+
+**Preferred method for streaming TOON output.** Converts any JSON-serializable value to TOON format as a sequence of lines, without building the full string in memory. Suitable for streaming large outputs to files, HTTP responses, or process stdout.
+
+```ts
+import { encodeLines } from '@toon-format/toon'
+
+// Stream to stdout (Node.js)
+for (const line of encodeLines(data)) {
+  process.stdout.write(`${line}\n`)
+}
+
+// Write to file line-by-line
+const lines = encodeLines(data, { indent: 2, delimiter: '\t' })
+for (const line of lines) {
+  await writeToStream(`${line}\n`)
+}
+
+// Collect to array
+const lineArray = Array.from(encodeLines(data))
+```
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `input` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
+| `options` | `EncodeOptions?` | Optional encoding options (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns an `Iterable<string>` that yields TOON lines one at a time. **Each yielded string is a single line without a trailing newline character** – you must add `\n` when writing to streams or stdout.
+
+::: info Relationship to `encode()`
+`encode(value, options)` is equivalent to:
+```ts
+Array.from(encodeLines(value, options)).join('\n')
+```
+:::
+
+#### Example
+
+```ts
+import { createWriteStream } from 'node:fs'
+import { encodeLines } from '@toon-format/toon'
+
+const data = {
+  items: Array.from({ length: 100000 }, (_, i) => ({
+    id: i,
+    name: `Item ${i}`,
+    value: Math.random()
+  }))
+}
+
+// Stream large dataset to file
+const stream = createWriteStream('output.toon')
+for (const line of encodeLines(data, { delimiter: '\t' })) {
+  stream.write(`${line}\n`)
+}
+stream.end()
+```
+
+### Replacer Function
+
+The `replacer` option allows you to transform or filter values during encoding. It works similarly to `JSON.stringify`'s replacer parameter, but with path tracking for more precise control.
+
+#### Type Signature
+
+```typescript
+type EncodeReplacer = (
+  key: string,
+  value: JsonValue,
+  path: readonly (string | number)[]
+) => unknown
+```
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `key` | `string` | Property name, array index (as string), or empty string for root |
+| `value` | `JsonValue` | The normalized value at this location |
+| `path` | `readonly (string \| number)[]` | Path from root to current value |
+
+#### Return Value
+
+- Return the value unchanged to keep it
+- Return a different value to replace it (will be normalized)
+- Return `undefined` to omit properties/array elements
+- For root value, `undefined` means "no change" (root cannot be omitted)
+
+#### Examples
+
+**Filtering sensitive data:**
+
+```typescript
+import { encode } from '@toon-format/toon'
+
+const data = {
+  user: { name: 'Alice', password: 'secret123', email: 'alice@example.com' }
+}
+
+function replacer(key, value) {
+  if (key === 'password')
+    return undefined
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+user:
+  name: Alice
+  email: alice@example.com
+```
+
+**Transforming values:**
+
+```typescript
+const data = { user: 'alice', role: 'admin' }
+
+function replacer(key, value) {
+  if (typeof value === 'string')
+    return value.toUpperCase()
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+user: ALICE
+role: ADMIN
+```
+
+**Path-based transformations:**
+
+```typescript
+const data = {
+  metadata: { created: '2025-01-01' },
+  user: { created: '2025-01-02' }
+}
+
+function replacer(key, value, path) {
+  // Add timezone info only to top-level metadata
+  if (path.length === 1 && path[0] === 'metadata' && key === 'created') {
+    return `${value}T00:00:00Z`
+  }
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+metadata:
+  created: "2025-01-01T00:00:00Z"
+user:
+  created: 2025-01-02
+```
+
+::: info Replacer Execution Order
+The replacer is called in a depth-first manner:
+1. Root value first (key = `''`, path = `[]`)
+2. Then each property/element (with proper key and path)
+3. Values are re-normalized after replacement
+4. Children are processed after parent transformation
+:::
+
+::: warning Array Indices as Strings
+Following `JSON.stringify` behavior, array indices are passed as strings (`'0'`, `'1'`, `'2'`, etc.) to the replacer, not as numbers.
+:::
+
+## Decoding Functions
+
+### `decode(input, options?)`
+
+Converts a TOON-formatted string back to JavaScript values.
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const data = decode(toon, {
+  indent: 2,
+  strict: true,
+  expandPaths: 'off'
+})
+```
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `input` | `string` | A TOON-formatted string to parse |
+| `options` | `DecodeOptions?` | Optional decoding options (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns a JavaScript value (object, array, or primitive) representing the parsed TOON data.
+
+#### Example
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const toon = `
+items[2]{sku,qty,price}:
+  A1,2,9.99
+  B2,1,14.5
+`
+
+const data = decode(toon)
+console.log(data)
+```
+
+**Output:**
+
+```json
+{
+  "items": [
+    { "sku": "A1", "qty": 2, "price": 9.99 },
+    { "sku": "B2", "qty": 1, "price": 14.5 }
+  ]
+}
+```
+
+### `decodeFromLines(lines, options?)`
+
+Decodes TOON format from pre-split lines into a JavaScript value. This is a streaming-friendly wrapper around the event-based decoder that builds the full value in memory.
+
+Useful when you already have lines as an array or iterable (e.g., from file streams, readline interfaces, or network responses) and want the standard decode behavior with path expansion support.
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `lines` | `Iterable<string>` | Iterable of TOON lines (without trailing newlines) |
+| `options` | `DecodeOptions?` | Optional decoding configuration (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns a `JsonValue` (the parsed JavaScript value: object, array, or primitive).
+
+#### Example
+
+**Basic usage with arrays:**
+
+```ts
+import { decodeFromLines } from '@toon-format/toon'
+
+const lines = ['name: Alice', 'age: 30']
+const value = decodeFromLines(lines)
+// { name: 'Alice', age: 30 }
+```
+
+**Streaming from Node.js readline:**
+
+```ts
+import { createReadStream } from 'node:fs'
+import { createInterface } from 'node:readline'
+import { decodeFromLines } from '@toon-format/toon'
+
+const rl = createInterface({
+  input: createReadStream('data.toon'),
+  crlfDelay: Infinity,
+})
+
+const value = decodeFromLines(rl)
+console.log(value)
+```
+
+**With path expansion:**
+
+```ts
+const lines = ['user.name: Alice', 'user.age: 30']
+const value = decodeFromLines(lines, { expandPaths: 'safe' })
+// { user: { name: 'Alice', age: 30 } }
+```
+
+### Choosing the Right Decoder
+
+| Function | Input | Output | Async | Path Expansion | Use When |
+|----------|-------|--------|-------|----------------|----------|
+| `decode()` | String | Value | No | Yes | You have a complete TOON string |
+| `decodeFromLines()` | Lines | Value | No | Yes | You have lines and want the full value |
+| `decodeStreamSync()` | Lines | Events | No | No | You need event-by-event processing (sync) |
+| `decodeStream()` | Lines | Events | Yes | No | You need event-by-event processing (async) |
+
+::: info Key Differences
+- **Value vs. Events**: Functions ending in `Stream` yield events without building the full value in memory.
+- **Path expansion**: Only `decode()` and `decodeFromLines()` support `expandPaths: 'safe'`.
+- **Async support**: Only `decodeStream()` accepts async iterables (useful for file/network streams).
+:::
+
+## Streaming Decoders
+
+### `decodeStreamSync(lines, options?)`
+
+Synchronously decodes TOON lines into a stream of JSON events. This function yields structured events that represent the JSON data model without building the full value tree.
+
+Useful for streaming processing, custom transformations, or memory-efficient parsing of large datasets where you don't need the full value in memory.
+
+::: tip Event Streaming
+This is a low-level API that returns individual parse events. For most use cases, [`decodeFromLines()`](#decodefromlines-lines-options) or [`decode()`](#decode-input-options) are more convenient.
+
+Path expansion (`expandPaths: 'safe'`) is **not supported** in streaming mode since it requires the full value tree.
+:::
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `lines` | `Iterable<string>` | Iterable of TOON lines (without trailing newlines) |
+| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns an `Iterable<JsonStreamEvent>` that yields structured events (see [TypeScript Types](#typescript-types) for event structure).
+
+#### Example
+
+**Basic event streaming:**
+
+```ts
+import { decodeStreamSync } from '@toon-format/toon'
+
+const lines = ['name: Alice', 'age: 30']
+
+for (const event of decodeStreamSync(lines)) {
+  console.log(event)
+}
+
+// Output:
+// { type: 'startObject' }
+// { type: 'key', key: 'name' }
+// { type: 'primitive', value: 'Alice' }
+// { type: 'key', key: 'age' }
+// { type: 'primitive', value: 30 }
+// { type: 'endObject' }
+```
+
+**Custom processing:**
+
+```ts
+import { decodeStreamSync } from '@toon-format/toon'
+
+const lines = ['users[2]{id,name}:', '  1,Alice', '  2,Bob']
+let userCount = 0
+
+for (const event of decodeStreamSync(lines)) {
+  if (event.type === 'endObject' && userCount < 2) {
+    userCount++
+    console.log(`Processed user ${userCount}`)
+  }
+}
+```
+
+### `decodeStream(source, options?)`
+
+Asynchronously decodes TOON lines into a stream of JSON events. This is the async version of [`decodeStreamSync()`](#decodestreamsync-lines-options), supporting both synchronous and asynchronous iterables.
+
+Useful for processing file streams, network responses, or other async sources where you want to handle data incrementally as it arrives.
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `source` | `AsyncIterable<string>` \| `Iterable<string>` | Async or sync iterable of TOON lines (without trailing newlines) |
+| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see [Configuration Reference](#configuration-reference)) |
+
+#### Return Value
+
+Returns an `AsyncIterable<JsonStreamEvent>` that yields structured events asynchronously (see [TypeScript Types](#typescript-types) for event structure).
+
+#### Example
+
+**Streaming from file:**
+
+```ts
+import { createReadStream } from 'node:fs'
+import { createInterface } from 'node:readline'
+import { decodeStream } from '@toon-format/toon'
+
+const fileStream = createReadStream('data.toon', 'utf-8')
+const rl = createInterface({ input: fileStream, crlfDelay: Infinity })
+
+for await (const event of decodeStream(rl)) {
+  console.log(event)
+  // Process events as they arrive
+}
+```
+
+## Error Handling
+
+Decoding throws a `ToonDecodeError` when input cannot be parsed. The class extends `SyntaxError`, so existing `error instanceof SyntaxError` checks keep working without code changes.
+
+### `ToonDecodeError`
+
+```ts
+import { ToonDecodeError } from '@toon-format/toon'
+```
+
+#### Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `name` | `'ToonDecodeError'` | Discriminator – `error.name === 'ToonDecodeError'` |
+| `message` | `string` | Human-readable message; prefixed with `Line N: ` when a line is known |
+| `line` | `number?` | 1-based line number where the error was detected |
+| `source` | `string?` | Raw source line (including its leading whitespace) |
+| `cause` | `unknown?` | The original error when the decoder enriched a lower-level parser failure |
+
+The `line` and `source` fields are populated for every error that has line context – essentially every parse error during normal decoding. The `cause` chain points back to the underlying `SyntaxError` or `TypeError` thrown by the token-level parser, so debuggers and verbose loggers can show the original frame.
+
+#### Example
+
+```ts
+import { decode, ToonDecodeError } from '@toon-format/toon'
+
+try {
+  decode('a:\n\tb: 1')
+}
+catch (error) {
+  if (error instanceof ToonDecodeError) {
+    console.error(`Line ${error.line}:`, error.source)
+    console.error(error.message)
+    // Line 2: 	b: 1
+    // Line 2: Tabs are not allowed in indentation in strict mode
+  }
+  else {
+    throw error
+  }
+}
+```
+
+::: info Backwards Compatibility
+`ToonDecodeError` extends `SyntaxError`. Code written against earlier versions that catches `SyntaxError` continues to match these errors. The class adds structured fields without removing anything.
+:::
+
+## Configuration Reference
+
+### `EncodeOptions`
+
+Configuration for [`encode()`](#encode-input-options) and [`encodeLines()`](#encodelines-input-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Number of spaces per indentation level |
+| `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows |
+| `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths |
+| `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) |
+| `replacer` | `EncodeReplacer` | `undefined` | Optional hook to transform or omit values before encoding (see [Replacer Function](#replacer-function)) |
+
+**Delimiter options:**
+
+::: code-group
+
+```ts [Comma (default)]
+encode(data, { delimiter: ',' })
+```
+
+```ts [Tab]
+encode(data, { delimiter: '\t' })
+```
+
+```ts [Pipe]
+encode(data, { delimiter: '|' })
+```
+
+:::
+
+See [Delimiter Strategies](#delimiter-strategies) for guidance on choosing delimiters.
+
+### `DecodeOptions`
+
+Configuration for [`decode()`](#decode-input-options) and [`decodeFromLines()`](#decodefromlines-lines-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Expected number of spaces per indentation level |
+| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+| `expandPaths` | `'off'` \| `'safe'` | `'off'` | Enable path expansion to reconstruct dotted keys into nested objects (pairs with `keyFolding: 'safe'`) |
+
+By default (`strict: true`), the decoder validates input strictly:
+
+- **Invalid escape sequences**: Throws on `\x`, unterminated strings, lone-surrogate `\uXXXX`
+- **Syntax errors**: Throws on missing colons, malformed headers
+- **Array length mismatches**: Throws when declared length doesn't match actual count
+- **Header delimiter mismatch**: Throws when the bracket-declared delimiter differs from the field-list delimiter (§14.2)
+- **Indentation errors**: Throws when leading spaces aren't exact multiples of `indent`
+- **Header structure**: Throws on leading-zero or non-integer array lengths and on intervening content between bracket/fields/colon
+- **Duplicate sibling keys**: Throws when an object has two children with the same key (§14.4)
+- **Path-expansion conflicts**: When `expandPaths: 'safe'` is set, throws on overlapping dotted paths that would collide
+
+All decode errors are thrown as [`ToonDecodeError`](#error-handling) instances with structured `line` and `source` fields.
+
+Set `strict: false` to skip these checks. Duplicate sibling keys and path-expansion conflicts then resolve with last-write-wins in document order.
+
+See [Key Folding & Path Expansion](#key-folding-path-expansion) for more details on path expansion behavior and conflict resolution.
+
+### `DecodeStreamOptions`
+
+Configuration for [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Expected number of spaces per indentation level |
+| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+
+::: warning Path Expansion Not Supported
+Path expansion requires building the full value tree, which is incompatible with event streaming. Use [`decodeFromLines()`](#decodefromlines-lines-options) if you need path expansion.
+:::
+
+## TypeScript Types
+
+### `JsonStreamEvent`
+
+Events emitted by [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options):
+
+```ts
+type JsonStreamEvent
+  = | { type: 'startObject' }
+    | { type: 'endObject' }
+    | { type: 'startArray', length: number }
+    | { type: 'endArray' }
+    | { type: 'key', key: string, wasQuoted?: boolean }
+    | { type: 'primitive', value: JsonPrimitive }
+```
+
+### Delimiters
+
+```ts
+import { DEFAULT_DELIMITER, DELIMITERS } from '@toon-format/toon'
+
+DEFAULT_DELIMITER // ','
+DELIMITERS // { comma: ',', tab: '\t', pipe: '|' }
+```
+
+| Export | Description |
+|--------|-------------|
+| `DEFAULT_DELIMITER` | The default delimiter character (`,`) used when none is specified |
+| `DELIMITERS` | Frozen record mapping delimiter names to their characters |
+| `Delimiter` | Type union of valid delimiter characters: `',' \| '\t' \| '\|'` |
+| `DelimiterKey` | Type union of delimiter names: `'comma' \| 'tab' \| 'pipe'` |
+
+### Option Types
+
+| Export | Description |
+|--------|-------------|
+| `EncodeOptions` | Options accepted by [`encode()`](#encode-input-options) and [`encodeLines()`](#encodelines-input-options) |
+| `DecodeOptions` | Options accepted by [`decode()`](#decode-input-options) and [`decodeFromLines()`](#decodefromlines-lines-options) |
+| `DecodeStreamOptions` | Options accepted by [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options) |
+| `EncodeReplacer` | Signature of the [replacer function](#replacer-function) |
+| `ResolvedEncodeOptions` | `EncodeOptions` after defaults are applied (advanced) |
+| `ResolvedDecodeOptions` | `DecodeOptions` after defaults are applied (advanced) |
+
+## Guides & Examples
+
+### Round-Trip Compatibility
+
+TOON provides lossless round-trips after normalization:
+
+```ts
+import { decode, encode } from '@toon-format/toon'
+
+const original = {
+  users: [
+    { id: 1, name: 'Alice', role: 'admin' },
+    { id: 2, name: 'Bob', role: 'user' }
+  ]
+}
+
+const toon = encode(original)
+const restored = decode(toon)
+
+console.log(JSON.stringify(original) === JSON.stringify(restored))
+// true
+```
+
+**With Key Folding:**
+
+```ts
+import { decode, encode } from '@toon-format/toon'
+
+const original = { data: { metadata: { items: ['a', 'b'] } } }
+
+// Encode with folding
+const toon = encode(original, { keyFolding: 'safe' })
+// → "data.metadata.items[2]: a,b"
+
+// Decode with expansion
+const restored = decode(toon, { expandPaths: 'safe' })
+// → { data: { metadata: { items: ['a', 'b'] } } }
+
+console.log(JSON.stringify(original) === JSON.stringify(restored))
+// true
+```
+
+### Key Folding & Path Expansion
+
+**Key Folding** (`keyFolding: 'safe'`) collapses single-key wrapper chains during encoding:
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const data = { data: { metadata: { items: ['a', 'b'] } } }
+
+// Without folding
+encode(data)
+// data:
+//   metadata:
+//     items[2]: a,b
+
+// With folding
+encode(data, { keyFolding: 'safe' })
+// data.metadata.items[2]: a,b
+```
+
+**Path Expansion** (`expandPaths: 'safe'`) reverses this during decoding:
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const toon = 'data.metadata.items[2]: a,b'
+
+const data = decode(toon, { expandPaths: 'safe' })
+console.log(data)
+// { data: { metadata: { items: ['a', 'b'] } } }
+```
+
+**Expansion Conflict Resolution:**
+
+When multiple expanded keys construct overlapping paths, the decoder merges them recursively:
+- **Object + Object**: Deep merge recursively
+- **Object + Non-object** (array or primitive): Conflict
+  - With `strict: true` (default): Error
+  - With `strict: false`: Last-write-wins (LWW)
+
+Duplicate sibling keys (independent of `expandPaths`) follow the same policy: strict mode throws, lenient mode keeps the last value seen.
+
+### Delimiter Strategies
+
+Tab delimiters (`\t`) often tokenize more efficiently than commas. Tabs are single characters that rarely appear in natural text, which reduces the need for quote-escaping and leads to smaller token counts in large datasets.
+
+Example:
+
+```yaml
+items[2	]{sku	name	qty	price}:
+  A1	Widget	2	9.99
+  B2	Gadget	1	14.5
+```
+
+For maximum token savings on large tabular data, combine tab delimiters with key folding:
+
+```ts
+encode(data, { delimiter: '\t', keyFolding: 'safe' })
+```
+
+**Choosing a Delimiter:**
+
+- **Comma (`,`)**: Default, widely understood, good for simple tabular data.
+- **Tab (`\t`)**: Best for LLM token efficiency, excellent for large datasets.
+- **Pipe (`|`)**: Alternative when commas appear frequently in data.

docs/reference/efficiency-formalization.md

@@ -0,0 +1,509 @@
+---
+description: Mathematical model of TOON's byte-level overhead vs JSON across structure families, with formulas and worked examples.
+---
+
+# TOON vs JSON: Byte-Level Efficiency Model
+
+A mathematical analysis of TOON's byte efficiency compared to JSON across different data structures.
+
+::: info Scope of This Document
+This page presents a theoretical, character-based comparison between TOON and JSON. For practical benchmarks and token counts, see [Benchmarks](/guide/benchmarks). It is an **advanced, non-normative** reference: it explains TOON's design from a mathematical angle but does not change the TOON specification.
+:::
+
+## Overview
+
+Standard JSON introduces structural verbosity that inflates token usage and inference cost. This page formalises a byte-level comparison between TOON and JSON to evaluate whether TOON achieves quantifiable efficiency gains by removing structural redundancy.
+
+Under the assumptions described below (compact JSON, canonical TOON, ASCII keys and punctuation, shallow to moderate nesting, and mostly unquoted TOON strings), TOON's **structural overhead is lower than compact JSON** for the structure families analyzed here, except arrays of arrays.
+
+### Key Findings
+
+- **Tabular arrays** represent TOON's optimal use case, with efficiency gains scaling linearly with both row count and field count.
+- **Simple objects and primitive arrays** show consistent byte reduction, with savings proportional to the number of fields or elements.
+- **Nested objects** benefit from reduced overhead, though efficiency decreases with depth due to indentation costs; at sufficient depth, compact JSON can become smaller.
+- **Arrays of arrays** are the only structure where TOON is less efficient than JSON in this analysis, due to TOON's explicit list markers and inner array headers.
+
+## Methodology
+
+We define recursive byte-length functions $L_{\text{json}}$ and $L_{\text{toon}}$ for both formats, then derive the efficiency delta:
+
+$$
+\Delta = L_{\text{json}}(\Omega) - L_{\text{toon}}(\Omega)
+$$
+
+Where $\Omega$ represents the data structure under comparison. If $\Delta > 0$, TOON uses fewer bytes than JSON for that structure.
+
+::: info Scope & Assumptions
+- **Compact JSON**: JSON is assumed to be compact (no spaces or newlines outside strings). Byte counts are computed on this compact form.
+- **Canonical TOON**: TOON is assumed to follow canonical formatting (indent = 2 spaces, exactly one space after `:`, no spaces after commas in arrays/field lists, no trailing spaces).
+- **Keys and strings**: All keys are "simple" ASCII identifier-style keys that:
+  - must be quoted in JSON, and
+  - can be left unquoted in TOON (no characters that would force quoting).
+  Many examples assume values are numbers, booleans, null, or TOON-safe strings that can be unquoted in TOON but must be quoted in JSON.
+- **Numbers**: For this analysis only, both formats are assumed to use the same canonical decimal representation. JSON could use exponent forms; we ignore that here to isolate structural differences.
+- **ASCII/UTF-8**: Keys and structural tokens are assumed ASCII, so byte length equals character count ($|x|_{\text{utf8}} = |x|_{\text{char}}$). Non-ASCII content affects both formats similarly and does not change the structural conclusions.
+- **Nesting depth**: Closed-form expressions are given for flat structures and a single level of nesting. Each additional nesting level in TOON adds 2 bytes of indentation per nested line. At sufficient depth, the braces of compact JSON can win over TOON's indentation (as seen in [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon)).
+- **Byte vs token count**: Modern LLM tokenizers operate over UTF-8 bytes, so byte length is a good upper bound and first-order proxy for token count, even though the mapping is not exactly linear.
+:::
+
+Think of this as a simplified structural model: we strip away real-world noise and ask, "if you only count structural characters, how do JSON and TOON compare?"
+
+## Formal Notation
+
+### Data Model
+
+Let $\omega$ be a primitive value such that $\omega \in \{\text{string, number, boolean, null}\}$.
+
+Let $\mathcal{O}$ be an object composed of $n$ key-value pairs:
+
+$$
+\mathcal{O} = \{(k_1, v_1), (k_2, v_2), \dots, (k_n, v_n)\}
+$$
+
+Let $\mathcal{A}$ be an array composed of $n$ elements:
+
+$$
+\mathcal{A} = \{v_1, v_2, \dots, v_n\}
+$$
+
+Where:
+- $k_i$ is a key (string)
+- $v_i$ can be a primitive value $\omega$, an object $\mathcal{O}$, or an array $\mathcal{A}$
+
+Therefore: $v_i \in \{\omega, \mathcal{O}, \mathcal{A}\}$
+
+### String Length
+
+Let $\mathcal{S}$ be the set of valid Unicode strings. For any string $x \in \mathcal{S}$, we denote $|x|_{\text{utf8}}$ as the byte-length of $x$ under UTF-8 encoding.
+
+### Integer Length
+
+Let $n \in \mathbb{Z}_{\ge 0}$ be a non-negative integer. The number of bytes required to represent $n$ in decimal format is:
+
+$$
+L_{\text{num}}(n) = \begin{cases}
+1 & \text{if } n = 0 \\
+\lfloor \log_{10}(|n|) \rfloor + 1 & \text{if } n > 0
+\end{cases}
+$$
+
+## JSON Size Functions
+
+For a flat object of $n$ keys:
+
+$$
+L_{\text{json}}(\mathcal{O}) = \underbrace{2}_{\{\}} + \sum_{i=1}^{n} (L_{\text{str}}(k_i) + \underbrace{1}_{:} + L_{\text{json}}(v_i)) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+Where $L_{\text{str}}(k)$ is the length of the key including its mandatory quotes:
+
+$$
+L_{\text{str}}(k) = |k|_{\text{utf8}} + \underbrace{2}_{\text{quotes}}
+$$
+
+### Primitive Values in JSON
+
+When $v_i$ is a primitive data type $\omega$:
+
+| Type | Formula |
+|------|---------|
+| String | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}} + 2$ |
+| Number | $L_{\text{num}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Boolean | $L_{\text{bool}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Null | $L_{\text{null}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+
+### Arrays in JSON
+
+When $v_i$ is an array $\mathcal{A}$:
+
+$$
+L_{\text{json}}(\mathcal{A}) = \underbrace{2}_{\text{[]}} + \sum_{i=1}^{n} L_{\text{json}}(v_i) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+## TOON Size Functions
+
+For a flat object of $n$ keys:
+
+$$
+L_{\text{toon}}(\mathcal{O}) = \sum_{i=1}^{n} (L_{\text{str}}(k_i) + \underbrace{1}_{:} + \underbrace{1}_{\text{space}} + L_{\text{toon}}(v_i)) + \underbrace{(n-1)}_{\text{newlines}}
+$$
+
+Where $L_{\text{str}}(k)$ is the length of the key (no quotes required for simple keys):
+
+$$
+L_{\text{str}}(k) = |k|_{\text{utf8}}
+$$
+
+### Primitive Values in TOON
+
+When $v_i$ is a primitive data type $\omega$:
+
+| Type | Formula |
+|------|---------|
+| String (normal) | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| String (looks like number/boolean) | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}} + 2$ |
+| Number | $L_{\text{num}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Boolean | $L_{\text{bool}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Null | $L_{\text{null}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+
+### Simple Arrays in TOON
+
+Here $L_{\text{toon}}(\mathcal{A})$ refers to the length of the whole field line `key[N]: ...`, not just the array value.
+
+When $v_i$ is a simple array $\mathcal{A}$:
+
+$$
+L_{\text{toon}}(\mathcal{A}) = L_{\text{str}}(k_i) + \underbrace{1}_{\text{[}} + L_{\text{num}}(n) + \underbrace{1}_{\text{]}} + \underbrace{1}_{:} + \underbrace{1}_{\text{space}} + \sum_{i=1}^{n} L_{\text{toon}}(v_i) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+### Tabular Arrays in TOON
+
+When $v_i$ is an array of objects with $m$ fields:
+
+$$
+\begin{split}
+L_{\text{toon}}(\mathcal{A}') = L_{\text{str}}(k_i) + \underbrace{1}_{\text{[}} + L_{\text{num}}(n) + \underbrace{1}_{\text{]}} + \underbrace{1}_{\{} + \\
+\sum_{i=1}^{m} L_{\text{str}}(k_i) + \underbrace{(m-1)}_{\text{commas}} + \underbrace{1}_{\}} + \underbrace{1}_{:} + \\
+\underbrace{2n}_{\text{indents}} + \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{toon}}(v_{ij}) + \underbrace{(m-1)n}_{\text{commas}} + \underbrace{n}_{\text{newlines}}
+\end{split}
+$$
+
+*Note: The term $2n$ assumes an indentation size of 2 spaces.*
+
+## Efficiency Analysis by Structure
+
+Each subsection below focuses on a particular structure family, states the resulting formula, and shows a small example. Intuitively, TOON tends to win when it can:
+
+- avoid repeating keys (tabular arrays),
+- avoid quoting keys and many values,
+- and replace braces with indentation,
+
+and tends to lose when it pays a fixed overhead per element (arrays of arrays) or deep indentation (heavily nested configs).
+
+### Simple Objects
+
+Flat objects with primitive string values are the easiest win: JSON pays for braces and quoted keys and strings, while TOON drops braces at the root, omits quotes on simple keys, and uses one line per field.
+
+For objects with only string primitives:
+
+$$
+\Delta_{\text{obj}} = 2 + n + \sum_{i=1}^{n}(L_{\text{json}}(v_i)) - \sum_{i=1}^{n}(L_{\text{toon}}(v_i))
+$$
+
+If all values are strings that can be unquoted in TOON, this simplifies to:
+
+$$
+f(n) = 2 + 3n
+$$
+
+**Example:** For 1,000,000 objects, TOON saves **3,000,002 bytes ≈ 2.86 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (21 bytes)]
+{ "id": 1, "name": "Ada" }
+```
+
+```yaml [TOON (15 bytes)]
+id: 1
+name: Ada
+```
+
+:::
+
+$$
+\Delta_{\text{obj}} = 2 + \underbrace{2}_{n} + \underbrace{6}_{\sum L_{\text{json}}(v_i)} - \underbrace{4}_{\sum L_{\text{toon}}(v_i)} = 6
+$$
+
+### Nested Objects
+
+Adding a wrapper object (one extra level of nesting) introduces extra braces for JSON and extra indentation and newlines for TOON. For a single level of nesting with primitive values, TOON still comes out ahead, but the net advantage is smaller.
+
+For a single level of nesting with primitives:
+
+$$
+f(n) = 5 + n
+$$
+
+**Example:** For 1,000,000 nested objects (depth 1), TOON saves **1,000,005 bytes ≈ 0.95 MB**.
+
+::: warning Caveat
+This formula is for a single nesting level. Each additional nesting level adds 2 spaces of indentation per nested line; at sufficient depth, compact JSON can become smaller, especially when tabular opportunities disappear (see [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon) and the "Deeply nested configuration" dataset in [Benchmarks](/guide/benchmarks)).
+:::
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (30 bytes)]
+{ "user": { "id": 1, "name": "Ada" } }
+```
+
+```yaml [TOON (25 bytes)]
+user:
+  id: 1
+  name: Ada
+```
+
+:::
+
+$$
+\Delta_{\text{nested}} = 5
+$$
+
+### Primitive Arrays
+
+For arrays of string primitives, JSON writes `["foo","bar","baz"]`, quoting every string and using `[]` for the array. TOON writes `key[N]: foo,bar,baz`, paying once for the length marker but omitting most quotes.
+
+For arrays of $n$ string primitives:
+
+$$
+\Delta_{\text{arr}} = 3 - L_{\text{num}}(n) + \sum_{i=1}^{n}(L_{\text{json}}(v_i)) - \sum_{i=1}^{n}(L_{\text{toon}}(v_i))
+$$
+
+With string values that can be unquoted in TOON, this simplifies to:
+
+$$
+f(n) = 2 + 2n - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 elements, TOON saves **1,999,996 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (28 bytes)]
+{ "tags": ["foo", "bar", "baz"] }
+```
+
+```yaml [TOON (20 bytes)]
+tags[3]: foo,bar,baz
+```
+
+:::
+
+$$
+\Delta_{\text{arr}} = 3 - \underbrace{1}_{L_{\text{num}}(3)} + \underbrace{15}_{\sum L_{\text{json}}} - \underbrace{9}_{\sum L_{\text{toon}}} = 8
+$$
+
+### Root Arrays
+
+At the root, JSON writes `["x","y","z"]`; TOON writes `[3]: x,y,z`. There is no object key cost, so the advantage mainly comes from not quoting TOON-safe strings and from replacing `[]` with `[N]:`.
+
+For root-level arrays of $n$ string primitives:
+
+$$
+f(n) = -3 + 2n - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 elements, TOON saves **1,999,991 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (13 bytes)]
+["x", "y", "z"]
+```
+
+```yaml [TOON (10 bytes)]
+[3]: x,y,z
+```
+
+:::
+
+$$
+\Delta_{\text{root}} = \underbrace{9}_{\sum L_{\text{json}}} - 2 - \underbrace{1}_{L_{\text{num}}(3)} - \underbrace{3}_{\sum L_{\text{toon}}} = 3
+$$
+
+### Tabular Arrays
+
+Uniform arrays of objects are TOON's sweet spot. JSON repeats every key for every row, while TOON declares the length and column names once (`key[N]{id,qty,...}:`) and streams rows as bare values.
+
+For arrays of objects with $n$ rows and $m$ fields, assuming numeric values and $|k| = 3$:
+
+$$
+f(n) = 1 + nm(3 + |k|) - m(1 + |k|) - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 rows with 2 fields and 3-character field names, TOON saves **11,999,987 bytes ≈ 11.44 MB**.
+
+This is where TOON's design (declare fields once, stream rows) pays off most strongly: savings grow linearly with both row count and field count.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (45 bytes)]
+{ "items": [{ "id": 1, "qty": 5 }, { "id": 2, "qty": 3 }] }
+```
+
+```yaml [TOON (29 bytes)]
+items[2]{id,qty}:
+  1,5
+  2,3
+```
+
+:::
+
+$$
+\Delta_{\text{tab}} = 2 + \underbrace{4}_{nm} - \underbrace{2}_{m} + \underbrace{22}_{\Sigma L_{\text{json}}} - \underbrace{1}_{L_{\text{num}}(n)} - \underbrace{5}_{\Sigma L_{\text{toon}}(k)} - \underbrace{4}_{\Sigma L_{\text{toon}}(v)} = 16
+$$
+
+### Arrays of Arrays
+
+Arrays of arrays of primitives are where TOON structurally loses: each inner array becomes a list item with its own header, so TOON pays a fixed overhead per inner array (`"- "` plus `"[m]: "`), while JSON just uses commas.
+
+::: info Practical Note
+For arrays of arrays of primitives, this model predicts that JSON is more byte-efficient than TOON, because TOON pays ~6 extra bytes per inner array (2 for `"- "`, 4 for `"[m]: "`), plus the length marker.
+:::
+
+For arrays of arrays with $n$ outer elements and $m$ inner elements:
+
+$$
+\begin{split}
+\Delta_{\text{arrarr}} = 2 - 6n - \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{num}}(m) + \\
+\sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{json}}(v_{ij}) - \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{toon}}(v_{ij})
+\end{split}
+$$
+
+With string primitives and $m = 2$:
+
+$$
+f(n) = 2 - 6n - \sum_{i=1}^{n}\sum_{j=1}^{m} (\lfloor \log_{10}(|m|) \rfloor + 1) + 2nm
+$$
+
+**Example:** For 1,000,000 arrays with $m = 2$, TOON **wastes 2,999,998 bytes ≈ 2.86 MB** relative to JSON under this model.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (23 bytes)]
+{ "pairs": [[1, 2], [3, 4]] }
+```
+
+```yaml [TOON (35 bytes)]
+pairs[2]:
+  - [2]: 1,2
+  - [2]: 3,4
+```
+
+:::
+
+$$
+\Delta_{\text{arrarr}} = 2 - \underbrace{12}_{6n} - \underbrace{2}_{\sum L_{\text{num}}(m)} + \underbrace{4}_{\sum L_{\text{json}}} - \underbrace{4}_{\sum L_{\text{toon}}} = -12
+$$
+
+### Strings That Look Like Literals
+
+Strings that look like numbers or booleans (e.g. `"123"`, `"true"`) must be quoted in both JSON and TOON, slightly reducing TOON's advantage because it no longer saves quotes on those values.
+
+For objects containing such strings:
+
+$$
+\Delta_{\text{strlit}} = 2 + n
+$$
+
+**Example:** For 1,000,000 objects, TOON saves **2,000,002 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (34 bytes)]
+{ "version": "123", "enabled": "true" }
+```
+
+```yaml [TOON (30 bytes)]
+version: "123"
+enabled: "true"
+```
+
+:::
+
+$$
+\Delta_{\text{str}} = 2 + \underbrace{2}_{n} = 4
+$$
+
+### Empty Structures
+
+Empty containers reveal structural differences even at minimal sizes.
+
+**Empty Object:**
+
+$$
+\Delta_{\text{EmptyObject}} = 2
+$$
+
+JSON requires `{}` (2 bytes), whereas a completely empty root object in TOON is represented as an empty document (0 bytes).
+
+**Empty Array (field):**
+
+$$
+\Delta_{\text{EmptyArray}} = 3
+$$
+
+For a field named `key`, JSON uses `{"key":[]}` in compact form, while TOON uses:
+
+```yaml
+key: []
+```
+
+Under this model, that yields a constant 3-byte advantage for TOON. The legacy `key[0]:` form remains decodable for backward compatibility.
+
+## Summary Table
+
+The table below summarizes the formulas and which side wins under the modeling assumptions.
+
+| Structure | Efficiency Formula | TOON Advantage? |
+|-----------|-------------------|-----------------|
+| Simple Objects | $f(n) = 2 + 3n$ | ✅ Yes |
+| Nested Objects (1 level) | $f(n) = 5 + n$ | ✅ Yes (shrinks with depth) |
+| Primitive Arrays | $f(n) = 2 + 2n - \lfloor \log_{10}(n) \rfloor$ | ✅ Yes |
+| Root Arrays | $f(n) = -3 + 2n - \lfloor \log_{10}(n) \rfloor$ | ✅ Yes |
+| Tabular Arrays | $f(n) = 1 + nm(3+\lvert k\rvert) - m(1+\lvert k\rvert) - \lfloor \log_{10}(n) \rfloor$ | ✅ **Best case** |
+| Arrays of Arrays | $f(n) = 2 - 6n + 2nm - \text{overhead}$ | ❌ JSON wins here |
+| String Literals | $f(n) = 2 + n$ | ✅ Yes (smaller gain) |
+| Empty Structures | $\Delta = 2$ or $3$ | ✅ Yes |
+
+In short:
+
+- TOON's gains are **linear in the number of fields** for flat objects.
+- For arrays, gains grow **linearly in the number of elements**, and for tabular arrays **linearly in both rows and fields**.
+- Arrays of arrays are the main structural case where JSON is smaller.
+- Deep nesting and heavy quoting can erode or reverse these advantages in real data.
+
+## Conclusion
+
+This simplified theoretical model supports TOON's design goal: structurally, it reduces overhead compared to compact JSON in many common patterns by:
+
+- avoiding repeated keys in tabular arrays,
+- omitting quotes on many keys and values,
+- and replacing braces with indentation at shallow depths.
+
+For the structure families examined here and under the stated assumptions, the structural overhead of TOON is lower than that of compact JSON except for arrays of arrays. Since UTF-8 byte length is a reasonable first-order proxy for tokens, these structural savings usually translate into lower token counts in those patterns.
+
+At the same time, this is deliberately a simplified model. In real datasets, additional factors – deeper or irregular nesting, heavily quoted strings, exponent notation in JSON, and tokenizer idiosyncrasies – can reduce or even reverse these gains. Our [Benchmarks](/guide/benchmarks) and [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon) show that compact JSON can be more efficient for deeply nested or low-tabularity data. Use this page as intuition for *why* TOON behaves the way it does, not as a universal guarantee.
+
+## Related Resources
+
+- [Benchmarks](/guide/benchmarks) – Empirical token count and accuracy comparisons across formats
+- [Specification](/reference/spec) – Formal TOON specification
+
+## References
+
+This analysis is based on:
+
+- **Original Research**: [TOON vs. JSON: A Mathematical Evaluation of Byte Efficiency in Structured Data](https://www.researchgate.net/publication/397903673_TOON_vs_JSON_A_Mathematical_Evaluation_of_Byte_Efficiency_in_Structured_Data)
+- **TOON Specification**: [toon-format/spec](https://github.com/toon-format/spec)
+- **JSON Specification**: [RFC 8259](https://datatracker.ietf.org/doc/html/rfc8259), [ECMA-404](https://www.ecma-international.org/publications-and-standards/standards/ecma-404/)
+
+---
+
+This page was contributed by Mateo Lafalce ([@mateolafalce](https://github.com/mateolafalce)).
+
+*Have questions or found an error in the formalization? Open an issue on [GitHub](https://github.com/toon-format/spec) or contribute improvements to this analysis.*

docs/reference/spec.md

@@ -0,0 +1,175 @@
+---
+description: Guided tour of the TOON specification – sections, conformance checklists, media type, and versioning.
+---
+
+# Specification
+
+The [TOON specification](https://github.com/toon-format/spec) is the authoritative reference for implementing encoders, decoders, and validators. It defines the concrete syntax, normative encoding/decoding behavior, and strict-mode validation rules.
+
+You don't need this page to *use* TOON. It's mainly for implementers and contributors. If you're looking to learn how to use TOON, start with the [Getting Started](/guide/getting-started) guide instead.
+
+> [!NOTE]
+> The TOON specification is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to it or sharing feedback.
+
+## Current Version
+
+**Spec v{{ $spec.version }}** (2026-05-20) is the current published Working Draft. It is stable for implementation but not yet finalized; see "Status of This Document" in the spec for details.
+
+## Media Type & File Extension
+
+The spec defines a provisional media type and file extension in [§17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
+
+- **Media type:** `text/toon` (provisional, not yet IANA‑registered; UTF‑8 only)
+- **File extension:** `.toon`
+
+TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `charset` parameter, when present, is `utf-8`.
+
+## Guided Tour of the Spec
+
+### Core Concepts
+
+[§1 Terminology and Conventions](https://github.com/toon-format/spec/blob/main/SPEC.md#1-terminology-and-conventions):
+Defines key terms like "indentation level", "active delimiter", "strict mode", and RFC2119 keywords (MUST, SHOULD, MAY).
+
+[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model):
+Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (canonical decimal for values in `[1e-6, 1e21)` or zero; exponent form permitted outside).
+
+[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder):
+Defines how non-JSON types (Date, BigInt, NaN, Infinity, undefined, etc.) are normalized before encoding. Required reading for encoder implementers.
+
+[§4 Decoding Interpretation](https://github.com/toon-format/spec/blob/main/SPEC.md#4-decoding-interpretation-reference-decoder):
+Specifies how decoders map text tokens to host values (quoted strings, unquoted primitives, numeric parsing with leading-zero handling). Decoders default to strict mode (`strict = true`) in the reference implementation; strict-mode errors are enumerated in §14.
+
+### Syntax Rules
+
+[§5 Concrete Syntax and Root Form](https://github.com/toon-format/spec/blob/main/SPEC.md#5-concrete-syntax-and-root-form):
+Defines TOON's line-oriented, indentation-based notation and how to determine whether the root is an object, array, or primitive.
+
+[§6 Header Syntax](https://github.com/toon-format/spec/blob/main/SPEC.md#6-header-syntax-normative):
+Normative ABNF grammar for array headers: `key[N<delim?>]{fields}:`. Specifies bracket segments, delimiter symbols, and field lists.
+
+[§7 Strings and Keys](https://github.com/toon-format/spec/blob/main/SPEC.md#7-strings-and-keys):
+Complete quoting rules (when strings MUST be quoted), escape sequences (only `\\`, `\"`, `\n`, `\r`, `\t`, and `\uXXXX` for other U+0000–U+001F controls are valid), and key encoding requirements.
+
+[§8 Objects](https://github.com/toon-format/spec/blob/main/SPEC.md#8-objects):
+Object field encoding (key: value), nesting rules, key order preservation, and empty object handling.
+
+[§9 Arrays](https://github.com/toon-format/spec/blob/main/SPEC.md#9-arrays):
+Covers all array forms: primitive (inline), arrays of objects (tabular), mixed/non-uniform (list), and arrays of arrays. Includes tabular detection requirements.
+
+[§10 Objects as List Items](https://github.com/toon-format/spec/blob/main/SPEC.md#10-objects-as-list-items):
+Indentation rules for objects appearing in list items (first field on the hyphen line), including the canonical pattern when the first field is a tabular array (header on the hyphen line, rows at depth +2, sibling fields at depth +1).
+
+[§11 Delimiters](https://github.com/toon-format/spec/blob/main/SPEC.md#11-delimiters):
+Delimiter scoping (document vs active), delimiter-aware quoting, and parsing rules for comma/tab/pipe delimiters.
+
+[§12 Indentation and Whitespace](https://github.com/toon-format/spec/blob/main/SPEC.md#12-indentation-and-whitespace):
+Encoding requirements (consistent spaces, no tabs in indentation, no trailing spaces/newlines) and decoding rules (strict vs non-strict indentation handling).
+
+### Conformance and Validation
+
+[§13 Conformance and Options](https://github.com/toon-format/spec/blob/main/SPEC.md#13-conformance-and-options):
+Defines conformance classes (encoder, decoder, validator), standardized options, and conformance checklists.
+
+[§13.4 Key Folding and Path Expansion](https://github.com/toon-format/spec/blob/main/SPEC.md#134-key-folding-and-path-expansion):
+Optional encoder feature (key folding) and decoder feature (path expansion) for collapsing/expanding dotted paths, with deep-merge semantics and strict/non-strict conflict resolution.
+
+[§14 Strict Mode Errors and Diagnostics](https://github.com/toon-format/spec/blob/main/SPEC.md#14-strict-mode-errors-and-diagnostics-authoritative-checklist):
+**Authoritative checklist** of all strict-mode errors: array count and width mismatches (§14.1), syntax and structural errors (§14.2), path expansion conflicts (§14.3), and duplicate sibling keys (§14.4).
+
+### Implementation Guidance
+
+[§15 Security Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#15-security-considerations):
+Injection risks, quoting rules, and strict-mode checks relevant to security.
+
+[§16 Internationalization](https://github.com/toon-format/spec/blob/main/SPEC.md#16-internationalization):
+Unicode handling and locale-independent number formatting.
+
+[§17 IANA Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
+Media type registration plans and provisional status.
+
+[§18 Versioning and Extensibility](https://github.com/toon-format/spec/blob/main/SPEC.md#18-versioning-and-extensibility):
+How the spec evolves: major vs minor bumps and the extensibility policy.
+
+[§19 Intellectual Property Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#19-intellectual-property-considerations):
+Licensing and IP terms for the specification.
+
+[Appendix F: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-f-host-type-normalization-examples-informative):
+Non-normative guidance for Go, JavaScript, Python, Rust, and Java implementations on normalizing language-specific types.
+
+[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative):
+Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://github.com/toon-format/spec/tree/main/tests) for validating implementations.
+
+## Spec Sections at a Glance
+
+| Section | Topic | When to Read |
+|---------|-------|--------------|
+| §1–4 | Data model, normalization, decoding | Implementing encoders/decoders |
+| §5–6 | Syntax, headers, root form | Implementing parsers |
+| §7 | Strings, keys, quoting, escaping | Implementing string handling |
+| §8–10 | Objects, arrays, list items | Implementing structure encoding |
+| §11–12 | Delimiters, indentation, whitespace | Implementing formatting and validation |
+| §13 | Conformance, options, key folding/path expansion | Implementing options and features |
+| §14 | Strict-mode errors | Implementing validators |
+| §15–16 | Security, internationalization | Operational considerations |
+| §17–19 | IANA, versioning, IP | Ecosystem and licensing |
+
+## Conformance Checklists
+
+The spec includes three conformance checklists:
+
+### Encoder Checklist (§13.1) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#131-encoder-conformance-checklist)</sup>
+
+Key requirements:
+- Produce UTF-8 with LF line endings
+- Use consistent indentation (default 2 spaces, no tabs)
+- Escape `\\`, `\"`, `\n`, `\r`, `\t` in quoted strings, and use `\uXXXX` for any other U+0000–U+001F control character; lone surrogates are rejected
+- Quote strings with active delimiter, colon, or structural characters
+- Emit array lengths `[N]` matching actual count
+- Preserve object key order
+- Emit numbers per §2 (canonical decimal in `[1e-6, 1e21)` or zero; exponent form permitted outside)
+- Convert `-0` to `0`, `NaN`/±Infinity to `null`
+- Emit booleans and null as lowercase literals (`true`, `false`, `null`)
+- No trailing spaces or trailing newline
+- When `keyFolding="safe"` is enabled, folding MUST follow §13.4:
+  - Only fold IdentifierSegment keys (letters/digits/underscores, no dots),
+  - Do not introduce collisions with existing sibling keys,
+  - Do not fold segments that would require quoting.
+- When `flattenDepth` is set, folding MUST stop at the configured number of segments (§13.4).
+
+### Decoder Checklist (§13.2) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#132-decoder-conformance-checklist)</sup>
+
+Key requirements:
+- Parse array headers per §6 (length, delimiter, fields)
+- Split inline arrays and tabular rows using active delimiter only
+- Unescape quoted strings with only valid escapes
+- Type unquoted primitives: true/false/null → booleans/null, numeric → number, else → string
+- Enforce strict-mode rules when `strict=true`
+- Preserve array order and object key order
+- When `expandPaths="safe"` is enabled, expand dotted keys into nested objects per §13.4:
+  - Split on `.`, only expand when all segments are IdentifierSegments,
+  - Deep-merge overlapping paths (object + object),
+  - Do not perform element-wise array merges.
+- With `expandPaths="safe"` and `strict=true` (default), MUST error on any expansion conflict (§14.3).
+- With `expandPaths="safe"` and `strict=false`, MUST apply deterministic last-write-wins (LWW) conflict resolution (§13.4).
+
+### Validator Checklist (§13.3) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#133-validator-conformance-checklist)</sup>
+
+Validators should verify:
+- Structural conformance (headers, indentation, list markers)
+- Whitespace invariants (no trailing spaces/newlines)
+- Delimiter consistency between headers and rows
+- Array length counts match declared `[N]`
+- All strict-mode requirements (including path-expansion conflicts when enabled)
+
+## Versioning
+
+The spec uses semantic versioning (major.minor):
+- **Major version** (e.g., v2 → v3): Breaking changes, incompatible with previous versions
+- **Minor version** (e.g., v3.1 → v3.2): Clarifications, additional requirements, or backward-compatible additions
+
+See [Appendix D: Document Changelog](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-d-document-changelog-informative) for detailed version history.
+
+## Contributing to the Spec
+
+The spec is community-maintained at [github.com/toon-format/spec](https://github.com/toon-format/spec). We welcome contributions of all kinds: reporting ambiguities or errors, proposing clarifications and examples, adding test cases to the reference suite, or discussing edge cases and normative behavior. Your feedback helps shape the format.

docs/reference/syntax-cheatsheet.md

@@ -0,0 +1,367 @@
+---
+description: JSON-to-TOON mappings at a glance for objects, arrays, quoting, key folding, and type conversions.
+---
+
+# Syntax Cheatsheet
+
+Quick reference for mapping JSON to TOON format. For rigorous, normative syntax rules and edge cases, see the [Specification](/reference/spec).
+
+## Objects
+
+::: code-group
+
+```json [JSON]
+{
+  "id": 1,
+  "name": "Ada"
+}
+```
+
+```yaml [TOON]
+id: 1
+name: Ada
+```
+
+:::
+
+## Nested Objects
+
+::: code-group
+
+```json [JSON]
+{
+  "user": {
+    "id": 1,
+    "name": "Ada"
+  }
+}
+```
+
+```yaml [TOON]
+user:
+  id: 1
+  name: Ada
+```
+
+:::
+
+## Primitive Arrays
+
+::: code-group
+
+```json [JSON]
+{
+  "tags": ["foo", "bar", "baz"]
+}
+```
+
+```yaml [TOON]
+tags[3]: foo,bar,baz
+```
+
+:::
+
+## Tabular Arrays
+
+::: code-group
+
+```json [JSON]
+{
+  "items": [
+    { "id": 1, "qty": 5 },
+    { "id": 2, "qty": 3 }
+  ]
+}
+```
+
+```yaml [TOON]
+items[2]{id,qty}:
+  1,5
+  2,3
+```
+
+:::
+
+## Mixed and Non-Uniform Arrays
+
+::: code-group
+
+```json [JSON]
+{
+  "items": [1, { "a": 1 }, "x"]
+}
+```
+
+```yaml [TOON]
+items[3]:
+  - 1
+  - a: 1
+  - x
+```
+
+:::
+
+> [!NOTE]
+> When a list-item object has a tabular array as its first field, the tabular header appears on the hyphen line. Rows are indented two levels deeper than the hyphen, and other fields are indented one level deeper. This is the canonical encoding for this pattern.
+
+::: code-group
+
+```yaml [Multi-field object]
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+    status: active
+```
+
+```yaml [Single-field object]
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+```
+
+:::
+
+## Arrays of Arrays
+
+::: code-group
+
+```json [JSON]
+{
+  "pairs": [[1, 2], [3, 4]]
+}
+```
+
+```yaml [TOON]
+pairs[2]:
+  - [2]: 1,2
+  - [2]: 3,4
+```
+
+:::
+
+## Root Arrays
+
+::: code-group
+
+```json [JSON]
+["x", "y", "z"]
+```
+
+```yaml [TOON]
+[3]: x,y,z
+```
+
+:::
+
+## Empty Containers
+
+::: code-group
+
+```json [Empty Object]
+{}
+```
+
+```yaml [Empty Object]
+(empty output)
+```
+
+:::
+
+::: code-group
+
+```json [Empty Array]
+{
+  "items": []
+}
+```
+
+```yaml [Empty Array]
+items: []
+```
+
+:::
+
+## Quoting Special Cases
+
+### Strings That Look Like Literals
+
+::: code-group
+
+```json [JSON]
+{
+  "version": "123",
+  "enabled": "true"
+}
+```
+
+```yaml [TOON]
+version: "123"
+enabled: "true"
+```
+
+:::
+
+These strings must be quoted because they look like numbers/booleans.
+
+### Strings Containing Delimiters
+
+::: code-group
+
+```json [JSON]
+{
+  "note": "hello, world"
+}
+```
+
+```yaml [TOON]
+note: "hello, world"
+```
+
+:::
+
+Strings must be quoted when they contain the active delimiter (inside an array scope) or the document delimiter (object field values, comma by default).
+
+### Strings with Leading/Trailing Spaces
+
+::: code-group
+
+```json [JSON]
+{
+  "message": " padded "
+}
+```
+
+```yaml [TOON]
+message: " padded "
+```
+
+:::
+
+### Empty String
+
+::: code-group
+
+```json [JSON]
+{
+  "name": ""
+}
+```
+
+```yaml [TOON]
+name: ""
+```
+
+:::
+
+## Quoting Rules Summary
+
+Strings **must** be quoted if they:
+
+- Are empty (`""`)
+- Have leading or trailing whitespace
+- Equal `true`, `false`, or `null` (case-sensitive)
+- Look like numbers (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
+- Contain special characters: `:`, `"`, `\`, `[`, `]`, `{`, `}`, or any control character (U+0000–U+001F, including newline/tab/CR)
+- Contain the relevant delimiter – the active delimiter inside an array scope, or the document delimiter (comma by default) for object field values
+- Equal `"-"` or start with `"-"` followed by any character
+
+Otherwise, strings can be unquoted. Unicode and emoji are safe:
+
+```yaml
+message: Hello 世界 👋
+note: This has inner spaces
+```
+
+## Escape Sequences
+
+Six escape sequences are valid in quoted strings:
+
+| Character | Escape |
+|-----------|--------|
+| Backslash (`\`) | `\\` |
+| Double quote (`"`) | `\"` |
+| Newline | `\n` |
+| Carriage return | `\r` |
+| Tab | `\t` |
+| Any other U+0000–U+001F control character | `\uXXXX` |
+
+Other escapes (e.g., `\x`, `\0`, `\b`) are invalid, and lone-surrogate `\uXXXX` values (U+D800–U+DFFF) are rejected.
+
+## Array Headers
+
+### Basic Header
+
+```
+key[N]:
+```
+
+- `N` = array length
+- Default delimiter: comma
+
+### Tabular Header
+
+```
+key[N]{field1,field2,field3}:
+```
+
+- `N` = array length
+- `{fields}` = column names
+- Default delimiter: comma
+
+### Alternative Delimiters
+
+::: code-group
+
+```yaml [Tab Delimiter]
+items[2	]{id	name}:
+  1	Alice
+  2	Bob
+```
+
+```yaml [Pipe Delimiter]
+items[2|]{id|name}:
+  1|Alice
+  2|Bob
+```
+
+:::
+
+The delimiter symbol appears inside the brackets and braces.
+
+## Key Folding (Optional)
+
+Standard nesting:
+
+```yaml
+data:
+  metadata:
+    items[2]: a,b
+```
+
+With key folding (`keyFolding: 'safe'`):
+
+```yaml
+data.metadata.items[2]: a,b
+```
+
+See [Format Overview – Key Folding](/guide/format-overview#key-folding-optional) for details.
+
+## Type Conversions
+
+| Input | Output |
+|-------|--------|
+| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal |
+| Finite number outside that range | Exponent form permitted |
+| `NaN`, `Infinity`, `-Infinity` | `null` |
+| `BigInt` (safe range) | Number |
+| `BigInt` (out of range) | Quoted decimal string |
+| `Date` | ISO string (quoted) |
+| `Set` | Array of normalized values |
+| `Map` | Object with `String(key)` keys |
+| `undefined`, `function`, `symbol` | `null` |
+
+::: info
+TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
+:::

docs/uno.config.ts

@@ -0,0 +1,14 @@
+import type { UserConfig } from 'unocss'
+import { defineConfig, presetIcons, presetWind4, transformerDirectives } from 'unocss'
+
+const config: UserConfig = defineConfig({
+  presets: [
+    presetWind4(),
+    presetIcons(),
+  ],
+  transformers: [
+    transformerDirectives(),
+  ],
+})
+
+export default config

docs/wrangler.toml

@@ -0,0 +1,10 @@
+name = "toon-docs"
+compatibility_date = "2025-10-01"
+
+[[routes]]
+pattern = "toonformat.dev"
+custom_domain = true
+
+[assets]
+directory = "./.vitepress/dist/"
+not_found_handling = "404-page"

eslint.config.mjs

@@ -1,9 +0,0 @@
-// @ts-check
-import antfu from '@antfu/eslint-config'
-
-export default antfu().append({
-  files: ['README.md', 'SPEC.md'],
-  rules: {
-    'style/no-tabs': 'off',
-  },
-})

eslint.config.ts

@@ -0,0 +1,22 @@
+import type { ConfigNames, TypedFlatConfigItem } from '@antfu/eslint-config'
+import type { FlatConfigComposer } from 'eslint-flat-config-utils'
+import antfu from '@antfu/eslint-config'
+
+const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
+  rules: {
+    'no-cond-assign': 'off',
+  },
+}).append({
+  files: ['**/README.md', 'SPEC.md', '**/benchmarks/**/*', '**/docs/**/*'],
+  rules: {
+    'markdown/no-missing-link-fragments': 'off',
+    'markdown/fenced-code-language': 'off',
+    'markdown/heading-increment': 'off',
+    'import/no-duplicates': 'off',
+    'style/no-tabs': 'off',
+    'yaml/quotes': 'off',
+    'yaml/indent': 'off',
+  },
+})
+
+export default config

package.json

@@ -1,55 +1,32 @@
 {
-  "name": "@byjohann/toon",
+  "name": "@toon-format/monorepo",
   "type": "module",
-  "version": "0.4.1",
-  "packageManager": "pnpm@10.19.0",
-  "description": "Token-Oriented Object Notation – a token-efficient JSON alternative for LLM prompts",
-  "author": "Johann Schopplich <hello@johannschopplich.com>",
-  "license": "MIT",
-  "homepage": "https://github.com/johannschopplich/toon#readme",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/johannschopplich/toon.git"
-  },
-  "bugs": {
-    "url": "https://github.com/johannschopplich/toon/issues"
-  },
-  "sideEffects": false,
-  "exports": {
-    ".": {
-      "types": "./dist/index.d.ts",
-      "default": "./dist/index.js"
-    }
-  },
-  "types": "./dist/index.d.ts",
-  "files": [
-    "dist"
-  ],
+  "version": "2.3.1",
+  "private": true,
+  "packageManager": "pnpm@11.13.0",
   "scripts": {
+    "build": "pnpm -r --filter='./packages/**' run build",
     "automd": "automd",
-    "build": "tsdown",
+    "docs:dev": "pnpm -C docs run dev",
+    "docs:build": "pnpm -C docs run build",
+    "docs:preview": "pnpm -C docs run preview",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
-    "test": "vitest",
+    "test": "pnpm -r test",
     "test:types": "tsc --noEmit",
-    "release": "bumpp"
+    "release": "bumpp -r"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^6.1.0",
-    "@types/node": "^24.9.1",
-    "automd": "^0.4.2",
-    "bumpp": "^10.3.1",
-    "eslint": "^9.38.0",
-    "gpt-tokenizer": "^3.2.0",
-    "tsdown": "^0.15.10",
-    "tsx": "^4.20.6",
-    "typescript": "^5.9.3",
-    "vitest": "^4.0.3"
-  },
-  "pnpm": {
-    "onlyBuiltDependencies": [
-      "@parcel/watcher",
-      "esbuild"
-    ]
+    "@antfu/eslint-config": "^9.1.0",
+    "@commitlint/types": "^21.2.0",
+    "@types/node": "^26.1.1",
+    "automd": "^0.4.3",
+    "bumpp": "^11.1.0",
+    "eslint": "^10.7.0",
+    "eslint-flat-config-utils": "^3.2.0",
+    "tsdown": "^0.22.8",
+    "typescript": "^6.0.3",
+    "vite": "^8.1.4",
+    "vitest": "^4.1.10"
   }
 }

packages/cli/README.md

@@ -0,0 +1,247 @@
+# @toon-format/cli
+
+Command-line tool for converting JSON to TOON and back, with token analysis and streaming support.
+
+[TOON (Token-Oriented Object Notation)](https://toonformat.dev) is a compact, human-readable encoding of the JSON data model that minimizes tokens for LLM input. The CLI lets you test conversions, analyze token savings, and integrate TOON into shell pipelines with stdin/stdout support.
+
+## Installation
+
+```bash
+# npm
+npm install -g @toon-format/cli
+
+# pnpm
+pnpm add -g @toon-format/cli
+
+# yarn
+yarn global add @toon-format/cli
+```
+
+Or use directly with `npx`:
+
+```bash
+npx @toon-format/cli [options] [input]
+```
+
+## Usage
+
+```bash
+toon [options] [input]
+```
+
+**Standard input:** Omit the input argument or use `-` to read from stdin. This enables piping data directly from other commands.
+
+**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode). When reading from stdin, use `--encode` or `--decode` flags to specify the operation (defaults to encode).
+
+### Basic Examples
+
+```bash
+# Encode JSON to TOON (auto-detected)
+toon input.json -o output.toon
+
+# Decode TOON to JSON (auto-detected)
+toon data.toon -o output.json
+
+# Output to stdout
+toon input.json
+
+# Pipe from stdin
+cat data.json | toon
+echo '{"name": "Ada"}' | toon
+
+# Decode from stdin
+cat data.toon | toon --decode
+```
+
+## Options
+
+| Option | Description |
+| ------ | ----------- |
+| `-o, --output <file>` | Output file path (prints to stdout if omitted) |
+| `-e, --encode` | Force encode mode (overrides auto-detection) |
+| `-d, --decode` | Force decode mode (overrides auto-detection) |
+| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
+| `--indent <number>` | Indentation size (default: `2`) |
+| `--stats` | Show token count estimates and savings (encode only) |
+| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
+| `--keyFolding <mode>` | Enable key folding: `off`, `safe` (default: `off`) |
+| `--flattenDepth <number>` | Maximum folded segment count when key folding is enabled (default: `Infinity`) |
+| `--expandPaths <mode>` | Enable path expansion: `off`, `safe` (default: `off`) |
+| `--verbose` | Show full stack traces and cause chains for errors (default: `false`) |
+
+## Advanced Examples
+
+### Token Statistics
+
+Show token savings when encoding:
+
+```bash
+toon data.json --stats -o output.toon
+```
+
+Example output:
+
+```
+✔ Encoded data.json → output.toon
+
+ℹ Token estimates: ~15,145 (JSON) → ~8,745 (TOON)
+✔ Saved ~6,400 tokens (-42.3%)
+```
+
+### Alternative Delimiters
+
+#### Tab-separated (often more token-efficient)
+
+```bash
+toon data.json --delimiter $'\t' -o output.toon
+```
+
+The `--delimiter` value must be the actual delimiter character. In bash/zsh, use `$'\t'` to pass a real tab; literal `"\t"` is rejected as an invalid delimiter.
+
+### Lenient Decoding
+
+Skip validation for faster, more forgiving decoding:
+
+```bash
+toon data.toon --no-strict -o output.json
+```
+
+With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
+
+### Decode Error Output
+
+When a TOON document fails to parse, the CLI renders the offending line with a caret pointing at the first non-whitespace character. Tabs are shown as `→` so the caret column reflects what the decoder actually saw:
+
+```
+ ERROR  Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode
+
+  2 | →b: 1
+      ^
+```
+
+The exit code is `1` on any error. Stack traces are suppressed by default. Pass `--verbose` to include the full stack and the underlying cause chain.
+
+### Stdin Workflows
+
+```bash
+# Convert API response to TOON
+curl https://api.example.com/data | toon --stats
+
+# Process large dataset
+cat large-dataset.json | toon --delimiter $'\t' > output.toon
+
+# Chain with other tools
+jq '.results' data.json | toon > filtered.toon
+```
+
+### Large Dataset Processing
+
+The CLI uses streaming output for both encoding and decoding, writing incrementally without building the full output string in memory:
+
+```bash
+# Encode large JSON file with minimal memory usage
+toon huge-dataset.json -o output.toon
+
+# Decode large TOON file with streaming JSON output
+toon huge-dataset.toon -o output.json
+
+# Process millions of records efficiently via stdin
+cat million-records.json | toon > output.toon
+cat million-records.toon | toon --decode > output.json
+```
+
+**Memory efficiency:**
+- **Encode (JSON → TOON)**: Streams TOON lines to output without full string in memory
+- **Decode (TOON → JSON)**: Uses the same event-based streaming decoder as the `decodeStream` API in `@toon-format/toon`, streaming JSON tokens to output without full string in memory
+- Peak memory usage scales with data depth, not total size
+- When `--expandPaths safe` is enabled, decode falls back to non-streaming mode internally to apply deep-merge expansion before writing JSON
+
+> [!TIP]
+> When using `--stats` with encode, the full output string is kept in memory for token counting. Omit `--stats` for maximum memory efficiency with very large datasets.
+
+### Key Folding (Since v1.5)
+
+Collapse nested wrapper chains to reduce tokens:
+
+#### Basic key folding
+
+```bash
+# Encode with key folding
+toon input.json --keyFolding safe -o output.toon
+```
+
+For data like:
+```json
+{
+  "data": {
+    "metadata": {
+      "items": ["a", "b"]
+    }
+  }
+}
+```
+
+Output becomes:
+```
+data.metadata.items[2]: a,b
+```
+
+Instead of:
+```
+data:
+  metadata:
+    items[2]: a,b
+```
+
+#### Limit folding depth
+
+```bash
+# Fold maximum 2 levels deep
+toon input.json --keyFolding safe --flattenDepth 2 -o output.toon
+```
+
+#### Path expansion on decode
+
+```bash
+# Reconstruct nested structure from folded keys
+toon data.toon --expandPaths safe -o output.json
+```
+
+#### Round-trip workflow
+
+```bash
+# Encode with folding
+toon input.json --keyFolding safe -o compressed.toon
+
+# Decode with expansion (restores original structure)
+toon compressed.toon --expandPaths safe -o output.json
+
+# Verify round-trip
+diff input.json output.json
+```
+
+#### Combined with other options
+
+```bash
+# Key folding + tab delimiter + stats
+toon data.json --keyFolding safe --delimiter $'\t' --stats -o output.toon
+```
+
+## Why Use the CLI?
+
+- **Quick conversions** between formats without writing code
+- **Token analysis** to see potential savings before sending to LLMs
+- **Pipeline integration** with existing JSON-based workflows
+- **Flexible formatting** with delimiter and indentation options
+- **Key folding** to collapse nested wrappers for additional token savings
+- **Memory-efficient streaming** for both encode and decode operations - process large datasets without loading entire outputs into memory
+
+## Related
+
+- [@toon-format/toon](https://www.npmjs.com/package/@toon-format/toon) – JavaScript/TypeScript library
+- [Full specification](https://github.com/toon-format/spec) – Complete format documentation
+- [Website](https://toonformat.dev) – Interactive examples and guides
+
+## License
+
+[MIT](https://github.com/toon-format/toon/blob/main/LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

packages/cli/bin/toon.mjs

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+'use strict'
+import('../dist/index.mjs')

packages/cli/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@toon-format/cli",
+  "type": "module",
+  "version": "2.3.1",
+  "packageManager": "pnpm@11.13.0",
+  "description": "CLI for JSON ↔ TOON conversion using @toon-format/toon",
+  "author": "Johann Schopplich <hello@johannschopplich.com>",
+  "license": "MIT",
+  "homepage": "https://toonformat.dev",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toon-format/toon.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toon-format/toon/issues"
+  },
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "toon": "bin/toon.mjs"
+  },
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "scripts": {
+    "dev": "node ./src/cli-entry.ts --help",
+    "build": "tsdown",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "citty": "^0.2.2",
+    "consola": "^3.4.2",
+    "tokenx": "^1.3.0"
+  }
+}

packages/cli/src/cli-entry.ts

@@ -0,0 +1,4 @@
+import { runMain } from 'citty'
+import { mainCommand } from './index.ts'
+
+runMain(mainCommand)

packages/cli/src/conversion.ts

@@ -0,0 +1,194 @@
+import type { FileHandle } from 'node:fs/promises'
+import type { DecodeOptions, DecodeStreamOptions, EncodeOptions } from '../../toon/src/index.ts'
+import type { InputSource } from './types.ts'
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import process from 'node:process'
+import { consola } from 'consola'
+import { estimateTokenCount } from 'tokenx'
+import { decode, decodeStream, encode, encodeLines } from '../../toon/src/index.ts'
+import { jsonStreamFromEvents } from './json-from-events.ts'
+import { jsonStringifyLines } from './json-stringify-stream.ts'
+import { formatInputLabel, readInput, readLinesFromSource } from './utils.ts'
+
+export async function encodeToToon(config: {
+  input: InputSource
+  output?: string
+  indent: NonNullable<EncodeOptions['indent']>
+  delimiter: NonNullable<EncodeOptions['delimiter']>
+  keyFolding?: NonNullable<EncodeOptions['keyFolding']>
+  flattenDepth?: number
+  printStats: boolean
+}): Promise<void> {
+  const jsonContent = await readInput(config.input)
+
+  let data: unknown
+  try {
+    data = JSON.parse(jsonContent)
+  }
+  catch (error) {
+    throw new Error(`Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`)
+  }
+
+  const encodeOptions: EncodeOptions = {
+    delimiter: config.delimiter,
+    indent: config.indent,
+    keyFolding: config.keyFolding,
+    flattenDepth: config.flattenDepth,
+  }
+
+  // When printing stats, we need the full string for token counting
+  if (config.printStats) {
+    const toonOutput = encode(data, encodeOptions)
+
+    if (config.output) {
+      await fsp.writeFile(config.output, toonOutput, 'utf-8')
+    }
+    else {
+      console.log(toonOutput)
+    }
+
+    const jsonTokens = estimateTokenCount(jsonContent)
+    const toonTokens = estimateTokenCount(toonOutput)
+    const diff = jsonTokens - toonTokens
+    const percent = ((diff / jsonTokens) * 100).toFixed(1)
+
+    if (config.output) {
+      const relativeInputPath = formatInputLabel(config.input)
+      const relativeOutputPath = path.relative(process.cwd(), config.output)
+      consola.success(`Encoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
+    }
+
+    console.log()
+    consola.info(`Token estimates: ~${jsonTokens} (JSON) → ~${toonTokens} (TOON)`)
+    consola.success(`Saved ~${diff} tokens (-${percent}%)`)
+  }
+  else {
+    await writeStreamingToon(encodeLines(data, encodeOptions), config.output)
+
+    if (config.output) {
+      const relativeInputPath = formatInputLabel(config.input)
+      const relativeOutputPath = path.relative(process.cwd(), config.output)
+      consola.success(`Encoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
+    }
+  }
+}
+
+export async function decodeToJson(config: {
+  input: InputSource
+  output?: string
+  indent: NonNullable<DecodeOptions['indent']>
+  strict: NonNullable<DecodeOptions['strict']>
+  expandPaths?: NonNullable<DecodeOptions['expandPaths']>
+}): Promise<void> {
+  // Path expansion requires full value in memory, so use non-streaming path
+  if (config.expandPaths === 'safe') {
+    const toonContent = await readInput(config.input)
+
+    const decodeOptions: DecodeOptions = {
+      indent: config.indent,
+      strict: config.strict,
+      expandPaths: config.expandPaths,
+    }
+    const data = decode(toonContent, decodeOptions)
+
+    await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
+  }
+  else {
+    const lineSource = readLinesFromSource(config.input)
+
+    const decodeStreamOptions: DecodeStreamOptions = {
+      indent: config.indent,
+      strict: config.strict,
+    }
+
+    const events = decodeStream(lineSource, decodeStreamOptions)
+    const jsonChunks = jsonStreamFromEvents(events, config.indent)
+
+    await writeStreamingJson(jsonChunks, config.output)
+  }
+
+  if (config.output) {
+    const relativeInputPath = formatInputLabel(config.input)
+    const relativeOutputPath = path.relative(process.cwd(), config.output)
+    consola.success(`Decoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
+  }
+}
+
+/**
+ * Writes JSON chunks to a file or stdout using streaming approach.
+ * Chunks are written one at a time without building the full string in memory.
+ */
+async function writeStreamingJson(
+  chunks: AsyncIterable<string> | Iterable<string>,
+  outputPath?: string,
+): Promise<void> {
+  // Stream to file using fs/promises API
+  if (outputPath) {
+    let fileHandle: FileHandle | undefined
+
+    try {
+      fileHandle = await fsp.open(outputPath, 'w')
+
+      for await (const chunk of chunks) {
+        await fileHandle.write(chunk)
+      }
+    }
+    finally {
+      await fileHandle?.close()
+    }
+  }
+  // Stream to stdout
+  else {
+    for await (const chunk of chunks) {
+      process.stdout.write(chunk)
+    }
+
+    // Add final newline for stdout
+    process.stdout.write('\n')
+  }
+}
+
+/**
+ * Writes TOON lines to a file or stdout using streaming approach.
+ * Lines are written one at a time without building the full string in memory.
+ */
+async function writeStreamingToon(
+  lines: Iterable<string>,
+  outputPath?: string,
+): Promise<void> {
+  let isFirst = true
+
+  // Stream to file using fs/promises API
+  if (outputPath) {
+    let fileHandle: FileHandle | undefined
+
+    try {
+      fileHandle = await fsp.open(outputPath, 'w')
+
+      for (const line of lines) {
+        if (!isFirst)
+          await fileHandle.write('\n')
+
+        await fileHandle.write(line)
+        isFirst = false
+      }
+    }
+    finally {
+      await fileHandle?.close()
+    }
+  }
+  // Stream to stdout
+  else {
+    for (const line of lines) {
+      if (!isFirst)
+        process.stdout.write('\n')
+
+      process.stdout.write(line)
+      isFirst = false
+    }
+
+    // Add final newline for stdout
+    process.stdout.write('\n')
+  }
+}

packages/cli/src/format-error.ts

@@ -0,0 +1,70 @@
+import { ToonDecodeError } from '../../toon/src/index.ts'
+
+export interface FormatErrorOptions {
+  isVerbose: boolean
+}
+
+// #region Public API
+
+export function formatError(error: unknown, options: FormatErrorOptions): string {
+  const sections: string[] = []
+
+  if (error instanceof ToonDecodeError && error.line !== undefined) {
+    sections.push(formatDecodeError(error))
+  }
+  else {
+    sections.push(String(error))
+  }
+
+  if (options.isVerbose) {
+    const causeChain = formatCauseChain(error)
+    if (causeChain) {
+      sections.push(causeChain)
+    }
+
+    if (error instanceof Error && error.stack) {
+      sections.push(error.stack)
+    }
+  }
+
+  return sections.join('\n\n')
+}
+
+// #endregion
+
+// #region Internal renderers
+
+function formatDecodeError(error: ToonDecodeError): string {
+  const linePrefix = `Line ${error.line}: `
+  const messageWithoutPrefix = error.message.startsWith(linePrefix)
+    ? error.message.slice(linePrefix.length)
+    : error.message
+
+  const header = `Failed to decode TOON at line ${error.line}: ${messageWithoutPrefix}`
+
+  if (error.source === undefined) {
+    return header
+  }
+
+  const visibleSource = error.source.replace(/\t/g, '→')
+  const firstNonWhitespaceIndex = visibleSource.search(/\S/)
+  const gutter = `  ${error.line} | `
+  const caretIndent = ' '.repeat(gutter.length + Math.max(firstNonWhitespaceIndex, 0))
+
+  return `${header}\n\n${gutter}${visibleSource}\n${caretIndent}^`
+}
+
+function formatCauseChain(error: unknown): string {
+  const causeLines: string[] = []
+  let current: unknown = error instanceof Error ? error.cause : undefined
+
+  while (current instanceof Error) {
+    const name = current.name || 'Error'
+    causeLines.push(`Caused by: ${name}: ${current.message}`)
+    current = current.cause
+  }
+
+  return causeLines.join('\n')
+}
+
+// #endregion

packages/cli/src/index.ts

@@ -0,0 +1,155 @@
+import type { ArgsDef, CommandDef } from 'citty'
+import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src/index.ts'
+import type { InputSource } from './types.ts'
+import * as path from 'node:path'
+import process from 'node:process'
+import { defineCommand } from 'citty'
+import { consola } from 'consola'
+import { DEFAULT_DELIMITER, DELIMITERS } from '../../toon/src/index.ts'
+import pkg from '../package.json' with { type: 'json' }
+import { decodeToJson, encodeToToon } from './conversion.ts'
+import { formatError } from './format-error.ts'
+import { detectMode } from './utils.ts'
+
+const { name, version } = pkg
+
+const args: ArgsDef = {
+  input: {
+    type: 'positional',
+    description: 'Input file path (omit or use "-" to read from stdin)',
+    required: false,
+  },
+  output: {
+    type: 'string',
+    description: 'Output file path',
+    alias: 'o',
+  },
+  encode: {
+    type: 'boolean',
+    description: 'Encode JSON to TOON (auto-detected by default)',
+    alias: 'e',
+  },
+  decode: {
+    type: 'boolean',
+    description: 'Decode TOON to JSON (auto-detected by default)',
+    alias: 'd',
+  },
+  delimiter: {
+    type: 'string',
+    description: 'Delimiter for arrays: comma (,), tab (\\t), or pipe (|)',
+    default: ',',
+  },
+  indent: {
+    type: 'string',
+    description: 'Indentation size',
+    default: '2',
+  },
+  strict: {
+    type: 'boolean',
+    description: 'Strict decode validation (disable with --no-strict)',
+    default: true,
+  },
+  keyFolding: {
+    type: 'string',
+    description: 'Enable key folding: off, safe (default: off)',
+    default: 'off',
+  },
+  flattenDepth: {
+    type: 'string',
+    description: 'Maximum folded segment count when key folding is enabled (default: Infinity)',
+  },
+  expandPaths: {
+    type: 'string',
+    description: 'Enable path expansion: off, safe (default: off)',
+    default: 'off',
+  },
+  stats: {
+    type: 'boolean',
+    description: 'Show token statistics',
+    default: false,
+  },
+  verbose: {
+    type: 'boolean',
+    description: 'Show full stack traces and cause chains for errors',
+    default: false,
+  },
+} as const
+
+export const mainCommand: CommandDef<ArgsDef> = defineCommand({
+  meta: {
+    name,
+    description: 'TOON CLI – Convert between JSON and TOON formats',
+    version,
+  },
+  args,
+  async run({ args }) {
+    const input = args.input
+
+    const inputSource: InputSource = !input || input === '-'
+      ? { type: 'stdin' }
+      : { type: 'file', path: path.resolve(input) }
+    const outputPath = args.output ? path.resolve(args.output) : undefined
+
+    // Parse and validate indent
+    const indent = Number.parseInt(args.indent || '2', 10)
+    if (Number.isNaN(indent) || indent < 0) {
+      throw new Error(`Invalid indent value: ${args.indent}`)
+    }
+
+    // Validate delimiter
+    const delimiter = args.delimiter || DEFAULT_DELIMITER
+    if (!(Object.values(DELIMITERS)).includes(delimiter as Delimiter)) {
+      throw new Error(`Invalid delimiter "${delimiter}". Valid delimiters are: comma (,), tab (\\t), pipe (|)`)
+    }
+
+    // Validate `keyFolding`
+    const keyFolding = args.keyFolding || 'off'
+    if (keyFolding !== 'off' && keyFolding !== 'safe') {
+      throw new Error(`Invalid keyFolding value "${keyFolding}". Valid values are: off, safe`)
+    }
+
+    // Parse and validate `flattenDepth`
+    let flattenDepth: number | undefined
+    if (args.flattenDepth !== undefined) {
+      flattenDepth = Number.parseInt(args.flattenDepth, 10)
+      if (Number.isNaN(flattenDepth) || flattenDepth < 0) {
+        throw new Error(`Invalid flattenDepth value: ${args.flattenDepth}`)
+      }
+    }
+
+    // Validate `expandPaths`
+    const expandPaths = args.expandPaths || 'off'
+    if (expandPaths !== 'off' && expandPaths !== 'safe') {
+      throw new Error(`Invalid expandPaths value "${expandPaths}". Valid values are: off, safe`)
+    }
+
+    const mode = detectMode(inputSource, args.encode, args.decode)
+
+    try {
+      if (mode === 'encode') {
+        await encodeToToon({
+          input: inputSource,
+          output: outputPath,
+          delimiter: delimiter as Delimiter,
+          indent,
+          keyFolding: keyFolding as NonNullable<EncodeOptions['keyFolding']>,
+          flattenDepth,
+          printStats: args.stats === true,
+        })
+      }
+      else {
+        await decodeToJson({
+          input: inputSource,
+          output: outputPath,
+          indent,
+          strict: args.strict !== false,
+          expandPaths: expandPaths as NonNullable<DecodeOptions['expandPaths']>,
+        })
+      }
+    }
+    catch (error) {
+      consola.error(formatError(error, { isVerbose: args.verbose === true }))
+      process.exit(1)
+    }
+  },
+})

packages/cli/src/json-from-events.ts

@@ -0,0 +1,217 @@
+import type { JsonStreamEvent } from '../../toon/src/types.ts'
+
+/**
+ * Context for tracking JSON structure state during event streaming.
+ */
+type JsonContext
+  = | { type: 'object', needsComma: boolean, expectValue: boolean }
+    | { type: 'array', needsComma: boolean }
+
+/**
+ * Converts a stream of `JsonStreamEvent` into formatted JSON string chunks.
+ *
+ * Similar to `jsonStringifyLines` but driven by events instead of a value tree.
+ * Useful for streaming TOON decode directly to JSON output without building
+ * the full data structure in memory.
+ *
+ * @param events - Async iterable of JSON stream events
+ * @param indent - Number of spaces for indentation (0 = compact, >0 = pretty)
+ * @returns Async iterable of JSON string chunks
+ *
+ * @example
+ * ```ts
+ * const lines = readLinesFromSource(input)
+ * const events = decodeStream(lines)
+ * for await (const chunk of jsonStreamFromEvents(events, 2)) {
+ *   process.stdout.write(chunk)
+ * }
+ * ```
+ */
+export async function* jsonStreamFromEvents(
+  events: AsyncIterable<JsonStreamEvent>,
+  indent: number = 2,
+): AsyncIterable<string> {
+  const stack: JsonContext[] = []
+  let depth = 0
+
+  for await (const event of events) {
+    const parent = stack.length > 0 ? stack[stack.length - 1] : undefined
+
+    switch (event.type) {
+      case 'startObject': {
+        // Emit comma if needed (inside array or after previous object field value)
+        if (parent) {
+          if (parent.type === 'array' && parent.needsComma) {
+            yield ','
+          }
+          else if (parent.type === 'object' && !parent.expectValue) {
+            // Object field value already emitted, this is a nested object after a key
+            // The comma is handled by the key event
+          }
+        }
+
+        // Emit newline and indent for pretty printing
+        if (indent > 0 && parent) {
+          if (parent.type === 'array') {
+            yield '\n'
+            yield ' '.repeat(depth * indent)
+          }
+        }
+
+        yield '{'
+        stack.push({ type: 'object', needsComma: false, expectValue: false })
+        depth++
+        break
+      }
+
+      case 'endObject': {
+        const context = stack.pop()
+        if (!context || context.type !== 'object') {
+          throw new Error('Mismatched endObject event')
+        }
+
+        depth--
+
+        // Emit newline and indent for closing brace (pretty print)
+        if (indent > 0 && context.needsComma) {
+          yield '\n'
+          yield ' '.repeat(depth * indent)
+        }
+
+        yield '}'
+
+        // Mark parent as needing comma for next item
+        const newParent = stack.length > 0 ? stack[stack.length - 1] : undefined
+        if (newParent) {
+          if (newParent.type === 'object') {
+            newParent.expectValue = false
+            newParent.needsComma = true
+          }
+          else if (newParent.type === 'array') {
+            newParent.needsComma = true
+          }
+        }
+        break
+      }
+
+      case 'startArray': {
+        // Emit comma if needed
+        if (parent) {
+          if (parent.type === 'array' && parent.needsComma) {
+            yield ','
+          }
+        }
+
+        // Emit newline and indent for pretty printing
+        if (indent > 0 && parent) {
+          if (parent.type === 'array') {
+            yield '\n'
+            yield ' '.repeat(depth * indent)
+          }
+        }
+
+        yield '['
+        stack.push({
+          type: 'array',
+          needsComma: false,
+        })
+        depth++
+        break
+      }
+
+      case 'endArray': {
+        const context = stack.pop()
+        if (!context || context.type !== 'array') {
+          throw new Error('Mismatched endArray event')
+        }
+
+        depth--
+
+        // Emit newline and indent for closing bracket (pretty print)
+        if (indent > 0 && context.needsComma) {
+          yield '\n'
+          yield ' '.repeat(depth * indent)
+        }
+
+        yield ']'
+
+        // Mark parent as needing comma for next item
+        const newParent = stack.length > 0 ? stack[stack.length - 1] : undefined
+        if (newParent) {
+          if (newParent.type === 'object') {
+            newParent.expectValue = false
+            newParent.needsComma = true
+          }
+          else if (newParent.type === 'array') {
+            newParent.needsComma = true
+          }
+        }
+        break
+      }
+
+      case 'key': {
+        if (!parent || parent.type !== 'object') {
+          throw new Error('Key event outside of object context')
+        }
+
+        // Emit comma before this field if needed
+        if (parent.needsComma) {
+          yield ','
+        }
+
+        // Emit newline and indent (pretty print)
+        if (indent > 0) {
+          yield '\n'
+          yield ' '.repeat(depth * indent)
+        }
+
+        // Emit key
+        yield JSON.stringify(event.key)
+        yield indent > 0 ? ': ' : ':'
+
+        parent.expectValue = true
+        parent.needsComma = true
+        break
+      }
+
+      case 'primitive': {
+        // Emit comma if needed
+        if (parent) {
+          if (parent.type === 'array' && parent.needsComma) {
+            yield ','
+          }
+          else if (parent.type === 'object' && !parent.expectValue) {
+            // This shouldn't happen in well-formed events
+            throw new Error('Primitive event in object without preceding key')
+          }
+        }
+
+        // Emit newline and indent for array items (pretty print)
+        if (indent > 0 && parent && parent.type === 'array') {
+          yield '\n'
+          yield ' '.repeat(depth * indent)
+        }
+
+        // Emit primitive value
+        yield JSON.stringify(event.value)
+
+        // Update parent context
+        if (parent) {
+          if (parent.type === 'object') {
+            parent.expectValue = false
+            // needsComma already true from key event
+          }
+          else if (parent.type === 'array') {
+            parent.needsComma = true
+          }
+        }
+        break
+      }
+    }
+  }
+
+  // Ensure stack is empty
+  if (stack.length !== 0) {
+    throw new Error('Incomplete event stream: unclosed objects or arrays')
+  }
+}

packages/cli/src/json-stringify-stream.ts

@@ -0,0 +1,161 @@
+/**
+ * Streaming JSON stringifier.
+ *
+ * Yields JSON tokens one at a time, allowing streaming output without holding
+ * the entire JSON string in memory.
+ *
+ * @param value - The value to stringify (must be JSON-serializable)
+ * @param indent - Number of spaces for indentation (0 = compact, >0 = pretty)
+ * @returns Generator that yields JSON string chunks
+ *
+ * @example
+ * ```ts
+ * const data = { name: "Alice", scores: [95, 87, 92] }
+ * for (const chunk of jsonStringifyLines(data, 2)) {
+ *   process.stdout.write(chunk)
+ * }
+ * ```
+ */
+export function* jsonStringifyLines(
+  value: unknown,
+  indent: number = 2,
+): Iterable<string> {
+  yield* stringifyValue(value, 0, indent)
+}
+
+/**
+ * Internal generator for recursive stringification.
+ */
+function* stringifyValue(
+  value: unknown,
+  depth: number,
+  indent: number,
+): Iterable<string> {
+  // Handle null
+  if (value === null) {
+    yield 'null'
+    return
+  }
+
+  const type = typeof value
+
+  // Handle primitives
+  if (type === 'boolean' || type === 'number') {
+    yield JSON.stringify(value)
+    return
+  }
+
+  if (type === 'string') {
+    yield JSON.stringify(value)
+    return
+  }
+
+  // Handle arrays
+  if (Array.isArray(value)) {
+    yield* stringifyArray(value, depth, indent)
+    return
+  }
+
+  // Handle objects
+  if (type === 'object') {
+    yield* stringifyObject(value as Record<string, unknown>, depth, indent)
+    return
+  }
+
+  // Undefined, functions, symbols become null in JSON
+  yield 'null'
+}
+
+/**
+ * Stringify an array with proper formatting.
+ */
+function* stringifyArray(
+  arr: unknown[],
+  depth: number,
+  indent: number,
+): Iterable<string> {
+  if (arr.length === 0) {
+    yield '[]'
+    return
+  }
+
+  yield '['
+
+  if (indent > 0) {
+    // Pretty-printed format
+    for (let i = 0; i < arr.length; i++) {
+      yield '\n'
+      yield ' '.repeat((depth + 1) * indent)
+      yield* stringifyValue(arr[i], depth + 1, indent)
+      if (i < arr.length - 1) {
+        yield ','
+      }
+    }
+    yield '\n'
+    yield ' '.repeat(depth * indent)
+    yield ']'
+  }
+  else {
+    // Compact format
+    for (let i = 0; i < arr.length; i++) {
+      yield* stringifyValue(arr[i], depth + 1, indent)
+      if (i < arr.length - 1) {
+        yield ','
+      }
+    }
+    yield ']'
+  }
+}
+
+/**
+ * Stringify an object with proper formatting.
+ */
+function* stringifyObject(
+  obj: Record<string, unknown>,
+  depth: number,
+  indent: number,
+): Iterable<string> {
+  const keys = Object.keys(obj)
+
+  if (keys.length === 0) {
+    yield '{}'
+    return
+  }
+
+  yield '{'
+
+  if (indent > 0) {
+    // Pretty-printed format
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]!
+      const value = obj[key]
+
+      yield '\n'
+      yield ' '.repeat((depth + 1) * indent)
+      yield JSON.stringify(key)
+      yield ': '
+      yield* stringifyValue(value, depth + 1, indent)
+      if (i < keys.length - 1) {
+        yield ','
+      }
+    }
+    yield '\n'
+    yield ' '.repeat(depth * indent)
+    yield '}'
+  }
+  else {
+    // Compact format
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]!
+      const value = obj[key]
+
+      yield JSON.stringify(key)
+      yield ':'
+      yield* stringifyValue(value, depth + 1, indent)
+      if (i < keys.length - 1) {
+        yield ','
+      }
+    }
+    yield '}'
+  }
+}

packages/cli/src/types.ts

@@ -0,0 +1,3 @@
+export type InputSource
+  = | { type: 'stdin' }
+    | { type: 'file', path: string }

packages/cli/src/utils.ts

@@ -0,0 +1,109 @@
+import type { InputSource } from './types.ts'
+import { createReadStream } from 'node:fs'
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import process from 'node:process'
+
+export function detectMode(
+  input: InputSource,
+  encodeFlag?: boolean,
+  decodeFlag?: boolean,
+): 'encode' | 'decode' {
+  // Explicit flags take precedence
+  if (encodeFlag)
+    return 'encode'
+  if (decodeFlag)
+    return 'decode'
+
+  // Auto-detect based on file extension
+  if (input.type === 'file') {
+    if (input.path.endsWith('.json'))
+      return 'encode'
+    if (input.path.endsWith('.toon'))
+      return 'decode'
+  }
+
+  // Default to encode
+  return 'encode'
+}
+
+export async function readInput(source: InputSource): Promise<string> {
+  if (source.type === 'stdin')
+    return readFromStdin()
+
+  return fsp.readFile(source.path, 'utf-8')
+}
+
+export function formatInputLabel(source: InputSource): string {
+  if (source.type === 'stdin')
+    return 'stdin'
+
+  const relativePath = path.relative(process.cwd(), source.path)
+  return relativePath || path.basename(source.path)
+}
+
+function readFromStdin(): Promise<string> {
+  const { stdin } = process
+
+  if (stdin.readableEnded)
+    return Promise.resolve('')
+
+  return new Promise((resolve, reject) => {
+    let data = ''
+
+    const onData = (chunk: string) => {
+      data += chunk
+    }
+
+    function cleanup() {
+      stdin.off('data', onData)
+      stdin.off('error', onError)
+      stdin.off('end', onEnd)
+    }
+
+    function onError(error: Error) {
+      cleanup()
+      reject(error)
+    }
+
+    function onEnd() {
+      cleanup()
+      resolve(data)
+    }
+
+    stdin.setEncoding('utf-8')
+    stdin.on('data', onData)
+    stdin.once('error', onError)
+    stdin.once('end', onEnd)
+    stdin.resume()
+  })
+}
+
+export async function* readLinesFromSource(source: InputSource): AsyncIterable<string> {
+  const stream = source.type === 'stdin'
+    ? process.stdin
+    : createReadStream(source.path, { encoding: 'utf-8' })
+
+  // Explicitly set encoding for stdin
+  if (source.type === 'stdin') {
+    stream.setEncoding('utf-8')
+  }
+
+  let buffer = ''
+
+  for await (const chunk of stream) {
+    buffer += chunk
+    let index: number
+
+    while ((index = buffer.indexOf('\n')) !== -1) {
+      const line = buffer.slice(0, index)
+      buffer = buffer.slice(index + 1)
+      yield line
+    }
+  }
+
+  // Emit last line if buffer is not empty and doesn't end with newline
+  if (buffer.length > 0) {
+    yield buffer
+  }
+}

packages/cli/test/format-error.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it } from 'vitest'
+import { ToonDecodeError } from '../../toon/src/index'
+import { formatError } from '../src/format-error'
+
+describe('formatError', () => {
+  it('renders a decode error with line and source as a header, source line, and caret', () => {
+    const error = new ToonDecodeError(
+      'Tabs are not allowed in indentation in strict mode',
+      { line: 2, source: '\tb: 1' },
+    )
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe(
+      'Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode\n'
+      + '\n'
+      + '  2 | →b: 1\n'
+      + '      ^',
+    )
+  })
+
+  it('renders a decode error without source as a header only', () => {
+    const error = new ToonDecodeError('Something went wrong', { line: 5 })
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe('Failed to decode TOON at line 5: Something went wrong')
+  })
+
+  it('appends the cause chain under verbose mode', () => {
+    const cause = new SyntaxError('Unterminated string: missing closing quote')
+    const error = new ToonDecodeError(
+      'Unterminated string: missing closing quote',
+      { line: 2, source: 'greeting: "hello', cause },
+    )
+
+    const output = formatError(error, { isVerbose: true })
+
+    expect(output).toContain('Failed to decode TOON at line 2:')
+    expect(output).toContain('  2 | greeting: "hello')
+    expect(output).toContain('Caused by: SyntaxError: Unterminated string: missing closing quote')
+  })
+
+  it('appends the stack trace under verbose mode and omits it otherwise', () => {
+    const error = new ToonDecodeError('Boom', { line: 1, source: 'x' })
+    error.stack = 'ToonDecodeError: Line 1: Boom\n    at fakeFrame (file.ts:1:1)'
+
+    const verbose = formatError(error, { isVerbose: true })
+    const quiet = formatError(error, { isVerbose: false })
+
+    expect(verbose).toContain('at fakeFrame (file.ts:1:1)')
+    expect(quiet).not.toContain('at fakeFrame')
+  })
+
+  it('renders a generic Error as its message only when not verbose', () => {
+    const error = new Error('something went wrong')
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe('Error: something went wrong')
+  })
+
+  it('places the caret under the first non-whitespace character of the source line', () => {
+    const error = new ToonDecodeError(
+      'Indentation must be exact multiple of 2, but found 3 spaces',
+      { line: 2, source: '   b: 1' },
+    )
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe(
+      'Failed to decode TOON at line 2: Indentation must be exact multiple of 2, but found 3 spaces\n'
+      + '\n'
+      + '  2 |    b: 1\n'
+      + '         ^',
+    )
+  })
+})

packages/cli/test/index.test.ts

@@ -0,0 +1,832 @@
+import process from 'node:process'
+import { consola } from 'consola'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { DEFAULT_DELIMITER, encode } from '../../toon/src'
+import { version } from '../package.json' with { type: 'json' }
+import { createCliTestContext, mockStdin, runCli } from './utils'
+
+describe('toon CLI', () => {
+  beforeEach(() => {
+    vi.spyOn(process, 'exit').mockImplementation(() => 0 as never)
+    vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    vi.spyOn(process.stdout, 'write').mockImplementation(() => true)
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  describe('version', () => {
+    it('prints the version when using --version', async () => {
+      const consoleLog = vi.mocked(console.log)
+
+      await runCli({ rawArgs: ['--version'] })
+
+      expect(consoleLog).toHaveBeenCalledWith(version)
+    })
+  })
+
+  describe('encode (JSON → TOON)', () => {
+    it('encodes JSON from stdin', async () => {
+      const data = {
+        title: 'TOON test',
+        count: 3,
+        nested: { ok: true },
+      }
+      const cleanup = mockStdin(JSON.stringify(data))
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli()
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe(`${encode(data)}\n`)
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('encodes a JSON file into a TOON file', async () => {
+      const data = {
+        title: 'TOON test',
+        count: 3,
+        nested: { ok: true },
+      }
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data, undefined, 2),
+      })
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['input.json', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        const expected = encode(data, {
+          delimiter: DEFAULT_DELIMITER,
+          indent: 2,
+        })
+
+        expect(output).toBe(expected)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Encoded .* → .*/))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('writes to stdout when output not specified', async () => {
+      const data = { ok: true }
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await context.run(['input.json'])
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe(`${encode(data)}\n`)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('encodes JSON from stdin to output file', async () => {
+      const data = { key: 'value' }
+      const context = await createCliTestContext({})
+      const cleanup = mockStdin(JSON.stringify(data))
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        expect(output).toBe(encode(data))
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Encoded.*stdin[^\n\r\u2028\u2029\u2192]*\u2192.*output\.toon/))
+      }
+      finally {
+        cleanup()
+        await context.cleanup()
+      }
+    })
+  })
+
+  describe('decode (TOON → JSON)', () => {
+    it('decodes a TOON file into a JSON file', async () => {
+      const data = {
+        items: ['alpha', 'beta'],
+        meta: { done: false },
+      }
+      const toonInput = encode(data)
+      const context = await createCliTestContext({
+        'input.toon': toonInput,
+      })
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['input.toon', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        expect(JSON.parse(output)).toEqual(data)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Decoded .* → .*/))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('decodes TOON from stdin', async () => {
+      const data = { items: ['a', 'b'], count: 2 }
+      const toonInput = encode(data)
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+        const fullOutput = writeChunks.join('')
+        // Remove trailing newline before parsing
+        const jsonOutput = fullOutput.endsWith('\n') ? fullOutput.slice(0, -1) : fullOutput
+        const result = JSON.parse(jsonOutput)
+        expect(result).toEqual(data)
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes TOON from stdin to output file', async () => {
+      const data = { name: 'test', values: [1, 2, 3] }
+      const toonInput = encode(data)
+      const context = await createCliTestContext({})
+      const cleanup = mockStdin(toonInput)
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['--decode', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        expect(JSON.parse(output)).toEqual(data)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Decoded.*stdin[^\n\r\u2028\u2029\u2192]*\u2192.*output\.json/))
+      }
+      finally {
+        cleanup()
+        await context.cleanup()
+      }
+    })
+  })
+
+  describe('stdin edge cases', () => {
+    it('handles invalid JSON from stdin', async () => {
+      const cleanup = mockStdin('{ invalid json }')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await runCli({ rawArgs: [] })
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+        expect(consolaError).toHaveBeenCalled()
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('handles invalid TOON from stdin', async () => {
+      const cleanup = mockStdin('key: "unterminated string')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+        expect(consolaError).toHaveBeenCalled()
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('renders a TOON decode error with line context, source, and caret', async () => {
+      const cleanup = mockStdin('a:\n\tb: 1\n')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+        const errorCall = consolaError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [rendered] = errorCall!
+        expect(rendered).toEqual(expect.stringContaining('Failed to decode TOON at line 2:'))
+        expect(rendered).toEqual(expect.stringContaining('  2 | →b: 1'))
+        expect(rendered).toEqual(expect.stringContaining('      ^'))
+        expect(rendered).not.toEqual(expect.stringMatching(/^\s+at \S+/m))
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('includes the stack trace when --verbose is passed', async () => {
+      const cleanup = mockStdin('a:\n\tb: 1\n')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+
+      try {
+        await runCli({ rawArgs: ['--decode', '--verbose'] })
+
+        const errorCall = consolaError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [rendered] = errorCall!
+        expect(rendered).toEqual(expect.stringContaining('Failed to decode TOON at line 2:'))
+        expect(rendered).toEqual(expect.stringMatching(/at \S+/))
+      }
+      finally {
+        cleanup()
+      }
+    })
+  })
+
+  describe('stdin with options', () => {
+    it('encodes JSON from stdin with custom delimiter', async () => {
+      const data = { items: [1, 2, 3] }
+      const cleanup = mockStdin(JSON.stringify(data))
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--delimiter', '|'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe(`${encode(data, { delimiter: '|' })}\n`)
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('encodes JSON from stdin with custom indent', async () => {
+      const data = {
+        nested: {
+          deep: { value: 1 },
+        },
+      }
+      const cleanup = mockStdin(JSON.stringify(data))
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--indent', '4'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe(`${encode(data, { indent: 4 })}\n`)
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes TOON from stdin with --no-strict', async () => {
+      const data = { test: true }
+      const toonInput = encode(data)
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode', '--no-strict'] })
+
+        const fullOutput = writeChunks.join('')
+        // Remove trailing newline before parsing
+        const jsonOutput = fullOutput.endsWith('\n') ? fullOutput.slice(0, -1) : fullOutput
+        const result = JSON.parse(jsonOutput)
+        expect(result).toEqual(data)
+      }
+      finally {
+        cleanup()
+      }
+    })
+  })
+
+  describe('encode options', () => {
+    it('encodes with --keyFolding safe', async () => {
+      const data = {
+        data: {
+          metadata: {
+            items: ['a', 'b'],
+          },
+        },
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'safe', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        const expected = encode(data, { keyFolding: 'safe' })
+
+        expect(output).toBe(expected)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('encodes with --flattenDepth', async () => {
+      const data = {
+        level1: {
+          level2: {
+            level3: {
+              value: 'deep',
+            },
+          },
+        },
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'safe', '--flattenDepth', '2', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        const expected = encode(data, { keyFolding: 'safe', flattenDepth: 2 })
+
+        expect(output).toBe(expected)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+  })
+
+  describe('decode options', () => {
+    it('decodes with --expandPaths safe', async () => {
+      const data = {
+        data: {
+          metadata: {
+            items: ['a', 'b'],
+          },
+        },
+      }
+      const toonInput = encode(data, { keyFolding: 'safe' })
+
+      const context = await createCliTestContext({
+        'input.toon': toonInput,
+      })
+
+      try {
+        await context.run(['input.toon', '--decode', '--expandPaths', 'safe', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('decodes with --indent for JSON formatting', async () => {
+      const data = {
+        a: 1,
+        b: [2, 3],
+        c: { nested: true },
+      }
+      const toonInput = encode(data, { indent: 4 })
+
+      const context = await createCliTestContext({
+        'input.toon': toonInput,
+      })
+
+      try {
+        await context.run(['input.toon', '--decode', '--indent', '4', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+        expect(output).toContain('    ') // Should have 4-space indentation
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('decodes root primitive number', async () => {
+      const toonInput = '42'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe('42\n')
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes root primitive string', async () => {
+      const toonInput = '"Hello World"'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        const jsonOutput = fullOutput.endsWith('\n') ? fullOutput.slice(0, -1) : fullOutput
+        expect(JSON.parse(jsonOutput)).toBe('Hello World')
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes root primitive boolean', async () => {
+      const toonInput = 'true'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe('true\n')
+      }
+      finally {
+        cleanup()
+      }
+    })
+  })
+
+  describe('streaming output', () => {
+    it('streams large JSON to TOON file with identical output', async () => {
+      const data = {
+        items: Array.from({ length: 1000 }, (_, i) => ({
+          id: i,
+          name: `Item ${i}`,
+          value: Math.random(),
+        })),
+      }
+
+      const context = await createCliTestContext({
+        'large-input.json': JSON.stringify(data, undefined, 2),
+      })
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['large-input.json', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        // Verify streaming produces identical output to `encode()`
+        const expected = encode(data, {
+          delimiter: DEFAULT_DELIMITER,
+          indent: 2,
+        })
+
+        expect(output).toBe(expected)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Encoded .* → .*/))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('streams large TOON to JSON file with streaming decode', async () => {
+      const data = {
+        records: Array.from({ length: 1000 }, (_, i) => ({
+          id: i,
+          title: `Record ${i}`,
+          score: Math.random() * 100,
+        })),
+      }
+
+      const toonContent = encode(data, {
+        delimiter: DEFAULT_DELIMITER,
+        indent: 2,
+      })
+
+      const context = await createCliTestContext({
+        'large-input.toon': toonContent,
+      })
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['large-input.toon', '--decode', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Decoded .* → .*/))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('streams to stdout using process.stdout.write', async () => {
+      const data = {
+        users: [
+          { id: 1, name: 'Alice' },
+          { id: 2, name: 'Bob' },
+        ],
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      const writeChunks: string[] = []
+      const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await context.run(['input.json'])
+
+        expect(writeSpy).toHaveBeenCalled()
+
+        // Verify complete output matches `encode()`
+        const fullOutput = writeChunks.join('')
+        const expected = `${encode(data)}\n`
+        expect(fullOutput).toBe(expected)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('handles empty object streaming correctly', async () => {
+      const data = {}
+
+      const context = await createCliTestContext({
+        'empty.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['empty.json', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        expect(output).toBe(encode(data))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('handles single-line output streaming correctly', async () => {
+      const data = { key: 'value' }
+
+      const context = await createCliTestContext({
+        'single.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['single.json', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        expect(output).toBe(encode(data))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('uses non-streaming path when stats are enabled', async () => {
+      const data = {
+        items: [
+          { id: 1, value: 'test' },
+          { id: 2, value: 'data' },
+        ],
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      const consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+      const consolaInfo = vi.spyOn(consola, 'info').mockImplementation(() => undefined)
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['input.json', '--stats'])
+
+        expect(consolaInfo).toHaveBeenCalledWith(expect.stringMatching(/Token estimates:/))
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Saved.*tokens/))
+        expect(consoleLogSpy).toHaveBeenCalledWith(encode(data))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+  })
+
+  describe('error handling', () => {
+    it('rejects invalid delimiter', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--delimiter', ';'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid delimiter')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid indent value', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--indent', 'abc'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid indent value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('handles missing input file', async () => {
+      const context = await createCliTestContext({})
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['nonexistent.json'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+        expect(consolaError).toHaveBeenCalled()
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid --keyFolding value', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'invalid'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid keyFolding value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid --expandPaths value', async () => {
+      const context = await createCliTestContext({
+        'input.toon': 'key: value',
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.toon', '--decode', '--expandPaths', 'invalid'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid expandPaths value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid --flattenDepth value', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--flattenDepth', '-1'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid flattenDepth value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+  })
+})

packages/cli/test/json-from-events.test.ts

@@ -0,0 +1,423 @@
+import type { JsonStreamEvent } from '../../toon/src/types'
+import { describe, expect, it } from 'vitest'
+import { jsonStreamFromEvents } from '../src/json-from-events'
+
+describe('jsonStreamFromEvents', () => {
+  describe('primitives', () => {
+    it('converts null event', async () => {
+      const events = [
+        { type: 'primitive' as const, value: null },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(null))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(null, null, 2))
+    })
+
+    it('converts boolean events', async () => {
+      const eventsTrue = [{ type: 'primitive' as const, value: true }]
+      const eventsFalse = [{ type: 'primitive' as const, value: false }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 0))).toBe(JSON.stringify(true))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsFalse), 0))).toBe(JSON.stringify(false))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 2))).toBe(JSON.stringify(true, null, 2))
+    })
+
+    it('converts number events', async () => {
+      const events0 = [{ type: 'primitive' as const, value: 0 }]
+      const events42 = [{ type: 'primitive' as const, value: 42 }]
+      const eventsNeg = [{ type: 'primitive' as const, value: -17 }]
+      const eventsFloat = [{ type: 'primitive' as const, value: 3.14159 }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(events0), 0))).toBe(JSON.stringify(0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events42), 0))).toBe(JSON.stringify(42))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsNeg), 0))).toBe(JSON.stringify(-17))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsFloat), 0))).toBe(JSON.stringify(3.14159))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events42), 2))).toBe(JSON.stringify(42, null, 2))
+    })
+
+    it('converts string events', async () => {
+      const eventsEmpty = [{ type: 'primitive' as const, value: '' }]
+      const eventsHello = [{ type: 'primitive' as const, value: 'hello' }]
+      const eventsQuotes = [{ type: 'primitive' as const, value: 'with "quotes"' }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsEmpty), 0))).toBe(JSON.stringify(''))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsHello), 0))).toBe(JSON.stringify('hello'))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsQuotes), 0))).toBe(JSON.stringify('with "quotes"'))
+    })
+  })
+
+  describe('empty containers', () => {
+    it('converts empty array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 0 },
+        { type: 'endArray' as const },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify([], null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify([], null, 2))
+    })
+
+    it('converts empty object events', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'endObject' as const },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify({}, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify({}, null, 2))
+    })
+  })
+
+  describe('arrays', () => {
+    it('converts simple array events with compact formatting', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 2, 3]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('converts simple array events with pretty formatting', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 2, 3]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts mixed-type array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 5 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 'two' },
+        { type: 'primitive' as const, value: true },
+        { type: 'primitive' as const, value: null },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'key' },
+        { type: 'primitive' as const, value: 'value' },
+        { type: 'endObject' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 'two', true, null, { key: 'value' }]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts nested array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'endArray' as const },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'primitive' as const, value: 4 },
+        { type: 'endArray' as const },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 5 },
+        { type: 'primitive' as const, value: 6 },
+        { type: 'endArray' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [[1, 2], [3, 4], [5, 6]]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('objects', () => {
+    it('converts simple object events with compact formatting', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'a' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'b' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'c' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endObject' as const },
+      ]
+      const value = { a: 1, b: 2, c: 3 }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('converts simple object events with pretty formatting', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'a' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'b' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'c' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endObject' as const },
+      ]
+      const value = { a: 1, b: 2, c: 3 }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts object events with mixed value types', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'num' },
+        { type: 'primitive' as const, value: 42 },
+        { type: 'key' as const, key: 'str' },
+        { type: 'primitive' as const, value: 'hello' },
+        { type: 'key' as const, key: 'bool' },
+        { type: 'primitive' as const, value: true },
+        { type: 'key' as const, key: 'nil' },
+        { type: 'primitive' as const, value: null },
+        { type: 'key' as const, key: 'arr' },
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        num: 42,
+        str: 'hello',
+        bool: true,
+        nil: null,
+        arr: [1, 2, 3],
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts nested object events', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level1' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level2' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level3' },
+        { type: 'primitive' as const, value: 'deep' },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        level1: {
+          level2: {
+            level3: 'deep',
+          },
+        },
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles special characters in keys', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'normal-key' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'key with spaces' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'key:with:colons' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'key' as const, key: 'key"with"quotes' },
+        { type: 'primitive' as const, value: 4 },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        'normal-key': 1,
+        'key with spaces': 2,
+        'key:with:colons': 3,
+        'key"with"quotes': 4,
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('complex nested structures', () => {
+    it('converts object containing arrays', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        { type: 'key' as const, key: 'scores' },
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 95 },
+        { type: 'primitive' as const, value: 87 },
+        { type: 'primitive' as const, value: 92 },
+        { type: 'endArray' as const },
+        { type: 'key' as const, key: 'metadata' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'tags' },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 'math' },
+        { type: 'primitive' as const, value: 'science' },
+        { type: 'endArray' as const },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        name: 'Alice',
+        scores: [95, 87, 92],
+        metadata: {
+          tags: ['math', 'science'],
+        },
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts array of objects', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        { type: 'endObject' as const },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Bob' },
+        { type: 'endObject' as const },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Charlie' },
+        { type: 'endObject' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [
+        { id: 1, name: 'Alice' },
+        { id: 2, name: 'Bob' },
+        { id: 3, name: 'Charlie' },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('indentation levels', () => {
+    const events = [
+      { type: 'startObject' as const },
+      { type: 'key' as const, key: 'a' },
+      { type: 'startArray' as const, length: 2 },
+      { type: 'primitive' as const, value: 1 },
+      { type: 'primitive' as const, value: 2 },
+      { type: 'endArray' as const },
+      { type: 'key' as const, key: 'b' },
+      { type: 'startObject' as const },
+      { type: 'key' as const, key: 'c' },
+      { type: 'primitive' as const, value: 3 },
+      { type: 'endObject' as const },
+      { type: 'endObject' as const },
+    ]
+    const value = { a: [1, 2], b: { c: 3 } }
+
+    it('handles indent=0 (compact)', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('handles indent=2', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles indent=4', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 4))).toBe(JSON.stringify(value, null, 4))
+    })
+
+    it('handles indent=8', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 8))).toBe(JSON.stringify(value, null, 8))
+    })
+  })
+
+  describe('error handling', () => {
+    it('throws on mismatched endObject event', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 0 },
+        { type: 'endObject' as const }, // Wrong closing event
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Mismatched endObject event')
+    })
+
+    it('throws on mismatched endArray event', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'endArray' as const }, // Wrong closing event
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Mismatched endArray event')
+    })
+
+    it('throws on key event outside object context', async () => {
+      const events = [
+        { type: 'key' as const, key: 'invalid' },
+        { type: 'primitive' as const, value: 1 },
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Key event outside of object context')
+    })
+
+    it('throws on primitive in object without preceding key', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'primitive' as const, value: 'invalid' }, // No key before primitive
+        { type: 'endObject' as const },
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Primitive event in object without preceding key')
+    })
+
+    it('throws on incomplete event stream', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        // Missing `endObject`
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Incomplete event stream: unclosed objects or arrays')
+    })
+  })
+})
+
+/**
+ * Converts array of events to async iterable.
+ */
+async function* asyncEvents(events: JsonStreamEvent[]): AsyncIterable<JsonStreamEvent> {
+  for (const event of events) {
+    await Promise.resolve()
+    yield event
+  }
+}
+
+/**
+ * Joins chunks from an async iterable into a single string.
+ */
+async function join(iter: AsyncIterable<string>): Promise<string> {
+  const chunks: string[] = []
+  for await (const chunk of iter) {
+    chunks.push(chunk)
+  }
+  return chunks.join('')
+}

packages/cli/test/json-stringify-stream.test.ts

@@ -0,0 +1,245 @@
+import { describe, expect, it } from 'vitest'
+import { jsonStringifyLines } from '../src/json-stringify-stream'
+
+describe('jsonStringifyLines', () => {
+  describe('primitives', () => {
+    it('stringifies null', () => {
+      expect(join(jsonStringifyLines(null, 0))).toBe(JSON.stringify(null))
+      expect(join(jsonStringifyLines(null, 2))).toBe(JSON.stringify(null, null, 2))
+    })
+
+    it('stringifies booleans', () => {
+      expect(join(jsonStringifyLines(true, 0))).toBe(JSON.stringify(true))
+      expect(join(jsonStringifyLines(false, 0))).toBe(JSON.stringify(false))
+      expect(join(jsonStringifyLines(true, 2))).toBe(JSON.stringify(true, null, 2))
+    })
+
+    it('stringifies numbers', () => {
+      expect(join(jsonStringifyLines(0, 0))).toBe(JSON.stringify(0))
+      expect(join(jsonStringifyLines(42, 0))).toBe(JSON.stringify(42))
+      expect(join(jsonStringifyLines(-17, 0))).toBe(JSON.stringify(-17))
+      expect(join(jsonStringifyLines(3.14159, 0))).toBe(JSON.stringify(3.14159))
+      expect(join(jsonStringifyLines(1e10, 2))).toBe(JSON.stringify(1e10, null, 2))
+    })
+
+    it('stringifies strings', () => {
+      expect(join(jsonStringifyLines('', 0))).toBe(JSON.stringify(''))
+      expect(join(jsonStringifyLines('hello', 0))).toBe(JSON.stringify('hello'))
+      expect(join(jsonStringifyLines('with "quotes"', 0))).toBe(JSON.stringify('with "quotes"'))
+      expect(join(jsonStringifyLines('with\nnewlines', 2))).toBe(JSON.stringify('with\nnewlines', null, 2))
+      expect(join(jsonStringifyLines('with\ttabs', 0))).toBe(JSON.stringify('with\ttabs'))
+    })
+
+    it('converts undefined to null', () => {
+      expect(join(jsonStringifyLines(undefined, 0))).toBe('null')
+      expect(join(jsonStringifyLines(undefined, 2))).toBe('null')
+    })
+  })
+
+  describe('empty containers', () => {
+    it('stringifies empty arrays', () => {
+      expect(join(jsonStringifyLines([], 0))).toBe(JSON.stringify([], null, 0))
+      expect(join(jsonStringifyLines([], 2))).toBe(JSON.stringify([], null, 2))
+    })
+
+    it('stringifies empty objects', () => {
+      expect(join(jsonStringifyLines({}, 0))).toBe(JSON.stringify({}, null, 0))
+      expect(join(jsonStringifyLines({}, 2))).toBe(JSON.stringify({}, null, 2))
+    })
+  })
+
+  describe('arrays', () => {
+    it('stringifies arrays with compact formatting (indent=0)', () => {
+      const value = [1, 2, 3]
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('stringifies arrays with pretty formatting (indent=2)', () => {
+      const value = [1, 2, 3]
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies mixed-type arrays', () => {
+      const value = [1, 'two', true, null, { key: 'value' }]
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies nested arrays', () => {
+      const value = [[1, 2], [3, 4], [5, 6]]
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies deeply nested arrays', () => {
+      const value = [[[1]], [[2]], [[3]]]
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+      expect(join(jsonStringifyLines(value, 4))).toBe(JSON.stringify(value, null, 4))
+    })
+  })
+
+  describe('objects', () => {
+    it('stringifies simple objects with compact formatting', () => {
+      const value = { a: 1, b: 2, c: 3 }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('stringifies simple objects with pretty formatting', () => {
+      const value = { a: 1, b: 2, c: 3 }
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies objects with mixed value types', () => {
+      const value = {
+        num: 42,
+        str: 'hello',
+        bool: true,
+        nil: null,
+        arr: [1, 2, 3],
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies nested objects', () => {
+      const value = {
+        level1: {
+          level2: {
+            level3: 'deep',
+          },
+        },
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('preserves key order', () => {
+      const value = { z: 1, a: 2, m: 3 }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles special characters in keys', () => {
+      const value = {
+        'normal-key': 1,
+        'key with spaces': 2,
+        'key:with:colons': 3,
+        'key"with"quotes': 4,
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('complex nested structures', () => {
+    it('stringifies objects containing arrays', () => {
+      const value = {
+        name: 'Alice',
+        scores: [95, 87, 92],
+        metadata: {
+          tags: ['math', 'science'],
+        },
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies arrays of objects', () => {
+      const value = [
+        { id: 1, name: 'Alice' },
+        { id: 2, name: 'Bob' },
+        { id: 3, name: 'Charlie' },
+      ]
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('stringifies deeply nested mixed structures', () => {
+      const value = {
+        users: [
+          {
+            name: 'Alice',
+            roles: ['admin', 'user'],
+            settings: {
+              theme: 'dark',
+              notifications: true,
+            },
+          },
+          {
+            name: 'Bob',
+            roles: ['user'],
+            settings: {
+              theme: 'light',
+              notifications: false,
+            },
+          },
+        ],
+        count: 2,
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('indentation levels', () => {
+    const value = { a: [1, 2], b: { c: 3 } }
+
+    it('handles indent=0 (compact)', () => {
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('handles indent=2', () => {
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles indent=4', () => {
+      expect(join(jsonStringifyLines(value, 4))).toBe(JSON.stringify(value, null, 4))
+    })
+
+    it('handles indent=8', () => {
+      expect(join(jsonStringifyLines(value, 8))).toBe(JSON.stringify(value, null, 8))
+    })
+  })
+
+  describe('edge cases', () => {
+    it('handles arrays with undefined values (converted to null)', () => {
+      const value = [1, undefined, 3]
+      const expected = JSON.stringify(value, null, 2)
+      expect(join(jsonStringifyLines(value, 2))).toBe(expected)
+    })
+
+    it('handles single-element arrays', () => {
+      const value = [42]
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles single-property objects', () => {
+      const value = { only: 'one' }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles objects with many properties', () => {
+      const value: Record<string, number> = {}
+      for (let i = 0; i < 100; i++) {
+        value[`key${i}`] = i
+      }
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles large arrays', () => {
+      const value = Array.from({ length: 1000 }, (_, i) => i)
+      expect(join(jsonStringifyLines(value, 0))).toBe(JSON.stringify(value, null, 0))
+      expect(join(jsonStringifyLines(value, 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+})
+
+/**
+ * Joins chunks from an iterable into a single string.
+ */
+function join(iter: Iterable<string>): string {
+  return Array.from(iter).join('')
+}

packages/cli/test/utils.ts

@@ -0,0 +1,96 @@
+import * as fsp from 'node:fs/promises'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import process from 'node:process'
+import { Readable } from 'node:stream'
+import { runMain } from 'citty'
+import { mainCommand } from '../src/index'
+
+interface FileRecord {
+  [relativePath: string]: string
+}
+
+export function runCli(options?: Parameters<typeof runMain>[1]): Promise<void> {
+  return runMain(mainCommand, options)
+}
+
+export interface CliTestContext {
+  readonly dir: string
+  run: (args?: string[]) => Promise<void>
+  read: (relativePath: string) => Promise<string>
+  write: (relativePath: string, contents: string) => Promise<void>
+  resolve: (relativePath: string) => string
+  cleanup: () => Promise<void>
+}
+
+const TEMP_PREFIX = path.join(os.tmpdir(), 'toon-cli-test-')
+
+export async function createCliTestContext(initialFiles: FileRecord = {}): Promise<CliTestContext> {
+  const dir = await fsp.mkdtemp(TEMP_PREFIX)
+  await writeFiles(dir, initialFiles)
+
+  async function run(args: string[] = []): Promise<void> {
+    const previousCwd = process.cwd()
+    process.chdir(dir)
+    try {
+      await runCli({ rawArgs: args })
+    }
+    finally {
+      process.chdir(previousCwd)
+    }
+  }
+
+  function resolvePath(relativePath: string): string {
+    return path.join(dir, relativePath)
+  }
+
+  async function read(relativePath: string): Promise<string> {
+    return fsp.readFile(resolvePath(relativePath), 'utf8')
+  }
+
+  async function write(relativePath: string, contents: string): Promise<void> {
+    const targetPath = resolvePath(relativePath)
+    await fsp.mkdir(path.dirname(targetPath), { recursive: true })
+    await fsp.writeFile(targetPath, contents, 'utf8')
+  }
+
+  async function cleanup(): Promise<void> {
+    await fsp.rm(dir, { recursive: true, force: true })
+  }
+
+  return {
+    dir,
+    run,
+    read,
+    write,
+    resolve: resolvePath,
+    cleanup,
+  }
+}
+
+async function writeFiles(baseDir: string, files: FileRecord): Promise<void> {
+  await Promise.all(
+    Object.entries(files).map(async ([relativePath, contents]) => {
+      const filePath = path.join(baseDir, relativePath)
+      await fsp.mkdir(path.dirname(filePath), { recursive: true })
+      await fsp.writeFile(filePath, contents, 'utf8')
+    }),
+  )
+}
+
+export function mockStdin(input: string): () => void {
+  const mockStream = Readable.from([input])
+
+  const originalStdin = process.stdin
+  Object.defineProperty(process, 'stdin', {
+    value: mockStream,
+    writable: true,
+  })
+
+  return () => {
+    Object.defineProperty(process, 'stdin', {
+      value: originalStdin,
+      writable: true,
+    })
+  }
+}

packages/cli/tsdown.config.ts

@@ -0,0 +1,11 @@
+import type { UserConfig } from 'tsdown/config'
+import { defineConfig } from 'tsdown/config'
+
+const config: UserConfig = defineConfig({
+  entry: {
+    index: 'src/cli-entry.ts',
+  },
+  dts: true,
+})
+
+export default config

packages/toon/README.md

@@ -0,0 +1,923 @@
+![TOON logo with step‑by‑step guide](./.github/og.png)
+
+# Token-Oriented Object Notation (TOON)
+
+[![CI](https://github.com/toon-format/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon/actions)
+[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
+[![SPEC v3.3](https://img.shields.io/badge/spec-v3.3-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
+[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
+[![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE)
+
+**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
+
+TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably. For deeply nested or non-uniform data, JSON may be more efficient.
+
+The similarity to CSV is intentional: CSV is simple and ubiquitous, and TOON aims to keep that familiarity while remaining a lossless, drop-in representation of JSON for Large Language Models.
+
+Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input.
+
+> [!TIP]
+> The TOON format is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to the [spec](https://github.com/toon-format/spec) or sharing feedback.
+
+## Table of Contents
+
+- [Why TOON?](#why-toon)
+- [Key Features](#key-features)
+- [When Not to Use TOON](#when-not-to-use-toon)
+- [Benchmarks](#benchmarks)
+- [Installation & Quick Start](#installation--quick-start)
+- [Playgrounds](#playgrounds)
+- [Editor Support](#editor-support)
+- [CLI](#cli)
+- [Format Overview](#format-overview)
+- [Using TOON with LLMs](#using-toon-with-llms)
+- [Documentation](#documentation)
+- [Other Implementations](#other-implementations)
+- [📋 Full Specification](https://github.com/toon-format/spec/blob/main/SPEC.md)
+
+## Why TOON?
+
+AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – and standard JSON is verbose and token-expensive:
+
+```json
+{
+  "context": {
+    "task": "Our favorite hikes together",
+    "location": "Boulder",
+    "season": "spring_2025"
+  },
+  "friends": ["ana", "luis", "sam"],
+  "hikes": [
+    {
+      "id": 1,
+      "name": "Blue Lake Trail",
+      "distanceKm": 7.5,
+      "elevationGain": 320,
+      "companion": "ana",
+      "wasSunny": true
+    },
+    {
+      "id": 2,
+      "name": "Ridge Overlook",
+      "distanceKm": 9.2,
+      "elevationGain": 540,
+      "companion": "luis",
+      "wasSunny": false
+    },
+    {
+      "id": 3,
+      "name": "Wildflower Loop",
+      "distanceKm": 5.1,
+      "elevationGain": 180,
+      "companion": "sam",
+      "wasSunny": true
+    }
+  ]
+}
+```
+
+<details>
+<summary>YAML already conveys the same information with <strong>fewer tokens</strong>.</summary>
+
+```yaml
+context:
+  task: Our favorite hikes together
+  location: Boulder
+  season: spring_2025
+friends:
+  - ana
+  - luis
+  - sam
+hikes:
+  - id: 1
+    name: Blue Lake Trail
+    distanceKm: 7.5
+    elevationGain: 320
+    companion: ana
+    wasSunny: true
+  - id: 2
+    name: Ridge Overlook
+    distanceKm: 9.2
+    elevationGain: 540
+    companion: luis
+    wasSunny: false
+  - id: 3
+    name: Wildflower Loop
+    distanceKm: 5.1
+    elevationGain: 180
+    companion: sam
+    wasSunny: true
+```
+
+</details>
+
+TOON conveys the same information with **even fewer tokens** – combining YAML-like indentation with CSV-style tabular arrays:
+
+```yaml
+context:
+  task: Our favorite hikes together
+  location: Boulder
+  season: spring_2025
+friends[3]: ana,luis,sam
+hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
+  1,Blue Lake Trail,7.5,320,ana,true
+  2,Ridge Overlook,9.2,540,luis,false
+  3,Wildflower Loop,5.1,180,sam,true
+```
+
+## Key Features
+
+- 📊 **Token-Efficient & Accurate:** TOON reaches 76.4% accuracy (vs JSON's 75.0%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
+- 🔁 **JSON Data Model:** Encodes the same objects, arrays, and primitives as JSON with deterministic, lossless round-trips.
+- 🛤️ **LLM-Friendly Guardrails:** Explicit [N] lengths and {fields} headers give models a clear schema to follow, improving parsing reliability.
+- 📐 **Minimal Syntax:** Uses indentation instead of braces and minimizes quoting, giving YAML-like readability with CSV-style compactness.
+- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
+- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
+
+## Media Type & File Extension
+
+By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations) for normative details.
+
+## When Not to Use TOON
+
+TOON excels with uniform arrays of objects, but there are cases where other formats are better:
+
+- **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels.
+- **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it.
+- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5–10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
+- **Latency-critical applications**: If end-to-end response time is your top priority, benchmark on your exact setup. Some deployments (especially local/quantized models like Ollama) may process compact JSON faster despite TOON's lower token count. Measure TTFT, tokens/sec, and total time for both formats and use whichever is faster.
+
+See [benchmarks](#benchmarks) for concrete comparisons across different data structures.
+
+## Benchmarks
+
+Benchmarks are organized into two tracks to ensure fair comparisons:
+
+- **Mixed-Structure Track**: Datasets with nested or semi-uniform structures (TOON vs JSON, YAML, XML). CSV excluded as it cannot properly represent these structures.
+- **Flat-Only Track**: Datasets with flat tabular structures where CSV is applicable (CSV vs TOON vs JSON, YAML, XML).
+
+### Retrieval Accuracy
+
+<!-- automd:file src="./benchmarks/results/retrieval-accuracy.md" -->
+
+Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
+
+<details>
+<summary><strong>Show Dataset Catalog</strong></summary>
+
+#### Dataset Catalog
+
+| Dataset | Rows | Structure | CSV Support | Eligibility |
+| ------- | ---- | --------- | ----------- | ----------- |
+| Uniform employee records | 100 | uniform | ✓ | 100% |
+| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
+| Time-series analytics data | 60 | uniform | ✓ | 100% |
+| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
+| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
+| Deeply nested configuration | 11 | deep | ✗ | 0% |
+| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
+| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
+| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
+| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
+| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
+
+**Structure classes:**
+- **uniform**: All objects have identical fields with primitive values
+- **semi-uniform**: Mix of uniform and non-uniform structures
+- **nested**: Objects with nested structures (nested objects or arrays)
+- **deep**: Highly nested with minimal tabular eligibility
+
+**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
+
+**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
+
+</details>
+
+#### Efficiency Ranking (Accuracy per 1K Tokens)
+
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
+
+```
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
+```
+
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
+
+**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
+
+#### Per-Model Accuracy
+
+Accuracy across 4 LLMs on 209 data retrieval questions:
+
+```
+claude-haiku-4-5-20251001
+→ TOON           ████████████░░░░░░░░    59.8% (125/209)
+  JSON           ███████████░░░░░░░░░    57.4% (120/209)
+  YAML           ███████████░░░░░░░░░    56.0% (117/209)
+  XML            ███████████░░░░░░░░░    55.5% (116/209)
+  JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
+  CSV            ██████████░░░░░░░░░░    50.5% (55/109)
+
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
+
+gpt-5-nano
+→ TOON           ██████████████████░░    90.9% (190/209)
+  JSON compact   ██████████████████░░    90.9% (190/209)
+  JSON           ██████████████████░░    89.0% (186/209)
+  CSV            ██████████████████░░    89.0% (97/109)
+  YAML           █████████████████░░░    87.1% (182/209)
+  XML            ████████████████░░░░    80.9% (169/209)
+
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
+```
+
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
+
+<details>
+<summary><strong>Performance by dataset, model, and question type</strong></summary>
+
+#### Performance by Question Type
+
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
+| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
+
+#### Performance by Dataset
+
+##### Uniform employee records
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
+
+##### E-commerce orders with nested structures
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
+
+##### Time-series analytics data
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
+
+##### Top 100 GitHub repositories
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
+
+##### Semi-uniform event logs
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
+
+##### Deeply nested configuration
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
+
+##### Valid complete dataset (control)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
+
+##### Array truncated: 3 rows removed from end
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
+
+##### Extra rows added beyond declared length
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
+
+##### Inconsistent field count (missing salary in row 10)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
+
+##### Missing required fields (no email in multiple rows)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
+
+#### Performance by Model
+
+##### claude-haiku-4-5-20251001
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 59.8% | 125/209 |
+| `json-pretty` | 57.4% | 120/209 |
+| `yaml` | 56.0% | 117/209 |
+| `xml` | 55.5% | 116/209 |
+| `json-compact` | 55.0% | 115/209 |
+| `csv` | 50.5% | 55/109 |
+
+##### gemini-3-flash-preview
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
+
+##### gpt-5-nano
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 90.9% | 190/209 |
+| `json-compact` | 90.9% | 190/209 |
+| `json-pretty` | 89.0% | 186/209 |
+| `csv` | 89.0% | 97/109 |
+| `yaml` | 87.1% | 182/209 |
+| `xml` | 80.9% | 169/209 |
+
+##### grok-4-1-fast-non-reasoning
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
+
+</details>
+
+#### What's Being Measured
+
+This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
+
+#### Datasets Tested
+
+Eleven datasets designed to test different structural patterns and validation capabilities:
+
+**Primary datasets:**
+
+1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
+2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
+3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
+4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
+5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
+6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
+
+**Structural validation datasets:**
+
+7. **Control**: Valid complete dataset (baseline for validation)
+8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
+9. **Extra rows**: Array with 3 additional rows beyond declared length
+10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
+11. **Missing fields**: Systematic field omissions (no email in multiple rows)
+
+#### Question Types
+
+209 questions are generated dynamically across five categories:
+
+- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
+  - Example: "What is Alice's salary?" → `75000`
+  - Example: "How many items are in order ORD-0042?" → `3`
+  - Example: "What is the customer name for order ORD-0042?" → `John Doe`
+
+- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
+  - Example: "How many employees work in Engineering?" → `17`
+  - Example: "What is the total revenue across all orders?" → `45123.50`
+  - Example: "How many employees have salary > 80000?" → `23`
+
+- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
+  - Example: "How many employees in Sales have salary > 80000?" → `5`
+  - Example: "How many active employees have more than 10 years of experience?" → `8`
+
+- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
+  - Example: "How many employees are in the dataset?" → `100`
+  - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
+  - Example: "What is the department of the last employee?" → `Sales`
+
+- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
+  - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
+  - Tests TOON's `[N]` length validation and `{fields}` consistency checking
+  - Demonstrates CSV's lack of structural validation capabilities
+
+#### Evaluation Process
+
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
+2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
+3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
+
+#### Models & Configuration
+
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
+- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
+- **Temperature**: Not set (models use their defaults)
+- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
+
+<!-- /automd -->
+
+### Token Efficiency
+
+Token counts are measured using the GPT-5 `o200k_base` tokenizer via [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer). Savings are calculated against formatted JSON (2-space indentation) as the primary baseline, with additional comparisons to compact JSON (minified), YAML, and XML. Actual savings vary by model and tokenizer.
+
+The benchmarks test datasets across different structural patterns (uniform, semi-uniform, nested, deeply nested) to show where TOON excels and where other formats may be better.
+
+<!-- automd:file src="./benchmarks/results/token-efficiency.md" -->
+
+#### Mixed-Structure Track
+
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
+
+```
+🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
+   │
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
+
+🧾 Semi-uniform event logs  ┊  Tabular: 50%
+   │
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
+
+🧩 Deeply nested configuration  ┊  Tabular: 0%
+   │
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
+```
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+```
+👥 Uniform employee records  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
+
+📈 Time-series analytics data  ┊  Tabular: 100%
+   │
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
+
+⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
+```
+
+<details>
+<summary><strong>Show detailed examples</strong></summary>
+
+#### 📈 Time-series analytics data
+
+**Savings:** 13,130 tokens (59.0% reduction vs JSON)
+
+**JSON** (22,245 tokens):
+
+```json
+{
+  "metrics": [
+    {
+      "date": "2025-01-01",
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
+    },
+    {
+      "date": "2025-01-02",
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
+    },
+    {
+      "date": "2025-01-03",
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
+    },
+    {
+      "date": "2025-01-04",
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
+    },
+    {
+      "date": "2025-01-05",
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
+    }
+  ]
+}
+```
+
+**TOON** (9,115 tokens):
+
+```
+metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
+```
+
+---
+
+#### ⭐ Top 100 GitHub repositories
+
+**Savings:** 6,400 tokens (42.3% reduction vs JSON)
+
+**JSON** (15,144 tokens):
+
+```json
+{
+  "repositories": [
+    {
+      "id": 28457823,
+      "name": "freeCodeCamp",
+      "repo": "freeCodeCamp/freeCodeCamp",
+      "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
+      "createdAt": "2014-12-24T17:49:19Z",
+      "updatedAt": "2025-10-28T11:58:08Z",
+      "pushedAt": "2025-10-28T10:17:16Z",
+      "stars": 430886,
+      "watchers": 8583,
+      "forks": 42146,
+      "defaultBranch": "main"
+    },
+    {
+      "id": 132750724,
+      "name": "build-your-own-x",
+      "repo": "codecrafters-io/build-your-own-x",
+      "description": "Master programming by recreating your favorite technologies from scratch.",
+      "createdAt": "2018-05-09T12:03:18Z",
+      "updatedAt": "2025-10-28T12:37:11Z",
+      "pushedAt": "2025-10-10T18:45:01Z",
+      "stars": 430877,
+      "watchers": 6332,
+      "forks": 40453,
+      "defaultBranch": "master"
+    },
+    {
+      "id": 21737465,
+      "name": "awesome",
+      "repo": "sindresorhus/awesome",
+      "description": "😎 Awesome lists about all kinds of interesting topics",
+      "createdAt": "2014-07-11T13:42:37Z",
+      "updatedAt": "2025-10-28T12:40:21Z",
+      "pushedAt": "2025-10-27T17:57:31Z",
+      "stars": 410052,
+      "watchers": 8017,
+      "forks": 32029,
+      "defaultBranch": "main"
+    }
+  ]
+}
+```
+
+**TOON** (8,744 tokens):
+
+```
+repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
+  28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
+  132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
+  21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
+```
+
+</details>
+
+<!-- /automd -->
+
+## Installation & Quick Start
+
+### CLI (No Installation Required)
+
+Try TOON instantly with npx:
+
+```bash
+# Convert JSON to TOON
+npx @toon-format/cli input.json -o output.toon
+
+# Pipe from stdin
+echo '{"name": "Ada", "role": "dev"}' | npx @toon-format/cli
+```
+
+See the [CLI section](#cli) for all options and examples.
+
+### TypeScript Library
+
+```bash
+# npm
+npm install @toon-format/toon
+
+# pnpm
+pnpm add @toon-format/toon
+
+# yarn
+yarn add @toon-format/toon
+```
+
+**Example usage:**
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const data = {
+  users: [
+    { id: 1, name: 'Alice', role: 'admin' },
+    { id: 2, name: 'Bob', role: 'user' }
+  ]
+}
+
+console.log(encode(data))
+// users[2]{id,name,role}:
+//   1,Alice,admin
+//   2,Bob,user
+```
+
+**Streaming large datasets:**
+
+```ts
+import { encodeLines } from '@toon-format/toon'
+
+const largeData = await fetchThousandsOfRecords()
+
+// Memory-efficient streaming for large data
+for (const line of encodeLines(largeData)) {
+  process.stdout.write(`${line}\n`)
+}
+```
+
+> [!TIP]
+> For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options).
+
+**Transforming values with replacer:**
+
+```ts
+import { encode } from '@toon-format/toon'
+
+// Remove sensitive fields
+const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
+const safe = encode(user, {
+  replacer: (key, value) => key === 'password' ? undefined : value
+})
+// name: Alice
+// email: alice@example.com
+
+// Transform values
+const data = { status: 'active', count: 5 }
+const transformed = encode(data, {
+  replacer: (key, value) =>
+    typeof value === 'string' ? value.toUpperCase() : value
+})
+// status: ACTIVE
+// count: 5
+```
+
+> [!TIP]
+> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples.
+
+## Playgrounds
+
+Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.
+
+### Official Playground
+
+The [TOON Playground](https://toonformat.dev/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
+
+### Community Playgrounds
+
+- [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration)
+- [TOON Tools](https://toontools.vercel.app/)
+
+## Editor Support
+
+### VS Code
+
+[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) – Syntax highlighting, validation, conversion, and token analysis.
+
+```bash
+code --install-extension vishalraut.vscode-toon
+```
+
+### Tree-sitter Grammar
+
+[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) – Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
+
+### Neovim
+
+[toon.nvim](https://github.com/thalesgelinger/toon.nvim) – Lua-based plugin.
+
+### Other Editors
+
+Use YAML syntax highlighting as a close approximation.
+
+## CLI
+
+Command-line tool for quick JSON↔TOON conversions, token analysis, and pipeline integration. Auto-detects format from file extension, supports stdin/stdout workflows, and offers delimiter options for maximum efficiency.
+
+```bash
+# Encode JSON to TOON (auto-detected)
+npx @toon-format/cli input.json -o output.toon
+
+# Decode TOON to JSON (auto-detected)
+npx @toon-format/cli data.toon -o output.json
+
+# Pipe from stdin (no argument needed)
+cat data.json | npx @toon-format/cli
+echo '{"name": "Ada"}' | npx @toon-format/cli
+
+# Output to stdout
+npx @toon-format/cli input.json
+
+# Show token savings
+npx @toon-format/cli data.json --stats
+```
+
+> [!TIP]
+> See the full [CLI documentation](https://toonformat.dev/cli/) for all options, examples, and advanced usage.
+
+## Format Overview
+
+Detailed syntax references, implementation guides, and quick lookups for understanding and using the TOON format.
+
+- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax documentation
+- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick reference
+- [API Reference](https://toonformat.dev/reference/api) – Encode/decode usage (TypeScript)
+
+## Using TOON with LLMs
+
+TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern. Wrap data in ` ```toon` code blocks for input, and show the expected header template when asking models to generate TOON. Use tab delimiters for even better token efficiency.
+
+Follow the detailed [LLM integration guide](https://toonformat.dev/guide/llm-prompts) for strategies, examples, and validation techniques.
+
+## Documentation
+
+Comprehensive guides, references, and resources to help you get the most out of the TOON format and tools.
+
+### Getting Started
+
+- [Introduction & Installation](https://toonformat.dev/guide/getting-started) – What TOON is, when to use it, first steps
+- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax with examples
+- [Benchmarks](https://toonformat.dev/guide/benchmarks) – Accuracy & token efficiency results
+
+### Tools & Integration
+
+- [CLI](https://toonformat.dev/cli/) – Command-line tool for JSON↔TOON conversions
+- [Playgrounds](https://toonformat.dev/ecosystem/tools-and-playgrounds) – Interactive tools
+- [Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON
+- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation
+
+### References
+
+- [API Reference](https://toonformat.dev/reference/api) – TypeScript/JavaScript encode/decode API
+- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick format lookup
+- [Specification](https://github.com/toon-format/spec/blob/main/SPEC.md) – Normative rules for implementers
+
+## Other Implementations
+
+TOON has official and community implementations across multiple languages including Python, Rust, Go, Java, Swift, .NET, and many more.
+
+See the full list of implementations in the [documentation](https://toonformat.dev/ecosystem/implementations).
+
+## Credits
+
+- Logo design by [鈴木ックス(SZKX)](https://x.com/szkx_art)
+
+## License
+
+[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

packages/toon/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@toon-format/toon",
+  "type": "module",
+  "version": "2.3.1",
+  "packageManager": "pnpm@11.13.0",
+  "description": "Token-Oriented Object Notation (TOON) – Compact, human-readable, schema-aware encoding of JSON for LLM prompts",
+  "author": "Johann Schopplich <hello@johannschopplich.com>",
+  "license": "MIT",
+  "homepage": "https://toonformat.dev",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/toon-format/toon.git"
+  },
+  "bugs": {
+    "url": "https://github.com/toon-format/toon/issues"
+  },
+  "keywords": [
+    "toon",
+    "format",
+    "specification",
+    "llm",
+    "token-efficiency",
+    "data-format"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest"
+  },
+  "devDependencies": {
+    "@toon-format/spec": "^3.3.2"
+  }
+}

packages/toon/src/constants.ts

@@ -11,7 +11,7 @@ export const COMMA = ','
 export const COLON = ':'
 export const SPACE = ' '
 export const PIPE = '|'
-export const HASH = '#'
+export const DOT = '.'
 
 // #endregion