fix(decode): prevent false array header detection for unquoted keys c… (#314)

* fix(decode): prevent false array header detection for unquoted keys containing brackets and colons

* test(decode): pin stream parity for an unquoted bracket-colon scalar

* chore(toon): bump `@toon-format/spec` to ^3.3.2

---------

Co-authored-by: Johann Schopplich <johann@schopplich.com>

.github/workflows/ci.yml

@@ -9,78 +9,50 @@ on:
       - main
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 permissions:
   contents: read
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
+  ci:
+    runs-on: ubuntu-slim
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v5
+      - name: Checkout
+        uses: actions/checkout@v6
         with:
-          fetch-depth: 1
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v6
-        with:
-          node-version: 24
-      - name: Get pnpm store directory
-        id: pnpm-cache
-        run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v4
-        with:
-          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
-          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-store-
-      - run: pnpm install --frozen-lockfile
-      - run: pnpm run lint
+          persist-credentials: false
 
-  typecheck:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          fetch-depth: 1
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v6
-        with:
-          node-version: 24
-      - name: Get pnpm store directory
-        id: pnpm-cache
-        run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v4
-        with:
-          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
-          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-          restore-keys: |
-            ${{ runner.os }}-pnpm-store-
-      - run: pnpm install --frozen-lockfile
-      - run: pnpm run test:types
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
 
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          fetch-depth: 1
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v6
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
           node-version: 24
+
       - name: Get pnpm store directory
         id: pnpm-cache
         run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v4
+
+      - name: Cache pnpm dependencies
+        uses: actions/cache@v5
         with:
           path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
-      - run: pnpm install --frozen-lockfile
-      - run: pnpm run test
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Lint
+        run: pnpm run lint
+
+      - name: Typecheck
+        run: pnpm run test:types
+
+      - name: Test
+        run: pnpm run test

.github/workflows/deploy.yml

@@ -1,37 +1,56 @@
-name: Deploy
+name: Deploy Docs
 
 on:
   push:
     branches:
       - main
+    paths:
+      - docs/**
+      - automd.config.ts
+      - package.json
+      - eslint.config.mjs
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions: {}
+
 jobs:
   deploy:
-    runs-on: ubuntu-latest
+    name: Deploy Docs
+    runs-on: ubuntu-slim
     steps:
-      - uses: actions/checkout@v5
+      - name: Checkout
+        uses: actions/checkout@v6
         with:
-          fetch-depth: 0
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v6
+          persist-credentials: false
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
           node-version: 24
+
       - name: Get pnpm store directory
         id: pnpm-cache
         run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v4
+
+      - name: Cache pnpm dependencies
+        uses: actions/cache@v5
         with:
           path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
 
-      - run: pnpm install --frozen-lockfile
-      - run: pnpm run docs:build
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build docs
+        run: pnpm run docs:build
 
       - name: Deploy to Cloudflare
         run: cd docs && npx wrangler deploy

.github/workflows/pr-title.yml

@@ -0,0 +1,36 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    types: [opened, edited]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-pr-title:
+    name: Lint PR title
+    runs-on: ubuntu-slim
+
+    if: ${{ (github.event.action == 'opened' || github.event.changes.title != null) && github.actor != 'renovate[bot]' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          # Only fetch the config file from the repository
+          sparse-checkout-cone-mode: false
+          sparse-checkout: commitlint.config.ts
+
+      - name: Install dependencies
+        run: npm install -D @commitlint/cli @commitlint/config-conventional
+
+      - name: Validate PR title with commitlint
+        run: echo "$PR_TITLE" | npx commitlint
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}

.github/workflows/release.yml

@@ -1,43 +1,52 @@
-name: Release
+name: Release + Publish
 
 on:
   push:
     tags:
       - 'v*'
 
-permissions:
-  id-token: write
-  contents: write
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    name: Release
+    runs-on: ubuntu-slim
+
+    permissions:
+      id-token: write
+      contents: write
+
     steps:
-      - uses: actions/checkout@v5
+      - name: Checkout
+        uses: actions/checkout@v6
         with:
-          fetch-depth: 0
-      - uses: pnpm/action-setup@v4
-      - uses: actions/setup-node@v6
+          fetch-depth: 0 # Required for fetching tags and generating release notes
+          persist-credentials: true
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
           node-version: 24
           registry-url: https://registry.npmjs.org/
           cache: pnpm
 
-      - name: Publish changelog
+      - name: Generate changelog and create GitHub release
         run: npx changelogithub
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - run: pnpm install
-      - run: pnpm run build
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
 
-      - name: Copy READMEs for npm packages
-        run: |
-          cp README.md packages/toon/README.md
-          echo "✓ Copied root README to @toon-format/toon"
-          echo "✓ CLI package uses its own README"
+      - name: Build packages
+        run: pnpm run build
 
-      - name: Publish to npm
+      - name: Publish packages to npm
         run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.npmrc

@@ -1 +0,0 @@
-shamefully-hoist=true

.vscode/settings.json

@@ -1,7 +1,4 @@
 {
-  // Enable the ESLint flat config support
-  "eslint.useFlatConfig": true,
-
   // Disable the default formatter, use ESLint instead
   "prettier.enable": false,
   "editor.formatOnSave": false,
@@ -37,6 +34,17 @@
     "markdown",
     "json",
     "jsonc",
-    "yaml"
+    "yaml",
+    "toml",
+    "xml",
+    "gql",
+    "graphql",
+    "astro",
+    "svelte",
+    "css",
+    "less",
+    "scss",
+    "pcss",
+    "postcss"
   ]
 }

README.md

@@ -1,950 +0,0 @@
-![TOON logo with step‑by‑step guide](./.github/og.png)
-
-# Token-Oriented Object Notation (TOON)
-
-[![CI](https://github.com/toon-format/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon/actions)
-[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
-[![SPEC v2.0](https://img.shields.io/badge/spec-v2.0-lightgray)](https://github.com/toon-format/spec)
-[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
-[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
-
-**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
-
-TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably. For deeply nested or non-uniform data, JSON may be more efficient.
-
-The similarity to CSV is intentional: CSV is simple and ubiquitous, and TOON aims to keep that familiarity while remaining a lossless, drop-in representation of JSON for Large Language Models.
-
-Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input.
-
-> [!TIP]
-> The TOON format is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to the [spec](https://github.com/toon-format/spec) or sharing feedback.
-
-## Table of Contents
-
-- [Why TOON?](#why-toon)
-- [Key Features](#key-features)
-- [When Not to Use TOON](#when-not-to-use-toon)
-- [Benchmarks](#benchmarks)
-- [Installation & Quick Start](#installation--quick-start)
-- [Playgrounds](#playgrounds)
-- [Editor Support](#editor-support)
-- [CLI](#cli)
-- [Format Overview](#format-overview)
-- [Using TOON with LLMs](#using-toon-with-llms)
-- [Documentation](#documentation)
-- [Other Implementations](#other-implementations)
-- [📋 Full Specification](https://github.com/toon-format/spec/blob/main/SPEC.md)
-
-## Why TOON?
-
-AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – and standard JSON is verbose and token-expensive:
-
-```json
-{
-  "context": {
-    "task": "Our favorite hikes together",
-    "location": "Boulder",
-    "season": "spring_2025"
-  },
-  "friends": ["ana", "luis", "sam"],
-  "hikes": [
-    {
-      "id": 1,
-      "name": "Blue Lake Trail",
-      "distanceKm": 7.5,
-      "elevationGain": 320,
-      "companion": "ana",
-      "wasSunny": true
-    },
-    {
-      "id": 2,
-      "name": "Ridge Overlook",
-      "distanceKm": 9.2,
-      "elevationGain": 540,
-      "companion": "luis",
-      "wasSunny": false
-    },
-    {
-      "id": 3,
-      "name": "Wildflower Loop",
-      "distanceKm": 5.1,
-      "elevationGain": 180,
-      "companion": "sam",
-      "wasSunny": true
-    }
-  ]
-}
-```
-
-<details>
-<summary>YAML already conveys the same information with <strong>fewer tokens</strong>.</summary>
-
-```yaml
-context:
-  task: Our favorite hikes together
-  location: Boulder
-  season: spring_2025
-
-friends:
-  - ana
-  - luis
-  - sam
-
-hikes:
-  - id: 1
-    name: Blue Lake Trail
-    distanceKm: 7.5
-    elevationGain: 320
-    companion: ana
-    wasSunny: true
-  - id: 2
-    name: Ridge Overlook
-    distanceKm: 9.2
-    elevationGain: 540
-    companion: luis
-    wasSunny: false
-  - id: 3
-    name: Wildflower Loop
-    distanceKm: 5.1
-    elevationGain: 180
-    companion: sam
-    wasSunny: true
-```
-
-</details>
-
-TOON conveys the same information with **even fewer tokens** – combining YAML-like indentation with CSV-style tabular arrays:
-
-```yaml
-context:
-  task: Our favorite hikes together
-  location: Boulder
-  season: spring_2025
-friends[3]: ana,luis,sam
-hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
-  1,Blue Lake Trail,7.5,320,ana,true
-  2,Ridge Overlook,9.2,540,luis,false
-  3,Wildflower Loop,5.1,180,sam,true
-```
-
-## Key Features
-
-- 📊 **Token-Efficient & Accurate:** TOON reaches 74% accuracy (vs JSON's 70%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
-- 🔁 **JSON Data Model:** Encodes the same objects, arrays, and primitives as JSON with deterministic, lossless round-trips.
-- 🛤️ **LLM-Friendly Guardrails:** Explicit [N] lengths and {fields} headers give models a clear schema to follow, improving parsing reliability.
-- 📐 **Minimal Syntax:** Uses indentation instead of braces and minimizes quoting, giving YAML-like readability with CSV-style compactness.
-- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
-- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
-
-## When Not to Use TOON
-
-TOON excels with uniform arrays of objects, but there are cases where other formats are better:
-
-- **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels.
-- **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it.
-- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5-10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
-- **Latency-critical applications**: If end-to-end response time is your top priority, benchmark on your exact setup. Some deployments (especially local/quantized models like Ollama) may process compact JSON faster despite TOON's lower token count. Measure TTFT, tokens/sec, and total time for both formats and use whichever is faster.
-
-See [benchmarks](#benchmarks) for concrete comparisons across different data structures.
-
-## Benchmarks
-
-Benchmarks are organized into two tracks to ensure fair comparisons:
-
-- **Mixed-Structure Track**: Datasets with nested or semi-uniform structures (TOON vs JSON, YAML, XML). CSV excluded as it cannot properly represent these structures.
-- **Flat-Only Track**: Datasets with flat tabular structures where CSV is applicable (CSV vs TOON vs JSON, YAML, XML).
-
-### Retrieval Accuracy
-
-<!-- automd:file src="./benchmarks/results/retrieval-accuracy.md" -->
-
-Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
-
-<details>
-<summary><strong>Show Dataset Catalog</strong></summary>
-
-#### Dataset Catalog
-
-| Dataset | Rows | Structure | CSV Support | Eligibility |
-| ------- | ---- | --------- | ----------- | ----------- |
-| Uniform employee records | 100 | uniform | ✓ | 100% |
-| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
-| Time-series analytics data | 60 | uniform | ✓ | 100% |
-| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
-| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
-| Deeply nested configuration | 11 | deep | ✗ | 0% |
-| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
-| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
-| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
-| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
-| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
-
-**Structure classes:**
-- **uniform**: All objects have identical fields with primitive values
-- **semi-uniform**: Mix of uniform and non-uniform structures
-- **nested**: Objects with nested structures (nested objects or arrays)
-- **deep**: Highly nested with minimal tabular eligibility
-
-**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
-
-**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
-
-</details>
-
-#### Efficiency Ranking (Accuracy per 1K Tokens)
-
-Each format's overall performance, balancing accuracy against token cost:
-
-```
-TOON           ████████████████████   26.9  │  73.9% acc  │  2,744 tokens
-JSON compact   █████████████████░░░   22.9  │  70.7% acc  │  3,081 tokens
-YAML           ██████████████░░░░░░   18.6  │  69.0% acc  │  3,719 tokens
-JSON           ███████████░░░░░░░░░   15.3  │  69.7% acc  │  4,545 tokens
-XML            ██████████░░░░░░░░░░   13.0  │  67.1% acc  │  5,167 tokens
-```
-
-TOON achieves **73.9%** accuracy (vs JSON's 69.7%) while using **39.6% fewer tokens**.
-
-**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
-
-#### Per-Model Accuracy
-
-Accuracy across 4 LLMs on 209 data retrieval questions:
-
-```
-claude-haiku-4-5-20251001
-→ TOON           ████████████░░░░░░░░    59.8% (125/209)
-  JSON           ███████████░░░░░░░░░    57.4% (120/209)
-  YAML           ███████████░░░░░░░░░    56.0% (117/209)
-  XML            ███████████░░░░░░░░░    55.5% (116/209)
-  JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
-  CSV            ██████████░░░░░░░░░░    50.5% (55/109)
-
-gemini-2.5-flash
-→ TOON           ██████████████████░░    87.6% (183/209)
-  CSV            █████████████████░░░    86.2% (94/109)
-  JSON compact   ████████████████░░░░    82.3% (172/209)
-  YAML           ████████████████░░░░    79.4% (166/209)
-  XML            ████████████████░░░░    79.4% (166/209)
-  JSON           ███████████████░░░░░    77.0% (161/209)
-
-gpt-5-nano
-→ TOON           ██████████████████░░    90.9% (190/209)
-  JSON compact   ██████████████████░░    90.9% (190/209)
-  JSON           ██████████████████░░    89.0% (186/209)
-  CSV            ██████████████████░░    89.0% (97/109)
-  YAML           █████████████████░░░    87.1% (182/209)
-  XML            ████████████████░░░░    80.9% (169/209)
-
-grok-4-fast-non-reasoning
-→ TOON           ███████████░░░░░░░░░    57.4% (120/209)
-  JSON           ███████████░░░░░░░░░    55.5% (116/209)
-  JSON compact   ███████████░░░░░░░░░    54.5% (114/209)
-  YAML           ███████████░░░░░░░░░    53.6% (112/209)
-  XML            ███████████░░░░░░░░░    52.6% (110/209)
-  CSV            ██████████░░░░░░░░░░    52.3% (57/109)
-```
-
-> [!TIP] Results Summary
-> TOON achieves **73.9% accuracy** (vs JSON's 69.7%) while using **39.6% fewer tokens** on these datasets.
-
-<details>
-<summary><strong>Performance by dataset, model, and question type</strong></summary>
-
-#### Performance by Question Type
-
-| Question Type | TOON | JSON compact | JSON | CSV | YAML | XML |
-| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Field Retrieval | 99.6% | 99.3% | 99.3% | 100.0% | 98.2% | 98.9% |
-| Aggregation | 54.4% | 47.2% | 48.8% | 44.0% | 47.6% | 41.3% |
-| Filtering | 56.3% | 57.3% | 50.5% | 49.1% | 51.0% | 47.9% |
-| Structure Awareness | 88.0% | 83.0% | 83.0% | 85.9% | 80.0% | 80.0% |
-| Structural Validation | 70.0% | 45.0% | 50.0% | 80.0% | 60.0% | 80.0% |
-
-#### Performance by Dataset
-
-##### Uniform employee records
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 72.0% | 2,352 | 118/164 |
-| `toon` | 73.8% | 2,518 | 121/164 |
-| `json-compact` | 69.5% | 3,953 | 114/164 |
-| `yaml` | 68.3% | 4,982 | 112/164 |
-| `json-pretty` | 68.3% | 6,360 | 112/164 |
-| `xml` | 69.5% | 7,324 | 114/164 |
-
-##### E-commerce orders with nested structures
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `toon` | 81.1% | 7,232 | 133/164 |
-| `json-compact` | 76.8% | 6,794 | 126/164 |
-| `yaml` | 75.6% | 8,347 | 124/164 |
-| `json-pretty` | 76.2% | 10,713 | 125/164 |
-| `xml` | 74.4% | 12,023 | 122/164 |
-
-##### Time-series analytics data
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 73.3% | 1,406 | 88/120 |
-| `toon` | 72.5% | 1,548 | 87/120 |
-| `json-compact` | 71.7% | 2,349 | 86/120 |
-| `yaml` | 71.7% | 2,949 | 86/120 |
-| `json-pretty` | 68.3% | 3,676 | 82/120 |
-| `xml` | 68.3% | 4,384 | 82/120 |
-
-##### Top 100 GitHub repositories
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `toon` | 62.9% | 8,779 | 83/132 |
-| `csv` | 61.4% | 8,527 | 81/132 |
-| `yaml` | 59.8% | 13,141 | 79/132 |
-| `json-compact` | 55.3% | 11,464 | 73/132 |
-| `json-pretty` | 56.1% | 15,157 | 74/132 |
-| `xml` | 48.5% | 17,105 | 64/132 |
-
-##### Semi-uniform event logs
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `json-compact` | 63.3% | 4,819 | 76/120 |
-| `toon` | 57.5% | 5,799 | 69/120 |
-| `json-pretty` | 59.2% | 6,797 | 71/120 |
-| `yaml` | 48.3% | 5,827 | 58/120 |
-| `xml` | 46.7% | 7,709 | 56/120 |
-
-##### Deeply nested configuration
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `json-compact` | 92.2% | 574 | 107/116 |
-| `toon` | 95.7% | 666 | 111/116 |
-| `yaml` | 91.4% | 686 | 106/116 |
-| `json-pretty` | 94.0% | 932 | 109/116 |
-| `xml` | 92.2% | 1,018 | 107/116 |
-
-##### Valid complete dataset (control)
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `toon` | 100.0% | 544 | 4/4 |
-| `json-compact` | 100.0% | 795 | 4/4 |
-| `yaml` | 100.0% | 1,003 | 4/4 |
-| `json-pretty` | 100.0% | 1,282 | 4/4 |
-| `csv` | 25.0% | 492 | 1/4 |
-| `xml` | 0.0% | 1,467 | 0/4 |
-
-##### Array truncated: 3 rows removed from end
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 425 | 4/4 |
-| `xml` | 100.0% | 1,251 | 4/4 |
-| `toon` | 0.0% | 474 | 0/4 |
-| `json-compact` | 0.0% | 681 | 0/4 |
-| `json-pretty` | 0.0% | 1,096 | 0/4 |
-| `yaml` | 0.0% | 859 | 0/4 |
-
-##### Extra rows added beyond declared length
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 566 | 4/4 |
-| `toon` | 75.0% | 621 | 3/4 |
-| `xml` | 100.0% | 1,692 | 4/4 |
-| `yaml` | 75.0% | 1,157 | 3/4 |
-| `json-compact` | 50.0% | 917 | 2/4 |
-| `json-pretty` | 50.0% | 1,476 | 2/4 |
-
-##### Inconsistent field count (missing salary in row 10)
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 75.0% | 489 | 3/4 |
-| `yaml` | 100.0% | 996 | 4/4 |
-| `toon` | 100.0% | 1,019 | 4/4 |
-| `json-compact` | 75.0% | 790 | 3/4 |
-| `xml` | 100.0% | 1,458 | 4/4 |
-| `json-pretty` | 75.0% | 1,274 | 3/4 |
-
-##### Missing required fields (no email in multiple rows)
-
-| Format | Accuracy | Tokens | Correct/Total |
-| ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 329 | 4/4 |
-| `xml` | 100.0% | 1,411 | 4/4 |
-| `toon` | 75.0% | 983 | 3/4 |
-| `yaml` | 25.0% | 960 | 1/4 |
-| `json-pretty` | 25.0% | 1,230 | 1/4 |
-| `json-compact` | 0.0% | 755 | 0/4 |
-
-#### Performance by Model
-
-##### claude-haiku-4-5-20251001
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 59.8% | 125/209 |
-| `json-pretty` | 57.4% | 120/209 |
-| `yaml` | 56.0% | 117/209 |
-| `xml` | 55.5% | 116/209 |
-| `json-compact` | 55.0% | 115/209 |
-| `csv` | 50.5% | 55/109 |
-
-##### gemini-2.5-flash
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 87.6% | 183/209 |
-| `csv` | 86.2% | 94/109 |
-| `json-compact` | 82.3% | 172/209 |
-| `yaml` | 79.4% | 166/209 |
-| `xml` | 79.4% | 166/209 |
-| `json-pretty` | 77.0% | 161/209 |
-
-##### gpt-5-nano
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 90.9% | 190/209 |
-| `json-compact` | 90.9% | 190/209 |
-| `json-pretty` | 89.0% | 186/209 |
-| `csv` | 89.0% | 97/109 |
-| `yaml` | 87.1% | 182/209 |
-| `xml` | 80.9% | 169/209 |
-
-##### grok-4-fast-non-reasoning
-
-| Format | Accuracy | Correct/Total |
-| ------ | -------- | ------------- |
-| `toon` | 57.4% | 120/209 |
-| `json-pretty` | 55.5% | 116/209 |
-| `json-compact` | 54.5% | 114/209 |
-| `yaml` | 53.6% | 112/209 |
-| `xml` | 52.6% | 110/209 |
-| `csv` | 52.3% | 57/109 |
-
-</details>
-
-#### What's Being Measured
-
-This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
-
-#### Datasets Tested
-
-Eleven datasets designed to test different structural patterns and validation capabilities:
-
-**Primary datasets:**
-
-1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
-2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
-3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
-4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
-5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
-6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
-
-**Structural validation datasets:**
-
-7. **Control**: Valid complete dataset (baseline for validation)
-8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
-9. **Extra rows**: Array with 3 additional rows beyond declared length
-10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
-11. **Missing fields**: Systematic field omissions (no email in multiple rows)
-
-#### Question Types
-
-209 questions are generated dynamically across five categories:
-
-- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
-  - Example: "What is Alice's salary?" → `75000`
-  - Example: "How many items are in order ORD-0042?" → `3`
-  - Example: "What is the customer name for order ORD-0042?" → `John Doe`
-
-- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
-  - Example: "How many employees work in Engineering?" → `17`
-  - Example: "What is the total revenue across all orders?" → `45123.50`
-  - Example: "How many employees have salary > 80000?" → `23`
-
-- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
-  - Example: "How many employees in Sales have salary > 80000?" → `5`
-  - Example: "How many active employees have more than 10 years of experience?" → `8`
-
-- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
-  - Example: "How many employees are in the dataset?" → `100`
-  - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
-  - Example: "What is the department of the last employee?" → `Sales`
-
-- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
-  - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
-  - Tests TOON's `[N]` length validation and `{fields}` consistency checking
-  - Demonstrates CSV's lack of structural validation capabilities
-
-#### Evaluation Process
-
-1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON compact, JSON, CSV, YAML, XML).
-2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
-3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
-
-#### Models & Configuration
-
-- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `gpt-5-nano`, `grok-4-fast-non-reasoning`
-- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
-- **Temperature**: Not set (models use their defaults)
-- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
-
-<!-- /automd -->
-
-### Token Efficiency
-
-Token counts are measured using the GPT-5 `o200k_base` tokenizer via [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer). Savings are calculated against formatted JSON (2-space indentation) as the primary baseline, with additional comparisons to compact JSON (minified), YAML, and XML. Actual savings vary by model and tokenizer.
-
-The benchmarks test datasets across different structural patterns (uniform, semi-uniform, nested, deeply nested) to show where TOON excels and where other formats may be better.
-
-<!-- automd:file src="./benchmarks/results/token-efficiency.md" -->
-
-#### Mixed-Structure Track
-
-Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
-
-```
-🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
-   │
-   TOON                █████████████░░░░░░░    72,771 tokens
-   ├─ vs JSON          (−33.1%)               108,806 tokens
-   ├─ vs JSON compact  (+5.5%)                 68,975 tokens
-   ├─ vs YAML          (−14.2%)                84,780 tokens
-   └─ vs XML           (−40.5%)               122,406 tokens
-
-🧾 Semi-uniform event logs  ┊  Tabular: 50%
-   │
-   TOON                █████████████████░░░   153,211 tokens
-   ├─ vs JSON          (−15.0%)               180,176 tokens
-   ├─ vs JSON compact  (+19.9%)               127,731 tokens
-   ├─ vs YAML          (−0.8%)                154,505 tokens
-   └─ vs XML           (−25.2%)               204,777 tokens
-
-🧩 Deeply nested configuration  ┊  Tabular: 0%
-   │
-   TOON                ██████████████░░░░░░       631 tokens
-   ├─ vs JSON          (−31.3%)                   919 tokens
-   ├─ vs JSON compact  (+11.9%)                   564 tokens
-   ├─ vs YAML          (−6.2%)                    673 tokens
-   └─ vs XML           (−37.4%)                 1,008 tokens
-
-──────────────────────────────────── Total ────────────────────────────────────
-   TOON                ████████████████░░░░   226,613 tokens
-   ├─ vs JSON          (−21.8%)               289,901 tokens
-   ├─ vs JSON compact  (+14.9%)               197,270 tokens
-   ├─ vs YAML          (−5.6%)                239,958 tokens
-   └─ vs XML           (−31.0%)               328,191 tokens
-```
-
-#### Flat-Only Track
-
-Datasets with flat tabular structures where CSV is applicable.
-
-```
-👥 Uniform employee records  ┊  Tabular: 100%
-   │
-   CSV                 ███████████████████░    46,954 tokens
-   TOON                ████████████████████    49,831 tokens   (+6.1% vs CSV)
-   ├─ vs JSON          (−60.7%)               126,860 tokens
-   ├─ vs JSON compact  (−36.8%)                78,856 tokens
-   ├─ vs YAML          (−50.0%)                99,706 tokens
-   └─ vs XML           (−66.0%)               146,444 tokens
-
-📈 Time-series analytics data  ┊  Tabular: 100%
-   │
-   CSV                 ██████████████████░░     8,388 tokens
-   TOON                ████████████████████     9,120 tokens   (+8.7% vs CSV)
-   ├─ vs JSON          (−59.0%)                22,250 tokens
-   ├─ vs JSON compact  (−35.8%)                14,216 tokens
-   ├─ vs YAML          (−48.9%)                17,863 tokens
-   └─ vs XML           (−65.7%)                26,621 tokens
-
-⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
-   │
-   CSV                 ███████████████████░     8,513 tokens
-   TOON                ████████████████████     8,745 tokens   (+2.7% vs CSV)
-   ├─ vs JSON          (−42.3%)                15,145 tokens
-   ├─ vs JSON compact  (−23.7%)                11,455 tokens
-   ├─ vs YAML          (−33.4%)                13,129 tokens
-   └─ vs XML           (−48.8%)                17,095 tokens
-
-──────────────────────────────────── Total ────────────────────────────────────
-   CSV                 ███████████████████░    63,855 tokens
-   TOON                ████████████████████    67,696 tokens   (+6.0% vs CSV)
-   ├─ vs JSON          (−58.8%)               164,255 tokens
-   ├─ vs JSON compact  (−35.2%)               104,527 tokens
-   ├─ vs YAML          (−48.2%)               130,698 tokens
-   └─ vs XML           (−64.4%)               190,160 tokens
-```
-
-<details>
-<summary><strong>Show detailed examples</strong></summary>
-
-#### 📈 Time-series analytics data
-
-**Savings:** 13,130 tokens (59.0% reduction vs JSON)
-
-**JSON** (22,250 tokens):
-
-```json
-{
-  "metrics": [
-    {
-      "date": "2025-01-01",
-      "views": 5715,
-      "clicks": 211,
-      "conversions": 28,
-      "revenue": 7976.46,
-      "bounceRate": 0.47
-    },
-    {
-      "date": "2025-01-02",
-      "views": 7103,
-      "clicks": 393,
-      "conversions": 28,
-      "revenue": 8360.53,
-      "bounceRate": 0.32
-    },
-    {
-      "date": "2025-01-03",
-      "views": 7248,
-      "clicks": 378,
-      "conversions": 24,
-      "revenue": 3212.57,
-      "bounceRate": 0.5
-    },
-    {
-      "date": "2025-01-04",
-      "views": 2927,
-      "clicks": 77,
-      "conversions": 11,
-      "revenue": 1211.69,
-      "bounceRate": 0.62
-    },
-    {
-      "date": "2025-01-05",
-      "views": 3530,
-      "clicks": 82,
-      "conversions": 8,
-      "revenue": 462.77,
-      "bounceRate": 0.56
-    }
-  ]
-}
-```
-
-**TOON** (9,120 tokens):
-
-```
-metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
-  2025-01-01,5715,211,28,7976.46,0.47
-  2025-01-02,7103,393,28,8360.53,0.32
-  2025-01-03,7248,378,24,3212.57,0.5
-  2025-01-04,2927,77,11,1211.69,0.62
-  2025-01-05,3530,82,8,462.77,0.56
-```
-
----
-
-#### ⭐ Top 100 GitHub repositories
-
-**Savings:** 6,400 tokens (42.3% reduction vs JSON)
-
-**JSON** (15,145 tokens):
-
-```json
-{
-  "repositories": [
-    {
-      "id": 28457823,
-      "name": "freeCodeCamp",
-      "repo": "freeCodeCamp/freeCodeCamp",
-      "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
-      "createdAt": "2014-12-24T17:49:19Z",
-      "updatedAt": "2025-10-28T11:58:08Z",
-      "pushedAt": "2025-10-28T10:17:16Z",
-      "stars": 430886,
-      "watchers": 8583,
-      "forks": 42146,
-      "defaultBranch": "main"
-    },
-    {
-      "id": 132750724,
-      "name": "build-your-own-x",
-      "repo": "codecrafters-io/build-your-own-x",
-      "description": "Master programming by recreating your favorite technologies from scratch.",
-      "createdAt": "2018-05-09T12:03:18Z",
-      "updatedAt": "2025-10-28T12:37:11Z",
-      "pushedAt": "2025-10-10T18:45:01Z",
-      "stars": 430877,
-      "watchers": 6332,
-      "forks": 40453,
-      "defaultBranch": "master"
-    },
-    {
-      "id": 21737465,
-      "name": "awesome",
-      "repo": "sindresorhus/awesome",
-      "description": "😎 Awesome lists about all kinds of interesting topics",
-      "createdAt": "2014-07-11T13:42:37Z",
-      "updatedAt": "2025-10-28T12:40:21Z",
-      "pushedAt": "2025-10-27T17:57:31Z",
-      "stars": 410052,
-      "watchers": 8017,
-      "forks": 32029,
-      "defaultBranch": "main"
-    }
-  ]
-}
-```
-
-**TOON** (8,745 tokens):
-
-```
-repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
-  28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
-  132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
-  21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
-```
-
-</details>
-
-<!-- /automd -->
-
-## Installation & Quick Start
-
-### CLI (No Installation Required)
-
-Try TOON instantly with npx:
-
-```bash
-# Convert JSON to TOON
-npx @toon-format/cli input.json -o output.toon
-
-# Pipe from stdin
-echo '{"name": "Ada", "role": "dev"}' | npx @toon-format/cli
-```
-
-See the [CLI section](#cli) for all options and examples.
-
-### TypeScript Library
-
-```bash
-# npm
-npm install @toon-format/toon
-
-# pnpm
-pnpm add @toon-format/toon
-
-# yarn
-yarn add @toon-format/toon
-```
-
-**Example usage:**
-
-```ts
-import { encode } from '@toon-format/toon'
-
-const data = {
-  users: [
-    { id: 1, name: 'Alice', role: 'admin' },
-    { id: 2, name: 'Bob', role: 'user' }
-  ]
-}
-
-console.log(encode(data))
-// users[2]{id,name,role}:
-//   1,Alice,admin
-//   2,Bob,user
-```
-
-**Streaming large datasets:**
-
-```ts
-import { encodeLines } from '@toon-format/toon'
-
-const largeData = await fetchThousandsOfRecords()
-
-// Memory-efficient streaming for large data
-for (const line of encodeLines(largeData)) {
-  process.stdout.write(`${line}\n`)
-}
-```
-
-**Streaming decode:**
-
-```ts
-import { decodeFromLines, decodeStreamSync } from '@toon-format/toon'
-
-// 1. Lines → value (build full JSON value)
-const value = decodeFromLines([
-  'users[2]{id,name}:',
-  '  1,Alice',
-  '  2,Bob',
-])
-// { users: [{ id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }] }
-
-// 2. Lines → events (for custom streaming consumers)
-const lines = [
-  'users[2]{id,name}:',
-  '  1,Alice',
-  '  2,Bob',
-]
-for (const event of decodeStreamSync(lines)) {
-  // { type: 'startObject' }, { type: 'key', key: 'users' }, ...
-}
-```
-
-**Async streaming decode:**
-
-```ts
-// 3. Async streaming from files or network
-import { createReadStream } from 'node:fs'
-import { createInterface } from 'node:readline'
-import { decodeStream } from '@toon-format/toon'
-
-const fileStream = createReadStream('data.toon', 'utf-8')
-const rl = createInterface({ input: fileStream })
-
-for await (const event of decodeStream(rl)) {
-  // Process events as they arrive
-}
-```
-
-## Playgrounds
-
-Experiment with TOON format interactively using these community-built tools for token comparison, format conversion, and validation:
-
-- [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration)
-- [TOON Tools](https://toontools.vercel.app/)
-
-## Editor Support
-
-### VS Code
-
-[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) - Syntax highlighting, validation, conversion, and token analysis.
-
-```bash
-code --install-extension vishalraut.vscode-toon
-```
-
-### Tree-sitter Grammar
-
-[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) - Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
-
-### Neovim
-
-[toon.nvim](https://github.com/thalesgelinger/toon.nvim) - Lua-based plugin.
-
-### Other Editors
-
-Use YAML syntax highlighting as a close approximation.
-
-## CLI
-
-Command-line tool for quick JSON↔TOON conversions, token analysis, and pipeline integration. Auto-detects format from file extension, supports stdin/stdout workflows, and offers delimiter options for maximum efficiency.
-
-```bash
-# Encode JSON to TOON (auto-detected)
-npx @toon-format/cli input.json -o output.toon
-
-# Decode TOON to JSON (auto-detected)
-npx @toon-format/cli data.toon -o output.json
-
-# Pipe from stdin (no argument needed)
-cat data.json | npx @toon-format/cli
-echo '{"name": "Ada"}' | npx @toon-format/cli
-
-# Output to stdout
-npx @toon-format/cli input.json
-
-# Show token savings
-npx @toon-format/cli data.json --stats
-```
-
-> [!TIP]
-> See the full [CLI documentation](https://toonformat.dev/cli/) for all options, examples, and advanced usage.
-
-## Format Overview
-
-Detailed syntax references, implementation guides, and quick lookups for understanding and using the TOON format.
-
-- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax documentation
-- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick reference
-- [API Reference](https://toonformat.dev/reference/api) – Encode/decode usage (TypeScript)
-
-## Using TOON with LLMs
-
-TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern. Wrap data in ` ```toon` code blocks for input, and show the expected header template when asking models to generate TOON. Use tab delimiters for even better token efficiency.
-
-Follow the detailed [LLM integration guide](https://toonformat.dev/guide/llm-prompts) for strategies, examples, and validation techniques.
-
-## Documentation
-
-Comprehensive guides, references, and resources to help you get the most out of the TOON format and tools.
-
-**Getting Started**
-- [Introduction & Installation](https://toonformat.dev/guide/getting-started) – What TOON is, when to use it, first steps
-- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax with examples
-- [Benchmarks](https://toonformat.dev/guide/benchmarks) – Accuracy & token efficiency results
-
-**Tools & Integration**
-- [CLI](https://toonformat.dev/cli/) – Command-line tool for  JSON↔TOON conversions
-- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation
-- [Playgrounds](https://toonformat.dev/ecosystem/tools-and-playgrounds) – Interactive tools
-
-**Reference**
-- [API Reference](https://toonformat.dev/reference/api) – TypeScript/JavaScript encode/decode API
-- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick format lookup
-- [Specification v2.0](https://github.com/toon-format/spec/blob/main/SPEC.md) – Normative rules for implementers
-
-## Other Implementations
-
-> [!NOTE]
-> When implementing TOON in other languages, please follow the [specification](https://github.com/toon-format/spec/blob/main/SPEC.md) (currently v2.0) to ensure compatibility across implementations. The [conformance tests](https://github.com/toon-format/spec/tree/main/tests) provide language-agnostic test fixtures that validate your implementations.
-
-### Official Implementations
-
-> [!TIP]
-> These implementations are actively being developed by dedicated teams. Contributions are welcome! Join the effort by opening issues, submitting PRs, or discussing implementation details in the respective repositories.
-
-- **.NET:** [toon_format](https://github.com/toon-format/toon-dotnet) *(in development)*
-- **Dart:** [toon](https://github.com/toon-format/toon-dart) *(in development)*
-- **Go:** [toon-go](https://github.com/toon-format/toon-go) *(in development)*
-- **Python:** [toon_format](https://github.com/toon-format/toon-python) *(in development)*
-- **Rust:** [toon_format](https://github.com/toon-format/toon-rust) *(in development)*
-
-### Community Implementations
-
-- **Apex:** [ApexToon](https://github.com/Eacaw/ApexToon)
-- **C++:** [ctoon](https://github.com/mohammadraziei/ctoon)
-- **Clojure:** [toon](https://github.com/vadelabs/toon)
-- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)
-- **Elixir:** [toon_ex](https://github.com/kentaro/toon_ex)
-- **Gleam:** [toon_codec](https://github.com/axelbellec/toon_codec)
-- **Go:** [gotoon](https://github.com/alpkeskin/gotoon)
-- **Java:** [JToon](https://github.com/felipestanzani/JToon)
-- **Scala:** [toon4s](https://github.com/vim89/toon4s)
-- **Lua/Neovim:** [toon.nvim](https://github.com/thalesgelinger/toon.nvim)
-- **OCaml:** [ocaml-toon](https://github.com/davesnx/ocaml-toon)
-- **Perl:** [Data::TOON](https://github.com/ytnobody/p5-Data-TOON)
-- **PHP:** [toon-php](https://github.com/HelgeSverre/toon-php)
-- **Laravel Framework:** [laravel-toon](https://github.com/jobmetric/laravel-toon)
-- **R**: [toon](https://github.com/laresbernardo/toon)
-- **Ruby:** [toon-ruby](https://github.com/andrepcg/toon-ruby)
-- **Swift:** [TOONEncoder](https://github.com/mattt/TOONEncoder)
-- **Kotlin:** [Kotlin-Toon Encoder/Decoder](https://github.com/vexpera-br/kotlin-toon)
-
-## Credits
-
-- Logo design by [鈴木ックス(SZKX)](https://x.com/szkx_art)
-
-## License
-
-[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

README.md

@@ -0,0 +1 @@
+packages/toon/README.md

SPEC.md

@@ -4,7 +4,7 @@ The TOON specification has moved to a dedicated repository: [github.com/toon-for
 
 ## Current Version
 
-**Version 2.0** (2025-11-10)
+**Version 3.3** (2026-05-21)
 
 ## Quick Links
 

benchmarks/README.md

@@ -3,7 +3,7 @@
 Benchmarks measuring TOON's **token efficiency** and **retrieval accuracy** compared to JSON, XML, YAML, and CSV.
 
 > [!NOTE]
-> Results are automatically embedded in the [main README](../README.md#benchmarks). This guide focuses on running the benchmarks locally.
+> Results are automatically embedded in the [main README](https://github.com/toon-format/toon/#benchmarks). This guide focuses on running the benchmarks locally.
 
 ## Quick Start
 
@@ -44,11 +44,11 @@ Tests how well LLMs can answer questions about data in different formats (TOON,
 
 1. Edit [`src/evaluate.ts`](./src/evaluate.ts) and add models to the exported `models` array:
    ```ts
-   export const models: LanguageModelV2[] = [
+   export const models: LanguageModelV3[] = [
      openai('gpt-5-nano'),
      anthropic('claude-haiku-4-5-20251001'),
-     google('gemini-2.5-flash'),
-     xai('grok-4-fast-non-reasoning'),
+     google('gemini-3-flash-preview'),
+     xai('grok-4-1-fast-non-reasoning'),
      // Add your models here
    ]
    ```

benchmarks/package.json

@@ -3,26 +3,26 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "benchmark:tokens": "tsx scripts/token-efficiency-benchmark.ts",
-    "benchmark:accuracy": "tsx --env-file=.env scripts/accuracy-benchmark.ts",
-    "fetch:github-repos": "tsx scripts/fetch-github-repos.ts"
+    "benchmark:tokens": "node scripts/token-efficiency-benchmark.ts",
+    "benchmark:accuracy": "node --env-file=.env scripts/accuracy-benchmark.ts",
+    "fetch:github-repos": "node scripts/fetch-github-repos.ts"
   },
   "devDependencies": {
-    "@ai-sdk/anthropic": "^2.0.44",
-    "@ai-sdk/google": "^2.0.31",
-    "@ai-sdk/openai": "^2.0.65",
-    "@ai-sdk/provider": "^2.0.0",
-    "@ai-sdk/xai": "^2.0.32",
-    "@clack/prompts": "^0.11.0",
-    "@faker-js/faker": "^10.1.0",
-    "ai": "^5.0.92",
-    "csv-stringify": "^6.6.0",
-    "fast-xml-parser": "^5.3.1",
+    "@ai-sdk/anthropic": "^3.0.97",
+    "@ai-sdk/google": "^3.0.94",
+    "@ai-sdk/openai": "^3.0.85",
+    "@ai-sdk/provider": "^3.0.14",
+    "@ai-sdk/xai": "^3.0.108",
+    "@clack/prompts": "^1.7.0",
+    "@faker-js/faker": "^10.5.0",
+    "ai": "^6.0.228",
+    "csv-stringify": "^6.8.1",
+    "fast-xml-parser": "^5.10.0",
     "gpt-tokenizer": "^3.4.0",
     "ofetch": "^1.5.1",
-    "p-map": "^7.0.4",
-    "p-queue": "^9.0.0",
-    "unstorage": "^1.17.2",
-    "yaml": "^2.8.1"
+    "p-map": "^7.0.5",
+    "p-queue": "^9.3.1",
+    "unstorage": "^1.17.5",
+    "yaml": "^2.9.0"
   }
 }

benchmarks/results/retrieval-accuracy.md

@@ -33,17 +33,20 @@ Benchmarks test LLM comprehension across different input formats using 209 data
 
 #### Efficiency Ranking (Accuracy per 1K Tokens)
 
-Each format's overall performance, balancing accuracy against token cost:
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
 
 ```
-TOON           ████████████████████   26.9  │  73.9% acc  │  2,744 tokens
-JSON compact   █████████████████░░░   22.9  │  70.7% acc  │  3,081 tokens
-YAML           ██████████████░░░░░░   18.6  │  69.0% acc  │  3,719 tokens
-JSON           ███████████░░░░░░░░░   15.3  │  69.7% acc  │  4,545 tokens
-XML            ██████████░░░░░░░░░░   13.0  │  67.1% acc  │  5,167 tokens
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
 ```
 
-TOON achieves **73.9%** accuracy (vs JSON's 69.7%) while using **39.6% fewer tokens**.
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
 
 **Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
 
@@ -60,13 +63,13 @@ claude-haiku-4-5-20251001
   JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
   CSV            ██████████░░░░░░░░░░    50.5% (55/109)
 
-gemini-2.5-flash
-→ TOON           ██████████████████░░    87.6% (183/209)
-  CSV            █████████████████░░░    86.2% (94/109)
-  JSON compact   ████████████████░░░░    82.3% (172/209)
-  YAML           ████████████████░░░░    79.4% (166/209)
-  XML            ████████████████░░░░    79.4% (166/209)
-  JSON           ███████████████░░░░░    77.0% (161/209)
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
 
 gpt-5-nano
 → TOON           ██████████████████░░    90.9% (190/209)
@@ -76,30 +79,30 @@ gpt-5-nano
   YAML           █████████████████░░░    87.1% (182/209)
   XML            ████████████████░░░░    80.9% (169/209)
 
-grok-4-fast-non-reasoning
-→ TOON           ███████████░░░░░░░░░    57.4% (120/209)
-  JSON           ███████████░░░░░░░░░    55.5% (116/209)
-  JSON compact   ███████████░░░░░░░░░    54.5% (114/209)
-  YAML           ███████████░░░░░░░░░    53.6% (112/209)
-  XML            ███████████░░░░░░░░░    52.6% (110/209)
-  CSV            ██████████░░░░░░░░░░    52.3% (57/109)
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
 ```
 
-> [!TIP] Results Summary
-> TOON achieves **73.9% accuracy** (vs JSON's 69.7%) while using **39.6% fewer tokens** on these datasets.
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
 
 <details>
 <summary><strong>Performance by dataset, model, and question type</strong></summary>
 
 #### Performance by Question Type
 
-| Question Type | TOON | JSON compact | JSON | CSV | YAML | XML |
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
 | ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Field Retrieval | 99.6% | 99.3% | 99.3% | 100.0% | 98.2% | 98.9% |
-| Aggregation | 54.4% | 47.2% | 48.8% | 44.0% | 47.6% | 41.3% |
-| Filtering | 56.3% | 57.3% | 50.5% | 49.1% | 51.0% | 47.9% |
-| Structure Awareness | 88.0% | 83.0% | 83.0% | 85.9% | 80.0% | 80.0% |
-| Structural Validation | 70.0% | 45.0% | 50.0% | 80.0% | 60.0% | 80.0% |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
 
 #### Performance by Dataset
 
@@ -107,119 +110,119 @@ grok-4-fast-non-reasoning
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 72.0% | 2,352 | 118/164 |
-| `toon` | 73.8% | 2,518 | 121/164 |
-| `json-compact` | 69.5% | 3,953 | 114/164 |
-| `yaml` | 68.3% | 4,982 | 112/164 |
-| `json-pretty` | 68.3% | 6,360 | 112/164 |
-| `xml` | 69.5% | 7,324 | 114/164 |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
 
 ##### E-commerce orders with nested structures
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 81.1% | 7,232 | 133/164 |
-| `json-compact` | 76.8% | 6,794 | 126/164 |
-| `yaml` | 75.6% | 8,347 | 124/164 |
-| `json-pretty` | 76.2% | 10,713 | 125/164 |
-| `xml` | 74.4% | 12,023 | 122/164 |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
 
 ##### Time-series analytics data
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 73.3% | 1,406 | 88/120 |
-| `toon` | 72.5% | 1,548 | 87/120 |
-| `json-compact` | 71.7% | 2,349 | 86/120 |
-| `yaml` | 71.7% | 2,949 | 86/120 |
-| `json-pretty` | 68.3% | 3,676 | 82/120 |
-| `xml` | 68.3% | 4,384 | 82/120 |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
 
 ##### Top 100 GitHub repositories
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 62.9% | 8,779 | 83/132 |
-| `csv` | 61.4% | 8,527 | 81/132 |
-| `yaml` | 59.8% | 13,141 | 79/132 |
-| `json-compact` | 55.3% | 11,464 | 73/132 |
-| `json-pretty` | 56.1% | 15,157 | 74/132 |
-| `xml` | 48.5% | 17,105 | 64/132 |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
 
 ##### Semi-uniform event logs
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `json-compact` | 63.3% | 4,819 | 76/120 |
-| `toon` | 57.5% | 5,799 | 69/120 |
-| `json-pretty` | 59.2% | 6,797 | 71/120 |
-| `yaml` | 48.3% | 5,827 | 58/120 |
-| `xml` | 46.7% | 7,709 | 56/120 |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
 
 ##### Deeply nested configuration
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `json-compact` | 92.2% | 574 | 107/116 |
-| `toon` | 95.7% | 666 | 111/116 |
-| `yaml` | 91.4% | 686 | 106/116 |
-| `json-pretty` | 94.0% | 932 | 109/116 |
-| `xml` | 92.2% | 1,018 | 107/116 |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
 
 ##### Valid complete dataset (control)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 100.0% | 544 | 4/4 |
-| `json-compact` | 100.0% | 795 | 4/4 |
-| `yaml` | 100.0% | 1,003 | 4/4 |
-| `json-pretty` | 100.0% | 1,282 | 4/4 |
-| `csv` | 25.0% | 492 | 1/4 |
-| `xml` | 0.0% | 1,467 | 0/4 |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
 
 ##### Array truncated: 3 rows removed from end
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 425 | 4/4 |
-| `xml` | 100.0% | 1,251 | 4/4 |
-| `toon` | 0.0% | 474 | 0/4 |
-| `json-compact` | 0.0% | 681 | 0/4 |
-| `json-pretty` | 0.0% | 1,096 | 0/4 |
-| `yaml` | 0.0% | 859 | 0/4 |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
 
 ##### Extra rows added beyond declared length
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 566 | 4/4 |
-| `toon` | 75.0% | 621 | 3/4 |
-| `xml` | 100.0% | 1,692 | 4/4 |
-| `yaml` | 75.0% | 1,157 | 3/4 |
-| `json-compact` | 50.0% | 917 | 2/4 |
-| `json-pretty` | 50.0% | 1,476 | 2/4 |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
 
 ##### Inconsistent field count (missing salary in row 10)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 75.0% | 489 | 3/4 |
-| `yaml` | 100.0% | 996 | 4/4 |
-| `toon` | 100.0% | 1,019 | 4/4 |
-| `json-compact` | 75.0% | 790 | 3/4 |
-| `xml` | 100.0% | 1,458 | 4/4 |
-| `json-pretty` | 75.0% | 1,274 | 3/4 |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
 
 ##### Missing required fields (no email in multiple rows)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 329 | 4/4 |
-| `xml` | 100.0% | 1,411 | 4/4 |
-| `toon` | 75.0% | 983 | 3/4 |
-| `yaml` | 25.0% | 960 | 1/4 |
-| `json-pretty` | 25.0% | 1,230 | 1/4 |
-| `json-compact` | 0.0% | 755 | 0/4 |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
 
 #### Performance by Model
 
@@ -234,16 +237,16 @@ grok-4-fast-non-reasoning
 | `json-compact` | 55.0% | 115/209 |
 | `csv` | 50.5% | 55/109 |
 
-##### gemini-2.5-flash
+##### gemini-3-flash-preview
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `toon` | 87.6% | 183/209 |
-| `csv` | 86.2% | 94/109 |
-| `json-compact` | 82.3% | 172/209 |
-| `yaml` | 79.4% | 166/209 |
-| `xml` | 79.4% | 166/209 |
-| `json-pretty` | 77.0% | 161/209 |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
 
 ##### gpt-5-nano
 
@@ -256,16 +259,16 @@ grok-4-fast-non-reasoning
 | `yaml` | 87.1% | 182/209 |
 | `xml` | 80.9% | 169/209 |
 
-##### grok-4-fast-non-reasoning
+##### grok-4-1-fast-non-reasoning
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `toon` | 57.4% | 120/209 |
-| `json-pretty` | 55.5% | 116/209 |
-| `json-compact` | 54.5% | 114/209 |
-| `yaml` | 53.6% | 112/209 |
-| `xml` | 52.6% | 110/209 |
-| `csv` | 52.3% | 57/109 |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
 
 </details>
 
@@ -324,13 +327,13 @@ Eleven datasets designed to test different structural patterns and validation ca
 
 #### Evaluation Process
 
-1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON compact, JSON, CSV, YAML, XML).
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
 2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
 3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
 
 #### Models & Configuration
 
-- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `gpt-5-nano`, `grok-4-fast-non-reasoning`
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
 - **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
 - **Temperature**: Not set (models use their defaults)
 - **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls

benchmarks/results/token-efficiency.md

@@ -5,34 +5,34 @@ Datasets with nested or semi-uniform structures. CSV excluded as it cannot prope
 ```
 🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
    │
-   TOON                █████████████░░░░░░░    72,771 tokens
-   ├─ vs JSON          (−33.1%)               108,806 tokens
-   ├─ vs JSON compact  (+5.5%)                 68,975 tokens
-   ├─ vs YAML          (−14.2%)                84,780 tokens
-   └─ vs XML           (−40.5%)               122,406 tokens
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
 
 🧾 Semi-uniform event logs  ┊  Tabular: 50%
    │
-   TOON                █████████████████░░░   153,211 tokens
-   ├─ vs JSON          (−15.0%)               180,176 tokens
-   ├─ vs JSON compact  (+19.9%)               127,731 tokens
-   ├─ vs YAML          (−0.8%)                154,505 tokens
-   └─ vs XML           (−25.2%)               204,777 tokens
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
 
 🧩 Deeply nested configuration  ┊  Tabular: 0%
    │
-   TOON                ██████████████░░░░░░       631 tokens
-   ├─ vs JSON          (−31.3%)                   919 tokens
-   ├─ vs JSON compact  (+11.9%)                   564 tokens
-   ├─ vs YAML          (−6.2%)                    673 tokens
-   └─ vs XML           (−37.4%)                 1,008 tokens
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
 
 ──────────────────────────────────── Total ────────────────────────────────────
-   TOON                ████████████████░░░░   226,613 tokens
-   ├─ vs JSON          (−21.8%)               289,901 tokens
-   ├─ vs JSON compact  (+14.9%)               197,270 tokens
-   ├─ vs YAML          (−5.6%)                239,958 tokens
-   └─ vs XML           (−31.0%)               328,191 tokens
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
 ```
 
 #### Flat-Only Track
@@ -42,38 +42,38 @@ Datasets with flat tabular structures where CSV is applicable.
 ```
 👥 Uniform employee records  ┊  Tabular: 100%
    │
-   CSV                 ███████████████████░    46,954 tokens
-   TOON                ████████████████████    49,831 tokens   (+6.1% vs CSV)
-   ├─ vs JSON          (−60.7%)               126,860 tokens
-   ├─ vs JSON compact  (−36.8%)                78,856 tokens
-   ├─ vs YAML          (−50.0%)                99,706 tokens
-   └─ vs XML           (−66.0%)               146,444 tokens
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
 
 📈 Time-series analytics data  ┊  Tabular: 100%
    │
-   CSV                 ██████████████████░░     8,388 tokens
-   TOON                ████████████████████     9,120 tokens   (+8.7% vs CSV)
-   ├─ vs JSON          (−59.0%)                22,250 tokens
-   ├─ vs JSON compact  (−35.8%)                14,216 tokens
-   ├─ vs YAML          (−48.9%)                17,863 tokens
-   └─ vs XML           (−65.7%)                26,621 tokens
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
 
 ⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
    │
-   CSV                 ███████████████████░     8,513 tokens
-   TOON                ████████████████████     8,745 tokens   (+2.7% vs CSV)
-   ├─ vs JSON          (−42.3%)                15,145 tokens
-   ├─ vs JSON compact  (−23.7%)                11,455 tokens
-   ├─ vs YAML          (−33.4%)                13,129 tokens
-   └─ vs XML           (−48.8%)                17,095 tokens
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
 
 ──────────────────────────────────── Total ────────────────────────────────────
-   CSV                 ███████████████████░    63,855 tokens
-   TOON                ████████████████████    67,696 tokens   (+6.0% vs CSV)
-   ├─ vs JSON          (−58.8%)               164,255 tokens
-   ├─ vs JSON compact  (−35.2%)               104,527 tokens
-   ├─ vs YAML          (−48.2%)               130,698 tokens
-   └─ vs XML           (−64.4%)               190,160 tokens
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
 ```
 
 <details>
@@ -83,64 +83,64 @@ Datasets with flat tabular structures where CSV is applicable.
 
 **Savings:** 13,130 tokens (59.0% reduction vs JSON)
 
-**JSON** (22,250 tokens):
+**JSON** (22,245 tokens):
 
 ```json
 {
   "metrics": [
     {
       "date": "2025-01-01",
-      "views": 5715,
-      "clicks": 211,
-      "conversions": 28,
-      "revenue": 7976.46,
-      "bounceRate": 0.47
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
     },
     {
       "date": "2025-01-02",
-      "views": 7103,
-      "clicks": 393,
-      "conversions": 28,
-      "revenue": 8360.53,
-      "bounceRate": 0.32
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
     },
     {
       "date": "2025-01-03",
-      "views": 7248,
-      "clicks": 378,
-      "conversions": 24,
-      "revenue": 3212.57,
-      "bounceRate": 0.5
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
     },
     {
       "date": "2025-01-04",
-      "views": 2927,
-      "clicks": 77,
-      "conversions": 11,
-      "revenue": 1211.69,
-      "bounceRate": 0.62
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
     },
     {
       "date": "2025-01-05",
-      "views": 3530,
-      "clicks": 82,
-      "conversions": 8,
-      "revenue": 462.77,
-      "bounceRate": 0.56
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
     }
   ]
 }
 ```
 
-**TOON** (9,120 tokens):
+**TOON** (9,115 tokens):
 
 ```
 metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
-  2025-01-01,5715,211,28,7976.46,0.47
-  2025-01-02,7103,393,28,8360.53,0.32
-  2025-01-03,7248,378,24,3212.57,0.5
-  2025-01-04,2927,77,11,1211.69,0.62
-  2025-01-05,3530,82,8,462.77,0.56
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
 ```
 
 ---
@@ -149,7 +149,7 @@ metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
 
 **Savings:** 6,400 tokens (42.3% reduction vs JSON)
 
-**JSON** (15,145 tokens):
+**JSON** (15,144 tokens):
 
 ```json
 {
@@ -197,7 +197,7 @@ metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
 }
 ```
 
-**TOON** (8,745 tokens):
+**TOON** (8,744 tokens):
 
 ```
 repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:

benchmarks/scripts/accuracy-benchmark.ts

@@ -1,17 +1,17 @@
-import type { Question } from '../src/types'
+import type { Question } from '../src/types.ts'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import process from 'node:process'
 import * as prompts from '@clack/prompts'
 import PQueue from 'p-queue'
-import { BENCHMARKS_DIR, DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants'
-import { ACCURACY_DATASETS } from '../src/datasets'
-import { evaluateQuestion, models } from '../src/evaluate'
-import { formatters, supportsCSV } from '../src/formatters'
-import { generateQuestions } from '../src/questions'
-import { calculateFormatResults, calculateTokenCounts, generateAccuracyReport } from '../src/report'
-import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage'
-import { ensureDir } from '../src/utils'
+import { BENCHMARKS_DIR, DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants.ts'
+import { ACCURACY_DATASETS } from '../src/datasets.ts'
+import { evaluateQuestion, models } from '../src/evaluate.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { generateQuestions } from '../src/questions/index.ts'
+import { calculateFormatResults, calculateTokenCounts, generateAccuracyReport } from '../src/report.ts'
+import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage.ts'
+import { ensureDir } from '../src/utils.ts'
 
 // Constants
 const PROGRESS_UPDATE_INTERVAL = 10

benchmarks/scripts/fetch-github-repos.ts

@@ -4,8 +4,8 @@ import process from 'node:process'
 import * as prompts from '@clack/prompts'
 import { ofetch } from 'ofetch'
 import pMap from 'p-map'
-import { BENCHMARKS_DIR } from '../src/constants'
-import { ensureDir } from '../src/utils'
+import { BENCHMARKS_DIR } from '../src/constants.ts'
+import { ensureDir } from '../src/utils.ts'
 
 prompts.intro('GitHub Repositories Fetcher')
 

benchmarks/scripts/token-efficiency-benchmark.ts

@@ -1,12 +1,12 @@
-import type { Dataset } from '../src/types'
+import type { Dataset } from '../src/types.ts'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import * as prompts from '@clack/prompts'
-import { encode } from '../../packages/toon/src'
-import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants'
-import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets'
-import { formatters, supportsCSV } from '../src/formatters'
-import { createProgressBar, ensureDir, tokenize } from '../src/utils'
+import { encode } from '../../packages/toon/src/index.ts'
+import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants.ts'
+import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { createProgressBar, ensureDir, tokenize } from '../src/utils.ts'
 
 interface FormatMetrics {
   name: string

benchmarks/src/constants.ts

@@ -34,9 +34,9 @@ export const DRY_RUN_LIMITS = {
 /// keep-sorted
 export const MODEL_RPM_LIMITS: Record<string, number | undefined> = {
   'claude-haiku-4-5-20251001': 50,
-  'gemini-2.5-flash': 25,
+  'gemini-3-flash-preview': 25,
   'gpt-5-nano': 50,
-  'grok-4-fast-non-reasoning': 50,
+  'grok-4-1-fast-non-reasoning': 25,
 }
 
 /**

benchmarks/src/datasets.ts

@@ -1,4 +1,4 @@
-import type { Dataset } from './types'
+import type { Dataset } from './types.ts'
 import { faker } from '@faker-js/faker'
 import githubRepos from '../data/github-repos.json' with { type: 'json' }
 

benchmarks/src/evaluate.ts

@@ -1,20 +1,20 @@
-import type { LanguageModelV2 } from '@ai-sdk/provider'
-import type { EvaluationResult, Question } from './types'
+import type { LanguageModelV3 } from '@ai-sdk/provider'
+import type { EvaluationResult, Question } from './types.ts'
 import { anthropic } from '@ai-sdk/anthropic'
 import { google } from '@ai-sdk/google'
 import { openai } from '@ai-sdk/openai'
 import { xai } from '@ai-sdk/xai'
 import { generateText } from 'ai'
-import { compareAnswers } from './normalize'
+import { compareAnswers } from './normalize.ts'
 
 /**
  * Models used for evaluation
  */
-export const models: LanguageModelV2[] = [
+export const models: LanguageModelV3[] = [
   anthropic('claude-haiku-4-5-20251001'),
-  google('gemini-2.5-flash'),
+  google('gemini-3-flash-preview'),
   openai('gpt-5-nano'),
-  xai('grok-4-fast-non-reasoning'),
+  xai('grok-4-1-fast-non-reasoning'),
 ]
 
 /**
@@ -58,7 +58,7 @@ export async function evaluateQuestion(
     question: Question
     formatName: string
     formattedData: string
-    model: LanguageModelV2
+    model: LanguageModelV3
   },
 ): Promise<EvaluationResult> {
   const primer = PRIMERS[formatName] ?? ''

benchmarks/src/formatters.ts

@@ -1,8 +1,8 @@
-import type { Dataset } from './types'
+import type { Dataset } from './types.ts'
 import { stringify as stringifyCSV } from 'csv-stringify/sync'
 import { XMLBuilder } from 'fast-xml-parser'
 import { stringify as stringifyYAML } from 'yaml'
-import { encode as encodeToon } from '../../packages/toon/src'
+import { encode as encodeToon } from '../../packages/toon/src/index.ts'
 
 /**
  * Format converters registry

benchmarks/src/questions/analytics.ts

@@ -1,7 +1,7 @@
-import type { AnalyticsMetric } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants'
-import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils'
+import type { AnalyticsMetric } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
 
 /**
  * Generate analytics (website metrics) questions

benchmarks/src/questions/event-logs.ts

@@ -1,7 +1,7 @@
-import type { EventLog } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS } from '../constants'
-import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils'
+import type { EventLog } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
 
 /**
  * Generate event log questions

benchmarks/src/questions/github.ts

@@ -1,7 +1,7 @@
-import type { Repository } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants'
-import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils'
+import type { Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
 
 /**
  * Generate GitHub repository questions

benchmarks/src/questions/index.ts

@@ -1,15 +1,15 @@
-import type { AnalyticsMetric, Employee, EventLog, NestedConfig, Order, Repository } from '../datasets'
-import type { Question } from '../types'
-import { ACCURACY_DATASETS } from '../datasets'
-import { generateAnalyticsQuestions } from './analytics'
-import { generateEventLogsQuestions } from './event-logs'
-import { generateGithubQuestions } from './github'
-import { generateNestedQuestions } from './nested'
-import { generateNestedConfigQuestions } from './nested-config'
-import { generateStructuralValidationQuestions } from './structural-validation'
-import { generateStructureQuestions } from './structure'
-import { generateTabularQuestions } from './tabular'
-import { createIdGenerator } from './utils'
+import type { AnalyticsMetric, Employee, EventLog, NestedConfig, Order, Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { ACCURACY_DATASETS } from '../datasets.ts'
+import { generateAnalyticsQuestions } from './analytics.ts'
+import { generateEventLogsQuestions } from './event-logs.ts'
+import { generateGithubQuestions } from './github.ts'
+import { generateNestedConfigQuestions } from './nested-config.ts'
+import { generateNestedQuestions } from './nested.ts'
+import { generateStructuralValidationQuestions } from './structural-validation.ts'
+import { generateStructureQuestions } from './structure.ts'
+import { generateTabularQuestions } from './tabular.ts'
+import { createIdGenerator } from './utils.ts'
 
 /**
  * Generate questions from all datasets

benchmarks/src/questions/nested-config.ts

@@ -1,7 +1,7 @@
-import type { NestedConfig } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS } from '../constants'
-import { QuestionBuilder } from './utils'
+import type { NestedConfig } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS } from '../constants.ts'
+import { QuestionBuilder } from './utils.ts'
 
 /**
  * Generate nested configuration questions

benchmarks/src/questions/nested.ts

@@ -1,7 +1,7 @@
-import type { Order } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants'
-import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils'
+import type { Order } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
 
 /**
  * Generate nested (orders) questions

benchmarks/src/questions/structural-validation.ts

@@ -1,5 +1,5 @@
-import type { Question } from '../types'
-import { QuestionBuilder } from './utils'
+import type { Question } from '../types.ts'
+import { QuestionBuilder } from './utils.ts'
 
 /**
  * Generate structural validation questions for all incompleteness fixtures

benchmarks/src/questions/structure.ts

@@ -1,6 +1,6 @@
-import type { AnalyticsMetric, Employee, EventLog, Order, Repository } from '../datasets'
-import type { Question } from '../types'
-import { QuestionBuilder } from './utils'
+import type { AnalyticsMetric, Employee, EventLog, Order, Repository } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QuestionBuilder } from './utils.ts'
 
 /**
  * Generate structure-awareness questions across all datasets

benchmarks/src/questions/tabular.ts

@@ -1,7 +1,7 @@
-import type { Employee } from '../datasets'
-import type { Question } from '../types'
-import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants'
-import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils'
+import type { Employee } from '../datasets.ts'
+import type { Question } from '../types.ts'
+import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
+import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
 
 /**
  * Generate tabular (employee) questions

benchmarks/src/questions/utils.ts

@@ -1,5 +1,5 @@
-import type { AnswerType, NormalizationOptions } from '../normalize'
-import type { Question } from '../types'
+import type { AnswerType, NormalizationOptions } from '../normalize.ts'
+import type { Question } from '../types.ts'
 
 // Constants for sampling strides
 export const SAMPLE_STRIDES = {

benchmarks/src/report.ts

@@ -1,10 +1,10 @@
-import type { Dataset, EfficiencyRanking, EvaluationResult, FormatResult, Question } from './types'
-import { FORMATTER_DISPLAY_NAMES, QUESTION_TYPE_LABELS, QUESTION_TYPES } from './constants'
-import { ACCURACY_DATASETS } from './datasets'
-import { models, PRIMERS } from './evaluate'
-import { supportsCSV } from './formatters'
-import { generateQuestions } from './questions'
-import { createProgressBar, tokenize } from './utils'
+import type { Dataset, EfficiencyRanking, EvaluationResult, FormatResult, Question } from './types.ts'
+import { FORMATTER_DISPLAY_NAMES, QUESTION_TYPE_LABELS, QUESTION_TYPES } from './constants.ts'
+import { ACCURACY_DATASETS } from './datasets.ts'
+import { models, PRIMERS } from './evaluate.ts'
+import { supportsCSV } from './formatters.ts'
+import { generateQuestions } from './questions/index.ts'
+import { createProgressBar, tokenize } from './utils.ts'
 
 const EFFICIENCY_CHART_STYLE: 'vertical' | 'horizontal' = 'horizontal'
 
@@ -179,17 +179,22 @@ function generateEfficiencyRankingReport(
   if (csv) {
     // CSV totalCount is evaluations (questions × models), so divide by number of models to get question count
     const csvQuestionCount = csv.totalCount / modelCount
-    csvNote = `\n\n**Note on CSV:** Excluded from ranking as it only supports ${csvQuestionCount} of ${totalQuestions} questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.`
+    csvNote = `**Note on CSV:** Excluded from ranking as it only supports ${csvQuestionCount} of ${totalQuestions} questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.`
   }
 
   return `
-Each format's overall performance, balancing accuracy against token cost:
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
 
 \`\`\`
 ${efficiencyChart}
 \`\`\`
 
-${summary}${csvNote}
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> ${summary}
+
+${csvNote}
 `.trim()
 }
 
@@ -396,7 +401,7 @@ function generateSummaryComparison(
     return ''
 
   return `
-> [!TIP] Results Summary
+> [!TIP]
 > TOON achieves **${(toon.accuracy * 100).toFixed(1)}% accuracy** (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using **${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens** on these datasets.
 `.trim()
 }
@@ -566,7 +571,7 @@ function generateHorizontalEfficiencyChart(
       const accuracy = `${(r.accuracy * 100).toFixed(1)}%`.padStart(5)
       const tokens = r.tokens.toLocaleString('en-US').padStart(5)
 
-      return `${formatName}   ${bar}   ${efficiency}  │  ${accuracy} acc  │  ${tokens} tokens`
+      return `${formatName}   ${bar}   ${efficiency} acc%/1K tok  │  ${accuracy} acc  │  ${tokens} tokens`
     })
     .join('\n')
 }

benchmarks/src/storage.ts

@@ -1,10 +1,9 @@
 import type { Storage, StorageValue } from 'unstorage'
-import type { EvaluationResult } from './types'
+import type { EvaluationResult } from './types.ts'
 import * as path from 'node:path'
 import { createStorage } from 'unstorage'
-// @ts-expect-error: No types available
 import fsDriver from 'unstorage/drivers/fs'
-import { BENCHMARKS_DIR } from './constants'
+import { BENCHMARKS_DIR } from './constants.ts'
 
 /**
  * Storage instance for model results

benchmarks/src/types.ts

@@ -1,5 +1,5 @@
-import type { DATASET_NAMES, QUESTION_TYPES, STRUCTURE_CLASSES } from './constants'
-import type { AnswerType, NormalizationOptions } from './normalize'
+import type { DATASET_NAMES, QUESTION_TYPES, STRUCTURE_CLASSES } from './constants.ts'
+import type { AnswerType, NormalizationOptions } from './normalize.ts'
 
 export type QuestionType = typeof QUESTION_TYPES[number]
 export type DatasetName = typeof DATASET_NAMES[number]

commitlint.config.ts

@@ -0,0 +1,38 @@
+import type { Rule, UserConfig } from '@commitlint/types'
+import { RuleConfigSeverity } from '@commitlint/types'
+
+// #region Rules
+
+/**
+ * Rule to ensure the first letter of the commit subject is lowercase.
+ *
+ * @param parsed - Parsed commit object containing commit message parts.
+ * @returns A tuple where the first element is a boolean indicating
+ * if the rule passed, and the second is an optional error message.
+ */
+const subjectLowercaseFirst: Rule = async (parsed) => {
+  const firstChar = parsed.subject!.match(/[a-z]/i)?.[0]
+  if (firstChar && firstChar === firstChar.toUpperCase()) {
+    return [false, 'Subject must start with a lowercase letter']
+  }
+  return [true]
+}
+
+// #endregion
+
+const Configuration: UserConfig = {
+  extends: ['@commitlint/config-conventional'],
+  rules: {
+    'subject-case': [RuleConfigSeverity.Disabled],
+    'subject-lowercase-first': [RuleConfigSeverity.Error, 'always'],
+  },
+  plugins: [
+    {
+      rules: {
+        'subject-lowercase-first': subjectLowercaseFirst,
+      },
+    },
+  ],
+}
+
+export default Configuration

docs/.vitepress/config.ts

@@ -32,6 +32,10 @@ export default defineConfig({
     logo: '/favicon.svg',
 
     nav: [
+      {
+        text: 'Playground',
+        link: '/playground',
+      },
       {
         text: 'Guide',
         activeMatch: '^/guide/',
@@ -53,6 +57,7 @@ export default defineConfig({
           { text: 'API', link: '/reference/api' },
           { text: 'Syntax Cheatsheet', link: '/reference/syntax-cheatsheet' },
           { text: 'Specification', link: '/reference/spec' },
+          { text: 'Efficiency Formalization', link: '/reference/efficiency-formalization' },
         ],
       },
       {
@@ -87,7 +92,7 @@ export default defineConfig({
 
     footer: {
       message: 'Released under the <a href="https://opensource.org/licenses/MIT" target="_blank">MIT License</a>.',
-      copyright: 'Copyright © 2025-PRESENT <a href="https://github.com/johannschopplich" target="_blank">Johann Schopplich</a>',
+      copyright: 'Copyright © 2025-PRESENT <a href="https://johannschopplich.com" target="_blank">Johann Schopplich</a>',
     },
 
     search: {
@@ -98,6 +103,7 @@ export default defineConfig({
     config(md) {
       md.use(copyOrDownloadAsMarkdownButtons)
     },
+    math: true,
   },
 })
 
@@ -115,13 +121,14 @@ function sidebarPrimary(): DefaultTheme.SidebarItem[] {
     {
       text: 'Tooling',
       items: [
+        { text: 'Playground', link: '/playground' },
         { text: 'CLI Reference', link: '/cli/' },
-        { text: 'Tools & Playgrounds', link: '/ecosystem/tools-and-playgrounds' },
       ],
     },
     {
       text: 'Ecosystem',
       items: [
+        { text: 'Tools & Playgrounds', link: '/ecosystem/tools-and-playgrounds' },
         { text: 'Implementations', link: '/ecosystem/implementations' },
       ],
     },
@@ -131,6 +138,7 @@ function sidebarPrimary(): DefaultTheme.SidebarItem[] {
         { text: 'API (TypeScript)', link: '/reference/api' },
         { text: 'Syntax Cheatsheet', link: '/reference/syntax-cheatsheet' },
         { text: 'Specification', link: '/reference/spec' },
+        { text: 'Efficiency Formalization', link: '/reference/efficiency-formalization' },
       ],
     },
   ]

docs/.vitepress/theme/components/PlaygroundLayout.vue

@@ -0,0 +1,764 @@
+<script setup lang="ts">
+import type { Delimiter, EncodeOptions } from '../../../../packages/toon/src'
+import { useClipboard, useDebounceFn } from '@vueuse/core'
+import { unzlibSync, zlibSync } from 'fflate'
+import { base64ToUint8Array, stringToUint8Array, uint8ArrayToBase64, uint8ArrayToString } from 'uint8array-extras'
+import { computed, onMounted, ref, shallowRef, watch } from 'vue'
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
+import { DEFAULT_DELIMITER, encode } from '../../../../packages/toon/src'
+import VPInput from './VPInput.vue'
+
+type InputFormat = 'json' | 'yaml'
+type JsonFormat = 'pretty-2' | 'pretty-4' | 'pretty-tab' | 'compact'
+
+interface PlaygroundState extends Required<Pick<EncodeOptions, 'delimiter' | 'indent' | 'keyFolding' | 'flattenDepth'>> {
+  input: string
+  inputFormat: InputFormat
+  jsonFormat: JsonFormat
+  /** Pre-YAML share URLs stored input under `json`. Read-only fallback. */
+  json?: string
+}
+
+function parseInput(text: string, format: InputFormat): unknown {
+  return format === 'yaml' ? parseYaml(text) : JSON.parse(text)
+}
+
+function stringifyInputYaml(value: unknown): string {
+  return stringifyYaml(value, { lineWidth: 0 })
+}
+
+const PRESETS = {
+  hikes: {
+    context: {
+      task: 'Our favorite hikes together',
+      location: 'Boulder',
+      season: 'spring_2025',
+    },
+    friends: ['ana', 'luis', 'sam'],
+    hikes: [
+      { id: 1, name: 'Blue Lake Trail', distanceKm: 7.5, elevationGain: 320, companion: 'ana', wasSunny: true },
+      { id: 2, name: 'Ridge Overlook', distanceKm: 9.2, elevationGain: 540, companion: 'luis', wasSunny: false },
+      { id: 3, name: 'Wildflower Loop', distanceKm: 5.1, elevationGain: 180, companion: 'sam', wasSunny: true },
+    ],
+  },
+  orders: {
+    orders: [
+      {
+        orderId: 'ORD-001',
+        customer: { name: 'Alice Chen', email: 'alice@example.com' },
+        items: [
+          { sku: 'WIDGET-A', quantity: 2, price: 29.99 },
+          { sku: 'GADGET-B', quantity: 1, price: 49.99 },
+        ],
+        total: 109.97,
+        status: 'shipped',
+      },
+      {
+        orderId: 'ORD-002',
+        customer: { name: 'Bob Smith', email: 'bob@example.com' },
+        items: [
+          { sku: 'THING-C', quantity: 3, price: 15.00 },
+        ],
+        total: 45.00,
+        status: 'delivered',
+      },
+    ],
+  },
+  metrics: {
+    metrics: [
+      { date: '2025-01-01', views: 5200, clicks: 180, conversions: 24, revenue: 2890.50 },
+      { date: '2025-01-02', views: 6100, clicks: 220, conversions: 31, revenue: 3450.00 },
+      { date: '2025-01-03', views: 4800, clicks: 165, conversions: 19, revenue: 2100.25 },
+      { date: '2025-01-04', views: 5900, clicks: 205, conversions: 28, revenue: 3200.00 },
+    ],
+  },
+  events: {
+    logs: [
+      { timestamp: '2025-01-15T10:23:45Z', level: 'info', endpoint: '/api/users', statusCode: 200, responseTime: 45 },
+      { timestamp: '2025-01-15T10:24:12Z', level: 'error', endpoint: '/api/orders', statusCode: 500, responseTime: 120, error: { message: 'Database timeout', retryable: true } },
+      { timestamp: '2025-01-15T10:25:03Z', level: 'info', endpoint: '/api/products', statusCode: 200, responseTime: 32 },
+      { timestamp: '2025-01-15T10:26:47Z', level: 'warn', endpoint: '/api/payment', statusCode: 429, responseTime: 5, error: { message: 'Rate limit exceeded', retryable: true } },
+    ],
+  },
+} as const
+const DELIMITER_OPTIONS: { value: Delimiter, label: string }[] = [
+  { value: ',', label: 'Comma (,)' },
+  { value: '\t', label: 'Tab (\\t)' },
+  { value: '|', label: 'Pipe (|)' },
+]
+const JSON_FORMAT_OPTIONS: { value: JsonFormat, label: string, indent: string | number | undefined }[] = [
+  { value: 'pretty-2', label: 'Pretty (2 spaces)', indent: 2 },
+  { value: 'pretty-4', label: 'Pretty (4 spaces)', indent: 4 },
+  { value: 'pretty-tab', label: 'Pretty (tabs)', indent: '\t' },
+  { value: 'compact', label: 'Compact', indent: undefined },
+]
+const DEFAULT_JSON = JSON.stringify(PRESETS.hikes, undefined, 2)
+const SHARE_URL_LIMIT = 8 * 1024
+
+// Input state
+const inputText = ref(DEFAULT_JSON)
+const inputFormat = ref<InputFormat>('json')
+const jsonFormat = ref<JsonFormat>('pretty-2')
+const currentFormatIndent = computed(() =>
+  JSON_FORMAT_OPTIONS.find(opt => opt.value === jsonFormat.value)?.indent,
+)
+const formattedInput = computed(() => {
+  try {
+    const data = parseInput(inputText.value, inputFormat.value)
+    return inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+  }
+  catch {
+    return inputText.value
+  }
+})
+
+// Encoder options
+const delimiter = ref<Delimiter>(DEFAULT_DELIMITER)
+const indent = ref(2)
+const keyFolding = ref<'off' | 'safe'>('safe')
+const flattenDepth = ref(2)
+
+// Encoding output
+const encodingResult = computed(() => {
+  try {
+    const parsedInput = parseInput(inputText.value, inputFormat.value)
+    return {
+      output: encode(parsedInput, {
+        indent: indent.value,
+        delimiter: delimiter.value,
+        keyFolding: keyFolding.value,
+        flattenDepth: flattenDepth.value,
+      }),
+      error: undefined,
+    }
+  }
+  catch (error) {
+    const fallback = inputFormat.value === 'yaml' ? 'Invalid YAML' : 'Invalid JSON'
+    return {
+      output: '',
+      error: error instanceof Error ? error.message : fallback,
+    }
+  }
+})
+const toonOutput = computed(() => encodingResult.value.output)
+const error = computed(() => encodingResult.value.error)
+
+// Token analysis
+const tokenizer = shallowRef<typeof import('gpt-tokenizer') | undefined>()
+const inputTokens = computed(() =>
+  tokenizer.value?.encode(formattedInput.value).length,
+)
+const toonTokens = computed(() =>
+  tokenizer.value && toonOutput.value ? tokenizer.value.encode(toonOutput.value).length : undefined,
+)
+const tokenSavings = computed(() => {
+  if (!inputTokens.value || !toonTokens.value)
+    return
+
+  const diff = inputTokens.value - toonTokens.value
+  const percent = Math.abs((diff / inputTokens.value) * 100).toFixed(1)
+  const sign = diff > 0 ? '−' : '+'
+
+  return { diff, percent, sign, isSavings: diff > 0 }
+})
+
+// UI state
+const canShareState = ref(true)
+const hasCopiedUrl = ref(false)
+
+const { copy, copied } = useClipboard({ source: toonOutput })
+const updateUrl = useDebounceFn(() => {
+  const hash = encodeState()
+  const baseUrl = `${window.location.origin}${window.location.pathname}${window.location.search}`
+  const targetUrl = `${baseUrl}#${hash}`
+
+  if (targetUrl.length > SHARE_URL_LIMIT) {
+    canShareState.value = false
+    return
+  }
+
+  canShareState.value = true
+  window.history.replaceState(null, '', `#${hash}`)
+}, 300)
+
+watch([inputText, delimiter, indent, keyFolding, flattenDepth, jsonFormat, inputFormat], () => {
+  updateUrl()
+})
+
+watch(jsonFormat, () => {
+  if (inputFormat.value !== 'json')
+    return
+  try {
+    inputText.value = formatJson(JSON.parse(inputText.value))
+  }
+  catch {}
+})
+
+watch(inputFormat, (next, prev) => {
+  if (prev === next)
+    return
+  try {
+    const data = parseInput(inputText.value, prev)
+    inputText.value = next === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+  }
+  catch {}
+})
+
+onMounted(() => {
+  loadTokenizer()
+
+  const hash = window.location.hash.slice(1)
+  if (!hash)
+    return
+
+  const state = decodeState(hash)
+  if (state) {
+    inputText.value = state.input ?? state.json
+    delimiter.value = state.delimiter
+    indent.value = state.indent
+    keyFolding.value = state.keyFolding ?? 'safe'
+    flattenDepth.value = state.flattenDepth ?? 2
+    jsonFormat.value = state.jsonFormat ?? 'pretty-2'
+    inputFormat.value = state.inputFormat ?? 'json'
+  }
+})
+
+function formatJson(value: unknown) {
+  return JSON.stringify(value, undefined, currentFormatIndent.value)
+}
+
+function encodeState() {
+  const state: PlaygroundState = {
+    input: inputText.value,
+    inputFormat: inputFormat.value,
+    delimiter: delimiter.value,
+    indent: indent.value,
+    keyFolding: keyFolding.value,
+    flattenDepth: flattenDepth.value,
+    jsonFormat: jsonFormat.value,
+  }
+
+  const compressedData = zlibSync(stringToUint8Array(JSON.stringify(state)))
+  return uint8ArrayToBase64(compressedData, { urlSafe: true })
+}
+
+function decodeState(hash: string) {
+  try {
+    const bytes = base64ToUint8Array(hash)
+    const decompressedData = unzlibSync(bytes)
+    const decodedData = uint8ArrayToString(decompressedData)
+    if (decodedData)
+      return JSON.parse(decodedData) as PlaygroundState
+  }
+  catch {}
+}
+
+function loadPreset(name: keyof typeof PRESETS) {
+  const data = PRESETS[name]
+  inputText.value = inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
+}
+
+async function copyShareUrl() {
+  if (!canShareState.value)
+    return
+
+  await navigator.clipboard.writeText(window.location.href)
+  hasCopiedUrl.value = true
+  setTimeout(() => (hasCopiedUrl.value = false), 2000)
+}
+
+async function loadTokenizer() {
+  tokenizer.value ??= await import('gpt-tokenizer')
+}
+</script>
+
+<template>
+  <div class="playground">
+    <div class="playground-container">
+      <!-- Header -->
+      <header class="playground-header">
+        <h1>Playground</h1>
+        <p>Convert JSON or YAML to TOON in real time.</p>
+      </header>
+
+      <!-- Options Bar -->
+      <div class="options-bar">
+        <VPInput id="inputFormat" label="Input format">
+          <select id="inputFormat" v-model="inputFormat">
+            <option value="json">
+              JSON
+            </option>
+            <option value="yaml">
+              YAML
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="delimiter" label="Delimiter">
+          <select id="delimiter" v-model="delimiter">
+            <option v-for="opt in DELIMITER_OPTIONS" :key="opt.value" :value="opt.value">
+              {{ opt.label }}
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="indent" label="Indent">
+          <input
+            id="indent"
+            v-model.number="indent"
+            type="number"
+            min="0"
+            max="8"
+          >
+        </VPInput>
+
+        <VPInput id="keyFolding" label="Key Folding">
+          <select id="keyFolding" v-model="keyFolding">
+            <option value="off">
+              Off
+            </option>
+            <option value="safe">
+              Safe
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput id="flattenDepth" label="Flatten Depth">
+          <input
+            id="flattenDepth"
+            v-model.number="flattenDepth"
+            type="number"
+            min="1"
+            max="10"
+            :disabled="keyFolding === 'off'"
+          >
+        </VPInput>
+
+        <VPInput id="preset" label="Preset">
+          <select id="preset" @change="(e) => loadPreset((e.target as HTMLSelectElement).value as keyof typeof PRESETS)">
+            <option value="" disabled selected>
+              Load example…
+            </option>
+            <option value="hikes">
+              Hikes (mixed structure)
+            </option>
+            <option value="orders">
+              Orders (nested objects)
+            </option>
+            <option value="metrics">
+              Metrics (tabular data)
+            </option>
+            <option value="events">
+              Events (semi-uniform)
+            </option>
+          </select>
+        </VPInput>
+
+        <VPInput v-if="inputFormat === 'json'" id="jsonFormat" label="JSON Baseline">
+          <select id="jsonFormat" v-model="jsonFormat">
+            <option v-for="opt in JSON_FORMAT_OPTIONS" :key="opt.value" :value="opt.value">
+              {{ opt.label }}
+            </option>
+          </select>
+        </VPInput>
+
+        <button
+          class="share-button"
+          :class="[hasCopiedUrl && 'copied']"
+          :aria-label="
+            !canShareState
+              ? 'State too large to share via URL'
+              : hasCopiedUrl
+                ? 'Link copied!'
+                : 'Copy shareable URL'
+          "
+          :title="!canShareState ? 'State too large to share via URL' : undefined"
+          :disabled="!canShareState"
+          :aria-disabled="!canShareState"
+          @click="copyShareUrl"
+        >
+          <span class="vpi-link" :class="[hasCopiedUrl && 'check']" aria-hidden="true" />
+          <template v-if="!canShareState">
+            Too large to share
+          </template>
+          <template v-else>
+            {{ hasCopiedUrl ? 'Copied!' : 'Share' }}
+          </template>
+        </button>
+      </div>
+
+      <!-- Editor Container -->
+      <div class="editor-container">
+        <!-- Input -->
+        <div class="editor-pane">
+          <div class="pane-header">
+            <span class="pane-title">{{ inputFormat === 'yaml' ? 'YAML Input' : 'JSON Input' }}</span>
+            <span class="pane-stats">
+              <span class="stat-primary" title="Token count of the formatted input">{{ inputTokens ?? '…' }} tokens</span>
+              <span class="stat-secondary">{{ formattedInput.length }} chars</span>
+            </span>
+          </div>
+          <textarea
+            id="input"
+            v-model="inputText"
+            class="editor-textarea"
+            spellcheck="false"
+            :aria-label="inputFormat === 'yaml' ? 'YAML input' : 'JSON input'"
+            :aria-describedby="error ? 'parse-error' : undefined"
+            :aria-invalid="!!error"
+            :placeholder="inputFormat === 'yaml' ? 'Enter YAML here…' : 'Enter JSON here…'"
+          />
+        </div>
+
+        <!-- TOON Output -->
+        <div class="editor-pane">
+          <div class="pane-header">
+            <span class="pane-title">
+              TOON Output
+              <span v-if="tokenSavings" class="savings-badge" :class="[!tokenSavings.isSavings && 'increase']">
+                {{ tokenSavings.sign }}{{ tokenSavings.percent }}%
+              </span>
+            </span>
+            <span class="pane-stats">
+              <span class="stat-primary">{{ toonTokens ?? '…' }} tokens</span>
+              <span class="stat-secondary">{{ toonOutput.length }} chars</span>
+            </span>
+          </div>
+          <div class="editor-output">
+            <button
+              v-if="!error"
+              class="copy-button"
+              :class="[copied && 'copied']"
+              :aria-label="copied ? 'Copied to clipboard' : 'Copy to clipboard'"
+              :aria-pressed="copied"
+              @click="copy()"
+            />
+            <pre v-if="!error"><code>{{ toonOutput }}</code></pre>
+            <div v-else id="parse-error" role="alert" class="error-message">
+              {{ error }}
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.playground {
+  padding: 32px 24px 32px;
+}
+
+@media (min-width: 768px) {
+  .playground {
+    padding: 48px 32px 48px;
+  }
+}
+
+@media (min-width: 960px) {
+  .playground {
+    padding: 48px 32px 48px;
+  }
+}
+
+.playground-container {
+  max-width: 1400px;
+  margin: 0 auto;
+}
+
+.playground-header {
+  margin-bottom: 24px;
+}
+
+.playground-header h1 {
+  font-size: 28px;
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  line-height: 40px;
+  color: var(--vp-c-text-1);
+  margin: 0 0 8px;
+}
+
+@media (min-width: 768px) {
+  .playground-header h1 {
+    font-size: 32px;
+  }
+}
+
+.playground-header p {
+  font-size: 16px;
+  line-height: 28px;
+  color: var(--vp-c-text-2);
+}
+
+.options-bar {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 12px;
+  align-items: flex-end;
+  margin-bottom: 16px;
+  padding: 12px 16px;
+  background: var(--vp-c-bg-soft);
+  border-radius: 8px;
+  border: 1px solid var(--vp-c-divider);
+}
+
+@media (max-width: 768px) {
+  .options-bar {
+    gap: 8px;
+  }
+}
+
+.vpi-link {
+  --icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' viewBox='0 0 24 24'%3E%3Cpath d='M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71'/%3E%3Cpath d='M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71'/%3E%3C/svg%3E");
+  display: inline-block;
+  width: 1em;
+  height: 1em;
+  -webkit-mask: var(--icon) no-repeat;
+  mask: var(--icon) no-repeat;
+  -webkit-mask-size: 100% 100%;
+  mask-size: 100% 100%;
+  background-color: currentColor;
+}
+
+.vpi-link.check {
+  --icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' stroke='currentColor' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' viewBox='0 0 24 24'%3E%3Cpath d='M20 6 9 17l-5-5'/%3E%3C/svg%3E");
+}
+
+.share-button {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 0 12px;
+  height: 32px;
+  font-size: 13px;
+  font-weight: 500;
+  color: var(--vp-c-text-1);
+  background: var(--vp-c-bg);
+  border: 1px solid var(--vp-c-border);
+  border-radius: 6px;
+  transition: border-color 0.25s, color 0.25s;
+  margin-left: auto;
+}
+
+.share-button:hover {
+  border-color: var(--vp-c-brand-1);
+  color: var(--vp-c-brand-1);
+}
+
+.share-button:focus-visible {
+  outline: 2px solid var(--vp-c-brand-1);
+  outline-offset: 2px;
+}
+
+.share-button.copied {
+  border-color: var(--vp-c-green-1);
+  color: var(--vp-c-green-1);
+}
+
+.share-button:disabled {
+  color: var(--vp-c-text-3);
+  border-color: var(--vp-c-divider);
+  background: var(--vp-c-bg-soft);
+  cursor: not-allowed;
+}
+
+.editor-container {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 16px;
+}
+
+@media (max-width: 768px) {
+  .editor-container {
+    grid-template-columns: 1fr;
+  }
+}
+
+.editor-pane {
+  display: flex;
+  flex-direction: column;
+  min-height: 500px;
+  border: 1px solid var(--vp-c-divider);
+  border-radius: 8px;
+  overflow: hidden;
+  background: var(--vp-c-bg-soft);
+  transition: border-color 0.25s;
+}
+
+@media (max-width: 768px) {
+  .editor-pane {
+    min-height: 400px;
+  }
+}
+
+.editor-pane:focus-within {
+  border-color: var(--vp-c-brand-1);
+}
+
+.pane-header {
+  display: flex;
+  align-items: center;
+  gap: 12px;
+  padding: 12px 16px;
+  background: var(--vp-c-bg-alt);
+  border-bottom: 1px solid var(--vp-c-divider);
+}
+
+.pane-title {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-size: 0.75rem;
+  font-weight: 600;
+  color: var(--vp-c-text-2);
+  text-transform: uppercase;
+  letter-spacing: 0.05em;
+  line-height: 1.5;
+}
+
+.pane-stats {
+  display: flex;
+  gap: 12px;
+  margin-left: auto;
+  font-size: 0.75rem;
+  font-weight: 400;
+  color: var(--vp-c-text-2);
+  text-transform: none;
+  letter-spacing: normal;
+}
+
+.stat-primary {
+  font-weight: 600;
+  color: var(--vp-c-text-1);
+}
+
+.stat-secondary {
+  color: var(--vp-c-text-3);
+}
+
+.savings-badge {
+  display: inline-flex;
+  padding: 2px 6px;
+  font-size: 0.625rem;
+  font-weight: 600;
+  color: var(--vp-c-green-1);
+  background: var(--vp-c-green-soft);
+  border-radius: 4px;
+  text-transform: none;
+  letter-spacing: normal;
+}
+
+.savings-badge.increase {
+  color: var(--vp-c-yellow-1);
+  background: var(--vp-c-yellow-soft);
+}
+
+.copy-button {
+  position: absolute;
+  top: 12px;
+  right: 12px;
+  z-index: 3;
+  border: 1px solid var(--vp-code-copy-code-border-color);
+  border-radius: 4px;
+  width: 40px;
+  height: 40px;
+  background-color: var(--vp-code-copy-code-bg);
+  opacity: 0;
+  cursor: pointer;
+  background-image: var(--vp-icon-copy);
+  background-position: 50%;
+  background-size: 20px;
+  background-repeat: no-repeat;
+  transition: border-color 0.25s, background-color 0.25s, opacity 0.25s;
+}
+
+.editor-output:hover .copy-button,
+.copy-button:focus {
+  opacity: 1;
+}
+
+.copy-button:hover:not(:disabled),
+.copy-button.copied {
+  border-color: var(--vp-code-copy-code-hover-border-color);
+  background-color: var(--vp-code-copy-code-hover-bg);
+}
+
+.copy-button:focus-visible {
+  outline: 2px solid var(--vp-c-brand-1);
+  outline-offset: 2px;
+}
+
+.copy-button:disabled {
+  opacity: 0.3;
+  cursor: not-allowed;
+}
+
+.copy-button.copied,
+.copy-button:hover.copied {
+  border-radius: 0 4px 4px 0;
+  background-image: var(--vp-icon-copied);
+}
+
+.copy-button.copied::before,
+.copy-button:hover.copied::before {
+  position: relative;
+  top: -1px;
+  transform: translateX(calc(-100% - 1px));
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  border: 1px solid var(--vp-code-copy-code-hover-border-color);
+  border-right: 0;
+  border-radius: 4px 0 0 4px;
+  padding: 0 10px;
+  width: fit-content;
+  height: 40px;
+  text-align: center;
+  font-size: 12px;
+  font-weight: 500;
+  color: var(--vp-code-copy-code-active-text);
+  background-color: var(--vp-code-copy-code-hover-bg);
+  white-space: nowrap;
+  content: var(--vp-code-copy-copied-text-content);
+}
+
+.copy-button[aria-pressed="true"] {
+  opacity: 1;
+}
+
+.editor-textarea,
+.editor-output {
+  flex: 1;
+  padding: 16px;
+  font-family: var(--vp-font-family-mono);
+  font-size: 0.875rem;
+  line-height: 1.7;
+}
+
+.editor-textarea {
+  resize: none;
+  color: var(--vp-c-text-1);
+  background: var(--vp-c-bg);
+}
+
+.editor-output {
+  position: relative;
+  overflow: auto;
+  background: var(--vp-code-block-bg);
+}
+
+.editor-output pre {
+  margin: 0;
+  white-space: pre;
+}
+
+.error-message {
+  color: var(--vp-c-danger-1);
+  padding: 8px 12px;
+  background: var(--vp-c-danger-soft);
+  border-radius: 4px;
+  font-size: 0.875rem;
+  font-family: var(--vp-font-family-base);
+}
+</style>

docs/.vitepress/theme/components/VPInput.vue

@@ -0,0 +1,68 @@
+<script setup lang="ts">
+defineProps<{
+  label: string
+  id: string
+}>()
+</script>
+
+<template>
+  <div class="VPInput">
+    <label :for="id" class="label">{{ label }}</label>
+    <div class="input-wrapper">
+      <slot />
+    </div>
+  </div>
+</template>
+
+<style scoped>
+.VPInput {
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+}
+
+.label {
+  font-size: 11px;
+  font-weight: 500;
+  color: var(--vp-c-text-2);
+}
+
+.input-wrapper :deep(select),
+.input-wrapper :deep(input) {
+  padding: 0 10px;
+  height: 32px;
+  font-size: 13px;
+  font-weight: 500;
+  color: var(--vp-c-text-1);
+  background-color: var(--vp-c-bg);
+  border: 1px solid var(--vp-c-border);
+  border-radius: 6px;
+  transition: border-color 0.25s;
+}
+
+.input-wrapper :deep(select):hover,
+.input-wrapper :deep(input):hover,
+.input-wrapper :deep(select):focus,
+.input-wrapper :deep(input):focus {
+  border-color: var(--vp-c-brand-1);
+}
+
+.input-wrapper :deep(select:disabled),
+.input-wrapper :deep(input:disabled) {
+  color: var(--vp-c-text-3);
+  background-color: var(--vp-c-bg-soft);
+  border-color: var(--vp-c-divider);
+  cursor: not-allowed;
+}
+
+.input-wrapper :deep(select:disabled):hover,
+.input-wrapper :deep(input:disabled):hover,
+.input-wrapper :deep(select:disabled):focus,
+.input-wrapper :deep(input:disabled):focus {
+  border-color: var(--vp-c-divider);
+}
+
+.input-wrapper :deep(input[type="number"]) {
+  width: 70px;
+}
+</style>

docs/.vitepress/theme/index.ts

@@ -1,6 +1,8 @@
 import type { Theme } from 'vitepress'
 import CopyOrDownloadAsMarkdownButtons from 'vitepress-plugin-llms/vitepress-components/CopyOrDownloadAsMarkdownButtons.vue'
 import DefaultTheme from 'vitepress/theme'
+import PlaygroundLayout from './components/PlaygroundLayout.vue'
+import VPInput from './components/VPInput.vue'
 
 import './vars.css'
 import './overrides.css'
@@ -10,9 +12,11 @@ const config: Theme = {
   extends: DefaultTheme,
   enhanceApp({ app }) {
     app.config.globalProperties.$spec = {
-      version: '2.0',
+      version: '3.3',
     }
     app.component('CopyOrDownloadAsMarkdownButtons', CopyOrDownloadAsMarkdownButtons)
+    app.component('PlaygroundLayout', PlaygroundLayout)
+    app.component('VPInput', VPInput)
   },
 }
 

docs/cli/index.md

@@ -1,8 +1,12 @@
+---
+description: Convert JSON to TOON and back from the command line, with token statistics, streaming, and delimiter options.
+---
+
 # Command Line Interface
 
-The `@toon-format/cli` package provides a command-line interface for encoding JSON to TOON and decoding TOON back to JSON. Use it to analyze token savings before integrating TOON into your application, or to process JSON data through TOON in shell pipelines using stdin/stdout with tools like curl and jq. The CLI supports token statistics, streaming for large datasets, and all encoding options available in the library.
+The `@toon-format/cli` package converts JSON to TOON and TOON to JSON. Use it to measure token savings before integrating TOON into your application, or to pipe JSON through TOON in shell workflows alongside tools like `curl` and `jq`. The CLI supports stdin/stdout, token statistics, streaming for large datasets, and every encoding option in the library.
 
-The CLI is built on top of the `@toon-format/toon` TypeScript implementation and adheres to the [latest specification](/reference/spec).
+The CLI is built on the `@toon-format/toon` TypeScript implementation and follows the [latest specification](/reference/spec).
 
 ## Usage
 
@@ -87,6 +91,8 @@ cat data.toon | toon --decode
 
 :::
 
+By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` (see [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations)).
+
 ### Standard Input
 
 Omit the input argument or use `-` to read from stdin. This enables piping data directly from other commands:
@@ -118,7 +124,7 @@ Both encoding and decoding operations use streaming output, writing incrementall
 - Uses the same event-based streaming decoder as the `decodeStream` API in `@toon-format/toon`.
 - Streams JSON tokens to output.
 - No full JSON string in memory.
-- When `--expand-paths safe` is enabled, falls back to non-streaming decode internally to apply deep-merge expansion before writing JSON.
+- When `--expandPaths safe` is enabled, falls back to non-streaming decode internally to apply deep-merge expansion before writing JSON.
 
 Process large files with minimal memory usage:
 
@@ -136,7 +142,7 @@ cat million-records.toon | toon --decode > output.json
 
 Peak memory usage scales with data depth, not total size. This allows processing arbitrarily large files as long as individual nested structures fit in memory.
 
-::: info Token Statistics
+::: tip Token Statistics
 When using the `--stats` flag with encode, the CLI builds the full TOON string once to compute accurate token counts. For maximum memory efficiency on very large files, omit `--stats`.
 :::
 
@@ -147,13 +153,14 @@ When using the `--stats` flag with encode, the CLI builds the full TOON string o
 | `-o, --output <file>` | Output file path (prints to stdout if omitted) |
 | `-e, --encode` | Force encode mode (overrides auto-detection) |
 | `-d, --decode` | Force decode mode (overrides auto-detection) |
-| `--delimiter <char>` | Array delimiter: `,` (comma), `\t` (tab), `\|` (pipe) |
+| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
 | `--indent <number>` | Indentation size (default: `2`) |
 | `--stats` | Show token count estimates and savings (encode only) |
-| `--no-strict` | Disable strict validation when decoding |
-| `--key-folding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
-| `--flatten-depth <number>` | Maximum segments to fold (default: `Infinity`) – requires `--key-folding safe` |
-| `--expand-paths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
+| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
+| `--keyFolding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
+| `--flattenDepth <number>` | Maximum segments to fold (default: `Infinity`) – requires `--keyFolding safe` |
+| `--expandPaths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
+| `--verbose` | Show full stack traces and cause chains for errors (default: `false`) |
 
 ## Advanced Examples
 
@@ -178,12 +185,12 @@ Example output:
 
 ### Alternative Delimiters
 
-TOON supports three delimiters: comma (default), tab, and pipe. Alternative delimiters can provide additional token savings in specific contexts.
+TOON supports three delimiters: comma (default), tab, and pipe. Alternative delimiters can save additional tokens depending on the data.
 
 ::: code-group
 
-```bash [Tab-separated]
-toon data.json --delimiter "\t" -o output.toon
+```bash [Tab-separated (bash/zsh)]
+toon data.json --delimiter $'\t' -o output.toon
 ```
 
 ```bash [Pipe-separated]
@@ -192,6 +199,8 @@ toon data.json --delimiter "|" -o output.toon
 
 :::
 
+The `--delimiter` value must be the actual delimiter character. In bash/zsh, use `$'\t'` to pass a real tab; literal `"\t"` is rejected as an invalid delimiter.
+
 **Tab delimiter example:**
 
 ::: code-group
@@ -210,18 +219,49 @@ items[2]{id,name,qty,price}:
 
 :::
 
-> [!TIP]
-> Tab delimiters often tokenize more efficiently than commas and reduce the need for quote-escaping. Use `--delimiter "\t"` for maximum token savings on large tabular data.
+::: tip
+Tab delimiters often tokenize more efficiently than commas and reduce the need for quote-escaping. Use `--delimiter $'\t'` (bash/zsh) for maximum token savings on large tabular data. See [Delimiter Strategies](/reference/api#delimiter-strategies) for full guidance.
+:::
 
 ### Lenient Decoding
 
-Skip validation for faster processing:
+Skip validation for faster, more forgiving decoding:
 
 ```bash
 toon data.toon --no-strict -o output.json
 ```
 
-Lenient mode (`--no-strict`) disables strict validation checks like array count matching, indentation multiples, and delimiter consistency. Use this when you trust the input and want faster decoding.
+With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
+
+### Decode Error Output
+
+When a TOON document fails to parse, the CLI renders the offending line with a caret pointing at the first non-whitespace character. Tabs are shown as `→` so the caret column reflects what the decoder actually saw.
+
+For an input file that uses a tab to indent the second line (rendered here with `→`):
+
+```
+a:
+→b: 1
+```
+
+The CLI prints:
+
+```
+ ERROR  Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode
+
+  2 | →b: 1
+      ^
+```
+
+The exit code is `1` on any error. Stack traces are suppressed by default. Pass `--verbose` to include the full stack and the underlying cause chain – useful when filing a bug report or diagnosing an unexpected error path:
+
+```bash
+cat broken.toon | toon --decode --verbose
+```
+
+::: tip Programmatic Access
+Decode errors are thrown as `ToonDecodeError` instances by the library. The CLI's caret rendering is built on the structured `line` and `source` fields exposed on that class. See the [Error Handling](/reference/api#error-handling) section of the API reference if you want the same diagnostic detail in your own code.
+:::
 
 ### Stdin Workflows
 
@@ -232,7 +272,7 @@ The CLI integrates seamlessly with Unix pipes and other command-line tools:
 curl https://api.example.com/data | toon --stats
 
 # Process large dataset
-cat large-dataset.json | toon --delimiter "\t" > output.toon
+cat large-dataset.json | toon --delimiter $'\t' > output.toon
 
 # Chain with jq
 jq '.results' data.json | toon > filtered.toon
@@ -245,11 +285,11 @@ Collapse nested wrapper chains to reduce tokens (since spec v1.5):
 ::: code-group
 
 ```bash [Basic key folding]
-toon input.json --key-folding safe -o output.toon
+toon input.json --keyFolding safe -o output.toon
 ```
 
 ```bash [Limit folding depth]
-toon input.json --key-folding safe --flatten-depth 2 -o output.toon
+toon input.json --keyFolding safe --flattenDepth 2 -o output.toon
 ```
 
 :::
@@ -268,7 +308,7 @@ For data like:
 }
 ```
 
-With `--key-folding safe`, output becomes:
+With `--keyFolding safe`, output becomes:
 
 ```yaml
 data.metadata.items[2]: a,b
@@ -287,19 +327,19 @@ data:
 Reconstruct nested structure from folded keys when decoding:
 
 ```bash
-toon data.toon --expand-paths safe -o output.json
+toon data.toon --expandPaths safe -o output.json
 ```
 
-This pairs with `--key-folding safe` for lossless round-trips.
+This pairs with `--keyFolding safe` for lossless round-trips.
 
 ### Round-Trip Workflow
 
 ```bash
 # Encode with folding
-toon input.json --key-folding safe -o compressed.toon
+toon input.json --keyFolding safe -o compressed.toon
 
 # Decode with expansion (restores original structure)
-toon compressed.toon --expand-paths safe -o output.json
+toon compressed.toon --expandPaths safe -o output.json
 
 # Verify round-trip
 diff input.json output.json
@@ -311,5 +351,5 @@ Combine multiple options for maximum efficiency:
 
 ```bash
 # Key folding + tab delimiter + stats
-toon data.json --key-folding safe --delimiter "\t" --stats -o output.toon
+toon data.json --keyFolding safe --delimiter $'\t' --stats -o output.toon
 ```

docs/ecosystem/implementations.md

@@ -1,6 +1,10 @@
+---
+description: Official and community TOON implementations across languages, plus contribution pointers.
+---
+
 # Implementations
 
-TOON has official and community implementations across multiple programming languages. All implementations are intended to conform to the same [specification](https://github.com/toon-format/spec) to ensure compatibility and interoperability.
+TOON has official and community implementations across multiple programming languages. All implementations are intended to conform to the same [Specification](https://github.com/toon-format/spec) to ensure compatibility and interoperability.
 
 The code examples throughout this documentation site use the TypeScript implementation by default, but the format and concepts apply equally to all languages.
 
@@ -16,8 +20,11 @@ These implementations are actively being developed by dedicated teams. Contribut
 | **.NET** | [toon-dotnet](https://github.com/toon-format/toon-dotnet) | In Development | Official Team |
 | **Dart** | [toon-dart](https://github.com/toon-format/toon-dart) | In Development | Official Team |
 | **Go** | [toon-go](https://github.com/toon-format/toon-go) | In Development | Official Team |
-| **Python** | [toon-python](https://github.com/toon-format/toon-python) | In Development | Official Team |
-| **Rust** | [toon-rust](https://github.com/toon-format/toon-rust) | In Development | Official Team |
+| **Java** | [toon-java](https://github.com/toon-format/toon-java) | ✅ Stable | Official Team |
+| **Julia** | [ToonFormat.jl](https://github.com/toon-format/ToonFormat.jl) | ✅ Stable | Official Team |
+| **Python** | [toon-python](https://github.com/toon-format/toon-python) | ✅ Stable | Official Team |
+| **Rust** | [toon-rust](https://github.com/toon-format/toon-rust) | ✅ Stable | Official Team |
+| **Swift** | [toon-swift](https://github.com/toon-format/toon-swift) | ✅ Stable | Official Team |
 | **TypeScript/JavaScript** | [toon](https://github.com/toon-format/toon/tree/main/packages/toon) | ✅ Stable | Official Team |
 
 ## Community Implementations
@@ -27,23 +34,29 @@ Community members have created implementations in additional languages:
 | Language | Repository | Maintainer |
 |----------|------------|------------|
 | **Apex** | [ApexToon](https://github.com/Eacaw/ApexToon) | [@Eacaw](https://github.com/Eacaw) |
+| **C** | [TOONc](https://github.com/UsboKirishima/TOONc) | [@UsboKirishima](https://github.com/UsboKirishima) |
 | **C++** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
+| **C#** | [ToonEncoder](https://github.com/Cysharp/ToonEncoder) | [@Cysharp](https://github.com/Cysharp/ToonEncoder) |
 | **Clojure** | [toon](https://github.com/vadelabs/toon) | [@vadelabs](https://github.com/vadelabs) |
 | **Crystal** | [toon-crystal](https://github.com/mamantoha/toon-crystal) | [@mamantoha](https://github.com/mamantoha) |
+| **Delphi** | [delphi-toon](https://github.com/ernestoalconada/delphi-toon) | [@ernestoalconada](https://github.com/ernestoalconada) |
 | **Elixir** | [toon_ex](https://github.com/kentaro/toon_ex) | [@kentaro](https://github.com/kentaro) |
 | **Gleam** | [toon_codec](https://github.com/axelbellec/toon_codec) | [@axelbellec](https://github.com/axelbellec) |
 | **Go** | [gotoon](https://github.com/alpkeskin/gotoon) | [@alpkeskin](https://github.com/alpkeskin) |
-| **Java** | [JToon](https://github.com/felipestanzani/JToon) | [@felipestanzani](https://github.com/felipestanzani) |
-| **Kotlin** | [kotlin-toon](https://github.com/vexpera-br/kotlin-toon) | [@vexpera-br](https://github.com/vexpera-br) |
-| **Laravel Framework** | [laravel-toon](https://github.com/jobmetric/laravel-toon) | [@jobmetric](https://github.com/jobmetric) |
+| **Java** | [json-io](https://github.com/jdereg/json-io) | [@jdereg](https://github.com/jdereg) |
+| **Kotlin** | [ktoon](https://github.com/lukelast/ktoon)| [@lukelast](https://github.com/lukelast) |
+| **Laravel Framework** | [laravel-toon](https://github.com/mischasigtermans/laravel-toon) | [@mischasigtermans](https://github.com/mischasigtermans) |
 | **Lua/Neovim** | [toon.nvim](https://github.com/thalesgelinger/toon.nvim) | [@thalesgelinger](https://github.com/thalesgelinger) |
+| **Matlab** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
 | **OCaml** | [ocaml-toon](https://github.com/davesnx/ocaml-toon) | [@davesnx](https://github.com/davesnx) |
 | **Perl** | [Data::TOON](https://github.com/ytnobody/p5-Data-TOON) | [@ytnobody](https://github.com/ytnobody) |
 | **PHP** | [toon-php](https://github.com/HelgeSverre/toon-php) | [@HelgeSverre](https://github.com/HelgeSverre) |
+| **Python** (C++ backend) | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
+| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
 | **R** | [toon](https://github.com/laresbernardo/toon) | [@laresbernardo](https://github.com/laresbernardo) |
 | **Ruby** | [toon-ruby](https://github.com/andrepcg/toon-ruby) | [@andrepcg](https://github.com/andrepcg) |
 | **Scala** | [toon4s](https://github.com/vim89/toon4s) | [@vim89](https://github.com/vim89) |
-| **Swift** | [TOONEncoder](https://github.com/mattt/TOONEncoder) | [@mattt](https://github.com/mattt) |
+| **Zig** | [toon-zig](https://github.com/LatentEvals/toon-zig) | [@montanaflynn](https://github.com/montanaflynn) |
 
 ## Contributing an Implementation
 
@@ -51,5 +64,5 @@ Building a TOON implementation for a new language? Great! Here are some steps to
 
 1. **Follow the spec**: Implement the [latest specification](https://github.com/toon-format/spec/blob/main/SPEC.md).
 2. **Add tests**: Run the [reference test suite](https://github.com/toon-format/spec/tree/main/tests).
-3. **Document usage**: Provide clear README with installation and usage examples.
+3. **Document usage**: Provide a clear README with installation and usage examples.
 4. **Share it**: Open a PR to add your implementation to the README at [github.com/toon-format/toon](https://github.com/toon-format/toon).

docs/ecosystem/tools-and-playgrounds.md

@@ -1,10 +1,18 @@
-# Tools & Playgrounds
+---
+description: TOON playgrounds, CLI, editor support, and ecosystem tools.
+---
 
-Experiment with TOON format interactively using these community-built tools for token comparison, format conversion, and validation.
+# Tools and Playgrounds
+
+Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.
 
 ## Playgrounds
 
-Experiment with TOON format interactively using these community-built tools for token comparison, format conversion, and validation:
+### Official Playground
+
+The [TOON Playground](/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
+
+### Community Playgrounds
 
 - [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration)
 - [TOON Tools](https://toontools.vercel.app/)
@@ -21,7 +29,7 @@ npx @toon-format/cli input.json --stats -o output.toon
 
 ### VS Code
 
-[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) - Syntax highlighting, validation, conversion, and token analysis.
+[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) – Syntax highlighting, validation, conversion, and token analysis.
 
 Install from the [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) or via command line:
 
@@ -31,16 +39,28 @@ code --install-extension vishalraut.vscode-toon
 
 ### Tree-sitter Grammar
 
-[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) - Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
+[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) – Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
 
 ### Neovim
 
-[toon.nvim](https://github.com/thalesgelinger/toon.nvim) - Lua-based plugin for Neovim.
+[toon.nvim](https://github.com/thalesgelinger/toon.nvim) – Lua-based plugin for Neovim.
 
 ### Other Editors
 
 Use YAML syntax highlighting as a close approximation. Most editors allow associating `.toon` files with YAML language mode.
 
+## Databases
+
+### ToonStore
+
+[ToonStore](https://github.com/Kalama-Tech/toonstoredb) – Redis-compatible embedded database (Rust) that stores data in TOON format.
+
+## ORMs
+
+### TORM
+
+[TORM](https://github.com/Kalama-Tech/torm) – ORM that works with the ToonStore database, with SDKs for Node.js, Python, Go, and PHP.
+
 ## Web APIs
 
 If you're building web applications that work with TOON, you can use the TypeScript library in the browser:
@@ -53,4 +73,10 @@ const toon = encode(data)
 const data = decode(toon)
 ```
 
-See the [API reference](/reference/api) for details.
+See the [API Reference](/reference/api) for details.
+
+## MCP
+
+### Tooner
+
+[Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON.

docs/guide/benchmarks.md

@@ -1,3 +1,7 @@
+---
+description: Retrieval accuracy and token efficiency results for TOON across mixed-structure and flat-only tracks.
+---
+
 # Benchmarks
 
 The benchmarks on this page measure TOON's performance across two key dimensions:
@@ -49,17 +53,20 @@ Benchmarks test LLM comprehension across different input formats using 209 data
 
 #### Efficiency Ranking (Accuracy per 1K Tokens)
 
-Each format's overall performance, balancing accuracy against token cost:
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
 
 ```
-TOON           ████████████████████   26.9  │  73.9% acc  │  2,744 tokens
-JSON compact   █████████████████░░░   22.9  │  70.7% acc  │  3,081 tokens
-YAML           ██████████████░░░░░░   18.6  │  69.0% acc  │  3,719 tokens
-JSON           ███████████░░░░░░░░░   15.3  │  69.7% acc  │  4,545 tokens
-XML            ██████████░░░░░░░░░░   13.0  │  67.1% acc  │  5,167 tokens
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
 ```
 
-TOON achieves **73.9%** accuracy (vs JSON's 69.7%) while using **39.6% fewer tokens**.
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
 
 **Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
 
@@ -76,13 +83,13 @@ claude-haiku-4-5-20251001
   JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
   CSV            ██████████░░░░░░░░░░    50.5% (55/109)
 
-gemini-2.5-flash
-→ TOON           ██████████████████░░    87.6% (183/209)
-  CSV            █████████████████░░░    86.2% (94/109)
-  JSON compact   ████████████████░░░░    82.3% (172/209)
-  YAML           ████████████████░░░░    79.4% (166/209)
-  XML            ████████████████░░░░    79.4% (166/209)
-  JSON           ███████████████░░░░░    77.0% (161/209)
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
 
 gpt-5-nano
 → TOON           ██████████████████░░    90.9% (190/209)
@@ -92,30 +99,30 @@ gpt-5-nano
   YAML           █████████████████░░░    87.1% (182/209)
   XML            ████████████████░░░░    80.9% (169/209)
 
-grok-4-fast-non-reasoning
-→ TOON           ███████████░░░░░░░░░    57.4% (120/209)
-  JSON           ███████████░░░░░░░░░    55.5% (116/209)
-  JSON compact   ███████████░░░░░░░░░    54.5% (114/209)
-  YAML           ███████████░░░░░░░░░    53.6% (112/209)
-  XML            ███████████░░░░░░░░░    52.6% (110/209)
-  CSV            ██████████░░░░░░░░░░    52.3% (57/109)
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
 ```
 
-> [!TIP] Results Summary
-> TOON achieves **73.9% accuracy** (vs JSON's 69.7%) while using **39.6% fewer tokens** on these datasets.
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
 
 <details>
 <summary><strong>Performance by dataset, model, and question type</strong></summary>
 
 #### Performance by Question Type
 
-| Question Type | TOON | JSON compact | JSON | CSV | YAML | XML |
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
 | ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
-| Field Retrieval | 99.6% | 99.3% | 99.3% | 100.0% | 98.2% | 98.9% |
-| Aggregation | 54.4% | 47.2% | 48.8% | 44.0% | 47.6% | 41.3% |
-| Filtering | 56.3% | 57.3% | 50.5% | 49.1% | 51.0% | 47.9% |
-| Structure Awareness | 88.0% | 83.0% | 83.0% | 85.9% | 80.0% | 80.0% |
-| Structural Validation | 70.0% | 45.0% | 50.0% | 80.0% | 60.0% | 80.0% |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
 
 #### Performance by Dataset
 
@@ -123,119 +130,119 @@ grok-4-fast-non-reasoning
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 72.0% | 2,352 | 118/164 |
-| `toon` | 73.8% | 2,518 | 121/164 |
-| `json-compact` | 69.5% | 3,953 | 114/164 |
-| `yaml` | 68.3% | 4,982 | 112/164 |
-| `json-pretty` | 68.3% | 6,360 | 112/164 |
-| `xml` | 69.5% | 7,324 | 114/164 |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
 
 ##### E-commerce orders with nested structures
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 81.1% | 7,232 | 133/164 |
-| `json-compact` | 76.8% | 6,794 | 126/164 |
-| `yaml` | 75.6% | 8,347 | 124/164 |
-| `json-pretty` | 76.2% | 10,713 | 125/164 |
-| `xml` | 74.4% | 12,023 | 122/164 |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
 
 ##### Time-series analytics data
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 73.3% | 1,406 | 88/120 |
-| `toon` | 72.5% | 1,548 | 87/120 |
-| `json-compact` | 71.7% | 2,349 | 86/120 |
-| `yaml` | 71.7% | 2,949 | 86/120 |
-| `json-pretty` | 68.3% | 3,676 | 82/120 |
-| `xml` | 68.3% | 4,384 | 82/120 |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
 
 ##### Top 100 GitHub repositories
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 62.9% | 8,779 | 83/132 |
-| `csv` | 61.4% | 8,527 | 81/132 |
-| `yaml` | 59.8% | 13,141 | 79/132 |
-| `json-compact` | 55.3% | 11,464 | 73/132 |
-| `json-pretty` | 56.1% | 15,157 | 74/132 |
-| `xml` | 48.5% | 17,105 | 64/132 |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
 
 ##### Semi-uniform event logs
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `json-compact` | 63.3% | 4,819 | 76/120 |
-| `toon` | 57.5% | 5,799 | 69/120 |
-| `json-pretty` | 59.2% | 6,797 | 71/120 |
-| `yaml` | 48.3% | 5,827 | 58/120 |
-| `xml` | 46.7% | 7,709 | 56/120 |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
 
 ##### Deeply nested configuration
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `json-compact` | 92.2% | 574 | 107/116 |
-| `toon` | 95.7% | 666 | 111/116 |
-| `yaml` | 91.4% | 686 | 106/116 |
-| `json-pretty` | 94.0% | 932 | 109/116 |
-| `xml` | 92.2% | 1,018 | 107/116 |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
 
 ##### Valid complete dataset (control)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `toon` | 100.0% | 544 | 4/4 |
-| `json-compact` | 100.0% | 795 | 4/4 |
-| `yaml` | 100.0% | 1,003 | 4/4 |
-| `json-pretty` | 100.0% | 1,282 | 4/4 |
-| `csv` | 25.0% | 492 | 1/4 |
-| `xml` | 0.0% | 1,467 | 0/4 |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
 
 ##### Array truncated: 3 rows removed from end
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 425 | 4/4 |
-| `xml` | 100.0% | 1,251 | 4/4 |
-| `toon` | 0.0% | 474 | 0/4 |
-| `json-compact` | 0.0% | 681 | 0/4 |
-| `json-pretty` | 0.0% | 1,096 | 0/4 |
-| `yaml` | 0.0% | 859 | 0/4 |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
 
 ##### Extra rows added beyond declared length
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 566 | 4/4 |
-| `toon` | 75.0% | 621 | 3/4 |
-| `xml` | 100.0% | 1,692 | 4/4 |
-| `yaml` | 75.0% | 1,157 | 3/4 |
-| `json-compact` | 50.0% | 917 | 2/4 |
-| `json-pretty` | 50.0% | 1,476 | 2/4 |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
 
 ##### Inconsistent field count (missing salary in row 10)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 75.0% | 489 | 3/4 |
-| `yaml` | 100.0% | 996 | 4/4 |
-| `toon` | 100.0% | 1,019 | 4/4 |
-| `json-compact` | 75.0% | 790 | 3/4 |
-| `xml` | 100.0% | 1,458 | 4/4 |
-| `json-pretty` | 75.0% | 1,274 | 3/4 |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
 
 ##### Missing required fields (no email in multiple rows)
 
 | Format | Accuracy | Tokens | Correct/Total |
 | ------ | -------- | ------ | ------------- |
-| `csv` | 100.0% | 329 | 4/4 |
-| `xml` | 100.0% | 1,411 | 4/4 |
-| `toon` | 75.0% | 983 | 3/4 |
-| `yaml` | 25.0% | 960 | 1/4 |
-| `json-pretty` | 25.0% | 1,230 | 1/4 |
-| `json-compact` | 0.0% | 755 | 0/4 |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
 
 #### Performance by Model
 
@@ -250,16 +257,16 @@ grok-4-fast-non-reasoning
 | `json-compact` | 55.0% | 115/209 |
 | `csv` | 50.5% | 55/109 |
 
-##### gemini-2.5-flash
+##### gemini-3-flash-preview
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `toon` | 87.6% | 183/209 |
-| `csv` | 86.2% | 94/109 |
-| `json-compact` | 82.3% | 172/209 |
-| `yaml` | 79.4% | 166/209 |
-| `xml` | 79.4% | 166/209 |
-| `json-pretty` | 77.0% | 161/209 |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
 
 ##### gpt-5-nano
 
@@ -272,16 +279,16 @@ grok-4-fast-non-reasoning
 | `yaml` | 87.1% | 182/209 |
 | `xml` | 80.9% | 169/209 |
 
-##### grok-4-fast-non-reasoning
+##### grok-4-1-fast-non-reasoning
 
 | Format | Accuracy | Correct/Total |
 | ------ | -------- | ------------- |
-| `toon` | 57.4% | 120/209 |
-| `json-pretty` | 55.5% | 116/209 |
-| `json-compact` | 54.5% | 114/209 |
-| `yaml` | 53.6% | 112/209 |
-| `xml` | 52.6% | 110/209 |
-| `csv` | 52.3% | 57/109 |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
 
 </details>
 
@@ -340,13 +347,13 @@ Eleven datasets designed to test different structural patterns and validation ca
 
 #### Evaluation Process
 
-1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON compact, JSON, CSV, YAML, XML).
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
 2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
 3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
 
 #### Models & Configuration
 
-- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-2.5-flash`, `gpt-5-nano`, `grok-4-fast-non-reasoning`
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
 - **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
 - **Temperature**: Not set (models use their defaults)
 - **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
@@ -368,34 +375,34 @@ Datasets with nested or semi-uniform structures. CSV excluded as it cannot prope
 ```
 🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
    │
-   TOON                █████████████░░░░░░░    72,771 tokens
-   ├─ vs JSON          (−33.1%)               108,806 tokens
-   ├─ vs JSON compact  (+5.5%)                 68,975 tokens
-   ├─ vs YAML          (−14.2%)                84,780 tokens
-   └─ vs XML           (−40.5%)               122,406 tokens
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
 
 🧾 Semi-uniform event logs  ┊  Tabular: 50%
    │
-   TOON                █████████████████░░░   153,211 tokens
-   ├─ vs JSON          (−15.0%)               180,176 tokens
-   ├─ vs JSON compact  (+19.9%)               127,731 tokens
-   ├─ vs YAML          (−0.8%)                154,505 tokens
-   └─ vs XML           (−25.2%)               204,777 tokens
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
 
 🧩 Deeply nested configuration  ┊  Tabular: 0%
    │
-   TOON                ██████████████░░░░░░       631 tokens
-   ├─ vs JSON          (−31.3%)                   919 tokens
-   ├─ vs JSON compact  (+11.9%)                   564 tokens
-   ├─ vs YAML          (−6.2%)                    673 tokens
-   └─ vs XML           (−37.4%)                 1,008 tokens
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
 
 ──────────────────────────────────── Total ────────────────────────────────────
-   TOON                ████████████████░░░░   226,613 tokens
-   ├─ vs JSON          (−21.8%)               289,901 tokens
-   ├─ vs JSON compact  (+14.9%)               197,270 tokens
-   ├─ vs YAML          (−5.6%)                239,958 tokens
-   └─ vs XML           (−31.0%)               328,191 tokens
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
 ```
 
 #### Flat-Only Track
@@ -405,38 +412,38 @@ Datasets with flat tabular structures where CSV is applicable.
 ```
 👥 Uniform employee records  ┊  Tabular: 100%
    │
-   CSV                 ███████████████████░    46,954 tokens
-   TOON                ████████████████████    49,831 tokens   (+6.1% vs CSV)
-   ├─ vs JSON          (−60.7%)               126,860 tokens
-   ├─ vs JSON compact  (−36.8%)                78,856 tokens
-   ├─ vs YAML          (−50.0%)                99,706 tokens
-   └─ vs XML           (−66.0%)               146,444 tokens
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
 
 📈 Time-series analytics data  ┊  Tabular: 100%
    │
-   CSV                 ██████████████████░░     8,388 tokens
-   TOON                ████████████████████     9,120 tokens   (+8.7% vs CSV)
-   ├─ vs JSON          (−59.0%)                22,250 tokens
-   ├─ vs JSON compact  (−35.8%)                14,216 tokens
-   ├─ vs YAML          (−48.9%)                17,863 tokens
-   └─ vs XML           (−65.7%)                26,621 tokens
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
 
 ⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
    │
-   CSV                 ███████████████████░     8,513 tokens
-   TOON                ████████████████████     8,745 tokens   (+2.7% vs CSV)
-   ├─ vs JSON          (−42.3%)                15,145 tokens
-   ├─ vs JSON compact  (−23.7%)                11,455 tokens
-   ├─ vs YAML          (−33.4%)                13,129 tokens
-   └─ vs XML           (−48.8%)                17,095 tokens
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
 
 ──────────────────────────────────── Total ────────────────────────────────────
-   CSV                 ███████████████████░    63,855 tokens
-   TOON                ████████████████████    67,696 tokens   (+6.0% vs CSV)
-   ├─ vs JSON          (−58.8%)               164,255 tokens
-   ├─ vs JSON compact  (−35.2%)               104,527 tokens
-   ├─ vs YAML          (−48.2%)               130,698 tokens
-   └─ vs XML           (−64.4%)               190,160 tokens
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
 ```
 
 <details>
@@ -446,64 +453,64 @@ Datasets with flat tabular structures where CSV is applicable.
 
 **Savings:** 13,130 tokens (59.0% reduction vs JSON)
 
-**JSON** (22,250 tokens):
+**JSON** (22,245 tokens):
 
 ```json
 {
   "metrics": [
     {
       "date": "2025-01-01",
-      "views": 5715,
-      "clicks": 211,
-      "conversions": 28,
-      "revenue": 7976.46,
-      "bounceRate": 0.47
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
     },
     {
       "date": "2025-01-02",
-      "views": 7103,
-      "clicks": 393,
-      "conversions": 28,
-      "revenue": 8360.53,
-      "bounceRate": 0.32
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
     },
     {
       "date": "2025-01-03",
-      "views": 7248,
-      "clicks": 378,
-      "conversions": 24,
-      "revenue": 3212.57,
-      "bounceRate": 0.5
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
     },
     {
       "date": "2025-01-04",
-      "views": 2927,
-      "clicks": 77,
-      "conversions": 11,
-      "revenue": 1211.69,
-      "bounceRate": 0.62
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
     },
     {
       "date": "2025-01-05",
-      "views": 3530,
-      "clicks": 82,
-      "conversions": 8,
-      "revenue": 462.77,
-      "bounceRate": 0.56
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
     }
   ]
 }
 ```
 
-**TOON** (9,120 tokens):
+**TOON** (9,115 tokens):
 
 ```
 metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
-  2025-01-01,5715,211,28,7976.46,0.47
-  2025-01-02,7103,393,28,8360.53,0.32
-  2025-01-03,7248,378,24,3212.57,0.5
-  2025-01-04,2927,77,11,1211.69,0.62
-  2025-01-05,3530,82,8,462.77,0.56
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
 ```
 
 ---
@@ -512,7 +519,7 @@ metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
 
 **Savings:** 6,400 tokens (42.3% reduction vs JSON)
 
-**JSON** (15,145 tokens):
+**JSON** (15,144 tokens):
 
 ```json
 {
@@ -560,7 +567,7 @@ metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
 }
 ```
 
-**TOON** (8,745 tokens):
+**TOON** (8,744 tokens):
 
 ```
 repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
@@ -572,3 +579,8 @@ repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watc
 </details>
 
 <!-- /automd -->
+
+## Related Resources
+
+- [Formal Byte-Level Model](/reference/efficiency-formalization) – Mathematical analysis of byte efficiency compared to JSON
+- [Specification](/reference/spec) – Formal TOON specification

docs/guide/format-overview.md

@@ -1,6 +1,10 @@
+---
+description: TOON syntax with concrete examples – objects, arrays, headers, key folding, and quoting rules.
+---
+
 # Format Overview
 
-TOON syntax reference with concrete examples. See [Getting Started](/guide/getting-started) for introduction.
+TOON syntax reference with concrete examples. See [Getting Started](/guide/getting-started) for an introduction.
 
 ## Data Model
 
@@ -92,7 +96,7 @@ The header `items[2]{sku,qty,price}:` declares:
 Each row contains values in the same order as the field list. Values are encoded as primitives (strings, numbers, booleans, null) and separated by the delimiter.
 
 > [!NOTE]
-> Tabular format requires identical field sets across all objects (same keys, order per object may vary) and primitive values only (no nested arrays/objects).
+> Tabular format requires identical field sets across all objects (same keys, order per object may vary), primitive values only (no nested arrays/objects), and at least one key per object – arrays that contain an empty `{}` element fall back to the expanded list form below.
 
 ### Mixed and Non-Uniform Arrays
 
@@ -107,6 +111,40 @@ items[3]:
 
 Each element starts with `- ` at one indentation level deeper than the parent array header.
 
+### Objects as List Items
+
+When an array element is an object, it appears as a list item:
+
+```yaml
+items[2]:
+  - id: 1
+    name: First
+  - id: 2
+    name: Second
+    extra: true
+```
+
+When a tabular array is the first field of a list-item object, the tabular header appears on the hyphen line, with rows indented two levels deeper and other fields indented one level deeper:
+
+```yaml
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+    status: active
+```
+
+When the object has only a single tabular field, the same pattern applies:
+
+```yaml
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+```
+
+This is the canonical encoding for list-item objects whose first field is a tabular array.
+
 ### Arrays of Arrays
 
 When you have arrays containing primitive inner arrays:
@@ -119,15 +157,27 @@ pairs[2]:
 
 Each inner array gets its own header on the list-item line.
 
-### Empty Arrays
-
-Empty arrays have special representations:
+When the inner arrays are themselves arrays of objects or non-uniform arrays, the same `- [N]:` header appears on the hyphen line and the nested items follow one indent deeper:
 
 ```yaml
-items[0]:
+items[3]:
+  - summary
+  - id: 1
+    name: Ada
+  - [2]:
+    - id: 2
+    - status: draft
 ```
 
-The header declares length zero, with no elements following.
+### Empty Arrays
+
+Empty arrays render as `key: []` for fields and `[]` at the root:
+
+```yaml
+items: []
+```
+
+The legacy `items[0]:` form is still decoded for backward compatibility.
 
 ## Array Headers
 
@@ -147,7 +197,7 @@ Where:
   - `|` → pipe delimiter
 - **fields** (optional) for tabular arrays: `{field1,field2,field3}`
 
-> [!TIP]
+> [!NOTE]
 > The array length `[N]` helps LLMs validate structure. If you ask a model to generate TOON output, explicit lengths let you detect truncation or malformed data.
 
 ### Delimiter Options
@@ -176,7 +226,7 @@ items[2|]{sku|name|qty|price}:
 
 :::
 
-Tab and pipe delimiters are explicitly encoded in the header brackets and field braces. Commas don't require quoting when tab or pipe is active, and vice versa.
+Tab and pipe delimiters are explicitly encoded in the header brackets and field braces. Inside an array scope, only the active delimiter triggers quoting – the others are literal data. Object field values (`key: value`) follow the document delimiter (§11.1) regardless of any surrounding array's active delimiter.
 
 > [!TIP]
 > Tab delimiters often tokenize more efficiently than commas, especially for data with few quoted strings. Use `encode(data, { delimiter: '\t' })` for additional token savings.
@@ -255,8 +305,8 @@ TOON quotes strings **only when necessary** to maximize token efficiency. A stri
 - It's empty (`""`)
 - It has leading or trailing whitespace
 - It equals `true`, `false`, or `null` (case-sensitive)
-- It looks like a number (e.g., `"42"`, `"-3.14"`, `"1e-6"`, or `"05"` with leading zeros)
-- It contains special characters: colon (`:`), quote (`"`), backslash (`\`), brackets, braces, or control characters (newline, tab, carriage return)
+- It looks like a number (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
+- It contains special characters: colon (`:`), quote (`"`), backslash (`\`), brackets, braces, or any control character in U+0000–U+001F
 - It contains the relevant delimiter (the active delimiter inside an array scope, or the document delimiter elsewhere)
 - It equals `"-"` or starts with `"-"` followed by any character
 
@@ -269,7 +319,7 @@ note: This has inner spaces
 
 ### Escape Sequences
 
-In quoted strings and keys, only five escape sequences are valid:
+In quoted strings and keys, six escape sequences are valid:
 
 | Character | Escape |
 |-----------|--------|
@@ -278,22 +328,38 @@ In quoted strings and keys, only five escape sequences are valid:
 | Newline (U+000A) | `\n` |
 | Carriage return (U+000D) | `\r` |
 | Tab (U+0009) | `\t` |
+| Any other U+0000–U+001F control character | `\uXXXX` |
 
-All other escape sequences (e.g., `\x`, `\u`) are invalid and will cause an error in strict mode.
+Other escapes (e.g., `\x`, `\0`, `\b`) are always rejected, as are lone-surrogate `\uXXXX` values (U+D800–U+DFFF).
 
 ### Type Conversions
 
-Numbers are emitted in canonical decimal form (no exponent notation, no trailing zeros). Non-JSON types are normalized before encoding:
-
-| Input | Output |
-|-------|--------|
-| Finite number | Canonical decimal (e.g., `1e6` → `1000000`, `1.5000` → `1.5`, `-0` → `0`) |
-| `NaN`, `Infinity`, `-Infinity` | `null` |
-| `BigInt` (within safe range) | Number |
-| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
-| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
-| `undefined`, `function`, `symbol` | `null` |
+Numbers are emitted in canonical decimal form for values in the §2 carve-out range; exponent notation is permitted outside. Non-JSON types (`NaN`, `Infinity`, `BigInt`, `Date`, `Set`, `Map`, `undefined`, etc.) are normalized before encoding – see [API Reference – Type Normalization](/reference/api#type-normalization) for the full mapping.
 
 Decoders accept both decimal and exponent forms on input (e.g., `42`, `-3.14`, `1e-6`), and treat tokens with forbidden leading zeros (e.g., `"05"`) as strings, not numbers.
 
+### Custom Serialization with toJSON
+
+Objects with a `toJSON()` method are serialized by calling the method and normalizing its result before encoding, similar to `JSON.stringify`:
+
+```ts
+const obj = {
+  data: 'example',
+  toJSON() {
+    return { info: this.data }
+  }
+}
+
+encode(obj)
+// info: example
+```
+
+The `toJSON()` method:
+
+- Takes precedence over built-in normalization (Date, Array, Set, Map)
+- Results are recursively normalized
+- Is called for objects with `toJSON` in their prototype chain
+
+---
+
 For complete rules on quoting, escaping, type conversions, and strict-mode decoding, see [spec §2–4 (data model), §7 (strings and keys), and §14 (strict mode)](https://github.com/toon-format/spec/blob/main/SPEC.md).

docs/guide/getting-started.md

@@ -1,14 +1,18 @@
+---
+description: What TOON is, when to use it, and a first encode/decode example with the TypeScript library.
+---
+
 # Getting Started
 
-## What is TOON?
+## What Is TOON?
 
-**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
+**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It is intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
 
 TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably.
 
 Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input.
 
-###  Why TOON?
+### Why TOON?
 
 Standard JSON is verbose and token-expensive. For uniform arrays of objects, JSON repeats every field name for every record:
 
@@ -41,7 +45,7 @@ users[2]{id,name,role}:
   2,Bob,user
 ```
 
-The `[2]` declares the array length, enabling LLMs to answer dataset size questions and detect truncation. The `{id,name,role}` declares the field names. Each row is then a compact, comma-separated list of values. This is the core pattern: declare structure once, stream data compactly. The format approaches CSV's efficiency while adding explicit structure.
+The `[2]` declares the array length, letting LLMs answer dataset-size questions and detect truncation. The `{id,name,role}` declares the field names. Each row is a compact, comma-separated list of values. The pattern is the same throughout TOON: declare structure once, stream data compactly. The result lands close to CSV density with explicit structure preserved.
 
 For a more realistic example, here's how TOON handles a dataset with both nested objects and tabular arrays:
 
@@ -123,11 +127,12 @@ TOON is not always the best choice. Consider alternatives when:
 
 - **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels.
 - **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it.
-- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5-10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
+- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5–10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
 - **Latency-critical applications**: Benchmark on your exact setup. Some deployments (especially local/quantized models) may process compact JSON faster despite TOON's lower token count.
 
-> [!NOTE]
-> For data-driven comparisons across different structures, see [benchmarks](/guide/benchmarks). When optimizing for latency, measure TTFT, tokens/sec, and total time for both TOON and JSON-compact and use whichever performs better in your specific environment.
+::: info
+For data-driven comparisons across different structures, see [Benchmarks](/guide/benchmarks). When optimizing for latency, measure TTFT, tokens/sec, and total time for both TOON and JSON-compact, and use whichever is faster in your specific environment.
+:::
 
 ## Installation
 
@@ -177,6 +182,10 @@ yarn global add @toon-format/cli
 
 For full CLI documentation, see the [CLI reference](/cli/).
 
+## Media Type & File Extension
+
+TOON files conventionally use the `.toon` extension. For HTTP transmission, the provisional media type is `text/toon`, always with UTF-8 encoding. While you may specify `charset=utf-8` explicitly, it's optional – UTF-8 is the default assumption. This follows the registration process outlined in [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations).
+
 ## Your First Example
 
 The examples below use the TypeScript library for demonstration, but the same operations work in any language with a TOON implementation.
@@ -236,6 +245,4 @@ Round-tripping is lossless: `decode(encode(x))` always equals `x` (after normali
 
 ## Where to Go Next
 
-Now that you've seen your first TOON document, read the [Format Overview](/guide/format-overview) for complete syntax details (objects, arrays, quoting rules, key folding), then explore [Using TOON with LLMs](/guide/llm-prompts) to see how to use it effectively in prompts. For implementation details, check the [API reference](/reference/api) (TypeScript) or the [specification](/reference/spec) (language-agnostic normative rules).
-
-For large datasets or streaming use-cases, see `encodeLines`, `decodeFromLines`, and `decodeStream` in the [API reference](/reference/api).
+Now that you've seen your first TOON document, read the [Format Overview](/guide/format-overview) for complete syntax details (objects, arrays, quoting rules, key folding), then explore [Using TOON with LLMs](/guide/llm-prompts) to see how to use it effectively in prompts. For implementation details, check the [API Reference](/reference/api) (TypeScript) or the [Specification](/reference/spec) (language-agnostic normative rules).

docs/guide/llm-prompts.md

@@ -1,3 +1,7 @@
+---
+description: Prompting strategies for sending TOON to LLMs and validating TOON they generate, with examples.
+---
+
 # Using TOON with LLMs
 
 TOON is designed for passing structured data to Large Language Models with reduced token costs and improved reliability. This guide shows how to use TOON effectively in prompts, both for input (sending data to models) and output (getting models to generate TOON).
@@ -6,7 +10,7 @@ This guide is about the TOON format itself. Code examples use the TypeScript lib
 
 ## Why TOON for LLMs
 
-LLM tokens cost money, and JSON is verbose – repeating every field name for every record in an array. TOON minimizes tokens especially for uniform arrays by declaring fields once and streaming data as rows, typically saving 30-60% compared to formatted JSON.
+LLM tokens cost money, and JSON is verbose – repeating every field name for every record in an array. TOON minimizes tokens especially for uniform arrays by declaring fields once and streaming data as rows, typically saving 30–60% compared to formatted JSON.
 
 TOON adds structure guardrails: explicit `[N]` lengths and `{fields}` headers make it easier for models to track rows and for you to validate output. Strict mode helps detect truncation and malformed TOON when decoding model responses.
 
@@ -21,9 +25,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
 
 ```toon
 users[3]{id,name,role,lastLogin}:
-  1,Alice,admin,2025-01-15T10:30:00Z
-  2,Bob,user,2025-01-14T15:22:00Z
-  3,Charlie,user,2025-01-13T09:45:00Z
+  1,Alice,admin,"2025-01-15T10:30:00Z"
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
 ```
 
 Task: Summarize the user roles and their last activity.
@@ -48,9 +52,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
 
 ```toon
 users[3]{id,name,role,lastLogin}:
-  1,Alice,admin,2025-01-15T10:30:00Z
-  2,Bob,user,2025-01-14T15:22:00Z
-  3,Charlie,user,2025-01-13T09:45:00Z
+  1,Alice,admin,"2025-01-15T10:30:00Z"
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
 ```
 
 Task: Return only users with role "user" as TOON. Use the same header format. Set [N] to match the row count. Output only the code block.
@@ -60,8 +64,8 @@ Task: Return only users with role "user" as TOON. Use the same header format. Se
 
 ```toon
 users[2]{id,name,role,lastLogin}:
-  2,Bob,user,2025-01-14T15:22:00Z
-  3,Charlie,user,2025-01-13T09:45:00Z
+  2,Bob,user,"2025-01-14T15:22:00Z"
+  3,Charlie,user,"2025-01-13T09:45:00Z"
 ```
 
 The model adjusts `[N]` to `2` and generates two rows.
@@ -83,7 +87,7 @@ catch (error) {
 }
 ```
 
-Strict mode checks counts, indentation, and escaping so you can detect truncation or malformed TOON. For complete details, see the [API reference](/reference/api#decode).
+Strict mode checks counts, indentation, and escaping so you can detect truncation or malformed TOON. For complete details, see the [API Reference](/reference/api#decode-input-options).
 
 ## Delimiter Choices for Token Efficiency
 
@@ -113,10 +117,10 @@ for (const line of encodeLines(largeData, { delimiter: '\t' })) {
 The CLI also supports streaming for memory-efficient JSON-to-TOON conversion:
 
 ```bash
-toon large-dataset.json --output output.toon
+toon large-dataset.json -o output.toon
 ```
 
-This streaming approach prevents out-of-memory errors when preparing large context windows for LLMs. For complete details on `encodeLines()`, see the [API reference](/reference/api#encodelines).
+This streaming approach prevents out-of-memory errors when preparing large context windows for LLMs. For complete details on `encodeLines()`, see the [API Reference](/reference/api#encodelines-input-options).
 
 **Consuming streaming LLM outputs:** If your LLM client exposes streaming text and you buffer by lines, you can decode TOON incrementally:
 
@@ -141,13 +145,13 @@ for await (const chunk of modelStream) {
 const data = decodeFromLines(lines)
 ```
 
-For streaming decode APIs, see [`decodeFromLines()`](/reference/api#decodeFromLines-lines-options) and [`decodeStream()`](/reference/api#decodeStream-source-options).
+For streaming decode APIs, see [`decodeFromLines()`](/reference/api#decodefromlines-lines-options) and [`decodeStream()`](/reference/api#decodestream-source-options).
 
 ## Tips and Pitfalls
 
-**Show, don't describe.** Don't explain TOON syntax in detail – just show an example. Models learn the pattern from context. A simple code block with 2-5 rows is more effective than paragraphs of explanation.
+**Show, don't describe.** Don't explain TOON syntax in detail – just show an example. Models learn the pattern from context. A simple code block with 2–5 rows is more effective than paragraphs of explanation.
 
-**Keep examples small.** Use 2-5 rows in your examples, not hundreds. The model generalizes from the pattern. Large examples waste tokens without improving accuracy.
+**Keep examples small.** Use 2–5 rows in your examples, not hundreds. The model generalizes from the pattern. Large examples waste tokens without improving accuracy.
 
 **Always validate output.** Decode generated TOON with `strict: true` (default) to catch errors early. Don't assume model output is valid TOON without checking.
 
@@ -162,10 +166,10 @@ System logs in TOON format (tab-separated):
 
 ```toon
 events[4	]{id	level	message	timestamp}:
-  1	error	Connection timeout	2025-01-15T10:00:00Z
-  2	warn	Slow query	2025-01-15T10:05:00Z
-  3	info	User login	2025-01-15T10:10:00Z
-  4	error	Database error	2025-01-15T10:15:00Z
+  1	error	Connection timeout	"2025-01-15T10:00:00Z"
+  2	warn	Slow query	"2025-01-15T10:05:00Z"
+  3	info	User login	"2025-01-15T10:10:00Z"
+  4	error	Database error	"2025-01-15T10:15:00Z"
 ```
 
 Task: Return only error-level events as TOON. Use the same format.
@@ -178,8 +182,8 @@ import { decode } from '@toon-format/toon'
 
 const modelResponse = `
 events[2	]{id	level	message	timestamp}:
-  1	error	Connection timeout	2025-01-15T10:00:00Z
-  4	error	Database error	2025-01-15T10:15:00Z
+  1	error	Connection timeout	"2025-01-15T10:00:00Z"
+  4	error	Database error	"2025-01-15T10:15:00Z"
 `
 
 const filtered = decode(modelResponse, { strict: true })

docs/index.md

@@ -11,22 +11,22 @@ hero:
     alt: TOON Logo
   actions:
     - theme: brand
-      text: Get Started
+      text: What is TOON?
       link: /guide/getting-started
     - theme: alt
       text: Benchmarks
       link: /guide/benchmarks
+    - theme: alt
+      text: Playground
+      link: /playground
     - theme: alt
       text: CLI
       link: /cli/
-    - theme: alt
-      text: Spec v2.0
-      link: /reference/spec
 
 features:
   - title: Token-Efficient & Accurate
     icon: 📊
-    details: TOON reaches 74% accuracy (vs JSON's 70%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
+    details: TOON reaches 76.4% accuracy (vs JSON's 75.0%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
     link: /guide/benchmarks
   - title: JSON Data Model
     icon: 🔁

docs/package.json

@@ -8,8 +8,14 @@
     "preview": "vitepress preview"
   },
   "devDependencies": {
-    "unocss": "^66.5.6",
+    "@vueuse/core": "^14.3.0",
+    "fflate": "^0.8.3",
+    "gpt-tokenizer": "^3.4.0",
+    "markdown-it-mathjax3": "^4.3.2",
+    "uint8array-extras": "^1.5.0",
+    "unocss": "^66.7.5",
     "vitepress": "^1.6.4",
-    "vitepress-plugin-llms": "^1.9.3"
+    "vitepress-plugin-llms": "^1.13.3",
+    "yaml": "^2.9.0"
   }
 }

docs/playground.md

@@ -0,0 +1,4 @@
+---
+layout: PlaygroundLayout
+title: Playground
+---

docs/reference/api.md

@@ -1,3 +1,7 @@
+---
+description: TypeScript and JavaScript encode and decode functions, options, error types, and streaming decoders for @toon-format/toon.
+---
+
 # API Reference
 
 TypeScript/JavaScript API documentation for the `@toon-format/toon` package. For format rules, see the [Format Overview](/guide/format-overview) or the [Specification](/reference/spec). For other languages, see [Implementations](/ecosystem/implementations).
@@ -20,7 +24,9 @@ yarn add @toon-format/toon
 
 :::
 
-## `encode(value, options?)`
+## Encoding Functions
+
+### `encode(input, options?)`
 
 Converts any JSON-serializable value to TOON format.
 
@@ -35,40 +41,39 @@ const toon = encode(data, {
 })
 ```
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
-| `value` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
-| `options` | `EncodeOptions?` | Optional encoding options (see below) |
+| `input` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
+| `options` | `EncodeOptions?` | Optional encoding options (see [Configuration Reference](#configuration-reference)) |
 
-### Options
-
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `indent` | `number` | `2` | Number of spaces per indentation level |
-| `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows |
-| `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths |
-| `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) |
-
-### Return Value
+#### Return Value
 
 Returns a TOON-formatted string with no trailing newline or spaces.
 
-### Type Normalization
+#### Type Normalization
 
 Non-JSON-serializable values are normalized before encoding:
 
 | Input | Output |
 |-------|--------|
-| Finite number | Canonical decimal (no exponent, no leading/trailing zeros: `1e6` → `1000000`, `-0` → `0`) |
+| `Object` with `toJSON()` method | Result of calling `toJSON()`, recursively normalized |
+| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal (e.g., `1e6` → `1000000`, `-0` → `0`) |
+| Finite number outside that range | Exponent form permitted (e.g., `1e-7`, `1e+21`) |
 | `NaN`, `Infinity`, `-Infinity` | `null` |
 | `BigInt` (within safe range) | Number |
 | `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
 | `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
+| `Set` | Array of normalized values |
+| `Map` | Object with `String(key)` keys |
 | `undefined`, `function`, `symbol` | `null` |
 
-### Example
+::: info
+TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
+:::
+
+#### Example
 
 ```ts
 import { encode } from '@toon-format/toon'
@@ -89,45 +94,7 @@ items[2]{sku,qty,price}:
   B2,1,14.5
 ```
 
-### Delimiter Options
-
-::: code-group
-
-```ts [Comma (default)]
-encode(data, { delimiter: ',' })
-```
-
-```ts [Tab]
-encode(data, { delimiter: '\t' })
-```
-
-```ts [Pipe]
-encode(data, { delimiter: '|' })
-```
-
-:::
-
-::: details Why Use Tab Delimiters?
-Tab delimiters (`\t`) often tokenize more efficiently than commas:
-- Tabs are single characters
-- Tabs rarely appear in natural text, reducing quote-escaping
-- The delimiter is explicitly encoded in the array header
-
-Example:
-
-```yaml
-items[2	]{sku	name	qty	price}:
-  A1	Widget	2	9.99
-  B2	Gadget	1	14.5
-```
-
-For maximum token savings on large tabular data, combine tab delimiters with key folding:
-```ts
-encode(data, { delimiter: '\t', keyFolding: 'safe' })
-```
-:::
-
-## `encodeLines(value, options?)`
+### `encodeLines(input, options?)`
 
 **Preferred method for streaming TOON output.** Converts any JSON-serializable value to TOON format as a sequence of lines, without building the full string in memory. Suitable for streaming large outputs to files, HTTP responses, or process stdout.
 
@@ -149,16 +116,16 @@ for (const line of lines) {
 const lineArray = Array.from(encodeLines(data))
 ```
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
-| `value` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
-| `options` | `EncodeOptions?` | Optional encoding options (same as `encode()`) |
+| `input` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
+| `options` | `EncodeOptions?` | Optional encoding options (see [Configuration Reference](#configuration-reference)) |
 
-### Return Value
+#### Return Value
 
-Returns an `Iterable<string>` that yields TOON lines one at a time. **Each yielded string is a single line without a trailing newline character** — you must add `\n` when writing to streams or stdout.
+Returns an `Iterable<string>` that yields TOON lines one at a time. **Each yielded string is a single line without a trailing newline character** – you must add `\n` when writing to streams or stdout.
 
 ::: info Relationship to `encode()`
 `encode(value, options)` is equivalent to:
@@ -167,7 +134,7 @@ Array.from(encodeLines(value, options)).join('\n')
 ```
 :::
 
-### Example
+#### Example
 
 ```ts
 import { createWriteStream } from 'node:fs'
@@ -189,7 +156,127 @@ for (const line of encodeLines(data, { delimiter: '\t' })) {
 stream.end()
 ```
 
-## `decode(input, options?)`
+### Replacer Function
+
+The `replacer` option allows you to transform or filter values during encoding. It works similarly to `JSON.stringify`'s replacer parameter, but with path tracking for more precise control.
+
+#### Type Signature
+
+```typescript
+type EncodeReplacer = (
+  key: string,
+  value: JsonValue,
+  path: readonly (string | number)[]
+) => unknown
+```
+
+#### Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `key` | `string` | Property name, array index (as string), or empty string for root |
+| `value` | `JsonValue` | The normalized value at this location |
+| `path` | `readonly (string \| number)[]` | Path from root to current value |
+
+#### Return Value
+
+- Return the value unchanged to keep it
+- Return a different value to replace it (will be normalized)
+- Return `undefined` to omit properties/array elements
+- For root value, `undefined` means "no change" (root cannot be omitted)
+
+#### Examples
+
+**Filtering sensitive data:**
+
+```typescript
+import { encode } from '@toon-format/toon'
+
+const data = {
+  user: { name: 'Alice', password: 'secret123', email: 'alice@example.com' }
+}
+
+function replacer(key, value) {
+  if (key === 'password')
+    return undefined
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+user:
+  name: Alice
+  email: alice@example.com
+```
+
+**Transforming values:**
+
+```typescript
+const data = { user: 'alice', role: 'admin' }
+
+function replacer(key, value) {
+  if (typeof value === 'string')
+    return value.toUpperCase()
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+user: ALICE
+role: ADMIN
+```
+
+**Path-based transformations:**
+
+```typescript
+const data = {
+  metadata: { created: '2025-01-01' },
+  user: { created: '2025-01-02' }
+}
+
+function replacer(key, value, path) {
+  // Add timezone info only to top-level metadata
+  if (path.length === 1 && path[0] === 'metadata' && key === 'created') {
+    return `${value}T00:00:00Z`
+  }
+  return value
+}
+
+console.log(encode(data, { replacer }))
+```
+
+**Output:**
+
+```yaml
+metadata:
+  created: "2025-01-01T00:00:00Z"
+user:
+  created: 2025-01-02
+```
+
+::: info Replacer Execution Order
+The replacer is called in a depth-first manner:
+1. Root value first (key = `''`, path = `[]`)
+2. Then each property/element (with proper key and path)
+3. Values are re-normalized after replacement
+4. Children are processed after parent transformation
+:::
+
+::: warning Array Indices as Strings
+Following `JSON.stringify` behavior, array indices are passed as strings (`'0'`, `'1'`, `'2'`, etc.) to the replacer, not as numbers.
+:::
+
+## Decoding Functions
+
+### `decode(input, options?)`
 
 Converts a TOON-formatted string back to JavaScript values.
 
@@ -203,38 +290,18 @@ const data = decode(toon, {
 })
 ```
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
 | `input` | `string` | A TOON-formatted string to parse |
-| `options` | `DecodeOptions?` | Optional decoding options (see below) |
+| `options` | `DecodeOptions?` | Optional decoding options (see [Configuration Reference](#configuration-reference)) |
 
-### Options
-
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `indent` | `number` | `2` | Expected number of spaces per indentation level |
-| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
-| `expandPaths` | `'off'` \| `'safe'` | `'off'` | Enable path expansion to reconstruct dotted keys into nested objects (pairs with `keyFolding: 'safe'`) |
-
-### Return Value
+#### Return Value
 
 Returns a JavaScript value (object, array, or primitive) representing the parsed TOON data.
 
-### Strict Mode
-
-By default (`strict: true`), the decoder validates input strictly:
-
-- **Invalid escape sequences**: Throws on `\x`, unterminated strings
-- **Syntax errors**: Throws on missing colons, malformed headers
-- **Array length mismatches**: Throws when declared length doesn't match actual count
-- **Delimiter mismatches**: Throws when row delimiters don't match header
-- **Indentation errors**: Throws when leading spaces aren't exact multiples of `indentSize`
-
-Set `strict: false` to skip validation for lenient parsing.
-
-### Example
+#### Example
 
 ```ts
 import { decode } from '@toon-format/toon'
@@ -260,72 +327,24 @@ console.log(data)
 }
 ```
 
-### Path Expansion
-
-When `expandPaths: 'safe'` is enabled, dotted keys are split into nested objects:
-
-```ts
-import { decode } from '@toon-format/toon'
-
-const toon = 'data.metadata.items[2]: a,b'
-
-const data = decode(toon, { expandPaths: 'safe' })
-console.log(data)
-// { data: { metadata: { items: ['a', 'b'] } } }
-```
-
-This pairs with `keyFolding: 'safe'` for lossless round-trips.
-
-::: details Expansion Conflict Resolution
-When multiple expanded keys construct overlapping paths, the decoder merges them recursively:
-- **Object + Object**: Deep merge recursively
-- **Object + Non-object** (array or primitive): Conflict
-  - With `strict: true` (default): Error
-  - With `strict: false`: Last-write-wins (LWW)
-
-Example conflict (strict mode):
-
-```ts
-const toon = 'a.b: 1\na: 2'
-decode(toon, { expandPaths: 'safe', strict: true })
-// Error: "Expansion conflict at path 'a' (object vs primitive)"
-```
-
-Example conflict (lenient mode):
-
-```ts
-const toon = 'a.b: 1\na: 2'
-decode(toon, { expandPaths: 'safe', strict: false })
-// { a: 2 } (last write wins)
-```
-:::
-
-## `decodeFromLines(lines, options?)`
+### `decodeFromLines(lines, options?)`
 
 Decodes TOON format from pre-split lines into a JavaScript value. This is a streaming-friendly wrapper around the event-based decoder that builds the full value in memory.
 
 Useful when you already have lines as an array or iterable (e.g., from file streams, readline interfaces, or network responses) and want the standard decode behavior with path expansion support.
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
 | `lines` | `Iterable<string>` | Iterable of TOON lines (without trailing newlines) |
-| `options` | `DecodeOptions?` | Optional decoding configuration (see below) |
+| `options` | `DecodeOptions?` | Optional decoding configuration (see [Configuration Reference](#configuration-reference)) |
 
-### Options
-
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `indent` | `number` | `2` | Expected number of spaces per indentation level |
-| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
-| `expandPaths` | `'off'` \| `'safe'` | `'off'` | Enable path expansion to reconstruct dotted keys into nested objects |
-
-### Return Value
+#### Return Value
 
 Returns a `JsonValue` (the parsed JavaScript value: object, array, or primitive).
 
-### Example
+#### Example
 
 **Basic usage with arrays:**
 
@@ -361,53 +380,47 @@ const value = decodeFromLines(lines, { expandPaths: 'safe' })
 // { user: { name: 'Alice', age: 30 } }
 ```
 
-## `decodeStreamSync(lines, options?)`
+### Choosing the Right Decoder
+
+| Function | Input | Output | Async | Path Expansion | Use When |
+|----------|-------|--------|-------|----------------|----------|
+| `decode()` | String | Value | No | Yes | You have a complete TOON string |
+| `decodeFromLines()` | Lines | Value | No | Yes | You have lines and want the full value |
+| `decodeStreamSync()` | Lines | Events | No | No | You need event-by-event processing (sync) |
+| `decodeStream()` | Lines | Events | Yes | No | You need event-by-event processing (async) |
+
+::: info Key Differences
+- **Value vs. Events**: Functions ending in `Stream` yield events without building the full value in memory.
+- **Path expansion**: Only `decode()` and `decodeFromLines()` support `expandPaths: 'safe'`.
+- **Async support**: Only `decodeStream()` accepts async iterables (useful for file/network streams).
+:::
+
+## Streaming Decoders
+
+### `decodeStreamSync(lines, options?)`
 
 Synchronously decodes TOON lines into a stream of JSON events. This function yields structured events that represent the JSON data model without building the full value tree.
 
 Useful for streaming processing, custom transformations, or memory-efficient parsing of large datasets where you don't need the full value in memory.
 
-::: info Event Streaming
-This is a low-level API that returns individual parse events. For most use cases, [`decodeFromLines()`](#decodeFromLines-lines-options) or [`decode()`](#decode-input-options) are more convenient.
+::: tip Event Streaming
+This is a low-level API that returns individual parse events. For most use cases, [`decodeFromLines()`](#decodefromlines-lines-options) or [`decode()`](#decode-input-options) are more convenient.
 
 Path expansion (`expandPaths: 'safe'`) is **not supported** in streaming mode since it requires the full value tree.
 :::
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
 | `lines` | `Iterable<string>` | Iterable of TOON lines (without trailing newlines) |
-| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see below) |
+| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see [Configuration Reference](#configuration-reference)) |
 
-### Options
+#### Return Value
 
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `indent` | `number` | `2` | Expected number of spaces per indentation level |
-| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+Returns an `Iterable<JsonStreamEvent>` that yields structured events (see [TypeScript Types](#typescript-types) for event structure).
 
-### Return Value
-
-Returns an `Iterable<JsonStreamEvent>` that yields structured events.
-
-### Event Types
-
-Events represent the structure of the JSON data model:
-
-```ts
-type JsonStreamEvent
-  = | { type: 'startObject' }
-    | { type: 'endObject' }
-    | { type: 'startArray' }
-    | { type: 'endArray' }
-    | { type: 'key', key: string }
-    | { type: 'primitive', value: JsonPrimitive }
-
-type JsonPrimitive = string | number | boolean | null
-```
-
-### Example
+#### Example
 
 **Basic event streaming:**
 
@@ -445,31 +458,24 @@ for (const event of decodeStreamSync(lines)) {
 }
 ```
 
-## `decodeStream(source, options?)`
+### `decodeStream(source, options?)`
 
-Asynchronously decodes TOON lines into a stream of JSON events. This is the async version of [`decodeStreamSync()`](#decodeStreamSync-lines-options), supporting both synchronous and asynchronous iterables.
+Asynchronously decodes TOON lines into a stream of JSON events. This is the async version of [`decodeStreamSync()`](#decodestreamsync-lines-options), supporting both synchronous and asynchronous iterables.
 
 Useful for processing file streams, network responses, or other async sources where you want to handle data incrementally as it arrives.
 
-### Parameters
+#### Parameters
 
 | Parameter | Type | Description |
 |-----------|------|-------------|
 | `source` | `AsyncIterable<string>` \| `Iterable<string>` | Async or sync iterable of TOON lines (without trailing newlines) |
-| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see below) |
+| `options` | `DecodeStreamOptions?` | Optional streaming decoding configuration (see [Configuration Reference](#configuration-reference)) |
 
-### Options
+#### Return Value
 
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `indent` | `number` | `2` | Expected number of spaces per indentation level |
-| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+Returns an `AsyncIterable<JsonStreamEvent>` that yields structured events asynchronously (see [TypeScript Types](#typescript-types) for event structure).
 
-### Return Value
-
-Returns an `AsyncIterable<JsonStreamEvent>` that yields structured events asynchronously.
-
-### Example
+#### Example
 
 **Streaming from file:**
 
@@ -487,41 +493,173 @@ for await (const event of decodeStream(rl)) {
 }
 ```
 
-**Processing events incrementally:**
+## Error Handling
+
+Decoding throws a `ToonDecodeError` when input cannot be parsed. The class extends `SyntaxError`, so existing `error instanceof SyntaxError` checks keep working without code changes.
+
+### `ToonDecodeError`
 
 ```ts
-import { decodeStream } from '@toon-format/toon'
+import { ToonDecodeError } from '@toon-format/toon'
+```
 
-const lines = getAsyncLineSource() // AsyncIterable<string>
+#### Fields
 
-for await (const event of decodeStream(lines, { strict: true })) {
-  if (event.type === 'key' && event.key === 'id') {
-    // Next event will be the id value
-    const valueEvent = await decodeStream(lines).next()
-    if (valueEvent.value?.type === 'primitive') {
-      console.log('Found ID:', valueEvent.value.value)
-    }
+| Field | Type | Description |
+|-------|------|-------------|
+| `name` | `'ToonDecodeError'` | Discriminator – `error.name === 'ToonDecodeError'` |
+| `message` | `string` | Human-readable message; prefixed with `Line N: ` when a line is known |
+| `line` | `number?` | 1-based line number where the error was detected |
+| `source` | `string?` | Raw source line (including its leading whitespace) |
+| `cause` | `unknown?` | The original error when the decoder enriched a lower-level parser failure |
+
+The `line` and `source` fields are populated for every error that has line context – essentially every parse error during normal decoding. The `cause` chain points back to the underlying `SyntaxError` or `TypeError` thrown by the token-level parser, so debuggers and verbose loggers can show the original frame.
+
+#### Example
+
+```ts
+import { decode, ToonDecodeError } from '@toon-format/toon'
+
+try {
+  decode('a:\n\tb: 1')
+}
+catch (error) {
+  if (error instanceof ToonDecodeError) {
+    console.error(`Line ${error.line}:`, error.source)
+    console.error(error.message)
+    // Line 2: 	b: 1
+    // Line 2: Tabs are not allowed in indentation in strict mode
+  }
+  else {
+    throw error
   }
 }
 ```
 
-**Auto-detection of sync/async sources:**
+::: info Backwards Compatibility
+`ToonDecodeError` extends `SyntaxError`. Code written against earlier versions that catches `SyntaxError` continues to match these errors. The class adds structured fields without removing anything.
+:::
 
-```ts
-// Works with sync iterables
-const syncLines = ['name: Alice', 'age: 30']
-for await (const event of decodeStream(syncLines)) {
-  console.log(event)
-}
+## Configuration Reference
 
-// Works with async iterables
-const asyncLines = readLinesFromNetwork()
-for await (const event of decodeStream(asyncLines)) {
-  console.log(event)
-}
+### `EncodeOptions`
+
+Configuration for [`encode()`](#encode-input-options) and [`encodeLines()`](#encodelines-input-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Number of spaces per indentation level |
+| `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows |
+| `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths |
+| `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) |
+| `replacer` | `EncodeReplacer` | `undefined` | Optional hook to transform or omit values before encoding (see [Replacer Function](#replacer-function)) |
+
+**Delimiter options:**
+
+::: code-group
+
+```ts [Comma (default)]
+encode(data, { delimiter: ',' })
 ```
 
-## Round-Trip Compatibility
+```ts [Tab]
+encode(data, { delimiter: '\t' })
+```
+
+```ts [Pipe]
+encode(data, { delimiter: '|' })
+```
+
+:::
+
+See [Delimiter Strategies](#delimiter-strategies) for guidance on choosing delimiters.
+
+### `DecodeOptions`
+
+Configuration for [`decode()`](#decode-input-options) and [`decodeFromLines()`](#decodefromlines-lines-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Expected number of spaces per indentation level |
+| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+| `expandPaths` | `'off'` \| `'safe'` | `'off'` | Enable path expansion to reconstruct dotted keys into nested objects (pairs with `keyFolding: 'safe'`) |
+
+By default (`strict: true`), the decoder validates input strictly:
+
+- **Invalid escape sequences**: Throws on `\x`, unterminated strings, lone-surrogate `\uXXXX`
+- **Syntax errors**: Throws on missing colons, malformed headers
+- **Array length mismatches**: Throws when declared length doesn't match actual count
+- **Header delimiter mismatch**: Throws when the bracket-declared delimiter differs from the field-list delimiter (§14.2)
+- **Indentation errors**: Throws when leading spaces aren't exact multiples of `indent`
+- **Header structure**: Throws on leading-zero or non-integer array lengths and on intervening content between bracket/fields/colon
+- **Duplicate sibling keys**: Throws when an object has two children with the same key (§14.4)
+- **Path-expansion conflicts**: When `expandPaths: 'safe'` is set, throws on overlapping dotted paths that would collide
+
+All decode errors are thrown as [`ToonDecodeError`](#error-handling) instances with structured `line` and `source` fields.
+
+Set `strict: false` to skip these checks. Duplicate sibling keys and path-expansion conflicts then resolve with last-write-wins in document order.
+
+See [Key Folding & Path Expansion](#key-folding-path-expansion) for more details on path expansion behavior and conflict resolution.
+
+### `DecodeStreamOptions`
+
+Configuration for [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options):
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `indent` | `number` | `2` | Expected number of spaces per indentation level |
+| `strict` | `boolean` | `true` | Enable strict validation (array counts, indentation, delimiter consistency) |
+
+::: warning Path Expansion Not Supported
+Path expansion requires building the full value tree, which is incompatible with event streaming. Use [`decodeFromLines()`](#decodefromlines-lines-options) if you need path expansion.
+:::
+
+## TypeScript Types
+
+### `JsonStreamEvent`
+
+Events emitted by [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options):
+
+```ts
+type JsonStreamEvent
+  = | { type: 'startObject' }
+    | { type: 'endObject' }
+    | { type: 'startArray', length: number }
+    | { type: 'endArray' }
+    | { type: 'key', key: string, wasQuoted?: boolean }
+    | { type: 'primitive', value: JsonPrimitive }
+```
+
+### Delimiters
+
+```ts
+import { DEFAULT_DELIMITER, DELIMITERS } from '@toon-format/toon'
+
+DEFAULT_DELIMITER // ','
+DELIMITERS // { comma: ',', tab: '\t', pipe: '|' }
+```
+
+| Export | Description |
+|--------|-------------|
+| `DEFAULT_DELIMITER` | The default delimiter character (`,`) used when none is specified |
+| `DELIMITERS` | Frozen record mapping delimiter names to their characters |
+| `Delimiter` | Type union of valid delimiter characters: `',' \| '\t' \| '\|'` |
+| `DelimiterKey` | Type union of delimiter names: `'comma' \| 'tab' \| 'pipe'` |
+
+### Option Types
+
+| Export | Description |
+|--------|-------------|
+| `EncodeOptions` | Options accepted by [`encode()`](#encode-input-options) and [`encodeLines()`](#encodelines-input-options) |
+| `DecodeOptions` | Options accepted by [`decode()`](#decode-input-options) and [`decodeFromLines()`](#decodefromlines-lines-options) |
+| `DecodeStreamOptions` | Options accepted by [`decodeStreamSync()`](#decodestreamsync-lines-options) and [`decodeStream()`](#decodestream-source-options) |
+| `EncodeReplacer` | Signature of the [replacer function](#replacer-function) |
+| `ResolvedEncodeOptions` | `EncodeOptions` after defaults are applied (advanced) |
+| `ResolvedDecodeOptions` | `DecodeOptions` after defaults are applied (advanced) |
+
+## Guides & Examples
+
+### Round-Trip Compatibility
 
 TOON provides lossless round-trips after normalization:
 
@@ -542,7 +680,7 @@ console.log(JSON.stringify(original) === JSON.stringify(restored))
 // true
 ```
 
-### With Key Folding
+**With Key Folding:**
 
 ```ts
 import { decode, encode } from '@toon-format/toon'
@@ -561,19 +699,68 @@ console.log(JSON.stringify(original) === JSON.stringify(restored))
 // true
 ```
 
-## Types
+### Key Folding & Path Expansion
+
+**Key Folding** (`keyFolding: 'safe'`) collapses single-key wrapper chains during encoding:
 
 ```ts
-interface EncodeOptions {
-  indent?: number
-  delimiter?: ',' | '\t' | '|'
-  keyFolding?: 'off' | 'safe'
-  flattenDepth?: number
-}
+import { encode } from '@toon-format/toon'
 
-interface DecodeOptions {
-  indent?: number
-  strict?: boolean
-  expandPaths?: 'off' | 'safe'
-}
+const data = { data: { metadata: { items: ['a', 'b'] } } }
+
+// Without folding
+encode(data)
+// data:
+//   metadata:
+//     items[2]: a,b
+
+// With folding
+encode(data, { keyFolding: 'safe' })
+// data.metadata.items[2]: a,b
 ```
+
+**Path Expansion** (`expandPaths: 'safe'`) reverses this during decoding:
+
+```ts
+import { decode } from '@toon-format/toon'
+
+const toon = 'data.metadata.items[2]: a,b'
+
+const data = decode(toon, { expandPaths: 'safe' })
+console.log(data)
+// { data: { metadata: { items: ['a', 'b'] } } }
+```
+
+**Expansion Conflict Resolution:**
+
+When multiple expanded keys construct overlapping paths, the decoder merges them recursively:
+- **Object + Object**: Deep merge recursively
+- **Object + Non-object** (array or primitive): Conflict
+  - With `strict: true` (default): Error
+  - With `strict: false`: Last-write-wins (LWW)
+
+Duplicate sibling keys (independent of `expandPaths`) follow the same policy: strict mode throws, lenient mode keeps the last value seen.
+
+### Delimiter Strategies
+
+Tab delimiters (`\t`) often tokenize more efficiently than commas. Tabs are single characters that rarely appear in natural text, which reduces the need for quote-escaping and leads to smaller token counts in large datasets.
+
+Example:
+
+```yaml
+items[2	]{sku	name	qty	price}:
+  A1	Widget	2	9.99
+  B2	Gadget	1	14.5
+```
+
+For maximum token savings on large tabular data, combine tab delimiters with key folding:
+
+```ts
+encode(data, { delimiter: '\t', keyFolding: 'safe' })
+```
+
+**Choosing a Delimiter:**
+
+- **Comma (`,`)**: Default, widely understood, good for simple tabular data.
+- **Tab (`\t`)**: Best for LLM token efficiency, excellent for large datasets.
+- **Pipe (`|`)**: Alternative when commas appear frequently in data.

docs/reference/efficiency-formalization.md

@@ -0,0 +1,509 @@
+---
+description: Mathematical model of TOON's byte-level overhead vs JSON across structure families, with formulas and worked examples.
+---
+
+# TOON vs JSON: Byte-Level Efficiency Model
+
+A mathematical analysis of TOON's byte efficiency compared to JSON across different data structures.
+
+::: info Scope of This Document
+This page presents a theoretical, character-based comparison between TOON and JSON. For practical benchmarks and token counts, see [Benchmarks](/guide/benchmarks). It is an **advanced, non-normative** reference: it explains TOON's design from a mathematical angle but does not change the TOON specification.
+:::
+
+## Overview
+
+Standard JSON introduces structural verbosity that inflates token usage and inference cost. This page formalises a byte-level comparison between TOON and JSON to evaluate whether TOON achieves quantifiable efficiency gains by removing structural redundancy.
+
+Under the assumptions described below (compact JSON, canonical TOON, ASCII keys and punctuation, shallow to moderate nesting, and mostly unquoted TOON strings), TOON's **structural overhead is lower than compact JSON** for the structure families analyzed here, except arrays of arrays.
+
+### Key Findings
+
+- **Tabular arrays** represent TOON's optimal use case, with efficiency gains scaling linearly with both row count and field count.
+- **Simple objects and primitive arrays** show consistent byte reduction, with savings proportional to the number of fields or elements.
+- **Nested objects** benefit from reduced overhead, though efficiency decreases with depth due to indentation costs; at sufficient depth, compact JSON can become smaller.
+- **Arrays of arrays** are the only structure where TOON is less efficient than JSON in this analysis, due to TOON's explicit list markers and inner array headers.
+
+## Methodology
+
+We define recursive byte-length functions $L_{\text{json}}$ and $L_{\text{toon}}$ for both formats, then derive the efficiency delta:
+
+$$
+\Delta = L_{\text{json}}(\Omega) - L_{\text{toon}}(\Omega)
+$$
+
+Where $\Omega$ represents the data structure under comparison. If $\Delta > 0$, TOON uses fewer bytes than JSON for that structure.
+
+::: info Scope & Assumptions
+- **Compact JSON**: JSON is assumed to be compact (no spaces or newlines outside strings). Byte counts are computed on this compact form.
+- **Canonical TOON**: TOON is assumed to follow canonical formatting (indent = 2 spaces, exactly one space after `:`, no spaces after commas in arrays/field lists, no trailing spaces).
+- **Keys and strings**: All keys are "simple" ASCII identifier-style keys that:
+  - must be quoted in JSON, and
+  - can be left unquoted in TOON (no characters that would force quoting).
+  Many examples assume values are numbers, booleans, null, or TOON-safe strings that can be unquoted in TOON but must be quoted in JSON.
+- **Numbers**: For this analysis only, both formats are assumed to use the same canonical decimal representation. JSON could use exponent forms; we ignore that here to isolate structural differences.
+- **ASCII/UTF-8**: Keys and structural tokens are assumed ASCII, so byte length equals character count ($|x|_{\text{utf8}} = |x|_{\text{char}}$). Non-ASCII content affects both formats similarly and does not change the structural conclusions.
+- **Nesting depth**: Closed-form expressions are given for flat structures and a single level of nesting. Each additional nesting level in TOON adds 2 bytes of indentation per nested line. At sufficient depth, the braces of compact JSON can win over TOON's indentation (as seen in [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon)).
+- **Byte vs token count**: Modern LLM tokenizers operate over UTF-8 bytes, so byte length is a good upper bound and first-order proxy for token count, even though the mapping is not exactly linear.
+:::
+
+Think of this as a simplified structural model: we strip away real-world noise and ask, "if you only count structural characters, how do JSON and TOON compare?"
+
+## Formal Notation
+
+### Data Model
+
+Let $\omega$ be a primitive value such that $\omega \in \{\text{string, number, boolean, null}\}$.
+
+Let $\mathcal{O}$ be an object composed of $n$ key-value pairs:
+
+$$
+\mathcal{O} = \{(k_1, v_1), (k_2, v_2), \dots, (k_n, v_n)\}
+$$
+
+Let $\mathcal{A}$ be an array composed of $n$ elements:
+
+$$
+\mathcal{A} = \{v_1, v_2, \dots, v_n\}
+$$
+
+Where:
+- $k_i$ is a key (string)
+- $v_i$ can be a primitive value $\omega$, an object $\mathcal{O}$, or an array $\mathcal{A}$
+
+Therefore: $v_i \in \{\omega, \mathcal{O}, \mathcal{A}\}$
+
+### String Length
+
+Let $\mathcal{S}$ be the set of valid Unicode strings. For any string $x \in \mathcal{S}$, we denote $|x|_{\text{utf8}}$ as the byte-length of $x$ under UTF-8 encoding.
+
+### Integer Length
+
+Let $n \in \mathbb{Z}_{\ge 0}$ be a non-negative integer. The number of bytes required to represent $n$ in decimal format is:
+
+$$
+L_{\text{num}}(n) = \begin{cases}
+1 & \text{if } n = 0 \\
+\lfloor \log_{10}(|n|) \rfloor + 1 & \text{if } n > 0
+\end{cases}
+$$
+
+## JSON Size Functions
+
+For a flat object of $n$ keys:
+
+$$
+L_{\text{json}}(\mathcal{O}) = \underbrace{2}_{\{\}} + \sum_{i=1}^{n} (L_{\text{str}}(k_i) + \underbrace{1}_{:} + L_{\text{json}}(v_i)) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+Where $L_{\text{str}}(k)$ is the length of the key including its mandatory quotes:
+
+$$
+L_{\text{str}}(k) = |k|_{\text{utf8}} + \underbrace{2}_{\text{quotes}}
+$$
+
+### Primitive Values in JSON
+
+When $v_i$ is a primitive data type $\omega$:
+
+| Type | Formula |
+|------|---------|
+| String | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}} + 2$ |
+| Number | $L_{\text{num}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Boolean | $L_{\text{bool}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Null | $L_{\text{null}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+
+### Arrays in JSON
+
+When $v_i$ is an array $\mathcal{A}$:
+
+$$
+L_{\text{json}}(\mathcal{A}) = \underbrace{2}_{\text{[]}} + \sum_{i=1}^{n} L_{\text{json}}(v_i) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+## TOON Size Functions
+
+For a flat object of $n$ keys:
+
+$$
+L_{\text{toon}}(\mathcal{O}) = \sum_{i=1}^{n} (L_{\text{str}}(k_i) + \underbrace{1}_{:} + \underbrace{1}_{\text{space}} + L_{\text{toon}}(v_i)) + \underbrace{(n-1)}_{\text{newlines}}
+$$
+
+Where $L_{\text{str}}(k)$ is the length of the key (no quotes required for simple keys):
+
+$$
+L_{\text{str}}(k) = |k|_{\text{utf8}}
+$$
+
+### Primitive Values in TOON
+
+When $v_i$ is a primitive data type $\omega$:
+
+| Type | Formula |
+|------|---------|
+| String (normal) | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| String (looks like number/boolean) | $L_{\text{str}}(v_i) = \lvert v_i\rvert_{\text{utf8}} + 2$ |
+| Number | $L_{\text{num}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Boolean | $L_{\text{bool}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+| Null | $L_{\text{null}}(v_i) = \lvert v_i\rvert_{\text{utf8}}$ |
+
+### Simple Arrays in TOON
+
+Here $L_{\text{toon}}(\mathcal{A})$ refers to the length of the whole field line `key[N]: ...`, not just the array value.
+
+When $v_i$ is a simple array $\mathcal{A}$:
+
+$$
+L_{\text{toon}}(\mathcal{A}) = L_{\text{str}}(k_i) + \underbrace{1}_{\text{[}} + L_{\text{num}}(n) + \underbrace{1}_{\text{]}} + \underbrace{1}_{:} + \underbrace{1}_{\text{space}} + \sum_{i=1}^{n} L_{\text{toon}}(v_i) + \underbrace{(n-1)}_{\text{commas}}
+$$
+
+### Tabular Arrays in TOON
+
+When $v_i$ is an array of objects with $m$ fields:
+
+$$
+\begin{split}
+L_{\text{toon}}(\mathcal{A}') = L_{\text{str}}(k_i) + \underbrace{1}_{\text{[}} + L_{\text{num}}(n) + \underbrace{1}_{\text{]}} + \underbrace{1}_{\{} + \\
+\sum_{i=1}^{m} L_{\text{str}}(k_i) + \underbrace{(m-1)}_{\text{commas}} + \underbrace{1}_{\}} + \underbrace{1}_{:} + \\
+\underbrace{2n}_{\text{indents}} + \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{toon}}(v_{ij}) + \underbrace{(m-1)n}_{\text{commas}} + \underbrace{n}_{\text{newlines}}
+\end{split}
+$$
+
+*Note: The term $2n$ assumes an indentation size of 2 spaces.*
+
+## Efficiency Analysis by Structure
+
+Each subsection below focuses on a particular structure family, states the resulting formula, and shows a small example. Intuitively, TOON tends to win when it can:
+
+- avoid repeating keys (tabular arrays),
+- avoid quoting keys and many values,
+- and replace braces with indentation,
+
+and tends to lose when it pays a fixed overhead per element (arrays of arrays) or deep indentation (heavily nested configs).
+
+### Simple Objects
+
+Flat objects with primitive string values are the easiest win: JSON pays for braces and quoted keys and strings, while TOON drops braces at the root, omits quotes on simple keys, and uses one line per field.
+
+For objects with only string primitives:
+
+$$
+\Delta_{\text{obj}} = 2 + n + \sum_{i=1}^{n}(L_{\text{json}}(v_i)) - \sum_{i=1}^{n}(L_{\text{toon}}(v_i))
+$$
+
+If all values are strings that can be unquoted in TOON, this simplifies to:
+
+$$
+f(n) = 2 + 3n
+$$
+
+**Example:** For 1,000,000 objects, TOON saves **3,000,002 bytes ≈ 2.86 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (21 bytes)]
+{ "id": 1, "name": "Ada" }
+```
+
+```yaml [TOON (15 bytes)]
+id: 1
+name: Ada
+```
+
+:::
+
+$$
+\Delta_{\text{obj}} = 2 + \underbrace{2}_{n} + \underbrace{6}_{\sum L_{\text{json}}(v_i)} - \underbrace{4}_{\sum L_{\text{toon}}(v_i)} = 6
+$$
+
+### Nested Objects
+
+Adding a wrapper object (one extra level of nesting) introduces extra braces for JSON and extra indentation and newlines for TOON. For a single level of nesting with primitive values, TOON still comes out ahead, but the net advantage is smaller.
+
+For a single level of nesting with primitives:
+
+$$
+f(n) = 5 + n
+$$
+
+**Example:** For 1,000,000 nested objects (depth 1), TOON saves **1,000,005 bytes ≈ 0.95 MB**.
+
+::: warning Caveat
+This formula is for a single nesting level. Each additional nesting level adds 2 spaces of indentation per nested line; at sufficient depth, compact JSON can become smaller, especially when tabular opportunities disappear (see [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon) and the "Deeply nested configuration" dataset in [Benchmarks](/guide/benchmarks)).
+:::
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (30 bytes)]
+{ "user": { "id": 1, "name": "Ada" } }
+```
+
+```yaml [TOON (25 bytes)]
+user:
+  id: 1
+  name: Ada
+```
+
+:::
+
+$$
+\Delta_{\text{nested}} = 5
+$$
+
+### Primitive Arrays
+
+For arrays of string primitives, JSON writes `["foo","bar","baz"]`, quoting every string and using `[]` for the array. TOON writes `key[N]: foo,bar,baz`, paying once for the length marker but omitting most quotes.
+
+For arrays of $n$ string primitives:
+
+$$
+\Delta_{\text{arr}} = 3 - L_{\text{num}}(n) + \sum_{i=1}^{n}(L_{\text{json}}(v_i)) - \sum_{i=1}^{n}(L_{\text{toon}}(v_i))
+$$
+
+With string values that can be unquoted in TOON, this simplifies to:
+
+$$
+f(n) = 2 + 2n - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 elements, TOON saves **1,999,996 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (28 bytes)]
+{ "tags": ["foo", "bar", "baz"] }
+```
+
+```yaml [TOON (20 bytes)]
+tags[3]: foo,bar,baz
+```
+
+:::
+
+$$
+\Delta_{\text{arr}} = 3 - \underbrace{1}_{L_{\text{num}}(3)} + \underbrace{15}_{\sum L_{\text{json}}} - \underbrace{9}_{\sum L_{\text{toon}}} = 8
+$$
+
+### Root Arrays
+
+At the root, JSON writes `["x","y","z"]`; TOON writes `[3]: x,y,z`. There is no object key cost, so the advantage mainly comes from not quoting TOON-safe strings and from replacing `[]` with `[N]:`.
+
+For root-level arrays of $n$ string primitives:
+
+$$
+f(n) = -3 + 2n - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 elements, TOON saves **1,999,991 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (13 bytes)]
+["x", "y", "z"]
+```
+
+```yaml [TOON (10 bytes)]
+[3]: x,y,z
+```
+
+:::
+
+$$
+\Delta_{\text{root}} = \underbrace{9}_{\sum L_{\text{json}}} - 2 - \underbrace{1}_{L_{\text{num}}(3)} - \underbrace{3}_{\sum L_{\text{toon}}} = 3
+$$
+
+### Tabular Arrays
+
+Uniform arrays of objects are TOON's sweet spot. JSON repeats every key for every row, while TOON declares the length and column names once (`key[N]{id,qty,...}:`) and streams rows as bare values.
+
+For arrays of objects with $n$ rows and $m$ fields, assuming numeric values and $|k| = 3$:
+
+$$
+f(n) = 1 + nm(3 + |k|) - m(1 + |k|) - \lfloor \log_{10}(|n|) \rfloor
+$$
+
+**Example:** For 1,000,000 rows with 2 fields and 3-character field names, TOON saves **11,999,987 bytes ≈ 11.44 MB**.
+
+This is where TOON's design (declare fields once, stream rows) pays off most strongly: savings grow linearly with both row count and field count.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (45 bytes)]
+{ "items": [{ "id": 1, "qty": 5 }, { "id": 2, "qty": 3 }] }
+```
+
+```yaml [TOON (29 bytes)]
+items[2]{id,qty}:
+  1,5
+  2,3
+```
+
+:::
+
+$$
+\Delta_{\text{tab}} = 2 + \underbrace{4}_{nm} - \underbrace{2}_{m} + \underbrace{22}_{\Sigma L_{\text{json}}} - \underbrace{1}_{L_{\text{num}}(n)} - \underbrace{5}_{\Sigma L_{\text{toon}}(k)} - \underbrace{4}_{\Sigma L_{\text{toon}}(v)} = 16
+$$
+
+### Arrays of Arrays
+
+Arrays of arrays of primitives are where TOON structurally loses: each inner array becomes a list item with its own header, so TOON pays a fixed overhead per inner array (`"- "` plus `"[m]: "`), while JSON just uses commas.
+
+::: info Practical Note
+For arrays of arrays of primitives, this model predicts that JSON is more byte-efficient than TOON, because TOON pays ~6 extra bytes per inner array (2 for `"- "`, 4 for `"[m]: "`), plus the length marker.
+:::
+
+For arrays of arrays with $n$ outer elements and $m$ inner elements:
+
+$$
+\begin{split}
+\Delta_{\text{arrarr}} = 2 - 6n - \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{num}}(m) + \\
+\sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{json}}(v_{ij}) - \sum_{i=1}^{n}\sum_{j=1}^{m} L_{\text{toon}}(v_{ij})
+\end{split}
+$$
+
+With string primitives and $m = 2$:
+
+$$
+f(n) = 2 - 6n - \sum_{i=1}^{n}\sum_{j=1}^{m} (\lfloor \log_{10}(|m|) \rfloor + 1) + 2nm
+$$
+
+**Example:** For 1,000,000 arrays with $m = 2$, TOON **wastes 2,999,998 bytes ≈ 2.86 MB** relative to JSON under this model.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (23 bytes)]
+{ "pairs": [[1, 2], [3, 4]] }
+```
+
+```yaml [TOON (35 bytes)]
+pairs[2]:
+  - [2]: 1,2
+  - [2]: 3,4
+```
+
+:::
+
+$$
+\Delta_{\text{arrarr}} = 2 - \underbrace{12}_{6n} - \underbrace{2}_{\sum L_{\text{num}}(m)} + \underbrace{4}_{\sum L_{\text{json}}} - \underbrace{4}_{\sum L_{\text{toon}}} = -12
+$$
+
+### Strings That Look Like Literals
+
+Strings that look like numbers or booleans (e.g. `"123"`, `"true"`) must be quoted in both JSON and TOON, slightly reducing TOON's advantage because it no longer saves quotes on those values.
+
+For objects containing such strings:
+
+$$
+\Delta_{\text{strlit}} = 2 + n
+$$
+
+**Example:** For 1,000,000 objects, TOON saves **2,000,002 bytes ≈ 1.91 MB**.
+
+#### Empirical Validation
+
+::: code-group
+
+```json [JSON (34 bytes)]
+{ "version": "123", "enabled": "true" }
+```
+
+```yaml [TOON (30 bytes)]
+version: "123"
+enabled: "true"
+```
+
+:::
+
+$$
+\Delta_{\text{str}} = 2 + \underbrace{2}_{n} = 4
+$$
+
+### Empty Structures
+
+Empty containers reveal structural differences even at minimal sizes.
+
+**Empty Object:**
+
+$$
+\Delta_{\text{EmptyObject}} = 2
+$$
+
+JSON requires `{}` (2 bytes), whereas a completely empty root object in TOON is represented as an empty document (0 bytes).
+
+**Empty Array (field):**
+
+$$
+\Delta_{\text{EmptyArray}} = 3
+$$
+
+For a field named `key`, JSON uses `{"key":[]}` in compact form, while TOON uses:
+
+```yaml
+key: []
+```
+
+Under this model, that yields a constant 3-byte advantage for TOON. The legacy `key[0]:` form remains decodable for backward compatibility.
+
+## Summary Table
+
+The table below summarizes the formulas and which side wins under the modeling assumptions.
+
+| Structure | Efficiency Formula | TOON Advantage? |
+|-----------|-------------------|-----------------|
+| Simple Objects | $f(n) = 2 + 3n$ | ✅ Yes |
+| Nested Objects (1 level) | $f(n) = 5 + n$ | ✅ Yes (shrinks with depth) |
+| Primitive Arrays | $f(n) = 2 + 2n - \lfloor \log_{10}(n) \rfloor$ | ✅ Yes |
+| Root Arrays | $f(n) = -3 + 2n - \lfloor \log_{10}(n) \rfloor$ | ✅ Yes |
+| Tabular Arrays | $f(n) = 1 + nm(3+\lvert k\rvert) - m(1+\lvert k\rvert) - \lfloor \log_{10}(n) \rfloor$ | ✅ **Best case** |
+| Arrays of Arrays | $f(n) = 2 - 6n + 2nm - \text{overhead}$ | ❌ JSON wins here |
+| String Literals | $f(n) = 2 + n$ | ✅ Yes (smaller gain) |
+| Empty Structures | $\Delta = 2$ or $3$ | ✅ Yes |
+
+In short:
+
+- TOON's gains are **linear in the number of fields** for flat objects.
+- For arrays, gains grow **linearly in the number of elements**, and for tabular arrays **linearly in both rows and fields**.
+- Arrays of arrays are the main structural case where JSON is smaller.
+- Deep nesting and heavy quoting can erode or reverse these advantages in real data.
+
+## Conclusion
+
+This simplified theoretical model supports TOON's design goal: structurally, it reduces overhead compared to compact JSON in many common patterns by:
+
+- avoiding repeated keys in tabular arrays,
+- omitting quotes on many keys and values,
+- and replacing braces with indentation at shallow depths.
+
+For the structure families examined here and under the stated assumptions, the structural overhead of TOON is lower than that of compact JSON except for arrays of arrays. Since UTF-8 byte length is a reasonable first-order proxy for tokens, these structural savings usually translate into lower token counts in those patterns.
+
+At the same time, this is deliberately a simplified model. In real datasets, additional factors – deeper or irregular nesting, heavily quoted strings, exponent notation in JSON, and tokenizer idiosyncrasies – can reduce or even reverse these gains. Our [Benchmarks](/guide/benchmarks) and [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon) show that compact JSON can be more efficient for deeply nested or low-tabularity data. Use this page as intuition for *why* TOON behaves the way it does, not as a universal guarantee.
+
+## Related Resources
+
+- [Benchmarks](/guide/benchmarks) – Empirical token count and accuracy comparisons across formats
+- [Specification](/reference/spec) – Formal TOON specification
+
+## References
+
+This analysis is based on:
+
+- **Original Research**: [TOON vs. JSON: A Mathematical Evaluation of Byte Efficiency in Structured Data](https://www.researchgate.net/publication/397903673_TOON_vs_JSON_A_Mathematical_Evaluation_of_Byte_Efficiency_in_Structured_Data)
+- **TOON Specification**: [toon-format/spec](https://github.com/toon-format/spec)
+- **JSON Specification**: [RFC 8259](https://datatracker.ietf.org/doc/html/rfc8259), [ECMA-404](https://www.ecma-international.org/publications-and-standards/standards/ecma-404/)
+
+---
+
+This page was contributed by Mateo Lafalce ([@mateolafalce](https://github.com/mateolafalce)).
+
+*Have questions or found an error in the formalization? Open an issue on [GitHub](https://github.com/toon-format/spec) or contribute improvements to this analysis.*

docs/reference/spec.md

@@ -1,111 +1,143 @@
+---
+description: Guided tour of the TOON specification – sections, conformance checklists, media type, and versioning.
+---
+
 # Specification
 
 The [TOON specification](https://github.com/toon-format/spec) is the authoritative reference for implementing encoders, decoders, and validators. It defines the concrete syntax, normative encoding/decoding behavior, and strict-mode validation rules.
 
 You don't need this page to *use* TOON. It's mainly for implementers and contributors. If you're looking to learn how to use TOON, start with the [Getting Started](/guide/getting-started) guide instead.
 
-> [!TIP]
-> The TOON specification is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to it or sharing feedback!
+> [!NOTE]
+> The TOON specification is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to it or sharing feedback.
 
 ## Current Version
 
-**Spec v{{ $spec.version }}** (2025-11-10) is the current stable version.
+**Spec v{{ $spec.version }}** (2026-05-20) is the current published Working Draft. It is stable for implementation but not yet finalized; see "Status of This Document" in the spec for details.
+
+## Media Type & File Extension
+
+The spec defines a provisional media type and file extension in [§17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
+
+- **Media type:** `text/toon` (provisional, not yet IANA‑registered; UTF‑8 only)
+- **File extension:** `.toon`
+
+TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `charset` parameter, when present, is `utf-8`.
 
 ## Guided Tour of the Spec
 
 ### Core Concepts
 
-**[§1 Terminology and Conventions](https://github.com/toon-format/spec/blob/main/SPEC.md#1-terminology-and-conventions)**
+[§1 Terminology and Conventions](https://github.com/toon-format/spec/blob/main/SPEC.md#1-terminology-and-conventions):
 Defines key terms like "indentation level", "active delimiter", "strict mode", and RFC2119 keywords (MUST, SHOULD, MAY).
 
-**[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model)**
-Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (no exponent notation, no leading/trailing zeros).
+[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model):
+Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (canonical decimal for values in `[1e-6, 1e21)` or zero; exponent form permitted outside).
 
-**[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder)**
+[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder):
 Defines how non-JSON types (Date, BigInt, NaN, Infinity, undefined, etc.) are normalized before encoding. Required reading for encoder implementers.
 
-**[§4 Decoding Interpretation](https://github.com/toon-format/spec/blob/main/SPEC.md#4-decoding-interpretation-reference-decoder)**
+[§4 Decoding Interpretation](https://github.com/toon-format/spec/blob/main/SPEC.md#4-decoding-interpretation-reference-decoder):
 Specifies how decoders map text tokens to host values (quoted strings, unquoted primitives, numeric parsing with leading-zero handling). Decoders default to strict mode (`strict = true`) in the reference implementation; strict-mode errors are enumerated in §14.
 
 ### Syntax Rules
 
-**[§5 Concrete Syntax and Root Form](https://github.com/toon-format/spec/blob/main/SPEC.md#5-concrete-syntax-and-root-form)**
+[§5 Concrete Syntax and Root Form](https://github.com/toon-format/spec/blob/main/SPEC.md#5-concrete-syntax-and-root-form):
 Defines TOON's line-oriented, indentation-based notation and how to determine whether the root is an object, array, or primitive.
 
-**[§6 Header Syntax](https://github.com/toon-format/spec/blob/main/SPEC.md#6-header-syntax-normative)**
+[§6 Header Syntax](https://github.com/toon-format/spec/blob/main/SPEC.md#6-header-syntax-normative):
 Normative ABNF grammar for array headers: `key[N<delim?>]{fields}:`. Specifies bracket segments, delimiter symbols, and field lists.
 
-**[§7 Strings and Keys](https://github.com/toon-format/spec/blob/main/SPEC.md#7-strings-and-keys)**
-Complete quoting rules (when strings MUST be quoted), escape sequences (only `\\`, `\"`, `\n`, `\r`, `\t` are valid), and key encoding requirements.
+[§7 Strings and Keys](https://github.com/toon-format/spec/blob/main/SPEC.md#7-strings-and-keys):
+Complete quoting rules (when strings MUST be quoted), escape sequences (only `\\`, `\"`, `\n`, `\r`, `\t`, and `\uXXXX` for other U+0000–U+001F controls are valid), and key encoding requirements.
 
-**[§8 Objects](https://github.com/toon-format/spec/blob/main/SPEC.md#8-objects)**
+[§8 Objects](https://github.com/toon-format/spec/blob/main/SPEC.md#8-objects):
 Object field encoding (key: value), nesting rules, key order preservation, and empty object handling.
 
-**[§9 Arrays](https://github.com/toon-format/spec/blob/main/SPEC.md#9-arrays)**
+[§9 Arrays](https://github.com/toon-format/spec/blob/main/SPEC.md#9-arrays):
 Covers all array forms: primitive (inline), arrays of objects (tabular), mixed/non-uniform (list), and arrays of arrays. Includes tabular detection requirements.
 
-**[§10 Objects as List Items](https://github.com/toon-format/spec/blob/main/SPEC.md#10-objects-as-list-items)**
-Indentation rules for objects appearing in list items (first field on hyphen line, nested object rules).
+[§10 Objects as List Items](https://github.com/toon-format/spec/blob/main/SPEC.md#10-objects-as-list-items):
+Indentation rules for objects appearing in list items (first field on the hyphen line), including the canonical pattern when the first field is a tabular array (header on the hyphen line, rows at depth +2, sibling fields at depth +1).
 
-**[§11 Delimiters](https://github.com/toon-format/spec/blob/main/SPEC.md#11-delimiters)**
+[§11 Delimiters](https://github.com/toon-format/spec/blob/main/SPEC.md#11-delimiters):
 Delimiter scoping (document vs active), delimiter-aware quoting, and parsing rules for comma/tab/pipe delimiters.
 
-**[§12 Indentation and Whitespace](https://github.com/toon-format/spec/blob/main/SPEC.md#12-indentation-and-whitespace)**
+[§12 Indentation and Whitespace](https://github.com/toon-format/spec/blob/main/SPEC.md#12-indentation-and-whitespace):
 Encoding requirements (consistent spaces, no tabs in indentation, no trailing spaces/newlines) and decoding rules (strict vs non-strict indentation handling).
 
 ### Conformance and Validation
 
-**[§13 Conformance and Options](https://github.com/toon-format/spec/blob/main/SPEC.md#13-conformance-and-options)**
-Defines conformance classes (encoder, decoder, validator), required options, and conformance checklists.
+[§13 Conformance and Options](https://github.com/toon-format/spec/blob/main/SPEC.md#13-conformance-and-options):
+Defines conformance classes (encoder, decoder, validator), standardized options, and conformance checklists.
 
-**[§13.4 Key Folding and Path Expansion](https://github.com/toon-format/spec/blob/main/SPEC.md#134-key-folding-and-path-expansion)**
-Optional encoder feature (key folding) and decoder feature (path expansion) for collapsing/expanding dotted paths. Specifies safety requirements and conflict resolution.
+[§13.4 Key Folding and Path Expansion](https://github.com/toon-format/spec/blob/main/SPEC.md#134-key-folding-and-path-expansion):
+Optional encoder feature (key folding) and decoder feature (path expansion) for collapsing/expanding dotted paths, with deep-merge semantics and strict/non-strict conflict resolution.
 
-**[§14 Strict Mode Errors and Diagnostics](https://github.com/toon-format/spec/blob/main/SPEC.md#14-strict-mode-errors-and-diagnostics-authoritative-checklist)**
-**Authoritative checklist** of all strict-mode errors: array count mismatches, syntax errors, indentation errors, structural errors, and path expansion conflicts.
+[§14 Strict Mode Errors and Diagnostics](https://github.com/toon-format/spec/blob/main/SPEC.md#14-strict-mode-errors-and-diagnostics-authoritative-checklist):
+**Authoritative checklist** of all strict-mode errors: array count and width mismatches (§14.1), syntax and structural errors (§14.2), path expansion conflicts (§14.3), and duplicate sibling keys (§14.4).
 
 ### Implementation Guidance
 
-**[§19 TOON Core Profile](https://github.com/toon-format/spec/blob/main/SPEC.md#19-toon-core-profile-normative-subset)**
-Normative subset of the most common, memory-friendly rules. Useful for minimal implementations.
+[§15 Security Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#15-security-considerations):
+Injection risks, quoting rules, and strict-mode checks relevant to security.
 
-**[Appendix G: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-g-host-type-normalization-examples-informative)**
-Non-normative guidance for Go, JavaScript, Python, and Rust implementations on normalizing language-specific types.
+[§16 Internationalization](https://github.com/toon-format/spec/blob/main/SPEC.md#16-internationalization):
+Unicode handling and locale-independent number formatting.
 
-**[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative)**
+[§17 IANA Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
+Media type registration plans and provisional status.
+
+[§18 Versioning and Extensibility](https://github.com/toon-format/spec/blob/main/SPEC.md#18-versioning-and-extensibility):
+How the spec evolves: major vs minor bumps and the extensibility policy.
+
+[§19 Intellectual Property Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#19-intellectual-property-considerations):
+Licensing and IP terms for the specification.
+
+[Appendix F: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-f-host-type-normalization-examples-informative):
+Non-normative guidance for Go, JavaScript, Python, Rust, and Java implementations on normalizing language-specific types.
+
+[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative):
 Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://github.com/toon-format/spec/tree/main/tests) for validating implementations.
 
 ## Spec Sections at a Glance
 
 | Section | Topic | When to Read |
 |---------|-------|--------------|
-| §1-4 | Data model, normalization, decoding | Implementing encoders/decoders |
-| §5-6 | Syntax, headers, root form | Implementing parsers |
+| §1–4 | Data model, normalization, decoding | Implementing encoders/decoders |
+| §5–6 | Syntax, headers, root form | Implementing parsers |
 | §7 | Strings, keys, quoting, escaping | Implementing string handling |
-| §8-10 | Objects, arrays, list items | Implementing structure encoding |
-| §11-12 | Delimiters, indentation, whitespace | Implementing formatting and validation |
-| §13 | Conformance, options, key folding | Implementing options and features |
+| §8–10 | Objects, arrays, list items | Implementing structure encoding |
+| §11–12 | Delimiters, indentation, whitespace | Implementing formatting and validation |
+| §13 | Conformance, options, key folding/path expansion | Implementing options and features |
 | §14 | Strict-mode errors | Implementing validators |
-| §19 | Core profile | Minimal implementations |
+| §15–16 | Security, internationalization | Operational considerations |
+| §17–19 | IANA, versioning, IP | Ecosystem and licensing |
 
 ## Conformance Checklists
 
 The spec includes three conformance checklists:
 
-### [Encoder Checklist (§13.1)](https://github.com/toon-format/spec/blob/main/SPEC.md#131-encoder-conformance-checklist)
+### Encoder Checklist (§13.1) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#131-encoder-conformance-checklist)</sup>
 
 Key requirements:
 - Produce UTF-8 with LF line endings
 - Use consistent indentation (default 2 spaces, no tabs)
-- Escape only `\\`, `\"`, `\n`, `\r`, `\t` in quoted strings
+- Escape `\\`, `\"`, `\n`, `\r`, `\t` in quoted strings, and use `\uXXXX` for any other U+0000–U+001F control character; lone surrogates are rejected
 - Quote strings with active delimiter, colon, or structural characters
 - Emit array lengths `[N]` matching actual count
 - Preserve object key order
-- Normalize numbers to non-exponential decimal form
+- Emit numbers per §2 (canonical decimal in `[1e-6, 1e21)` or zero; exponent form permitted outside)
 - Convert `-0` to `0`, `NaN`/±Infinity to `null`
+- Emit booleans and null as lowercase literals (`true`, `false`, `null`)
 - No trailing spaces or trailing newline
+- When `keyFolding="safe"` is enabled, folding MUST follow §13.4:
+  - Only fold IdentifierSegment keys (letters/digits/underscores, no dots),
+  - Do not introduce collisions with existing sibling keys,
+  - Do not fold segments that would require quoting.
+- When `flattenDepth` is set, folding MUST stop at the configured number of segments (§13.4).
 
-### [Decoder Checklist (§13.2)](https://github.com/toon-format/spec/blob/main/SPEC.md#132-decoder-conformance-checklist)
+### Decoder Checklist (§13.2) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#132-decoder-conformance-checklist)</sup>
 
 Key requirements:
 - Parse array headers per §6 (length, delimiter, fields)
@@ -114,21 +146,27 @@ Key requirements:
 - Type unquoted primitives: true/false/null → booleans/null, numeric → number, else → string
 - Enforce strict-mode rules when `strict=true`
 - Preserve array order and object key order
+- When `expandPaths="safe"` is enabled, expand dotted keys into nested objects per §13.4:
+  - Split on `.`, only expand when all segments are IdentifierSegments,
+  - Deep-merge overlapping paths (object + object),
+  - Do not perform element-wise array merges.
+- With `expandPaths="safe"` and `strict=true` (default), MUST error on any expansion conflict (§14.3).
+- With `expandPaths="safe"` and `strict=false`, MUST apply deterministic last-write-wins (LWW) conflict resolution (§13.4).
 
-### [Validator Checklist (§13.3)](https://github.com/toon-format/spec/blob/main/SPEC.md#133-validator-conformance-checklist)
+### Validator Checklist (§13.3) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#133-validator-conformance-checklist)</sup>
 
 Validators should verify:
 - Structural conformance (headers, indentation, list markers)
 - Whitespace invariants (no trailing spaces/newlines)
 - Delimiter consistency between headers and rows
 - Array length counts match declared `[N]`
-- All strict-mode requirements
+- All strict-mode requirements (including path-expansion conflicts when enabled)
 
 ## Versioning
 
 The spec uses semantic versioning (major.minor):
-- **Major version** (e.g., v{{ $spec.version }}): Breaking changes, incompatible with previous versions
-- **Minor version** (e.g., v1.5 → v1.6): Clarifications, additional requirements, or backward-compatible additions
+- **Major version** (e.g., v2 → v3): Breaking changes, incompatible with previous versions
+- **Minor version** (e.g., v3.1 → v3.2): Clarifications, additional requirements, or backward-compatible additions
 
 See [Appendix D: Document Changelog](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-d-document-changelog-informative) for detailed version history.
 

docs/reference/syntax-cheatsheet.md

@@ -1,6 +1,10 @@
+---
+description: JSON-to-TOON mappings at a glance for objects, arrays, quoting, key folding, and type conversions.
+---
+
 # Syntax Cheatsheet
 
-Quick reference for mapping JSON to TOON format. For rigorous, normative syntax rules and edge cases, see the [specification](/reference/spec).
+Quick reference for mapping JSON to TOON format. For rigorous, normative syntax rules and edge cases, see the [Specification](/reference/spec).
 
 ## Objects
 
@@ -78,7 +82,7 @@ items[2]{id,qty}:
 
 :::
 
-## Mixed / Non-Uniform Arrays
+## Mixed and Non-Uniform Arrays
 
 ::: code-group
 
@@ -97,6 +101,28 @@ items[3]:
 
 :::
 
+> [!NOTE]
+> When a list-item object has a tabular array as its first field, the tabular header appears on the hyphen line. Rows are indented two levels deeper than the hyphen, and other fields are indented one level deeper. This is the canonical encoding for this pattern.
+
+::: code-group
+
+```yaml [Multi-field object]
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+    status: active
+```
+
+```yaml [Single-field object]
+items[1]:
+  - users[2]{id,name}:
+      1,Ada
+      2,Bob
+```
+
+:::
+
 ## Arrays of Arrays
 
 ::: code-group
@@ -152,7 +178,7 @@ pairs[2]:
 ```
 
 ```yaml [Empty Array]
-items[0]:
+items: []
 ```
 
 :::
@@ -179,7 +205,7 @@ enabled: "true"
 
 These strings must be quoted because they look like numbers/booleans.
 
-### Strings with Active Delimiter
+### Strings Containing Delimiters
 
 ::: code-group
 
@@ -195,7 +221,7 @@ note: "hello, world"
 
 :::
 
-Strings containing the active delimiter (comma by default) must be quoted.
+Strings must be quoted when they contain the active delimiter (inside an array scope) or the document delimiter (object field values, comma by default).
 
 ### Strings with Leading/Trailing Spaces
 
@@ -237,8 +263,8 @@ Strings **must** be quoted if they:
 - Have leading or trailing whitespace
 - Equal `true`, `false`, or `null` (case-sensitive)
 - Look like numbers (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
-- Contain special characters: `:`, `"`, `\`, `[`, `]`, `{`, `}`, newline, tab, carriage return
-- Contain the active delimiter (comma by default, or tab/pipe if declared in header)
+- Contain special characters: `:`, `"`, `\`, `[`, `]`, `{`, `}`, or any control character (U+0000–U+001F, including newline/tab/CR)
+- Contain the relevant delimiter – the active delimiter inside an array scope, or the document delimiter (comma by default) for object field values
 - Equal `"-"` or start with `"-"` followed by any character
 
 Otherwise, strings can be unquoted. Unicode and emoji are safe:
@@ -250,7 +276,7 @@ note: This has inner spaces
 
 ## Escape Sequences
 
-Only five escape sequences are valid in quoted strings:
+Six escape sequences are valid in quoted strings:
 
 | Character | Escape |
 |-----------|--------|
@@ -259,8 +285,9 @@ Only five escape sequences are valid in quoted strings:
 | Newline | `\n` |
 | Carriage return | `\r` |
 | Tab | `\t` |
+| Any other U+0000–U+001F control character | `\uXXXX` |
 
-All other escapes (e.g., `\x`, `\u`) are invalid.
+Other escapes (e.g., `\x`, `\0`, `\b`) are invalid, and lone-surrogate `\uXXXX` values (U+D800–U+DFFF) are rejected.
 
 ## Array Headers
 
@@ -325,9 +352,16 @@ See [Format Overview – Key Folding](/guide/format-overview#key-folding-optiona
 
 | Input | Output |
 |-------|--------|
-| Finite number | Canonical decimal (no exponent, no trailing zeros) |
+| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal |
+| Finite number outside that range | Exponent form permitted |
 | `NaN`, `Infinity`, `-Infinity` | `null` |
 | `BigInt` (safe range) | Number |
 | `BigInt` (out of range) | Quoted decimal string |
 | `Date` | ISO string (quoted) |
+| `Set` | Array of normalized values |
+| `Map` | Object with `String(key)` keys |
 | `undefined`, `function`, `symbol` | `null` |
+
+::: info
+TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
+:::

docs/wrangler.toml

@@ -1,6 +1,9 @@
 name = "toon-docs"
 compatibility_date = "2025-10-01"
-routes = [ { pattern = "toonformat.dev", custom_domain = true } ]
+
+[[routes]]
+pattern = "toonformat.dev"
+custom_domain = true
 
 [assets]
 directory = "./.vitepress/dist/"

eslint.config.mjs

@@ -1,15 +0,0 @@
-// @ts-check
-import antfu from '@antfu/eslint-config'
-
-export default antfu({
-  rules: {
-    'no-cond-assign': 'off',
-  },
-}).append({
-  files: ['README.md', 'SPEC.md', '**/docs/**/*'],
-  rules: {
-    'import/no-duplicates': 'off',
-    'style/no-tabs': 'off',
-    'yaml/quotes': 'off',
-  },
-})

eslint.config.ts

@@ -0,0 +1,22 @@
+import type { ConfigNames, TypedFlatConfigItem } from '@antfu/eslint-config'
+import type { FlatConfigComposer } from 'eslint-flat-config-utils'
+import antfu from '@antfu/eslint-config'
+
+const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
+  rules: {
+    'no-cond-assign': 'off',
+  },
+}).append({
+  files: ['**/README.md', 'SPEC.md', '**/benchmarks/**/*', '**/docs/**/*'],
+  rules: {
+    'markdown/no-missing-link-fragments': 'off',
+    'markdown/fenced-code-language': 'off',
+    'markdown/heading-increment': 'off',
+    'import/no-duplicates': 'off',
+    'style/no-tabs': 'off',
+    'yaml/quotes': 'off',
+    'yaml/indent': 'off',
+  },
+})
+
+export default config

package.json

@@ -1,15 +1,15 @@
 {
   "name": "@toon-format/monorepo",
   "type": "module",
-  "version": "1.3.0",
+  "version": "2.3.1",
   "private": true,
-  "packageManager": "pnpm@10.21.0",
+  "packageManager": "pnpm@11.13.0",
   "scripts": {
-    "build": "pnpm -r --filter=./packages/** run build",
+    "build": "pnpm -r --filter='./packages/**' run build",
     "automd": "automd",
-    "docs:dev": "vitepress dev docs",
-    "docs:build": "vitepress build docs",
-    "docs:preview": "vitepress preview docs",
+    "docs:dev": "pnpm -C docs run dev",
+    "docs:build": "pnpm -C docs run build",
+    "docs:preview": "pnpm -C docs run preview",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "test": "pnpm -r test",
@@ -17,14 +17,16 @@
     "release": "bumpp -r"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^6.2.0",
-    "@types/node": "^24.10.0",
-    "automd": "^0.4.2",
-    "bumpp": "^10.3.1",
-    "eslint": "^9.39.1",
-    "tsdown": "^0.16.2",
-    "tsx": "^4.20.6",
-    "typescript": "^5.9.3",
-    "vitest": "^4.0.8"
+    "@antfu/eslint-config": "^9.1.0",
+    "@commitlint/types": "^21.2.0",
+    "@types/node": "^26.1.1",
+    "automd": "^0.4.3",
+    "bumpp": "^11.1.0",
+    "eslint": "^10.7.0",
+    "eslint-flat-config-utils": "^3.2.0",
+    "tsdown": "^0.22.8",
+    "typescript": "^6.0.3",
+    "vite": "^8.1.4",
+    "vitest": "^4.1.10"
   }
 }

packages/cli/README.md

@@ -2,7 +2,7 @@
 
 Command-line tool for converting JSON to TOON and back, with token analysis and streaming support.
 
-[TOON (Token-Oriented Object Notation)](https://toonformat.dev) is a compact, human-readable encoding of the JSON data model that minimizes tokens for LLM input. The CLI lets you test conversions, analyze token savings, and integrate TOON into shell pipelines with stdin/stdout support—no code required.
+[TOON (Token-Oriented Object Notation)](https://toonformat.dev) is a compact, human-readable encoding of the JSON data model that minimizes tokens for LLM input. The CLI lets you test conversions, analyze token savings, and integrate TOON into shell pipelines with stdin/stdout support.
 
 ## Installation
 
@@ -60,13 +60,14 @@ cat data.toon | toon --decode
 | `-o, --output <file>` | Output file path (prints to stdout if omitted) |
 | `-e, --encode` | Force encode mode (overrides auto-detection) |
 | `-d, --decode` | Force decode mode (overrides auto-detection) |
-| `--delimiter <char>` | Array delimiter: `,` (comma), `\t` (tab), `\|` (pipe) |
+| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
 | `--indent <number>` | Indentation size (default: `2`) |
 | `--stats` | Show token count estimates and savings (encode only) |
-| `--no-strict` | Disable strict validation when decoding |
-| `--key-folding <mode>` | Enable key folding: `off`, `safe` (default: `off`) |
-| `--flatten-depth <number>` | Maximum folded segment count when key folding is enabled (default: `Infinity`) |
-| `--expand-paths <mode>` | Enable path expansion: `off`, `safe` (default: `off`) |
+| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
+| `--keyFolding <mode>` | Enable key folding: `off`, `safe` (default: `off`) |
+| `--flattenDepth <number>` | Maximum folded segment count when key folding is enabled (default: `Infinity`) |
+| `--expandPaths <mode>` | Enable path expansion: `off`, `safe` (default: `off`) |
+| `--verbose` | Show full stack traces and cause chains for errors (default: `false`) |
 
 ## Advanced Examples
 
@@ -92,17 +93,34 @@ Example output:
 #### Tab-separated (often more token-efficient)
 
 ```bash
-toon data.json --delimiter "\t" -o output.toon
+toon data.json --delimiter $'\t' -o output.toon
 ```
 
+The `--delimiter` value must be the actual delimiter character. In bash/zsh, use `$'\t'` to pass a real tab; literal `"\t"` is rejected as an invalid delimiter.
+
 ### Lenient Decoding
 
-Skip validation for faster processing:
+Skip validation for faster, more forgiving decoding:
 
 ```bash
 toon data.toon --no-strict -o output.json
 ```
 
+With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
+
+### Decode Error Output
+
+When a TOON document fails to parse, the CLI renders the offending line with a caret pointing at the first non-whitespace character. Tabs are shown as `→` so the caret column reflects what the decoder actually saw:
+
+```
+ ERROR  Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode
+
+  2 | →b: 1
+      ^
+```
+
+The exit code is `1` on any error. Stack traces are suppressed by default. Pass `--verbose` to include the full stack and the underlying cause chain.
+
 ### Stdin Workflows
 
 ```bash
@@ -110,7 +128,7 @@ toon data.toon --no-strict -o output.json
 curl https://api.example.com/data | toon --stats
 
 # Process large dataset
-cat large-dataset.json | toon --delimiter "\t" > output.toon
+cat large-dataset.json | toon --delimiter $'\t' > output.toon
 
 # Chain with other tools
 jq '.results' data.json | toon > filtered.toon
@@ -136,9 +154,9 @@ cat million-records.toon | toon --decode > output.json
 - **Encode (JSON → TOON)**: Streams TOON lines to output without full string in memory
 - **Decode (TOON → JSON)**: Uses the same event-based streaming decoder as the `decodeStream` API in `@toon-format/toon`, streaming JSON tokens to output without full string in memory
 - Peak memory usage scales with data depth, not total size
-- When `--expand-paths safe` is enabled, decode falls back to non-streaming mode internally to apply deep-merge expansion before writing JSON
+- When `--expandPaths safe` is enabled, decode falls back to non-streaming mode internally to apply deep-merge expansion before writing JSON
 
-> [!NOTE]
+> [!TIP]
 > When using `--stats` with encode, the full output string is kept in memory for token counting. Omit `--stats` for maximum memory efficiency with very large datasets.
 
 ### Key Folding (Since v1.5)
@@ -149,7 +167,7 @@ Collapse nested wrapper chains to reduce tokens:
 
 ```bash
 # Encode with key folding
-toon input.json --key-folding safe -o output.toon
+toon input.json --keyFolding safe -o output.toon
 ```
 
 For data like:
@@ -179,24 +197,24 @@ data:
 
 ```bash
 # Fold maximum 2 levels deep
-toon input.json --key-folding safe --flatten-depth 2 -o output.toon
+toon input.json --keyFolding safe --flattenDepth 2 -o output.toon
 ```
 
 #### Path expansion on decode
 
 ```bash
 # Reconstruct nested structure from folded keys
-toon data.toon --expand-paths safe -o output.json
+toon data.toon --expandPaths safe -o output.json
 ```
 
 #### Round-trip workflow
 
 ```bash
 # Encode with folding
-toon input.json --key-folding safe -o compressed.toon
+toon input.json --keyFolding safe -o compressed.toon
 
 # Decode with expansion (restores original structure)
-toon compressed.toon --expand-paths safe -o output.json
+toon compressed.toon --expandPaths safe -o output.json
 
 # Verify round-trip
 diff input.json output.json
@@ -206,7 +224,7 @@ diff input.json output.json
 
 ```bash
 # Key folding + tab delimiter + stats
-toon data.json --key-folding safe --delimiter "\t" --stats -o output.toon
+toon data.json --keyFolding safe --delimiter $'\t' --stats -o output.toon
 ```
 
 ## Why Use the CLI?
@@ -220,9 +238,9 @@ toon data.json --key-folding safe --delimiter "\t" --stats -o output.toon
 
 ## Related
 
-- [@toon-format/toon](https://www.npmjs.com/package/@toon-format/toon) - JavaScript/TypeScript library
-- [Full specification](https://github.com/toon-format/spec) - Complete format documentation
-- [Website](https://toonformat.dev) - Interactive examples and guides
+- [@toon-format/toon](https://www.npmjs.com/package/@toon-format/toon) – JavaScript/TypeScript library
+- [Full specification](https://github.com/toon-format/spec) – Complete format documentation
+- [Website](https://toonformat.dev) – Interactive examples and guides
 
 ## License
 

packages/cli/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@toon-format/cli",
   "type": "module",
-  "version": "1.3.0",
-  "packageManager": "pnpm@10.21.0",
+  "version": "2.3.1",
+  "packageManager": "pnpm@11.13.0",
   "description": "CLI for JSON ↔ TOON conversion using @toon-format/toon",
   "author": "Johann Schopplich <hello@johannschopplich.com>",
   "license": "MIT",
@@ -30,13 +30,13 @@
     "dist"
   ],
   "scripts": {
-    "dev": "tsx ./src/cli-entry.ts --help",
+    "dev": "node ./src/cli-entry.ts --help",
     "build": "tsdown",
     "test": "vitest"
   },
   "dependencies": {
-    "citty": "^0.1.6",
+    "citty": "^0.2.2",
     "consola": "^3.4.2",
-    "tokenx": "^1.2.1"
+    "tokenx": "^1.3.0"
   }
 }

packages/cli/src/cli-entry.ts

@@ -1,4 +1,4 @@
 import { runMain } from 'citty'
-import { mainCommand } from '.'
+import { mainCommand } from './index.ts'
 
 runMain(mainCommand)

packages/cli/src/conversion.ts

@@ -1,15 +1,15 @@
 import type { FileHandle } from 'node:fs/promises'
-import type { DecodeOptions, DecodeStreamOptions, EncodeOptions } from '../../toon/src'
-import type { InputSource } from './types'
+import type { DecodeOptions, DecodeStreamOptions, EncodeOptions } from '../../toon/src/index.ts'
+import type { InputSource } from './types.ts'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
 import process from 'node:process'
 import { consola } from 'consola'
 import { estimateTokenCount } from 'tokenx'
-import { decode, decodeStream, encode, encodeLines } from '../../toon/src'
-import { jsonStreamFromEvents } from './json-from-events'
-import { jsonStringifyLines } from './json-stringify-stream'
-import { formatInputLabel, readInput, readLinesFromSource } from './utils'
+import { decode, decodeStream, encode, encodeLines } from '../../toon/src/index.ts'
+import { jsonStreamFromEvents } from './json-from-events.ts'
+import { jsonStringifyLines } from './json-stringify-stream.ts'
+import { formatInputLabel, readInput, readLinesFromSource } from './utils.ts'
 
 export async function encodeToToon(config: {
   input: InputSource
@@ -85,38 +85,27 @@ export async function decodeToJson(config: {
   if (config.expandPaths === 'safe') {
     const toonContent = await readInput(config.input)
 
-    let data: unknown
-    try {
-      const decodeOptions: DecodeOptions = {
-        indent: config.indent,
-        strict: config.strict,
-        expandPaths: config.expandPaths,
-      }
-      data = decode(toonContent, decodeOptions)
-    }
-    catch (error) {
-      throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
+    const decodeOptions: DecodeOptions = {
+      indent: config.indent,
+      strict: config.strict,
+      expandPaths: config.expandPaths,
     }
+    const data = decode(toonContent, decodeOptions)
 
     await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
   }
   else {
-    try {
-      const lineSource = readLinesFromSource(config.input)
+    const lineSource = readLinesFromSource(config.input)
 
-      const decodeStreamOptions: DecodeStreamOptions = {
-        indent: config.indent,
-        strict: config.strict,
-      }
-
-      const events = decodeStream(lineSource, decodeStreamOptions)
-      const jsonChunks = jsonStreamFromEvents(events, config.indent)
-
-      await writeStreamingJson(jsonChunks, config.output)
-    }
-    catch (error) {
-      throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
+    const decodeStreamOptions: DecodeStreamOptions = {
+      indent: config.indent,
+      strict: config.strict,
     }
+
+    const events = decodeStream(lineSource, decodeStreamOptions)
+    const jsonChunks = jsonStreamFromEvents(events, config.indent)
+
+    await writeStreamingJson(jsonChunks, config.output)
   }
 
   if (config.output) {

packages/cli/src/format-error.ts

@@ -0,0 +1,70 @@
+import { ToonDecodeError } from '../../toon/src/index.ts'
+
+export interface FormatErrorOptions {
+  isVerbose: boolean
+}
+
+// #region Public API
+
+export function formatError(error: unknown, options: FormatErrorOptions): string {
+  const sections: string[] = []
+
+  if (error instanceof ToonDecodeError && error.line !== undefined) {
+    sections.push(formatDecodeError(error))
+  }
+  else {
+    sections.push(String(error))
+  }
+
+  if (options.isVerbose) {
+    const causeChain = formatCauseChain(error)
+    if (causeChain) {
+      sections.push(causeChain)
+    }
+
+    if (error instanceof Error && error.stack) {
+      sections.push(error.stack)
+    }
+  }
+
+  return sections.join('\n\n')
+}
+
+// #endregion
+
+// #region Internal renderers
+
+function formatDecodeError(error: ToonDecodeError): string {
+  const linePrefix = `Line ${error.line}: `
+  const messageWithoutPrefix = error.message.startsWith(linePrefix)
+    ? error.message.slice(linePrefix.length)
+    : error.message
+
+  const header = `Failed to decode TOON at line ${error.line}: ${messageWithoutPrefix}`
+
+  if (error.source === undefined) {
+    return header
+  }
+
+  const visibleSource = error.source.replace(/\t/g, '→')
+  const firstNonWhitespaceIndex = visibleSource.search(/\S/)
+  const gutter = `  ${error.line} | `
+  const caretIndent = ' '.repeat(gutter.length + Math.max(firstNonWhitespaceIndex, 0))
+
+  return `${header}\n\n${gutter}${visibleSource}\n${caretIndent}^`
+}
+
+function formatCauseChain(error: unknown): string {
+  const causeLines: string[] = []
+  let current: unknown = error instanceof Error ? error.cause : undefined
+
+  while (current instanceof Error) {
+    const name = current.name || 'Error'
+    causeLines.push(`Caused by: ${name}: ${current.message}`)
+    current = current.cause
+  }
+
+  return causeLines.join('\n')
+}
+
+// #endregion

packages/cli/src/index.ts

@@ -1,132 +1,87 @@
-import type { CommandDef } from 'citty'
-import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
-import type { InputSource } from './types'
+import type { ArgsDef, CommandDef } from 'citty'
+import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src/index.ts'
+import type { InputSource } from './types.ts'
 import * as path from 'node:path'
 import process from 'node:process'
 import { defineCommand } from 'citty'
 import { consola } from 'consola'
-import { DEFAULT_DELIMITER, DELIMITERS } from '../../toon/src'
-import { name, version } from '../package.json' with { type: 'json' }
-import { decodeToJson, encodeToToon } from './conversion'
-import { detectMode } from './utils'
+import { DEFAULT_DELIMITER, DELIMITERS } from '../../toon/src/index.ts'
+import pkg from '../package.json' with { type: 'json' }
+import { decodeToJson, encodeToToon } from './conversion.ts'
+import { formatError } from './format-error.ts'
+import { detectMode } from './utils.ts'
 
-export const mainCommand: CommandDef<{
+const { name, version } = pkg
+
+const args: ArgsDef = {
   input: {
-    type: 'positional'
-    description: string
-    required: false
-  }
+    type: 'positional',
+    description: 'Input file path (omit or use "-" to read from stdin)',
+    required: false,
+  },
   output: {
-    type: 'string'
-    description: string
-    alias: string
-  }
+    type: 'string',
+    description: 'Output file path',
+    alias: 'o',
+  },
   encode: {
-    type: 'boolean'
-    description: string
-    alias: string
-  }
+    type: 'boolean',
+    description: 'Encode JSON to TOON (auto-detected by default)',
+    alias: 'e',
+  },
   decode: {
-    type: 'boolean'
-    description: string
-    alias: string
-  }
+    type: 'boolean',
+    description: 'Decode TOON to JSON (auto-detected by default)',
+    alias: 'd',
+  },
   delimiter: {
-    type: 'string'
-    description: string
-    default: string
-  }
+    type: 'string',
+    description: 'Delimiter for arrays: comma (,), tab (\\t), or pipe (|)',
+    default: ',',
+  },
   indent: {
-    type: 'string'
-    description: string
-    default: string
-  }
+    type: 'string',
+    description: 'Indentation size',
+    default: '2',
+  },
   strict: {
-    type: 'boolean'
-    description: string
-    default: true
-  }
+    type: 'boolean',
+    description: 'Strict decode validation (disable with --no-strict)',
+    default: true,
+  },
   keyFolding: {
-    type: 'string'
-    description: string
-    default: string
-  }
+    type: 'string',
+    description: 'Enable key folding: off, safe (default: off)',
+    default: 'off',
+  },
   flattenDepth: {
-    type: 'string'
-    description: string
-  }
+    type: 'string',
+    description: 'Maximum folded segment count when key folding is enabled (default: Infinity)',
+  },
   expandPaths: {
-    type: 'string'
-    description: string
-    default: string
-  }
+    type: 'string',
+    description: 'Enable path expansion: off, safe (default: off)',
+    default: 'off',
+  },
   stats: {
-    type: 'boolean'
-    description: string
-    default: false
-  }
-}> = defineCommand({
+    type: 'boolean',
+    description: 'Show token statistics',
+    default: false,
+  },
+  verbose: {
+    type: 'boolean',
+    description: 'Show full stack traces and cause chains for errors',
+    default: false,
+  },
+} as const
+
+export const mainCommand: CommandDef<ArgsDef> = defineCommand({
   meta: {
     name,
-    description: 'TOON CLI — Convert between JSON and TOON formats',
+    description: 'TOON CLI – Convert between JSON and TOON formats',
     version,
   },
-  args: {
-    input: {
-      type: 'positional',
-      description: 'Input file path (omit or use "-" to read from stdin)',
-      required: false,
-    },
-    output: {
-      type: 'string',
-      description: 'Output file path',
-      alias: 'o',
-    },
-    encode: {
-      type: 'boolean',
-      description: 'Encode JSON to TOON (auto-detected by default)',
-      alias: 'e',
-    },
-    decode: {
-      type: 'boolean',
-      description: 'Decode TOON to JSON (auto-detected by default)',
-      alias: 'd',
-    },
-    delimiter: {
-      type: 'string',
-      description: 'Delimiter for arrays: comma (,), tab (\\t), or pipe (|)',
-      default: ',',
-    },
-    indent: {
-      type: 'string',
-      description: 'Indentation size',
-      default: '2',
-    },
-    strict: {
-      type: 'boolean',
-      description: 'Enable strict mode for decoding',
-      default: true,
-    },
-    keyFolding: {
-      type: 'string',
-      description: 'Enable key folding: off, safe (default: off)',
-      default: 'off',
-    },
-    flattenDepth: {
-      type: 'string',
-      description: 'Maximum folded segment count when key folding is enabled (default: Infinity)',
-    },
-    expandPaths: {
-      type: 'string',
-      description: 'Enable path expansion: off, safe (default: off)',
-      default: 'off',
-    },
-    stats: {
-      type: 'boolean',
-      description: 'Show token statistics',
-      default: false,
-    },
-  },
+  args,
   async run({ args }) {
     const input = args.input
 
@@ -193,7 +148,7 @@ export const mainCommand: CommandDef<{
       }
     }
     catch (error) {
-      consola.error(error)
+      consola.error(formatError(error, { isVerbose: args.verbose === true }))
       process.exit(1)
     }
   },

packages/cli/src/json-from-events.ts

@@ -1,4 +1,4 @@
-import type { JsonStreamEvent } from '../../toon/src/types'
+import type { JsonStreamEvent } from '../../toon/src/types.ts'
 
 /**
  * Context for tracking JSON structure state during event streaming.

packages/cli/src/utils.ts

@@ -1,4 +1,4 @@
-import type { InputSource } from './types'
+import type { InputSource } from './types.ts'
 import { createReadStream } from 'node:fs'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'

packages/cli/test/format-error.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it } from 'vitest'
+import { ToonDecodeError } from '../../toon/src/index'
+import { formatError } from '../src/format-error'
+
+describe('formatError', () => {
+  it('renders a decode error with line and source as a header, source line, and caret', () => {
+    const error = new ToonDecodeError(
+      'Tabs are not allowed in indentation in strict mode',
+      { line: 2, source: '\tb: 1' },
+    )
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe(
+      'Failed to decode TOON at line 2: Tabs are not allowed in indentation in strict mode\n'
+      + '\n'
+      + '  2 | →b: 1\n'
+      + '      ^',
+    )
+  })
+
+  it('renders a decode error without source as a header only', () => {
+    const error = new ToonDecodeError('Something went wrong', { line: 5 })
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe('Failed to decode TOON at line 5: Something went wrong')
+  })
+
+  it('appends the cause chain under verbose mode', () => {
+    const cause = new SyntaxError('Unterminated string: missing closing quote')
+    const error = new ToonDecodeError(
+      'Unterminated string: missing closing quote',
+      { line: 2, source: 'greeting: "hello', cause },
+    )
+
+    const output = formatError(error, { isVerbose: true })
+
+    expect(output).toContain('Failed to decode TOON at line 2:')
+    expect(output).toContain('  2 | greeting: "hello')
+    expect(output).toContain('Caused by: SyntaxError: Unterminated string: missing closing quote')
+  })
+
+  it('appends the stack trace under verbose mode and omits it otherwise', () => {
+    const error = new ToonDecodeError('Boom', { line: 1, source: 'x' })
+    error.stack = 'ToonDecodeError: Line 1: Boom\n    at fakeFrame (file.ts:1:1)'
+
+    const verbose = formatError(error, { isVerbose: true })
+    const quiet = formatError(error, { isVerbose: false })
+
+    expect(verbose).toContain('at fakeFrame (file.ts:1:1)')
+    expect(quiet).not.toContain('at fakeFrame')
+  })
+
+  it('renders a generic Error as its message only when not verbose', () => {
+    const error = new Error('something went wrong')
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe('Error: something went wrong')
+  })
+
+  it('places the caret under the first non-whitespace character of the source line', () => {
+    const error = new ToonDecodeError(
+      'Indentation must be exact multiple of 2, but found 3 spaces',
+      { line: 2, source: '   b: 1' },
+    )
+
+    const output = formatError(error, { isVerbose: false })
+
+    expect(output).toBe(
+      'Failed to decode TOON at line 2: Indentation must be exact multiple of 2, but found 3 spaces\n'
+      + '\n'
+      + '  2 |    b: 1\n'
+      + '         ^',
+    )
+  })
+})

packages/cli/test/index.test.ts

@@ -8,6 +8,8 @@ import { createCliTestContext, mockStdin, runCli } from './utils'
 describe('toon CLI', () => {
   beforeEach(() => {
     vi.spyOn(process, 'exit').mockImplementation(() => 0 as never)
+    vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    vi.spyOn(process.stdout, 'write').mockImplementation(() => true)
   })
 
   afterEach(() => {
@@ -16,11 +18,11 @@ describe('toon CLI', () => {
 
   describe('version', () => {
     it('prints the version when using --version', async () => {
-      const consolaLog = vi.spyOn(consola, 'log').mockImplementation(() => undefined)
+      const consoleLog = vi.mocked(console.log)
 
       await runCli({ rawArgs: ['--version'] })
 
-      expect(consolaLog).toHaveBeenCalledWith(version)
+      expect(consoleLog).toHaveBeenCalledWith(version)
     })
   })
 
@@ -228,6 +230,48 @@ describe('toon CLI', () => {
         cleanup()
       }
     })
+
+    it('renders a TOON decode error with line context, source, and caret', async () => {
+      const cleanup = mockStdin('a:\n\tb: 1\n')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+        const errorCall = consolaError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [rendered] = errorCall!
+        expect(rendered).toEqual(expect.stringContaining('Failed to decode TOON at line 2:'))
+        expect(rendered).toEqual(expect.stringContaining('  2 | →b: 1'))
+        expect(rendered).toEqual(expect.stringContaining('      ^'))
+        expect(rendered).not.toEqual(expect.stringMatching(/^\s+at \S+/m))
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('includes the stack trace when --verbose is passed', async () => {
+      const cleanup = mockStdin('a:\n\tb: 1\n')
+
+      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+
+      try {
+        await runCli({ rawArgs: ['--decode', '--verbose'] })
+
+        const errorCall = consolaError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [rendered] = errorCall!
+        expect(rendered).toEqual(expect.stringContaining('Failed to decode TOON at line 2:'))
+        expect(rendered).toEqual(expect.stringMatching(/at \S+/))
+      }
+      finally {
+        cleanup()
+      }
+    })
   })
 
   describe('stdin with options', () => {
@@ -303,6 +347,184 @@ describe('toon CLI', () => {
     })
   })
 
+  describe('encode options', () => {
+    it('encodes with --keyFolding safe', async () => {
+      const data = {
+        data: {
+          metadata: {
+            items: ['a', 'b'],
+          },
+        },
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'safe', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        const expected = encode(data, { keyFolding: 'safe' })
+
+        expect(output).toBe(expected)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('encodes with --flattenDepth', async () => {
+      const data = {
+        level1: {
+          level2: {
+            level3: {
+              value: 'deep',
+            },
+          },
+        },
+      }
+
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify(data),
+      })
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'safe', '--flattenDepth', '2', '--output', 'output.toon'])
+
+        const output = await context.read('output.toon')
+        const expected = encode(data, { keyFolding: 'safe', flattenDepth: 2 })
+
+        expect(output).toBe(expected)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+  })
+
+  describe('decode options', () => {
+    it('decodes with --expandPaths safe', async () => {
+      const data = {
+        data: {
+          metadata: {
+            items: ['a', 'b'],
+          },
+        },
+      }
+      const toonInput = encode(data, { keyFolding: 'safe' })
+
+      const context = await createCliTestContext({
+        'input.toon': toonInput,
+      })
+
+      try {
+        await context.run(['input.toon', '--decode', '--expandPaths', 'safe', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('decodes with --indent for JSON formatting', async () => {
+      const data = {
+        a: 1,
+        b: [2, 3],
+        c: { nested: true },
+      }
+      const toonInput = encode(data, { indent: 4 })
+
+      const context = await createCliTestContext({
+        'input.toon': toonInput,
+      })
+
+      try {
+        await context.run(['input.toon', '--decode', '--indent', '4', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+        expect(output).toContain('    ') // Should have 4-space indentation
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('decodes root primitive number', async () => {
+      const toonInput = '42'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe('42\n')
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes root primitive string', async () => {
+      const toonInput = '"Hello World"'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        const jsonOutput = fullOutput.endsWith('\n') ? fullOutput.slice(0, -1) : fullOutput
+        expect(JSON.parse(jsonOutput)).toBe('Hello World')
+      }
+      finally {
+        cleanup()
+      }
+    })
+
+    it('decodes root primitive boolean', async () => {
+      const toonInput = 'true'
+
+      const cleanup = mockStdin(toonInput)
+
+      const writeChunks: string[] = []
+      vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+        writeChunks.push(String(chunk))
+        return true
+      })
+
+      try {
+        await runCli({ rawArgs: ['--decode'] })
+
+        const fullOutput = writeChunks.join('')
+        expect(fullOutput).toBe('true\n')
+      }
+      finally {
+        cleanup()
+      }
+    })
+  })
+
   describe('streaming output', () => {
     it('streams large JSON to TOON file with identical output', async () => {
       const data = {
@@ -337,6 +559,40 @@ describe('toon CLI', () => {
       }
     })
 
+    it('streams large TOON to JSON file with streaming decode', async () => {
+      const data = {
+        records: Array.from({ length: 1000 }, (_, i) => ({
+          id: i,
+          title: `Record ${i}`,
+          score: Math.random() * 100,
+        })),
+      }
+
+      const toonContent = encode(data, {
+        delimiter: DEFAULT_DELIMITER,
+        indent: 2,
+      })
+
+      const context = await createCliTestContext({
+        'large-input.toon': toonContent,
+      })
+
+      const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined)
+
+      try {
+        await context.run(['large-input.toon', '--decode', '--output', 'output.json'])
+
+        const output = await context.read('output.json')
+        const result = JSON.parse(output)
+
+        expect(result).toEqual(data)
+        expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Decoded .* → .*/))
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
     it('streams to stdout using process.stdout.write', async () => {
       const data = {
         users: [
@@ -441,7 +697,7 @@ describe('toon CLI', () => {
         'input.json': JSON.stringify({ value: 1 }),
       })
 
-      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
       const exitSpy = vi.mocked(process.exit)
 
       try {
@@ -449,7 +705,7 @@ describe('toon CLI', () => {
 
         expect(exitSpy).toHaveBeenCalledWith(1)
 
-        const errorCall = consolaError.mock.calls.at(0)
+        const errorCall = consoleError.mock.calls.at(0)
         expect(errorCall).toBeDefined()
         const [error] = errorCall!
         expect(error).toBeInstanceOf(Error)
@@ -465,7 +721,7 @@ describe('toon CLI', () => {
         'input.json': JSON.stringify({ value: 1 }),
       })
 
-      const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined)
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
       const exitSpy = vi.mocked(process.exit)
 
       try {
@@ -473,7 +729,7 @@ describe('toon CLI', () => {
 
         expect(exitSpy).toHaveBeenCalledWith(1)
 
-        const errorCall = consolaError.mock.calls.at(0)
+        const errorCall = consoleError.mock.calls.at(0)
         expect(errorCall).toBeDefined()
         const [error] = errorCall!
         expect(error).toBeInstanceOf(Error)
@@ -500,5 +756,77 @@ describe('toon CLI', () => {
         await context.cleanup()
       }
     })
+
+    it('rejects invalid --keyFolding value', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--keyFolding', 'invalid'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid keyFolding value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid --expandPaths value', async () => {
+      const context = await createCliTestContext({
+        'input.toon': 'key: value',
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.toon', '--decode', '--expandPaths', 'invalid'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid expandPaths value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
+
+    it('rejects invalid --flattenDepth value', async () => {
+      const context = await createCliTestContext({
+        'input.json': JSON.stringify({ value: 1 }),
+      })
+
+      const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+      const exitSpy = vi.mocked(process.exit)
+
+      try {
+        await context.run(['input.json', '--flattenDepth', '-1'])
+
+        expect(exitSpy).toHaveBeenCalledWith(1)
+
+        const errorCall = consoleError.mock.calls.at(0)
+        expect(errorCall).toBeDefined()
+        const [error] = errorCall!
+        expect(error).toBeInstanceOf(Error)
+        expect(error.message).toContain('Invalid flattenDepth value')
+      }
+      finally {
+        await context.cleanup()
+      }
+    })
   })
 })

packages/cli/test/json-from-events.test.ts

@@ -0,0 +1,423 @@
+import type { JsonStreamEvent } from '../../toon/src/types'
+import { describe, expect, it } from 'vitest'
+import { jsonStreamFromEvents } from '../src/json-from-events'
+
+describe('jsonStreamFromEvents', () => {
+  describe('primitives', () => {
+    it('converts null event', async () => {
+      const events = [
+        { type: 'primitive' as const, value: null },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(null))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(null, null, 2))
+    })
+
+    it('converts boolean events', async () => {
+      const eventsTrue = [{ type: 'primitive' as const, value: true }]
+      const eventsFalse = [{ type: 'primitive' as const, value: false }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 0))).toBe(JSON.stringify(true))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsFalse), 0))).toBe(JSON.stringify(false))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 2))).toBe(JSON.stringify(true, null, 2))
+    })
+
+    it('converts number events', async () => {
+      const events0 = [{ type: 'primitive' as const, value: 0 }]
+      const events42 = [{ type: 'primitive' as const, value: 42 }]
+      const eventsNeg = [{ type: 'primitive' as const, value: -17 }]
+      const eventsFloat = [{ type: 'primitive' as const, value: 3.14159 }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(events0), 0))).toBe(JSON.stringify(0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events42), 0))).toBe(JSON.stringify(42))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsNeg), 0))).toBe(JSON.stringify(-17))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsFloat), 0))).toBe(JSON.stringify(3.14159))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events42), 2))).toBe(JSON.stringify(42, null, 2))
+    })
+
+    it('converts string events', async () => {
+      const eventsEmpty = [{ type: 'primitive' as const, value: '' }]
+      const eventsHello = [{ type: 'primitive' as const, value: 'hello' }]
+      const eventsQuotes = [{ type: 'primitive' as const, value: 'with "quotes"' }]
+
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsEmpty), 0))).toBe(JSON.stringify(''))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsHello), 0))).toBe(JSON.stringify('hello'))
+      expect(await join(jsonStreamFromEvents(asyncEvents(eventsQuotes), 0))).toBe(JSON.stringify('with "quotes"'))
+    })
+  })
+
+  describe('empty containers', () => {
+    it('converts empty array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 0 },
+        { type: 'endArray' as const },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify([], null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify([], null, 2))
+    })
+
+    it('converts empty object events', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'endObject' as const },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify({}, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify({}, null, 2))
+    })
+  })
+
+  describe('arrays', () => {
+    it('converts simple array events with compact formatting', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 2, 3]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('converts simple array events with pretty formatting', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 2, 3]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts mixed-type array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 5 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 'two' },
+        { type: 'primitive' as const, value: true },
+        { type: 'primitive' as const, value: null },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'key' },
+        { type: 'primitive' as const, value: 'value' },
+        { type: 'endObject' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [1, 'two', true, null, { key: 'value' }]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts nested array events', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'endArray' as const },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'primitive' as const, value: 4 },
+        { type: 'endArray' as const },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 5 },
+        { type: 'primitive' as const, value: 6 },
+        { type: 'endArray' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [[1, 2], [3, 4], [5, 6]]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('objects', () => {
+    it('converts simple object events with compact formatting', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'a' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'b' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'c' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endObject' as const },
+      ]
+      const value = { a: 1, b: 2, c: 3 }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('converts simple object events with pretty formatting', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'a' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'b' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'c' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endObject' as const },
+      ]
+      const value = { a: 1, b: 2, c: 3 }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts object events with mixed value types', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'num' },
+        { type: 'primitive' as const, value: 42 },
+        { type: 'key' as const, key: 'str' },
+        { type: 'primitive' as const, value: 'hello' },
+        { type: 'key' as const, key: 'bool' },
+        { type: 'primitive' as const, value: true },
+        { type: 'key' as const, key: 'nil' },
+        { type: 'primitive' as const, value: null },
+        { type: 'key' as const, key: 'arr' },
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'endArray' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        num: 42,
+        str: 'hello',
+        bool: true,
+        nil: null,
+        arr: [1, 2, 3],
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts nested object events', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level1' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level2' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'level3' },
+        { type: 'primitive' as const, value: 'deep' },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        level1: {
+          level2: {
+            level3: 'deep',
+          },
+        },
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles special characters in keys', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'normal-key' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'key with spaces' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'key:with:colons' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'key' as const, key: 'key"with"quotes' },
+        { type: 'primitive' as const, value: 4 },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        'normal-key': 1,
+        'key with spaces': 2,
+        'key:with:colons': 3,
+        'key"with"quotes': 4,
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('complex nested structures', () => {
+    it('converts object containing arrays', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        { type: 'key' as const, key: 'scores' },
+        { type: 'startArray' as const, length: 3 },
+        { type: 'primitive' as const, value: 95 },
+        { type: 'primitive' as const, value: 87 },
+        { type: 'primitive' as const, value: 92 },
+        { type: 'endArray' as const },
+        { type: 'key' as const, key: 'metadata' },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'tags' },
+        { type: 'startArray' as const, length: 2 },
+        { type: 'primitive' as const, value: 'math' },
+        { type: 'primitive' as const, value: 'science' },
+        { type: 'endArray' as const },
+        { type: 'endObject' as const },
+        { type: 'endObject' as const },
+      ]
+      const value = {
+        name: 'Alice',
+        scores: [95, 87, 92],
+        metadata: {
+          tags: ['math', 'science'],
+        },
+      }
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('converts array of objects', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 3 },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 1 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        { type: 'endObject' as const },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 2 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Bob' },
+        { type: 'endObject' as const },
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'id' },
+        { type: 'primitive' as const, value: 3 },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Charlie' },
+        { type: 'endObject' as const },
+        { type: 'endArray' as const },
+      ]
+      const value = [
+        { id: 1, name: 'Alice' },
+        { id: 2, name: 'Bob' },
+        { id: 3, name: 'Charlie' },
+      ]
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+  })
+
+  describe('indentation levels', () => {
+    const events = [
+      { type: 'startObject' as const },
+      { type: 'key' as const, key: 'a' },
+      { type: 'startArray' as const, length: 2 },
+      { type: 'primitive' as const, value: 1 },
+      { type: 'primitive' as const, value: 2 },
+      { type: 'endArray' as const },
+      { type: 'key' as const, key: 'b' },
+      { type: 'startObject' as const },
+      { type: 'key' as const, key: 'c' },
+      { type: 'primitive' as const, value: 3 },
+      { type: 'endObject' as const },
+      { type: 'endObject' as const },
+    ]
+    const value = { a: [1, 2], b: { c: 3 } }
+
+    it('handles indent=0 (compact)', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0))
+    })
+
+    it('handles indent=2', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2))
+    })
+
+    it('handles indent=4', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 4))).toBe(JSON.stringify(value, null, 4))
+    })
+
+    it('handles indent=8', async () => {
+      expect(await join(jsonStreamFromEvents(asyncEvents(events), 8))).toBe(JSON.stringify(value, null, 8))
+    })
+  })
+
+  describe('error handling', () => {
+    it('throws on mismatched endObject event', async () => {
+      const events = [
+        { type: 'startArray' as const, length: 0 },
+        { type: 'endObject' as const }, // Wrong closing event
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Mismatched endObject event')
+    })
+
+    it('throws on mismatched endArray event', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'endArray' as const }, // Wrong closing event
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Mismatched endArray event')
+    })
+
+    it('throws on key event outside object context', async () => {
+      const events = [
+        { type: 'key' as const, key: 'invalid' },
+        { type: 'primitive' as const, value: 1 },
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Key event outside of object context')
+    })
+
+    it('throws on primitive in object without preceding key', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'primitive' as const, value: 'invalid' }, // No key before primitive
+        { type: 'endObject' as const },
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Primitive event in object without preceding key')
+    })
+
+    it('throws on incomplete event stream', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'name' },
+        { type: 'primitive' as const, value: 'Alice' },
+        // Missing `endObject`
+      ]
+
+      await expect(async () => {
+        await join(jsonStreamFromEvents(asyncEvents(events), 0))
+      }).rejects.toThrow('Incomplete event stream: unclosed objects or arrays')
+    })
+  })
+})
+
+/**
+ * Converts array of events to async iterable.
+ */
+async function* asyncEvents(events: JsonStreamEvent[]): AsyncIterable<JsonStreamEvent> {
+  for (const event of events) {
+    await Promise.resolve()
+    yield event
+  }
+}
+
+/**
+ * Joins chunks from an async iterable into a single string.
+ */
+async function join(iter: AsyncIterable<string>): Promise<string> {
+  const chunks: string[] = []
+  for await (const chunk of iter) {
+    chunks.push(chunk)
+  }
+  return chunks.join('')
+}

packages/cli/tsdown.config.ts

@@ -1,7 +1,7 @@
-import type { UserConfig, UserConfigFn } from 'tsdown/config'
+import type { UserConfig } from 'tsdown/config'
 import { defineConfig } from 'tsdown/config'
 
-const config: UserConfig | UserConfigFn = defineConfig({
+const config: UserConfig = defineConfig({
   entry: {
     index: 'src/cli-entry.ts',
   },

packages/toon/README.md

@@ -0,0 +1,923 @@
+![TOON logo with step‑by‑step guide](./.github/og.png)
+
+# Token-Oriented Object Notation (TOON)
+
+[![CI](https://github.com/toon-format/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon/actions)
+[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
+[![SPEC v3.3](https://img.shields.io/badge/spec-v3.3-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
+[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
+[![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE)
+
+**Token-Oriented Object Notation** is a compact, human-readable encoding of the JSON data model that minimizes tokens and makes structure easy for models to follow. It's intended for *LLM input* as a drop-in, lossless representation of your existing JSON.
+
+TOON combines YAML's indentation-based structure for nested objects with a CSV-style tabular layout for uniform arrays. TOON's sweet spot is uniform arrays of objects (multiple fields per row, same structure across items), achieving CSV-like compactness while adding explicit structure that helps LLMs parse and validate data reliably. For deeply nested or non-uniform data, JSON may be more efficient.
+
+The similarity to CSV is intentional: CSV is simple and ubiquitous, and TOON aims to keep that familiarity while remaining a lossless, drop-in representation of JSON for Large Language Models.
+
+Think of it as a translation layer: use JSON programmatically, and encode it as TOON for LLM input.
+
+> [!TIP]
+> The TOON format is stable, but also an idea in progress. Nothing's set in stone – help shape where it goes by contributing to the [spec](https://github.com/toon-format/spec) or sharing feedback.
+
+## Table of Contents
+
+- [Why TOON?](#why-toon)
+- [Key Features](#key-features)
+- [When Not to Use TOON](#when-not-to-use-toon)
+- [Benchmarks](#benchmarks)
+- [Installation & Quick Start](#installation--quick-start)
+- [Playgrounds](#playgrounds)
+- [Editor Support](#editor-support)
+- [CLI](#cli)
+- [Format Overview](#format-overview)
+- [Using TOON with LLMs](#using-toon-with-llms)
+- [Documentation](#documentation)
+- [Other Implementations](#other-implementations)
+- [📋 Full Specification](https://github.com/toon-format/spec/blob/main/SPEC.md)
+
+## Why TOON?
+
+AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – and standard JSON is verbose and token-expensive:
+
+```json
+{
+  "context": {
+    "task": "Our favorite hikes together",
+    "location": "Boulder",
+    "season": "spring_2025"
+  },
+  "friends": ["ana", "luis", "sam"],
+  "hikes": [
+    {
+      "id": 1,
+      "name": "Blue Lake Trail",
+      "distanceKm": 7.5,
+      "elevationGain": 320,
+      "companion": "ana",
+      "wasSunny": true
+    },
+    {
+      "id": 2,
+      "name": "Ridge Overlook",
+      "distanceKm": 9.2,
+      "elevationGain": 540,
+      "companion": "luis",
+      "wasSunny": false
+    },
+    {
+      "id": 3,
+      "name": "Wildflower Loop",
+      "distanceKm": 5.1,
+      "elevationGain": 180,
+      "companion": "sam",
+      "wasSunny": true
+    }
+  ]
+}
+```
+
+<details>
+<summary>YAML already conveys the same information with <strong>fewer tokens</strong>.</summary>
+
+```yaml
+context:
+  task: Our favorite hikes together
+  location: Boulder
+  season: spring_2025
+friends:
+  - ana
+  - luis
+  - sam
+hikes:
+  - id: 1
+    name: Blue Lake Trail
+    distanceKm: 7.5
+    elevationGain: 320
+    companion: ana
+    wasSunny: true
+  - id: 2
+    name: Ridge Overlook
+    distanceKm: 9.2
+    elevationGain: 540
+    companion: luis
+    wasSunny: false
+  - id: 3
+    name: Wildflower Loop
+    distanceKm: 5.1
+    elevationGain: 180
+    companion: sam
+    wasSunny: true
+```
+
+</details>
+
+TOON conveys the same information with **even fewer tokens** – combining YAML-like indentation with CSV-style tabular arrays:
+
+```yaml
+context:
+  task: Our favorite hikes together
+  location: Boulder
+  season: spring_2025
+friends[3]: ana,luis,sam
+hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
+  1,Blue Lake Trail,7.5,320,ana,true
+  2,Ridge Overlook,9.2,540,luis,false
+  3,Wildflower Loop,5.1,180,sam,true
+```
+
+## Key Features
+
+- 📊 **Token-Efficient & Accurate:** TOON reaches 76.4% accuracy (vs JSON's 75.0%) while using ~40% fewer tokens in mixed-structure benchmarks across 4 models.
+- 🔁 **JSON Data Model:** Encodes the same objects, arrays, and primitives as JSON with deterministic, lossless round-trips.
+- 🛤️ **LLM-Friendly Guardrails:** Explicit [N] lengths and {fields} headers give models a clear schema to follow, improving parsing reliability.
+- 📐 **Minimal Syntax:** Uses indentation instead of braces and minimizes quoting, giving YAML-like readability with CSV-style compactness.
+- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
+- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
+
+## Media Type & File Extension
+
+By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations) for normative details.
+
+## When Not to Use TOON
+
+TOON excels with uniform arrays of objects, but there are cases where other formats are better:
+
+- **Deeply nested or non-uniform structures** (tabular eligibility ≈ 0%): JSON-compact often uses fewer tokens. Example: complex configuration objects with many nested levels.
+- **Semi-uniform arrays** (~40–60% tabular eligibility): Token savings diminish. Prefer JSON if your pipelines already rely on it.
+- **Pure tabular data**: CSV is smaller than TOON for flat tables. TOON adds minimal overhead (~5–10%) to provide structure (array length declarations, field headers, delimiter scoping) that improves LLM reliability.
+- **Latency-critical applications**: If end-to-end response time is your top priority, benchmark on your exact setup. Some deployments (especially local/quantized models like Ollama) may process compact JSON faster despite TOON's lower token count. Measure TTFT, tokens/sec, and total time for both formats and use whichever is faster.
+
+See [benchmarks](#benchmarks) for concrete comparisons across different data structures.
+
+## Benchmarks
+
+Benchmarks are organized into two tracks to ensure fair comparisons:
+
+- **Mixed-Structure Track**: Datasets with nested or semi-uniform structures (TOON vs JSON, YAML, XML). CSV excluded as it cannot properly represent these structures.
+- **Flat-Only Track**: Datasets with flat tabular structures where CSV is applicable (CSV vs TOON vs JSON, YAML, XML).
+
+### Retrieval Accuracy
+
+<!-- automd:file src="./benchmarks/results/retrieval-accuracy.md" -->
+
+Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
+
+<details>
+<summary><strong>Show Dataset Catalog</strong></summary>
+
+#### Dataset Catalog
+
+| Dataset | Rows | Structure | CSV Support | Eligibility |
+| ------- | ---- | --------- | ----------- | ----------- |
+| Uniform employee records | 100 | uniform | ✓ | 100% |
+| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
+| Time-series analytics data | 60 | uniform | ✓ | 100% |
+| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
+| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
+| Deeply nested configuration | 11 | deep | ✗ | 0% |
+| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
+| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
+| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
+| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
+| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
+
+**Structure classes:**
+- **uniform**: All objects have identical fields with primitive values
+- **semi-uniform**: Mix of uniform and non-uniform structures
+- **nested**: Objects with nested structures (nested objects or arrays)
+- **deep**: Highly nested with minimal tabular eligibility
+
+**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
+
+**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
+
+</details>
+
+#### Efficiency Ranking (Accuracy per 1K Tokens)
+
+Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
+
+```
+TOON           ████████████████████   27.7 acc%/1K tok  │  76.4% acc  │  2,759 tokens
+JSON compact   █████████████████░░░   23.7 acc%/1K tok  │  73.7% acc  │  3,104 tokens
+YAML           ██████████████░░░░░░   19.9 acc%/1K tok  │  74.5% acc  │  3,749 tokens
+JSON           ████████████░░░░░░░░   16.4 acc%/1K tok  │  75.0% acc  │  4,587 tokens
+XML            ██████████░░░░░░░░░░   13.8 acc%/1K tok  │  72.1% acc  │  5,221 tokens
+```
+
+*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
+
+> [!TIP]
+> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
+
+**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
+
+#### Per-Model Accuracy
+
+Accuracy across 4 LLMs on 209 data retrieval questions:
+
+```
+claude-haiku-4-5-20251001
+→ TOON           ████████████░░░░░░░░    59.8% (125/209)
+  JSON           ███████████░░░░░░░░░    57.4% (120/209)
+  YAML           ███████████░░░░░░░░░    56.0% (117/209)
+  XML            ███████████░░░░░░░░░    55.5% (116/209)
+  JSON compact   ███████████░░░░░░░░░    55.0% (115/209)
+  CSV            ██████████░░░░░░░░░░    50.5% (55/109)
+
+gemini-3-flash-preview
+  XML            ████████████████████    98.1% (205/209)
+  JSON           ███████████████████░    97.1% (203/209)
+  YAML           ███████████████████░    97.1% (203/209)
+→ TOON           ███████████████████░    96.7% (202/209)
+  JSON compact   ███████████████████░    96.7% (202/209)
+  CSV            ███████████████████░    96.3% (105/109)
+
+gpt-5-nano
+→ TOON           ██████████████████░░    90.9% (190/209)
+  JSON compact   ██████████████████░░    90.9% (190/209)
+  JSON           ██████████████████░░    89.0% (186/209)
+  CSV            ██████████████████░░    89.0% (97/109)
+  YAML           █████████████████░░░    87.1% (182/209)
+  XML            ████████████████░░░░    80.9% (169/209)
+
+grok-4-1-fast-non-reasoning
+→ TOON           ████████████░░░░░░░░    58.4% (122/209)
+  YAML           ████████████░░░░░░░░    57.9% (121/209)
+  JSON           ███████████░░░░░░░░░    56.5% (118/209)
+  XML            ███████████░░░░░░░░░    54.1% (113/209)
+  JSON compact   ██████████░░░░░░░░░░    52.2% (109/209)
+  CSV            ██████████░░░░░░░░░░    51.4% (56/109)
+```
+
+> [!TIP]
+> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
+
+<details>
+<summary><strong>Performance by dataset, model, and question type</strong></summary>
+
+#### Performance by Question Type
+
+| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
+| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
+| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
+| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
+| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
+| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
+
+#### Performance by Dataset
+
+##### Uniform employee records
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 73.2% | 2,334 | 120/164 |
+| `toon` | 73.2% | 2,498 | 120/164 |
+| `json-compact` | 73.8% | 3,924 | 121/164 |
+| `yaml` | 73.8% | 4,959 | 121/164 |
+| `json-pretty` | 73.8% | 6,331 | 121/164 |
+| `xml` | 74.4% | 7,296 | 122/164 |
+
+##### E-commerce orders with nested structures
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 82.3% | 7,458 | 135/164 |
+| `json-compact` | 78.7% | 7,110 | 129/164 |
+| `yaml` | 79.9% | 8,755 | 131/164 |
+| `json-pretty` | 79.3% | 11,234 | 130/164 |
+| `xml` | 77.4% | 12,649 | 127/164 |
+
+##### Time-series analytics data
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 75.0% | 1,411 | 90/120 |
+| `toon` | 78.3% | 1,553 | 94/120 |
+| `json-compact` | 74.2% | 2,354 | 89/120 |
+| `yaml` | 75.8% | 2,954 | 91/120 |
+| `json-pretty` | 75.0% | 3,681 | 90/120 |
+| `xml` | 72.5% | 4,389 | 87/120 |
+
+##### Top 100 GitHub repositories
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 65.9% | 8,527 | 87/132 |
+| `toon` | 66.7% | 8,779 | 88/132 |
+| `yaml` | 65.2% | 13,141 | 86/132 |
+| `json-compact` | 59.8% | 11,464 | 79/132 |
+| `json-pretty` | 63.6% | 15,157 | 84/132 |
+| `xml` | 56.1% | 17,105 | 74/132 |
+
+##### Semi-uniform event logs
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 68.3% | 4,839 | 82/120 |
+| `toon` | 65.0% | 5,819 | 78/120 |
+| `json-pretty` | 69.2% | 6,817 | 83/120 |
+| `yaml` | 61.7% | 5,847 | 74/120 |
+| `xml` | 58.3% | 7,729 | 70/120 |
+
+##### Deeply nested configuration
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `json-compact` | 90.5% | 568 | 105/116 |
+| `toon` | 94.8% | 655 | 110/116 |
+| `yaml` | 93.1% | 675 | 108/116 |
+| `json-pretty` | 92.2% | 924 | 107/116 |
+| `xml` | 91.4% | 1,013 | 106/116 |
+
+##### Valid complete dataset (control)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `toon` | 100.0% | 535 | 4/4 |
+| `json-compact` | 100.0% | 787 | 4/4 |
+| `yaml` | 100.0% | 992 | 4/4 |
+| `json-pretty` | 100.0% | 1,274 | 4/4 |
+| `xml` | 25.0% | 1,462 | 1/4 |
+| `csv` | 0.0% | 483 | 0/4 |
+
+##### Array truncated: 3 rows removed from end
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 413 | 4/4 |
+| `xml` | 100.0% | 1,243 | 4/4 |
+| `toon` | 0.0% | 462 | 0/4 |
+| `json-pretty` | 0.0% | 1,085 | 0/4 |
+| `yaml` | 0.0% | 843 | 0/4 |
+| `json-compact` | 0.0% | 670 | 0/4 |
+
+##### Extra rows added beyond declared length
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 550 | 4/4 |
+| `toon` | 75.0% | 605 | 3/4 |
+| `json-compact` | 75.0% | 901 | 3/4 |
+| `xml` | 100.0% | 1,678 | 4/4 |
+| `yaml` | 75.0% | 1,138 | 3/4 |
+| `json-pretty` | 50.0% | 1,460 | 2/4 |
+
+##### Inconsistent field count (missing salary in row 10)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 480 | 4/4 |
+| `json-compact` | 100.0% | 782 | 4/4 |
+| `yaml` | 100.0% | 985 | 4/4 |
+| `toon` | 100.0% | 1,008 | 4/4 |
+| `json-pretty` | 100.0% | 1,266 | 4/4 |
+| `xml` | 100.0% | 1,453 | 4/4 |
+
+##### Missing required fields (no email in multiple rows)
+
+| Format | Accuracy | Tokens | Correct/Total |
+| ------ | -------- | ------ | ------------- |
+| `csv` | 100.0% | 340 | 4/4 |
+| `xml` | 100.0% | 1,409 | 4/4 |
+| `toon` | 75.0% | 974 | 3/4 |
+| `json-pretty` | 50.0% | 1,225 | 2/4 |
+| `yaml` | 25.0% | 951 | 1/4 |
+| `json-compact` | 0.0% | 750 | 0/4 |
+
+#### Performance by Model
+
+##### claude-haiku-4-5-20251001
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 59.8% | 125/209 |
+| `json-pretty` | 57.4% | 120/209 |
+| `yaml` | 56.0% | 117/209 |
+| `xml` | 55.5% | 116/209 |
+| `json-compact` | 55.0% | 115/209 |
+| `csv` | 50.5% | 55/109 |
+
+##### gemini-3-flash-preview
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `xml` | 98.1% | 205/209 |
+| `json-pretty` | 97.1% | 203/209 |
+| `yaml` | 97.1% | 203/209 |
+| `toon` | 96.7% | 202/209 |
+| `json-compact` | 96.7% | 202/209 |
+| `csv` | 96.3% | 105/109 |
+
+##### gpt-5-nano
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 90.9% | 190/209 |
+| `json-compact` | 90.9% | 190/209 |
+| `json-pretty` | 89.0% | 186/209 |
+| `csv` | 89.0% | 97/109 |
+| `yaml` | 87.1% | 182/209 |
+| `xml` | 80.9% | 169/209 |
+
+##### grok-4-1-fast-non-reasoning
+
+| Format | Accuracy | Correct/Total |
+| ------ | -------- | ------------- |
+| `toon` | 58.4% | 122/209 |
+| `yaml` | 57.9% | 121/209 |
+| `json-pretty` | 56.5% | 118/209 |
+| `xml` | 54.1% | 113/209 |
+| `json-compact` | 52.2% | 109/209 |
+| `csv` | 51.4% | 56/109 |
+
+</details>
+
+#### What's Being Measured
+
+This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
+
+#### Datasets Tested
+
+Eleven datasets designed to test different structural patterns and validation capabilities:
+
+**Primary datasets:**
+
+1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
+2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
+3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
+4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
+5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
+6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
+
+**Structural validation datasets:**
+
+7. **Control**: Valid complete dataset (baseline for validation)
+8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
+9. **Extra rows**: Array with 3 additional rows beyond declared length
+10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
+11. **Missing fields**: Systematic field omissions (no email in multiple rows)
+
+#### Question Types
+
+209 questions are generated dynamically across five categories:
+
+- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
+  - Example: "What is Alice's salary?" → `75000`
+  - Example: "How many items are in order ORD-0042?" → `3`
+  - Example: "What is the customer name for order ORD-0042?" → `John Doe`
+
+- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
+  - Example: "How many employees work in Engineering?" → `17`
+  - Example: "What is the total revenue across all orders?" → `45123.50`
+  - Example: "How many employees have salary > 80000?" → `23`
+
+- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
+  - Example: "How many employees in Sales have salary > 80000?" → `5`
+  - Example: "How many active employees have more than 10 years of experience?" → `8`
+
+- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
+  - Example: "How many employees are in the dataset?" → `100`
+  - Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
+  - Example: "What is the department of the last employee?" → `Sales`
+
+- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
+  - Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
+  - Tests TOON's `[N]` length validation and `{fields}` consistency checking
+  - Demonstrates CSV's lack of structural validation capabilities
+
+#### Evaluation Process
+
+1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
+2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
+3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
+
+#### Models & Configuration
+
+- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
+- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
+- **Temperature**: Not set (models use their defaults)
+- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
+
+<!-- /automd -->
+
+### Token Efficiency
+
+Token counts are measured using the GPT-5 `o200k_base` tokenizer via [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer). Savings are calculated against formatted JSON (2-space indentation) as the primary baseline, with additional comparisons to compact JSON (minified), YAML, and XML. Actual savings vary by model and tokenizer.
+
+The benchmarks test datasets across different structural patterns (uniform, semi-uniform, nested, deeply nested) to show where TOON excels and where other formats may be better.
+
+<!-- automd:file src="./benchmarks/results/token-efficiency.md" -->
+
+#### Mixed-Structure Track
+
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
+
+```
+🛒 E-commerce orders with nested structures  ┊  Tabular: 33%
+   │
+   TOON                █████████████░░░░░░░    73,126 tokens
+   ├─ vs JSON          (−33.3%)               109,599 tokens
+   ├─ vs JSON compact  (+5.3%)                 69,459 tokens
+   ├─ vs YAML          (−14.4%)                85,415 tokens
+   └─ vs XML           (−40.7%)               123,344 tokens
+
+🧾 Semi-uniform event logs  ┊  Tabular: 50%
+   │
+   TOON                █████████████████░░░   154,084 tokens
+   ├─ vs JSON          (−15.0%)               181,201 tokens
+   ├─ vs JSON compact  (+19.9%)               128,529 tokens
+   ├─ vs YAML          (−0.8%)                155,397 tokens
+   └─ vs XML           (−25.2%)               205,859 tokens
+
+🧩 Deeply nested configuration  ┊  Tabular: 0%
+   │
+   TOON                ██████████████░░░░░░       620 tokens
+   ├─ vs JSON          (−31.9%)                   911 tokens
+   ├─ vs JSON compact  (+11.1%)                   558 tokens
+   ├─ vs YAML          (−6.3%)                    662 tokens
+   └─ vs XML           (−38.2%)                 1,003 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   TOON                ████████████████░░░░   227,830 tokens
+   ├─ vs JSON          (−21.9%)               291,711 tokens
+   ├─ vs JSON compact  (+14.7%)               198,546 tokens
+   ├─ vs YAML          (−5.7%)                241,474 tokens
+   └─ vs XML           (−31.0%)               330,206 tokens
+```
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+```
+👥 Uniform employee records  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░    47,102 tokens
+   TOON                ████████████████████    49,919 tokens   (+6.0% vs CSV)
+   ├─ vs JSON          (−60.7%)               127,063 tokens
+   ├─ vs JSON compact  (−36.9%)                79,059 tokens
+   ├─ vs YAML          (−50.1%)               100,011 tokens
+   └─ vs XML           (−65.9%)               146,579 tokens
+
+📈 Time-series analytics data  ┊  Tabular: 100%
+   │
+   CSV                 ██████████████████░░     8,383 tokens
+   TOON                ████████████████████     9,115 tokens   (+8.7% vs CSV)
+   ├─ vs JSON          (−59.0%)                22,245 tokens
+   ├─ vs JSON compact  (−35.9%)                14,211 tokens
+   ├─ vs YAML          (−49.0%)                17,858 tokens
+   └─ vs XML           (−65.8%)                26,616 tokens
+
+⭐ Top 100 GitHub repositories  ┊  Tabular: 100%
+   │
+   CSV                 ███████████████████░     8,512 tokens
+   TOON                ████████████████████     8,744 tokens   (+2.7% vs CSV)
+   ├─ vs JSON          (−42.3%)                15,144 tokens
+   ├─ vs JSON compact  (−23.7%)                11,454 tokens
+   ├─ vs YAML          (−33.4%)                13,128 tokens
+   └─ vs XML           (−48.9%)                17,095 tokens
+
+──────────────────────────────────── Total ────────────────────────────────────
+   CSV                 ███████████████████░    63,997 tokens
+   TOON                ████████████████████    67,778 tokens   (+5.9% vs CSV)
+   ├─ vs JSON          (−58.8%)               164,452 tokens
+   ├─ vs JSON compact  (−35.3%)               104,724 tokens
+   ├─ vs YAML          (−48.3%)               130,997 tokens
+   └─ vs XML           (−64.4%)               190,290 tokens
+```
+
+<details>
+<summary><strong>Show detailed examples</strong></summary>
+
+#### 📈 Time-series analytics data
+
+**Savings:** 13,130 tokens (59.0% reduction vs JSON)
+
+**JSON** (22,245 tokens):
+
+```json
+{
+  "metrics": [
+    {
+      "date": "2025-01-01",
+      "views": 6138,
+      "clicks": 174,
+      "conversions": 12,
+      "revenue": 2712.49,
+      "bounceRate": 0.35
+    },
+    {
+      "date": "2025-01-02",
+      "views": 4616,
+      "clicks": 274,
+      "conversions": 34,
+      "revenue": 9156.29,
+      "bounceRate": 0.56
+    },
+    {
+      "date": "2025-01-03",
+      "views": 4460,
+      "clicks": 143,
+      "conversions": 8,
+      "revenue": 1317.98,
+      "bounceRate": 0.59
+    },
+    {
+      "date": "2025-01-04",
+      "views": 4740,
+      "clicks": 125,
+      "conversions": 13,
+      "revenue": 2934.77,
+      "bounceRate": 0.37
+    },
+    {
+      "date": "2025-01-05",
+      "views": 6428,
+      "clicks": 369,
+      "conversions": 19,
+      "revenue": 1317.24,
+      "bounceRate": 0.3
+    }
+  ]
+}
+```
+
+**TOON** (9,115 tokens):
+
+```
+metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
+  2025-01-01,6138,174,12,2712.49,0.35
+  2025-01-02,4616,274,34,9156.29,0.56
+  2025-01-03,4460,143,8,1317.98,0.59
+  2025-01-04,4740,125,13,2934.77,0.37
+  2025-01-05,6428,369,19,1317.24,0.3
+```
+
+---
+
+#### ⭐ Top 100 GitHub repositories
+
+**Savings:** 6,400 tokens (42.3% reduction vs JSON)
+
+**JSON** (15,144 tokens):
+
+```json
+{
+  "repositories": [
+    {
+      "id": 28457823,
+      "name": "freeCodeCamp",
+      "repo": "freeCodeCamp/freeCodeCamp",
+      "description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
+      "createdAt": "2014-12-24T17:49:19Z",
+      "updatedAt": "2025-10-28T11:58:08Z",
+      "pushedAt": "2025-10-28T10:17:16Z",
+      "stars": 430886,
+      "watchers": 8583,
+      "forks": 42146,
+      "defaultBranch": "main"
+    },
+    {
+      "id": 132750724,
+      "name": "build-your-own-x",
+      "repo": "codecrafters-io/build-your-own-x",
+      "description": "Master programming by recreating your favorite technologies from scratch.",
+      "createdAt": "2018-05-09T12:03:18Z",
+      "updatedAt": "2025-10-28T12:37:11Z",
+      "pushedAt": "2025-10-10T18:45:01Z",
+      "stars": 430877,
+      "watchers": 6332,
+      "forks": 40453,
+      "defaultBranch": "master"
+    },
+    {
+      "id": 21737465,
+      "name": "awesome",
+      "repo": "sindresorhus/awesome",
+      "description": "😎 Awesome lists about all kinds of interesting topics",
+      "createdAt": "2014-07-11T13:42:37Z",
+      "updatedAt": "2025-10-28T12:40:21Z",
+      "pushedAt": "2025-10-27T17:57:31Z",
+      "stars": 410052,
+      "watchers": 8017,
+      "forks": 32029,
+      "defaultBranch": "main"
+    }
+  ]
+}
+```
+
+**TOON** (8,744 tokens):
+
+```
+repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
+  28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
+  132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
+  21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
+```
+
+</details>
+
+<!-- /automd -->
+
+## Installation & Quick Start
+
+### CLI (No Installation Required)
+
+Try TOON instantly with npx:
+
+```bash
+# Convert JSON to TOON
+npx @toon-format/cli input.json -o output.toon
+
+# Pipe from stdin
+echo '{"name": "Ada", "role": "dev"}' | npx @toon-format/cli
+```
+
+See the [CLI section](#cli) for all options and examples.
+
+### TypeScript Library
+
+```bash
+# npm
+npm install @toon-format/toon
+
+# pnpm
+pnpm add @toon-format/toon
+
+# yarn
+yarn add @toon-format/toon
+```
+
+**Example usage:**
+
+```ts
+import { encode } from '@toon-format/toon'
+
+const data = {
+  users: [
+    { id: 1, name: 'Alice', role: 'admin' },
+    { id: 2, name: 'Bob', role: 'user' }
+  ]
+}
+
+console.log(encode(data))
+// users[2]{id,name,role}:
+//   1,Alice,admin
+//   2,Bob,user
+```
+
+**Streaming large datasets:**
+
+```ts
+import { encodeLines } from '@toon-format/toon'
+
+const largeData = await fetchThousandsOfRecords()
+
+// Memory-efficient streaming for large data
+for (const line of encodeLines(largeData)) {
+  process.stdout.write(`${line}\n`)
+}
+```
+
+> [!TIP]
+> For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options).
+
+**Transforming values with replacer:**
+
+```ts
+import { encode } from '@toon-format/toon'
+
+// Remove sensitive fields
+const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
+const safe = encode(user, {
+  replacer: (key, value) => key === 'password' ? undefined : value
+})
+// name: Alice
+// email: alice@example.com
+
+// Transform values
+const data = { status: 'active', count: 5 }
+const transformed = encode(data, {
+  replacer: (key, value) =>
+    typeof value === 'string' ? value.toUpperCase() : value
+})
+// status: ACTIVE
+// count: 5
+```
+
+> [!TIP]
+> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples.
+
+## Playgrounds
+
+Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.
+
+### Official Playground
+
+The [TOON Playground](https://toonformat.dev/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
+
+### Community Playgrounds
+
+- [Format Tokenization Playground](https://www.curiouslychase.com/playground/format-tokenization-exploration)
+- [TOON Tools](https://toontools.vercel.app/)
+
+## Editor Support
+
+### VS Code
+
+[TOON Language Support](https://marketplace.visualstudio.com/items?itemName=vishalraut.vscode-toon) – Syntax highlighting, validation, conversion, and token analysis.
+
+```bash
+code --install-extension vishalraut.vscode-toon
+```
+
+### Tree-sitter Grammar
+
+[tree-sitter-toon](https://github.com/3swordman/tree-sitter-toon) – Grammar for Tree-sitter-compatible editors (Neovim, Helix, Emacs, Zed).
+
+### Neovim
+
+[toon.nvim](https://github.com/thalesgelinger/toon.nvim) – Lua-based plugin.
+
+### Other Editors
+
+Use YAML syntax highlighting as a close approximation.
+
+## CLI
+
+Command-line tool for quick JSON↔TOON conversions, token analysis, and pipeline integration. Auto-detects format from file extension, supports stdin/stdout workflows, and offers delimiter options for maximum efficiency.
+
+```bash
+# Encode JSON to TOON (auto-detected)
+npx @toon-format/cli input.json -o output.toon
+
+# Decode TOON to JSON (auto-detected)
+npx @toon-format/cli data.toon -o output.json
+
+# Pipe from stdin (no argument needed)
+cat data.json | npx @toon-format/cli
+echo '{"name": "Ada"}' | npx @toon-format/cli
+
+# Output to stdout
+npx @toon-format/cli input.json
+
+# Show token savings
+npx @toon-format/cli data.json --stats
+```
+
+> [!TIP]
+> See the full [CLI documentation](https://toonformat.dev/cli/) for all options, examples, and advanced usage.
+
+## Format Overview
+
+Detailed syntax references, implementation guides, and quick lookups for understanding and using the TOON format.
+
+- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax documentation
+- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick reference
+- [API Reference](https://toonformat.dev/reference/api) – Encode/decode usage (TypeScript)
+
+## Using TOON with LLMs
+
+TOON works best when you show the format instead of describing it. The structure is self-documenting – models parse it naturally once they see the pattern. Wrap data in ` ```toon` code blocks for input, and show the expected header template when asking models to generate TOON. Use tab delimiters for even better token efficiency.
+
+Follow the detailed [LLM integration guide](https://toonformat.dev/guide/llm-prompts) for strategies, examples, and validation techniques.
+
+## Documentation
+
+Comprehensive guides, references, and resources to help you get the most out of the TOON format and tools.
+
+### Getting Started
+
+- [Introduction & Installation](https://toonformat.dev/guide/getting-started) – What TOON is, when to use it, first steps
+- [Format Overview](https://toonformat.dev/guide/format-overview) – Complete syntax with examples
+- [Benchmarks](https://toonformat.dev/guide/benchmarks) – Accuracy & token efficiency results
+
+### Tools & Integration
+
+- [CLI](https://toonformat.dev/cli/) – Command-line tool for JSON↔TOON conversions
+- [Playgrounds](https://toonformat.dev/ecosystem/tools-and-playgrounds) – Interactive tools
+- [Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON
+- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation
+
+### References
+
+- [API Reference](https://toonformat.dev/reference/api) – TypeScript/JavaScript encode/decode API
+- [Syntax Cheatsheet](https://toonformat.dev/reference/syntax-cheatsheet) – Quick format lookup
+- [Specification](https://github.com/toon-format/spec/blob/main/SPEC.md) – Normative rules for implementers
+
+## Other Implementations
+
+TOON has official and community implementations across multiple languages including Python, Rust, Go, Java, Swift, .NET, and many more.
+
+See the full list of implementations in the [documentation](https://toonformat.dev/ecosystem/implementations).
+
+## Credits
+
+- Logo design by [鈴木ックス(SZKX)](https://x.com/szkx_art)
+
+## License
+
+[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

packages/toon/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@toon-format/toon",
   "type": "module",
-  "version": "1.3.0",
-  "packageManager": "pnpm@10.21.0",
+  "version": "2.3.1",
+  "packageManager": "pnpm@11.13.0",
   "description": "Token-Oriented Object Notation (TOON) – Compact, human-readable, schema-aware encoding of JSON for LLM prompts",
   "author": "Johann Schopplich <hello@johannschopplich.com>",
   "license": "MIT",
@@ -38,6 +38,6 @@
     "test": "vitest"
   },
   "devDependencies": {
-    "@toon-format/spec": "^2.0.1"
+    "@toon-format/spec": "^3.3.2"
   }
 }

packages/toon/src/decode/decoders.ts

@@ -1,10 +1,11 @@
-import type { ArrayHeaderInfo, DecodeStreamOptions, Depth, JsonStreamEvent, ParsedLine } from '../types'
-import type { StreamingScanState } from './scanner'
-import { COLON, DEFAULT_DELIMITER, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants'
-import { findClosingQuote } from '../shared/string-utils'
-import { isArrayHeaderContent, isKeyValueContent, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser'
-import { createScanState, parseLinesAsync, parseLinesSync } from './scanner'
-import { assertExpectedCount, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation'
+import type { ArrayHeaderInfo, DecodeStreamOptions, Depth, JsonPrimitive, JsonStreamEvent, ParsedLine } from '../types.ts'
+import type { StreamingScanState } from './scanner.ts'
+import { COLON, DEFAULT_DELIMITER, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants.ts'
+import { findClosingQuote } from '../shared/string-utils.ts'
+import { ToonDecodeError, withLine } from './errors.ts'
+import { isArrayHeaderContent, isKeyValueContent, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser.ts'
+import { createScanState, parseLinesAsync, parseLinesSync } from './scanner.ts'
+import { assertExpectedCount, isDataRow, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
 
 interface DecoderContext { indent: number, strict: boolean }
 
@@ -139,12 +140,19 @@ export function* decodeStreamSync(
     return
   }
 
+  if (first.content.trim() === '[]') {
+    cursor.advanceSync()
+    yield { type: 'startArray', length: 0 }
+    yield { type: 'endArray' }
+    return
+  }
+
   // Check for root array
   if (isArrayHeaderContent(first.content)) {
-    const headerInfo = parseArrayHeaderLine(first.content, DEFAULT_DELIMITER)
+    const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER, resolvedOptions.strict))
     if (headerInfo) {
       cursor.advanceSync()
-      yield* decodeArrayFromHeaderSync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions)
+      yield* decodeArrayFromHeaderSync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions, first)
       return
     }
   }
@@ -154,13 +162,21 @@ export function* decodeStreamSync(
   const hasMore = !cursor.atEndSync()
   if (!hasMore && !isKeyValueLineSync(first)) {
     // Single non-key-value line is root primitive
-    yield { type: 'primitive', value: parsePrimitiveToken(first.content.trim()) }
+    yield { type: 'primitive', value: withLine(first, () => parsePrimitiveToken(first.content.trim())) }
     return
   }
 
+  if (!isKeyValueLineSync(first) && cursor.peekSync()?.depth === 0) {
+    throw new ToonDecodeError(
+      'Top-level document must start with a key-value or array-header line',
+      { line: first.lineNumber, source: first.raw },
+    )
+  }
+
   // Root object
+  const rootSeenKeys = resolvedOptions.strict ? new Set<string>() : undefined
   yield { type: 'startObject' }
-  yield* decodeKeyValueSync(first.content, cursor, 0, resolvedOptions)
+  yield* decodeKeyValueSync(first, cursor, 0, resolvedOptions, rootSeenKeys)
 
   // Process remaining object fields
   while (!cursor.atEndSync()) {
@@ -170,31 +186,47 @@ export function* decodeStreamSync(
     }
 
     cursor.advanceSync()
-    yield* decodeKeyValueSync(line.content, cursor, 0, resolvedOptions)
+    yield* decodeKeyValueSync(line, cursor, 0, resolvedOptions, rootSeenKeys)
   }
 
   yield { type: 'endObject' }
 }
 
+function assertNoDuplicateKey(key: string, line: ParsedLine, seenKeys: Set<string> | undefined): void {
+  if (!seenKeys)
+    return
+  if (seenKeys.has(key)) {
+    throw new ToonDecodeError(
+      `Duplicate sibling key "${key}"`,
+      { line: line.lineNumber, source: line.raw },
+    )
+  }
+  seenKeys.add(key)
+}
+
 function* decodeKeyValueSync(
-  content: string,
+  line: ParsedLine,
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  seenKeys?: Set<string>,
 ): Generator<JsonStreamEvent> {
+  const content = line.content
+
   // Check for array header first
-  const arrayHeader = parseArrayHeaderLine(content, DEFAULT_DELIMITER)
-  if (arrayHeader && arrayHeader.header.key) {
+  const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER, options.strict))
+  if (arrayHeader && arrayHeader.header.key !== undefined) {
+    assertNoDuplicateKey(arrayHeader.header.key, line, seenKeys)
     yield { type: 'key', key: arrayHeader.header.key }
-    yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
+    yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, line)
     return
   }
 
   // Regular key-value pair
-  const { key, isQuoted } = parseKeyToken(content, 0)
-  const colonIndex = content.indexOf(COLON, key.length)
-  const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
+  const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
+  const rest = content.slice(end).trim()
 
+  assertNoDuplicateKey(key, line, seenKeys)
   yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
 
   // No value after colon - expect nested object or empty
@@ -213,8 +245,14 @@ function* decodeKeyValueSync(
     return
   }
 
+  if (rest === '[]') {
+    yield { type: 'startArray', length: 0 }
+    yield { type: 'endArray' }
+    return
+  }
+
   // Inline primitive value
-  yield { type: 'primitive', value: parsePrimitiveToken(rest) }
+  yield { type: 'primitive', value: withLine(line, () => parsePrimitiveToken(rest)) }
 }
 
 function* decodeObjectFieldsSync(
@@ -223,6 +261,7 @@ function* decodeObjectFieldsSync(
   options: DecoderContext,
 ): Generator<JsonStreamEvent> {
   let computedDepth: Depth | undefined
+  const seenKeys = options.strict ? new Set<string>() : undefined
 
   while (!cursor.atEndSync()) {
     const line = cursor.peekSync()
@@ -236,7 +275,7 @@ function* decodeObjectFieldsSync(
 
     if (line.depth === computedDepth) {
       cursor.advanceSync()
-      yield* decodeKeyValueSync(line.content, cursor, computedDepth, options)
+      yield* decodeKeyValueSync(line, cursor, computedDepth, options, seenKeys)
     }
     else {
       break
@@ -250,25 +289,26 @@ function* decodeArrayFromHeaderSync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): Generator<JsonStreamEvent> {
   yield { type: 'startArray', length: header.length }
 
   // Inline primitive array
   if (inlineValues) {
-    yield* decodeInlinePrimitiveArraySync(header, inlineValues, options)
+    yield* decodeInlinePrimitiveArraySync(header, inlineValues, options, headerLine)
     yield { type: 'endArray' }
     return
   }
 
   // Tabular array
   if (header.fields && header.fields.length > 0) {
-    yield* decodeTabularArraySync(header, cursor, baseDepth, options)
+    yield* decodeTabularArraySync(header, cursor, baseDepth, options, headerLine)
     yield { type: 'endArray' }
     return
   }
 
   // List array
-  yield* decodeListArraySync(header, cursor, baseDepth, options)
+  yield* decodeListArraySync(header, cursor, baseDepth, options, headerLine)
   yield { type: 'endArray' }
 }
 
@@ -276,16 +316,17 @@ function* decodeInlinePrimitiveArraySync(
   header: ArrayHeaderInfo,
   inlineValues: string,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): Generator<JsonStreamEvent> {
   if (!inlineValues.trim()) {
-    assertExpectedCount(0, header.length, 'inline array items', options)
+    assertExpectedCount(0, header.length, 'inline array items', options, headerLine)
     return
   }
 
-  const values = parseDelimitedValues(inlineValues, header.delimiter)
-  const primitives = mapRowValuesToPrimitives(values)
+  const values = withLine(headerLine, () => parseDelimitedValues(inlineValues, header.delimiter))
+  const primitives = withLine(headerLine, () => mapRowValuesToPrimitives(values))
 
-  assertExpectedCount(primitives.length, header.length, 'inline array items', options)
+  assertExpectedCount(primitives.length, header.length, 'inline array items', options, headerLine)
 
   for (const primitive of primitives) {
     yield { type: 'primitive', value: primitive }
@@ -297,11 +338,13 @@ function* decodeTabularArraySync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): Generator<JsonStreamEvent> {
   const rowDepth = baseDepth + 1
   let rowCount = 0
   let startLine: number | undefined
   let endLine: number | undefined
+  let lastRowLine: ParsedLine = headerLine
 
   while (!cursor.atEndSync() && rowCount < header.length) {
     const line = cursor.peekSync()
@@ -310,23 +353,22 @@ function* decodeTabularArraySync(
     }
 
     if (line.depth === rowDepth) {
+      if (!isDataRow(line.content, header.delimiter)) {
+        break
+      }
+
       if (startLine === undefined) {
         startLine = line.lineNumber
       }
       endLine = line.lineNumber
+      lastRowLine = line
 
       cursor.advanceSync()
-      const values = parseDelimitedValues(line.content, header.delimiter)
-      assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options)
+      const values = withLine(line, () => parseDelimitedValues(line.content, header.delimiter))
+      assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options, line)
 
-      const primitives = mapRowValuesToPrimitives(values)
-
-      yield { type: 'startObject' }
-      for (let i = 0; i < header.fields!.length; i++) {
-        yield { type: 'key', key: header.fields![i]! }
-        yield { type: 'primitive', value: primitives[i]! }
-      }
-      yield { type: 'endObject' }
+      const primitives = withLine(line, () => mapRowValuesToPrimitives(values))
+      yield* yieldObjectFromFields(header.fields!, primitives)
 
       rowCount++
     }
@@ -335,7 +377,7 @@ function* decodeTabularArraySync(
     }
   }
 
-  assertExpectedCount(rowCount, header.length, 'tabular rows', options)
+  assertExpectedCount(rowCount, header.length, 'tabular rows', options, lastRowLine)
 
   if (options.strict && startLine !== undefined && endLine !== undefined) {
     validateNoBlankLinesInRange(startLine, endLine, cursor.getBlankLines(), options.strict, 'tabular array')
@@ -352,11 +394,13 @@ function* decodeListArraySync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): Generator<JsonStreamEvent> {
   const itemDepth = baseDepth + 1
   let itemCount = 0
   let startLine: number | undefined
   let endLine: number | undefined
+  let lastItemLine: ParsedLine = headerLine
 
   while (!cursor.atEndSync() && itemCount < header.length) {
     const line = cursor.peekSync()
@@ -371,12 +415,14 @@ function* decodeListArraySync(
         startLine = line.lineNumber
       }
       endLine = line.lineNumber
+      lastItemLine = line
 
       yield* decodeListItemSync(cursor, itemDepth, options)
 
       const currentLine = cursor.current()
       if (currentLine) {
         endLine = currentLine.lineNumber
+        lastItemLine = currentLine
       }
 
       itemCount++
@@ -386,7 +432,7 @@ function* decodeListArraySync(
     }
   }
 
-  assertExpectedCount(itemCount, header.length, 'list array items', options)
+  assertExpectedCount(itemCount, header.length, 'list array items', options, lastItemLine)
 
   if (options.strict && startLine !== undefined && endLine !== undefined) {
     validateNoBlankLinesInRange(startLine, endLine, cursor.getBlankLines(), options.strict, 'list array')
@@ -411,6 +457,7 @@ function* decodeListItemSync(
   let afterHyphen: string
 
   if (line.content === LIST_ITEM_MARKER) {
+    // Bare list item marker: always an empty object
     yield { type: 'startObject' }
     yield { type: 'endObject' }
     return
@@ -419,7 +466,10 @@ function* decodeListItemSync(
     afterHyphen = line.content.slice(LIST_ITEM_PREFIX.length)
   }
   else {
-    throw new SyntaxError(`Expected list item to start with "${LIST_ITEM_PREFIX}"`)
+    throw new ToonDecodeError(
+      `Expected list item to start with "${LIST_ITEM_PREFIX}"`,
+      { line: line.lineNumber, source: line.raw },
+    )
   }
 
   if (!afterHyphen.trim()) {
@@ -428,19 +478,61 @@ function* decodeListItemSync(
     return
   }
 
+  if (afterHyphen.trim() === '[]') {
+    yield { type: 'startArray', length: 0 }
+    yield { type: 'endArray' }
+    return
+  }
+
+  const itemLine: ParsedLine = { ...line, content: afterHyphen }
+
   // Check for array header after hyphen
   if (isArrayHeaderContent(afterHyphen)) {
-    const arrayHeader = parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER)
+    const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
     if (arrayHeader) {
-      yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
+      yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, itemLine)
       return
     }
   }
 
+  // Check for tabular-first list-item object: `- key[N]{fields}:`
+  const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
+  if (headerInfo && headerInfo.header.key !== undefined && headerInfo.header.fields !== undefined) {
+    // Object with tabular array as first field
+    const header = headerInfo.header
+    const seenKeys = options.strict ? new Set<string>([header.key!]) : undefined
+    yield { type: 'startObject' }
+    yield { type: 'key', key: header.key! }
+
+    // Use baseDepth + 1 for the array so rows are at baseDepth + 2
+    yield* decodeArrayFromHeaderSync(header, headerInfo.inlineValues, cursor, baseDepth + 1, options, itemLine)
+
+    // Read sibling fields at depth = baseDepth + 1
+    const followDepth = baseDepth + 1
+    while (!cursor.atEndSync()) {
+      const nextLine = cursor.peekSync()
+      if (!nextLine || nextLine.depth < followDepth) {
+        break
+      }
+
+      if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
+        cursor.advanceSync()
+        yield* decodeKeyValueSync(nextLine, cursor, followDepth, options, seenKeys)
+      }
+      else {
+        break
+      }
+    }
+
+    yield { type: 'endObject' }
+    return
+  }
+
   // Check for object first field after hyphen
   if (isKeyValueContent(afterHyphen)) {
+    const seenKeys = options.strict ? new Set<string>() : undefined
     yield { type: 'startObject' }
-    yield* decodeKeyValueSync(afterHyphen, cursor, baseDepth, options)
+    yield* decodeKeyValueSync(itemLine, cursor, baseDepth + 1, options, seenKeys)
 
     // Read subsequent fields
     const followDepth = baseDepth + 1
@@ -452,7 +544,7 @@ function* decodeListItemSync(
 
       if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
         cursor.advanceSync()
-        yield* decodeKeyValueSync(nextLine.content, cursor, followDepth, options)
+        yield* decodeKeyValueSync(nextLine, cursor, followDepth, options, seenKeys)
       }
       else {
         break
@@ -464,7 +556,7 @@ function* decodeListItemSync(
   }
 
   // Primitive value
-  yield { type: 'primitive', value: parsePrimitiveToken(afterHyphen) }
+  yield { type: 'primitive', value: withLine(itemLine, () => parsePrimitiveToken(afterHyphen)) }
 }
 
 function isKeyValueLineSync(line: ParsedLine): boolean {
@@ -509,18 +601,25 @@ export async function* decodeStream(
     // Get first line to determine root form
     const first = await cursor.peek()
     if (!first) {
-      // Empty input decodes to empty object (matches decode('') behavior)
+      // Empty input decodes to empty object
       yield { type: 'startObject' }
       yield { type: 'endObject' }
       return
     }
 
+    if (first.content.trim() === '[]') {
+      await cursor.advance()
+      yield { type: 'startArray', length: 0 }
+      yield { type: 'endArray' }
+      return
+    }
+
     // Check for root array
     if (isArrayHeaderContent(first.content)) {
-      const headerInfo = parseArrayHeaderLine(first.content, DEFAULT_DELIMITER)
+      const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER, resolvedOptions.strict))
       if (headerInfo) {
         await cursor.advance()
-        yield* decodeArrayFromHeaderAsync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions)
+        yield* decodeArrayFromHeaderAsync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions, first)
         return
       }
     }
@@ -529,13 +628,21 @@ export async function* decodeStream(
     await cursor.advance()
     const hasMore = !(await cursor.atEnd())
     if (!hasMore && !isKeyValueLineSync(first)) {
-      yield { type: 'primitive', value: parsePrimitiveToken(first.content.trim()) }
+      yield { type: 'primitive', value: withLine(first, () => parsePrimitiveToken(first.content.trim())) }
       return
     }
 
+    if (!isKeyValueLineSync(first) && (await cursor.peek())?.depth === 0) {
+      throw new ToonDecodeError(
+        'Top-level document must start with a key-value or array-header line',
+        { line: first.lineNumber, source: first.raw },
+      )
+    }
+
     // Root object
+    const rootSeenKeys = resolvedOptions.strict ? new Set<string>() : undefined
     yield { type: 'startObject' }
-    yield* decodeKeyValueAsync(first.content, cursor, 0, resolvedOptions)
+    yield* decodeKeyValueAsync(first, cursor, 0, resolvedOptions, rootSeenKeys)
 
     // Process remaining object fields
     while (!(await cursor.atEnd())) {
@@ -544,7 +651,7 @@ export async function* decodeStream(
         break
       }
       await cursor.advance()
-      yield* decodeKeyValueAsync(line.content, cursor, 0, resolvedOptions)
+      yield* decodeKeyValueAsync(line, cursor, 0, resolvedOptions, rootSeenKeys)
     }
 
     yield { type: 'endObject' }
@@ -556,24 +663,28 @@ export async function* decodeStream(
 }
 
 async function* decodeKeyValueAsync(
-  content: string,
+  line: ParsedLine,
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  seenKeys?: Set<string>,
 ): AsyncGenerator<JsonStreamEvent> {
+  const content = line.content
+
   // Check for array header first
-  const arrayHeader = parseArrayHeaderLine(content, DEFAULT_DELIMITER)
-  if (arrayHeader && arrayHeader.header.key) {
+  const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER, options.strict))
+  if (arrayHeader && arrayHeader.header.key !== undefined) {
+    assertNoDuplicateKey(arrayHeader.header.key, line, seenKeys)
     yield { type: 'key', key: arrayHeader.header.key }
-    yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
+    yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, line)
     return
   }
 
   // Regular key-value pair
-  const { key, isQuoted } = parseKeyToken(content, 0)
-  const colonIndex = content.indexOf(COLON, key.length)
-  const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
+  const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
+  const rest = content.slice(end).trim()
 
+  assertNoDuplicateKey(key, line, seenKeys)
   yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
 
   // No value after colon - expect nested object or empty
@@ -592,8 +703,14 @@ async function* decodeKeyValueAsync(
     return
   }
 
+  if (rest === '[]') {
+    yield { type: 'startArray', length: 0 }
+    yield { type: 'endArray' }
+    return
+  }
+
   // Inline primitive value
-  yield { type: 'primitive', value: parsePrimitiveToken(rest) }
+  yield { type: 'primitive', value: withLine(line, () => parsePrimitiveToken(rest)) }
 }
 
 async function* decodeObjectFieldsAsync(
@@ -602,6 +719,7 @@ async function* decodeObjectFieldsAsync(
   options: DecoderContext,
 ): AsyncGenerator<JsonStreamEvent> {
   let computedDepth: Depth | undefined
+  const seenKeys = options.strict ? new Set<string>() : undefined
 
   while (!(await cursor.atEnd())) {
     const line = await cursor.peek()
@@ -615,7 +733,7 @@ async function* decodeObjectFieldsAsync(
 
     if (line.depth === computedDepth) {
       await cursor.advance()
-      yield* decodeKeyValueAsync(line.content, cursor, computedDepth, options)
+      yield* decodeKeyValueAsync(line, cursor, computedDepth, options, seenKeys)
     }
     else {
       break
@@ -629,25 +747,26 @@ async function* decodeArrayFromHeaderAsync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): AsyncGenerator<JsonStreamEvent> {
   yield { type: 'startArray', length: header.length }
 
   // Inline primitive array
   if (inlineValues) {
-    yield* decodeInlinePrimitiveArraySync(header, inlineValues, options)
+    yield* decodeInlinePrimitiveArraySync(header, inlineValues, options, headerLine)
     yield { type: 'endArray' }
     return
   }
 
   // Tabular array
   if (header.fields && header.fields.length > 0) {
-    yield* decodeTabularArrayAsync(header, cursor, baseDepth, options)
+    yield* decodeTabularArrayAsync(header, cursor, baseDepth, options, headerLine)
     yield { type: 'endArray' }
     return
   }
 
   // List array
-  yield* decodeListArrayAsync(header, cursor, baseDepth, options)
+  yield* decodeListArrayAsync(header, cursor, baseDepth, options, headerLine)
   yield { type: 'endArray' }
 }
 
@@ -656,11 +775,13 @@ async function* decodeTabularArrayAsync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): AsyncGenerator<JsonStreamEvent> {
   const rowDepth = baseDepth + 1
   let rowCount = 0
   let startLine: number | undefined
   let endLine: number | undefined
+  let lastRowLine: ParsedLine = headerLine
 
   while (!(await cursor.atEnd()) && rowCount < header.length) {
     const line = await cursor.peek()
@@ -669,23 +790,22 @@ async function* decodeTabularArrayAsync(
     }
 
     if (line.depth === rowDepth) {
+      if (!isDataRow(line.content, header.delimiter)) {
+        break
+      }
+
       if (startLine === undefined) {
         startLine = line.lineNumber
       }
       endLine = line.lineNumber
+      lastRowLine = line
 
       await cursor.advance()
-      const values = parseDelimitedValues(line.content, header.delimiter)
-      assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options)
+      const values = withLine(line, () => parseDelimitedValues(line.content, header.delimiter))
+      assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options, line)
 
-      const primitives = mapRowValuesToPrimitives(values)
-
-      yield { type: 'startObject' }
-      for (let i = 0; i < header.fields!.length; i++) {
-        yield { type: 'key', key: header.fields![i]! }
-        yield { type: 'primitive', value: primitives[i]! }
-      }
-      yield { type: 'endObject' }
+      const primitives = withLine(line, () => mapRowValuesToPrimitives(values))
+      yield* yieldObjectFromFields(header.fields!, primitives)
 
       rowCount++
     }
@@ -694,7 +814,7 @@ async function* decodeTabularArrayAsync(
     }
   }
 
-  assertExpectedCount(rowCount, header.length, 'tabular rows', options)
+  assertExpectedCount(rowCount, header.length, 'tabular rows', options, lastRowLine)
 
   if (options.strict && startLine !== undefined && endLine !== undefined) {
     validateNoBlankLinesInRange(startLine, endLine, cursor.getBlankLines(), options.strict, 'tabular array')
@@ -711,11 +831,13 @@ async function* decodeListArrayAsync(
   cursor: StreamingLineCursor,
   baseDepth: Depth,
   options: DecoderContext,
+  headerLine: ParsedLine,
 ): AsyncGenerator<JsonStreamEvent> {
   const itemDepth = baseDepth + 1
   let itemCount = 0
   let startLine: number | undefined
   let endLine: number | undefined
+  let lastItemLine: ParsedLine = headerLine
 
   while (!(await cursor.atEnd()) && itemCount < header.length) {
     const line = await cursor.peek()
@@ -730,12 +852,14 @@ async function* decodeListArrayAsync(
         startLine = line.lineNumber
       }
       endLine = line.lineNumber
+      lastItemLine = line
 
       yield* decodeListItemAsync(cursor, itemDepth, options)
 
       const currentLine = cursor.current()
       if (currentLine) {
         endLine = currentLine.lineNumber
+        lastItemLine = currentLine
       }
 
       itemCount++
@@ -745,7 +869,7 @@ async function* decodeListArrayAsync(
     }
   }
 
-  assertExpectedCount(itemCount, header.length, 'list array items', options)
+  assertExpectedCount(itemCount, header.length, 'list array items', options, lastItemLine)
 
   if (options.strict && startLine !== undefined && endLine !== undefined) {
     validateNoBlankLinesInRange(startLine, endLine, cursor.getBlankLines(), options.strict, 'list array')
@@ -770,6 +894,7 @@ async function* decodeListItemAsync(
   let afterHyphen: string
 
   if (line.content === LIST_ITEM_MARKER) {
+    // Bare list item marker: always an empty object
     yield { type: 'startObject' }
     yield { type: 'endObject' }
     return
@@ -778,7 +903,10 @@ async function* decodeListItemAsync(
     afterHyphen = line.content.slice(LIST_ITEM_PREFIX.length)
   }
   else {
-    throw new SyntaxError(`Expected list item to start with "${LIST_ITEM_PREFIX}"`)
+    throw new ToonDecodeError(
+      `Expected list item to start with "${LIST_ITEM_PREFIX}"`,
+      { line: line.lineNumber, source: line.raw },
+    )
   }
 
   if (!afterHyphen.trim()) {
@@ -787,19 +915,61 @@ async function* decodeListItemAsync(
     return
   }
 
+  if (afterHyphen.trim() === '[]') {
+    yield { type: 'startArray', length: 0 }
+    yield { type: 'endArray' }
+    return
+  }
+
+  const itemLine: ParsedLine = { ...line, content: afterHyphen }
+
   // Check for array header after hyphen
   if (isArrayHeaderContent(afterHyphen)) {
-    const arrayHeader = parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER)
+    const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
     if (arrayHeader) {
-      yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
+      yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, itemLine)
       return
     }
   }
 
+  // Check for tabular-first list-item object: `- key[N]{fields}:`
+  const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
+  if (headerInfo && headerInfo.header.key !== undefined && headerInfo.header.fields !== undefined) {
+    // Object with tabular array as first field
+    const header = headerInfo.header
+    const seenKeys = options.strict ? new Set<string>([header.key!]) : undefined
+    yield { type: 'startObject' }
+    yield { type: 'key', key: header.key! }
+
+    // Use baseDepth + 1 for the array so rows are at baseDepth + 2
+    yield* decodeArrayFromHeaderAsync(header, headerInfo.inlineValues, cursor, baseDepth + 1, options, itemLine)
+
+    // Read sibling fields at depth = baseDepth + 1
+    const followDepth = baseDepth + 1
+    while (!(await cursor.atEnd())) {
+      const nextLine = await cursor.peek()
+      if (!nextLine || nextLine.depth < followDepth) {
+        break
+      }
+
+      if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
+        await cursor.advance()
+        yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options, seenKeys)
+      }
+      else {
+        break
+      }
+    }
+
+    yield { type: 'endObject' }
+    return
+  }
+
   // Check for object first field after hyphen
   if (isKeyValueContent(afterHyphen)) {
+    const seenKeys = options.strict ? new Set<string>() : undefined
     yield { type: 'startObject' }
-    yield* decodeKeyValueAsync(afterHyphen, cursor, baseDepth, options)
+    yield* decodeKeyValueAsync(itemLine, cursor, baseDepth + 1, options, seenKeys)
 
     // Read subsequent fields
     const followDepth = baseDepth + 1
@@ -811,7 +981,7 @@ async function* decodeListItemAsync(
 
       if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
         await cursor.advance()
-        yield* decodeKeyValueAsync(nextLine.content, cursor, followDepth, options)
+        yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options, seenKeys)
       }
       else {
         break
@@ -823,7 +993,23 @@ async function* decodeListItemAsync(
   }
 
   // Primitive value
-  yield { type: 'primitive', value: parsePrimitiveToken(afterHyphen) }
+  yield { type: 'primitive', value: withLine(itemLine, () => parsePrimitiveToken(afterHyphen)) }
+}
+
+// #endregion
+
+// #region Shared decoder helpers
+
+function* yieldObjectFromFields(
+  fields: string[],
+  primitives: JsonPrimitive[],
+): Generator<JsonStreamEvent> {
+  yield { type: 'startObject' }
+  for (let i = 0; i < fields.length; i++) {
+    yield { type: 'key', key: fields[i]! }
+    yield { type: 'primitive', value: primitives[i]! }
+  }
+  yield { type: 'endObject' }
 }
 
 // #endregion

packages/toon/src/decode/errors.ts

@@ -0,0 +1,47 @@
+import type { ParsedLine } from '../types.ts'
+
+/**
+ * Error thrown by the TOON decoder when input cannot be parsed.
+ *
+ * Extends `SyntaxError` so existing `instanceof SyntaxError` checks keep working.
+ * Adds structured location fields for programmatic consumers and richer CLI output.
+ */
+export class ToonDecodeError extends SyntaxError {
+  /** 1-based line number where the error was detected, if known. */
+  readonly line?: number
+  /** Raw source line (including indentation) where the error was detected, if known. */
+  readonly source?: string
+
+  constructor(message: string, context?: { line?: number, source?: string, cause?: unknown }) {
+    const prefix = context?.line !== undefined ? `Line ${context.line}: ` : ''
+    super(prefix + message, context?.cause !== undefined ? { cause: context.cause } : undefined)
+    this.name = 'ToonDecodeError'
+    this.line = context?.line
+    this.source = context?.source
+  }
+}
+
+/**
+ * Runs `fn` and re-throws any non-`ToonDecodeError` `Error` as a `ToonDecodeError`
+ * with line context attached and the original error preserved as `cause`.
+ *
+ * Pure parser helpers (parser.ts, string-utils.ts) don't know which line they're
+ * parsing; this wrapper is how the streaming decoder enriches their errors.
+ */
+export function withLine<T>(line: ParsedLine, fn: () => T): T {
+  try {
+    return fn()
+  }
+  catch (error) {
+    if (error instanceof ToonDecodeError)
+      throw error
+    if (error instanceof Error) {
+      throw new ToonDecodeError(error.message, {
+        line: line.lineNumber,
+        source: line.raw,
+        cause: error,
+      })
+    }
+    throw error
+  }
+}

packages/toon/src/decode/event-builder.ts

@@ -1,176 +1,30 @@
-import type { JsonObject, JsonStreamEvent, JsonValue } from '../types'
-import { QUOTED_KEY_MARKER } from './expand'
+import type { JsonObject, JsonStreamEvent, JsonValue } from '../types.ts'
+import { setOwnProperty } from '../shared/object-utils.ts'
+import { QUOTED_KEY_MARKER } from './expand.ts'
 
 // #region Build context types
 
-/**
- * Stack context for building JSON values from events.
- */
 type BuildContext
   = | { type: 'object', obj: JsonObject, currentKey?: string, quotedKeys: Set<string> }
     | { type: 'array', arr: JsonValue[] }
 
+interface BuildState {
+  stack: BuildContext[]
+  root: JsonValue | undefined
+}
+
 // #endregion
 
 // #region Synchronous AST builder
 
 export function buildValueFromEvents(events: Iterable<JsonStreamEvent>): JsonValue {
-  const stack: BuildContext[] = []
-  let root: JsonValue | undefined
+  const state: BuildState = { stack: [], root: undefined }
 
   for (const event of events) {
-    switch (event.type) {
-      case 'startObject': {
-        const obj: JsonObject = {}
-        const quotedKeys = new Set<string>()
-
-        if (stack.length === 0) {
-          // Root object
-          stack.push({ type: 'object', obj, quotedKeys })
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Object startObject event without preceding key')
-            }
-
-            parent.obj[parent.currentKey] = obj
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(obj)
-          }
-
-          stack.push({ type: 'object', obj, quotedKeys })
-        }
-
-        break
-      }
-
-      case 'endObject': {
-        if (stack.length === 0) {
-          throw new Error('Unexpected endObject event')
-        }
-
-        const context = stack.pop()!
-        if (context.type !== 'object') {
-          throw new Error('Mismatched endObject event')
-        }
-
-        // Attach quoted keys metadata if any keys were quoted
-        if (context.quotedKeys.size > 0) {
-          Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
-            value: context.quotedKeys,
-            enumerable: false,
-            writable: false,
-            configurable: false,
-          })
-        }
-
-        if (stack.length === 0) {
-          root = context.obj
-        }
-
-        break
-      }
-
-      case 'startArray': {
-        const arr: JsonValue[] = []
-
-        if (stack.length === 0) {
-          // Root array
-          stack.push({ type: 'array', arr })
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Array startArray event without preceding key')
-            }
-            parent.obj[parent.currentKey] = arr
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(arr)
-          }
-
-          stack.push({ type: 'array', arr })
-        }
-
-        break
-      }
-
-      case 'endArray': {
-        if (stack.length === 0) {
-          throw new Error('Unexpected endArray event')
-        }
-
-        const context = stack.pop()!
-        if (context.type !== 'array') {
-          throw new Error('Mismatched endArray event')
-        }
-
-        if (stack.length === 0) {
-          root = context.arr
-        }
-
-        break
-      }
-
-      case 'key': {
-        if (stack.length === 0) {
-          throw new Error('Key event outside of object context')
-        }
-
-        const parent = stack[stack.length - 1]!
-        if (parent.type !== 'object') {
-          throw new Error('Key event in non-object context')
-        }
-
-        parent.currentKey = event.key
-
-        // Track quoted keys for path expansion
-        if (event.wasQuoted) {
-          parent.quotedKeys.add(event.key)
-        }
-
-        break
-      }
-
-      case 'primitive': {
-        if (stack.length === 0) {
-          // Root primitive
-          root = event.value
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Primitive event without preceding key in object')
-            }
-            parent.obj[parent.currentKey] = event.value
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(event.value)
-          }
-        }
-
-        break
-      }
-    }
+    applyEvent(state, event)
   }
 
-  if (stack.length !== 0) {
-    throw new Error('Incomplete event stream: stack not empty at end')
-  }
-
-  if (root === undefined) {
-    throw new Error('No root value built from events')
-  }
-
-  return root
+  return finalizeState(state)
 }
 
 // #endregion
@@ -178,157 +32,175 @@ export function buildValueFromEvents(events: Iterable<JsonStreamEvent>): JsonVal
 // #region Asynchronous AST builder
 
 export async function buildValueFromEventsAsync(events: AsyncIterable<JsonStreamEvent>): Promise<JsonValue> {
-  const stack: BuildContext[] = []
-  let root: JsonValue | undefined
+  const state: BuildState = { stack: [], root: undefined }
 
   for await (const event of events) {
-    switch (event.type) {
-      case 'startObject': {
-        const obj: JsonObject = {}
-        const quotedKeys = new Set<string>()
-
-        if (stack.length === 0) {
-          stack.push({ type: 'object', obj, quotedKeys })
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Object startObject event without preceding key')
-            }
-            parent.obj[parent.currentKey] = obj
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(obj)
-          }
-
-          stack.push({ type: 'object', obj, quotedKeys })
-        }
-
-        break
-      }
-
-      case 'endObject': {
-        if (stack.length === 0) {
-          throw new Error('Unexpected endObject event')
-        }
-
-        const context = stack.pop()!
-        if (context.type !== 'object') {
-          throw new Error('Mismatched endObject event')
-        }
-
-        // Attach quoted keys metadata if any keys were quoted
-        if (context.quotedKeys.size > 0) {
-          Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
-            value: context.quotedKeys,
-            enumerable: false,
-            writable: false,
-            configurable: false,
-          })
-        }
-
-        if (stack.length === 0) {
-          root = context.obj
-        }
-
-        break
-      }
-
-      case 'startArray': {
-        const arr: JsonValue[] = []
-        if (stack.length === 0) {
-          stack.push({ type: 'array', arr })
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Array startArray event without preceding key')
-            }
-            parent.obj[parent.currentKey] = arr
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(arr)
-          }
-
-          stack.push({ type: 'array', arr })
-        }
-
-        break
-      }
-
-      case 'endArray': {
-        if (stack.length === 0) {
-          throw new Error('Unexpected endArray event')
-        }
-
-        const context = stack.pop()!
-        if (context.type !== 'array') {
-          throw new Error('Mismatched endArray event')
-        }
-
-        if (stack.length === 0) {
-          root = context.arr
-        }
-
-        break
-      }
-
-      case 'key': {
-        if (stack.length === 0) {
-          throw new Error('Key event outside of object context')
-        }
-
-        const parent = stack[stack.length - 1]!
-        if (parent.type !== 'object') {
-          throw new Error('Key event in non-object context')
-        }
-
-        parent.currentKey = event.key
-
-        // Track quoted keys for path expansion
-        if (event.wasQuoted) {
-          parent.quotedKeys.add(event.key)
-        }
-
-        break
-      }
-
-      case 'primitive': {
-        if (stack.length === 0) {
-          root = event.value
-        }
-        else {
-          const parent = stack[stack.length - 1]!
-          if (parent.type === 'object') {
-            if (parent.currentKey === undefined) {
-              throw new Error('Primitive event without preceding key in object')
-            }
-            parent.obj[parent.currentKey] = event.value
-            parent.currentKey = undefined
-          }
-          else if (parent.type === 'array') {
-            parent.arr.push(event.value)
-          }
-        }
-
-        break
-      }
-    }
+    applyEvent(state, event)
   }
 
-  if (stack.length !== 0) {
-    throw new Error('Incomplete event stream: stack not empty at end')
-  }
-
-  if (root === undefined) {
-    throw new Error('No root value built from events')
-  }
-
-  return root
+  return finalizeState(state)
+}
+
+// #endregion
+
+// #region Shared event handlers
+
+function applyEvent(state: BuildState, event: JsonStreamEvent): void {
+  const { stack } = state
+
+  switch (event.type) {
+    case 'startObject': {
+      const obj: JsonObject = {}
+      const quotedKeys = new Set<string>()
+
+      if (stack.length === 0) {
+        // Root object
+        stack.push({ type: 'object', obj, quotedKeys })
+      }
+      else {
+        const parent = stack[stack.length - 1]!
+        if (parent.type === 'object') {
+          if (parent.currentKey === undefined) {
+            throw new Error('Object startObject event without preceding key')
+          }
+
+          setOwnProperty(parent.obj, parent.currentKey, obj)
+          parent.currentKey = undefined
+        }
+        else if (parent.type === 'array') {
+          parent.arr.push(obj)
+        }
+
+        stack.push({ type: 'object', obj, quotedKeys })
+      }
+
+      break
+    }
+
+    case 'endObject': {
+      if (stack.length === 0) {
+        throw new Error('Unexpected endObject event')
+      }
+
+      const context = stack.pop()!
+      if (context.type !== 'object') {
+        throw new Error('Mismatched endObject event')
+      }
+
+      // Attach quoted keys metadata if any keys were quoted
+      if (context.quotedKeys.size > 0) {
+        Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
+          value: context.quotedKeys,
+          enumerable: false,
+          writable: false,
+          configurable: false,
+        })
+      }
+
+      if (stack.length === 0) {
+        state.root = context.obj
+      }
+
+      break
+    }
+
+    case 'startArray': {
+      const arr: JsonValue[] = []
+
+      if (stack.length === 0) {
+        // Root array
+        stack.push({ type: 'array', arr })
+      }
+      else {
+        const parent = stack[stack.length - 1]!
+        if (parent.type === 'object') {
+          if (parent.currentKey === undefined) {
+            throw new Error('Array startArray event without preceding key')
+          }
+          setOwnProperty(parent.obj, parent.currentKey, arr)
+          parent.currentKey = undefined
+        }
+        else if (parent.type === 'array') {
+          parent.arr.push(arr)
+        }
+
+        stack.push({ type: 'array', arr })
+      }
+
+      break
+    }
+
+    case 'endArray': {
+      if (stack.length === 0) {
+        throw new Error('Unexpected endArray event')
+      }
+
+      const context = stack.pop()!
+      if (context.type !== 'array') {
+        throw new Error('Mismatched endArray event')
+      }
+
+      if (stack.length === 0) {
+        state.root = context.arr
+      }
+
+      break
+    }
+
+    case 'key': {
+      if (stack.length === 0) {
+        throw new Error('Key event outside of object context')
+      }
+
+      const parent = stack[stack.length - 1]!
+      if (parent.type !== 'object') {
+        throw new Error('Key event in non-object context')
+      }
+
+      parent.currentKey = event.key
+
+      // Track quoted keys for path expansion
+      if (event.wasQuoted) {
+        parent.quotedKeys.add(event.key)
+      }
+
+      break
+    }
+
+    case 'primitive': {
+      if (stack.length === 0) {
+        // Root primitive
+        state.root = event.value
+      }
+      else {
+        const parent = stack[stack.length - 1]!
+        if (parent.type === 'object') {
+          if (parent.currentKey === undefined) {
+            throw new Error('Primitive event without preceding key in object')
+          }
+          setOwnProperty(parent.obj, parent.currentKey, event.value)
+          parent.currentKey = undefined
+        }
+        else if (parent.type === 'array') {
+          parent.arr.push(event.value)
+        }
+      }
+
+      break
+    }
+  }
+}
+
+function finalizeState(state: BuildState): JsonValue {
+  if (state.stack.length !== 0) {
+    throw new Error('Incomplete event stream: stack not empty at end')
+  }
+
+  if (state.root === undefined) {
+    throw new Error('No root value built from events')
+  }
+
+  return state.root
 }
 
 // #endregion

packages/toon/src/decode/expand.ts

@@ -1,7 +1,8 @@
-import type { JsonObject, JsonValue } from '../types'
-import { DOT } from '../constants'
-import { isJsonObject } from '../encode/normalize'
-import { isIdentifierSegment } from '../shared/validation'
+import type { JsonObject, JsonValue } from '../types.ts'
+import { DOT } from '../constants.ts'
+import { isJsonObject } from '../encode/normalize.ts'
+import { getOwnProperty, setOwnProperty } from '../shared/object-utils.ts'
+import { isIdentifierSegment } from '../shared/validation.ts'
 
 // #region Path expansion (safe)
 
@@ -73,7 +74,7 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
       const expandedValue = expandPathsSafe(keyValue, strict)
 
       // Check for conflicts with already-expanded keys
-      if (key in expandedObject) {
+      if (Object.hasOwn(expandedObject, key)) {
         const conflictingValue = expandedObject[key]!
         // If both are objects, try to merge them
         if (canMerge(conflictingValue, expandedValue)) {
@@ -87,12 +88,12 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
             )
           }
           // Non-strict: overwrite (LWW)
-          expandedObject[key] = expandedValue
+          setOwnProperty(expandedObject, key, expandedValue)
         }
       }
       else {
         // No conflict - insert directly
-        expandedObject[key] = expandedValue
+        setOwnProperty(expandedObject, key, expandedValue)
       }
     }
 
@@ -131,12 +132,12 @@ function insertPathSafe(
   // Walk to the penultimate segment, creating objects as needed
   for (let i = 0; i < segments.length - 1; i++) {
     const currentSegment = segments[i]!
-    const segmentValue = currentNode[currentSegment]
+    const segmentValue = getOwnProperty(currentNode, currentSegment)
 
     if (segmentValue === undefined) {
       // Create new intermediate object
       const newObj: JsonObject = {}
-      currentNode[currentSegment] = newObj
+      setOwnProperty(currentNode, currentSegment, newObj)
       currentNode = newObj
     }
     else if (isJsonObject(segmentValue)) {
@@ -152,18 +153,18 @@ function insertPathSafe(
       }
       // Non-strict: overwrite with new object
       const newObj: JsonObject = {}
-      currentNode[currentSegment] = newObj
+      setOwnProperty(currentNode, currentSegment, newObj)
       currentNode = newObj
     }
   }
 
   // Insert at the final segment
   const lastSeg = segments[segments.length - 1]!
-  const destinationValue = currentNode[lastSeg]
+  const destinationValue = getOwnProperty(currentNode, lastSeg)
 
   if (destinationValue === undefined) {
     // No conflict - insert directly
-    currentNode[lastSeg] = value
+    setOwnProperty(currentNode, lastSeg, value)
   }
   else if (canMerge(destinationValue, value)) {
     // Both are objects - deep merge
@@ -177,7 +178,7 @@ function insertPathSafe(
       )
     }
     // Non-strict: overwrite (LWW)
-    currentNode[lastSeg] = value
+    setOwnProperty(currentNode, lastSeg, value)
   }
 }
 
@@ -201,13 +202,14 @@ function mergeObjects(
   strict: boolean,
 ): void {
   for (const [key, sourceValue] of Object.entries(source)) {
-    const targetValue = target[key]
-
-    if (targetValue === undefined) {
+    if (!Object.hasOwn(target, key)) {
       // Key doesn't exist in target - copy it
-      target[key] = sourceValue
+      setOwnProperty(target, key, sourceValue)
+      continue
     }
-    else if (canMerge(targetValue, sourceValue)) {
+
+    const targetValue = target[key] as JsonValue
+    if (canMerge(targetValue, sourceValue)) {
       // Both are objects - recursively merge
       mergeObjects(targetValue as JsonObject, sourceValue as JsonObject, strict)
     }
@@ -219,7 +221,7 @@ function mergeObjects(
         )
       }
       // Non-strict: overwrite (LWW)
-      target[key] = sourceValue
+      setOwnProperty(target, key, sourceValue)
     }
   }
 }

packages/toon/src/decode/parser.ts

@@ -1,46 +1,53 @@
-import type { ArrayHeaderInfo, Delimiter, JsonPrimitive } from '../types'
-import { BACKSLASH, CLOSE_BRACE, CLOSE_BRACKET, COLON, DELIMITERS, DOUBLE_QUOTE, FALSE_LITERAL, NULL_LITERAL, OPEN_BRACE, OPEN_BRACKET, PIPE, TAB, TRUE_LITERAL } from '../constants'
-import { isBooleanOrNullLiteral, isNumericLiteral } from '../shared/literal-utils'
-import { findClosingQuote, findUnquotedChar, unescapeString } from '../shared/string-utils'
+import type { ArrayHeaderInfo, Delimiter, JsonPrimitive } from '../types.ts'
+import { BACKSLASH, CLOSE_BRACE, CLOSE_BRACKET, COLON, DELIMITERS, DOUBLE_QUOTE, FALSE_LITERAL, NULL_LITERAL, OPEN_BRACE, OPEN_BRACKET, PIPE, TAB, TRUE_LITERAL } from '../constants.ts'
+import { isBooleanOrNullLiteral, isNumericLiteral } from '../shared/literal-utils.ts'
+import { findClosingQuote, findUnquotedChar, unescapeString } from '../shared/string-utils.ts'
 
 // #region Array header parsing
 
 export function parseArrayHeaderLine(
   content: string,
   defaultDelimiter: Delimiter,
+  strict: boolean = false,
 ): { header: ArrayHeaderInfo, inlineValues?: string } | undefined {
-  const trimmed = content.trimStart()
+  const trimmedToken = content.trimStart()
 
   // Find the bracket segment, accounting for quoted keys that may contain brackets
   let bracketStart = -1
 
   // For quoted keys, find bracket after closing quote (not inside the quoted string)
-  if (trimmed.startsWith(DOUBLE_QUOTE)) {
-    const closingQuoteIndex = findClosingQuote(trimmed, 0)
+  if (trimmedToken.startsWith(DOUBLE_QUOTE)) {
+    const closingQuoteIndex = findClosingQuote(trimmedToken, 0)
     if (closingQuoteIndex === -1) {
       return
     }
 
-    const afterQuote = trimmed.slice(closingQuoteIndex + 1)
+    const afterQuote = trimmedToken.slice(closingQuoteIndex + 1)
     if (!afterQuote.startsWith(OPEN_BRACKET)) {
       return
     }
 
     // Calculate position in original content and find bracket after the quoted key
-    const leadingWhitespace = content.length - trimmed.length
+    const leadingWhitespace = content.length - trimmedToken.length
     const keyEndIndex = leadingWhitespace + closingQuoteIndex + 1
     bracketStart = content.indexOf(OPEN_BRACKET, keyEndIndex)
   }
   else {
     // Unquoted key - find first bracket
-    bracketStart = content.indexOf(OPEN_BRACKET)
+    bracketStart = findUnquotedChar(content, OPEN_BRACKET)
   }
 
   if (bracketStart === -1) {
     return
   }
 
-  const bracketEnd = content.indexOf(CLOSE_BRACKET, bracketStart)
+  // A header key can't contain an unquoted colon, so this is a key-value line
+  const firstColonIndex = findUnquotedChar(content, COLON)
+  if (firstColonIndex !== -1 && firstColonIndex < bracketStart) {
+    return
+  }
+
+  const bracketEnd = findUnquotedChar(content, CLOSE_BRACKET, bracketStart)
   if (bracketEnd === -1) {
     return
   }
@@ -50,20 +57,43 @@ export function parseArrayHeaderLine(
   let braceEnd = colonIndex
 
   // Check for fields segment (braces come after bracket)
-  const braceStart = content.indexOf(OPEN_BRACE, bracketEnd)
-  if (braceStart !== -1 && braceStart < content.indexOf(COLON, bracketEnd)) {
-    const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
+  const braceStart = findUnquotedChar(content, OPEN_BRACE, bracketEnd)
+  if (braceStart !== -1 && braceStart < findUnquotedChar(content, COLON, bracketEnd)) {
+    const gapBeforeBrace = content.slice(bracketEnd + 1, braceStart)
+    if (gapBeforeBrace !== '') {
+      if (strict) {
+        const trimmedGap = gapBeforeBrace.trim()
+        throw new SyntaxError(trimmedGap === ''
+          ? `Unexpected whitespace between bracket and fields segment`
+          : `Unexpected content "${trimmedGap}" between bracket and fields segment`)
+      }
+      return
+    }
+
+    const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
     if (foundBraceEnd !== -1) {
       braceEnd = foundBraceEnd + 1
     }
   }
 
   // Now find colon after brackets and braces
-  colonIndex = content.indexOf(COLON, Math.max(bracketEnd, braceEnd))
+  colonIndex = findUnquotedChar(content, COLON, Math.max(bracketEnd, braceEnd))
   if (colonIndex === -1) {
     return
   }
 
+  const gapStart = Math.max(bracketEnd + 1, braceEnd)
+  const gapBeforeColon = content.slice(gapStart, colonIndex)
+  if (gapBeforeColon !== '') {
+    if (strict) {
+      const trimmedGap = gapBeforeColon.trim()
+      throw new SyntaxError(trimmedGap === ''
+        ? `Unexpected whitespace between bracket segment and colon`
+        : `Unexpected content "${trimmedGap}" between bracket segment and colon`)
+    }
+    return
+  }
+
   // Extract and parse the key (might be quoted)
   let key: string | undefined
   if (bracketStart > 0) {
@@ -72,15 +102,15 @@ export function parseArrayHeaderLine(
   }
 
   const afterColon = content.slice(colonIndex + 1).trim()
-
   const bracketContent = content.slice(bracketStart + 1, bracketEnd)
 
-  // Try to parse bracket segment
   let parsedBracket: ReturnType<typeof parseBracketSegment>
   try {
     parsedBracket = parseBracketSegment(bracketContent, defaultDelimiter)
   }
-  catch {
+  catch (error) {
+    if (strict)
+      throw error
     return
   }
 
@@ -89,9 +119,17 @@ export function parseArrayHeaderLine(
   // Check for fields segment
   let fields: string[] | undefined
   if (braceStart !== -1 && braceStart < colonIndex) {
-    const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
+    const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
     if (foundBraceEnd !== -1 && foundBraceEnd < colonIndex) {
       const fieldsContent = content.slice(braceStart + 1, foundBraceEnd)
+
+      const mismatchedDelimiter = findUnquotedMismatchedDelimiter(fieldsContent, delimiter)
+      if (mismatchedDelimiter !== undefined) {
+        if (strict)
+          throw new SyntaxError(`Header delimiter mismatch: bracket declares "${formatDelimiter(delimiter)}" but fields segment contains unquoted "${formatDelimiter(mismatchedDelimiter)}"`)
+        return
+      }
+
       fields = parseDelimitedValues(fieldsContent, delimiter).map(field => parseStringLiteral(field.trim()))
     }
   }
@@ -107,6 +145,8 @@ export function parseArrayHeaderLine(
   }
 }
 
+const BRACKET_LENGTH_PATTERN = /^(?:0|[1-9]\d*)$/
+
 export function parseBracketSegment(
   seg: string,
   defaultDelimiter: Delimiter,
@@ -124,12 +164,28 @@ export function parseBracketSegment(
     content = content.slice(0, -1)
   }
 
-  const length = Number.parseInt(content, 10)
-  if (Number.isNaN(length)) {
-    throw new TypeError(`Invalid array length: ${seg}`)
+  if (!BRACKET_LENGTH_PATTERN.test(content)) {
+    throw new SyntaxError(`Invalid array length: "${seg}" (expected non-negative integer with no leading zeros)`)
   }
 
-  return { length, delimiter }
+  return { length: Number.parseInt(content, 10), delimiter }
+}
+
+const DELIMITER_CANDIDATES: readonly Delimiter[] = [',', '\t', '|']
+
+function findUnquotedMismatchedDelimiter(content: string, activeDelimiter: Delimiter): Delimiter | undefined {
+  for (const candidate of DELIMITER_CANDIDATES) {
+    if (candidate === activeDelimiter)
+      continue
+    if (findUnquotedChar(content, candidate) !== -1)
+      return candidate
+  }
+}
+
+function formatDelimiter(delimiter: Delimiter): string {
+  if (delimiter === '\t')
+    return '\\t'
+  return delimiter
 }
 
 // #endregion
@@ -196,37 +252,37 @@ export function mapRowValuesToPrimitives(values: string[]): JsonPrimitive[] {
 // #region Primitive and key parsing
 
 export function parsePrimitiveToken(token: string): JsonPrimitive {
-  const trimmed = token.trim()
+  const trimmedToken = token.trim()
 
   // Empty token
-  if (!trimmed) {
+  if (!trimmedToken) {
     return ''
   }
 
   // Quoted string (if starts with quote, it MUST be properly quoted)
-  if (trimmed.startsWith(DOUBLE_QUOTE)) {
-    return parseStringLiteral(trimmed)
+  if (trimmedToken.startsWith(DOUBLE_QUOTE)) {
+    return parseStringLiteral(trimmedToken)
   }
 
   // Boolean or null literals
-  if (isBooleanOrNullLiteral(trimmed)) {
-    if (trimmed === TRUE_LITERAL)
+  if (isBooleanOrNullLiteral(trimmedToken)) {
+    if (trimmedToken === TRUE_LITERAL)
       return true
-    if (trimmed === FALSE_LITERAL)
+    if (trimmedToken === FALSE_LITERAL)
       return false
-    if (trimmed === NULL_LITERAL)
+    if (trimmedToken === NULL_LITERAL)
       return null
   }
 
   // Numeric literal
-  if (isNumericLiteral(trimmed)) {
-    const parsedNumber = Number.parseFloat(trimmed)
+  if (isNumericLiteral(trimmedToken)) {
+    const parsedNumber = Number.parseFloat(trimmedToken)
     // Normalize negative zero to positive zero
     return Object.is(parsedNumber, -0) ? 0 : parsedNumber
   }
 
   // Unquoted string
-  return trimmed
+  return trimmedToken
 }
 
 export function parseStringLiteral(token: string): string {

packages/toon/src/decode/scanner.ts

@@ -1,5 +1,6 @@
-import type { BlankLineInfo, Depth, ParsedLine } from '../types'
-import { SPACE, TAB } from '../constants'
+import type { BlankLineInfo, Depth, ParsedLine } from '../types.ts'
+import { SPACE, TAB } from '../constants.ts'
+import { ToonDecodeError } from './errors.ts'
 
 // #region Scan state
 
@@ -58,13 +59,17 @@ export function parseLineIncremental(
 
     // Check for tabs in leading whitespace (before actual content)
     if (raw.slice(0, whitespaceEndIndex).includes(TAB)) {
-      throw new SyntaxError(`Line ${lineNumber}: Tabs are not allowed in indentation in strict mode`)
+      throw new ToonDecodeError(
+        'Tabs are not allowed in indentation in strict mode',
+        { line: lineNumber, source: raw },
+      )
     }
 
     // Check for exact multiples of indentSize
     if (indent > 0 && indent % indentSize !== 0) {
-      throw new SyntaxError(
-        `Line ${lineNumber}: Indentation must be exact multiple of ${indentSize}, but found ${indent} spaces`,
+      throw new ToonDecodeError(
+        `Indentation must be exact multiple of ${indentSize}, but found ${indent} spaces`,
+        { line: lineNumber, source: raw },
       )
     }
   }

packages/toon/src/decode/validation.ts

@@ -1,5 +1,7 @@
-import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ParsedLine } from '../types'
-import { COLON, LIST_ITEM_PREFIX } from '../constants'
+import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ParsedLine } from '../types.ts'
+import { COLON, LIST_ITEM_PREFIX } from '../constants.ts'
+import { findUnquotedChar } from '../shared/string-utils.ts'
+import { ToonDecodeError } from './errors.ts'
 
 // #region Count and structure validation
 
@@ -11,9 +13,13 @@ export function assertExpectedCount(
   expected: number,
   itemType: string,
   options: { strict: boolean },
+  line: ParsedLine,
 ): void {
   if (options.strict && actual !== expected) {
-    throw new RangeError(`Expected ${expected} ${itemType}, but got ${actual}`)
+    throw new ToonDecodeError(
+      `Expected ${expected} ${itemType}, but got ${actual}`,
+      { line: line.lineNumber, source: line.raw },
+    )
   }
 }
 
@@ -26,7 +32,10 @@ export function validateNoExtraListItems(
   expectedCount: number,
 ): void {
   if (nextLine?.depth === itemDepth && nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
-    throw new RangeError(`Expected ${expectedCount} list array items, but found more`)
+    throw new ToonDecodeError(
+      `Expected ${expectedCount} list array items, but found more`,
+      { line: nextLine.lineNumber, source: nextLine.raw },
+    )
   }
 }
 
@@ -43,7 +52,10 @@ export function validateNoExtraTabularRows(
     && !nextLine.content.startsWith(LIST_ITEM_PREFIX)
     && isDataRow(nextLine.content, header.delimiter)
   ) {
-    throw new RangeError(`Expected ${header.length} tabular rows, but found more`)
+    throw new ToonDecodeError(
+      `Expected ${header.length} tabular rows, but found more`,
+      { line: nextLine.lineNumber, source: nextLine.raw },
+    )
   }
 }
 
@@ -66,8 +78,9 @@ export function validateNoBlankLinesInRange(
   )
 
   if (firstBlank) {
-    throw new SyntaxError(
-      `Line ${firstBlank.lineNumber}: Blank lines inside ${context} are not allowed in strict mode`,
+    throw new ToonDecodeError(
+      `Blank lines inside ${context} are not allowed in strict mode`,
+      { line: firstBlank.lineNumber },
     )
   }
 }
@@ -79,9 +92,9 @@ export function validateNoBlankLinesInRange(
 /**
  * Checks if a line is a data row (vs a key-value pair) in a tabular array.
  */
-function isDataRow(content: string, delimiter: Delimiter): boolean {
-  const colonPos = content.indexOf(COLON)
-  const delimiterPos = content.indexOf(delimiter)
+export function isDataRow(content: string, delimiter: Delimiter): boolean {
+  const colonPos = findUnquotedChar(content, COLON)
+  const delimiterPos = findUnquotedChar(content, delimiter)
 
   // No colon = definitely a data row
   if (colonPos === -1) {

packages/toon/src/encode/encoders.ts

@@ -1,8 +1,8 @@
-import type { Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ResolvedEncodeOptions } from '../types'
-import { DOT, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants'
-import { tryFoldKeyChain } from './folding'
-import { isArrayOfArrays, isArrayOfObjects, isArrayOfPrimitives, isEmptyObject, isJsonArray, isJsonObject, isJsonPrimitive } from './normalize'
-import { encodeAndJoinPrimitives, encodeKey, encodePrimitive, formatHeader } from './primitives'
+import type { Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ResolvedEncodeOptions } from '../types.ts'
+import { DOT, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants.ts'
+import { tryFoldKeyChain } from './folding.ts'
+import { isArrayOfArrays, isArrayOfObjects, isArrayOfPrimitives, isEmptyObject, isJsonArray, isJsonObject, isJsonPrimitive } from './normalize.ts'
+import { encodeAndJoinPrimitives, encodeKey, encodePrimitive, formatHeader } from './primitives.ts'
 
 // #region Encode normalized JsonValue
 
@@ -101,7 +101,6 @@ export function* encodeKeyValuePairLines(
     }
   }
 
-  // No folding applied - use standard encoding
   const encodedKey = encodeKey(key)
 
   if (isJsonPrimitive(value)) {
@@ -129,8 +128,8 @@ export function* encodeArrayLines(
   options: ResolvedEncodeOptions,
 ): Generator<string> {
   if (value.length === 0) {
-    const header = formatHeader(0, { key, delimiter: options.delimiter })
-    yield indentedLine(depth, header, options.indent)
+    const line = key != null ? `${encodeKey(key)}: []` : '[]'
+    yield indentedLine(depth, line, options.indent)
     return
   }
 
@@ -190,10 +189,10 @@ export function* encodeArrayOfArraysAsListItemsLines(
 export function encodeInlineArrayLine(values: readonly JsonPrimitive[], delimiter: string, prefix?: string): string {
   const header = formatHeader(values.length, { key: prefix, delimiter })
   const joinedValue = encodeAndJoinPrimitives(values, delimiter)
-  // Only add space if there are values
-  if (values.length === 0) {
+
+  if (values.length === 0)
     return header
-  }
+
   return `${header} ${joinedValue}`
 }
 
@@ -242,7 +241,7 @@ export function isTabularArray(
 
     // Check that all header keys exist in the row and all values are primitives
     for (const key of header) {
-      if (!(key in row)) {
+      if (!Object.hasOwn(row, key)) {
         return false
       }
       if (!isJsonPrimitive(row[key])) {
@@ -297,55 +296,63 @@ export function* encodeObjectAsListItemLines(
 
   const entries = Object.entries(obj)
   const [firstKey, firstValue] = entries[0]!
+  const restEntries = entries.slice(1)
+
+  // Check if first field is a tabular array
+  if (isJsonArray(firstValue) && isArrayOfObjects(firstValue)) {
+    const header = extractTabularHeader(firstValue)
+    if (header) {
+      // Tabular array as first field
+      const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter })
+      yield indentedListItem(depth, formattedHeader, options.indent)
+      yield* writeTabularRowsLines(firstValue, header, depth + 2, options)
+
+      if (restEntries.length > 0) {
+        const restObj: JsonObject = Object.fromEntries(restEntries)
+        yield* encodeObjectLines(restObj, depth + 1, options)
+      }
+      return
+    }
+  }
+
   const encodedKey = encodeKey(firstKey)
 
   if (isJsonPrimitive(firstValue)) {
-    yield indentedListItem(depth, `${encodedKey}: ${encodePrimitive(firstValue, options.delimiter)}`, options.indent)
+    // Primitive value: `- key: value`
+    const encodedValue = encodePrimitive(firstValue, options.delimiter)
+    yield indentedListItem(depth, `${encodedKey}: ${encodedValue}`, options.indent)
   }
   else if (isJsonArray(firstValue)) {
-    if (isArrayOfPrimitives(firstValue)) {
-      // Inline format for primitive arrays
-      const arrayPropertyLine = encodeInlineArrayLine(firstValue, options.delimiter, firstKey)
-      yield indentedListItem(depth, arrayPropertyLine, options.indent)
+    if (firstValue.length === 0) {
+      // Empty array: `- key: []`
+      yield indentedListItem(depth, `${encodedKey}: []`, options.indent)
     }
-    else if (isArrayOfObjects(firstValue)) {
-      // Check if array of objects can use tabular format
-      const header = extractTabularHeader(firstValue)
-      if (header) {
-        // Tabular format for uniform arrays of objects
-        const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter })
-        yield indentedListItem(depth, formattedHeader, options.indent)
-        yield* writeTabularRowsLines(firstValue, header, depth + 1, options)
-      }
-      else {
-        // Fall back to list format for non-uniform arrays of objects
-        yield indentedListItem(depth, `${encodedKey}[${firstValue.length}]:`, options.indent)
-        for (const item of firstValue) {
-          yield* encodeObjectAsListItemLines(item, depth + 1, options)
-        }
-      }
+    else if (isArrayOfPrimitives(firstValue)) {
+      // Inline primitive array: `- key[N]: values`
+      const arrayLine = encodeInlineArrayLine(firstValue, options.delimiter)
+      yield indentedListItem(depth, `${encodedKey}${arrayLine}`, options.indent)
     }
     else {
-      // Complex arrays on separate lines (array of arrays, etc.)
-      yield indentedListItem(depth, `${encodedKey}[${firstValue.length}]:`, options.indent)
+      // Non-inline array: `- key[N]:` with items at depth + 2
+      const header = formatHeader(firstValue.length, { delimiter: options.delimiter })
+      yield indentedListItem(depth, `${encodedKey}${header}`, options.indent)
 
-      // Encode array contents at depth + 1
       for (const item of firstValue) {
-        yield* encodeListItemValueLines(item, depth + 1, options)
+        yield* encodeListItemValueLines(item, depth + 2, options)
       }
     }
   }
   else if (isJsonObject(firstValue)) {
+    // Object value: `- key:` with fields at depth + 2
     yield indentedListItem(depth, `${encodedKey}:`, options.indent)
     if (!isEmptyObject(firstValue)) {
       yield* encodeObjectLines(firstValue, depth + 2, options)
     }
   }
 
-  // Remaining entries on indented lines
-  for (let i = 1; i < entries.length; i++) {
-    const [key, value] = entries[i]!
-    yield* encodeKeyValuePairLines(key, value, depth + 1, options)
+  if (restEntries.length > 0) {
+    const restObj: JsonObject = Object.fromEntries(restEntries)
+    yield* encodeObjectLines(restObj, depth + 1, options)
   }
 }
 

packages/toon/src/encode/folding.ts

@@ -1,7 +1,7 @@
-import type { JsonValue, ResolvedEncodeOptions } from '../types'
-import { DOT } from '../constants'
-import { isIdentifierSegment } from '../shared/validation'
-import { isEmptyObject, isJsonObject } from './normalize'
+import type { JsonValue, ResolvedEncodeOptions } from '../types.ts'
+import { DOT } from '../constants.ts'
+import { isIdentifierSegment } from '../shared/validation.ts'
+import { isEmptyObject, isJsonObject } from './normalize.ts'
 
 // #region Key folding helpers
 

packages/toon/src/encode/normalize.ts

@@ -1,4 +1,5 @@
-import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types'
+import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types.ts'
+import { setOwnProperty } from '../shared/object-utils.ts'
 
 // #region Normalization (unknown → JsonValue)
 
@@ -8,6 +9,20 @@ export function normalizeValue(value: unknown): JsonValue {
     return null
   }
 
+  // Objects with toJSON: delegate to its result before host-type normalization
+  if (
+    typeof value === 'object'
+    && value !== null
+    && 'toJSON' in value
+    && typeof value.toJSON === 'function'
+  ) {
+    const next = value.toJSON()
+    // Avoid infinite recursion when toJSON returns the same object
+    if (next !== value) {
+      return normalizeValue(next)
+    }
+  }
+
   // Primitives
   if (typeof value === 'string' || typeof value === 'boolean') {
     return value
@@ -30,7 +45,7 @@ export function normalizeValue(value: unknown): JsonValue {
     if (value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) {
       return Number(value)
     }
-    // Otherwise convert to string (will be unquoted as it looks numeric)
+    // Otherwise convert to string (will be quoted in output)
     return value.toString()
   }
 
@@ -58,15 +73,15 @@ export function normalizeValue(value: unknown): JsonValue {
 
   // Plain object
   if (isPlainObject(value)) {
-    const normalized: Record<string, JsonValue> = {}
+    const encodedValues: Record<string, JsonValue> = {}
 
     for (const key in value) {
-      if (Object.prototype.hasOwnProperty.call(value, key)) {
-        normalized[key] = normalizeValue(value[key])
+      if (Object.hasOwn(value, key)) {
+        setOwnProperty(encodedValues, key, normalizeValue(value[key]))
       }
     }
 
-    return normalized
+    return encodedValues
   }
 
   // Fallback: function, symbol, undefined, or other → null

packages/toon/src/encode/primitives.ts

@@ -1,7 +1,7 @@
-import type { JsonPrimitive } from '../types'
-import { COMMA, DEFAULT_DELIMITER, DOUBLE_QUOTE, NULL_LITERAL } from '../constants'
-import { escapeString } from '../shared/string-utils'
-import { isSafeUnquoted, isValidUnquotedKey } from '../shared/validation'
+import type { JsonPrimitive } from '../types.ts'
+import { COMMA, DEFAULT_DELIMITER, DOUBLE_QUOTE, NULL_LITERAL } from '../constants.ts'
+import { escapeString } from '../shared/string-utils.ts'
+import { isSafeUnquoted, isValidUnquotedKey } from '../shared/validation.ts'
 
 // #region Primitive encoding
 
@@ -67,7 +67,7 @@ export function formatHeader(
 
   let header = ''
 
-  if (key) {
+  if (key != null) {
     header += encodeKey(key)
   }
 

packages/toon/src/encode/replacer.ts

@@ -0,0 +1,127 @@
+import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types.ts'
+import { setOwnProperty } from '../shared/object-utils.ts'
+import { isJsonArray, isJsonObject, normalizeValue } from './normalize.ts'
+
+/**
+ * Applies a replacer function to a `JsonValue` and all its descendants.
+ *
+ * The replacer is called for:
+ * - The root value (with key='', path=[])
+ * - Every object property (with the property name as key)
+ * - Every array element (with the string index as key: '0', '1', etc.)
+ *
+ * @param root - The normalized `JsonValue` to transform
+ * @param replacer - The replacer function to apply
+ * @returns The transformed `JsonValue`
+ */
+export function applyReplacer(root: JsonValue, replacer: EncodeReplacer): JsonValue {
+  // Call replacer on root with empty string key and empty path
+  const replacedRoot = replacer('', root, [])
+
+  // For root, undefined means "no change" (don't omit the root)
+  if (replacedRoot === undefined) {
+    return transformChildren(root, replacer, [])
+  }
+
+  // Normalize the replaced value (in case user returned non-JsonValue)
+  const normalizedRoot = normalizeValue(replacedRoot)
+
+  // Recursively transform children
+  return transformChildren(normalizedRoot, replacer, [])
+}
+
+/**
+ * Recursively transforms the children of a `JsonValue` using the replacer.
+ *
+ * @param value - The value whose children should be transformed
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns The value with transformed children
+ */
+function transformChildren(
+  value: JsonValue,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonValue {
+  if (isJsonObject(value)) {
+    return transformObject(value, replacer, path)
+  }
+
+  if (isJsonArray(value)) {
+    return transformArray(value, replacer, path)
+  }
+
+  // Primitives have no children
+  return value
+}
+
+/**
+ * Transforms an object by applying the replacer to each property.
+ *
+ * @param obj - The object to transform
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns A new object with transformed properties
+ */
+function transformObject(
+  obj: JsonObject,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonObject {
+  const result: Record<string, JsonValue> = {}
+
+  for (const [key, value] of Object.entries(obj)) {
+    // Call replacer with the property key and current path
+    const childPath = [...path, key]
+    const replacedValue = replacer(key, value, childPath)
+
+    // undefined means omit this property
+    if (replacedValue === undefined) {
+      continue
+    }
+
+    // Normalize the replaced value
+    const normalizedValue = normalizeValue(replacedValue)
+
+    // Recursively transform children of the replaced value
+    setOwnProperty(result, key, transformChildren(normalizedValue, replacer, childPath))
+  }
+
+  return result
+}
+
+/**
+ * Transforms an array by applying the replacer to each element.
+ *
+ * @param arr - The array to transform
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns A new array with transformed elements
+ */
+function transformArray(
+  arr: JsonArray,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonArray {
+  const result: JsonValue[] = []
+
+  for (let i = 0; i < arr.length; i++) {
+    const value = arr[i]!
+    // Call replacer with string index (`'0'`, `'1'`, etc.) to match `JSON.stringify` behavior
+    const childPath = [...path, i]
+    const replacedValue = replacer(String(i), value, childPath)
+
+    // undefined means omit this element
+    if (replacedValue === undefined) {
+      continue
+    }
+
+    // Normalize the replaced value
+    const normalizedValue = normalizeValue(replacedValue)
+
+    // Recursively transform children of the replaced value
+    result.push(transformChildren(normalizedValue, replacer, childPath))
+  }
+
+  return result
+}

packages/toon/src/index.ts

@@ -1,18 +1,21 @@
-import type { DecodeOptions, DecodeStreamOptions, EncodeOptions, JsonStreamEvent, JsonValue, ResolvedDecodeOptions, ResolvedEncodeOptions } from './types'
-import { DEFAULT_DELIMITER } from './constants'
-import { decodeStream as decodeStreamCore, decodeStreamSync as decodeStreamSyncCore } from './decode/decoders'
-import { buildValueFromEvents } from './decode/event-builder'
-import { expandPathsSafe } from './decode/expand'
-import { encodeJsonValue } from './encode/encoders'
-import { normalizeValue } from './encode/normalize'
+import type { DecodeOptions, DecodeStreamOptions, EncodeOptions, JsonStreamEvent, JsonValue, ResolvedDecodeOptions, ResolvedEncodeOptions } from './types.ts'
+import { DEFAULT_DELIMITER } from './constants.ts'
+import { decodeStream as decodeStreamCore, decodeStreamSync as decodeStreamSyncCore } from './decode/decoders.ts'
+import { buildValueFromEvents } from './decode/event-builder.ts'
+import { expandPathsSafe } from './decode/expand.ts'
+import { encodeJsonValue } from './encode/encoders.ts'
+import { normalizeValue } from './encode/normalize.ts'
+import { applyReplacer } from './encode/replacer.ts'
 
-export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
+export { DEFAULT_DELIMITER, DELIMITERS } from './constants.ts'
+export { ToonDecodeError } from './decode/errors.ts'
 export type {
   DecodeOptions,
   DecodeStreamOptions,
   Delimiter,
   DelimiterKey,
   EncodeOptions,
+  EncodeReplacer,
   JsonArray,
   JsonObject,
   JsonPrimitive,
@@ -20,7 +23,7 @@ export type {
   JsonValue,
   ResolvedDecodeOptions,
   ResolvedEncodeOptions,
-} from './types'
+} from './types.ts'
 
 /**
  * Encodes a JavaScript value into TOON format string.
@@ -36,9 +39,12 @@ export type {
  * // age: 30
  *
  * encode({ users: [{ id: 1 }, { id: 2 }] })
- * // users[]:
- * //   - id: 1
- * //   - id: 2
+ * // users[2]{id}:
+ * //   1
+ * //   2
+ *
+ * encode({ tags: [] })
+ * // tags: []
  *
  * encode(data, { indent: 4, keyFolding: 'safe' })
  * ```
@@ -59,9 +65,12 @@ export function encode(input: unknown, options?: EncodeOptions): string {
  * decode('name: Alice\nage: 30')
  * // { name: 'Alice', age: 30 }
  *
- * decode('users[]:\n  - id: 1\n  - id: 2')
+ * decode('users[2]:\n  - id: 1\n  - id: 2')
  * // { users: [{ id: 1 }, { id: 2 }] }
  *
+ * decode('tags: []')
+ * // { tags: [] }
+ *
  * decode(toonString, { strict: false, expandPaths: 'safe' })
  * ```
  */
@@ -97,7 +106,13 @@ export function decode(input: string, options?: DecodeOptions): JsonValue {
 export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> {
   const normalizedValue = normalizeValue(input)
   const resolvedOptions = resolveOptions(options)
-  return encodeJsonValue(normalizedValue, resolvedOptions, 0)
+
+  // Apply replacer if provided
+  const maybeReplacedValue = resolvedOptions.replacer
+    ? applyReplacer(normalizedValue, resolvedOptions.replacer)
+    : normalizedValue
+
+  return encodeJsonValue(maybeReplacedValue, resolvedOptions, 0)
 }
 
 /**
@@ -210,6 +225,7 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
     delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
     keyFolding: options?.keyFolding ?? 'off',
     flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
+    replacer: options?.replacer,
   }
 }
 

packages/toon/src/shared/literal-utils.ts

@@ -1,4 +1,6 @@
-import { FALSE_LITERAL, NULL_LITERAL, TRUE_LITERAL } from '../constants'
+import { FALSE_LITERAL, NULL_LITERAL, TRUE_LITERAL } from '../constants.ts'
+
+const NUMERIC_LITERAL_PATTERN = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:e[+-]?\d+)?$/i
 
 export function isBooleanOrNullLiteral(token: string): boolean {
   return token === TRUE_LITERAL || token === FALSE_LITERAL || token === NULL_LITERAL
@@ -14,12 +16,10 @@ export function isNumericLiteral(token: string): boolean {
   if (!token)
     return false
 
-  // Must not have leading zeros (except for `"0"` itself or decimals like `"0.5"`)
-  if (token.length > 1 && token[0] === '0' && token[1] !== '.') {
+  // Enforce JSON-like grammar with no forbidden leading zeros
+  if (!NUMERIC_LITERAL_PATTERN.test(token))
     return false
-  }
 
-  // Check if it's a valid number
   const numericValue = Number(token)
   return !Number.isNaN(numericValue) && Number.isFinite(numericValue)
 }

packages/toon/src/shared/object-utils.ts

@@ -0,0 +1,33 @@
+import type { JsonObject, JsonValue } from '../types.ts'
+
+/**
+ * Reads an own data property, treating inherited and absent keys alike.
+ *
+ * @remarks
+ * Keys such as `__proto__` must not resolve through the prototype chain.
+ */
+export function getOwnProperty(target: JsonObject, key: string): JsonValue | undefined {
+  return Object.hasOwn(target, key) ? target[key] : undefined
+}
+
+/**
+ * Assigns an own data property without invoking inherited accessors.
+ *
+ * @remarks
+ * Plain assignment of `__proto__` would hit the `Object.prototype` setter and
+ * corrupt the prototype chain; `defineProperty` avoids that but is markedly
+ * slower, so every other key takes plain assignment.
+ */
+export function setOwnProperty(target: JsonObject, key: string, value: JsonValue): void {
+  if (key === '__proto__') {
+    Object.defineProperty(target, key, {
+      value,
+      enumerable: true,
+      writable: true,
+      configurable: true,
+    })
+    return
+  }
+
+  target[key] = value
+}

packages/toon/src/shared/string-utils.ts

@@ -1,10 +1,11 @@
-import { BACKSLASH, CARRIAGE_RETURN, DOUBLE_QUOTE, NEWLINE, TAB } from '../constants'
+import { BACKSLASH, CARRIAGE_RETURN, DOUBLE_QUOTE, NEWLINE, TAB } from '../constants.ts'
 
 /**
  * Escapes special characters in a string for encoding.
  *
  * @remarks
  * Handles backslashes, quotes, newlines, carriage returns, and tabs.
+ * Other U+0000–U+001F control characters are emitted as `\uXXXX`.
  */
 export function escapeString(value: string): string {
   return value
@@ -13,13 +14,16 @@ export function escapeString(value: string): string {
     .replace(/\n/g, `${BACKSLASH}n`)
     .replace(/\r/g, `${BACKSLASH}r`)
     .replace(/\t/g, `${BACKSLASH}t`)
+    // eslint-disable-next-line no-control-regex
+    .replace(/[\u0000-\u001F]/g, c => `${BACKSLASH}u${c.charCodeAt(0).toString(16).padStart(4, '0')}`)
 }
 
 /**
  * Unescapes a string by processing escape sequences.
  *
  * @remarks
- * Handles `\n`, `\t`, `\r`, `\\`, and `\"` escape sequences.
+ * Handles `\n`, `\t`, `\r`, `\\`, `\"`, and `\uXXXX` escape sequences.
+ * Lone surrogates in `\uXXXX` are rejected.
  */
 export function unescapeString(value: string): string {
   let unescaped = ''
@@ -57,6 +61,22 @@ export function unescapeString(value: string): string {
         i += 2
         continue
       }
+      if (next === 'u') {
+        if (i + 6 > value.length) {
+          throw new SyntaxError(`Invalid escape sequence: truncated \\u escape at "${value.slice(i, i + 6)}"`)
+        }
+        const hex = value.slice(i + 2, i + 6)
+        if (!/^[0-9a-f]{4}$/i.test(hex)) {
+          throw new SyntaxError(`Invalid escape sequence: \\u must be followed by 4 hex digits, got "${hex}"`)
+        }
+        const codeUnit = Number.parseInt(hex, 16)
+        if (codeUnit >= 0xD800 && codeUnit <= 0xDFFF) {
+          throw new SyntaxError(`Invalid escape sequence: \\u${hex} is a lone surrogate; supplementary code points MUST appear as literal UTF-8`)
+        }
+        unescaped += String.fromCodePoint(codeUnit)
+        i += 6
+        continue
+      }
 
       throw new SyntaxError(`Invalid escape sequence: \\${next}`)
     }

packages/toon/src/shared/validation.ts

@@ -1,5 +1,8 @@
-import { DEFAULT_DELIMITER, LIST_ITEM_MARKER } from '../constants'
-import { isBooleanOrNullLiteral } from './literal-utils'
+import { DEFAULT_DELIMITER, LIST_ITEM_MARKER } from '../constants.ts'
+import { isBooleanOrNullLiteral } from './literal-utils.ts'
+
+const NUMERIC_LIKE_PATTERN = /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i
+const LEADING_ZERO_PATTERN = /^0\d+$/
 
 /**
  * Checks if a key can be used without quotes.
@@ -68,8 +71,9 @@ export function isSafeUnquoted(value: string, delimiter: string = DEFAULT_DELIMI
     return false
   }
 
-  // Check for control characters (newline, carriage return, tab - always need quoting/escaping)
-  if (/[\n\r\t]/.test(value)) {
+  // Check for control characters (any U+0000–U+001F always need quoting/escaping)
+  // eslint-disable-next-line no-control-regex
+  if (/[\u0000-\u001F]/.test(value)) {
     return false
   }
 
@@ -93,5 +97,5 @@ export function isSafeUnquoted(value: string, delimiter: string = DEFAULT_DELIMI
  * Match numbers like `42`, `-3.14`, `1e-6`, `05`, etc.
  */
 function isNumericLike(value: string): boolean {
-  return /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i.test(value) || /^0\d+$/.test(value)
+  return NUMERIC_LIKE_PATTERN.test(value) || LEADING_ZERO_PATTERN.test(value)
 }

packages/toon/src/types.ts

@@ -1,6 +1,6 @@
 // #region JSON types
 
-import type { Delimiter, DelimiterKey } from './constants'
+import type { Delimiter, DelimiterKey } from './constants.ts'
 
 export type JsonPrimitive = string | number | boolean | null
 export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined }
@@ -13,6 +13,42 @@ export type JsonValue = JsonPrimitive | JsonObject | JsonArray
 
 export type { Delimiter, DelimiterKey }
 
+/**
+ * A function that transforms or filters values during encoding.
+ *
+ * Called for every value (root, object properties, array elements) during the encoding process.
+ * Similar to `JSON.stringify`'s replacer, but with path tracking.
+ *
+ * @param key - The property key or array index (as string). Empty string (`''`) for root value.
+ * @param value - The normalized `JsonValue` at this location.
+ * @param path - Array representing the path from root to this value.
+ *
+ * @returns The replacement value (will be normalized again), or `undefined` to omit.
+ *          For root value, returning `undefined` means "no change" (don't omit root).
+ *
+ * @example
+ * ```ts
+ * // Remove password fields
+ * const replacer = (key, value) => {
+ *   if (key === 'password') return undefined
+ *   return value
+ * }
+ *
+ * // Add timestamps
+ * const replacer = (key, value, path) => {
+ *   if (path.length === 0 && typeof value === 'object' && value !== null) {
+ *     return { ...value, _timestamp: Date.now() }
+ *   }
+ *   return value
+ * }
+ * ```
+ */
+export type EncodeReplacer = (
+  key: string,
+  value: JsonValue,
+  path: readonly (string | number)[],
+) => unknown
+
 export interface EncodeOptions {
   /**
    * Number of spaces per indentation level.
@@ -38,9 +74,16 @@ export interface EncodeOptions {
    * @default Infinity
    */
   flattenDepth?: number
+  /**
+   * A function to transform or filter values during encoding.
+   * Called for the root value and every nested property/element.
+   * Return `undefined` to omit properties/elements (root cannot be omitted).
+   * @default undefined
+   */
+  replacer?: EncodeReplacer
 }
 
-export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>>
+export type ResolvedEncodeOptions = Readonly<Required<Omit<EncodeOptions, 'replacer'>>> & Pick<EncodeOptions, 'replacer'>
 
 // #endregion
 

packages/toon/test/decode-errors.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from 'vitest'
+import { decode, ToonDecodeError } from '../src/index'
+
+describe('toonDecodeError line context', () => {
+  it('reports line number when a parent key is missing its colon', () => {
+    const error = captureDecodeError('_meta\n  version: "1.0"\n  name: test\n')
+
+    expect(error).toBeInstanceOf(SyntaxError)
+    expect(error.line).toBe(1)
+    expect(error.source).toBe('_meta')
+    expect(error.message).toMatch(/^Line 1: /)
+    expect(error.message).toMatch(/missing colon/i)
+  })
+
+  it('reports the line of the missing-colon error in nested context', () => {
+    const error = captureDecodeError('wrapper:\n  inner\n    version: "1.0"\n')
+
+    expect(error.line).toBe(2)
+  })
+
+  it('includes line number when a list array has too few items', () => {
+    const error = captureDecodeError('_meta:\n  version: "1.0"\n\nrules[3]:\n  - first\n')
+
+    expect(error.line).toBeDefined()
+    expect(error.message).toMatch(/^Line \d+: /)
+    expect(error.message).toMatch(/3.*1/)
+    expect(error.message).toMatch(/list/i)
+  })
+
+  it('includes line number when a tabular row count is wrong', () => {
+    const error = captureDecodeError('rules[3]{id,rule}:\n  R1,first\n  R2,second\n')
+
+    expect(error.line).toBeDefined()
+    expect(error.message).toMatch(/^Line \d+: /)
+    expect(error.message).toMatch(/3.*2/)
+    expect(error.message).toMatch(/tabular|rows?/i)
+  })
+
+  it('reports a malformed array length marker with line and source', () => {
+    const error = captureDecodeError('rules[-1]:\n  - a\n')
+
+    expect(error).toBeInstanceOf(SyntaxError)
+    expect(error.line).toBe(1)
+    expect(error.source).toBe('rules[-1]:')
+    expect(error.message).toMatch(/^Line 1: /)
+    expect(error.message).toMatch(/invalid array length/i)
+  })
+
+  it('reports indentation errors with line and source', () => {
+    const error = captureDecodeError('a:\n   b: 1\n')
+
+    expect(error.line).toBe(2)
+    expect(error.source).toBe('   b: 1')
+  })
+
+  it('attaches line context to errors raised during value parsing', () => {
+    const error = captureDecodeError('name: alice\ngreeting: "hello\n')
+
+    expect(error.line).toBe(2)
+    expect(error.source).toBe('greeting: "hello')
+    expect(error.message).toMatch(/^Line 2: /)
+    expect(error.message).toMatch(/unterminated|closing quote/i)
+  })
+
+  it('reports tabs in indentation with line and source', () => {
+    const error = captureDecodeError('a:\n\tb: 1\n')
+
+    expect(error.line).toBe(2)
+    expect(error.source).toBe('\tb: 1')
+    expect(error.message).toMatch(/^Line 2: /)
+    expect(error.message).toMatch(/tabs?/i)
+  })
+
+  it('reports blank lines inside an array with the line number of the blank', () => {
+    const error = captureDecodeError('rules[3]{id,rule}:\n  R1,first\n\n  R2,second\n  R3,third\n')
+
+    expect(error.line).toBe(3)
+    expect(error.message).toMatch(/^Line 3: /)
+    expect(error.message).toMatch(/blank lines?/i)
+  })
+
+  it('points to the first extra item when an array exceeds its declared count', () => {
+    const error = captureDecodeError('items[2]:\n  - a\n  - b\n  - c\n')
+
+    expect(error.line).toBe(4)
+    expect(error.source).toBe('  - c')
+    expect(error.message).toMatch(/^Line 4: /)
+    expect(error.message).toMatch(/list/i)
+    expect(error.message).toMatch(/2|more/i)
+  })
+
+  it('points to the offending row when a tabular row width does not match the field count', () => {
+    const error = captureDecodeError('rules[2]{id,rule,priority}:\n  R1,first\n  R2,second,high\n')
+
+    expect(error.line).toBe(2)
+    expect(error.source).toBe('  R1,first')
+    expect(error.message).toMatch(/^Line 2: /)
+    expect(error.message).toMatch(/3.*2/)
+    expect(error.message).toMatch(/row|tabular/i)
+  })
+})
+
+function captureDecodeError(input: string): ToonDecodeError {
+  try {
+    decode(input)
+  }
+  catch (error) {
+    if (error instanceof ToonDecodeError)
+      return error
+    throw error
+  }
+  throw new Error('Expected decode to throw ToonDecodeError, but it returned normally')
+}

packages/toon/test/decode-security.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest'
+import { decode } from '../src/index'
+
+describe('decode security hardening', () => {
+  const prototypeKey = '__proto__'
+
+  it('keeps direct __proto__ keys as own data properties', () => {
+    const marker = '__toonDirectPolluted'
+
+    try {
+      const decoded = decode(`__proto__:\n  ${marker}: true\n`) as Record<string, any>
+
+      expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+      expect(decoded[prototypeKey][marker]).toBe(true)
+      expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[marker]
+    }
+  })
+
+  it.each([
+    ['primitive', '__proto__: true', true],
+    ['array', '__proto__[2]: 1,2', [1, 2]],
+  ])('keeps direct __proto__ %s values as own data properties', (_name, input, expected) => {
+    const decoded = decode(input) as Record<string, any>
+
+    expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+    expect(decoded[prototypeKey]).toEqual(expected)
+  })
+
+  it('does not follow Object.prototype during dotted path expansion', () => {
+    const marker = '__toonExpandedPolluted'
+
+    try {
+      const decoded = decode(`payload.__proto__.${marker}: true\n`, {
+        expandPaths: 'safe',
+      }) as Record<string, any>
+
+      expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
+      expect(decoded.payload[prototypeKey][marker]).toBe(true)
+      expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[marker]
+    }
+  })
+
+  it('keeps constructor.prototype expansion within own properties', () => {
+    const marker = '__toonConstructorPolluted'
+
+    try {
+      const decoded = decode(`constructor.prototype.${marker}: true\n`, {
+        expandPaths: 'safe',
+      }) as Record<string, any>
+
+      expect(Object.hasOwn(decoded, 'constructor')).toBe(true)
+      expect(Object.hasOwn(decoded.constructor, 'prototype')).toBe(true)
+      expect(decoded.constructor.prototype[marker]).toBe(true)
+      expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[marker]
+    }
+  })
+
+  it('safely merges an expanded __proto__ path with a direct object', () => {
+    const decoded = decode(`__proto__.first: true\n__proto__:\n  second: true\n`, {
+      expandPaths: 'safe',
+    }) as Record<string, any>
+
+    expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+    expect(decoded[prototypeKey]).toEqual({ first: true, second: true })
+  })
+
+  it('safely overwrites a primitive __proto__ conflict in non-strict mode', () => {
+    const decoded = decode(`__proto__: 1\n__proto__.second: true\n`, {
+      expandPaths: 'safe',
+      strict: false,
+    }) as Record<string, any>
+
+    expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+    expect(decoded[prototypeKey]).toEqual({ second: true })
+  })
+})

packages/toon/test/decode.test.ts

@@ -13,7 +13,7 @@ import rootForm from '@toon-format/spec/tests/fixtures/decode/root-form.json'
 import validationErrors from '@toon-format/spec/tests/fixtures/decode/validation-errors.json'
 import whitespace from '@toon-format/spec/tests/fixtures/decode/whitespace.json'
 import { describe, expect, it } from 'vitest'
-import { decode } from '../src/index'
+import { decode, encode } from '../src/index'
 
 const fixtureFiles = [
   primitives,
@@ -47,3 +47,42 @@ for (const fixtures of fixtureFiles) {
     }
   })
 }
+
+describe('quoted content opacity (round-trip)', () => {
+  const cases: [name: string, value: unknown][] = [
+    ['bracket segment then colon in scalar', { a: '[2]: x' }],
+    ['bracket segment then colon, count matches length', { a: '[1]: x' }],
+    ['bracketed index with a later colon', { content: 'accept (process.argv[2]) and greet: name' }],
+    ['brackets, braces, colons, commas and pipes in a scalar', { s: '{k: v}, [0]: a | b' }],
+    ['structural characters in tabular cells', { rows: [{ id: 1, note: '[2]: x' }, { id: 2, note: 'a: b, c' }] }],
+    ['structural characters in list items', { xs: ['[2]: x', 'a: b', 'p | q'] }],
+    ['bracket-shaped text inside a quoted key', { 'k [2]: y': 1 }],
+    ['quoted key opens an array header', { 'quoted key': [1, 2] }],
+    ['embedded quoted bracket in a key that opens an array header', { 'has "[2]" in it': [3, 4, 5] }],
+  ]
+
+  for (const [name, value] of cases) {
+    it(`round-trips ${name}`, () => {
+      expect(decode(encode(value))).toEqual(value)
+    })
+  }
+
+  it('round-trips the Nostr event with a bracketed index and later colon in content', () => {
+    const event = {
+      id: '68f5c016e5a3128d7af740e088fc5d94e56edda4205fffa56aa3d58fe6bb55ee',
+      pubkey: '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5',
+      kind: 1621,
+      content: 'Currently index.js always prints "Hello, world!". It should accept an optional name as the first CLI argument (process.argv[2]) and greet that name instead, falling back to "world" when no argument is given. Example: `node index.js Ada` -> "Hello, Ada!".',
+      tags: [
+        ['a', '30617:3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5:hello-compare-rig'],
+        ['p', '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5'],
+        ['subject', 'greeting should accept a name argument'],
+        ['t', 'enhancement'],
+      ],
+      created_at: 1783027036,
+      sig: '3220fa9dcb7af14b9a970f5d25f4ac13ebb4848c91a83bf496cdef17a839d4475773defb38a20d64b250227eb5aa457eb77358c5f32c4c7935397a3430b102db',
+    }
+
+    expect(decode(encode(event))).toEqual(event)
+  })
+})

packages/toon/test/decodeStream.test.ts

@@ -1,10 +1,11 @@
+import type { JsonStreamEvent } from '../src/index'
 import { describe, expect, it } from 'vitest'
-import { buildValueFromEvents } from '../src/decode/event-builder'
-import { decode, decodeFromLines, decodeStreamSync } from '../src/index'
+import { buildValueFromEvents, buildValueFromEventsAsync } from '../src/decode/event-builder'
+import { decode, decodeFromLines, decodeStream, decodeStreamSync } from '../src/index'
 
 describe('streaming decode', () => {
   describe('decodeStreamSync', () => {
-    it('should decode simple object', () => {
+    it('decodes simple object', () => {
       const input = 'name: Alice\nage: 30'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -19,7 +20,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode nested object', () => {
+    it('decodes nested object', () => {
       const input = 'user:\n  name: Alice\n  age: 30'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -37,7 +38,17 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode inline primitive array', () => {
+    it('materializes __proto__ as an own property', () => {
+      const prototypeKey = '__proto__'
+      const lines = ['__proto__:', '  safe: true']
+      const result = buildValueFromEvents(decodeStreamSync(lines)) as Record<string, unknown>
+
+      expect(Object.hasOwn(result, prototypeKey)).toBe(true)
+      expect(result[prototypeKey]).toEqual({ safe: true })
+      expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
+    })
+
+    it('decodes inline primitive array', () => {
       const input = 'scores[3]: 95, 87, 92'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -54,7 +65,23 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode list array', () => {
+    it('decodes inline array with empty string key', () => {
+      const input = '""[2]: 1,2'
+      const lines = input.split('\n')
+      const events = Array.from(decodeStreamSync(lines))
+
+      expect(events).toEqual([
+        { type: 'startObject' },
+        { type: 'key', key: '' },
+        { type: 'startArray', length: 2 },
+        { type: 'primitive', value: 1 },
+        { type: 'primitive', value: 2 },
+        { type: 'endArray' },
+        { type: 'endObject' },
+      ])
+    })
+
+    it('decodes list array', () => {
       const input = 'items[2]:\n  - Apple\n  - Banana'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -70,7 +97,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode tabular array', () => {
+    it('decodes tabular array', () => {
       const input = 'users[2]{name,age}:\n  Alice, 30\n  Bob, 25'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -96,7 +123,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode root primitive', () => {
+    it('decodes root primitive', () => {
       const input = 'Hello World'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -106,7 +133,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode root array', () => {
+    it('decodes root array', () => {
       const input = '[2]:\n  - Apple\n  - Banana'
       const lines = input.split('\n')
       const events = Array.from(decodeStreamSync(lines))
@@ -119,7 +146,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should decode empty input as empty object', () => {
+    it('decodes empty input as empty object', () => {
       const lines: string[] = []
       const events = Array.from(decodeStreamSync(lines))
 
@@ -129,7 +156,7 @@ describe('streaming decode', () => {
       ])
     })
 
-    it('should throw on expandPaths option', () => {
+    it('throws on expandPaths option', () => {
       const input = 'name: Alice'
       const lines = input.split('\n')
 
@@ -137,7 +164,7 @@ describe('streaming decode', () => {
         .toThrow('expandPaths is not supported in streaming decode')
     })
 
-    it('should enforce strict mode validation', () => {
+    it('enforces strict mode validation', () => {
       const input = 'items[2]:\n  - Apple'
       const lines = input.split('\n')
 
@@ -145,11 +172,10 @@ describe('streaming decode', () => {
         .toThrow()
     })
 
-    it('should allow count mismatch in non-strict mode', () => {
+    it('allows count mismatch in non-strict mode', () => {
       const input = 'items[2]:\n  - Apple'
       const lines = input.split('\n')
 
-      // Should not throw in non-strict mode
       const events = Array.from(decodeStreamSync(lines, { strict: false }))
 
       expect(events).toBeDefined()
@@ -157,8 +183,122 @@ describe('streaming decode', () => {
     })
   })
 
+  describe('decodeStream (async)', () => {
+    const equivalenceCases = [
+      { name: 'simple object', input: 'name: Alice\nage: 30' },
+      { name: 'nested object', input: 'user:\n  name: Alice\n  age: 30' },
+      { name: 'tabular array', input: 'users[2]{name,age}:\n  Alice, 30\n  Bob, 25' },
+      { name: 'list array', input: 'items[2]:\n  - Apple\n  - Banana' },
+      { name: 'root primitive', input: 'Hello World' },
+      { name: 'root array', input: '[2]:\n  - Apple\n  - Banana' },
+      { name: 'empty input', input: '' },
+    ]
+
+    for (const { name, input } of equivalenceCases) {
+      it(`emits the same events as decodeStreamSync for ${name}`, async () => {
+        const lines = input === '' ? [] : input.split('\n')
+        const syncResult = Array.from(decodeStreamSync(lines))
+        const asyncResult = await collect(decodeStream(asyncLines(lines)))
+        expect(asyncResult).toEqual(syncResult)
+      })
+    }
+
+    it('accepts a sync iterable as source', async () => {
+      const lines = ['name: Alice', 'age: 30']
+      const events = await collect(decodeStream(lines))
+
+      expect(events).toEqual(Array.from(decodeStreamSync(lines)))
+    })
+
+    it('locates the value after an escaped quoted key containing a colon', async () => {
+      const lines = ['"\\t:x": v']
+      const events = await collect(decodeStream(asyncLines(lines)))
+
+      expect(events).toEqual([
+        { type: 'startObject' },
+        { type: 'key', key: '\t:x', wasQuoted: true },
+        { type: 'primitive', value: 'v' },
+        { type: 'endObject' },
+      ])
+      expect(events).toEqual(Array.from(decodeStreamSync(lines)))
+    })
+
+    it('keeps a quoted bracket-then-colon scalar opaque, matching decodeStreamSync', async () => {
+      const lines = ['a: "[1]: x"']
+      const events = await collect(decodeStream(asyncLines(lines)))
+
+      expect(events).toEqual([
+        { type: 'startObject' },
+        { type: 'key', key: 'a' },
+        { type: 'primitive', value: '[1]: x' },
+        { type: 'endObject' },
+      ])
+      expect(events).toEqual(Array.from(decodeStreamSync(lines)))
+    })
+
+    it('keeps an unquoted bracket-colon scalar whole, matching decodeStreamSync', async () => {
+      const lines = ['key: foo [2]: bar']
+      const events = await collect(decodeStream(asyncLines(lines)))
+
+      expect(events).toEqual([
+        { type: 'startObject' },
+        { type: 'key', key: 'key' },
+        { type: 'primitive', value: 'foo [2]: bar' },
+        { type: 'endObject' },
+      ])
+      expect(events).toEqual(Array.from(decodeStreamSync(lines)))
+    })
+
+    it('keeps a colon-bearing value such as a URL intact', async () => {
+      const lines = ['a: http://x']
+      const events = await collect(decodeStream(asyncLines(lines)))
+
+      expect(events).toEqual([
+        { type: 'startObject' },
+        { type: 'key', key: 'a' },
+        { type: 'primitive', value: 'http://x' },
+        { type: 'endObject' },
+      ])
+      expect(events).toEqual(Array.from(decodeStreamSync(lines)))
+    })
+
+    it('materializes __proto__ as an own property', async () => {
+      const prototypeKey = '__proto__'
+      const lines = ['__proto__:', '  safe: true']
+      const events = await collect(decodeStream(asyncLines(lines)))
+      const result = await buildValueFromEventsAsync(asyncEvents(events)) as Record<string, unknown>
+
+      expect(Object.hasOwn(result, prototypeKey)).toBe(true)
+      expect(result[prototypeKey]).toEqual({ safe: true })
+      expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
+    })
+
+    it('rejects expandPaths option', async () => {
+      const lines = ['name: Alice']
+
+      await expect(async () => {
+        await collect(decodeStream(asyncLines(lines), { expandPaths: 'safe' } as any))
+      }).rejects.toThrow('expandPaths is not supported in streaming decode')
+    })
+
+    it('enforces strict mode validation', async () => {
+      const lines = ['items[2]:', '  - Apple']
+
+      await expect(async () => {
+        await collect(decodeStream(asyncLines(lines), { strict: true }))
+      }).rejects.toThrow()
+    })
+
+    it('allows count mismatch in non-strict mode', async () => {
+      const lines = ['items[2]:', '  - Apple']
+      const events = await collect(decodeStream(asyncLines(lines), { strict: false }))
+
+      expect(events[0]).toEqual({ type: 'startObject' })
+    })
+  })
+
   describe('buildValueFromEvents', () => {
-    it('should build object from events', () => {
+    it('builds object from events', () => {
       const events = [
         { type: 'startObject' as const },
         { type: 'key' as const, key: 'name' },
@@ -173,7 +313,7 @@ describe('streaming decode', () => {
       expect(result).toEqual({ name: 'Alice', age: 30 })
     })
 
-    it('should build nested object from events', () => {
+    it('builds nested object from events', () => {
       const events = [
         { type: 'startObject' as const },
         { type: 'key' as const, key: 'user' },
@@ -189,7 +329,7 @@ describe('streaming decode', () => {
       expect(result).toEqual({ user: { name: 'Alice' } })
     })
 
-    it('should build array from events', () => {
+    it('builds array from events', () => {
       const events = [
         { type: 'startArray' as const, length: 3 },
         { type: 'primitive' as const, value: 1 },
@@ -203,7 +343,7 @@ describe('streaming decode', () => {
       expect(result).toEqual([1, 2, 3])
     })
 
-    it('should build primitive from events', () => {
+    it('builds primitive from events', () => {
       const events = [
         { type: 'primitive' as const, value: 'Hello' },
       ]
@@ -213,11 +353,10 @@ describe('streaming decode', () => {
       expect(result).toEqual('Hello')
     })
 
-    it('should throw on incomplete event stream', () => {
+    it('throws on incomplete event stream', () => {
       const events = [
         { type: 'startObject' as const },
         { type: 'key' as const, key: 'name' },
-        // Missing primitive and endObject
       ]
 
       expect(() => buildValueFromEvents(events))
@@ -225,119 +364,115 @@ describe('streaming decode', () => {
     })
   })
 
+  describe('buildValueFromEventsAsync', () => {
+    it('matches buildValueFromEvents for representative shapes', async () => {
+      const cases: JsonStreamEvent[][] = [
+        [
+          { type: 'startObject' },
+          { type: 'key', key: 'name' },
+          { type: 'primitive', value: 'Alice' },
+          { type: 'endObject' },
+        ],
+        [
+          { type: 'startArray', length: 2 },
+          { type: 'primitive', value: 1 },
+          { type: 'primitive', value: 2 },
+          { type: 'endArray' },
+        ],
+        [
+          { type: 'primitive', value: 'Hello' },
+        ],
+      ]
+
+      for (const events of cases) {
+        const syncResult = buildValueFromEvents(events)
+        const asyncResult = await buildValueFromEventsAsync(asyncEvents(events))
+        expect(asyncResult).toEqual(syncResult)
+      }
+    })
+
+    it('throws on incomplete event stream', async () => {
+      const events = [
+        { type: 'startObject' as const },
+        { type: 'key' as const, key: 'name' },
+      ]
+
+      await expect(buildValueFromEventsAsync(asyncEvents(events)))
+        .rejects
+        .toThrow('Incomplete event stream')
+    })
+  })
+
   describe('decodeFromLines', () => {
-    it('should produce same result as decode', () => {
+    it('produces same result as decode', () => {
       const input = 'name: Alice\nage: 30\nscores[3]: 95, 87, 92'
       const lines = input.split('\n')
 
-      const fromLines = decodeFromLines(lines)
-      const fromString = decode(input)
-
-      expect(fromLines).toEqual(fromString)
+      expect(decodeFromLines(lines)).toEqual(decode(input))
     })
 
-    it('should support expandPaths option', () => {
-      const input = 'user.name: Alice\nuser.age: 30'
-      const lines = input.split('\n')
+    it('supports expandPaths option', () => {
+      const lines = ['user.name: Alice', 'user.age: 30']
 
-      const result = decodeFromLines(lines, { expandPaths: 'safe' })
-
-      expect(result).toEqual({
-        user: {
-          name: 'Alice',
-          age: 30,
-        },
+      expect(decodeFromLines(lines, { expandPaths: 'safe' })).toEqual({
+        user: { name: 'Alice', age: 30 },
       })
     })
 
-    it('should handle complex nested structures', () => {
+    it('handles list item objects with empty string keyed tabular fields', () => {
       const input = [
-        'users[2]:',
-        '  - name: Alice',
-        '    scores[3]: 95, 87, 92',
-        '  - name: Bob',
-        '    scores[3]: 88, 91, 85',
+        'items[1]:',
+        '  - ""[2]{a}:',
+        '      1',
+        '      2',
       ].join('\n')
 
-      const fromLines = decodeFromLines(input.split('\n'))
-      const fromString = decode(input)
-
-      expect(fromLines).toEqual(fromString)
-      expect(fromLines).toEqual({
-        users: [
-          { name: 'Alice', scores: [95, 87, 92] },
-          { name: 'Bob', scores: [88, 91, 85] },
-        ],
-      })
-    })
-
-    it('should handle tabular arrays', () => {
-      const input = [
-        'users[3]{name,age,city}:',
-        '  Alice, 30, NYC',
-        '  Bob, 25, LA',
-        '  Charlie, 35, SF',
-      ].join('\n')
-
-      const fromLines = decodeFromLines(input.split('\n'))
-      const fromString = decode(input)
-
-      expect(fromLines).toEqual(fromString)
-      expect(fromLines).toEqual({
-        users: [
-          { name: 'Alice', age: 30, city: 'NYC' },
-          { name: 'Bob', age: 25, city: 'LA' },
-          { name: 'Charlie', age: 35, city: 'SF' },
-        ],
+      expect(decodeFromLines(input.split('\n'))).toEqual({
+        items: [{ '': [{ a: 1 }, { a: 2 }] }],
       })
     })
   })
 
   describe('streaming equivalence', () => {
-    // Test that streaming produces same results as non-streaming for various inputs
     const testCases = [
-      {
-        name: 'simple object',
-        input: 'name: Alice\nage: 30',
-      },
-      {
-        name: 'nested objects',
-        input: 'user:\n  profile:\n    name: Alice\n    age: 30',
-      },
-      {
-        name: 'mixed structures',
-        input: 'name: Alice\nscores[3]: 95, 87, 92\naddress:\n  city: NYC\n  zip: 10001',
-      },
-      {
-        name: 'list array with objects',
-        input: 'users[2]:\n  - name: Alice\n    age: 30\n  - name: Bob\n    age: 25',
-      },
-      {
-        name: 'root primitive number',
-        input: '42',
-      },
-      {
-        name: 'root primitive string',
-        input: 'Hello World',
-      },
-      {
-        name: 'root primitive boolean',
-        input: 'true',
-      },
-      {
-        name: 'root primitive null',
-        input: 'null',
-      },
+      { name: 'simple object', input: 'name: Alice\nage: 30' },
+      { name: 'nested objects', input: 'user:\n  profile:\n    name: Alice\n    age: 30' },
+      { name: 'mixed structures', input: 'name: Alice\nscores[3]: 95, 87, 92\naddress:\n  city: NYC\n  zip: 10001' },
+      { name: 'list array with objects', input: 'users[2]:\n  - name: Alice\n    age: 30\n  - name: Bob\n    age: 25' },
+      { name: 'tabular array', input: 'users[3]{name,age,city}:\n  Alice, 30, NYC\n  Bob, 25, LA\n  Charlie, 35, SF' },
+      { name: 'root primitive number', input: '42' },
+      { name: 'root primitive string', input: 'Hello World' },
+      { name: 'root primitive boolean', input: 'true' },
+      { name: 'root primitive null', input: 'null' },
     ]
 
     for (const testCase of testCases) {
-      it(`should match decode() for: ${testCase.name}`, () => {
+      it(`decodeFromLines matches decode() for: ${testCase.name}`, () => {
         const lines = testCase.input.split('\n')
-        const streamResult = decodeFromLines(lines)
-        const regularResult = decode(testCase.input)
-
-        expect(streamResult).toEqual(regularResult)
+        expect(decodeFromLines(lines)).toEqual(decode(testCase.input))
       })
     }
   })
 })
+
+async function collect<T>(iterable: AsyncIterable<T>): Promise<T[]> {
+  const results: T[] = []
+  for await (const item of iterable) {
+    results.push(item)
+  }
+  return results
+}
+
+async function* asyncLines(lines: string[]): AsyncGenerator<string> {
+  for (const line of lines) {
+    await Promise.resolve()
+    yield line
+  }
+}
+
+async function* asyncEvents<T>(events: T[]): AsyncGenerator<T> {
+  for (const event of events) {
+    await Promise.resolve()
+    yield event
+  }
+}

packages/toon/test/encode-security.test.ts

@@ -0,0 +1,71 @@
+import type { EncodeReplacer } from '../src/index'
+import { describe, expect, it } from 'vitest'
+import { decode, encode } from '../src/index'
+
+describe('encode security hardening', () => {
+  const prototypeKey = '__proto__'
+
+  it('preserves own __proto__ properties through an encode/decode round trip', () => {
+    const marker = '__toonEncodedPolluted'
+    const input = JSON.parse(`{"__proto__":{"${marker}":true},"b":2}`)
+
+    try {
+      const decoded = decode(encode(input)) as Record<string, any>
+
+      expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+      expect(decoded[prototypeKey][marker]).toBe(true)
+      expect(decoded.b).toBe(2)
+      expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[marker]
+    }
+  })
+
+  it('preserves nested own __proto__ properties introduced by a replacer', () => {
+    const marker = '__toonNestedReplacerPolluted'
+    const replacement = JSON.parse(`{"__proto__":{"${marker}":true},"safe":1}`)
+    const replacer: EncodeReplacer = (key, value) => key === 'payload' ? replacement : value
+
+    try {
+      const decoded = decode(encode({ payload: null }, { replacer })) as Record<string, any>
+
+      expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
+      expect(decoded.payload[prototypeKey][marker]).toBe(true)
+      expect(decoded.payload.safe).toBe(1)
+      expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[marker]
+    }
+  })
+
+  it('preserves own __proto__ properties through key folding and expansion', () => {
+    const input = JSON.parse('{"__proto__":{"inner":{"folded":true}}}')
+    const encoded = encode(input, { keyFolding: 'safe' })
+    const decoded = decode(encoded, { expandPaths: 'safe' }) as Record<string, any>
+
+    expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
+    expect(decoded[prototypeKey].inner.folded).toBe(true)
+  })
+
+  it('does not use inherited properties to classify tabular rows', () => {
+    const inheritedKey = '__toonInheritedTabularField'
+    const input = [{ [inheritedKey]: 'user' }, { other: 'kept' }]
+
+    // eslint-disable-next-line no-extend-native -- Simulate a pre-polluted environment
+    Object.defineProperty(Object.prototype, inheritedKey, {
+      value: 'admin',
+      enumerable: true,
+      writable: true,
+      configurable: true,
+    })
+
+    try {
+      expect(decode(encode(input))).toEqual(input)
+    }
+    finally {
+      delete (Object.prototype as Record<string, unknown>)[inheritedKey]
+    }
+  })
+})

packages/toon/test/encodeLines.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'
 import { encodeLines } from '../src/index'
 
 describe('encodeLines', () => {
-  it('should yield lines without newline characters', () => {
+  it('yields lines without newline characters', () => {
     const value = { name: 'Alice', age: 30, city: 'Paris' }
     const lines = Array.from(encodeLines(value))
 
@@ -11,26 +11,13 @@ describe('encodeLines', () => {
     }
   })
 
-  it('should yield zero lines for empty object', () => {
+  it('yields zero lines for empty object', () => {
     const lines = Array.from(encodeLines({}))
 
     expect(lines.length).toBe(0)
   })
 
-  it('should be iterable with for-of loop', () => {
-    const value = { x: 10, y: 20 }
-    const collectedLines: string[] = []
-
-    for (const line of encodeLines(value)) {
-      collectedLines.push(line)
-    }
-
-    expect(collectedLines.length).toBe(2)
-    expect(collectedLines[0]).toBe('x: 10')
-    expect(collectedLines[1]).toBe('y: 20')
-  })
-
-  it('should not have trailing spaces in lines', () => {
+  it('yields lines without trailing spaces', () => {
     const value = {
       user: {
         name: 'Alice',
@@ -46,11 +33,4 @@ describe('encodeLines', () => {
       expect(line).not.toMatch(/\s$/)
     }
   })
-
-  it('should yield correct number of lines', () => {
-    const value = { a: 1, b: 2, c: 3 }
-    const lines = Array.from(encodeLines(value))
-
-    expect(lines.length).toBe(3)
-  })
 })

packages/toon/test/normalization.test.ts

@@ -1,4 +1,5 @@
 /* eslint-disable test/prefer-lowercase-title */
+import type { EncodeReplacer } from '../src/index'
 import { describe, expect, it } from 'vitest'
 import { decode, encode } from '../src/index'
 
@@ -47,7 +48,7 @@ describe('JavaScript-specific type normalization', () => {
 
     it('converts empty Set to empty array', () => {
       const result = encode(new Set())
-      expect(result).toBe('[0]:')
+      expect(result).toBe('[]')
     })
   })
 
@@ -112,4 +113,190 @@ describe('JavaScript-specific type normalization', () => {
       expect(result).toBe('0')
     })
   })
+
+  describe('canonical number form', () => {
+    it('encodes zero as canonical decimal', () => {
+      expect(encode(0)).toBe('0')
+      expect(decode(encode(0))).toBe(0)
+    })
+
+    it('encodes 1e-6 (lower boundary) as canonical decimal', () => {
+      expect(encode(1e-6)).toBe('0.000001')
+      expect(decode(encode(1e-6))).toBe(1e-6)
+    })
+
+    it('encodes 1e-7 (just below lower boundary) as exponent form', () => {
+      expect(encode(1e-7)).toBe('1e-7')
+      expect(decode(encode(1e-7))).toBe(1e-7)
+    })
+
+    it('encodes 9.999999e20 (near upper boundary) as canonical decimal', () => {
+      expect(encode(9.999999e20)).toBe('999999900000000000000')
+      expect(decode(encode(9.999999e20))).toBe(9.999999e20)
+    })
+
+    it('encodes 1e21 (upper boundary) as exponent form', () => {
+      expect(encode(1e21)).toBe('1e+21')
+      expect(decode(encode(1e21))).toBe(1e21)
+    })
+
+    it('encodes 1.5e20 as canonical decimal', () => {
+      expect(encode(1.5e20)).toBe('150000000000000000000')
+      expect(decode(encode(1.5e20))).toBe(1.5e20)
+    })
+
+    it('encodes 5e-324 (smallest subnormal) as exponent form', () => {
+      expect(encode(5e-324)).toBe('5e-324')
+      expect(decode(encode(5e-324))).toBe(5e-324)
+    })
+  })
+
+  describe('toJSON method support', () => {
+    it('calls toJSON method when object has it', () => {
+      const obj = {
+        data: 'example',
+        toJSON() {
+          return { info: this.data }
+        },
+      }
+      const result = encode(obj)
+      expect(result).toBe('info: example')
+    })
+
+    it('calls toJSON returning a primitive', () => {
+      const obj = {
+        value: 42,
+        toJSON() {
+          return 'custom-string'
+        },
+      }
+      const result = encode(obj)
+      expect(result).toBe('custom-string')
+    })
+
+    it('calls toJSON returning an array', () => {
+      const obj = {
+        items: [1, 2, 3],
+        toJSON() {
+          return ['a', 'b', 'c']
+        },
+      }
+      const result = encode(obj)
+      expect(result).toBe('[3]: a,b,c')
+    })
+
+    it('calls toJSON in nested object properties', () => {
+      const nestedObj = {
+        secret: 'hidden',
+        toJSON() {
+          return { public: 'visible' }
+        },
+      }
+      const obj = {
+        nested: nestedObj,
+        other: 'value',
+      }
+      const result = encode(obj)
+      expect(result).toBe('nested:\n  public: visible\nother: value')
+    })
+
+    it('calls toJSON in array elements', () => {
+      const obj1 = {
+        data: 'first',
+        toJSON() {
+          return { transformed: 'first-transformed' }
+        },
+      }
+      const obj2 = {
+        data: 'second',
+        toJSON() {
+          return { transformed: 'second-transformed' }
+        },
+      }
+      const arr = [obj1, obj2]
+      const result = encode(arr)
+      expect(result).toBe('[2]{transformed}:\n  first-transformed\n  second-transformed')
+    })
+
+    it('toJSON takes precedence over Date normalization', () => {
+      const customDate = {
+        toJSON() {
+          return { type: 'custom-date', value: '2025-01-01' }
+        },
+      }
+      // Make it look like a Date but with toJSON
+      Object.setPrototypeOf(customDate, Date.prototype)
+      const result = encode(customDate)
+      expect(result).toBe('type: custom-date\nvalue: 2025-01-01')
+    })
+
+    it('works with toJSON inherited from prototype', () => {
+      class CustomClass {
+        value: string
+
+        constructor(value: string) {
+          this.value = value
+        }
+
+        toJSON() {
+          return { classValue: this.value }
+        }
+      }
+
+      const instance = new CustomClass('test-value')
+      const result = encode(instance)
+      expect(result).toBe('classValue: test-value')
+    })
+
+    it('handles toJSON returning undefined (normalizes to null)', () => {
+      const obj = {
+        data: 'test',
+        toJSON() {
+          return undefined
+        },
+      }
+      const result = encode(obj)
+      expect(result).toBe('null')
+    })
+
+    it('works with replacer function', () => {
+      const obj = {
+        id: 1,
+        secret: 'hidden',
+        toJSON() {
+          return { id: this.id, public: 'visible' }
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Replacer should see the toJSON result, not the original object
+        if (typeof value === 'object' && value !== null && 'public' in value) {
+          return { ...value, extra: 'added' }
+        }
+        return value
+      }
+      const result = encode(obj, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual({ id: 1, public: 'visible', extra: 'added' })
+      expect(decoded).not.toHaveProperty('secret')
+    })
+
+    it('toJSON result is normalized before replacer is applied', () => {
+      const dateObj = {
+        date: new Date('2025-01-01T00:00:00.000Z'),
+        toJSON() {
+          return { date: this.date }
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        // The date should already be normalized to ISO string by the time replacer sees it
+        if (key === 'date' && typeof value === 'string') {
+          return value.replace('2025', 'YEAR')
+        }
+        return value
+      }
+      const result = encode(dateObj, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual({ date: 'YEAR-01-01T00:00:00.000Z' })
+    })
+  })
 })

packages/toon/test/replacer.test.ts

@@ -0,0 +1,407 @@
+import type { EncodeReplacer, JsonObject, JsonValue } from '../src/types'
+import { describe, expect, it } from 'vitest'
+import { decode, encode } from '../src/index'
+
+describe('replacer function', () => {
+  describe('basic filtering', () => {
+    it('removes properties by returning undefined', () => {
+      const input = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'Alice', email: 'alice@example.com' })
+      expect(decoded).not.toHaveProperty('password')
+    })
+
+    it('removes array elements by returning undefined', () => {
+      const input = [1, 2, 3, 4, 5]
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'number' && value % 2 === 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual([1, 3, 5])
+    })
+
+    it('handles deeply nested filtering', () => {
+      const input = {
+        users: [
+          { name: 'Alice', password: 'secret1', role: 'admin' },
+          { name: 'Bob', password: 'secret2', role: 'user' },
+        ],
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        users: [
+          { name: 'Alice', role: 'admin' },
+          { name: 'Bob', role: 'user' },
+        ],
+      })
+    })
+  })
+
+  describe('value transformation', () => {
+    it('transforms primitive values', () => {
+      const input = { name: 'alice', age: 30 }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'ALICE', age: 30 })
+    })
+
+    it('transforms objects', () => {
+      const input = { user: { name: 'Alice' } }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length === 1 && typeof value === 'object' && value !== null && !Array.isArray(value)) {
+          return { ...value as object, _id: `${key}_123` }
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        user: { name: 'Alice', _id: 'user_123' },
+      })
+    })
+
+    it('transforms arrays', () => {
+      const input = { numbers: [1, 2, 3] }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'number')
+          return value * 2
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ numbers: [2, 4, 6] })
+    })
+  })
+
+  describe('root value handling', () => {
+    it('calls replacer on root value with empty string key', () => {
+      const input = { value: 42 }
+      let rootKeySeen = false
+      let rootPathSeen = false
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (key === '' && path.length === 0) {
+          rootKeySeen = true
+          rootPathSeen = true
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(rootKeySeen).toBe(true)
+      expect(rootPathSeen).toBe(true)
+    })
+
+    it('transforms root object', () => {
+      const input = { name: 'Alice' }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length === 0) {
+          return { ...value as object, timestamp: 1234567890 }
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'Alice', timestamp: 1234567890 })
+    })
+
+    it('does not omit root when replacer returns undefined', () => {
+      const input = { name: 'Alice' }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length === 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'Alice' })
+    })
+
+    it('handles primitive root values', () => {
+      const input = 'hello'
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      expect(result).toBe('HELLO')
+    })
+
+    it('provides correct arguments to root call', () => {
+      const input = { data: 'test' }
+      const calls: { key: string, path: (string | number)[] }[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        calls.push({ key, path: [...path] })
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(calls[0]).toEqual({ key: '', path: [] })
+    })
+  })
+
+  describe('path tracking', () => {
+    it('provides correct paths for nested objects', () => {
+      const input = {
+        user: {
+          profile: {
+            name: 'Alice',
+          },
+        },
+      }
+      const paths: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        paths.push(path.join('.'))
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(paths).toContain('') // root
+      expect(paths).toContain('user')
+      expect(paths).toContain('user.profile')
+      expect(paths).toContain('user.profile.name')
+    })
+
+    it('provides correct paths for arrays with string indices', () => {
+      const input = { items: ['a', 'b', 'c'] }
+      const seenKeys: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length > 0 && path[path.length - 1] !== 'items') {
+          seenKeys.push(key)
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(seenKeys).toEqual(['0', '1', '2'])
+    })
+
+    it('provides correct paths for nested arrays', () => {
+      const input = {
+        matrix: [
+          [1, 2],
+          [3, 4],
+        ],
+      }
+      const paths: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (typeof value === 'number') {
+          paths.push(`${path.join('.')} (key="${key}")`)
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(paths).toContain('matrix.0.0 (key="0")')
+      expect(paths).toContain('matrix.0.1 (key="1")')
+      expect(paths).toContain('matrix.1.0 (key="0")')
+      expect(paths).toContain('matrix.1.1 (key="1")')
+    })
+  })
+
+  describe('edge cases', () => {
+    it('handles empty objects', () => {
+      const input = {}
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer })
+      expect(result).toBe('')
+    })
+
+    it('handles empty arrays', () => {
+      const input: JsonValue[] = []
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual([])
+    })
+
+    it('handles null values', () => {
+      const input = { value: null }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (value === null)
+          return 'NULL'
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual({ value: 'NULL' })
+    })
+
+    it('re-normalizes non-JsonValue returns', () => {
+      const input = { date: '2025-01-01' }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Return a Date object (will be normalized to ISO string)
+        if (key === 'date')
+          return new Date(value as string)
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result) as JsonObject
+
+      expect(typeof decoded.date).toBe('string')
+      expect(decoded.date).toMatch(/^\d{4}-\d{2}-\d{2}T/)
+    })
+
+    it('handles all properties being filtered out', () => {
+      const input = { a: 1, b: 2, c: 3 }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length > 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({})
+    })
+
+    it('handles all array elements being filtered out', () => {
+      const input = [1, 2, 3]
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length > 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual([])
+    })
+
+    it('handles nested objects with mixed omissions', () => {
+      const input = {
+        keep: 'this',
+        remove: 'that',
+        nested: {
+          keep: 'nested keep',
+          remove: 'nested remove',
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'remove')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        keep: 'this',
+        nested: {
+          keep: 'nested keep',
+        },
+      })
+    })
+
+    it('handles arrays with some elements removed', () => {
+      const input = { items: [{ id: 1, keep: true }, { id: 2, keep: false }, { id: 3, keep: true }] }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'keep' in value && value.keep === false) {
+          return undefined
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        items: [{ id: 1, keep: true }, { id: 3, keep: true }],
+      })
+    })
+  })
+
+  describe('integration with other options', () => {
+    it('works with keyFolding', () => {
+      const input = {
+        user: {
+          profile: {
+            name: 'Alice',
+          },
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer, keyFolding: 'safe' })
+      expect(result).toContain('user.profile.name: ALICE')
+    })
+
+    it('works with custom delimiters', () => {
+      const input = { items: [1, 2, 3] }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'number')
+          return value * 10
+        return value
+      }
+
+      const result = encode(input, { replacer, delimiter: '\t' })
+      expect(result).toContain('10\t20\t30')
+    })
+
+    it('works with custom indent', () => {
+      const input = { user: { name: 'Alice' } }
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer, indent: 4 })
+      expect(result).toContain('    name: Alice')
+    })
+  })
+})