Compare commits

...

16 Commits

Author SHA1 Message Date
Phumrapee Limpianchop 1336074d43 fix(decode): prevent false array header detection for unquoted keys c… (#314)
CI / ci (push) Waiting to run
Deploy Docs / Deploy Docs (push) Waiting to run
* fix(decode): prevent false array header detection for unquoted keys containing brackets and colons

* test(decode): pin stream parity for an unquoted bracket-colon scalar

* chore(toon): bump `@toon-format/spec` to ^3.3.2

---------

Co-authored-by: Johann Schopplich <johann@schopplich.com>
2026-07-18 21:15:25 +02:00
Johann Schopplich 352a5b017d chore: release v2.3.1
Release + Publish / Release (push) Waiting to run
2026-07-16 08:06:50 +02:00
Johann Schopplich e2003970dd chore: revert v2.3.1 version bump and quote build filter glob 2026-07-16 08:02:04 +02:00
Johann Schopplich bb9a09ad6c chore: remove pnpm shamefully hoisting 2026-07-16 07:31:14 +02:00
Johann Schopplich 2013a7e09c chore: release v2.3.1 2026-07-16 07:26:53 +02:00
Johann Schopplich 9024af9e6a fix(decode): treat quoted spans as opaque in line classification 2026-07-16 07:22:49 +02:00
Johann Schopplich 27b65c2931 chore: upgrade dependencies 2026-07-15 23:07:17 +02:00
Chris Korhonen 94a2b7560b fix: harden encode and decode against prototype pollution (#316)
* fix: prevent prototype pollution during decode

* fix: preserve __proto__ during encode

* fix: ignore inherited tabular fields

* refactor: consolidate own-property helper and tighten __proto__ regression tests

* perf: fast-path setOwnProperty for non-__proto__ keys

* docs: trim object-utils remarks to rationale

* chore: update comment

---------

Co-authored-by: Johann Schopplich <johann@schopplich.com>
2026-07-15 22:58:34 +02:00
Johann Schopplich 09a7b38f2c test: cover malformed array length marker error context 2026-07-15 18:30:22 +02:00
Johann Schopplich 702301231c fix(decode): slice key-value rest at key token end offset 2026-07-15 18:30:22 +02:00
Johann Schopplich a19a117919 ci: update GitHub Actions to latest versions 2026-06-12 17:56:06 +02:00
Mohammad Raziei a52f2d2815 docs: add Matlab and Python implementations to ecosystem list (CToon Project) (#309)
* docs: add Python (C++ backend) implementation to ecosystem list

Added a new entry for the Python implementation with a C++ backend (ctoon) to the ecosystem documentation, ensuring it appears alongside other language implementations.

* docs: add Matlab implementation to ecosystem implementations list

Add Matlab entry (ctoon) to the ecosystem implementations table, acknowledging the new community implementation and providing its repository link and author.
2026-05-23 10:00:46 +02:00
Johann Schopplich a956757de0 test: align canonical number form titles with file conventions
Drop the (§2 carve-out) suffix from the describe block — no other test in this file references spec sections. Switch it() titles from 'emits X' to 'encodes X as Y' to match the existing converts/normalizes verb pattern.
2026-05-21 10:39:05 +02:00
Johann Schopplich c0af15e592 chore: close v3.3 release cleanup
- docs/guide/format-overview.md: drop duplicated Type Conversions table; link to canonical api.md#type-normalization (drift surface 3 -> 2 tables)
- packages/toon/test/normalization.test.ts: add 7 §2 carve-out boundary tests (encode + round-trip per value); locks current conformance against future String(value) refactors
2026-05-21 10:33:14 +02:00
Johann Schopplich 5b66523e42 chore: align with spec v3.3
- bump @toon-format/spec to ^3.3.0; update SPEC.md pointer and badges
- docs: align number-form prose with §2 carve-out (canonical in [1e-6, 1e21) or zero; exponent form permitted outside)
- docs/reference/spec.md: add lowercase booleans/null to encoder checklist; add Java to Appendix F language list
- docs/reference/efficiency-formalization.md: drop overstated "matching TOON's requirement" claim
- packages/toon/README.md: drop "What's New in v3.2" section (now stale)

All 593 tests pass (packages/toon: 497, packages/cli: 96). Encoder remains conformant by default under the v3.3 §2 carve-out via String(value).
2026-05-21 10:10:20 +02:00
Johann Schopplich e2b5611d37 docs: quote §7.2 timestamps in examples and propagate dd856fe to docs/cli
Two regrade fixes surfaced by a post-v3.2 verification swarm:

* llm-prompts.md and api.md had ISO-8601 timestamps as unquoted TOON
  values, violating §7.2 (strings containing a colon MUST be quoted).
  The spec repo's examples and packages/toon/README.md already quote
  timestamps; the docs repo had drifted.

* docs/cli/index.md is the docs-site twin of packages/cli/README.md.
  The prior v3.2 swarm (dd856fe) updated the README's --no-strict
  options row and Lenient Decoding prose to cite "header delimiter"
  but did not propagate to the parallel paragraphs in docs/cli/index.md
  (lines 159 and 234).
2026-05-20 12:28:18 +02:00
39 changed files with 2151 additions and 1826 deletions
+3 -3
View File
@@ -21,12 +21,12 @@ jobs:
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@v5
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@v4
uses: pnpm/action-setup@v6
- name: Setup Node.js
uses: actions/setup-node@v6
@@ -38,7 +38,7 @@ jobs:
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
- name: Cache pnpm dependencies
uses: actions/cache@v4
uses: actions/cache@v5
with:
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+3 -3
View File
@@ -22,12 +22,12 @@ jobs:
runs-on: ubuntu-slim
steps:
- name: Checkout
uses: actions/checkout@v5
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@v4
uses: pnpm/action-setup@v6
- name: Setup Node.js
uses: actions/setup-node@v6
@@ -39,7 +39,7 @@ jobs:
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
- name: Cache pnpm dependencies
uses: actions/cache@v4
uses: actions/cache@v5
with:
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v5
uses: actions/checkout@v6
with:
persist-credentials: false
# Only fetch the config file from the repository
+2 -2
View File
@@ -20,13 +20,13 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v5
uses: actions/checkout@v6
with:
fetch-depth: 0 # Required for fetching tags and generating release notes
persist-credentials: true
- name: Setup pnpm
uses: pnpm/action-setup@v4
uses: pnpm/action-setup@v6
- name: Setup Node.js
uses: actions/setup-node@v6
-1
View File
@@ -1 +0,0 @@
shamefully-hoist=true
+1 -1
View File
@@ -4,7 +4,7 @@ The TOON specification has moved to a dedicated repository: [github.com/toon-for
## Current Version
**Version 3.2** (2026-05-20)
**Version 3.3** (2026-05-21)
## Quick Links
+12 -12
View File
@@ -8,20 +8,20 @@
"fetch:github-repos": "node scripts/fetch-github-repos.ts"
},
"devDependencies": {
"@ai-sdk/anthropic": "^3.0.78",
"@ai-sdk/google": "^3.0.75",
"@ai-sdk/openai": "^3.0.64",
"@ai-sdk/provider": "^3.0.10",
"@ai-sdk/xai": "^3.0.91",
"@clack/prompts": "^1.4.0",
"@faker-js/faker": "^10.4.0",
"ai": "^6.0.185",
"csv-stringify": "^6.7.0",
"fast-xml-parser": "^5.8.0",
"@ai-sdk/anthropic": "^3.0.97",
"@ai-sdk/google": "^3.0.94",
"@ai-sdk/openai": "^3.0.85",
"@ai-sdk/provider": "^3.0.14",
"@ai-sdk/xai": "^3.0.108",
"@clack/prompts": "^1.7.0",
"@faker-js/faker": "^10.5.0",
"ai": "^6.0.228",
"csv-stringify": "^6.8.1",
"fast-xml-parser": "^5.10.0",
"gpt-tokenizer": "^3.4.0",
"ofetch": "^1.5.1",
"p-map": "^7.0.4",
"p-queue": "^9.3.0",
"p-map": "^7.0.5",
"p-queue": "^9.3.1",
"unstorage": "^1.17.5",
"yaml": "^2.9.0"
}
+1 -1
View File
@@ -12,7 +12,7 @@ const config: Theme = {
extends: DefaultTheme,
enhanceApp({ app }) {
app.config.globalProperties.$spec = {
version: '3.2',
version: '3.3',
}
app.component('CopyOrDownloadAsMarkdownButtons', CopyOrDownloadAsMarkdownButtons)
app.component('PlaygroundLayout', PlaygroundLayout)
+2 -2
View File
@@ -156,7 +156,7 @@ When using the `--stats` flag with encode, the CLI builds the full TOON string o
| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
| `--indent <number>` | Indentation size (default: `2`) |
| `--stats` | Show token count estimates and savings (encode only) |
| `--no-strict` | Skip decode validation; last-write-wins on duplicate keys |
| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
| `--keyFolding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
| `--flattenDepth <number>` | Maximum segments to fold (default: `Infinity`) requires `--keyFolding safe` |
| `--expandPaths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
@@ -231,7 +231,7 @@ Skip validation for faster, more forgiving decoding:
toon data.toon --no-strict -o output.json
```
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and delimiter consistency. Duplicate sibling keys no longer throw the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
### Decode Error Output
+2
View File
@@ -47,9 +47,11 @@ Community members have created implementations in additional languages:
| **Kotlin** | [ktoon](https://github.com/lukelast/ktoon)| [@lukelast](https://github.com/lukelast) |
| **Laravel Framework** | [laravel-toon](https://github.com/mischasigtermans/laravel-toon) | [@mischasigtermans](https://github.com/mischasigtermans) |
| **Lua/Neovim** | [toon.nvim](https://github.com/thalesgelinger/toon.nvim) | [@thalesgelinger](https://github.com/thalesgelinger) |
| **Matlab** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
| **OCaml** | [ocaml-toon](https://github.com/davesnx/ocaml-toon) | [@davesnx](https://github.com/davesnx) |
| **Perl** | [Data::TOON](https://github.com/ytnobody/p5-Data-TOON) | [@ytnobody](https://github.com/ytnobody) |
| **PHP** | [toon-php](https://github.com/HelgeSverre/toon-php) | [@HelgeSverre](https://github.com/HelgeSverre) |
| **Python** (C++ backend) | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
| **R** | [toon](https://github.com/laresbernardo/toon) | [@laresbernardo](https://github.com/laresbernardo) |
| **Ruby** | [toon-ruby](https://github.com/andrepcg/toon-ruby) | [@andrepcg](https://github.com/andrepcg) |
+1 -12
View File
@@ -334,18 +334,7 @@ Other escapes (e.g., `\x`, `\0`, `\b`) are always rejected, as are lone-surrogat
### Type Conversions
Numbers are emitted in canonical decimal form (no exponent notation, no trailing zeros). Non-JSON types are normalized before encoding:
| Input | Output |
|-------|--------|
| Finite number | Canonical decimal (e.g., `1e6``1000000`, `1.5000``1.5`, `-0``0`) |
| `NaN`, `Infinity`, `-Infinity` | `null` |
| `BigInt` (within safe range) | Number |
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
| `Set` | Array of normalized values |
| `Map` | Object with `String(key)` keys |
| `undefined`, `function`, `symbol` | `null` |
Numbers are emitted in canonical decimal form for values in the §2 carve-out range; exponent notation is permitted outside. Non-JSON types (`NaN`, `Infinity`, `BigInt`, `Date`, `Set`, `Map`, `undefined`, etc.) are normalized before encoding see [API Reference Type Normalization](/reference/api#type-normalization) for the full mapping.
Decoders accept both decimal and exponent forms on input (e.g., `42`, `-3.14`, `1e-6`), and treat tokens with forbidden leading zeros (e.g., `"05"`) as strings, not numbers.
+14 -14
View File
@@ -25,9 +25,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
```toon
users[3]{id,name,role,lastLogin}:
1,Alice,admin,2025-01-15T10:30:00Z
2,Bob,user,2025-01-14T15:22:00Z
3,Charlie,user,2025-01-13T09:45:00Z
1,Alice,admin,"2025-01-15T10:30:00Z"
2,Bob,user,"2025-01-14T15:22:00Z"
3,Charlie,user,"2025-01-13T09:45:00Z"
```
Task: Summarize the user roles and their last activity.
@@ -52,9 +52,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
```toon
users[3]{id,name,role,lastLogin}:
1,Alice,admin,2025-01-15T10:30:00Z
2,Bob,user,2025-01-14T15:22:00Z
3,Charlie,user,2025-01-13T09:45:00Z
1,Alice,admin,"2025-01-15T10:30:00Z"
2,Bob,user,"2025-01-14T15:22:00Z"
3,Charlie,user,"2025-01-13T09:45:00Z"
```
Task: Return only users with role "user" as TOON. Use the same header format. Set [N] to match the row count. Output only the code block.
@@ -64,8 +64,8 @@ Task: Return only users with role "user" as TOON. Use the same header format. Se
```toon
users[2]{id,name,role,lastLogin}:
2,Bob,user,2025-01-14T15:22:00Z
3,Charlie,user,2025-01-13T09:45:00Z
2,Bob,user,"2025-01-14T15:22:00Z"
3,Charlie,user,"2025-01-13T09:45:00Z"
```
The model adjusts `[N]` to `2` and generates two rows.
@@ -166,10 +166,10 @@ System logs in TOON format (tab-separated):
```toon
events[4 ]{id level message timestamp}:
1 error Connection timeout 2025-01-15T10:00:00Z
2 warn Slow query 2025-01-15T10:05:00Z
3 info User login 2025-01-15T10:10:00Z
4 error Database error 2025-01-15T10:15:00Z
1 error Connection timeout "2025-01-15T10:00:00Z"
2 warn Slow query "2025-01-15T10:05:00Z"
3 info User login "2025-01-15T10:10:00Z"
4 error Database error "2025-01-15T10:15:00Z"
```
Task: Return only error-level events as TOON. Use the same format.
@@ -182,8 +182,8 @@ import { decode } from '@toon-format/toon'
const modelResponse = `
events[2 ]{id level message timestamp}:
1 error Connection timeout 2025-01-15T10:00:00Z
4 error Database error 2025-01-15T10:15:00Z
1 error Connection timeout "2025-01-15T10:00:00Z"
4 error Database error "2025-01-15T10:15:00Z"
`
const filtered = decode(modelResponse, { strict: true })
+2 -2
View File
@@ -13,9 +13,9 @@
"gpt-tokenizer": "^3.4.0",
"markdown-it-mathjax3": "^4.3.2",
"uint8array-extras": "^1.5.0",
"unocss": "^66.6.8",
"unocss": "^66.7.5",
"vitepress": "^1.6.4",
"vitepress-plugin-llms": "^1.12.2",
"vitepress-plugin-llms": "^1.13.3",
"yaml": "^2.9.0"
}
}
+3 -2
View File
@@ -59,7 +59,8 @@ Non-JSON-serializable values are normalized before encoding:
| Input | Output |
|-------|--------|
| `Object` with `toJSON()` method | Result of calling `toJSON()`, recursively normalized |
| Finite number | Canonical decimal (no exponent, no leading/trailing zeros: `1e6` → `1000000`, `-0` → `0`) |
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal (e.g., `1e6` → `1000000`, `-0` → `0`) |
| Finite number outside that range | Exponent form permitted (e.g., `1e-7`, `1e+21`) |
| `NaN`, `Infinity`, `-Infinity` | `null` |
| `BigInt` (within safe range) | Number |
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
@@ -256,7 +257,7 @@ console.log(encode(data, { replacer }))
```yaml
metadata:
created: 2025-01-01T00:00:00Z
created: "2025-01-01T00:00:00Z"
user:
created: 2025-01-02
```
+1 -1
View File
@@ -40,7 +40,7 @@ Where $\Omega$ represents the data structure under comparison. If $\Delta > 0$,
- must be quoted in JSON, and
- can be left unquoted in TOON (no characters that would force quoting).
Many examples assume values are numbers, booleans, null, or TOON-safe strings that can be unquoted in TOON but must be quoted in JSON.
- **Numbers**: Both formats are assumed to use the same canonical decimal representation (no exponent notation), matching TOON's requirement. JSON could use exponent forms; we ignore that here to isolate structural differences.
- **Numbers**: For this analysis only, both formats are assumed to use the same canonical decimal representation. JSON could use exponent forms; we ignore that here to isolate structural differences.
- **ASCII/UTF-8**: Keys and structural tokens are assumed ASCII, so byte length equals character count ($|x|_{\text{utf8}} = |x|_{\text{char}}$). Non-ASCII content affects both formats similarly and does not change the structural conclusions.
- **Nesting depth**: Closed-form expressions are given for flat structures and a single level of nesting. Each additional nesting level in TOON adds 2 bytes of indentation per nested line. At sufficient depth, the braces of compact JSON can win over TOON's indentation (as seen in [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon)).
- **Byte vs token count**: Modern LLM tokenizers operate over UTF-8 bytes, so byte length is a good upper bound and first-order proxy for token count, even though the mapping is not exactly linear.
+4 -3
View File
@@ -32,7 +32,7 @@ TOON documents are always UTF8 with LF (`\n`) line endings; the optional `cha
Defines key terms like "indentation level", "active delimiter", "strict mode", and RFC2119 keywords (MUST, SHOULD, MAY).
[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model):
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (no exponent notation, no leading/trailing zeros).
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (canonical decimal for values in `[1e-6, 1e21)` or zero; exponent form permitted outside).
[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder):
Defines how non-JSON types (Date, BigInt, NaN, Infinity, undefined, etc.) are normalized before encoding. Required reading for encoder implementers.
@@ -95,7 +95,7 @@ How the spec evolves: major vs minor bumps and the extensibility policy.
Licensing and IP terms for the specification.
[Appendix F: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-f-host-type-normalization-examples-informative):
Non-normative guidance for Go, JavaScript, Python, and Rust implementations on normalizing language-specific types.
Non-normative guidance for Go, JavaScript, Python, Rust, and Java implementations on normalizing language-specific types.
[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative):
Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://github.com/toon-format/spec/tree/main/tests) for validating implementations.
@@ -127,8 +127,9 @@ Key requirements:
- Quote strings with active delimiter, colon, or structural characters
- Emit array lengths `[N]` matching actual count
- Preserve object key order
- Normalize numbers to non-exponential decimal form
- Emit numbers per §2 (canonical decimal in `[1e-6, 1e21)` or zero; exponent form permitted outside)
- Convert `-0` to `0`, `NaN`/±Infinity to `null`
- Emit booleans and null as lowercase literals (`true`, `false`, `null`)
- No trailing spaces or trailing newline
- When `keyFolding="safe"` is enabled, folding MUST follow §13.4:
- Only fold IdentifierSegment keys (letters/digits/underscores, no dots),
+2 -1
View File
@@ -352,7 +352,8 @@ See [Format Overview Key Folding](/guide/format-overview#key-folding-optiona
| Input | Output |
|-------|--------|
| Finite number | Canonical decimal (no exponent, no trailing zeros) |
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal |
| Finite number outside that range | Exponent form permitted |
| `NaN`, `Infinity`, `-Infinity` | `null` |
| `BigInt` (safe range) | Number |
| `BigInt` (out of range) | Quoted decimal string |
-1
View File
@@ -3,7 +3,6 @@ import type { FlatConfigComposer } from 'eslint-flat-config-utils'
import antfu from '@antfu/eslint-config'
const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
pnpm: false,
rules: {
'no-cond-assign': 'off',
},
+13 -13
View File
@@ -1,15 +1,15 @@
{
"name": "@toon-format/monorepo",
"type": "module",
"version": "2.3.0",
"version": "2.3.1",
"private": true,
"packageManager": "pnpm@10.33.4",
"packageManager": "pnpm@11.13.0",
"scripts": {
"build": "pnpm -r --filter=./packages/** run build",
"build": "pnpm -r --filter='./packages/**' run build",
"automd": "automd",
"docs:dev": "vitepress dev docs",
"docs:build": "vitepress build docs",
"docs:preview": "vitepress preview docs",
"docs:dev": "pnpm -C docs run dev",
"docs:build": "pnpm -C docs run build",
"docs:preview": "pnpm -C docs run preview",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"test": "pnpm -r test",
@@ -17,16 +17,16 @@
"release": "bumpp -r"
},
"devDependencies": {
"@antfu/eslint-config": "^8.3.0",
"@commitlint/types": "^21.0.1",
"@types/node": "^25.9.1",
"@antfu/eslint-config": "^9.1.0",
"@commitlint/types": "^21.2.0",
"@types/node": "^26.1.1",
"automd": "^0.4.3",
"bumpp": "^11.1.0",
"eslint": "^10.4.0",
"eslint": "^10.7.0",
"eslint-flat-config-utils": "^3.2.0",
"tsdown": "^0.22.0",
"tsdown": "^0.22.8",
"typescript": "^6.0.3",
"vite": "^8.0.13",
"vitest": "^4.1.7"
"vite": "^8.1.4",
"vitest": "^4.1.10"
}
}
+2 -2
View File
@@ -1,8 +1,8 @@
{
"name": "@toon-format/cli",
"type": "module",
"version": "2.3.0",
"packageManager": "pnpm@10.33.4",
"version": "2.3.1",
"packageManager": "pnpm@11.13.0",
"description": "CLI for JSON ↔ TOON conversion using @toon-format/toon",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
+1 -8
View File
@@ -4,7 +4,7 @@
[![CI](https://github.com/toon-format/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/toon-format/toon/actions)
[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
[![SPEC v3.2](https://img.shields.io/badge/spec-v3.2-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
[![SPEC v3.3](https://img.shields.io/badge/spec-v3.3-fef3c0?labelColor=1b1b1f)](https://github.com/toon-format/spec)
[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg?labelColor=1b1b1f&color=fef3c0)](https://www.npmjs.com/package/@toon-format/toon)
[![License: MIT](https://img.shields.io/badge/license-MIT-fef3c0?labelColor=1b1b1f)](./LICENSE)
@@ -134,13 +134,6 @@ hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
## What's New in v3.2
- Strict mode rejects duplicate sibling keys; with `strict: false` the last value wins.
- Malformed array headers (`[03]`, `[1] foo:`, header/field delimiter mismatch) error in strict mode instead of silently degrading.
- Tabular form excludes arrays that contain an empty `{}` element those fall back to the expanded list form.
- Nested arrays-of-objects can appear as list items via an explicit `- [N]:` header.
## Media Type & File Extension
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-typeaware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations) for normative details.
+3 -3
View File
@@ -1,8 +1,8 @@
{
"name": "@toon-format/toon",
"type": "module",
"version": "2.3.0",
"packageManager": "pnpm@10.33.4",
"version": "2.3.1",
"packageManager": "pnpm@11.13.0",
"description": "Token-Oriented Object Notation (TOON) Compact, human-readable, schema-aware encoding of JSON for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
@@ -38,6 +38,6 @@
"test": "vitest"
},
"devDependencies": {
"@toon-format/spec": "^3.2.0"
"@toon-format/spec": "^3.3.2"
}
}
+13 -7
View File
@@ -5,7 +5,7 @@ import { findClosingQuote } from '../shared/string-utils.ts'
import { ToonDecodeError, withLine } from './errors.ts'
import { isArrayHeaderContent, isKeyValueContent, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser.ts'
import { createScanState, parseLinesAsync, parseLinesSync } from './scanner.ts'
import { assertExpectedCount, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
import { assertExpectedCount, isDataRow, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
interface DecoderContext { indent: number, strict: boolean }
@@ -223,9 +223,8 @@ function* decodeKeyValueSync(
}
// Regular key-value pair
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
const colonIndex = content.indexOf(COLON, key.length)
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
const rest = content.slice(end).trim()
assertNoDuplicateKey(key, line, seenKeys)
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
@@ -354,6 +353,10 @@ function* decodeTabularArraySync(
}
if (line.depth === rowDepth) {
if (!isDataRow(line.content, header.delimiter)) {
break
}
if (startLine === undefined) {
startLine = line.lineNumber
}
@@ -678,9 +681,8 @@ async function* decodeKeyValueAsync(
}
// Regular key-value pair
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
const colonIndex = content.indexOf(COLON, key.length)
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
const rest = content.slice(end).trim()
assertNoDuplicateKey(key, line, seenKeys)
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
@@ -788,6 +790,10 @@ async function* decodeTabularArrayAsync(
}
if (line.depth === rowDepth) {
if (!isDataRow(line.content, header.delimiter)) {
break
}
if (startLine === undefined) {
startLine = line.lineNumber
}
+4 -3
View File
@@ -1,4 +1,5 @@
import type { JsonObject, JsonStreamEvent, JsonValue } from '../types.ts'
import { setOwnProperty } from '../shared/object-utils.ts'
import { QUOTED_KEY_MARKER } from './expand.ts'
// #region Build context types
@@ -63,7 +64,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
throw new Error('Object startObject event without preceding key')
}
parent.obj[parent.currentKey] = obj
setOwnProperty(parent.obj, parent.currentKey, obj)
parent.currentKey = undefined
}
else if (parent.type === 'array') {
@@ -116,7 +117,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
if (parent.currentKey === undefined) {
throw new Error('Array startArray event without preceding key')
}
parent.obj[parent.currentKey] = arr
setOwnProperty(parent.obj, parent.currentKey, arr)
parent.currentKey = undefined
}
else if (parent.type === 'array') {
@@ -177,7 +178,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
if (parent.currentKey === undefined) {
throw new Error('Primitive event without preceding key in object')
}
parent.obj[parent.currentKey] = event.value
setOwnProperty(parent.obj, parent.currentKey, event.value)
parent.currentKey = undefined
}
else if (parent.type === 'array') {
+17 -15
View File
@@ -1,6 +1,7 @@
import type { JsonObject, JsonValue } from '../types.ts'
import { DOT } from '../constants.ts'
import { isJsonObject } from '../encode/normalize.ts'
import { getOwnProperty, setOwnProperty } from '../shared/object-utils.ts'
import { isIdentifierSegment } from '../shared/validation.ts'
// #region Path expansion (safe)
@@ -73,7 +74,7 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
const expandedValue = expandPathsSafe(keyValue, strict)
// Check for conflicts with already-expanded keys
if (key in expandedObject) {
if (Object.hasOwn(expandedObject, key)) {
const conflictingValue = expandedObject[key]!
// If both are objects, try to merge them
if (canMerge(conflictingValue, expandedValue)) {
@@ -87,12 +88,12 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
)
}
// Non-strict: overwrite (LWW)
expandedObject[key] = expandedValue
setOwnProperty(expandedObject, key, expandedValue)
}
}
else {
// No conflict - insert directly
expandedObject[key] = expandedValue
setOwnProperty(expandedObject, key, expandedValue)
}
}
@@ -131,12 +132,12 @@ function insertPathSafe(
// Walk to the penultimate segment, creating objects as needed
for (let i = 0; i < segments.length - 1; i++) {
const currentSegment = segments[i]!
const segmentValue = currentNode[currentSegment]
const segmentValue = getOwnProperty(currentNode, currentSegment)
if (segmentValue === undefined) {
// Create new intermediate object
const newObj: JsonObject = {}
currentNode[currentSegment] = newObj
setOwnProperty(currentNode, currentSegment, newObj)
currentNode = newObj
}
else if (isJsonObject(segmentValue)) {
@@ -152,18 +153,18 @@ function insertPathSafe(
}
// Non-strict: overwrite with new object
const newObj: JsonObject = {}
currentNode[currentSegment] = newObj
setOwnProperty(currentNode, currentSegment, newObj)
currentNode = newObj
}
}
// Insert at the final segment
const lastSeg = segments[segments.length - 1]!
const destinationValue = currentNode[lastSeg]
const destinationValue = getOwnProperty(currentNode, lastSeg)
if (destinationValue === undefined) {
// No conflict - insert directly
currentNode[lastSeg] = value
setOwnProperty(currentNode, lastSeg, value)
}
else if (canMerge(destinationValue, value)) {
// Both are objects - deep merge
@@ -177,7 +178,7 @@ function insertPathSafe(
)
}
// Non-strict: overwrite (LWW)
currentNode[lastSeg] = value
setOwnProperty(currentNode, lastSeg, value)
}
}
@@ -201,13 +202,14 @@ function mergeObjects(
strict: boolean,
): void {
for (const [key, sourceValue] of Object.entries(source)) {
const targetValue = target[key]
if (targetValue === undefined) {
if (!Object.hasOwn(target, key)) {
// Key doesn't exist in target - copy it
target[key] = sourceValue
setOwnProperty(target, key, sourceValue)
continue
}
else if (canMerge(targetValue, sourceValue)) {
const targetValue = target[key] as JsonValue
if (canMerge(targetValue, sourceValue)) {
// Both are objects - recursively merge
mergeObjects(targetValue as JsonObject, sourceValue as JsonObject, strict)
}
@@ -219,7 +221,7 @@ function mergeObjects(
)
}
// Non-strict: overwrite (LWW)
target[key] = sourceValue
setOwnProperty(target, key, sourceValue)
}
}
}
+13 -7
View File
@@ -34,14 +34,20 @@ export function parseArrayHeaderLine(
}
else {
// Unquoted key - find first bracket
bracketStart = content.indexOf(OPEN_BRACKET)
bracketStart = findUnquotedChar(content, OPEN_BRACKET)
}
if (bracketStart === -1) {
return
}
const bracketEnd = content.indexOf(CLOSE_BRACKET, bracketStart)
// A header key can't contain an unquoted colon, so this is a key-value line
const firstColonIndex = findUnquotedChar(content, COLON)
if (firstColonIndex !== -1 && firstColonIndex < bracketStart) {
return
}
const bracketEnd = findUnquotedChar(content, CLOSE_BRACKET, bracketStart)
if (bracketEnd === -1) {
return
}
@@ -51,8 +57,8 @@ export function parseArrayHeaderLine(
let braceEnd = colonIndex
// Check for fields segment (braces come after bracket)
const braceStart = content.indexOf(OPEN_BRACE, bracketEnd)
if (braceStart !== -1 && braceStart < content.indexOf(COLON, bracketEnd)) {
const braceStart = findUnquotedChar(content, OPEN_BRACE, bracketEnd)
if (braceStart !== -1 && braceStart < findUnquotedChar(content, COLON, bracketEnd)) {
const gapBeforeBrace = content.slice(bracketEnd + 1, braceStart)
if (gapBeforeBrace !== '') {
if (strict) {
@@ -64,14 +70,14 @@ export function parseArrayHeaderLine(
return
}
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
if (foundBraceEnd !== -1) {
braceEnd = foundBraceEnd + 1
}
}
// Now find colon after brackets and braces
colonIndex = content.indexOf(COLON, Math.max(bracketEnd, braceEnd))
colonIndex = findUnquotedChar(content, COLON, Math.max(bracketEnd, braceEnd))
if (colonIndex === -1) {
return
}
@@ -113,7 +119,7 @@ export function parseArrayHeaderLine(
// Check for fields segment
let fields: string[] | undefined
if (braceStart !== -1 && braceStart < colonIndex) {
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
if (foundBraceEnd !== -1 && foundBraceEnd < colonIndex) {
const fieldsContent = content.slice(braceStart + 1, foundBraceEnd)
+4 -3
View File
@@ -1,5 +1,6 @@
import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ParsedLine } from '../types.ts'
import { COLON, LIST_ITEM_PREFIX } from '../constants.ts'
import { findUnquotedChar } from '../shared/string-utils.ts'
import { ToonDecodeError } from './errors.ts'
// #region Count and structure validation
@@ -91,9 +92,9 @@ export function validateNoBlankLinesInRange(
/**
* Checks if a line is a data row (vs a key-value pair) in a tabular array.
*/
function isDataRow(content: string, delimiter: Delimiter): boolean {
const colonPos = content.indexOf(COLON)
const delimiterPos = content.indexOf(delimiter)
export function isDataRow(content: string, delimiter: Delimiter): boolean {
const colonPos = findUnquotedChar(content, COLON)
const delimiterPos = findUnquotedChar(content, delimiter)
// No colon = definitely a data row
if (colonPos === -1) {
+1 -1
View File
@@ -241,7 +241,7 @@ export function isTabularArray(
// Check that all header keys exist in the row and all values are primitives
for (const key of header) {
if (!(key in row)) {
if (!Object.hasOwn(row, key)) {
return false
}
if (!isJsonPrimitive(row[key])) {
+2 -1
View File
@@ -1,4 +1,5 @@
import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types.ts'
import { setOwnProperty } from '../shared/object-utils.ts'
// #region Normalization (unknown → JsonValue)
@@ -76,7 +77,7 @@ export function normalizeValue(value: unknown): JsonValue {
for (const key in value) {
if (Object.hasOwn(value, key)) {
encodedValues[key] = normalizeValue(value[key])
setOwnProperty(encodedValues, key, normalizeValue(value[key]))
}
}
+2 -1
View File
@@ -1,4 +1,5 @@
import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types.ts'
import { setOwnProperty } from '../shared/object-utils.ts'
import { isJsonArray, isJsonObject, normalizeValue } from './normalize.ts'
/**
@@ -83,7 +84,7 @@ function transformObject(
const normalizedValue = normalizeValue(replacedValue)
// Recursively transform children of the replaced value
result[key] = transformChildren(normalizedValue, replacer, childPath)
setOwnProperty(result, key, transformChildren(normalizedValue, replacer, childPath))
}
return result
+33
View File
@@ -0,0 +1,33 @@
import type { JsonObject, JsonValue } from '../types.ts'
/**
* Reads an own data property, treating inherited and absent keys alike.
*
* @remarks
* Keys such as `__proto__` must not resolve through the prototype chain.
*/
export function getOwnProperty(target: JsonObject, key: string): JsonValue | undefined {
return Object.hasOwn(target, key) ? target[key] : undefined
}
/**
* Assigns an own data property without invoking inherited accessors.
*
* @remarks
* Plain assignment of `__proto__` would hit the `Object.prototype` setter and
* corrupt the prototype chain; `defineProperty` avoids that but is markedly
* slower, so every other key takes plain assignment.
*/
export function setOwnProperty(target: JsonObject, key: string, value: JsonValue): void {
if (key === '__proto__') {
Object.defineProperty(target, key, {
value,
enumerable: true,
writable: true,
configurable: true,
})
return
}
target[key] = value
}
+10
View File
@@ -36,6 +36,16 @@ describe('toonDecodeError line context', () => {
expect(error.message).toMatch(/tabular|rows?/i)
})
it('reports a malformed array length marker with line and source', () => {
const error = captureDecodeError('rules[-1]:\n - a\n')
expect(error).toBeInstanceOf(SyntaxError)
expect(error.line).toBe(1)
expect(error.source).toBe('rules[-1]:')
expect(error.message).toMatch(/^Line 1: /)
expect(error.message).toMatch(/invalid array length/i)
})
it('reports indentation errors with line and source', () => {
const error = captureDecodeError('a:\n b: 1\n')
@@ -0,0 +1,85 @@
import { describe, expect, it } from 'vitest'
import { decode } from '../src/index'
describe('decode security hardening', () => {
const prototypeKey = '__proto__'
it('keeps direct __proto__ keys as own data properties', () => {
const marker = '__toonDirectPolluted'
try {
const decoded = decode(`__proto__:\n ${marker}: true\n`) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey][marker]).toBe(true)
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
}
finally {
delete (Object.prototype as Record<string, unknown>)[marker]
}
})
it.each([
['primitive', '__proto__: true', true],
['array', '__proto__[2]: 1,2', [1, 2]],
])('keeps direct __proto__ %s values as own data properties', (_name, input, expected) => {
const decoded = decode(input) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey]).toEqual(expected)
})
it('does not follow Object.prototype during dotted path expansion', () => {
const marker = '__toonExpandedPolluted'
try {
const decoded = decode(`payload.__proto__.${marker}: true\n`, {
expandPaths: 'safe',
}) as Record<string, any>
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
expect(decoded.payload[prototypeKey][marker]).toBe(true)
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
}
finally {
delete (Object.prototype as Record<string, unknown>)[marker]
}
})
it('keeps constructor.prototype expansion within own properties', () => {
const marker = '__toonConstructorPolluted'
try {
const decoded = decode(`constructor.prototype.${marker}: true\n`, {
expandPaths: 'safe',
}) as Record<string, any>
expect(Object.hasOwn(decoded, 'constructor')).toBe(true)
expect(Object.hasOwn(decoded.constructor, 'prototype')).toBe(true)
expect(decoded.constructor.prototype[marker]).toBe(true)
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
}
finally {
delete (Object.prototype as Record<string, unknown>)[marker]
}
})
it('safely merges an expanded __proto__ path with a direct object', () => {
const decoded = decode(`__proto__.first: true\n__proto__:\n second: true\n`, {
expandPaths: 'safe',
}) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey]).toEqual({ first: true, second: true })
})
it('safely overwrites a primitive __proto__ conflict in non-strict mode', () => {
const decoded = decode(`__proto__: 1\n__proto__.second: true\n`, {
expandPaths: 'safe',
strict: false,
}) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey]).toEqual({ second: true })
})
})
+40 -1
View File
@@ -13,7 +13,7 @@ import rootForm from '@toon-format/spec/tests/fixtures/decode/root-form.json'
import validationErrors from '@toon-format/spec/tests/fixtures/decode/validation-errors.json'
import whitespace from '@toon-format/spec/tests/fixtures/decode/whitespace.json'
import { describe, expect, it } from 'vitest'
import { decode } from '../src/index'
import { decode, encode } from '../src/index'
const fixtureFiles = [
primitives,
@@ -47,3 +47,42 @@ for (const fixtures of fixtureFiles) {
}
})
}
describe('quoted content opacity (round-trip)', () => {
const cases: [name: string, value: unknown][] = [
['bracket segment then colon in scalar', { a: '[2]: x' }],
['bracket segment then colon, count matches length', { a: '[1]: x' }],
['bracketed index with a later colon', { content: 'accept (process.argv[2]) and greet: name' }],
['brackets, braces, colons, commas and pipes in a scalar', { s: '{k: v}, [0]: a | b' }],
['structural characters in tabular cells', { rows: [{ id: 1, note: '[2]: x' }, { id: 2, note: 'a: b, c' }] }],
['structural characters in list items', { xs: ['[2]: x', 'a: b', 'p | q'] }],
['bracket-shaped text inside a quoted key', { 'k [2]: y': 1 }],
['quoted key opens an array header', { 'quoted key': [1, 2] }],
['embedded quoted bracket in a key that opens an array header', { 'has "[2]" in it': [3, 4, 5] }],
]
for (const [name, value] of cases) {
it(`round-trips ${name}`, () => {
expect(decode(encode(value))).toEqual(value)
})
}
it('round-trips the Nostr event with a bracketed index and later colon in content', () => {
const event = {
id: '68f5c016e5a3128d7af740e088fc5d94e56edda4205fffa56aa3d58fe6bb55ee',
pubkey: '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5',
kind: 1621,
content: 'Currently index.js always prints "Hello, world!". It should accept an optional name as the first CLI argument (process.argv[2]) and greet that name instead, falling back to "world" when no argument is given. Example: `node index.js Ada` -> "Hello, Ada!".',
tags: [
['a', '30617:3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5:hello-compare-rig'],
['p', '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5'],
['subject', 'greeting should accept a name argument'],
['t', 'enhancement'],
],
created_at: 1783027036,
sig: '3220fa9dcb7af14b9a970f5d25f4ac13ebb4848c91a83bf496cdef17a839d4475773defb38a20d64b250227eb5aa457eb77358c5f32c4c7935397a3430b102db',
}
expect(decode(encode(event))).toEqual(event)
})
})
+73
View File
@@ -38,6 +38,16 @@ describe('streaming decode', () => {
])
})
it('materializes __proto__ as an own property', () => {
const prototypeKey = '__proto__'
const lines = ['__proto__:', ' safe: true']
const result = buildValueFromEvents(decodeStreamSync(lines)) as Record<string, unknown>
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
expect(result[prototypeKey]).toEqual({ safe: true })
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
})
it('decodes inline primitive array', () => {
const input = 'scores[3]: 95, 87, 92'
const lines = input.split('\n')
@@ -200,6 +210,69 @@ describe('streaming decode', () => {
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
})
it('locates the value after an escaped quoted key containing a colon', async () => {
const lines = ['"\\t:x": v']
const events = await collect(decodeStream(asyncLines(lines)))
expect(events).toEqual([
{ type: 'startObject' },
{ type: 'key', key: '\t:x', wasQuoted: true },
{ type: 'primitive', value: 'v' },
{ type: 'endObject' },
])
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
})
it('keeps a quoted bracket-then-colon scalar opaque, matching decodeStreamSync', async () => {
const lines = ['a: "[1]: x"']
const events = await collect(decodeStream(asyncLines(lines)))
expect(events).toEqual([
{ type: 'startObject' },
{ type: 'key', key: 'a' },
{ type: 'primitive', value: '[1]: x' },
{ type: 'endObject' },
])
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
})
it('keeps an unquoted bracket-colon scalar whole, matching decodeStreamSync', async () => {
const lines = ['key: foo [2]: bar']
const events = await collect(decodeStream(asyncLines(lines)))
expect(events).toEqual([
{ type: 'startObject' },
{ type: 'key', key: 'key' },
{ type: 'primitive', value: 'foo [2]: bar' },
{ type: 'endObject' },
])
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
})
it('keeps a colon-bearing value such as a URL intact', async () => {
const lines = ['a: http://x']
const events = await collect(decodeStream(asyncLines(lines)))
expect(events).toEqual([
{ type: 'startObject' },
{ type: 'key', key: 'a' },
{ type: 'primitive', value: 'http://x' },
{ type: 'endObject' },
])
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
})
it('materializes __proto__ as an own property', async () => {
const prototypeKey = '__proto__'
const lines = ['__proto__:', ' safe: true']
const events = await collect(decodeStream(asyncLines(lines)))
const result = await buildValueFromEventsAsync(asyncEvents(events)) as Record<string, unknown>
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
expect(result[prototypeKey]).toEqual({ safe: true })
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
})
it('rejects expandPaths option', async () => {
const lines = ['name: Alice']
@@ -0,0 +1,71 @@
import type { EncodeReplacer } from '../src/index'
import { describe, expect, it } from 'vitest'
import { decode, encode } from '../src/index'
describe('encode security hardening', () => {
const prototypeKey = '__proto__'
it('preserves own __proto__ properties through an encode/decode round trip', () => {
const marker = '__toonEncodedPolluted'
const input = JSON.parse(`{"__proto__":{"${marker}":true},"b":2}`)
try {
const decoded = decode(encode(input)) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey][marker]).toBe(true)
expect(decoded.b).toBe(2)
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
}
finally {
delete (Object.prototype as Record<string, unknown>)[marker]
}
})
it('preserves nested own __proto__ properties introduced by a replacer', () => {
const marker = '__toonNestedReplacerPolluted'
const replacement = JSON.parse(`{"__proto__":{"${marker}":true},"safe":1}`)
const replacer: EncodeReplacer = (key, value) => key === 'payload' ? replacement : value
try {
const decoded = decode(encode({ payload: null }, { replacer })) as Record<string, any>
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
expect(decoded.payload[prototypeKey][marker]).toBe(true)
expect(decoded.payload.safe).toBe(1)
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
}
finally {
delete (Object.prototype as Record<string, unknown>)[marker]
}
})
it('preserves own __proto__ properties through key folding and expansion', () => {
const input = JSON.parse('{"__proto__":{"inner":{"folded":true}}}')
const encoded = encode(input, { keyFolding: 'safe' })
const decoded = decode(encoded, { expandPaths: 'safe' }) as Record<string, any>
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
expect(decoded[prototypeKey].inner.folded).toBe(true)
})
it('does not use inherited properties to classify tabular rows', () => {
const inheritedKey = '__toonInheritedTabularField'
const input = [{ [inheritedKey]: 'user' }, { other: 'kept' }]
// eslint-disable-next-line no-extend-native -- Simulate a pre-polluted environment
Object.defineProperty(Object.prototype, inheritedKey, {
value: 'admin',
enumerable: true,
writable: true,
configurable: true,
})
try {
expect(decode(encode(input))).toEqual(input)
}
finally {
delete (Object.prototype as Record<string, unknown>)[inheritedKey]
}
})
})
+37
View File
@@ -114,6 +114,43 @@ describe('JavaScript-specific type normalization', () => {
})
})
describe('canonical number form', () => {
it('encodes zero as canonical decimal', () => {
expect(encode(0)).toBe('0')
expect(decode(encode(0))).toBe(0)
})
it('encodes 1e-6 (lower boundary) as canonical decimal', () => {
expect(encode(1e-6)).toBe('0.000001')
expect(decode(encode(1e-6))).toBe(1e-6)
})
it('encodes 1e-7 (just below lower boundary) as exponent form', () => {
expect(encode(1e-7)).toBe('1e-7')
expect(decode(encode(1e-7))).toBe(1e-7)
})
it('encodes 9.999999e20 (near upper boundary) as canonical decimal', () => {
expect(encode(9.999999e20)).toBe('999999900000000000000')
expect(decode(encode(9.999999e20))).toBe(9.999999e20)
})
it('encodes 1e21 (upper boundary) as exponent form', () => {
expect(encode(1e21)).toBe('1e+21')
expect(decode(encode(1e21))).toBe(1e21)
})
it('encodes 1.5e20 as canonical decimal', () => {
expect(encode(1.5e20)).toBe('150000000000000000000')
expect(decode(encode(1.5e20))).toBe(1.5e20)
})
it('encodes 5e-324 (smallest subnormal) as exponent form', () => {
expect(encode(5e-324)).toBe('5e-324')
expect(decode(encode(5e-324))).toBe(5e-324)
})
})
describe('toJSON method support', () => {
it('calls toJSON method when object has it', () => {
const obj = {
+1656 -1696
View File
File diff suppressed because it is too large Load Diff
+17 -3
View File
@@ -1,8 +1,22 @@
minimumReleaseAgeExclude:
- '@ai-sdk/gateway@3.0.151'
- '@ai-sdk/google@3.0.94'
- '@ai-sdk/openai-compatible@2.0.61'
- '@ai-sdk/xai@3.0.108'
- ai@6.0.228
- tsdown@0.22.8
- '@toon-format/spec@3.3.2'
shellEmulator: true
trustPolicy: no-downgrade
trustPolicyExclude:
- vite@5.4.21
packages:
- docs
- benchmarks
- packages/*
onlyBuiltDependencies:
- '@parcel/watcher'
- esbuild
allowBuilds:
'@parcel/watcher': true
esbuild: true