Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1336074d43 | |||
| 352a5b017d | |||
| e2003970dd | |||
| bb9a09ad6c | |||
| 2013a7e09c | |||
| 9024af9e6a | |||
| 27b65c2931 | |||
| 94a2b7560b | |||
| 09a7b38f2c | |||
| 702301231c | |||
| a19a117919 | |||
| a52f2d2815 | |||
| a956757de0 | |||
| c0af15e592 | |||
| 5b66523e42 | |||
| e2b5611d37 |
@@ -21,12 +21,12 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
|
||||
@@ -22,12 +22,12 @@ jobs:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Only fetch the config file from the repository
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0 # Required for fetching tags and generating release notes
|
||||
persist-credentials: true
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
|
||||
@@ -4,7 +4,7 @@ The TOON specification has moved to a dedicated repository: [github.com/toon-for
|
||||
|
||||
## Current Version
|
||||
|
||||
**Version 3.2** (2026-05-20)
|
||||
**Version 3.3** (2026-05-21)
|
||||
|
||||
## Quick Links
|
||||
|
||||
|
||||
+12
-12
@@ -8,20 +8,20 @@
|
||||
"fetch:github-repos": "node scripts/fetch-github-repos.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.78",
|
||||
"@ai-sdk/google": "^3.0.75",
|
||||
"@ai-sdk/openai": "^3.0.64",
|
||||
"@ai-sdk/provider": "^3.0.10",
|
||||
"@ai-sdk/xai": "^3.0.91",
|
||||
"@clack/prompts": "^1.4.0",
|
||||
"@faker-js/faker": "^10.4.0",
|
||||
"ai": "^6.0.185",
|
||||
"csv-stringify": "^6.7.0",
|
||||
"fast-xml-parser": "^5.8.0",
|
||||
"@ai-sdk/anthropic": "^3.0.97",
|
||||
"@ai-sdk/google": "^3.0.94",
|
||||
"@ai-sdk/openai": "^3.0.85",
|
||||
"@ai-sdk/provider": "^3.0.14",
|
||||
"@ai-sdk/xai": "^3.0.108",
|
||||
"@clack/prompts": "^1.7.0",
|
||||
"@faker-js/faker": "^10.5.0",
|
||||
"ai": "^6.0.228",
|
||||
"csv-stringify": "^6.8.1",
|
||||
"fast-xml-parser": "^5.10.0",
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"ofetch": "^1.5.1",
|
||||
"p-map": "^7.0.4",
|
||||
"p-queue": "^9.3.0",
|
||||
"p-map": "^7.0.5",
|
||||
"p-queue": "^9.3.1",
|
||||
"unstorage": "^1.17.5",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ const config: Theme = {
|
||||
extends: DefaultTheme,
|
||||
enhanceApp({ app }) {
|
||||
app.config.globalProperties.$spec = {
|
||||
version: '3.2',
|
||||
version: '3.3',
|
||||
}
|
||||
app.component('CopyOrDownloadAsMarkdownButtons', CopyOrDownloadAsMarkdownButtons)
|
||||
app.component('PlaygroundLayout', PlaygroundLayout)
|
||||
|
||||
+2
-2
@@ -156,7 +156,7 @@ When using the `--stats` flag with encode, the CLI builds the full TOON string o
|
||||
| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
|
||||
| `--indent <number>` | Indentation size (default: `2`) |
|
||||
| `--stats` | Show token count estimates and savings (encode only) |
|
||||
| `--no-strict` | Skip decode validation; last-write-wins on duplicate keys |
|
||||
| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
|
||||
| `--keyFolding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
|
||||
| `--flattenDepth <number>` | Maximum segments to fold (default: `Infinity`) – requires `--keyFolding safe` |
|
||||
| `--expandPaths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
|
||||
@@ -231,7 +231,7 @@ Skip validation for faster, more forgiving decoding:
|
||||
toon data.toon --no-strict -o output.json
|
||||
```
|
||||
|
||||
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and delimiter consistency. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
|
||||
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
|
||||
|
||||
### Decode Error Output
|
||||
|
||||
|
||||
@@ -47,9 +47,11 @@ Community members have created implementations in additional languages:
|
||||
| **Kotlin** | [ktoon](https://github.com/lukelast/ktoon)| [@lukelast](https://github.com/lukelast) |
|
||||
| **Laravel Framework** | [laravel-toon](https://github.com/mischasigtermans/laravel-toon) | [@mischasigtermans](https://github.com/mischasigtermans) |
|
||||
| **Lua/Neovim** | [toon.nvim](https://github.com/thalesgelinger/toon.nvim) | [@thalesgelinger](https://github.com/thalesgelinger) |
|
||||
| **Matlab** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
|
||||
| **OCaml** | [ocaml-toon](https://github.com/davesnx/ocaml-toon) | [@davesnx](https://github.com/davesnx) |
|
||||
| **Perl** | [Data::TOON](https://github.com/ytnobody/p5-Data-TOON) | [@ytnobody](https://github.com/ytnobody) |
|
||||
| **PHP** | [toon-php](https://github.com/HelgeSverre/toon-php) | [@HelgeSverre](https://github.com/HelgeSverre) |
|
||||
| **Python** (C++ backend) | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
|
||||
| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
|
||||
| **R** | [toon](https://github.com/laresbernardo/toon) | [@laresbernardo](https://github.com/laresbernardo) |
|
||||
| **Ruby** | [toon-ruby](https://github.com/andrepcg/toon-ruby) | [@andrepcg](https://github.com/andrepcg) |
|
||||
|
||||
@@ -334,18 +334,7 @@ Other escapes (e.g., `\x`, `\0`, `\b`) are always rejected, as are lone-surrogat
|
||||
|
||||
### Type Conversions
|
||||
|
||||
Numbers are emitted in canonical decimal form (no exponent notation, no trailing zeros). Non-JSON types are normalized before encoding:
|
||||
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| Finite number | Canonical decimal (e.g., `1e6` → `1000000`, `1.5000` → `1.5`, `-0` → `0`) |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (within safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
|
||||
| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
|
||||
| `Set` | Array of normalized values |
|
||||
| `Map` | Object with `String(key)` keys |
|
||||
| `undefined`, `function`, `symbol` | `null` |
|
||||
Numbers are emitted in canonical decimal form for values in the §2 carve-out range; exponent notation is permitted outside. Non-JSON types (`NaN`, `Infinity`, `BigInt`, `Date`, `Set`, `Map`, `undefined`, etc.) are normalized before encoding – see [API Reference – Type Normalization](/reference/api#type-normalization) for the full mapping.
|
||||
|
||||
Decoders accept both decimal and exponent forms on input (e.g., `42`, `-3.14`, `1e-6`), and treat tokens with forbidden leading zeros (e.g., `"05"`) as strings, not numbers.
|
||||
|
||||
|
||||
+14
-14
@@ -25,9 +25,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
|
||||
|
||||
```toon
|
||||
users[3]{id,name,role,lastLogin}:
|
||||
1,Alice,admin,2025-01-15T10:30:00Z
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
1,Alice,admin,"2025-01-15T10:30:00Z"
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
Task: Summarize the user roles and their last activity.
|
||||
@@ -52,9 +52,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
|
||||
|
||||
```toon
|
||||
users[3]{id,name,role,lastLogin}:
|
||||
1,Alice,admin,2025-01-15T10:30:00Z
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
1,Alice,admin,"2025-01-15T10:30:00Z"
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
Task: Return only users with role "user" as TOON. Use the same header format. Set [N] to match the row count. Output only the code block.
|
||||
@@ -64,8 +64,8 @@ Task: Return only users with role "user" as TOON. Use the same header format. Se
|
||||
|
||||
```toon
|
||||
users[2]{id,name,role,lastLogin}:
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
The model adjusts `[N]` to `2` and generates two rows.
|
||||
@@ -166,10 +166,10 @@ System logs in TOON format (tab-separated):
|
||||
|
||||
```toon
|
||||
events[4 ]{id level message timestamp}:
|
||||
1 error Connection timeout 2025-01-15T10:00:00Z
|
||||
2 warn Slow query 2025-01-15T10:05:00Z
|
||||
3 info User login 2025-01-15T10:10:00Z
|
||||
4 error Database error 2025-01-15T10:15:00Z
|
||||
1 error Connection timeout "2025-01-15T10:00:00Z"
|
||||
2 warn Slow query "2025-01-15T10:05:00Z"
|
||||
3 info User login "2025-01-15T10:10:00Z"
|
||||
4 error Database error "2025-01-15T10:15:00Z"
|
||||
```
|
||||
|
||||
Task: Return only error-level events as TOON. Use the same format.
|
||||
@@ -182,8 +182,8 @@ import { decode } from '@toon-format/toon'
|
||||
|
||||
const modelResponse = `
|
||||
events[2 ]{id level message timestamp}:
|
||||
1 error Connection timeout 2025-01-15T10:00:00Z
|
||||
4 error Database error 2025-01-15T10:15:00Z
|
||||
1 error Connection timeout "2025-01-15T10:00:00Z"
|
||||
4 error Database error "2025-01-15T10:15:00Z"
|
||||
`
|
||||
|
||||
const filtered = decode(modelResponse, { strict: true })
|
||||
|
||||
+2
-2
@@ -13,9 +13,9 @@
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"markdown-it-mathjax3": "^4.3.2",
|
||||
"uint8array-extras": "^1.5.0",
|
||||
"unocss": "^66.6.8",
|
||||
"unocss": "^66.7.5",
|
||||
"vitepress": "^1.6.4",
|
||||
"vitepress-plugin-llms": "^1.12.2",
|
||||
"vitepress-plugin-llms": "^1.13.3",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +59,8 @@ Non-JSON-serializable values are normalized before encoding:
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| `Object` with `toJSON()` method | Result of calling `toJSON()`, recursively normalized |
|
||||
| Finite number | Canonical decimal (no exponent, no leading/trailing zeros: `1e6` → `1000000`, `-0` → `0`) |
|
||||
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal (e.g., `1e6` → `1000000`, `-0` → `0`) |
|
||||
| Finite number outside that range | Exponent form permitted (e.g., `1e-7`, `1e+21`) |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (within safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
|
||||
@@ -256,7 +257,7 @@ console.log(encode(data, { replacer }))
|
||||
|
||||
```yaml
|
||||
metadata:
|
||||
created: 2025-01-01T00:00:00Z
|
||||
created: "2025-01-01T00:00:00Z"
|
||||
user:
|
||||
created: 2025-01-02
|
||||
```
|
||||
|
||||
@@ -40,7 +40,7 @@ Where $\Omega$ represents the data structure under comparison. If $\Delta > 0$,
|
||||
- must be quoted in JSON, and
|
||||
- can be left unquoted in TOON (no characters that would force quoting).
|
||||
Many examples assume values are numbers, booleans, null, or TOON-safe strings that can be unquoted in TOON but must be quoted in JSON.
|
||||
- **Numbers**: Both formats are assumed to use the same canonical decimal representation (no exponent notation), matching TOON's requirement. JSON could use exponent forms; we ignore that here to isolate structural differences.
|
||||
- **Numbers**: For this analysis only, both formats are assumed to use the same canonical decimal representation. JSON could use exponent forms; we ignore that here to isolate structural differences.
|
||||
- **ASCII/UTF-8**: Keys and structural tokens are assumed ASCII, so byte length equals character count ($|x|_{\text{utf8}} = |x|_{\text{char}}$). Non-ASCII content affects both formats similarly and does not change the structural conclusions.
|
||||
- **Nesting depth**: Closed-form expressions are given for flat structures and a single level of nesting. Each additional nesting level in TOON adds 2 bytes of indentation per nested line. At sufficient depth, the braces of compact JSON can win over TOON's indentation (as seen in [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon)).
|
||||
- **Byte vs token count**: Modern LLM tokenizers operate over UTF-8 bytes, so byte length is a good upper bound and first-order proxy for token count, even though the mapping is not exactly linear.
|
||||
|
||||
@@ -32,7 +32,7 @@ TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `cha
|
||||
Defines key terms like "indentation level", "active delimiter", "strict mode", and RFC2119 keywords (MUST, SHOULD, MAY).
|
||||
|
||||
[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model):
|
||||
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (no exponent notation, no leading/trailing zeros).
|
||||
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (canonical decimal for values in `[1e-6, 1e21)` or zero; exponent form permitted outside).
|
||||
|
||||
[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder):
|
||||
Defines how non-JSON types (Date, BigInt, NaN, Infinity, undefined, etc.) are normalized before encoding. Required reading for encoder implementers.
|
||||
@@ -95,7 +95,7 @@ How the spec evolves: major vs minor bumps and the extensibility policy.
|
||||
Licensing and IP terms for the specification.
|
||||
|
||||
[Appendix F: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-f-host-type-normalization-examples-informative):
|
||||
Non-normative guidance for Go, JavaScript, Python, and Rust implementations on normalizing language-specific types.
|
||||
Non-normative guidance for Go, JavaScript, Python, Rust, and Java implementations on normalizing language-specific types.
|
||||
|
||||
[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative):
|
||||
Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://github.com/toon-format/spec/tree/main/tests) for validating implementations.
|
||||
@@ -127,8 +127,9 @@ Key requirements:
|
||||
- Quote strings with active delimiter, colon, or structural characters
|
||||
- Emit array lengths `[N]` matching actual count
|
||||
- Preserve object key order
|
||||
- Normalize numbers to non-exponential decimal form
|
||||
- Emit numbers per §2 (canonical decimal in `[1e-6, 1e21)` or zero; exponent form permitted outside)
|
||||
- Convert `-0` to `0`, `NaN`/±Infinity to `null`
|
||||
- Emit booleans and null as lowercase literals (`true`, `false`, `null`)
|
||||
- No trailing spaces or trailing newline
|
||||
- When `keyFolding="safe"` is enabled, folding MUST follow §13.4:
|
||||
- Only fold IdentifierSegment keys (letters/digits/underscores, no dots),
|
||||
|
||||
@@ -352,7 +352,8 @@ See [Format Overview – Key Folding](/guide/format-overview#key-folding-optiona
|
||||
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| Finite number | Canonical decimal (no exponent, no trailing zeros) |
|
||||
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal |
|
||||
| Finite number outside that range | Exponent form permitted |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string |
|
||||
|
||||
@@ -3,7 +3,6 @@ import type { FlatConfigComposer } from 'eslint-flat-config-utils'
|
||||
import antfu from '@antfu/eslint-config'
|
||||
|
||||
const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
|
||||
pnpm: false,
|
||||
rules: {
|
||||
'no-cond-assign': 'off',
|
||||
},
|
||||
|
||||
+13
-13
@@ -1,15 +1,15 @@
|
||||
{
|
||||
"name": "@toon-format/monorepo",
|
||||
"type": "module",
|
||||
"version": "2.3.0",
|
||||
"version": "2.3.1",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"scripts": {
|
||||
"build": "pnpm -r --filter=./packages/** run build",
|
||||
"build": "pnpm -r --filter='./packages/**' run build",
|
||||
"automd": "automd",
|
||||
"docs:dev": "vitepress dev docs",
|
||||
"docs:build": "vitepress build docs",
|
||||
"docs:preview": "vitepress preview docs",
|
||||
"docs:dev": "pnpm -C docs run dev",
|
||||
"docs:build": "pnpm -C docs run build",
|
||||
"docs:preview": "pnpm -C docs run preview",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"test": "pnpm -r test",
|
||||
@@ -17,16 +17,16 @@
|
||||
"release": "bumpp -r"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^8.3.0",
|
||||
"@commitlint/types": "^21.0.1",
|
||||
"@types/node": "^25.9.1",
|
||||
"@antfu/eslint-config": "^9.1.0",
|
||||
"@commitlint/types": "^21.2.0",
|
||||
"@types/node": "^26.1.1",
|
||||
"automd": "^0.4.3",
|
||||
"bumpp": "^11.1.0",
|
||||
"eslint": "^10.4.0",
|
||||
"eslint": "^10.7.0",
|
||||
"eslint-flat-config-utils": "^3.2.0",
|
||||
"tsdown": "^0.22.0",
|
||||
"tsdown": "^0.22.8",
|
||||
"typescript": "^6.0.3",
|
||||
"vite": "^8.0.13",
|
||||
"vitest": "^4.1.7"
|
||||
"vite": "^8.1.4",
|
||||
"vitest": "^4.1.10"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@toon-format/cli",
|
||||
"type": "module",
|
||||
"version": "2.3.0",
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"version": "2.3.1",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"description": "CLI for JSON ↔ TOON conversion using @toon-format/toon",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
[](https://github.com/toon-format/toon/actions)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](https://github.com/toon-format/spec)
|
||||
[](https://github.com/toon-format/spec)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](./LICENSE)
|
||||
|
||||
@@ -134,13 +134,6 @@ hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
|
||||
- 🧺 **Tabular Arrays:** Uniform arrays of objects collapse into tables that declare fields once and stream row values line by line.
|
||||
- 🌐 **Multi-Language Ecosystem:** Spec-driven implementations in TypeScript, Python, Go, Rust, .NET, and other languages.
|
||||
|
||||
## What's New in v3.2
|
||||
|
||||
- Strict mode rejects duplicate sibling keys; with `strict: false` the last value wins.
|
||||
- Malformed array headers (`[03]`, `[1] foo:`, header/field delimiter mismatch) error in strict mode instead of silently degrading.
|
||||
- Tabular form excludes arrays that contain an empty `{}` element – those fall back to the expanded list form.
|
||||
- Nested arrays-of-objects can appear as list items via an explicit `- [N]:` header.
|
||||
|
||||
## Media Type & File Extension
|
||||
|
||||
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations) for normative details.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@toon-format/toon",
|
||||
"type": "module",
|
||||
"version": "2.3.0",
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"version": "2.3.1",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"description": "Token-Oriented Object Notation (TOON) – Compact, human-readable, schema-aware encoding of JSON for LLM prompts",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
@@ -38,6 +38,6 @@
|
||||
"test": "vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@toon-format/spec": "^3.2.0"
|
||||
"@toon-format/spec": "^3.3.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { findClosingQuote } from '../shared/string-utils.ts'
|
||||
import { ToonDecodeError, withLine } from './errors.ts'
|
||||
import { isArrayHeaderContent, isKeyValueContent, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser.ts'
|
||||
import { createScanState, parseLinesAsync, parseLinesSync } from './scanner.ts'
|
||||
import { assertExpectedCount, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
|
||||
import { assertExpectedCount, isDataRow, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
|
||||
|
||||
interface DecoderContext { indent: number, strict: boolean }
|
||||
|
||||
@@ -223,9 +223,8 @@ function* decodeKeyValueSync(
|
||||
}
|
||||
|
||||
// Regular key-value pair
|
||||
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const colonIndex = content.indexOf(COLON, key.length)
|
||||
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
|
||||
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const rest = content.slice(end).trim()
|
||||
|
||||
assertNoDuplicateKey(key, line, seenKeys)
|
||||
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
|
||||
@@ -354,6 +353,10 @@ function* decodeTabularArraySync(
|
||||
}
|
||||
|
||||
if (line.depth === rowDepth) {
|
||||
if (!isDataRow(line.content, header.delimiter)) {
|
||||
break
|
||||
}
|
||||
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
@@ -678,9 +681,8 @@ async function* decodeKeyValueAsync(
|
||||
}
|
||||
|
||||
// Regular key-value pair
|
||||
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const colonIndex = content.indexOf(COLON, key.length)
|
||||
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
|
||||
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const rest = content.slice(end).trim()
|
||||
|
||||
assertNoDuplicateKey(key, line, seenKeys)
|
||||
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
|
||||
@@ -788,6 +790,10 @@ async function* decodeTabularArrayAsync(
|
||||
}
|
||||
|
||||
if (line.depth === rowDepth) {
|
||||
if (!isDataRow(line.content, header.delimiter)) {
|
||||
break
|
||||
}
|
||||
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { JsonObject, JsonStreamEvent, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { QUOTED_KEY_MARKER } from './expand.ts'
|
||||
|
||||
// #region Build context types
|
||||
@@ -63,7 +64,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
throw new Error('Object startObject event without preceding key')
|
||||
}
|
||||
|
||||
parent.obj[parent.currentKey] = obj
|
||||
setOwnProperty(parent.obj, parent.currentKey, obj)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
@@ -116,7 +117,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
if (parent.currentKey === undefined) {
|
||||
throw new Error('Array startArray event without preceding key')
|
||||
}
|
||||
parent.obj[parent.currentKey] = arr
|
||||
setOwnProperty(parent.obj, parent.currentKey, arr)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
@@ -177,7 +178,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
if (parent.currentKey === undefined) {
|
||||
throw new Error('Primitive event without preceding key in object')
|
||||
}
|
||||
parent.obj[parent.currentKey] = event.value
|
||||
setOwnProperty(parent.obj, parent.currentKey, event.value)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { JsonObject, JsonValue } from '../types.ts'
|
||||
import { DOT } from '../constants.ts'
|
||||
import { isJsonObject } from '../encode/normalize.ts'
|
||||
import { getOwnProperty, setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { isIdentifierSegment } from '../shared/validation.ts'
|
||||
|
||||
// #region Path expansion (safe)
|
||||
@@ -73,7 +74,7 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
|
||||
const expandedValue = expandPathsSafe(keyValue, strict)
|
||||
|
||||
// Check for conflicts with already-expanded keys
|
||||
if (key in expandedObject) {
|
||||
if (Object.hasOwn(expandedObject, key)) {
|
||||
const conflictingValue = expandedObject[key]!
|
||||
// If both are objects, try to merge them
|
||||
if (canMerge(conflictingValue, expandedValue)) {
|
||||
@@ -87,12 +88,12 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
expandedObject[key] = expandedValue
|
||||
setOwnProperty(expandedObject, key, expandedValue)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No conflict - insert directly
|
||||
expandedObject[key] = expandedValue
|
||||
setOwnProperty(expandedObject, key, expandedValue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,12 +132,12 @@ function insertPathSafe(
|
||||
// Walk to the penultimate segment, creating objects as needed
|
||||
for (let i = 0; i < segments.length - 1; i++) {
|
||||
const currentSegment = segments[i]!
|
||||
const segmentValue = currentNode[currentSegment]
|
||||
const segmentValue = getOwnProperty(currentNode, currentSegment)
|
||||
|
||||
if (segmentValue === undefined) {
|
||||
// Create new intermediate object
|
||||
const newObj: JsonObject = {}
|
||||
currentNode[currentSegment] = newObj
|
||||
setOwnProperty(currentNode, currentSegment, newObj)
|
||||
currentNode = newObj
|
||||
}
|
||||
else if (isJsonObject(segmentValue)) {
|
||||
@@ -152,18 +153,18 @@ function insertPathSafe(
|
||||
}
|
||||
// Non-strict: overwrite with new object
|
||||
const newObj: JsonObject = {}
|
||||
currentNode[currentSegment] = newObj
|
||||
setOwnProperty(currentNode, currentSegment, newObj)
|
||||
currentNode = newObj
|
||||
}
|
||||
}
|
||||
|
||||
// Insert at the final segment
|
||||
const lastSeg = segments[segments.length - 1]!
|
||||
const destinationValue = currentNode[lastSeg]
|
||||
const destinationValue = getOwnProperty(currentNode, lastSeg)
|
||||
|
||||
if (destinationValue === undefined) {
|
||||
// No conflict - insert directly
|
||||
currentNode[lastSeg] = value
|
||||
setOwnProperty(currentNode, lastSeg, value)
|
||||
}
|
||||
else if (canMerge(destinationValue, value)) {
|
||||
// Both are objects - deep merge
|
||||
@@ -177,7 +178,7 @@ function insertPathSafe(
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
currentNode[lastSeg] = value
|
||||
setOwnProperty(currentNode, lastSeg, value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,13 +202,14 @@ function mergeObjects(
|
||||
strict: boolean,
|
||||
): void {
|
||||
for (const [key, sourceValue] of Object.entries(source)) {
|
||||
const targetValue = target[key]
|
||||
|
||||
if (targetValue === undefined) {
|
||||
if (!Object.hasOwn(target, key)) {
|
||||
// Key doesn't exist in target - copy it
|
||||
target[key] = sourceValue
|
||||
setOwnProperty(target, key, sourceValue)
|
||||
continue
|
||||
}
|
||||
else if (canMerge(targetValue, sourceValue)) {
|
||||
|
||||
const targetValue = target[key] as JsonValue
|
||||
if (canMerge(targetValue, sourceValue)) {
|
||||
// Both are objects - recursively merge
|
||||
mergeObjects(targetValue as JsonObject, sourceValue as JsonObject, strict)
|
||||
}
|
||||
@@ -219,7 +221,7 @@ function mergeObjects(
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
target[key] = sourceValue
|
||||
setOwnProperty(target, key, sourceValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,14 +34,20 @@ export function parseArrayHeaderLine(
|
||||
}
|
||||
else {
|
||||
// Unquoted key - find first bracket
|
||||
bracketStart = content.indexOf(OPEN_BRACKET)
|
||||
bracketStart = findUnquotedChar(content, OPEN_BRACKET)
|
||||
}
|
||||
|
||||
if (bracketStart === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
const bracketEnd = content.indexOf(CLOSE_BRACKET, bracketStart)
|
||||
// A header key can't contain an unquoted colon, so this is a key-value line
|
||||
const firstColonIndex = findUnquotedChar(content, COLON)
|
||||
if (firstColonIndex !== -1 && firstColonIndex < bracketStart) {
|
||||
return
|
||||
}
|
||||
|
||||
const bracketEnd = findUnquotedChar(content, CLOSE_BRACKET, bracketStart)
|
||||
if (bracketEnd === -1) {
|
||||
return
|
||||
}
|
||||
@@ -51,8 +57,8 @@ export function parseArrayHeaderLine(
|
||||
let braceEnd = colonIndex
|
||||
|
||||
// Check for fields segment (braces come after bracket)
|
||||
const braceStart = content.indexOf(OPEN_BRACE, bracketEnd)
|
||||
if (braceStart !== -1 && braceStart < content.indexOf(COLON, bracketEnd)) {
|
||||
const braceStart = findUnquotedChar(content, OPEN_BRACE, bracketEnd)
|
||||
if (braceStart !== -1 && braceStart < findUnquotedChar(content, COLON, bracketEnd)) {
|
||||
const gapBeforeBrace = content.slice(bracketEnd + 1, braceStart)
|
||||
if (gapBeforeBrace !== '') {
|
||||
if (strict) {
|
||||
@@ -64,14 +70,14 @@ export function parseArrayHeaderLine(
|
||||
return
|
||||
}
|
||||
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1) {
|
||||
braceEnd = foundBraceEnd + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Now find colon after brackets and braces
|
||||
colonIndex = content.indexOf(COLON, Math.max(bracketEnd, braceEnd))
|
||||
colonIndex = findUnquotedChar(content, COLON, Math.max(bracketEnd, braceEnd))
|
||||
if (colonIndex === -1) {
|
||||
return
|
||||
}
|
||||
@@ -113,7 +119,7 @@ export function parseArrayHeaderLine(
|
||||
// Check for fields segment
|
||||
let fields: string[] | undefined
|
||||
if (braceStart !== -1 && braceStart < colonIndex) {
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1 && foundBraceEnd < colonIndex) {
|
||||
const fieldsContent = content.slice(braceStart + 1, foundBraceEnd)
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ParsedLine } from '../types.ts'
|
||||
import { COLON, LIST_ITEM_PREFIX } from '../constants.ts'
|
||||
import { findUnquotedChar } from '../shared/string-utils.ts'
|
||||
import { ToonDecodeError } from './errors.ts'
|
||||
|
||||
// #region Count and structure validation
|
||||
@@ -91,9 +92,9 @@ export function validateNoBlankLinesInRange(
|
||||
/**
|
||||
* Checks if a line is a data row (vs a key-value pair) in a tabular array.
|
||||
*/
|
||||
function isDataRow(content: string, delimiter: Delimiter): boolean {
|
||||
const colonPos = content.indexOf(COLON)
|
||||
const delimiterPos = content.indexOf(delimiter)
|
||||
export function isDataRow(content: string, delimiter: Delimiter): boolean {
|
||||
const colonPos = findUnquotedChar(content, COLON)
|
||||
const delimiterPos = findUnquotedChar(content, delimiter)
|
||||
|
||||
// No colon = definitely a data row
|
||||
if (colonPos === -1) {
|
||||
|
||||
@@ -241,7 +241,7 @@ export function isTabularArray(
|
||||
|
||||
// Check that all header keys exist in the row and all values are primitives
|
||||
for (const key of header) {
|
||||
if (!(key in row)) {
|
||||
if (!Object.hasOwn(row, key)) {
|
||||
return false
|
||||
}
|
||||
if (!isJsonPrimitive(row[key])) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
|
||||
// #region Normalization (unknown → JsonValue)
|
||||
|
||||
@@ -76,7 +77,7 @@ export function normalizeValue(value: unknown): JsonValue {
|
||||
|
||||
for (const key in value) {
|
||||
if (Object.hasOwn(value, key)) {
|
||||
encodedValues[key] = normalizeValue(value[key])
|
||||
setOwnProperty(encodedValues, key, normalizeValue(value[key]))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { isJsonArray, isJsonObject, normalizeValue } from './normalize.ts'
|
||||
|
||||
/**
|
||||
@@ -83,7 +84,7 @@ function transformObject(
|
||||
const normalizedValue = normalizeValue(replacedValue)
|
||||
|
||||
// Recursively transform children of the replaced value
|
||||
result[key] = transformChildren(normalizedValue, replacer, childPath)
|
||||
setOwnProperty(result, key, transformChildren(normalizedValue, replacer, childPath))
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { JsonObject, JsonValue } from '../types.ts'
|
||||
|
||||
/**
|
||||
* Reads an own data property, treating inherited and absent keys alike.
|
||||
*
|
||||
* @remarks
|
||||
* Keys such as `__proto__` must not resolve through the prototype chain.
|
||||
*/
|
||||
export function getOwnProperty(target: JsonObject, key: string): JsonValue | undefined {
|
||||
return Object.hasOwn(target, key) ? target[key] : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns an own data property without invoking inherited accessors.
|
||||
*
|
||||
* @remarks
|
||||
* Plain assignment of `__proto__` would hit the `Object.prototype` setter and
|
||||
* corrupt the prototype chain; `defineProperty` avoids that but is markedly
|
||||
* slower, so every other key takes plain assignment.
|
||||
*/
|
||||
export function setOwnProperty(target: JsonObject, key: string, value: JsonValue): void {
|
||||
if (key === '__proto__') {
|
||||
Object.defineProperty(target, key, {
|
||||
value,
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
configurable: true,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
target[key] = value
|
||||
}
|
||||
@@ -36,6 +36,16 @@ describe('toonDecodeError line context', () => {
|
||||
expect(error.message).toMatch(/tabular|rows?/i)
|
||||
})
|
||||
|
||||
it('reports a malformed array length marker with line and source', () => {
|
||||
const error = captureDecodeError('rules[-1]:\n - a\n')
|
||||
|
||||
expect(error).toBeInstanceOf(SyntaxError)
|
||||
expect(error.line).toBe(1)
|
||||
expect(error.source).toBe('rules[-1]:')
|
||||
expect(error.message).toMatch(/^Line 1: /)
|
||||
expect(error.message).toMatch(/invalid array length/i)
|
||||
})
|
||||
|
||||
it('reports indentation errors with line and source', () => {
|
||||
const error = captureDecodeError('a:\n b: 1\n')
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode } from '../src/index'
|
||||
|
||||
describe('decode security hardening', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
|
||||
it('keeps direct __proto__ keys as own data properties', () => {
|
||||
const marker = '__toonDirectPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`__proto__:\n ${marker}: true\n`) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey][marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
['primitive', '__proto__: true', true],
|
||||
['array', '__proto__[2]: 1,2', [1, 2]],
|
||||
])('keeps direct __proto__ %s values as own data properties', (_name, input, expected) => {
|
||||
const decoded = decode(input) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual(expected)
|
||||
})
|
||||
|
||||
it('does not follow Object.prototype during dotted path expansion', () => {
|
||||
const marker = '__toonExpandedPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`payload.__proto__.${marker}: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
|
||||
expect(decoded.payload[prototypeKey][marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps constructor.prototype expansion within own properties', () => {
|
||||
const marker = '__toonConstructorPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`constructor.prototype.${marker}: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, 'constructor')).toBe(true)
|
||||
expect(Object.hasOwn(decoded.constructor, 'prototype')).toBe(true)
|
||||
expect(decoded.constructor.prototype[marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('safely merges an expanded __proto__ path with a direct object', () => {
|
||||
const decoded = decode(`__proto__.first: true\n__proto__:\n second: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual({ first: true, second: true })
|
||||
})
|
||||
|
||||
it('safely overwrites a primitive __proto__ conflict in non-strict mode', () => {
|
||||
const decoded = decode(`__proto__: 1\n__proto__.second: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
strict: false,
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual({ second: true })
|
||||
})
|
||||
})
|
||||
@@ -13,7 +13,7 @@ import rootForm from '@toon-format/spec/tests/fixtures/decode/root-form.json'
|
||||
import validationErrors from '@toon-format/spec/tests/fixtures/decode/validation-errors.json'
|
||||
import whitespace from '@toon-format/spec/tests/fixtures/decode/whitespace.json'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode } from '../src/index'
|
||||
import { decode, encode } from '../src/index'
|
||||
|
||||
const fixtureFiles = [
|
||||
primitives,
|
||||
@@ -47,3 +47,42 @@ for (const fixtures of fixtureFiles) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
describe('quoted content opacity (round-trip)', () => {
|
||||
const cases: [name: string, value: unknown][] = [
|
||||
['bracket segment then colon in scalar', { a: '[2]: x' }],
|
||||
['bracket segment then colon, count matches length', { a: '[1]: x' }],
|
||||
['bracketed index with a later colon', { content: 'accept (process.argv[2]) and greet: name' }],
|
||||
['brackets, braces, colons, commas and pipes in a scalar', { s: '{k: v}, [0]: a | b' }],
|
||||
['structural characters in tabular cells', { rows: [{ id: 1, note: '[2]: x' }, { id: 2, note: 'a: b, c' }] }],
|
||||
['structural characters in list items', { xs: ['[2]: x', 'a: b', 'p | q'] }],
|
||||
['bracket-shaped text inside a quoted key', { 'k [2]: y': 1 }],
|
||||
['quoted key opens an array header', { 'quoted key': [1, 2] }],
|
||||
['embedded quoted bracket in a key that opens an array header', { 'has "[2]" in it': [3, 4, 5] }],
|
||||
]
|
||||
|
||||
for (const [name, value] of cases) {
|
||||
it(`round-trips ${name}`, () => {
|
||||
expect(decode(encode(value))).toEqual(value)
|
||||
})
|
||||
}
|
||||
|
||||
it('round-trips the Nostr event with a bracketed index and later colon in content', () => {
|
||||
const event = {
|
||||
id: '68f5c016e5a3128d7af740e088fc5d94e56edda4205fffa56aa3d58fe6bb55ee',
|
||||
pubkey: '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5',
|
||||
kind: 1621,
|
||||
content: 'Currently index.js always prints "Hello, world!". It should accept an optional name as the first CLI argument (process.argv[2]) and greet that name instead, falling back to "world" when no argument is given. Example: `node index.js Ada` -> "Hello, Ada!".',
|
||||
tags: [
|
||||
['a', '30617:3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5:hello-compare-rig'],
|
||||
['p', '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5'],
|
||||
['subject', 'greeting should accept a name argument'],
|
||||
['t', 'enhancement'],
|
||||
],
|
||||
created_at: 1783027036,
|
||||
sig: '3220fa9dcb7af14b9a970f5d25f4ac13ebb4848c91a83bf496cdef17a839d4475773defb38a20d64b250227eb5aa457eb77358c5f32c4c7935397a3430b102db',
|
||||
}
|
||||
|
||||
expect(decode(encode(event))).toEqual(event)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -38,6 +38,16 @@ describe('streaming decode', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('materializes __proto__ as an own property', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
const lines = ['__proto__:', ' safe: true']
|
||||
const result = buildValueFromEvents(decodeStreamSync(lines)) as Record<string, unknown>
|
||||
|
||||
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
|
||||
expect(result[prototypeKey]).toEqual({ safe: true })
|
||||
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
|
||||
})
|
||||
|
||||
it('decodes inline primitive array', () => {
|
||||
const input = 'scores[3]: 95, 87, 92'
|
||||
const lines = input.split('\n')
|
||||
@@ -200,6 +210,69 @@ describe('streaming decode', () => {
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('locates the value after an escaped quoted key containing a colon', async () => {
|
||||
const lines = ['"\\t:x": v']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: '\t:x', wasQuoted: true },
|
||||
{ type: 'primitive', value: 'v' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps a quoted bracket-then-colon scalar opaque, matching decodeStreamSync', async () => {
|
||||
const lines = ['a: "[1]: x"']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'a' },
|
||||
{ type: 'primitive', value: '[1]: x' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps an unquoted bracket-colon scalar whole, matching decodeStreamSync', async () => {
|
||||
const lines = ['key: foo [2]: bar']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'key' },
|
||||
{ type: 'primitive', value: 'foo [2]: bar' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps a colon-bearing value such as a URL intact', async () => {
|
||||
const lines = ['a: http://x']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'a' },
|
||||
{ type: 'primitive', value: 'http://x' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('materializes __proto__ as an own property', async () => {
|
||||
const prototypeKey = '__proto__'
|
||||
const lines = ['__proto__:', ' safe: true']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
const result = await buildValueFromEventsAsync(asyncEvents(events)) as Record<string, unknown>
|
||||
|
||||
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
|
||||
expect(result[prototypeKey]).toEqual({ safe: true })
|
||||
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
|
||||
})
|
||||
|
||||
it('rejects expandPaths option', async () => {
|
||||
const lines = ['name: Alice']
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { EncodeReplacer } from '../src/index'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode, encode } from '../src/index'
|
||||
|
||||
describe('encode security hardening', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
|
||||
it('preserves own __proto__ properties through an encode/decode round trip', () => {
|
||||
const marker = '__toonEncodedPolluted'
|
||||
const input = JSON.parse(`{"__proto__":{"${marker}":true},"b":2}`)
|
||||
|
||||
try {
|
||||
const decoded = decode(encode(input)) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey][marker]).toBe(true)
|
||||
expect(decoded.b).toBe(2)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves nested own __proto__ properties introduced by a replacer', () => {
|
||||
const marker = '__toonNestedReplacerPolluted'
|
||||
const replacement = JSON.parse(`{"__proto__":{"${marker}":true},"safe":1}`)
|
||||
const replacer: EncodeReplacer = (key, value) => key === 'payload' ? replacement : value
|
||||
|
||||
try {
|
||||
const decoded = decode(encode({ payload: null }, { replacer })) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
|
||||
expect(decoded.payload[prototypeKey][marker]).toBe(true)
|
||||
expect(decoded.payload.safe).toBe(1)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves own __proto__ properties through key folding and expansion', () => {
|
||||
const input = JSON.parse('{"__proto__":{"inner":{"folded":true}}}')
|
||||
const encoded = encode(input, { keyFolding: 'safe' })
|
||||
const decoded = decode(encoded, { expandPaths: 'safe' }) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey].inner.folded).toBe(true)
|
||||
})
|
||||
|
||||
it('does not use inherited properties to classify tabular rows', () => {
|
||||
const inheritedKey = '__toonInheritedTabularField'
|
||||
const input = [{ [inheritedKey]: 'user' }, { other: 'kept' }]
|
||||
|
||||
// eslint-disable-next-line no-extend-native -- Simulate a pre-polluted environment
|
||||
Object.defineProperty(Object.prototype, inheritedKey, {
|
||||
value: 'admin',
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
configurable: true,
|
||||
})
|
||||
|
||||
try {
|
||||
expect(decode(encode(input))).toEqual(input)
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[inheritedKey]
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -114,6 +114,43 @@ describe('JavaScript-specific type normalization', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('canonical number form', () => {
|
||||
it('encodes zero as canonical decimal', () => {
|
||||
expect(encode(0)).toBe('0')
|
||||
expect(decode(encode(0))).toBe(0)
|
||||
})
|
||||
|
||||
it('encodes 1e-6 (lower boundary) as canonical decimal', () => {
|
||||
expect(encode(1e-6)).toBe('0.000001')
|
||||
expect(decode(encode(1e-6))).toBe(1e-6)
|
||||
})
|
||||
|
||||
it('encodes 1e-7 (just below lower boundary) as exponent form', () => {
|
||||
expect(encode(1e-7)).toBe('1e-7')
|
||||
expect(decode(encode(1e-7))).toBe(1e-7)
|
||||
})
|
||||
|
||||
it('encodes 9.999999e20 (near upper boundary) as canonical decimal', () => {
|
||||
expect(encode(9.999999e20)).toBe('999999900000000000000')
|
||||
expect(decode(encode(9.999999e20))).toBe(9.999999e20)
|
||||
})
|
||||
|
||||
it('encodes 1e21 (upper boundary) as exponent form', () => {
|
||||
expect(encode(1e21)).toBe('1e+21')
|
||||
expect(decode(encode(1e21))).toBe(1e21)
|
||||
})
|
||||
|
||||
it('encodes 1.5e20 as canonical decimal', () => {
|
||||
expect(encode(1.5e20)).toBe('150000000000000000000')
|
||||
expect(decode(encode(1.5e20))).toBe(1.5e20)
|
||||
})
|
||||
|
||||
it('encodes 5e-324 (smallest subnormal) as exponent form', () => {
|
||||
expect(encode(5e-324)).toBe('5e-324')
|
||||
expect(decode(encode(5e-324))).toBe(5e-324)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toJSON method support', () => {
|
||||
it('calls toJSON method when object has it', () => {
|
||||
const obj = {
|
||||
|
||||
Generated
+1656
-1696
File diff suppressed because it is too large
Load Diff
+17
-3
@@ -1,8 +1,22 @@
|
||||
minimumReleaseAgeExclude:
|
||||
- '@ai-sdk/gateway@3.0.151'
|
||||
- '@ai-sdk/google@3.0.94'
|
||||
- '@ai-sdk/openai-compatible@2.0.61'
|
||||
- '@ai-sdk/xai@3.0.108'
|
||||
- ai@6.0.228
|
||||
- tsdown@0.22.8
|
||||
- '@toon-format/spec@3.3.2'
|
||||
|
||||
shellEmulator: true
|
||||
trustPolicy: no-downgrade
|
||||
trustPolicyExclude:
|
||||
- vite@5.4.21
|
||||
|
||||
packages:
|
||||
- docs
|
||||
- benchmarks
|
||||
- packages/*
|
||||
|
||||
onlyBuiltDependencies:
|
||||
- '@parcel/watcher'
|
||||
- esbuild
|
||||
allowBuilds:
|
||||
'@parcel/watcher': true
|
||||
esbuild: true
|
||||
|
||||
Reference in New Issue
Block a user