Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1336074d43 | |||
| 352a5b017d | |||
| e2003970dd | |||
| bb9a09ad6c | |||
| 2013a7e09c | |||
| 9024af9e6a | |||
| 27b65c2931 | |||
| 94a2b7560b | |||
| 09a7b38f2c | |||
| 702301231c | |||
| a19a117919 | |||
| a52f2d2815 | |||
| a956757de0 | |||
| c0af15e592 | |||
| 5b66523e42 | |||
| e2b5611d37 | |||
| 3c66a1bead | |||
| 4b8aa998c3 | |||
| 1d23fa36d2 | |||
| d6c64d23e4 | |||
| dd856fe590 | |||
| d514942fc0 | |||
| 979152508d | |||
| 1446cae85d | |||
| 13110eb975 | |||
| 1f53af3582 | |||
| 1a9c72a6c2 | |||
| bc1ae4e194 | |||
| a13dcb5eec | |||
| e6d97d232e | |||
| 178207636d | |||
| caa59ae4dc | |||
| 269e43e960 | |||
| 86006809cd | |||
| 90a04bd34b | |||
| 9ac3f9ed67 |
@@ -21,12 +21,12 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
|
||||
@@ -22,12 +22,12 @@ jobs:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm dependencies
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
|
||||
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Only fetch the config file from the repository
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0 # Required for fetching tags and generating release notes
|
||||
persist-credentials: true
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
|
||||
@@ -4,7 +4,7 @@ The TOON specification has moved to a dedicated repository: [github.com/toon-for
|
||||
|
||||
## Current Version
|
||||
|
||||
**Version 3.0** (2025-11-24)
|
||||
**Version 3.3** (2026-05-21)
|
||||
|
||||
## Quick Links
|
||||
|
||||
|
||||
+13
-13
@@ -8,21 +8,21 @@
|
||||
"fetch:github-repos": "node scripts/fetch-github-repos.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.76",
|
||||
"@ai-sdk/google": "^3.0.71",
|
||||
"@ai-sdk/openai": "^3.0.63",
|
||||
"@ai-sdk/provider": "^3.0.10",
|
||||
"@ai-sdk/xai": "^3.0.89",
|
||||
"@clack/prompts": "^1.3.0",
|
||||
"@faker-js/faker": "^10.4.0",
|
||||
"ai": "^6.0.177",
|
||||
"csv-stringify": "^6.7.0",
|
||||
"fast-xml-parser": "^5.7.3",
|
||||
"@ai-sdk/anthropic": "^3.0.97",
|
||||
"@ai-sdk/google": "^3.0.94",
|
||||
"@ai-sdk/openai": "^3.0.85",
|
||||
"@ai-sdk/provider": "^3.0.14",
|
||||
"@ai-sdk/xai": "^3.0.108",
|
||||
"@clack/prompts": "^1.7.0",
|
||||
"@faker-js/faker": "^10.5.0",
|
||||
"ai": "^6.0.228",
|
||||
"csv-stringify": "^6.8.1",
|
||||
"fast-xml-parser": "^5.10.0",
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"ofetch": "^1.5.1",
|
||||
"p-map": "^7.0.4",
|
||||
"p-queue": "^9.2.0",
|
||||
"p-map": "^7.0.5",
|
||||
"p-queue": "^9.3.1",
|
||||
"unstorage": "^1.17.5",
|
||||
"yaml": "^2.8.4"
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,14 +4,27 @@ import { useClipboard, useDebounceFn } from '@vueuse/core'
|
||||
import { unzlibSync, zlibSync } from 'fflate'
|
||||
import { base64ToUint8Array, stringToUint8Array, uint8ArrayToBase64, uint8ArrayToString } from 'uint8array-extras'
|
||||
import { computed, onMounted, ref, shallowRef, watch } from 'vue'
|
||||
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
|
||||
import { DEFAULT_DELIMITER, encode } from '../../../../packages/toon/src'
|
||||
import VPInput from './VPInput.vue'
|
||||
|
||||
type InputFormat = 'json' | 'yaml'
|
||||
type JsonFormat = 'pretty-2' | 'pretty-4' | 'pretty-tab' | 'compact'
|
||||
|
||||
interface PlaygroundState extends Required<Pick<EncodeOptions, 'delimiter' | 'indent' | 'keyFolding' | 'flattenDepth'>> {
|
||||
json: string
|
||||
input: string
|
||||
inputFormat: InputFormat
|
||||
jsonFormat: JsonFormat
|
||||
/** Pre-YAML share URLs stored input under `json`. Read-only fallback. */
|
||||
json?: string
|
||||
}
|
||||
|
||||
function parseInput(text: string, format: InputFormat): unknown {
|
||||
return format === 'yaml' ? parseYaml(text) : JSON.parse(text)
|
||||
}
|
||||
|
||||
function stringifyInputYaml(value: unknown): string {
|
||||
return stringifyYaml(value, { lineWidth: 0 })
|
||||
}
|
||||
|
||||
const PRESETS = {
|
||||
@@ -83,17 +96,19 @@ const DEFAULT_JSON = JSON.stringify(PRESETS.hikes, undefined, 2)
|
||||
const SHARE_URL_LIMIT = 8 * 1024
|
||||
|
||||
// Input state
|
||||
const jsonInput = ref(DEFAULT_JSON)
|
||||
const inputText = ref(DEFAULT_JSON)
|
||||
const inputFormat = ref<InputFormat>('json')
|
||||
const jsonFormat = ref<JsonFormat>('pretty-2')
|
||||
const currentFormatIndent = computed(() =>
|
||||
JSON_FORMAT_OPTIONS.find(opt => opt.value === jsonFormat.value)?.indent,
|
||||
)
|
||||
const formattedJson = computed(() => {
|
||||
const formattedInput = computed(() => {
|
||||
try {
|
||||
return formatJson(JSON.parse(jsonInput.value))
|
||||
const data = parseInput(inputText.value, inputFormat.value)
|
||||
return inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
|
||||
}
|
||||
catch {
|
||||
return jsonInput.value
|
||||
return inputText.value
|
||||
}
|
||||
})
|
||||
|
||||
@@ -106,7 +121,7 @@ const flattenDepth = ref(2)
|
||||
// Encoding output
|
||||
const encodingResult = computed(() => {
|
||||
try {
|
||||
const parsedInput = JSON.parse(jsonInput.value)
|
||||
const parsedInput = parseInput(inputText.value, inputFormat.value)
|
||||
return {
|
||||
output: encode(parsedInput, {
|
||||
indent: indent.value,
|
||||
@@ -118,9 +133,10 @@ const encodingResult = computed(() => {
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const fallback = inputFormat.value === 'yaml' ? 'Invalid YAML' : 'Invalid JSON'
|
||||
return {
|
||||
output: '',
|
||||
error: error instanceof Error ? error.message : 'Invalid JSON',
|
||||
error: error instanceof Error ? error.message : fallback,
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -129,18 +145,18 @@ const error = computed(() => encodingResult.value.error)
|
||||
|
||||
// Token analysis
|
||||
const tokenizer = shallowRef<typeof import('gpt-tokenizer') | undefined>()
|
||||
const jsonTokens = computed(() =>
|
||||
tokenizer.value?.encode(formattedJson.value).length,
|
||||
const inputTokens = computed(() =>
|
||||
tokenizer.value?.encode(formattedInput.value).length,
|
||||
)
|
||||
const toonTokens = computed(() =>
|
||||
tokenizer.value && toonOutput.value ? tokenizer.value.encode(toonOutput.value).length : undefined,
|
||||
)
|
||||
const tokenSavings = computed(() => {
|
||||
if (!jsonTokens.value || !toonTokens.value)
|
||||
if (!inputTokens.value || !toonTokens.value)
|
||||
return
|
||||
|
||||
const diff = jsonTokens.value - toonTokens.value
|
||||
const percent = Math.abs((diff / jsonTokens.value) * 100).toFixed(1)
|
||||
const diff = inputTokens.value - toonTokens.value
|
||||
const percent = Math.abs((diff / inputTokens.value) * 100).toFixed(1)
|
||||
const sign = diff > 0 ? '−' : '+'
|
||||
|
||||
return { diff, percent, sign, isSavings: diff > 0 }
|
||||
@@ -165,13 +181,25 @@ const updateUrl = useDebounceFn(() => {
|
||||
window.history.replaceState(null, '', `#${hash}`)
|
||||
}, 300)
|
||||
|
||||
watch([jsonInput, delimiter, indent, keyFolding, flattenDepth, jsonFormat], () => {
|
||||
watch([inputText, delimiter, indent, keyFolding, flattenDepth, jsonFormat, inputFormat], () => {
|
||||
updateUrl()
|
||||
})
|
||||
|
||||
watch(jsonFormat, () => {
|
||||
if (inputFormat.value !== 'json')
|
||||
return
|
||||
try {
|
||||
jsonInput.value = formatJson(JSON.parse(jsonInput.value))
|
||||
inputText.value = formatJson(JSON.parse(inputText.value))
|
||||
}
|
||||
catch {}
|
||||
})
|
||||
|
||||
watch(inputFormat, (next, prev) => {
|
||||
if (prev === next)
|
||||
return
|
||||
try {
|
||||
const data = parseInput(inputText.value, prev)
|
||||
inputText.value = next === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
|
||||
}
|
||||
catch {}
|
||||
})
|
||||
@@ -185,12 +213,13 @@ onMounted(() => {
|
||||
|
||||
const state = decodeState(hash)
|
||||
if (state) {
|
||||
jsonInput.value = state.json
|
||||
inputText.value = state.input ?? state.json
|
||||
delimiter.value = state.delimiter
|
||||
indent.value = state.indent
|
||||
keyFolding.value = state.keyFolding ?? 'safe'
|
||||
flattenDepth.value = state.flattenDepth ?? 2
|
||||
jsonFormat.value = state.jsonFormat ?? 'pretty-2'
|
||||
inputFormat.value = state.inputFormat ?? 'json'
|
||||
}
|
||||
})
|
||||
|
||||
@@ -200,7 +229,8 @@ function formatJson(value: unknown) {
|
||||
|
||||
function encodeState() {
|
||||
const state: PlaygroundState = {
|
||||
json: jsonInput.value,
|
||||
input: inputText.value,
|
||||
inputFormat: inputFormat.value,
|
||||
delimiter: delimiter.value,
|
||||
indent: indent.value,
|
||||
keyFolding: keyFolding.value,
|
||||
@@ -224,7 +254,8 @@ function decodeState(hash: string) {
|
||||
}
|
||||
|
||||
function loadPreset(name: keyof typeof PRESETS) {
|
||||
jsonInput.value = formatJson(PRESETS[name])
|
||||
const data = PRESETS[name]
|
||||
inputText.value = inputFormat.value === 'yaml' ? stringifyInputYaml(data) : formatJson(data)
|
||||
}
|
||||
|
||||
async function copyShareUrl() {
|
||||
@@ -247,11 +278,22 @@ async function loadTokenizer() {
|
||||
<!-- Header -->
|
||||
<header class="playground-header">
|
||||
<h1>Playground</h1>
|
||||
<p>Experiment with JSON to TOON encoding in real-time.</p>
|
||||
<p>Convert JSON or YAML to TOON in real time.</p>
|
||||
</header>
|
||||
|
||||
<!-- Options Bar -->
|
||||
<div class="options-bar">
|
||||
<VPInput id="inputFormat" label="Input format">
|
||||
<select id="inputFormat" v-model="inputFormat">
|
||||
<option value="json">
|
||||
JSON
|
||||
</option>
|
||||
<option value="yaml">
|
||||
YAML
|
||||
</option>
|
||||
</select>
|
||||
</VPInput>
|
||||
|
||||
<VPInput id="delimiter" label="Delimiter">
|
||||
<select id="delimiter" v-model="delimiter">
|
||||
<option v-for="opt in DELIMITER_OPTIONS" :key="opt.value" :value="opt.value">
|
||||
@@ -312,7 +354,7 @@ async function loadTokenizer() {
|
||||
</select>
|
||||
</VPInput>
|
||||
|
||||
<VPInput id="jsonFormat" label="JSON Baseline">
|
||||
<VPInput v-if="inputFormat === 'json'" id="jsonFormat" label="JSON Baseline">
|
||||
<select id="jsonFormat" v-model="jsonFormat">
|
||||
<option v-for="opt in JSON_FORMAT_OPTIONS" :key="opt.value" :value="opt.value">
|
||||
{{ opt.label }}
|
||||
@@ -347,24 +389,24 @@ async function loadTokenizer() {
|
||||
|
||||
<!-- Editor Container -->
|
||||
<div class="editor-container">
|
||||
<!-- JSON Input -->
|
||||
<!-- Input -->
|
||||
<div class="editor-pane">
|
||||
<div class="pane-header">
|
||||
<span class="pane-title">JSON Input</span>
|
||||
<span class="pane-title">{{ inputFormat === 'yaml' ? 'YAML Input' : 'JSON Input' }}</span>
|
||||
<span class="pane-stats">
|
||||
<span class="stat-primary" title="Token count using selected JSON baseline format">{{ jsonTokens ?? '…' }} tokens</span>
|
||||
<span class="stat-secondary">{{ formattedJson.length }} chars</span>
|
||||
<span class="stat-primary" title="Token count of the formatted input">{{ inputTokens ?? '…' }} tokens</span>
|
||||
<span class="stat-secondary">{{ formattedInput.length }} chars</span>
|
||||
</span>
|
||||
</div>
|
||||
<textarea
|
||||
id="json-input"
|
||||
v-model="jsonInput"
|
||||
id="input"
|
||||
v-model="inputText"
|
||||
class="editor-textarea"
|
||||
spellcheck="false"
|
||||
aria-label="JSON input"
|
||||
:aria-describedby="error ? 'json-error' : undefined"
|
||||
:aria-label="inputFormat === 'yaml' ? 'YAML input' : 'JSON input'"
|
||||
:aria-describedby="error ? 'parse-error' : undefined"
|
||||
:aria-invalid="!!error"
|
||||
placeholder="Enter JSON here…"
|
||||
:placeholder="inputFormat === 'yaml' ? 'Enter YAML here…' : 'Enter JSON here…'"
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -392,7 +434,7 @@ async function loadTokenizer() {
|
||||
@click="copy()"
|
||||
/>
|
||||
<pre v-if="!error"><code>{{ toonOutput }}</code></pre>
|
||||
<div v-else id="json-error" role="alert" class="error-message">
|
||||
<div v-else id="parse-error" role="alert" class="error-message">
|
||||
{{ error }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -12,7 +12,7 @@ const config: Theme = {
|
||||
extends: DefaultTheme,
|
||||
enhanceApp({ app }) {
|
||||
app.config.globalProperties.$spec = {
|
||||
version: '3.0',
|
||||
version: '3.3',
|
||||
}
|
||||
app.component('CopyOrDownloadAsMarkdownButtons', CopyOrDownloadAsMarkdownButtons)
|
||||
app.component('PlaygroundLayout', PlaygroundLayout)
|
||||
|
||||
+4
-4
@@ -91,7 +91,7 @@ cat data.toon | toon --decode
|
||||
|
||||
:::
|
||||
|
||||
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` (see [spec §18.2](https://github.com/toon-format/spec/blob/main/SPEC.md#182-provisional-media-type)).
|
||||
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` (see [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations)).
|
||||
|
||||
### Standard Input
|
||||
|
||||
@@ -156,7 +156,7 @@ When using the `--stats` flag with encode, the CLI builds the full TOON string o
|
||||
| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
|
||||
| `--indent <number>` | Indentation size (default: `2`) |
|
||||
| `--stats` | Show token count estimates and savings (encode only) |
|
||||
| `--no-strict` | Disable strict validation when decoding |
|
||||
| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
|
||||
| `--keyFolding <mode>` | Key folding mode: `off`, `safe` (default: `off`) |
|
||||
| `--flattenDepth <number>` | Maximum segments to fold (default: `Infinity`) – requires `--keyFolding safe` |
|
||||
| `--expandPaths <mode>` | Path expansion mode: `off`, `safe` (default: `off`) |
|
||||
@@ -225,13 +225,13 @@ Tab delimiters often tokenize more efficiently than commas and reduce the need f
|
||||
|
||||
### Lenient Decoding
|
||||
|
||||
Skip validation for faster processing:
|
||||
Skip validation for faster, more forgiving decoding:
|
||||
|
||||
```bash
|
||||
toon data.toon --no-strict -o output.json
|
||||
```
|
||||
|
||||
Lenient mode (`--no-strict`) disables strict validation checks like array count matching, indentation multiples, and delimiter consistency. Use this when you trust the input and want faster decoding.
|
||||
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
|
||||
|
||||
### Decode Error Output
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ Community members have created implementations in additional languages:
|
||||
| **C#** | [ToonEncoder](https://github.com/Cysharp/ToonEncoder) | [@Cysharp](https://github.com/Cysharp/ToonEncoder) |
|
||||
| **Clojure** | [toon](https://github.com/vadelabs/toon) | [@vadelabs](https://github.com/vadelabs) |
|
||||
| **Crystal** | [toon-crystal](https://github.com/mamantoha/toon-crystal) | [@mamantoha](https://github.com/mamantoha) |
|
||||
| **Delphi** | [delphi-toon](https://github.com/ernestoalconada/delphi-toon) | [@ernestoalconada](https://github.com/ernestoalconada) |
|
||||
| **Elixir** | [toon_ex](https://github.com/kentaro/toon_ex) | [@kentaro](https://github.com/kentaro) |
|
||||
| **Gleam** | [toon_codec](https://github.com/axelbellec/toon_codec) | [@axelbellec](https://github.com/axelbellec) |
|
||||
| **Go** | [gotoon](https://github.com/alpkeskin/gotoon) | [@alpkeskin](https://github.com/alpkeskin) |
|
||||
@@ -46,13 +47,16 @@ Community members have created implementations in additional languages:
|
||||
| **Kotlin** | [ktoon](https://github.com/lukelast/ktoon)| [@lukelast](https://github.com/lukelast) |
|
||||
| **Laravel Framework** | [laravel-toon](https://github.com/mischasigtermans/laravel-toon) | [@mischasigtermans](https://github.com/mischasigtermans) |
|
||||
| **Lua/Neovim** | [toon.nvim](https://github.com/thalesgelinger/toon.nvim) | [@thalesgelinger](https://github.com/thalesgelinger) |
|
||||
| **Matlab** | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
|
||||
| **OCaml** | [ocaml-toon](https://github.com/davesnx/ocaml-toon) | [@davesnx](https://github.com/davesnx) |
|
||||
| **Perl** | [Data::TOON](https://github.com/ytnobody/p5-Data-TOON) | [@ytnobody](https://github.com/ytnobody) |
|
||||
| **PHP** | [toon-php](https://github.com/HelgeSverre/toon-php) | [@HelgeSverre](https://github.com/HelgeSverre) |
|
||||
| **Python** (C++ backend) | [ctoon](https://github.com/mohammadraziei/ctoon) | [@mohammadraziei](https://github.com/mohammadraziei) |
|
||||
| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
|
||||
| **R** | [toon](https://github.com/laresbernardo/toon) | [@laresbernardo](https://github.com/laresbernardo) |
|
||||
| **Ruby** | [toon-ruby](https://github.com/andrepcg/toon-ruby) | [@andrepcg](https://github.com/andrepcg) |
|
||||
| **Scala** | [toon4s](https://github.com/vim89/toon4s) | [@vim89](https://github.com/vim89) |
|
||||
| **Python** (Rust backend) | [toons](https://github.com/alesanfra/toons) | [@alesanfra](https://github.com/alesanfra) |
|
||||
| **Zig** | [toon-zig](https://github.com/LatentEvals/toon-zig) | [@montanaflynn](https://github.com/montanaflynn) |
|
||||
|
||||
## Contributing an Implementation
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ Experiment with TOON format interactively using these tools for token comparison
|
||||
|
||||
### Official Playground
|
||||
|
||||
The [TOON Playground](/playground) lets you convert JSON to TOON in real-time, compare token counts, and share your experiments via URL.
|
||||
The [TOON Playground](/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
|
||||
|
||||
### Community Playgrounds
|
||||
|
||||
@@ -74,3 +74,9 @@ const data = decode(toon)
|
||||
```
|
||||
|
||||
See the [API Reference](/reference/api) for details.
|
||||
|
||||
## MCP
|
||||
|
||||
### Tooner
|
||||
|
||||
[Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON.
|
||||
|
||||
@@ -96,7 +96,7 @@ The header `items[2]{sku,qty,price}:` declares:
|
||||
Each row contains values in the same order as the field list. Values are encoded as primitives (strings, numbers, booleans, null) and separated by the delimiter.
|
||||
|
||||
> [!NOTE]
|
||||
> Tabular format requires identical field sets across all objects (same keys, order per object may vary) and primitive values only (no nested arrays/objects).
|
||||
> Tabular format requires identical field sets across all objects (same keys, order per object may vary), primitive values only (no nested arrays/objects), and at least one key per object – arrays that contain an empty `{}` element fall back to the expanded list form below.
|
||||
|
||||
### Mixed and Non-Uniform Arrays
|
||||
|
||||
@@ -157,15 +157,27 @@ pairs[2]:
|
||||
|
||||
Each inner array gets its own header on the list-item line.
|
||||
|
||||
### Empty Arrays
|
||||
|
||||
Empty arrays have special representations:
|
||||
When the inner arrays are themselves arrays of objects or non-uniform arrays, the same `- [N]:` header appears on the hyphen line and the nested items follow one indent deeper:
|
||||
|
||||
```yaml
|
||||
items[0]:
|
||||
items[3]:
|
||||
- summary
|
||||
- id: 1
|
||||
name: Ada
|
||||
- [2]:
|
||||
- id: 2
|
||||
- status: draft
|
||||
```
|
||||
|
||||
The header declares length zero, with no elements following.
|
||||
### Empty Arrays
|
||||
|
||||
Empty arrays render as `key: []` for fields and `[]` at the root:
|
||||
|
||||
```yaml
|
||||
items: []
|
||||
```
|
||||
|
||||
The legacy `items[0]:` form is still decoded for backward compatibility.
|
||||
|
||||
## Array Headers
|
||||
|
||||
@@ -214,7 +226,7 @@ items[2|]{sku|name|qty|price}:
|
||||
|
||||
:::
|
||||
|
||||
Tab and pipe delimiters are explicitly encoded in the header brackets and field braces. Commas don't require quoting when tab or pipe is active, and vice versa.
|
||||
Tab and pipe delimiters are explicitly encoded in the header brackets and field braces. Inside an array scope, only the active delimiter triggers quoting – the others are literal data. Object field values (`key: value`) follow the document delimiter (§11.1) regardless of any surrounding array's active delimiter.
|
||||
|
||||
> [!TIP]
|
||||
> Tab delimiters often tokenize more efficiently than commas, especially for data with few quoted strings. Use `encode(data, { delimiter: '\t' })` for additional token savings.
|
||||
@@ -293,8 +305,8 @@ TOON quotes strings **only when necessary** to maximize token efficiency. A stri
|
||||
- It's empty (`""`)
|
||||
- It has leading or trailing whitespace
|
||||
- It equals `true`, `false`, or `null` (case-sensitive)
|
||||
- It looks like a number (e.g., `"42"`, `"-3.14"`, `"1e-6"`, or `"05"` with leading zeros)
|
||||
- It contains special characters: colon (`:`), quote (`"`), backslash (`\`), brackets, braces, or control characters (newline, tab, carriage return)
|
||||
- It looks like a number (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
|
||||
- It contains special characters: colon (`:`), quote (`"`), backslash (`\`), brackets, braces, or any control character in U+0000–U+001F
|
||||
- It contains the relevant delimiter (the active delimiter inside an array scope, or the document delimiter elsewhere)
|
||||
- It equals `"-"` or starts with `"-"` followed by any character
|
||||
|
||||
@@ -307,7 +319,7 @@ note: This has inner spaces
|
||||
|
||||
### Escape Sequences
|
||||
|
||||
In quoted strings and keys, only five escape sequences are valid:
|
||||
In quoted strings and keys, six escape sequences are valid:
|
||||
|
||||
| Character | Escape |
|
||||
|-----------|--------|
|
||||
@@ -316,23 +328,13 @@ In quoted strings and keys, only five escape sequences are valid:
|
||||
| Newline (U+000A) | `\n` |
|
||||
| Carriage return (U+000D) | `\r` |
|
||||
| Tab (U+0009) | `\t` |
|
||||
| Any other U+0000–U+001F control character | `\uXXXX` |
|
||||
|
||||
All other escape sequences (e.g., `\x`, `\u`) are invalid and will cause an error in strict mode.
|
||||
Other escapes (e.g., `\x`, `\0`, `\b`) are always rejected, as are lone-surrogate `\uXXXX` values (U+D800–U+DFFF).
|
||||
|
||||
### Type Conversions
|
||||
|
||||
Numbers are emitted in canonical decimal form (no exponent notation, no trailing zeros). Non-JSON types are normalized before encoding:
|
||||
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| Finite number | Canonical decimal (e.g., `1e6` → `1000000`, `1.5000` → `1.5`, `-0` → `0`) |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (within safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
|
||||
| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
|
||||
| `Set` | Array of normalized values |
|
||||
| `Map` | Object with `String(key)` keys |
|
||||
| `undefined`, `function`, `symbol` | `null` |
|
||||
Numbers are emitted in canonical decimal form for values in the §2 carve-out range; exponent notation is permitted outside. Non-JSON types (`NaN`, `Infinity`, `BigInt`, `Date`, `Set`, `Map`, `undefined`, etc.) are normalized before encoding – see [API Reference – Type Normalization](/reference/api#type-normalization) for the full mapping.
|
||||
|
||||
Decoders accept both decimal and exponent forms on input (e.g., `42`, `-3.14`, `1e-6`), and treat tokens with forbidden leading zeros (e.g., `"05"`) as strings, not numbers.
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ For full CLI documentation, see the [CLI reference](/cli/).
|
||||
|
||||
## Media Type & File Extension
|
||||
|
||||
TOON files conventionally use the `.toon` extension. For HTTP transmission, the provisional media type is `text/toon`, always with UTF-8 encoding. While you may specify `charset=utf-8` explicitly, it's optional – UTF-8 is the default assumption. This follows the registration process outlined in [spec §18.2](https://github.com/toon-format/spec/blob/main/SPEC.md#182-provisional-media-type).
|
||||
TOON files conventionally use the `.toon` extension. For HTTP transmission, the provisional media type is `text/toon`, always with UTF-8 encoding. While you may specify `charset=utf-8` explicitly, it's optional – UTF-8 is the default assumption. This follows the registration process outlined in [spec §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations).
|
||||
|
||||
## Your First Example
|
||||
|
||||
|
||||
+17
-17
@@ -25,9 +25,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
|
||||
|
||||
```toon
|
||||
users[3]{id,name,role,lastLogin}:
|
||||
1,Alice,admin,2025-01-15T10:30:00Z
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
1,Alice,admin,"2025-01-15T10:30:00Z"
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
Task: Summarize the user roles and their last activity.
|
||||
@@ -52,9 +52,9 @@ Data is in TOON format (2-space indent, arrays show length and fields).
|
||||
|
||||
```toon
|
||||
users[3]{id,name,role,lastLogin}:
|
||||
1,Alice,admin,2025-01-15T10:30:00Z
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
1,Alice,admin,"2025-01-15T10:30:00Z"
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
Task: Return only users with role "user" as TOON. Use the same header format. Set [N] to match the row count. Output only the code block.
|
||||
@@ -64,8 +64,8 @@ Task: Return only users with role "user" as TOON. Use the same header format. Se
|
||||
|
||||
```toon
|
||||
users[2]{id,name,role,lastLogin}:
|
||||
2,Bob,user,2025-01-14T15:22:00Z
|
||||
3,Charlie,user,2025-01-13T09:45:00Z
|
||||
2,Bob,user,"2025-01-14T15:22:00Z"
|
||||
3,Charlie,user,"2025-01-13T09:45:00Z"
|
||||
```
|
||||
|
||||
The model adjusts `[N]` to `2` and generates two rows.
|
||||
@@ -87,7 +87,7 @@ catch (error) {
|
||||
}
|
||||
```
|
||||
|
||||
Strict mode checks counts, indentation, and escaping so you can detect truncation or malformed TOON. For complete details, see the [API Reference](/reference/api#decode).
|
||||
Strict mode checks counts, indentation, and escaping so you can detect truncation or malformed TOON. For complete details, see the [API Reference](/reference/api#decode-input-options).
|
||||
|
||||
## Delimiter Choices for Token Efficiency
|
||||
|
||||
@@ -117,10 +117,10 @@ for (const line of encodeLines(largeData, { delimiter: '\t' })) {
|
||||
The CLI also supports streaming for memory-efficient JSON-to-TOON conversion:
|
||||
|
||||
```bash
|
||||
toon large-dataset.json --output output.toon
|
||||
toon large-dataset.json -o output.toon
|
||||
```
|
||||
|
||||
This streaming approach prevents out-of-memory errors when preparing large context windows for LLMs. For complete details on `encodeLines()`, see the [API Reference](/reference/api#encodelines).
|
||||
This streaming approach prevents out-of-memory errors when preparing large context windows for LLMs. For complete details on `encodeLines()`, see the [API Reference](/reference/api#encodelines-input-options).
|
||||
|
||||
**Consuming streaming LLM outputs:** If your LLM client exposes streaming text and you buffer by lines, you can decode TOON incrementally:
|
||||
|
||||
@@ -166,10 +166,10 @@ System logs in TOON format (tab-separated):
|
||||
|
||||
```toon
|
||||
events[4 ]{id level message timestamp}:
|
||||
1 error Connection timeout 2025-01-15T10:00:00Z
|
||||
2 warn Slow query 2025-01-15T10:05:00Z
|
||||
3 info User login 2025-01-15T10:10:00Z
|
||||
4 error Database error 2025-01-15T10:15:00Z
|
||||
1 error Connection timeout "2025-01-15T10:00:00Z"
|
||||
2 warn Slow query "2025-01-15T10:05:00Z"
|
||||
3 info User login "2025-01-15T10:10:00Z"
|
||||
4 error Database error "2025-01-15T10:15:00Z"
|
||||
```
|
||||
|
||||
Task: Return only error-level events as TOON. Use the same format.
|
||||
@@ -182,8 +182,8 @@ import { decode } from '@toon-format/toon'
|
||||
|
||||
const modelResponse = `
|
||||
events[2 ]{id level message timestamp}:
|
||||
1 error Connection timeout 2025-01-15T10:00:00Z
|
||||
4 error Database error 2025-01-15T10:15:00Z
|
||||
1 error Connection timeout "2025-01-15T10:00:00Z"
|
||||
4 error Database error "2025-01-15T10:15:00Z"
|
||||
`
|
||||
|
||||
const filtered = decode(modelResponse, { strict: true })
|
||||
|
||||
+4
-3
@@ -9,12 +9,13 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vueuse/core": "^14.3.0",
|
||||
"fflate": "^0.8.2",
|
||||
"fflate": "^0.8.3",
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"markdown-it-mathjax3": "^4.3.2",
|
||||
"uint8array-extras": "^1.5.0",
|
||||
"unocss": "^66.6.8",
|
||||
"unocss": "^66.7.5",
|
||||
"vitepress": "^1.6.4",
|
||||
"vitepress-plugin-llms": "^1.12.2"
|
||||
"vitepress-plugin-llms": "^1.13.3",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
|
||||
+16
-15
@@ -59,7 +59,8 @@ Non-JSON-serializable values are normalized before encoding:
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| `Object` with `toJSON()` method | Result of calling `toJSON()`, recursively normalized |
|
||||
| Finite number | Canonical decimal (no exponent, no leading/trailing zeros: `1e6` → `1000000`, `-0` → `0`) |
|
||||
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal (e.g., `1e6` → `1000000`, `-0` → `0`) |
|
||||
| Finite number outside that range | Exponent form permitted (e.g., `1e-7`, `1e+21`) |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (within safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string (e.g., `"9007199254740993"`) |
|
||||
@@ -68,6 +69,10 @@ Non-JSON-serializable values are normalized before encoding:
|
||||
| `Map` | Object with `String(key)` keys |
|
||||
| `undefined`, `function`, `symbol` | `null` |
|
||||
|
||||
::: info
|
||||
TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
|
||||
:::
|
||||
|
||||
#### Example
|
||||
|
||||
```ts
|
||||
@@ -252,7 +257,7 @@ console.log(encode(data, { replacer }))
|
||||
|
||||
```yaml
|
||||
metadata:
|
||||
created: 2025-01-01T00:00:00Z
|
||||
created: "2025-01-01T00:00:00Z"
|
||||
user:
|
||||
created: 2025-01-02
|
||||
```
|
||||
@@ -581,15 +586,18 @@ Configuration for [`decode()`](#decode-input-options) and [`decodeFromLines()`](
|
||||
|
||||
By default (`strict: true`), the decoder validates input strictly:
|
||||
|
||||
- **Invalid escape sequences**: Throws on `\x`, unterminated strings
|
||||
- **Invalid escape sequences**: Throws on `\x`, unterminated strings, lone-surrogate `\uXXXX`
|
||||
- **Syntax errors**: Throws on missing colons, malformed headers
|
||||
- **Array length mismatches**: Throws when declared length doesn't match actual count
|
||||
- **Delimiter mismatches**: Throws when row delimiters don't match header
|
||||
- **Header delimiter mismatch**: Throws when the bracket-declared delimiter differs from the field-list delimiter (§14.2)
|
||||
- **Indentation errors**: Throws when leading spaces aren't exact multiples of `indent`
|
||||
- **Header structure**: Throws on leading-zero or non-integer array lengths and on intervening content between bracket/fields/colon
|
||||
- **Duplicate sibling keys**: Throws when an object has two children with the same key (§14.4)
|
||||
- **Path-expansion conflicts**: When `expandPaths: 'safe'` is set, throws on overlapping dotted paths that would collide
|
||||
|
||||
All decode errors are thrown as [`ToonDecodeError`](#error-handling) instances with structured `line` and `source` fields.
|
||||
|
||||
Set `strict: false` to skip validation for lenient parsing.
|
||||
Set `strict: false` to skip these checks. Duplicate sibling keys and path-expansion conflicts then resolve with last-write-wins in document order.
|
||||
|
||||
See [Key Folding & Path Expansion](#key-folding-path-expansion) for more details on path expansion behavior and conflict resolution.
|
||||
|
||||
@@ -622,22 +630,13 @@ type JsonStreamEvent
|
||||
| { type: 'primitive', value: JsonPrimitive }
|
||||
```
|
||||
|
||||
### JSON Value Types
|
||||
|
||||
```ts
|
||||
type JsonPrimitive = string | number | boolean | null
|
||||
type JsonArray = readonly JsonValue[]
|
||||
type JsonObject = { readonly [key: string]: JsonValue }
|
||||
type JsonValue = JsonPrimitive | JsonArray | JsonObject
|
||||
```
|
||||
|
||||
### Delimiters
|
||||
|
||||
```ts
|
||||
import { DEFAULT_DELIMITER, DELIMITERS } from '@toon-format/toon'
|
||||
|
||||
DEFAULT_DELIMITER // ','
|
||||
DELIMITERS // { comma: ',', tab: '\t', pipe: '|' }
|
||||
DELIMITERS // { comma: ',', tab: '\t', pipe: '|' }
|
||||
```
|
||||
|
||||
| Export | Description |
|
||||
@@ -740,6 +739,8 @@ When multiple expanded keys construct overlapping paths, the decoder merges them
|
||||
- With `strict: true` (default): Error
|
||||
- With `strict: false`: Last-write-wins (LWW)
|
||||
|
||||
Duplicate sibling keys (independent of `expandPaths`) follow the same policy: strict mode throws, lenient mode keeps the last value seen.
|
||||
|
||||
### Delimiter Strategies
|
||||
|
||||
Tab delimiters (`\t`) often tokenize more efficiently than commas. Tabs are single characters that rarely appear in natural text, which reduces the need for quote-escaping and leads to smaller token counts in large datasets.
|
||||
|
||||
@@ -40,7 +40,7 @@ Where $\Omega$ represents the data structure under comparison. If $\Delta > 0$,
|
||||
- must be quoted in JSON, and
|
||||
- can be left unquoted in TOON (no characters that would force quoting).
|
||||
Many examples assume values are numbers, booleans, null, or TOON-safe strings that can be unquoted in TOON but must be quoted in JSON.
|
||||
- **Numbers**: Both formats are assumed to use the same canonical decimal representation (no exponent notation), matching TOON's requirement. JSON could use exponent forms; we ignore that here to isolate structural differences.
|
||||
- **Numbers**: For this analysis only, both formats are assumed to use the same canonical decimal representation. JSON could use exponent forms; we ignore that here to isolate structural differences.
|
||||
- **ASCII/UTF-8**: Keys and structural tokens are assumed ASCII, so byte length equals character count ($|x|_{\text{utf8}} = |x|_{\text{char}}$). Non-ASCII content affects both formats similarly and does not change the structural conclusions.
|
||||
- **Nesting depth**: Closed-form expressions are given for flat structures and a single level of nesting. Each additional nesting level in TOON adds 2 bytes of indentation per nested line. At sufficient depth, the braces of compact JSON can win over TOON's indentation (as seen in [When Not to Use TOON](/guide/getting-started#when-not-to-use-toon)).
|
||||
- **Byte vs token count**: Modern LLM tokenizers operate over UTF-8 bytes, so byte length is a good upper bound and first-order proxy for token count, even though the mapping is not exactly linear.
|
||||
@@ -450,10 +450,10 @@ $$
|
||||
For a field named `key`, JSON uses `{"key":[]}` in compact form, while TOON uses:
|
||||
|
||||
```yaml
|
||||
key[0]:
|
||||
key: []
|
||||
```
|
||||
|
||||
Under this model, that yields a constant 3-byte advantage for TOON.
|
||||
Under this model, that yields a constant 3-byte advantage for TOON. The legacy `key[0]:` form remains decodable for backward compatibility.
|
||||
|
||||
## Summary Table
|
||||
|
||||
|
||||
+26
-26
@@ -13,16 +13,16 @@ You don't need this page to *use* TOON. It's mainly for implementers and contrib
|
||||
|
||||
## Current Version
|
||||
|
||||
**Spec v{{ $spec.version }}** (2025-11-24) is the current published Working Draft. It is stable for implementation but not yet finalized; see "Status of This Document" in the spec for details.
|
||||
**Spec v{{ $spec.version }}** (2026-05-20) is the current published Working Draft. It is stable for implementation but not yet finalized; see "Status of This Document" in the spec for details.
|
||||
|
||||
## Media Type & File Extension
|
||||
|
||||
The spec defines a provisional media type and file extension in [§18.2](https://github.com/toon-format/spec/blob/main/SPEC.md#182-provisional-media-type):
|
||||
The spec defines a provisional media type and file extension in [§17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
|
||||
|
||||
- **Media type:** `text/toon` (provisional, not yet IANA‑registered; UTF‑8 only)
|
||||
- **File extension:** `.toon`
|
||||
|
||||
TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `charset` parameter, when present, MUST be `utf-8` per the spec.
|
||||
TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `charset` parameter, when present, is `utf-8`.
|
||||
|
||||
## Guided Tour of the Spec
|
||||
|
||||
@@ -32,7 +32,7 @@ TOON documents are always UTF‑8 with LF (`\n`) line endings; the optional `cha
|
||||
Defines key terms like "indentation level", "active delimiter", "strict mode", and RFC2119 keywords (MUST, SHOULD, MAY).
|
||||
|
||||
[§2 Data Model](https://github.com/toon-format/spec/blob/main/SPEC.md#2-data-model):
|
||||
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (no exponent notation, no leading/trailing zeros).
|
||||
Specifies the JSON data model (objects, arrays, primitives), array/object ordering requirements, and canonical number formatting (canonical decimal for values in `[1e-6, 1e21)` or zero; exponent form permitted outside).
|
||||
|
||||
[§3 Encoding Normalization](https://github.com/toon-format/spec/blob/main/SPEC.md#3-encoding-normalization-reference-encoder):
|
||||
Defines how non-JSON types (Date, BigInt, NaN, Infinity, undefined, etc.) are normalized before encoding. Required reading for encoder implementers.
|
||||
@@ -49,7 +49,7 @@ Defines TOON's line-oriented, indentation-based notation and how to determine wh
|
||||
Normative ABNF grammar for array headers: `key[N<delim?>]{fields}:`. Specifies bracket segments, delimiter symbols, and field lists.
|
||||
|
||||
[§7 Strings and Keys](https://github.com/toon-format/spec/blob/main/SPEC.md#7-strings-and-keys):
|
||||
Complete quoting rules (when strings MUST be quoted), escape sequences (only `\\`, `\"`, `\n`, `\r`, `\t` are valid), and key encoding requirements.
|
||||
Complete quoting rules (when strings MUST be quoted), escape sequences (only `\\`, `\"`, `\n`, `\r`, `\t`, and `\uXXXX` for other U+0000–U+001F controls are valid), and key encoding requirements.
|
||||
|
||||
[§8 Objects](https://github.com/toon-format/spec/blob/main/SPEC.md#8-objects):
|
||||
Object field encoding (key: value), nesting rules, key order preservation, and empty object handling.
|
||||
@@ -75,7 +75,7 @@ Defines conformance classes (encoder, decoder, validator), standardized options,
|
||||
Optional encoder feature (key folding) and decoder feature (path expansion) for collapsing/expanding dotted paths, with deep-merge semantics and strict/non-strict conflict resolution.
|
||||
|
||||
[§14 Strict Mode Errors and Diagnostics](https://github.com/toon-format/spec/blob/main/SPEC.md#14-strict-mode-errors-and-diagnostics-authoritative-checklist):
|
||||
**Authoritative checklist** of all strict-mode errors: array count mismatches, syntax errors, indentation errors, structural errors, and path expansion conflicts.
|
||||
**Authoritative checklist** of all strict-mode errors: array count and width mismatches (§14.1), syntax and structural errors (§14.2), path expansion conflicts (§14.3), and duplicate sibling keys (§14.4).
|
||||
|
||||
### Implementation Guidance
|
||||
|
||||
@@ -85,17 +85,17 @@ Injection risks, quoting rules, and strict-mode checks relevant to security.
|
||||
[§16 Internationalization](https://github.com/toon-format/spec/blob/main/SPEC.md#16-internationalization):
|
||||
Unicode handling and locale-independent number formatting.
|
||||
|
||||
[§17 Interoperability and Mappings](https://github.com/toon-format/spec/blob/main/SPEC.md#17-interoperability-and-mappings):
|
||||
JSON/CSV/YAML mappings and conversion guidance.
|
||||
|
||||
[§18 IANA Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#18-iana-considerations):
|
||||
[§17 IANA Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations):
|
||||
Media type registration plans and provisional status.
|
||||
|
||||
[§19 TOON Core Profile](https://github.com/toon-format/spec/blob/main/SPEC.md#19-toon-core-profile-normative-subset):
|
||||
Normative subset of the most common, memory-friendly rules. Useful for minimal implementations.
|
||||
[§18 Versioning and Extensibility](https://github.com/toon-format/spec/blob/main/SPEC.md#18-versioning-and-extensibility):
|
||||
How the spec evolves: major vs minor bumps and the extensibility policy.
|
||||
|
||||
[Appendix G: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-g-host-type-normalization-examples-informative):
|
||||
Non-normative guidance for Go, JavaScript, Python, and Rust implementations on normalizing language-specific types.
|
||||
[§19 Intellectual Property Considerations](https://github.com/toon-format/spec/blob/main/SPEC.md#19-intellectual-property-considerations):
|
||||
Licensing and IP terms for the specification.
|
||||
|
||||
[Appendix F: Host Type Normalization Examples](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-f-host-type-normalization-examples-informative):
|
||||
Non-normative guidance for Go, JavaScript, Python, Rust, and Java implementations on normalizing language-specific types.
|
||||
|
||||
[Appendix C: Test Suite and Compliance](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-c-test-suite-and-compliance-informative):
|
||||
Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://github.com/toon-format/spec/tree/main/tests) for validating implementations.
|
||||
@@ -104,16 +104,15 @@ Reference test suite at [github.com/toon-format/spec/tree/main/tests](https://gi
|
||||
|
||||
| Section | Topic | When to Read |
|
||||
|---------|-------|--------------|
|
||||
| §1-4 | Data model, normalization, decoding | Implementing encoders/decoders |
|
||||
| §5-6 | Syntax, headers, root form | Implementing parsers |
|
||||
| §1–4 | Data model, normalization, decoding | Implementing encoders/decoders |
|
||||
| §5–6 | Syntax, headers, root form | Implementing parsers |
|
||||
| §7 | Strings, keys, quoting, escaping | Implementing string handling |
|
||||
| §8-10 | Objects, arrays, list items | Implementing structure encoding |
|
||||
| §11-12 | Delimiters, indentation, whitespace | Implementing formatting and validation |
|
||||
| §8–10 | Objects, arrays, list items | Implementing structure encoding |
|
||||
| §11–12 | Delimiters, indentation, whitespace | Implementing formatting and validation |
|
||||
| §13 | Conformance, options, key folding/path expansion | Implementing options and features |
|
||||
| §14 | Strict-mode errors | Implementing validators |
|
||||
| §15-18 | Security, i18n, interoperability, media type | Operational and ecosystem considerations |
|
||||
| §19 | Core profile | Minimal implementations |
|
||||
| §20-21 | Versioning, extensibility, IP | Long-term stability and licensing |
|
||||
| §15–16 | Security, internationalization | Operational considerations |
|
||||
| §17–19 | IANA, versioning, IP | Ecosystem and licensing |
|
||||
|
||||
## Conformance Checklists
|
||||
|
||||
@@ -124,12 +123,13 @@ The spec includes three conformance checklists:
|
||||
Key requirements:
|
||||
- Produce UTF-8 with LF line endings
|
||||
- Use consistent indentation (default 2 spaces, no tabs)
|
||||
- Escape only `\\`, `\"`, `\n`, `\r`, `\t` in quoted strings; any other escape is invalid
|
||||
- Escape `\\`, `\"`, `\n`, `\r`, `\t` in quoted strings, and use `\uXXXX` for any other U+0000–U+001F control character; lone surrogates are rejected
|
||||
- Quote strings with active delimiter, colon, or structural characters
|
||||
- Emit array lengths `[N]` matching actual count
|
||||
- Preserve object key order
|
||||
- Normalize numbers to non-exponential decimal form
|
||||
- Emit numbers per §2 (canonical decimal in `[1e-6, 1e21)` or zero; exponent form permitted outside)
|
||||
- Convert `-0` to `0`, `NaN`/±Infinity to `null`
|
||||
- Emit booleans and null as lowercase literals (`true`, `false`, `null`)
|
||||
- No trailing spaces or trailing newline
|
||||
- When `keyFolding="safe"` is enabled, folding MUST follow §13.4:
|
||||
- Only fold IdentifierSegment keys (letters/digits/underscores, no dots),
|
||||
@@ -150,7 +150,7 @@ Key requirements:
|
||||
- Split on `.`, only expand when all segments are IdentifierSegments,
|
||||
- Deep-merge overlapping paths (object + object),
|
||||
- Do not perform element-wise array merges.
|
||||
- With `expandPaths="safe"` and `strict=true` (default), MUST error on any expansion conflict (§14.5).
|
||||
- With `expandPaths="safe"` and `strict=true` (default), MUST error on any expansion conflict (§14.3).
|
||||
- With `expandPaths="safe"` and `strict=false`, MUST apply deterministic last-write-wins (LWW) conflict resolution (§13.4).
|
||||
|
||||
### Validator Checklist (§13.3) <sup>[↗ SPEC.md](https://github.com/toon-format/spec/blob/main/SPEC.md#133-validator-conformance-checklist)</sup>
|
||||
@@ -165,8 +165,8 @@ Validators should verify:
|
||||
## Versioning
|
||||
|
||||
The spec uses semantic versioning (major.minor):
|
||||
- **Major version** (e.g., v{{ $spec.version }}): Breaking changes, incompatible with previous versions
|
||||
- **Minor version** (e.g., v1.5 → v1.6): Clarifications, additional requirements, or backward-compatible additions
|
||||
- **Major version** (e.g., v2 → v3): Breaking changes, incompatible with previous versions
|
||||
- **Minor version** (e.g., v3.1 → v3.2): Clarifications, additional requirements, or backward-compatible additions
|
||||
|
||||
See [Appendix D: Document Changelog](https://github.com/toon-format/spec/blob/main/SPEC.md#appendix-d-document-changelog-informative) for detailed version history.
|
||||
|
||||
|
||||
@@ -178,7 +178,7 @@ pairs[2]:
|
||||
```
|
||||
|
||||
```yaml [Empty Array]
|
||||
items[0]:
|
||||
items: []
|
||||
```
|
||||
|
||||
:::
|
||||
@@ -205,7 +205,7 @@ enabled: "true"
|
||||
|
||||
These strings must be quoted because they look like numbers/booleans.
|
||||
|
||||
### Strings with Active Delimiter
|
||||
### Strings Containing Delimiters
|
||||
|
||||
::: code-group
|
||||
|
||||
@@ -221,7 +221,7 @@ note: "hello, world"
|
||||
|
||||
:::
|
||||
|
||||
Strings containing the active delimiter (comma by default) must be quoted.
|
||||
Strings must be quoted when they contain the active delimiter (inside an array scope) or the document delimiter (object field values, comma by default).
|
||||
|
||||
### Strings with Leading/Trailing Spaces
|
||||
|
||||
@@ -263,8 +263,8 @@ Strings **must** be quoted if they:
|
||||
- Have leading or trailing whitespace
|
||||
- Equal `true`, `false`, or `null` (case-sensitive)
|
||||
- Look like numbers (e.g., `"42"`, `"-3.14"`, `"1e-6"`, `"05"`)
|
||||
- Contain special characters: `:`, `"`, `\`, `[`, `]`, `{`, `}`, newline, tab, carriage return
|
||||
- Contain the active delimiter (comma by default, or tab/pipe if declared in header)
|
||||
- Contain special characters: `:`, `"`, `\`, `[`, `]`, `{`, `}`, or any control character (U+0000–U+001F, including newline/tab/CR)
|
||||
- Contain the relevant delimiter – the active delimiter inside an array scope, or the document delimiter (comma by default) for object field values
|
||||
- Equal `"-"` or start with `"-"` followed by any character
|
||||
|
||||
Otherwise, strings can be unquoted. Unicode and emoji are safe:
|
||||
@@ -276,7 +276,7 @@ note: This has inner spaces
|
||||
|
||||
## Escape Sequences
|
||||
|
||||
Only five escape sequences are valid in quoted strings:
|
||||
Six escape sequences are valid in quoted strings:
|
||||
|
||||
| Character | Escape |
|
||||
|-----------|--------|
|
||||
@@ -285,8 +285,9 @@ Only five escape sequences are valid in quoted strings:
|
||||
| Newline | `\n` |
|
||||
| Carriage return | `\r` |
|
||||
| Tab | `\t` |
|
||||
| Any other U+0000–U+001F control character | `\uXXXX` |
|
||||
|
||||
All other escapes (e.g., `\x`, `\u`) are invalid.
|
||||
Other escapes (e.g., `\x`, `\0`, `\b`) are invalid, and lone-surrogate `\uXXXX` values (U+D800–U+DFFF) are rejected.
|
||||
|
||||
## Array Headers
|
||||
|
||||
@@ -351,7 +352,8 @@ See [Format Overview – Key Folding](/guide/format-overview#key-folding-optiona
|
||||
|
||||
| Input | Output |
|
||||
|-------|--------|
|
||||
| Finite number | Canonical decimal (no exponent, no trailing zeros) |
|
||||
| Finite number in `[1e-6, 1e21)` (or zero) | Canonical decimal |
|
||||
| Finite number outside that range | Exponent form permitted |
|
||||
| `NaN`, `Infinity`, `-Infinity` | `null` |
|
||||
| `BigInt` (safe range) | Number |
|
||||
| `BigInt` (out of range) | Quoted decimal string |
|
||||
@@ -359,3 +361,7 @@ See [Format Overview – Key Folding](/guide/format-overview#key-folding-optiona
|
||||
| `Set` | Array of normalized values |
|
||||
| `Map` | Object with `String(key)` keys |
|
||||
| `undefined`, `function`, `symbol` | `null` |
|
||||
|
||||
::: info
|
||||
TOON itself doesn't specify how `Date` should be encoded – the spec leaves this to implementations. This library emits an ISO 8601 string in quotes; other implementations may choose differently.
|
||||
:::
|
||||
|
||||
+1
-1
@@ -3,7 +3,6 @@ import type { FlatConfigComposer } from 'eslint-flat-config-utils'
|
||||
import antfu from '@antfu/eslint-config'
|
||||
|
||||
const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
|
||||
pnpm: false,
|
||||
rules: {
|
||||
'no-cond-assign': 'off',
|
||||
},
|
||||
@@ -16,6 +15,7 @@ const config: FlatConfigComposer<TypedFlatConfigItem, ConfigNames> = antfu({
|
||||
'import/no-duplicates': 'off',
|
||||
'style/no-tabs': 'off',
|
||||
'yaml/quotes': 'off',
|
||||
'yaml/indent': 'off',
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
+13
-13
@@ -1,15 +1,15 @@
|
||||
{
|
||||
"name": "@toon-format/monorepo",
|
||||
"type": "module",
|
||||
"version": "2.2.0",
|
||||
"version": "2.3.1",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"scripts": {
|
||||
"build": "pnpm -r --filter=./packages/** run build",
|
||||
"build": "pnpm -r --filter='./packages/**' run build",
|
||||
"automd": "automd",
|
||||
"docs:dev": "vitepress dev docs",
|
||||
"docs:build": "vitepress build docs",
|
||||
"docs:preview": "vitepress preview docs",
|
||||
"docs:dev": "pnpm -C docs run dev",
|
||||
"docs:build": "pnpm -C docs run build",
|
||||
"docs:preview": "pnpm -C docs run preview",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"test": "pnpm -r test",
|
||||
@@ -17,16 +17,16 @@
|
||||
"release": "bumpp -r"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^8.2.0",
|
||||
"@commitlint/types": "^21.0.0",
|
||||
"@types/node": "^25.6.2",
|
||||
"@antfu/eslint-config": "^9.1.0",
|
||||
"@commitlint/types": "^21.2.0",
|
||||
"@types/node": "^26.1.1",
|
||||
"automd": "^0.4.3",
|
||||
"bumpp": "^11.1.0",
|
||||
"eslint": "^10.3.0",
|
||||
"eslint": "^10.7.0",
|
||||
"eslint-flat-config-utils": "^3.2.0",
|
||||
"tsdown": "^0.22.0",
|
||||
"tsdown": "^0.22.8",
|
||||
"typescript": "^6.0.3",
|
||||
"vite": "^8.0.11",
|
||||
"vitest": "^4.1.5"
|
||||
"vite": "^8.1.4",
|
||||
"vitest": "^4.1.10"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ cat data.toon | toon --decode
|
||||
| `--delimiter <char>` | Array delimiter: `,` (comma), tab character, `\|` (pipe). Pass tab as `$'\t'` in bash/zsh |
|
||||
| `--indent <number>` | Indentation size (default: `2`) |
|
||||
| `--stats` | Show token count estimates and savings (encode only) |
|
||||
| `--no-strict` | Disable strict validation when decoding |
|
||||
| `--no-strict` | Skip decode validation (array counts, indentation, header delimiter); last-write-wins on duplicate keys |
|
||||
| `--keyFolding <mode>` | Enable key folding: `off`, `safe` (default: `off`) |
|
||||
| `--flattenDepth <number>` | Maximum folded segment count when key folding is enabled (default: `Infinity`) |
|
||||
| `--expandPaths <mode>` | Enable path expansion: `off`, `safe` (default: `off`) |
|
||||
@@ -100,12 +100,14 @@ The `--delimiter` value must be the actual delimiter character. In bash/zsh, use
|
||||
|
||||
### Lenient Decoding
|
||||
|
||||
Skip validation for faster processing:
|
||||
Skip validation for faster, more forgiving decoding:
|
||||
|
||||
```bash
|
||||
toon data.toon --no-strict -o output.json
|
||||
```
|
||||
|
||||
With `--no-strict`, the decoder stops enforcing array count matches, indentation multiples, and header delimiter mismatches. Duplicate sibling keys no longer throw – the last value wins. Malformed array headers fall back to plain `key: value` lines instead of erroring.
|
||||
|
||||
### Decode Error Output
|
||||
|
||||
When a TOON document fails to parse, the CLI renders the offending line with a caret pointing at the first non-whitespace character. Tabs are shown as `→` so the caret column reflects what the decoder actually saw:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@toon-format/cli",
|
||||
"type": "module",
|
||||
"version": "2.2.0",
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"version": "2.3.1",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"description": "CLI for JSON ↔ TOON conversion using @toon-format/toon",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -46,7 +46,7 @@ const args: ArgsDef = {
|
||||
},
|
||||
strict: {
|
||||
type: 'boolean',
|
||||
description: 'Enable strict mode for decoding',
|
||||
description: 'Strict decode validation (disable with --no-strict)',
|
||||
default: true,
|
||||
},
|
||||
keyFolding: {
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
[](https://github.com/toon-format/toon/actions)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](https://github.com/toon-format/spec)
|
||||
[](https://github.com/toon-format/spec)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](./LICENSE)
|
||||
|
||||
@@ -136,7 +136,7 @@ hikes[3]{id,name,distanceKm,elevationGain,companion,wasSunny}:
|
||||
|
||||
## Media Type & File Extension
|
||||
|
||||
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §18.2](https://github.com/toon-format/spec/blob/main/SPEC.md#182-provisional-media-type) for normative details.
|
||||
By convention, TOON files use the `.toon` extension and the provisional media type `text/toon` for HTTP and content-type–aware contexts. TOON documents are always UTF-8 encoded; the `charset=utf-8` parameter may be specified but defaults to UTF-8 when omitted. See [SPEC.md §17](https://github.com/toon-format/spec/blob/main/SPEC.md#17-iana-considerations) for normative details.
|
||||
|
||||
## When Not to Use TOON
|
||||
|
||||
@@ -817,7 +817,7 @@ Experiment with TOON format interactively using these tools for token comparison
|
||||
|
||||
### Official Playground
|
||||
|
||||
The [TOON Playground](https://toonformat.dev/playground) lets you convert JSON to TOON in real-time, compare token counts, and share your experiments via URL.
|
||||
The [TOON Playground](https://toonformat.dev/playground) lets you convert JSON or YAML to TOON in real time, compare token counts, and share your experiments via URL.
|
||||
|
||||
### Community Playgrounds
|
||||
|
||||
@@ -898,8 +898,9 @@ Comprehensive guides, references, and resources to help you get the most out of
|
||||
### Tools & Integration
|
||||
|
||||
- [CLI](https://toonformat.dev/cli/) – Command-line tool for JSON↔TOON conversions
|
||||
- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation
|
||||
- [Playgrounds](https://toonformat.dev/ecosystem/tools-and-playgrounds) – Interactive tools
|
||||
- [Tooner](https://github.com/chaindead/tooner) – MCP proxy that converts JSON tool responses to TOON
|
||||
- [Using TOON with LLMs](https://toonformat.dev/guide/llm-prompts) – Prompting strategies & validation
|
||||
|
||||
### References
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@toon-format/toon",
|
||||
"type": "module",
|
||||
"version": "2.2.0",
|
||||
"packageManager": "pnpm@10.33.4",
|
||||
"version": "2.3.1",
|
||||
"packageManager": "pnpm@11.13.0",
|
||||
"description": "Token-Oriented Object Notation (TOON) – Compact, human-readable, schema-aware encoding of JSON for LLM prompts",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
@@ -38,6 +38,6 @@
|
||||
"test": "vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@toon-format/spec": "^3.0.3"
|
||||
"@toon-format/spec": "^3.3.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { findClosingQuote } from '../shared/string-utils.ts'
|
||||
import { ToonDecodeError, withLine } from './errors.ts'
|
||||
import { isArrayHeaderContent, isKeyValueContent, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser.ts'
|
||||
import { createScanState, parseLinesAsync, parseLinesSync } from './scanner.ts'
|
||||
import { assertExpectedCount, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
|
||||
import { assertExpectedCount, isDataRow, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation.ts'
|
||||
|
||||
interface DecoderContext { indent: number, strict: boolean }
|
||||
|
||||
@@ -140,9 +140,16 @@ export function* decodeStreamSync(
|
||||
return
|
||||
}
|
||||
|
||||
if (first.content.trim() === '[]') {
|
||||
cursor.advanceSync()
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
// Check for root array
|
||||
if (isArrayHeaderContent(first.content)) {
|
||||
const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER))
|
||||
const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER, resolvedOptions.strict))
|
||||
if (headerInfo) {
|
||||
cursor.advanceSync()
|
||||
yield* decodeArrayFromHeaderSync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions, first)
|
||||
@@ -159,9 +166,17 @@ export function* decodeStreamSync(
|
||||
return
|
||||
}
|
||||
|
||||
if (!isKeyValueLineSync(first) && cursor.peekSync()?.depth === 0) {
|
||||
throw new ToonDecodeError(
|
||||
'Top-level document must start with a key-value or array-header line',
|
||||
{ line: first.lineNumber, source: first.raw },
|
||||
)
|
||||
}
|
||||
|
||||
// Root object
|
||||
const rootSeenKeys = resolvedOptions.strict ? new Set<string>() : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield* decodeKeyValueSync(first, cursor, 0, resolvedOptions)
|
||||
yield* decodeKeyValueSync(first, cursor, 0, resolvedOptions, rootSeenKeys)
|
||||
|
||||
// Process remaining object fields
|
||||
while (!cursor.atEndSync()) {
|
||||
@@ -171,33 +186,47 @@ export function* decodeStreamSync(
|
||||
}
|
||||
|
||||
cursor.advanceSync()
|
||||
yield* decodeKeyValueSync(line, cursor, 0, resolvedOptions)
|
||||
yield* decodeKeyValueSync(line, cursor, 0, resolvedOptions, rootSeenKeys)
|
||||
}
|
||||
|
||||
yield { type: 'endObject' }
|
||||
}
|
||||
|
||||
function assertNoDuplicateKey(key: string, line: ParsedLine, seenKeys: Set<string> | undefined): void {
|
||||
if (!seenKeys)
|
||||
return
|
||||
if (seenKeys.has(key)) {
|
||||
throw new ToonDecodeError(
|
||||
`Duplicate sibling key "${key}"`,
|
||||
{ line: line.lineNumber, source: line.raw },
|
||||
)
|
||||
}
|
||||
seenKeys.add(key)
|
||||
}
|
||||
|
||||
function* decodeKeyValueSync(
|
||||
line: ParsedLine,
|
||||
cursor: StreamingLineCursor,
|
||||
baseDepth: Depth,
|
||||
options: DecoderContext,
|
||||
seenKeys?: Set<string>,
|
||||
): Generator<JsonStreamEvent> {
|
||||
const content = line.content
|
||||
|
||||
// Check for array header first
|
||||
const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER))
|
||||
const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER, options.strict))
|
||||
if (arrayHeader && arrayHeader.header.key !== undefined) {
|
||||
assertNoDuplicateKey(arrayHeader.header.key, line, seenKeys)
|
||||
yield { type: 'key', key: arrayHeader.header.key }
|
||||
yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, line)
|
||||
return
|
||||
}
|
||||
|
||||
// Regular key-value pair
|
||||
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const colonIndex = content.indexOf(COLON, key.length)
|
||||
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
|
||||
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const rest = content.slice(end).trim()
|
||||
|
||||
assertNoDuplicateKey(key, line, seenKeys)
|
||||
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
|
||||
|
||||
// No value after colon - expect nested object or empty
|
||||
@@ -216,6 +245,12 @@ function* decodeKeyValueSync(
|
||||
return
|
||||
}
|
||||
|
||||
if (rest === '[]') {
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
// Inline primitive value
|
||||
yield { type: 'primitive', value: withLine(line, () => parsePrimitiveToken(rest)) }
|
||||
}
|
||||
@@ -226,6 +261,7 @@ function* decodeObjectFieldsSync(
|
||||
options: DecoderContext,
|
||||
): Generator<JsonStreamEvent> {
|
||||
let computedDepth: Depth | undefined
|
||||
const seenKeys = options.strict ? new Set<string>() : undefined
|
||||
|
||||
while (!cursor.atEndSync()) {
|
||||
const line = cursor.peekSync()
|
||||
@@ -239,7 +275,7 @@ function* decodeObjectFieldsSync(
|
||||
|
||||
if (line.depth === computedDepth) {
|
||||
cursor.advanceSync()
|
||||
yield* decodeKeyValueSync(line, cursor, computedDepth, options)
|
||||
yield* decodeKeyValueSync(line, cursor, computedDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
@@ -317,6 +353,10 @@ function* decodeTabularArraySync(
|
||||
}
|
||||
|
||||
if (line.depth === rowDepth) {
|
||||
if (!isDataRow(line.content, header.delimiter)) {
|
||||
break
|
||||
}
|
||||
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
@@ -438,11 +478,17 @@ function* decodeListItemSync(
|
||||
return
|
||||
}
|
||||
|
||||
if (afterHyphen.trim() === '[]') {
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
const itemLine: ParsedLine = { ...line, content: afterHyphen }
|
||||
|
||||
// Check for array header after hyphen
|
||||
if (isArrayHeaderContent(afterHyphen)) {
|
||||
const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER))
|
||||
const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
|
||||
if (arrayHeader) {
|
||||
yield* decodeArrayFromHeaderSync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, itemLine)
|
||||
return
|
||||
@@ -450,10 +496,11 @@ function* decodeListItemSync(
|
||||
}
|
||||
|
||||
// Check for tabular-first list-item object: `- key[N]{fields}:`
|
||||
const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER))
|
||||
const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
|
||||
if (headerInfo && headerInfo.header.key !== undefined && headerInfo.header.fields !== undefined) {
|
||||
// Object with tabular array as first field
|
||||
const header = headerInfo.header
|
||||
const seenKeys = options.strict ? new Set<string>([header.key!]) : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield { type: 'key', key: header.key! }
|
||||
|
||||
@@ -470,7 +517,7 @@ function* decodeListItemSync(
|
||||
|
||||
if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
cursor.advanceSync()
|
||||
yield* decodeKeyValueSync(nextLine, cursor, followDepth, options)
|
||||
yield* decodeKeyValueSync(nextLine, cursor, followDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
@@ -483,8 +530,9 @@ function* decodeListItemSync(
|
||||
|
||||
// Check for object first field after hyphen
|
||||
if (isKeyValueContent(afterHyphen)) {
|
||||
const seenKeys = options.strict ? new Set<string>() : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield* decodeKeyValueSync(itemLine, cursor, baseDepth + 1, options)
|
||||
yield* decodeKeyValueSync(itemLine, cursor, baseDepth + 1, options, seenKeys)
|
||||
|
||||
// Read subsequent fields
|
||||
const followDepth = baseDepth + 1
|
||||
@@ -496,7 +544,7 @@ function* decodeListItemSync(
|
||||
|
||||
if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
cursor.advanceSync()
|
||||
yield* decodeKeyValueSync(nextLine, cursor, followDepth, options)
|
||||
yield* decodeKeyValueSync(nextLine, cursor, followDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
@@ -559,9 +607,16 @@ export async function* decodeStream(
|
||||
return
|
||||
}
|
||||
|
||||
if (first.content.trim() === '[]') {
|
||||
await cursor.advance()
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
// Check for root array
|
||||
if (isArrayHeaderContent(first.content)) {
|
||||
const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER))
|
||||
const headerInfo = withLine(first, () => parseArrayHeaderLine(first.content, DEFAULT_DELIMITER, resolvedOptions.strict))
|
||||
if (headerInfo) {
|
||||
await cursor.advance()
|
||||
yield* decodeArrayFromHeaderAsync(headerInfo.header, headerInfo.inlineValues, cursor, 0, resolvedOptions, first)
|
||||
@@ -577,9 +632,17 @@ export async function* decodeStream(
|
||||
return
|
||||
}
|
||||
|
||||
if (!isKeyValueLineSync(first) && (await cursor.peek())?.depth === 0) {
|
||||
throw new ToonDecodeError(
|
||||
'Top-level document must start with a key-value or array-header line',
|
||||
{ line: first.lineNumber, source: first.raw },
|
||||
)
|
||||
}
|
||||
|
||||
// Root object
|
||||
const rootSeenKeys = resolvedOptions.strict ? new Set<string>() : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield* decodeKeyValueAsync(first, cursor, 0, resolvedOptions)
|
||||
yield* decodeKeyValueAsync(first, cursor, 0, resolvedOptions, rootSeenKeys)
|
||||
|
||||
// Process remaining object fields
|
||||
while (!(await cursor.atEnd())) {
|
||||
@@ -588,7 +651,7 @@ export async function* decodeStream(
|
||||
break
|
||||
}
|
||||
await cursor.advance()
|
||||
yield* decodeKeyValueAsync(line, cursor, 0, resolvedOptions)
|
||||
yield* decodeKeyValueAsync(line, cursor, 0, resolvedOptions, rootSeenKeys)
|
||||
}
|
||||
|
||||
yield { type: 'endObject' }
|
||||
@@ -604,22 +667,24 @@ async function* decodeKeyValueAsync(
|
||||
cursor: StreamingLineCursor,
|
||||
baseDepth: Depth,
|
||||
options: DecoderContext,
|
||||
seenKeys?: Set<string>,
|
||||
): AsyncGenerator<JsonStreamEvent> {
|
||||
const content = line.content
|
||||
|
||||
// Check for array header first
|
||||
const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER))
|
||||
const arrayHeader = withLine(line, () => parseArrayHeaderLine(content, DEFAULT_DELIMITER, options.strict))
|
||||
if (arrayHeader && arrayHeader.header.key !== undefined) {
|
||||
assertNoDuplicateKey(arrayHeader.header.key, line, seenKeys)
|
||||
yield { type: 'key', key: arrayHeader.header.key }
|
||||
yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, line)
|
||||
return
|
||||
}
|
||||
|
||||
// Regular key-value pair
|
||||
const { key, isQuoted } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const colonIndex = content.indexOf(COLON, key.length)
|
||||
const rest = colonIndex >= 0 ? content.slice(colonIndex + 1).trim() : ''
|
||||
const { key, isQuoted, end } = withLine(line, () => parseKeyToken(content, 0))
|
||||
const rest = content.slice(end).trim()
|
||||
|
||||
assertNoDuplicateKey(key, line, seenKeys)
|
||||
yield isQuoted ? { type: 'key', key, wasQuoted: true } : { type: 'key', key }
|
||||
|
||||
// No value after colon - expect nested object or empty
|
||||
@@ -638,6 +703,12 @@ async function* decodeKeyValueAsync(
|
||||
return
|
||||
}
|
||||
|
||||
if (rest === '[]') {
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
// Inline primitive value
|
||||
yield { type: 'primitive', value: withLine(line, () => parsePrimitiveToken(rest)) }
|
||||
}
|
||||
@@ -648,6 +719,7 @@ async function* decodeObjectFieldsAsync(
|
||||
options: DecoderContext,
|
||||
): AsyncGenerator<JsonStreamEvent> {
|
||||
let computedDepth: Depth | undefined
|
||||
const seenKeys = options.strict ? new Set<string>() : undefined
|
||||
|
||||
while (!(await cursor.atEnd())) {
|
||||
const line = await cursor.peek()
|
||||
@@ -661,7 +733,7 @@ async function* decodeObjectFieldsAsync(
|
||||
|
||||
if (line.depth === computedDepth) {
|
||||
await cursor.advance()
|
||||
yield* decodeKeyValueAsync(line, cursor, computedDepth, options)
|
||||
yield* decodeKeyValueAsync(line, cursor, computedDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
@@ -718,6 +790,10 @@ async function* decodeTabularArrayAsync(
|
||||
}
|
||||
|
||||
if (line.depth === rowDepth) {
|
||||
if (!isDataRow(line.content, header.delimiter)) {
|
||||
break
|
||||
}
|
||||
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
@@ -839,11 +915,17 @@ async function* decodeListItemAsync(
|
||||
return
|
||||
}
|
||||
|
||||
if (afterHyphen.trim() === '[]') {
|
||||
yield { type: 'startArray', length: 0 }
|
||||
yield { type: 'endArray' }
|
||||
return
|
||||
}
|
||||
|
||||
const itemLine: ParsedLine = { ...line, content: afterHyphen }
|
||||
|
||||
// Check for array header after hyphen
|
||||
if (isArrayHeaderContent(afterHyphen)) {
|
||||
const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER))
|
||||
const arrayHeader = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
|
||||
if (arrayHeader) {
|
||||
yield* decodeArrayFromHeaderAsync(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options, itemLine)
|
||||
return
|
||||
@@ -851,10 +933,11 @@ async function* decodeListItemAsync(
|
||||
}
|
||||
|
||||
// Check for tabular-first list-item object: `- key[N]{fields}:`
|
||||
const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER))
|
||||
const headerInfo = withLine(itemLine, () => parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER, options.strict))
|
||||
if (headerInfo && headerInfo.header.key !== undefined && headerInfo.header.fields !== undefined) {
|
||||
// Object with tabular array as first field
|
||||
const header = headerInfo.header
|
||||
const seenKeys = options.strict ? new Set<string>([header.key!]) : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield { type: 'key', key: header.key! }
|
||||
|
||||
@@ -871,7 +954,7 @@ async function* decodeListItemAsync(
|
||||
|
||||
if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
await cursor.advance()
|
||||
yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options)
|
||||
yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
@@ -884,8 +967,9 @@ async function* decodeListItemAsync(
|
||||
|
||||
// Check for object first field after hyphen
|
||||
if (isKeyValueContent(afterHyphen)) {
|
||||
const seenKeys = options.strict ? new Set<string>() : undefined
|
||||
yield { type: 'startObject' }
|
||||
yield* decodeKeyValueAsync(itemLine, cursor, baseDepth + 1, options)
|
||||
yield* decodeKeyValueAsync(itemLine, cursor, baseDepth + 1, options, seenKeys)
|
||||
|
||||
// Read subsequent fields
|
||||
const followDepth = baseDepth + 1
|
||||
@@ -897,7 +981,7 @@ async function* decodeListItemAsync(
|
||||
|
||||
if (nextLine.depth === followDepth && !nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
await cursor.advance()
|
||||
yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options)
|
||||
yield* decodeKeyValueAsync(nextLine, cursor, followDepth, options, seenKeys)
|
||||
}
|
||||
else {
|
||||
break
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { JsonObject, JsonStreamEvent, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { QUOTED_KEY_MARKER } from './expand.ts'
|
||||
|
||||
// #region Build context types
|
||||
@@ -63,7 +64,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
throw new Error('Object startObject event without preceding key')
|
||||
}
|
||||
|
||||
parent.obj[parent.currentKey] = obj
|
||||
setOwnProperty(parent.obj, parent.currentKey, obj)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
@@ -116,7 +117,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
if (parent.currentKey === undefined) {
|
||||
throw new Error('Array startArray event without preceding key')
|
||||
}
|
||||
parent.obj[parent.currentKey] = arr
|
||||
setOwnProperty(parent.obj, parent.currentKey, arr)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
@@ -177,7 +178,7 @@ function applyEvent(state: BuildState, event: JsonStreamEvent): void {
|
||||
if (parent.currentKey === undefined) {
|
||||
throw new Error('Primitive event without preceding key in object')
|
||||
}
|
||||
parent.obj[parent.currentKey] = event.value
|
||||
setOwnProperty(parent.obj, parent.currentKey, event.value)
|
||||
parent.currentKey = undefined
|
||||
}
|
||||
else if (parent.type === 'array') {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { JsonObject, JsonValue } from '../types.ts'
|
||||
import { DOT } from '../constants.ts'
|
||||
import { isJsonObject } from '../encode/normalize.ts'
|
||||
import { getOwnProperty, setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { isIdentifierSegment } from '../shared/validation.ts'
|
||||
|
||||
// #region Path expansion (safe)
|
||||
@@ -73,7 +74,7 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
|
||||
const expandedValue = expandPathsSafe(keyValue, strict)
|
||||
|
||||
// Check for conflicts with already-expanded keys
|
||||
if (key in expandedObject) {
|
||||
if (Object.hasOwn(expandedObject, key)) {
|
||||
const conflictingValue = expandedObject[key]!
|
||||
// If both are objects, try to merge them
|
||||
if (canMerge(conflictingValue, expandedValue)) {
|
||||
@@ -87,12 +88,12 @@ export function expandPathsSafe(value: JsonValue, strict: boolean): JsonValue {
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
expandedObject[key] = expandedValue
|
||||
setOwnProperty(expandedObject, key, expandedValue)
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No conflict - insert directly
|
||||
expandedObject[key] = expandedValue
|
||||
setOwnProperty(expandedObject, key, expandedValue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,12 +132,12 @@ function insertPathSafe(
|
||||
// Walk to the penultimate segment, creating objects as needed
|
||||
for (let i = 0; i < segments.length - 1; i++) {
|
||||
const currentSegment = segments[i]!
|
||||
const segmentValue = currentNode[currentSegment]
|
||||
const segmentValue = getOwnProperty(currentNode, currentSegment)
|
||||
|
||||
if (segmentValue === undefined) {
|
||||
// Create new intermediate object
|
||||
const newObj: JsonObject = {}
|
||||
currentNode[currentSegment] = newObj
|
||||
setOwnProperty(currentNode, currentSegment, newObj)
|
||||
currentNode = newObj
|
||||
}
|
||||
else if (isJsonObject(segmentValue)) {
|
||||
@@ -152,18 +153,18 @@ function insertPathSafe(
|
||||
}
|
||||
// Non-strict: overwrite with new object
|
||||
const newObj: JsonObject = {}
|
||||
currentNode[currentSegment] = newObj
|
||||
setOwnProperty(currentNode, currentSegment, newObj)
|
||||
currentNode = newObj
|
||||
}
|
||||
}
|
||||
|
||||
// Insert at the final segment
|
||||
const lastSeg = segments[segments.length - 1]!
|
||||
const destinationValue = currentNode[lastSeg]
|
||||
const destinationValue = getOwnProperty(currentNode, lastSeg)
|
||||
|
||||
if (destinationValue === undefined) {
|
||||
// No conflict - insert directly
|
||||
currentNode[lastSeg] = value
|
||||
setOwnProperty(currentNode, lastSeg, value)
|
||||
}
|
||||
else if (canMerge(destinationValue, value)) {
|
||||
// Both are objects - deep merge
|
||||
@@ -177,7 +178,7 @@ function insertPathSafe(
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
currentNode[lastSeg] = value
|
||||
setOwnProperty(currentNode, lastSeg, value)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,13 +202,14 @@ function mergeObjects(
|
||||
strict: boolean,
|
||||
): void {
|
||||
for (const [key, sourceValue] of Object.entries(source)) {
|
||||
const targetValue = target[key]
|
||||
|
||||
if (targetValue === undefined) {
|
||||
if (!Object.hasOwn(target, key)) {
|
||||
// Key doesn't exist in target - copy it
|
||||
target[key] = sourceValue
|
||||
setOwnProperty(target, key, sourceValue)
|
||||
continue
|
||||
}
|
||||
else if (canMerge(targetValue, sourceValue)) {
|
||||
|
||||
const targetValue = target[key] as JsonValue
|
||||
if (canMerge(targetValue, sourceValue)) {
|
||||
// Both are objects - recursively merge
|
||||
mergeObjects(targetValue as JsonObject, sourceValue as JsonObject, strict)
|
||||
}
|
||||
@@ -219,7 +221,7 @@ function mergeObjects(
|
||||
)
|
||||
}
|
||||
// Non-strict: overwrite (LWW)
|
||||
target[key] = sourceValue
|
||||
setOwnProperty(target, key, sourceValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { findClosingQuote, findUnquotedChar, unescapeString } from '../shared/st
|
||||
export function parseArrayHeaderLine(
|
||||
content: string,
|
||||
defaultDelimiter: Delimiter,
|
||||
strict: boolean = false,
|
||||
): { header: ArrayHeaderInfo, inlineValues?: string } | undefined {
|
||||
const trimmedToken = content.trimStart()
|
||||
|
||||
@@ -33,14 +34,20 @@ export function parseArrayHeaderLine(
|
||||
}
|
||||
else {
|
||||
// Unquoted key - find first bracket
|
||||
bracketStart = content.indexOf(OPEN_BRACKET)
|
||||
bracketStart = findUnquotedChar(content, OPEN_BRACKET)
|
||||
}
|
||||
|
||||
if (bracketStart === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
const bracketEnd = content.indexOf(CLOSE_BRACKET, bracketStart)
|
||||
// A header key can't contain an unquoted colon, so this is a key-value line
|
||||
const firstColonIndex = findUnquotedChar(content, COLON)
|
||||
if (firstColonIndex !== -1 && firstColonIndex < bracketStart) {
|
||||
return
|
||||
}
|
||||
|
||||
const bracketEnd = findUnquotedChar(content, CLOSE_BRACKET, bracketStart)
|
||||
if (bracketEnd === -1) {
|
||||
return
|
||||
}
|
||||
@@ -50,30 +57,40 @@ export function parseArrayHeaderLine(
|
||||
let braceEnd = colonIndex
|
||||
|
||||
// Check for fields segment (braces come after bracket)
|
||||
const braceStart = content.indexOf(OPEN_BRACE, bracketEnd)
|
||||
if (braceStart !== -1 && braceStart < content.indexOf(COLON, bracketEnd)) {
|
||||
// Validate: no extraneous content between bracket end and brace start
|
||||
const braceStart = findUnquotedChar(content, OPEN_BRACE, bracketEnd)
|
||||
if (braceStart !== -1 && braceStart < findUnquotedChar(content, COLON, bracketEnd)) {
|
||||
const gapBeforeBrace = content.slice(bracketEnd + 1, braceStart)
|
||||
if (gapBeforeBrace.trim() !== '') {
|
||||
if (gapBeforeBrace !== '') {
|
||||
if (strict) {
|
||||
const trimmedGap = gapBeforeBrace.trim()
|
||||
throw new SyntaxError(trimmedGap === ''
|
||||
? `Unexpected whitespace between bracket and fields segment`
|
||||
: `Unexpected content "${trimmedGap}" between bracket and fields segment`)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1) {
|
||||
braceEnd = foundBraceEnd + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Now find colon after brackets and braces
|
||||
colonIndex = content.indexOf(COLON, Math.max(bracketEnd, braceEnd))
|
||||
colonIndex = findUnquotedChar(content, COLON, Math.max(bracketEnd, braceEnd))
|
||||
if (colonIndex === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate: no extraneous content between bracket/fields end and colon
|
||||
const gapStart = Math.max(bracketEnd + 1, braceEnd)
|
||||
const gapBeforeColon = content.slice(gapStart, colonIndex)
|
||||
if (gapBeforeColon.trim() !== '') {
|
||||
if (gapBeforeColon !== '') {
|
||||
if (strict) {
|
||||
const trimmedGap = gapBeforeColon.trim()
|
||||
throw new SyntaxError(trimmedGap === ''
|
||||
? `Unexpected whitespace between bracket segment and colon`
|
||||
: `Unexpected content "${trimmedGap}" between bracket segment and colon`)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -87,12 +104,13 @@ export function parseArrayHeaderLine(
|
||||
const afterColon = content.slice(colonIndex + 1).trim()
|
||||
const bracketContent = content.slice(bracketStart + 1, bracketEnd)
|
||||
|
||||
// Try to parse bracket segment
|
||||
let parsedBracket: ReturnType<typeof parseBracketSegment>
|
||||
try {
|
||||
parsedBracket = parseBracketSegment(bracketContent, defaultDelimiter)
|
||||
}
|
||||
catch {
|
||||
catch (error) {
|
||||
if (strict)
|
||||
throw error
|
||||
return
|
||||
}
|
||||
|
||||
@@ -101,9 +119,17 @@ export function parseArrayHeaderLine(
|
||||
// Check for fields segment
|
||||
let fields: string[] | undefined
|
||||
if (braceStart !== -1 && braceStart < colonIndex) {
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
const foundBraceEnd = findUnquotedChar(content, CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1 && foundBraceEnd < colonIndex) {
|
||||
const fieldsContent = content.slice(braceStart + 1, foundBraceEnd)
|
||||
|
||||
const mismatchedDelimiter = findUnquotedMismatchedDelimiter(fieldsContent, delimiter)
|
||||
if (mismatchedDelimiter !== undefined) {
|
||||
if (strict)
|
||||
throw new SyntaxError(`Header delimiter mismatch: bracket declares "${formatDelimiter(delimiter)}" but fields segment contains unquoted "${formatDelimiter(mismatchedDelimiter)}"`)
|
||||
return
|
||||
}
|
||||
|
||||
fields = parseDelimitedValues(fieldsContent, delimiter).map(field => parseStringLiteral(field.trim()))
|
||||
}
|
||||
}
|
||||
@@ -119,6 +145,8 @@ export function parseArrayHeaderLine(
|
||||
}
|
||||
}
|
||||
|
||||
const BRACKET_LENGTH_PATTERN = /^(?:0|[1-9]\d*)$/
|
||||
|
||||
export function parseBracketSegment(
|
||||
seg: string,
|
||||
defaultDelimiter: Delimiter,
|
||||
@@ -136,12 +164,28 @@ export function parseBracketSegment(
|
||||
content = content.slice(0, -1)
|
||||
}
|
||||
|
||||
const length = Number.parseInt(content, 10)
|
||||
if (Number.isNaN(length)) {
|
||||
throw new TypeError(`Invalid array length: ${seg}`)
|
||||
if (!BRACKET_LENGTH_PATTERN.test(content)) {
|
||||
throw new SyntaxError(`Invalid array length: "${seg}" (expected non-negative integer with no leading zeros)`)
|
||||
}
|
||||
|
||||
return { length, delimiter }
|
||||
return { length: Number.parseInt(content, 10), delimiter }
|
||||
}
|
||||
|
||||
const DELIMITER_CANDIDATES: readonly Delimiter[] = [',', '\t', '|']
|
||||
|
||||
function findUnquotedMismatchedDelimiter(content: string, activeDelimiter: Delimiter): Delimiter | undefined {
|
||||
for (const candidate of DELIMITER_CANDIDATES) {
|
||||
if (candidate === activeDelimiter)
|
||||
continue
|
||||
if (findUnquotedChar(content, candidate) !== -1)
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
function formatDelimiter(delimiter: Delimiter): string {
|
||||
if (delimiter === '\t')
|
||||
return '\\t'
|
||||
return delimiter
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ParsedLine } from '../types.ts'
|
||||
import { COLON, LIST_ITEM_PREFIX } from '../constants.ts'
|
||||
import { findUnquotedChar } from '../shared/string-utils.ts'
|
||||
import { ToonDecodeError } from './errors.ts'
|
||||
|
||||
// #region Count and structure validation
|
||||
@@ -91,9 +92,9 @@ export function validateNoBlankLinesInRange(
|
||||
/**
|
||||
* Checks if a line is a data row (vs a key-value pair) in a tabular array.
|
||||
*/
|
||||
function isDataRow(content: string, delimiter: Delimiter): boolean {
|
||||
const colonPos = content.indexOf(COLON)
|
||||
const delimiterPos = content.indexOf(delimiter)
|
||||
export function isDataRow(content: string, delimiter: Delimiter): boolean {
|
||||
const colonPos = findUnquotedChar(content, COLON)
|
||||
const delimiterPos = findUnquotedChar(content, delimiter)
|
||||
|
||||
// No colon = definitely a data row
|
||||
if (colonPos === -1) {
|
||||
|
||||
@@ -128,8 +128,8 @@ export function* encodeArrayLines(
|
||||
options: ResolvedEncodeOptions,
|
||||
): Generator<string> {
|
||||
if (value.length === 0) {
|
||||
const header = formatHeader(0, { key, delimiter: options.delimiter })
|
||||
yield indentedLine(depth, header, options.indent)
|
||||
const line = key != null ? `${encodeKey(key)}: []` : '[]'
|
||||
yield indentedLine(depth, line, options.indent)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -241,7 +241,7 @@ export function isTabularArray(
|
||||
|
||||
// Check that all header keys exist in the row and all values are primitives
|
||||
for (const key of header) {
|
||||
if (!(key in row)) {
|
||||
if (!Object.hasOwn(row, key)) {
|
||||
return false
|
||||
}
|
||||
if (!isJsonPrimitive(row[key])) {
|
||||
@@ -324,9 +324,8 @@ export function* encodeObjectAsListItemLines(
|
||||
}
|
||||
else if (isJsonArray(firstValue)) {
|
||||
if (firstValue.length === 0) {
|
||||
// Empty array: `- key[0]:`
|
||||
const header = formatHeader(0, { delimiter: options.delimiter })
|
||||
yield indentedListItem(depth, `${encodedKey}${header}`, options.indent)
|
||||
// Empty array: `- key: []`
|
||||
yield indentedListItem(depth, `${encodedKey}: []`, options.indent)
|
||||
}
|
||||
else if (isArrayOfPrimitives(firstValue)) {
|
||||
// Inline primitive array: `- key[N]: values`
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
|
||||
// #region Normalization (unknown → JsonValue)
|
||||
|
||||
@@ -76,7 +77,7 @@ export function normalizeValue(value: unknown): JsonValue {
|
||||
|
||||
for (const key in value) {
|
||||
if (Object.hasOwn(value, key)) {
|
||||
encodedValues[key] = normalizeValue(value[key])
|
||||
setOwnProperty(encodedValues, key, normalizeValue(value[key]))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types.ts'
|
||||
import { setOwnProperty } from '../shared/object-utils.ts'
|
||||
import { isJsonArray, isJsonObject, normalizeValue } from './normalize.ts'
|
||||
|
||||
/**
|
||||
@@ -83,7 +84,7 @@ function transformObject(
|
||||
const normalizedValue = normalizeValue(replacedValue)
|
||||
|
||||
// Recursively transform children of the replaced value
|
||||
result[key] = transformChildren(normalizedValue, replacer, childPath)
|
||||
setOwnProperty(result, key, transformChildren(normalizedValue, replacer, childPath))
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -39,9 +39,12 @@ export type {
|
||||
* // age: 30
|
||||
*
|
||||
* encode({ users: [{ id: 1 }, { id: 2 }] })
|
||||
* // users[]:
|
||||
* // - id: 1
|
||||
* // - id: 2
|
||||
* // users[2]{id}:
|
||||
* // 1
|
||||
* // 2
|
||||
*
|
||||
* encode({ tags: [] })
|
||||
* // tags: []
|
||||
*
|
||||
* encode(data, { indent: 4, keyFolding: 'safe' })
|
||||
* ```
|
||||
@@ -62,9 +65,12 @@ export function encode(input: unknown, options?: EncodeOptions): string {
|
||||
* decode('name: Alice\nage: 30')
|
||||
* // { name: 'Alice', age: 30 }
|
||||
*
|
||||
* decode('users[]:\n - id: 1\n - id: 2')
|
||||
* decode('users[2]:\n - id: 1\n - id: 2')
|
||||
* // { users: [{ id: 1 }, { id: 2 }] }
|
||||
*
|
||||
* decode('tags: []')
|
||||
* // { tags: [] }
|
||||
*
|
||||
* decode(toonString, { strict: false, expandPaths: 'safe' })
|
||||
* ```
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { JsonObject, JsonValue } from '../types.ts'
|
||||
|
||||
/**
|
||||
* Reads an own data property, treating inherited and absent keys alike.
|
||||
*
|
||||
* @remarks
|
||||
* Keys such as `__proto__` must not resolve through the prototype chain.
|
||||
*/
|
||||
export function getOwnProperty(target: JsonObject, key: string): JsonValue | undefined {
|
||||
return Object.hasOwn(target, key) ? target[key] : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns an own data property without invoking inherited accessors.
|
||||
*
|
||||
* @remarks
|
||||
* Plain assignment of `__proto__` would hit the `Object.prototype` setter and
|
||||
* corrupt the prototype chain; `defineProperty` avoids that but is markedly
|
||||
* slower, so every other key takes plain assignment.
|
||||
*/
|
||||
export function setOwnProperty(target: JsonObject, key: string, value: JsonValue): void {
|
||||
if (key === '__proto__') {
|
||||
Object.defineProperty(target, key, {
|
||||
value,
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
configurable: true,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
target[key] = value
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { BACKSLASH, CARRIAGE_RETURN, DOUBLE_QUOTE, NEWLINE, TAB } from '../const
|
||||
*
|
||||
* @remarks
|
||||
* Handles backslashes, quotes, newlines, carriage returns, and tabs.
|
||||
* Other U+0000–U+001F control characters are emitted as `\uXXXX`.
|
||||
*/
|
||||
export function escapeString(value: string): string {
|
||||
return value
|
||||
@@ -13,13 +14,16 @@ export function escapeString(value: string): string {
|
||||
.replace(/\n/g, `${BACKSLASH}n`)
|
||||
.replace(/\r/g, `${BACKSLASH}r`)
|
||||
.replace(/\t/g, `${BACKSLASH}t`)
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\u0000-\u001F]/g, c => `${BACKSLASH}u${c.charCodeAt(0).toString(16).padStart(4, '0')}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Unescapes a string by processing escape sequences.
|
||||
*
|
||||
* @remarks
|
||||
* Handles `\n`, `\t`, `\r`, `\\`, and `\"` escape sequences.
|
||||
* Handles `\n`, `\t`, `\r`, `\\`, `\"`, and `\uXXXX` escape sequences.
|
||||
* Lone surrogates in `\uXXXX` are rejected.
|
||||
*/
|
||||
export function unescapeString(value: string): string {
|
||||
let unescaped = ''
|
||||
@@ -57,6 +61,22 @@ export function unescapeString(value: string): string {
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (next === 'u') {
|
||||
if (i + 6 > value.length) {
|
||||
throw new SyntaxError(`Invalid escape sequence: truncated \\u escape at "${value.slice(i, i + 6)}"`)
|
||||
}
|
||||
const hex = value.slice(i + 2, i + 6)
|
||||
if (!/^[0-9a-f]{4}$/i.test(hex)) {
|
||||
throw new SyntaxError(`Invalid escape sequence: \\u must be followed by 4 hex digits, got "${hex}"`)
|
||||
}
|
||||
const codeUnit = Number.parseInt(hex, 16)
|
||||
if (codeUnit >= 0xD800 && codeUnit <= 0xDFFF) {
|
||||
throw new SyntaxError(`Invalid escape sequence: \\u${hex} is a lone surrogate; supplementary code points MUST appear as literal UTF-8`)
|
||||
}
|
||||
unescaped += String.fromCodePoint(codeUnit)
|
||||
i += 6
|
||||
continue
|
||||
}
|
||||
|
||||
throw new SyntaxError(`Invalid escape sequence: \\${next}`)
|
||||
}
|
||||
|
||||
@@ -71,8 +71,9 @@ export function isSafeUnquoted(value: string, delimiter: string = DEFAULT_DELIMI
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for control characters (newline, carriage return, tab - always need quoting/escaping)
|
||||
if (/[\n\r\t]/.test(value)) {
|
||||
// Check for control characters (any U+0000–U+001F always need quoting/escaping)
|
||||
// eslint-disable-next-line no-control-regex
|
||||
if (/[\u0000-\u001F]/.test(value)) {
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,16 @@ describe('toonDecodeError line context', () => {
|
||||
expect(error.message).toMatch(/tabular|rows?/i)
|
||||
})
|
||||
|
||||
it('reports a malformed array length marker with line and source', () => {
|
||||
const error = captureDecodeError('rules[-1]:\n - a\n')
|
||||
|
||||
expect(error).toBeInstanceOf(SyntaxError)
|
||||
expect(error.line).toBe(1)
|
||||
expect(error.source).toBe('rules[-1]:')
|
||||
expect(error.message).toMatch(/^Line 1: /)
|
||||
expect(error.message).toMatch(/invalid array length/i)
|
||||
})
|
||||
|
||||
it('reports indentation errors with line and source', () => {
|
||||
const error = captureDecodeError('a:\n b: 1\n')
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode } from '../src/index'
|
||||
|
||||
describe('decode security hardening', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
|
||||
it('keeps direct __proto__ keys as own data properties', () => {
|
||||
const marker = '__toonDirectPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`__proto__:\n ${marker}: true\n`) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey][marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
['primitive', '__proto__: true', true],
|
||||
['array', '__proto__[2]: 1,2', [1, 2]],
|
||||
])('keeps direct __proto__ %s values as own data properties', (_name, input, expected) => {
|
||||
const decoded = decode(input) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual(expected)
|
||||
})
|
||||
|
||||
it('does not follow Object.prototype during dotted path expansion', () => {
|
||||
const marker = '__toonExpandedPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`payload.__proto__.${marker}: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
|
||||
expect(decoded.payload[prototypeKey][marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps constructor.prototype expansion within own properties', () => {
|
||||
const marker = '__toonConstructorPolluted'
|
||||
|
||||
try {
|
||||
const decoded = decode(`constructor.prototype.${marker}: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, 'constructor')).toBe(true)
|
||||
expect(Object.hasOwn(decoded.constructor, 'prototype')).toBe(true)
|
||||
expect(decoded.constructor.prototype[marker]).toBe(true)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('safely merges an expanded __proto__ path with a direct object', () => {
|
||||
const decoded = decode(`__proto__.first: true\n__proto__:\n second: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual({ first: true, second: true })
|
||||
})
|
||||
|
||||
it('safely overwrites a primitive __proto__ conflict in non-strict mode', () => {
|
||||
const decoded = decode(`__proto__: 1\n__proto__.second: true\n`, {
|
||||
expandPaths: 'safe',
|
||||
strict: false,
|
||||
}) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey]).toEqual({ second: true })
|
||||
})
|
||||
})
|
||||
@@ -13,7 +13,7 @@ import rootForm from '@toon-format/spec/tests/fixtures/decode/root-form.json'
|
||||
import validationErrors from '@toon-format/spec/tests/fixtures/decode/validation-errors.json'
|
||||
import whitespace from '@toon-format/spec/tests/fixtures/decode/whitespace.json'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode } from '../src/index'
|
||||
import { decode, encode } from '../src/index'
|
||||
|
||||
const fixtureFiles = [
|
||||
primitives,
|
||||
@@ -47,3 +47,42 @@ for (const fixtures of fixtureFiles) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
describe('quoted content opacity (round-trip)', () => {
|
||||
const cases: [name: string, value: unknown][] = [
|
||||
['bracket segment then colon in scalar', { a: '[2]: x' }],
|
||||
['bracket segment then colon, count matches length', { a: '[1]: x' }],
|
||||
['bracketed index with a later colon', { content: 'accept (process.argv[2]) and greet: name' }],
|
||||
['brackets, braces, colons, commas and pipes in a scalar', { s: '{k: v}, [0]: a | b' }],
|
||||
['structural characters in tabular cells', { rows: [{ id: 1, note: '[2]: x' }, { id: 2, note: 'a: b, c' }] }],
|
||||
['structural characters in list items', { xs: ['[2]: x', 'a: b', 'p | q'] }],
|
||||
['bracket-shaped text inside a quoted key', { 'k [2]: y': 1 }],
|
||||
['quoted key opens an array header', { 'quoted key': [1, 2] }],
|
||||
['embedded quoted bracket in a key that opens an array header', { 'has "[2]" in it': [3, 4, 5] }],
|
||||
]
|
||||
|
||||
for (const [name, value] of cases) {
|
||||
it(`round-trips ${name}`, () => {
|
||||
expect(decode(encode(value))).toEqual(value)
|
||||
})
|
||||
}
|
||||
|
||||
it('round-trips the Nostr event with a bracketed index and later colon in content', () => {
|
||||
const event = {
|
||||
id: '68f5c016e5a3128d7af740e088fc5d94e56edda4205fffa56aa3d58fe6bb55ee',
|
||||
pubkey: '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5',
|
||||
kind: 1621,
|
||||
content: 'Currently index.js always prints "Hello, world!". It should accept an optional name as the first CLI argument (process.argv[2]) and greet that name instead, falling back to "world" when no argument is given. Example: `node index.js Ada` -> "Hello, Ada!".',
|
||||
tags: [
|
||||
['a', '30617:3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5:hello-compare-rig'],
|
||||
['p', '3cd318a74dbac2a29491ebf64db6ac66965c2ba907585d34705772f417aad6d5'],
|
||||
['subject', 'greeting should accept a name argument'],
|
||||
['t', 'enhancement'],
|
||||
],
|
||||
created_at: 1783027036,
|
||||
sig: '3220fa9dcb7af14b9a970f5d25f4ac13ebb4848c91a83bf496cdef17a839d4475773defb38a20d64b250227eb5aa457eb77358c5f32c4c7935397a3430b102db',
|
||||
}
|
||||
|
||||
expect(decode(encode(event))).toEqual(event)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -38,6 +38,16 @@ describe('streaming decode', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('materializes __proto__ as an own property', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
const lines = ['__proto__:', ' safe: true']
|
||||
const result = buildValueFromEvents(decodeStreamSync(lines)) as Record<string, unknown>
|
||||
|
||||
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
|
||||
expect(result[prototypeKey]).toEqual({ safe: true })
|
||||
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
|
||||
})
|
||||
|
||||
it('decodes inline primitive array', () => {
|
||||
const input = 'scores[3]: 95, 87, 92'
|
||||
const lines = input.split('\n')
|
||||
@@ -200,6 +210,69 @@ describe('streaming decode', () => {
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('locates the value after an escaped quoted key containing a colon', async () => {
|
||||
const lines = ['"\\t:x": v']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: '\t:x', wasQuoted: true },
|
||||
{ type: 'primitive', value: 'v' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps a quoted bracket-then-colon scalar opaque, matching decodeStreamSync', async () => {
|
||||
const lines = ['a: "[1]: x"']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'a' },
|
||||
{ type: 'primitive', value: '[1]: x' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps an unquoted bracket-colon scalar whole, matching decodeStreamSync', async () => {
|
||||
const lines = ['key: foo [2]: bar']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'key' },
|
||||
{ type: 'primitive', value: 'foo [2]: bar' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('keeps a colon-bearing value such as a URL intact', async () => {
|
||||
const lines = ['a: http://x']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: 'startObject' },
|
||||
{ type: 'key', key: 'a' },
|
||||
{ type: 'primitive', value: 'http://x' },
|
||||
{ type: 'endObject' },
|
||||
])
|
||||
expect(events).toEqual(Array.from(decodeStreamSync(lines)))
|
||||
})
|
||||
|
||||
it('materializes __proto__ as an own property', async () => {
|
||||
const prototypeKey = '__proto__'
|
||||
const lines = ['__proto__:', ' safe: true']
|
||||
const events = await collect(decodeStream(asyncLines(lines)))
|
||||
const result = await buildValueFromEventsAsync(asyncEvents(events)) as Record<string, unknown>
|
||||
|
||||
expect(Object.hasOwn(result, prototypeKey)).toBe(true)
|
||||
expect(result[prototypeKey]).toEqual({ safe: true })
|
||||
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
|
||||
})
|
||||
|
||||
it('rejects expandPaths option', async () => {
|
||||
const lines = ['name: Alice']
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { EncodeReplacer } from '../src/index'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode, encode } from '../src/index'
|
||||
|
||||
describe('encode security hardening', () => {
|
||||
const prototypeKey = '__proto__'
|
||||
|
||||
it('preserves own __proto__ properties through an encode/decode round trip', () => {
|
||||
const marker = '__toonEncodedPolluted'
|
||||
const input = JSON.parse(`{"__proto__":{"${marker}":true},"b":2}`)
|
||||
|
||||
try {
|
||||
const decoded = decode(encode(input)) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey][marker]).toBe(true)
|
||||
expect(decoded.b).toBe(2)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves nested own __proto__ properties introduced by a replacer', () => {
|
||||
const marker = '__toonNestedReplacerPolluted'
|
||||
const replacement = JSON.parse(`{"__proto__":{"${marker}":true},"safe":1}`)
|
||||
const replacer: EncodeReplacer = (key, value) => key === 'payload' ? replacement : value
|
||||
|
||||
try {
|
||||
const decoded = decode(encode({ payload: null }, { replacer })) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded.payload, prototypeKey)).toBe(true)
|
||||
expect(decoded.payload[prototypeKey][marker]).toBe(true)
|
||||
expect(decoded.payload.safe).toBe(1)
|
||||
expect(({} as Record<string, unknown>)[marker]).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[marker]
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves own __proto__ properties through key folding and expansion', () => {
|
||||
const input = JSON.parse('{"__proto__":{"inner":{"folded":true}}}')
|
||||
const encoded = encode(input, { keyFolding: 'safe' })
|
||||
const decoded = decode(encoded, { expandPaths: 'safe' }) as Record<string, any>
|
||||
|
||||
expect(Object.hasOwn(decoded, prototypeKey)).toBe(true)
|
||||
expect(decoded[prototypeKey].inner.folded).toBe(true)
|
||||
})
|
||||
|
||||
it('does not use inherited properties to classify tabular rows', () => {
|
||||
const inheritedKey = '__toonInheritedTabularField'
|
||||
const input = [{ [inheritedKey]: 'user' }, { other: 'kept' }]
|
||||
|
||||
// eslint-disable-next-line no-extend-native -- Simulate a pre-polluted environment
|
||||
Object.defineProperty(Object.prototype, inheritedKey, {
|
||||
value: 'admin',
|
||||
enumerable: true,
|
||||
writable: true,
|
||||
configurable: true,
|
||||
})
|
||||
|
||||
try {
|
||||
expect(decode(encode(input))).toEqual(input)
|
||||
}
|
||||
finally {
|
||||
delete (Object.prototype as Record<string, unknown>)[inheritedKey]
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -48,7 +48,7 @@ describe('JavaScript-specific type normalization', () => {
|
||||
|
||||
it('converts empty Set to empty array', () => {
|
||||
const result = encode(new Set())
|
||||
expect(result).toBe('[0]:')
|
||||
expect(result).toBe('[]')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -114,6 +114,43 @@ describe('JavaScript-specific type normalization', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('canonical number form', () => {
|
||||
it('encodes zero as canonical decimal', () => {
|
||||
expect(encode(0)).toBe('0')
|
||||
expect(decode(encode(0))).toBe(0)
|
||||
})
|
||||
|
||||
it('encodes 1e-6 (lower boundary) as canonical decimal', () => {
|
||||
expect(encode(1e-6)).toBe('0.000001')
|
||||
expect(decode(encode(1e-6))).toBe(1e-6)
|
||||
})
|
||||
|
||||
it('encodes 1e-7 (just below lower boundary) as exponent form', () => {
|
||||
expect(encode(1e-7)).toBe('1e-7')
|
||||
expect(decode(encode(1e-7))).toBe(1e-7)
|
||||
})
|
||||
|
||||
it('encodes 9.999999e20 (near upper boundary) as canonical decimal', () => {
|
||||
expect(encode(9.999999e20)).toBe('999999900000000000000')
|
||||
expect(decode(encode(9.999999e20))).toBe(9.999999e20)
|
||||
})
|
||||
|
||||
it('encodes 1e21 (upper boundary) as exponent form', () => {
|
||||
expect(encode(1e21)).toBe('1e+21')
|
||||
expect(decode(encode(1e21))).toBe(1e21)
|
||||
})
|
||||
|
||||
it('encodes 1.5e20 as canonical decimal', () => {
|
||||
expect(encode(1.5e20)).toBe('150000000000000000000')
|
||||
expect(decode(encode(1.5e20))).toBe(1.5e20)
|
||||
})
|
||||
|
||||
it('encodes 5e-324 (smallest subnormal) as exponent form', () => {
|
||||
expect(encode(5e-324)).toBe('5e-324')
|
||||
expect(decode(encode(5e-324))).toBe(5e-324)
|
||||
})
|
||||
})
|
||||
|
||||
describe('toJSON method support', () => {
|
||||
it('calls toJSON method when object has it', () => {
|
||||
const obj = {
|
||||
|
||||
Generated
+1721
-1763
File diff suppressed because it is too large
Load Diff
+17
-3
@@ -1,8 +1,22 @@
|
||||
minimumReleaseAgeExclude:
|
||||
- '@ai-sdk/gateway@3.0.151'
|
||||
- '@ai-sdk/google@3.0.94'
|
||||
- '@ai-sdk/openai-compatible@2.0.61'
|
||||
- '@ai-sdk/xai@3.0.108'
|
||||
- ai@6.0.228
|
||||
- tsdown@0.22.8
|
||||
- '@toon-format/spec@3.3.2'
|
||||
|
||||
shellEmulator: true
|
||||
trustPolicy: no-downgrade
|
||||
trustPolicyExclude:
|
||||
- vite@5.4.21
|
||||
|
||||
packages:
|
||||
- docs
|
||||
- benchmarks
|
||||
- packages/*
|
||||
|
||||
onlyBuiltDependencies:
|
||||
- '@parcel/watcher'
|
||||
- esbuild
|
||||
allowBuilds:
|
||||
'@parcel/watcher': true
|
||||
esbuild: true
|
||||
|
||||
Reference in New Issue
Block a user