chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,5 @@
|
||||
# Add keys for the models you're testing
|
||||
OPENAI_API_KEY=
|
||||
ANTHROPIC_API_KEY=
|
||||
GOOGLE_GENERATIVE_AI_API_KEY=
|
||||
XAI_API_KEY=
|
||||
@@ -0,0 +1,120 @@
|
||||
# TOON Benchmarks
|
||||
|
||||
Benchmarks measuring TOON's **token efficiency** and **retrieval accuracy** compared to JSON, XML, YAML, and CSV.
|
||||
|
||||
> [!NOTE]
|
||||
> Results are automatically embedded in the [main README](https://github.com/toon-format/toon/#benchmarks). This guide focuses on running the benchmarks locally.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Run token efficiency benchmark
|
||||
pnpm benchmark:tokens
|
||||
|
||||
# Run retrieval accuracy benchmark (requires API keys)
|
||||
pnpm benchmark:accuracy
|
||||
```
|
||||
|
||||
## Token Efficiency Benchmark
|
||||
|
||||
Measures token count reduction across JSON, XML, YAML, CSV, and TOON:
|
||||
|
||||
1. Generate datasets (GitHub repos, analytics, orders)
|
||||
2. Convert to all formats (TOON, JSON, XML, YAML, CSV)
|
||||
3. Tokenize using `gpt-tokenizer` (`o200k_base` encoding)
|
||||
4. Calculate savings and generate report
|
||||
|
||||
```bash
|
||||
pnpm benchmark:tokens
|
||||
```
|
||||
|
||||
Results are saved to `results/token-efficiency.md`.
|
||||
|
||||
## Retrieval Accuracy Benchmark
|
||||
|
||||
Tests how well LLMs can answer questions about data in different formats (TOON, JSON, JSON compact, XML, YAML, CSV):
|
||||
|
||||
1. Generate 209 questions across 11 datasets (6 primary + 5 structural validation; CSV only included for datasets with flat/tabular structure)
|
||||
2. Convert each dataset to all supported formats
|
||||
3. Query each LLM with formatted data + question
|
||||
4. Validate answers deterministically using type-aware comparison (no LLM judge needed)
|
||||
5. Aggregate metrics and generate report
|
||||
|
||||
### Setup
|
||||
|
||||
1. Edit [`src/evaluate.ts`](./src/evaluate.ts) and add models to the exported `models` array:
|
||||
```ts
|
||||
export const models: LanguageModelV3[] = [
|
||||
openai('gpt-5-nano'),
|
||||
anthropic('claude-haiku-4-5-20251001'),
|
||||
google('gemini-3-flash-preview'),
|
||||
xai('grok-4-1-fast-non-reasoning'),
|
||||
// Add your models here
|
||||
]
|
||||
```
|
||||
2. Duplicate `.env.example` to `.env` and add your API keys:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# Full benchmark
|
||||
pnpm benchmark:accuracy
|
||||
|
||||
# Dry run (10 questions only, for testing setup)
|
||||
DRY_RUN=true pnpm benchmark:accuracy
|
||||
```
|
||||
|
||||
Running the script will:
|
||||
|
||||
1. Prompt you to select which models to test.
|
||||
2. Skip models with existing results (rerun to overwrite).
|
||||
3. Show progress with rate limiting.
|
||||
4. Save results to `results/accuracy/models/{model-id}.json`.
|
||||
5. Generate report at `results/retrieval-accuracy.md`.
|
||||
|
||||
### Configuration
|
||||
|
||||
Edit [`src/constants.ts`](./src/constants.ts) to adjust:
|
||||
|
||||
- `MODEL_RPM_LIMITS` – Rate limits per model
|
||||
- `DEFAULT_CONCURRENCY` – Parallel tasks (default: 10)
|
||||
- `DRY_RUN_LIMITS` – Questions per dry run (default: 10)
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
scripts/
|
||||
├── accuracy-benchmark.ts # Retrieval accuracy benchmark
|
||||
├── token-efficiency-benchmark.ts # Token counting benchmark
|
||||
└── fetch-github-repos.ts # Update GitHub dataset
|
||||
src/
|
||||
├── constants.ts # Configuration
|
||||
├── datasets.ts # Test data generators
|
||||
├── evaluate.ts # LLM evaluation
|
||||
├── formatters.ts # Format converters
|
||||
├── normalize.ts # Answer normalization
|
||||
├── report.ts # Markdown reports
|
||||
├── storage.ts # Result caching
|
||||
├── types.ts # Type definitions
|
||||
├── utils.ts # Helpers
|
||||
└── questions/ # Question generators
|
||||
├── analytics.ts
|
||||
├── event-logs.ts
|
||||
├── github.ts
|
||||
├── index.ts
|
||||
├── nested-config.ts
|
||||
├── nested.ts
|
||||
├── structural-validation.ts
|
||||
├── structure.ts
|
||||
├── tabular.ts
|
||||
└── utils.ts
|
||||
data/
|
||||
└── github-repos.json # Top 100 GitHub repos
|
||||
results/
|
||||
├── token-efficiency.md # Token savings report
|
||||
├── retrieval-accuracy.md # Accuracy report
|
||||
└── accuracy/models/ # Per-model results (JSON)
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@toon/benchmarks",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"benchmark:tokens": "node scripts/token-efficiency-benchmark.ts",
|
||||
"benchmark:accuracy": "node --env-file=.env scripts/accuracy-benchmark.ts",
|
||||
"fetch:github-repos": "node scripts/fetch-github-repos.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.78",
|
||||
"@ai-sdk/google": "^3.0.75",
|
||||
"@ai-sdk/openai": "^3.0.64",
|
||||
"@ai-sdk/provider": "^3.0.10",
|
||||
"@ai-sdk/xai": "^3.0.91",
|
||||
"@clack/prompts": "^1.4.0",
|
||||
"@faker-js/faker": "^10.4.0",
|
||||
"ai": "^6.0.185",
|
||||
"csv-stringify": "^6.7.0",
|
||||
"fast-xml-parser": "^5.8.0",
|
||||
"gpt-tokenizer": "^3.4.0",
|
||||
"ofetch": "^1.5.1",
|
||||
"p-map": "^7.0.4",
|
||||
"p-queue": "^9.3.0",
|
||||
"unstorage": "^1.17.5",
|
||||
"yaml": "^2.9.0"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,339 @@
|
||||
Benchmarks test LLM comprehension across different input formats using 209 data retrieval questions on 4 models.
|
||||
|
||||
<details>
|
||||
<summary><strong>Show Dataset Catalog</strong></summary>
|
||||
|
||||
#### Dataset Catalog
|
||||
|
||||
| Dataset | Rows | Structure | CSV Support | Eligibility |
|
||||
| ------- | ---- | --------- | ----------- | ----------- |
|
||||
| Uniform employee records | 100 | uniform | ✓ | 100% |
|
||||
| E-commerce orders with nested structures | 50 | nested | ✗ | 33% |
|
||||
| Time-series analytics data | 60 | uniform | ✓ | 100% |
|
||||
| Top 100 GitHub repositories | 100 | uniform | ✓ | 100% |
|
||||
| Semi-uniform event logs | 75 | semi-uniform | ✗ | 50% |
|
||||
| Deeply nested configuration | 11 | deep | ✗ | 0% |
|
||||
| Valid complete dataset (control) | 20 | uniform | ✓ | 100% |
|
||||
| Array truncated: 3 rows removed from end | 17 | uniform | ✓ | 100% |
|
||||
| Extra rows added beyond declared length | 23 | uniform | ✓ | 100% |
|
||||
| Inconsistent field count (missing salary in row 10) | 20 | uniform | ✓ | 100% |
|
||||
| Missing required fields (no email in multiple rows) | 20 | uniform | ✓ | 100% |
|
||||
|
||||
**Structure classes:**
|
||||
- **uniform**: All objects have identical fields with primitive values
|
||||
- **semi-uniform**: Mix of uniform and non-uniform structures
|
||||
- **nested**: Objects with nested structures (nested objects or arrays)
|
||||
- **deep**: Highly nested with minimal tabular eligibility
|
||||
|
||||
**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
|
||||
|
||||
**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
|
||||
|
||||
</details>
|
||||
|
||||
#### Efficiency Ranking (Accuracy per 1K Tokens)
|
||||
|
||||
Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
|
||||
|
||||
```
|
||||
TOON ████████████████████ 27.7 acc%/1K tok │ 76.4% acc │ 2,759 tokens
|
||||
JSON compact █████████████████░░░ 23.7 acc%/1K tok │ 73.7% acc │ 3,104 tokens
|
||||
YAML ██████████████░░░░░░ 19.9 acc%/1K tok │ 74.5% acc │ 3,749 tokens
|
||||
JSON ████████████░░░░░░░░ 16.4 acc%/1K tok │ 75.0% acc │ 4,587 tokens
|
||||
XML ██████████░░░░░░░░░░ 13.8 acc%/1K tok │ 72.1% acc │ 5,221 tokens
|
||||
```
|
||||
|
||||
*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
|
||||
|
||||
> [!TIP]
|
||||
> TOON achieves **76.4%** accuracy (vs JSON's 75.0%) while using **39.9% fewer tokens**.
|
||||
|
||||
**Note on CSV:** Excluded from ranking as it only supports 109 of 209 questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.
|
||||
|
||||
#### Per-Model Accuracy
|
||||
|
||||
Accuracy across 4 LLMs on 209 data retrieval questions:
|
||||
|
||||
```
|
||||
claude-haiku-4-5-20251001
|
||||
→ TOON ████████████░░░░░░░░ 59.8% (125/209)
|
||||
JSON ███████████░░░░░░░░░ 57.4% (120/209)
|
||||
YAML ███████████░░░░░░░░░ 56.0% (117/209)
|
||||
XML ███████████░░░░░░░░░ 55.5% (116/209)
|
||||
JSON compact ███████████░░░░░░░░░ 55.0% (115/209)
|
||||
CSV ██████████░░░░░░░░░░ 50.5% (55/109)
|
||||
|
||||
gemini-3-flash-preview
|
||||
XML ████████████████████ 98.1% (205/209)
|
||||
JSON ███████████████████░ 97.1% (203/209)
|
||||
YAML ███████████████████░ 97.1% (203/209)
|
||||
→ TOON ███████████████████░ 96.7% (202/209)
|
||||
JSON compact ███████████████████░ 96.7% (202/209)
|
||||
CSV ███████████████████░ 96.3% (105/109)
|
||||
|
||||
gpt-5-nano
|
||||
→ TOON ██████████████████░░ 90.9% (190/209)
|
||||
JSON compact ██████████████████░░ 90.9% (190/209)
|
||||
JSON ██████████████████░░ 89.0% (186/209)
|
||||
CSV ██████████████████░░ 89.0% (97/109)
|
||||
YAML █████████████████░░░ 87.1% (182/209)
|
||||
XML ████████████████░░░░ 80.9% (169/209)
|
||||
|
||||
grok-4-1-fast-non-reasoning
|
||||
→ TOON ████████████░░░░░░░░ 58.4% (122/209)
|
||||
YAML ████████████░░░░░░░░ 57.9% (121/209)
|
||||
JSON ███████████░░░░░░░░░ 56.5% (118/209)
|
||||
XML ███████████░░░░░░░░░ 54.1% (113/209)
|
||||
JSON compact ██████████░░░░░░░░░░ 52.2% (109/209)
|
||||
CSV ██████████░░░░░░░░░░ 51.4% (56/109)
|
||||
```
|
||||
|
||||
> [!TIP]
|
||||
> TOON achieves **76.4% accuracy** (vs JSON's 75.0%) while using **39.9% fewer tokens** on these datasets.
|
||||
|
||||
<details>
|
||||
<summary><strong>Performance by dataset, model, and question type</strong></summary>
|
||||
|
||||
#### Performance by Question Type
|
||||
|
||||
| Question Type | TOON | JSON | YAML | JSON compact | XML | CSV |
|
||||
| ------------- | ---- | ---- | ---- | ---- | ---- | ---- |
|
||||
| Field Retrieval | 99.6% | 99.3% | 98.5% | 98.5% | 98.9% | 100.0% |
|
||||
| Aggregation | 61.9% | 61.9% | 59.9% | 58.3% | 54.4% | 50.9% |
|
||||
| Filtering | 56.8% | 53.1% | 56.3% | 55.2% | 51.6% | 50.9% |
|
||||
| Structure Awareness | 89.0% | 87.0% | 84.0% | 84.0% | 81.0% | 85.9% |
|
||||
| Structural Validation | 70.0% | 60.0% | 60.0% | 55.0% | 85.0% | 80.0% |
|
||||
|
||||
#### Performance by Dataset
|
||||
|
||||
##### Uniform employee records
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 73.2% | 2,334 | 120/164 |
|
||||
| `toon` | 73.2% | 2,498 | 120/164 |
|
||||
| `json-compact` | 73.8% | 3,924 | 121/164 |
|
||||
| `yaml` | 73.8% | 4,959 | 121/164 |
|
||||
| `json-pretty` | 73.8% | 6,331 | 121/164 |
|
||||
| `xml` | 74.4% | 7,296 | 122/164 |
|
||||
|
||||
##### E-commerce orders with nested structures
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `toon` | 82.3% | 7,458 | 135/164 |
|
||||
| `json-compact` | 78.7% | 7,110 | 129/164 |
|
||||
| `yaml` | 79.9% | 8,755 | 131/164 |
|
||||
| `json-pretty` | 79.3% | 11,234 | 130/164 |
|
||||
| `xml` | 77.4% | 12,649 | 127/164 |
|
||||
|
||||
##### Time-series analytics data
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 75.0% | 1,411 | 90/120 |
|
||||
| `toon` | 78.3% | 1,553 | 94/120 |
|
||||
| `json-compact` | 74.2% | 2,354 | 89/120 |
|
||||
| `yaml` | 75.8% | 2,954 | 91/120 |
|
||||
| `json-pretty` | 75.0% | 3,681 | 90/120 |
|
||||
| `xml` | 72.5% | 4,389 | 87/120 |
|
||||
|
||||
##### Top 100 GitHub repositories
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 65.9% | 8,527 | 87/132 |
|
||||
| `toon` | 66.7% | 8,779 | 88/132 |
|
||||
| `yaml` | 65.2% | 13,141 | 86/132 |
|
||||
| `json-compact` | 59.8% | 11,464 | 79/132 |
|
||||
| `json-pretty` | 63.6% | 15,157 | 84/132 |
|
||||
| `xml` | 56.1% | 17,105 | 74/132 |
|
||||
|
||||
##### Semi-uniform event logs
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `json-compact` | 68.3% | 4,839 | 82/120 |
|
||||
| `toon` | 65.0% | 5,819 | 78/120 |
|
||||
| `json-pretty` | 69.2% | 6,817 | 83/120 |
|
||||
| `yaml` | 61.7% | 5,847 | 74/120 |
|
||||
| `xml` | 58.3% | 7,729 | 70/120 |
|
||||
|
||||
##### Deeply nested configuration
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `json-compact` | 90.5% | 568 | 105/116 |
|
||||
| `toon` | 94.8% | 655 | 110/116 |
|
||||
| `yaml` | 93.1% | 675 | 108/116 |
|
||||
| `json-pretty` | 92.2% | 924 | 107/116 |
|
||||
| `xml` | 91.4% | 1,013 | 106/116 |
|
||||
|
||||
##### Valid complete dataset (control)
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `toon` | 100.0% | 535 | 4/4 |
|
||||
| `json-compact` | 100.0% | 787 | 4/4 |
|
||||
| `yaml` | 100.0% | 992 | 4/4 |
|
||||
| `json-pretty` | 100.0% | 1,274 | 4/4 |
|
||||
| `xml` | 25.0% | 1,462 | 1/4 |
|
||||
| `csv` | 0.0% | 483 | 0/4 |
|
||||
|
||||
##### Array truncated: 3 rows removed from end
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 100.0% | 413 | 4/4 |
|
||||
| `xml` | 100.0% | 1,243 | 4/4 |
|
||||
| `toon` | 0.0% | 462 | 0/4 |
|
||||
| `json-pretty` | 0.0% | 1,085 | 0/4 |
|
||||
| `yaml` | 0.0% | 843 | 0/4 |
|
||||
| `json-compact` | 0.0% | 670 | 0/4 |
|
||||
|
||||
##### Extra rows added beyond declared length
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 100.0% | 550 | 4/4 |
|
||||
| `toon` | 75.0% | 605 | 3/4 |
|
||||
| `json-compact` | 75.0% | 901 | 3/4 |
|
||||
| `xml` | 100.0% | 1,678 | 4/4 |
|
||||
| `yaml` | 75.0% | 1,138 | 3/4 |
|
||||
| `json-pretty` | 50.0% | 1,460 | 2/4 |
|
||||
|
||||
##### Inconsistent field count (missing salary in row 10)
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 100.0% | 480 | 4/4 |
|
||||
| `json-compact` | 100.0% | 782 | 4/4 |
|
||||
| `yaml` | 100.0% | 985 | 4/4 |
|
||||
| `toon` | 100.0% | 1,008 | 4/4 |
|
||||
| `json-pretty` | 100.0% | 1,266 | 4/4 |
|
||||
| `xml` | 100.0% | 1,453 | 4/4 |
|
||||
|
||||
##### Missing required fields (no email in multiple rows)
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
| `csv` | 100.0% | 340 | 4/4 |
|
||||
| `xml` | 100.0% | 1,409 | 4/4 |
|
||||
| `toon` | 75.0% | 974 | 3/4 |
|
||||
| `json-pretty` | 50.0% | 1,225 | 2/4 |
|
||||
| `yaml` | 25.0% | 951 | 1/4 |
|
||||
| `json-compact` | 0.0% | 750 | 0/4 |
|
||||
|
||||
#### Performance by Model
|
||||
|
||||
##### claude-haiku-4-5-20251001
|
||||
|
||||
| Format | Accuracy | Correct/Total |
|
||||
| ------ | -------- | ------------- |
|
||||
| `toon` | 59.8% | 125/209 |
|
||||
| `json-pretty` | 57.4% | 120/209 |
|
||||
| `yaml` | 56.0% | 117/209 |
|
||||
| `xml` | 55.5% | 116/209 |
|
||||
| `json-compact` | 55.0% | 115/209 |
|
||||
| `csv` | 50.5% | 55/109 |
|
||||
|
||||
##### gemini-3-flash-preview
|
||||
|
||||
| Format | Accuracy | Correct/Total |
|
||||
| ------ | -------- | ------------- |
|
||||
| `xml` | 98.1% | 205/209 |
|
||||
| `json-pretty` | 97.1% | 203/209 |
|
||||
| `yaml` | 97.1% | 203/209 |
|
||||
| `toon` | 96.7% | 202/209 |
|
||||
| `json-compact` | 96.7% | 202/209 |
|
||||
| `csv` | 96.3% | 105/109 |
|
||||
|
||||
##### gpt-5-nano
|
||||
|
||||
| Format | Accuracy | Correct/Total |
|
||||
| ------ | -------- | ------------- |
|
||||
| `toon` | 90.9% | 190/209 |
|
||||
| `json-compact` | 90.9% | 190/209 |
|
||||
| `json-pretty` | 89.0% | 186/209 |
|
||||
| `csv` | 89.0% | 97/109 |
|
||||
| `yaml` | 87.1% | 182/209 |
|
||||
| `xml` | 80.9% | 169/209 |
|
||||
|
||||
##### grok-4-1-fast-non-reasoning
|
||||
|
||||
| Format | Accuracy | Correct/Total |
|
||||
| ------ | -------- | ------------- |
|
||||
| `toon` | 58.4% | 122/209 |
|
||||
| `yaml` | 57.9% | 121/209 |
|
||||
| `json-pretty` | 56.5% | 118/209 |
|
||||
| `xml` | 54.1% | 113/209 |
|
||||
| `json-compact` | 52.2% | 109/209 |
|
||||
| `csv` | 51.4% | 56/109 |
|
||||
|
||||
</details>
|
||||
|
||||
#### What's Being Measured
|
||||
|
||||
This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
|
||||
|
||||
#### Datasets Tested
|
||||
|
||||
Eleven datasets designed to test different structural patterns and validation capabilities:
|
||||
|
||||
**Primary datasets:**
|
||||
|
||||
1. **Tabular** (100 employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
|
||||
2. **Nested** (50 e-commerce orders): Complex structures with nested customer objects and item arrays.
|
||||
3. **Analytics** (60 days of metrics): Time-series data with dates and numeric values.
|
||||
4. **GitHub** (100 repositories): Real-world data from top GitHub repos by stars.
|
||||
5. **Event Logs** (75 logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
|
||||
6. **Nested Config** (1 configuration): Deeply nested configuration with minimal tabular eligibility.
|
||||
|
||||
**Structural validation datasets:**
|
||||
|
||||
7. **Control**: Valid complete dataset (baseline for validation)
|
||||
8. **Truncated**: Array with 3 rows removed from end (tests `[N]` length detection)
|
||||
9. **Extra rows**: Array with 3 additional rows beyond declared length
|
||||
10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
|
||||
11. **Missing fields**: Systematic field omissions (no email in multiple rows)
|
||||
|
||||
#### Question Types
|
||||
|
||||
209 questions are generated dynamically across five categories:
|
||||
|
||||
- **Field retrieval (33%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
|
||||
- Example: "What is Alice's salary?" → `75000`
|
||||
- Example: "How many items are in order ORD-0042?" → `3`
|
||||
- Example: "What is the customer name for order ORD-0042?" → `John Doe`
|
||||
|
||||
- **Aggregation (30%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
|
||||
- Example: "How many employees work in Engineering?" → `17`
|
||||
- Example: "What is the total revenue across all orders?" → `45123.50`
|
||||
- Example: "How many employees have salary > 80000?" → `23`
|
||||
|
||||
- **Filtering (23%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
|
||||
- Example: "How many employees in Sales have salary > 80000?" → `5`
|
||||
- Example: "How many active employees have more than 10 years of experience?" → `8`
|
||||
|
||||
- **Structure awareness (12%)**: Tests format-native structural affordances (TOON's `[N]` count and `{fields}`, CSV's header row)
|
||||
- Example: "How many employees are in the dataset?" → `100`
|
||||
- Example: "List the field names for employees" → `id, name, email, department, salary, yearsExperience, active`
|
||||
- Example: "What is the department of the last employee?" → `Sales`
|
||||
|
||||
- **Structural validation (2%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
|
||||
- Example: "Is this data complete and valid?" → `YES` (control dataset) or `NO` (corrupted datasets)
|
||||
- Tests TOON's `[N]` length validation and `{fields}` consistency checking
|
||||
- Demonstrates CSV's lack of structural validation capabilities
|
||||
|
||||
#### Evaluation Process
|
||||
|
||||
1. **Format conversion**: Each dataset is converted to all 6 formats (TOON, JSON, YAML, JSON compact, XML, CSV).
|
||||
2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
|
||||
3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., `50000` = `$50,000`, `Engineering` = `engineering`, `2025-01-01` = `January 1, 2025`) without requiring an LLM judge.
|
||||
|
||||
#### Models & Configuration
|
||||
|
||||
- **Models tested**: `claude-haiku-4-5-20251001`, `gemini-3-flash-preview`, `gpt-5-nano`, `grok-4-1-fast-non-reasoning`
|
||||
- **Token counting**: Using `gpt-tokenizer` with `o200k_base` encoding (GPT-5 tokenizer)
|
||||
- **Temperature**: Not set (models use their defaults)
|
||||
- **Total evaluations**: 209 questions × 6 formats × 4 models = 5,016 LLM calls
|
||||
@@ -0,0 +1,209 @@
|
||||
#### Mixed-Structure Track
|
||||
|
||||
Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
|
||||
|
||||
```
|
||||
🛒 E-commerce orders with nested structures ┊ Tabular: 33%
|
||||
│
|
||||
TOON █████████████░░░░░░░ 73,126 tokens
|
||||
├─ vs JSON (−33.3%) 109,599 tokens
|
||||
├─ vs JSON compact (+5.3%) 69,459 tokens
|
||||
├─ vs YAML (−14.4%) 85,415 tokens
|
||||
└─ vs XML (−40.7%) 123,344 tokens
|
||||
|
||||
🧾 Semi-uniform event logs ┊ Tabular: 50%
|
||||
│
|
||||
TOON █████████████████░░░ 154,084 tokens
|
||||
├─ vs JSON (−15.0%) 181,201 tokens
|
||||
├─ vs JSON compact (+19.9%) 128,529 tokens
|
||||
├─ vs YAML (−0.8%) 155,397 tokens
|
||||
└─ vs XML (−25.2%) 205,859 tokens
|
||||
|
||||
🧩 Deeply nested configuration ┊ Tabular: 0%
|
||||
│
|
||||
TOON ██████████████░░░░░░ 620 tokens
|
||||
├─ vs JSON (−31.9%) 911 tokens
|
||||
├─ vs JSON compact (+11.1%) 558 tokens
|
||||
├─ vs YAML (−6.3%) 662 tokens
|
||||
└─ vs XML (−38.2%) 1,003 tokens
|
||||
|
||||
──────────────────────────────────── Total ────────────────────────────────────
|
||||
TOON ████████████████░░░░ 227,830 tokens
|
||||
├─ vs JSON (−21.9%) 291,711 tokens
|
||||
├─ vs JSON compact (+14.7%) 198,546 tokens
|
||||
├─ vs YAML (−5.7%) 241,474 tokens
|
||||
└─ vs XML (−31.0%) 330,206 tokens
|
||||
```
|
||||
|
||||
#### Flat-Only Track
|
||||
|
||||
Datasets with flat tabular structures where CSV is applicable.
|
||||
|
||||
```
|
||||
👥 Uniform employee records ┊ Tabular: 100%
|
||||
│
|
||||
CSV ███████████████████░ 47,102 tokens
|
||||
TOON ████████████████████ 49,919 tokens (+6.0% vs CSV)
|
||||
├─ vs JSON (−60.7%) 127,063 tokens
|
||||
├─ vs JSON compact (−36.9%) 79,059 tokens
|
||||
├─ vs YAML (−50.1%) 100,011 tokens
|
||||
└─ vs XML (−65.9%) 146,579 tokens
|
||||
|
||||
📈 Time-series analytics data ┊ Tabular: 100%
|
||||
│
|
||||
CSV ██████████████████░░ 8,383 tokens
|
||||
TOON ████████████████████ 9,115 tokens (+8.7% vs CSV)
|
||||
├─ vs JSON (−59.0%) 22,245 tokens
|
||||
├─ vs JSON compact (−35.9%) 14,211 tokens
|
||||
├─ vs YAML (−49.0%) 17,858 tokens
|
||||
└─ vs XML (−65.8%) 26,616 tokens
|
||||
|
||||
⭐ Top 100 GitHub repositories ┊ Tabular: 100%
|
||||
│
|
||||
CSV ███████████████████░ 8,512 tokens
|
||||
TOON ████████████████████ 8,744 tokens (+2.7% vs CSV)
|
||||
├─ vs JSON (−42.3%) 15,144 tokens
|
||||
├─ vs JSON compact (−23.7%) 11,454 tokens
|
||||
├─ vs YAML (−33.4%) 13,128 tokens
|
||||
└─ vs XML (−48.9%) 17,095 tokens
|
||||
|
||||
──────────────────────────────────── Total ────────────────────────────────────
|
||||
CSV ███████████████████░ 63,997 tokens
|
||||
TOON ████████████████████ 67,778 tokens (+5.9% vs CSV)
|
||||
├─ vs JSON (−58.8%) 164,452 tokens
|
||||
├─ vs JSON compact (−35.3%) 104,724 tokens
|
||||
├─ vs YAML (−48.3%) 130,997 tokens
|
||||
└─ vs XML (−64.4%) 190,290 tokens
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><strong>Show detailed examples</strong></summary>
|
||||
|
||||
#### 📈 Time-series analytics data
|
||||
|
||||
**Savings:** 13,130 tokens (59.0% reduction vs JSON)
|
||||
|
||||
**JSON** (22,245 tokens):
|
||||
|
||||
```json
|
||||
{
|
||||
"metrics": [
|
||||
{
|
||||
"date": "2025-01-01",
|
||||
"views": 6138,
|
||||
"clicks": 174,
|
||||
"conversions": 12,
|
||||
"revenue": 2712.49,
|
||||
"bounceRate": 0.35
|
||||
},
|
||||
{
|
||||
"date": "2025-01-02",
|
||||
"views": 4616,
|
||||
"clicks": 274,
|
||||
"conversions": 34,
|
||||
"revenue": 9156.29,
|
||||
"bounceRate": 0.56
|
||||
},
|
||||
{
|
||||
"date": "2025-01-03",
|
||||
"views": 4460,
|
||||
"clicks": 143,
|
||||
"conversions": 8,
|
||||
"revenue": 1317.98,
|
||||
"bounceRate": 0.59
|
||||
},
|
||||
{
|
||||
"date": "2025-01-04",
|
||||
"views": 4740,
|
||||
"clicks": 125,
|
||||
"conversions": 13,
|
||||
"revenue": 2934.77,
|
||||
"bounceRate": 0.37
|
||||
},
|
||||
{
|
||||
"date": "2025-01-05",
|
||||
"views": 6428,
|
||||
"clicks": 369,
|
||||
"conversions": 19,
|
||||
"revenue": 1317.24,
|
||||
"bounceRate": 0.3
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**TOON** (9,115 tokens):
|
||||
|
||||
```
|
||||
metrics[5]{date,views,clicks,conversions,revenue,bounceRate}:
|
||||
2025-01-01,6138,174,12,2712.49,0.35
|
||||
2025-01-02,4616,274,34,9156.29,0.56
|
||||
2025-01-03,4460,143,8,1317.98,0.59
|
||||
2025-01-04,4740,125,13,2934.77,0.37
|
||||
2025-01-05,6428,369,19,1317.24,0.3
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### ⭐ Top 100 GitHub repositories
|
||||
|
||||
**Savings:** 6,400 tokens (42.3% reduction vs JSON)
|
||||
|
||||
**JSON** (15,144 tokens):
|
||||
|
||||
```json
|
||||
{
|
||||
"repositories": [
|
||||
{
|
||||
"id": 28457823,
|
||||
"name": "freeCodeCamp",
|
||||
"repo": "freeCodeCamp/freeCodeCamp",
|
||||
"description": "freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…",
|
||||
"createdAt": "2014-12-24T17:49:19Z",
|
||||
"updatedAt": "2025-10-28T11:58:08Z",
|
||||
"pushedAt": "2025-10-28T10:17:16Z",
|
||||
"stars": 430886,
|
||||
"watchers": 8583,
|
||||
"forks": 42146,
|
||||
"defaultBranch": "main"
|
||||
},
|
||||
{
|
||||
"id": 132750724,
|
||||
"name": "build-your-own-x",
|
||||
"repo": "codecrafters-io/build-your-own-x",
|
||||
"description": "Master programming by recreating your favorite technologies from scratch.",
|
||||
"createdAt": "2018-05-09T12:03:18Z",
|
||||
"updatedAt": "2025-10-28T12:37:11Z",
|
||||
"pushedAt": "2025-10-10T18:45:01Z",
|
||||
"stars": 430877,
|
||||
"watchers": 6332,
|
||||
"forks": 40453,
|
||||
"defaultBranch": "master"
|
||||
},
|
||||
{
|
||||
"id": 21737465,
|
||||
"name": "awesome",
|
||||
"repo": "sindresorhus/awesome",
|
||||
"description": "😎 Awesome lists about all kinds of interesting topics",
|
||||
"createdAt": "2014-07-11T13:42:37Z",
|
||||
"updatedAt": "2025-10-28T12:40:21Z",
|
||||
"pushedAt": "2025-10-27T17:57:31Z",
|
||||
"stars": 410052,
|
||||
"watchers": 8017,
|
||||
"forks": 32029,
|
||||
"defaultBranch": "main"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**TOON** (8,744 tokens):
|
||||
|
||||
```
|
||||
repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watchers,forks,defaultBranch}:
|
||||
28457823,freeCodeCamp,freeCodeCamp/freeCodeCamp,"freeCodeCamp.org's open-source codebase and curriculum. Learn math, programming,…","2014-12-24T17:49:19Z","2025-10-28T11:58:08Z","2025-10-28T10:17:16Z",430886,8583,42146,main
|
||||
132750724,build-your-own-x,codecrafters-io/build-your-own-x,Master programming by recreating your favorite technologies from scratch.,"2018-05-09T12:03:18Z","2025-10-28T12:37:11Z","2025-10-10T18:45:01Z",430877,6332,40453,master
|
||||
21737465,awesome,sindresorhus/awesome,😎 Awesome lists about all kinds of interesting topics,"2014-07-11T13:42:37Z","2025-10-28T12:40:21Z","2025-10-27T17:57:31Z",410052,8017,32029,main
|
||||
```
|
||||
|
||||
</details>
|
||||
@@ -0,0 +1,207 @@
|
||||
import type { Question } from '../src/types.ts'
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import * as prompts from '@clack/prompts'
|
||||
import PQueue from 'p-queue'
|
||||
import { BENCHMARKS_DIR, DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants.ts'
|
||||
import { ACCURACY_DATASETS } from '../src/datasets.ts'
|
||||
import { evaluateQuestion, models } from '../src/evaluate.ts'
|
||||
import { formatters, supportsCSV } from '../src/formatters.ts'
|
||||
import { generateQuestions } from '../src/questions/index.ts'
|
||||
import { calculateFormatResults, calculateTokenCounts, generateAccuracyReport } from '../src/report.ts'
|
||||
import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage.ts'
|
||||
import { ensureDir } from '../src/utils.ts'
|
||||
|
||||
// Constants
|
||||
const PROGRESS_UPDATE_INTERVAL = 10
|
||||
const RATE_LIMIT_INTERVAL_MS = 60_000
|
||||
|
||||
prompts.intro('Retrieval Accuracy Benchmark')
|
||||
|
||||
/**
|
||||
* Generate evaluation tasks for a model
|
||||
*/
|
||||
function generateEvaluationTasks(questions: Question[]): { question: Question, formatName: string }[] {
|
||||
const tasks: { question: Question, formatName: string }[] = []
|
||||
|
||||
for (const question of questions) {
|
||||
for (const [formatName] of Object.entries(formatters)) {
|
||||
// Skip CSV for datasets that don't support it
|
||||
const dataset = ACCURACY_DATASETS.find(d => d.name === question.dataset)
|
||||
if (formatName === 'csv' && dataset && !supportsCSV(dataset))
|
||||
continue
|
||||
|
||||
tasks.push({ question, formatName })
|
||||
}
|
||||
}
|
||||
|
||||
return tasks
|
||||
}
|
||||
|
||||
/**
|
||||
* Check which models already have saved results
|
||||
*/
|
||||
async function checkExistingResults(activeModels: typeof models) {
|
||||
const existingModelResults: Record<string, boolean> = {}
|
||||
|
||||
for (const model of activeModels) {
|
||||
const existingResult = await hasModelResults(model.modelId)
|
||||
if (existingResult)
|
||||
existingModelResults[model.modelId] = existingResult
|
||||
}
|
||||
|
||||
return existingModelResults
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a progress updater function
|
||||
*/
|
||||
function createProgressUpdater(spinner: ReturnType<typeof prompts.spinner>, total: number) {
|
||||
let completed = 0
|
||||
|
||||
return () => {
|
||||
completed++
|
||||
if (completed % PROGRESS_UPDATE_INTERVAL === 0 || completed === total) {
|
||||
const percent = ((completed / total) * 100).toFixed(1)
|
||||
spinner.message(`Progress: ${completed}/${total} (${percent}%)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a rate-limited queue for model evaluation
|
||||
*/
|
||||
function createEvaluationQueue(modelId: string) {
|
||||
const rpmLimit = MODEL_RPM_LIMITS[modelId]
|
||||
|
||||
return new PQueue({
|
||||
concurrency: DEFAULT_CONCURRENCY,
|
||||
intervalCap: rpmLimit ?? Infinity,
|
||||
interval: rpmLimit ? RATE_LIMIT_INTERVAL_MS : 0,
|
||||
})
|
||||
}
|
||||
|
||||
// Prompt user to select which models to benchmark
|
||||
const modelChoices = models.map(({ modelId }) => ({
|
||||
value: modelId,
|
||||
label: modelId,
|
||||
}))
|
||||
|
||||
const selectedModels = await prompts.multiselect({
|
||||
message: 'Select models to benchmark (Space to select, Enter to confirm)',
|
||||
options: modelChoices,
|
||||
required: true,
|
||||
})
|
||||
|
||||
if (prompts.isCancel(selectedModels)) {
|
||||
prompts.cancel('Benchmark cancelled')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const activeModels = models.filter(m => selectedModels.includes(m.modelId))
|
||||
|
||||
prompts.log.info(`Selected ${activeModels.length} model(s): ${activeModels.map(m => m.modelId).join(', ')}`)
|
||||
|
||||
// Check which models already have results
|
||||
const existingModelResults = await checkExistingResults(activeModels)
|
||||
|
||||
if (Object.keys(existingModelResults).length > 0) {
|
||||
prompts.log.info(`Found existing results for ${Object.keys(existingModelResults).length} model(s)`)
|
||||
}
|
||||
|
||||
if (DRY_RUN) {
|
||||
prompts.log.info('Limiting questions and models for dry run')
|
||||
}
|
||||
|
||||
let questions = generateQuestions()
|
||||
|
||||
// Apply dry run limits if enabled
|
||||
if (DRY_RUN && DRY_RUN_LIMITS.maxQuestions) {
|
||||
questions = questions.slice(0, DRY_RUN_LIMITS.maxQuestions)
|
||||
}
|
||||
|
||||
prompts.log.info(`Evaluating ${questions.length} questions`)
|
||||
prompts.log.info(`Testing ${Object.keys(formatters).length} formats`)
|
||||
|
||||
// Evaluate each model separately and save results incrementally
|
||||
for (const model of activeModels) {
|
||||
const modelId = model.modelId
|
||||
|
||||
// Skip if results already exist
|
||||
if (existingModelResults[modelId]) {
|
||||
prompts.log.info(`Skipping ${modelId} (results already exist)`)
|
||||
continue
|
||||
}
|
||||
|
||||
prompts.log.step(`Running benchmark for ${modelId}`)
|
||||
|
||||
// Generate evaluation tasks for this model
|
||||
const tasks = generateEvaluationTasks(questions)
|
||||
|
||||
const total = tasks.length
|
||||
const rpmLimit = MODEL_RPM_LIMITS[modelId]
|
||||
const queue = createEvaluationQueue(modelId)
|
||||
|
||||
const evalSpinner = prompts.spinner()
|
||||
evalSpinner.start(`Running ${total} evaluations (concurrency: ${DEFAULT_CONCURRENCY}, RPM limit: ${rpmLimit ?? 'unlimited'})`)
|
||||
|
||||
const updateProgress = createProgressUpdater(evalSpinner, total)
|
||||
|
||||
// Queue all tasks
|
||||
const modelResultPromises = tasks.map(task =>
|
||||
queue.add(async () => {
|
||||
// Format data on-demand
|
||||
const dataset = ACCURACY_DATASETS.find(d => d.name === task.question.dataset)!
|
||||
const formatter = formatters[task.formatName]!
|
||||
const formattedData = formatter(dataset.data)
|
||||
|
||||
const result = await evaluateQuestion({
|
||||
question: task.question,
|
||||
formatName: task.formatName,
|
||||
formattedData,
|
||||
model,
|
||||
})
|
||||
|
||||
// Progress update after task completes
|
||||
updateProgress()
|
||||
|
||||
return result
|
||||
}),
|
||||
)
|
||||
|
||||
// Wait for all tasks to complete
|
||||
const modelResults = await Promise.all(modelResultPromises)
|
||||
|
||||
evalSpinner.stop(`Evaluation complete for ${modelId}`)
|
||||
|
||||
// Save results immediately for this model
|
||||
await saveModelResults(modelId, modelResults)
|
||||
prompts.log.success(`Saved results for ${modelId}`)
|
||||
}
|
||||
|
||||
// Generate/regenerate markdown report from all available model results
|
||||
const reportSpinner = prompts.spinner()
|
||||
reportSpinner.start('Generating report from all model results')
|
||||
|
||||
// Load all available model results (including any that were skipped)
|
||||
const allModelResults = await getAllModelResults()
|
||||
const allResults = Object.values(allModelResults).flat()
|
||||
|
||||
if (allResults.length === 0) {
|
||||
prompts.log.warn('No results available to generate report')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const tokenCounts = calculateTokenCounts(formatters)
|
||||
const formatResults = calculateFormatResults(allResults, tokenCounts)
|
||||
const accuracyReport = generateAccuracyReport(allResults, formatResults, tokenCounts)
|
||||
|
||||
const resultsDir = path.join(BENCHMARKS_DIR, 'results')
|
||||
await ensureDir(resultsDir)
|
||||
|
||||
const outputFilePath = path.join(resultsDir, 'retrieval-accuracy.md')
|
||||
await fsp.writeFile(outputFilePath, accuracyReport)
|
||||
|
||||
reportSpinner.stop('Report generation complete!')
|
||||
prompts.log.info(`Report saved to: \`${path.relative(ROOT_DIR, outputFilePath)}\``)
|
||||
@@ -0,0 +1,88 @@
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import * as prompts from '@clack/prompts'
|
||||
import { ofetch } from 'ofetch'
|
||||
import pMap from 'p-map'
|
||||
import { BENCHMARKS_DIR } from '../src/constants.ts'
|
||||
import { ensureDir } from '../src/utils.ts'
|
||||
|
||||
prompts.intro('GitHub Repositories Fetcher')
|
||||
|
||||
try {
|
||||
// Fetch top 100 repos from GitHub
|
||||
const repoList = await searchTop100Repos()
|
||||
const repos = await fetchRepoDetails(repoList)
|
||||
|
||||
if (repos.length === 0) {
|
||||
prompts.log.error('No repositories fetched. Exiting.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Sort by stars descending
|
||||
repos.sort((a, b) => b.stars - a.stars)
|
||||
|
||||
await saveRepos(repos)
|
||||
|
||||
prompts.log.success('Done!')
|
||||
}
|
||||
catch (error) {
|
||||
prompts.log.error(String(error))
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
async function searchTop100Repos(): Promise<string[]> {
|
||||
const s = prompts.spinner()
|
||||
s.start('Fetching top 100 starred repositories')
|
||||
|
||||
const response = await ofetch<{ items: { full_name: string }[] }>(
|
||||
'https://api.github.com/search/repositories',
|
||||
{
|
||||
query: {
|
||||
q: 'stars:>1',
|
||||
sort: 'stars',
|
||||
order: 'desc',
|
||||
per_page: 100,
|
||||
},
|
||||
headers: {
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
s.stop('Fetched top 100 repositories')
|
||||
|
||||
return response.items.map(item => item.full_name)
|
||||
}
|
||||
|
||||
async function fetchRepoDetails(repoList: string[]): Promise<Record<string, any>[]> {
|
||||
const s = prompts.spinner()
|
||||
s.start(`Fetching ${repoList.length} GitHub repositories`)
|
||||
|
||||
const repos = await pMap(
|
||||
repoList,
|
||||
async (repoPath, index) => {
|
||||
s.message(`[${index + 1}/${repoList.length}] Fetching ${repoPath}`)
|
||||
const { repo } = await ofetch(`https://ungh.cc/repos/${repoPath}`)
|
||||
return repo
|
||||
},
|
||||
{ concurrency: 5 },
|
||||
)
|
||||
|
||||
s.stop(`Successfully fetched ${repos.length}/${repoList.length} repositories`)
|
||||
|
||||
return repos
|
||||
}
|
||||
|
||||
async function saveRepos(repos: Record<string, any>[]): Promise<void> {
|
||||
const outputDir = path.join(BENCHMARKS_DIR, 'data')
|
||||
const outputFile = path.join(outputDir, 'github-repos.json')
|
||||
|
||||
await ensureDir(outputDir)
|
||||
const jsonOutput = JSON.stringify(repos, undefined, 2)
|
||||
await fsp.writeFile(outputFile, `${jsonOutput}\n`, 'utf-8')
|
||||
|
||||
const relativePath = path.relative(BENCHMARKS_DIR, outputFile)
|
||||
prompts.log.info(`Result saved to \`${relativePath}\``)
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
import type { Dataset } from '../src/types.ts'
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import * as prompts from '@clack/prompts'
|
||||
import { encode } from '../../packages/toon/src/index.ts'
|
||||
import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants.ts'
|
||||
import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets.ts'
|
||||
import { formatters, supportsCSV } from '../src/formatters.ts'
|
||||
import { createProgressBar, ensureDir, tokenize } from '../src/utils.ts'
|
||||
|
||||
interface FormatMetrics {
|
||||
name: string
|
||||
tokens: number
|
||||
savings: number
|
||||
savingsPercent: number
|
||||
}
|
||||
|
||||
interface BenchmarkResult {
|
||||
dataset: Dataset
|
||||
formats: FormatMetrics[]
|
||||
}
|
||||
|
||||
// Constants
|
||||
const DATASET_ICONS: Record<string, string> = {
|
||||
'tabular': '👥',
|
||||
'nested': '🛒',
|
||||
'analytics': '📈',
|
||||
'github': '⭐',
|
||||
'event-logs': '🧾',
|
||||
'nested-config': '🧩',
|
||||
}
|
||||
|
||||
const COMPARISON_FORMAT_ORDER = ['json-pretty', 'json-compact', 'yaml', 'xml'] as const
|
||||
|
||||
const PROGRESS_BAR_WIDTH = 20
|
||||
const TOKEN_PADDING = 7
|
||||
|
||||
const DEFAULT_DATASET_ICON = '📊'
|
||||
|
||||
const DETAILED_EXAMPLE_DATASETS = ['github', 'analytics'] as const
|
||||
const GITHUB_REPO_LIMIT = 3
|
||||
const GITHUB_DESC_LIMIT = 80
|
||||
const ANALYTICS_METRICS_LIMIT = 5
|
||||
|
||||
prompts.intro('Token Efficiency Benchmark')
|
||||
|
||||
/**
|
||||
* Format a comparison line showing savings vs TOON
|
||||
*/
|
||||
function formatComparisonLine(format: FormatMetrics, isLast: boolean = false): string {
|
||||
const label = FORMATTER_DISPLAY_NAMES[format.name] || format.name.toUpperCase()
|
||||
const signedPercent = format.savingsPercent >= 0
|
||||
? `−${format.savingsPercent.toFixed(1)}%`
|
||||
: `+${Math.abs(format.savingsPercent).toFixed(1)}%`
|
||||
const connector = isLast ? '└─' : '├─'
|
||||
const tokenStr = format.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||||
return `${connector} vs ${label.padEnd(13)} ${`(${signedPercent})`.padEnd(20)} ${tokenStr} tokens`
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate total tokens and savings for a set of datasets
|
||||
*/
|
||||
function calculateTotalMetrics(datasets: BenchmarkResult[], formatNames: readonly string[]) {
|
||||
const totalToonTokens = datasets.reduce((sum, r) => {
|
||||
const toon = r.formats.find(f => f.name === 'toon')!
|
||||
return sum + toon.tokens
|
||||
}, 0)
|
||||
|
||||
const totals = formatNames.map((formatName) => {
|
||||
const totalTokens = datasets.reduce((sum, r) => {
|
||||
const format = r.formats.find(f => f.name === formatName)
|
||||
return sum + (format?.tokens || 0)
|
||||
}, 0)
|
||||
const savings = totalTokens - totalToonTokens
|
||||
const savingsPercent = (savings / totalTokens) * 100
|
||||
return { name: formatName, tokens: totalTokens, savingsPercent }
|
||||
})
|
||||
|
||||
return { totalToonTokens, totals }
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate total lines for a track
|
||||
*/
|
||||
function generateTotalLines(
|
||||
totalToonTokens: number,
|
||||
totals: { name: string, tokens: number, savingsPercent: number }[],
|
||||
baselineFormat?: { name: string, tokens: number },
|
||||
) {
|
||||
const separatorHalf = '─'.repeat(36)
|
||||
const lines = [`${separatorHalf} Total ${separatorHalf}`]
|
||||
|
||||
if (baselineFormat) {
|
||||
// Flat-only track with CSV baseline
|
||||
const csvPercentage = Math.min(100, (baselineFormat.tokens / totalToonTokens) * 100)
|
||||
const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
|
||||
const csvStr = baselineFormat.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||||
lines.push(` CSV ${csvBar} ${csvStr} tokens`)
|
||||
|
||||
const overheadPercent = ((totalToonTokens - baselineFormat.tokens) / baselineFormat.tokens) * 100
|
||||
const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
|
||||
const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||||
lines.push(` TOON ${toonBar} ${toonStr} tokens (+${overheadPercent.toFixed(1)}% vs CSV)`)
|
||||
}
|
||||
else {
|
||||
// Mixed-structure track
|
||||
const totalPercentage = Math.min(100, (totalToonTokens / totals[0]!.tokens) * 100)
|
||||
const totalBar = createProgressBar(totalPercentage, 100, PROGRESS_BAR_WIDTH)
|
||||
const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||||
lines.push(` TOON ${totalBar} ${toonStr} tokens`)
|
||||
}
|
||||
|
||||
// Add comparison lines
|
||||
for (let i = 0; i < totals.length; i++) {
|
||||
const format = totals[i]!
|
||||
const isLast = i === totals.length - 1
|
||||
lines.push(` ${formatComparisonLine({
|
||||
name: format.name,
|
||||
tokens: format.tokens,
|
||||
savings: 0, // Not used in this context
|
||||
savingsPercent: format.savingsPercent,
|
||||
}, isLast)}`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate bar chart for a dataset
|
||||
*/
|
||||
function generateDatasetChart(result: BenchmarkResult): string {
|
||||
const { dataset, formats } = result
|
||||
const toon = formats.find(f => f.name === 'toon')!
|
||||
const jsonPretty = formats.find(f => f.name === 'json-pretty')!
|
||||
|
||||
const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
|
||||
const eligibility = dataset.metadata.tabularEligibility
|
||||
const name = dataset.description
|
||||
|
||||
const percentage = Math.min(100, 100 - jsonPretty.savingsPercent)
|
||||
const bar = createProgressBar(percentage, 100, PROGRESS_BAR_WIDTH)
|
||||
const toonStr = toon.tokens.toLocaleString('en-US')
|
||||
|
||||
const line1 = `${emoji} ${name} ┊ Tabular: ${eligibility}%`
|
||||
const line2 = ` │`
|
||||
const line3 = ` TOON ${bar} ${toonStr.padStart(TOKEN_PADDING)} tokens`
|
||||
|
||||
const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
|
||||
const format = formats.find(f => f.name === formatName)
|
||||
if (!format)
|
||||
return undefined
|
||||
|
||||
return ` ${formatComparisonLine(format, index === array.length - 1)}`
|
||||
}).filter(Boolean)
|
||||
|
||||
return [line1, line2, line3, ...comparisonLines].join('\n')
|
||||
}
|
||||
|
||||
const results: BenchmarkResult[] = []
|
||||
|
||||
// Calculate token counts for all datasets
|
||||
for (const dataset of TOKEN_EFFICIENCY_DATASETS) {
|
||||
const formatMetrics: FormatMetrics[] = []
|
||||
const tokensByFormat: Record<string, number> = {}
|
||||
|
||||
// Calculate tokens for each format
|
||||
for (const [formatName, formatter] of Object.entries(formatters)) {
|
||||
// Skip CSV for datasets that don't support it
|
||||
if (formatName === 'csv' && !supportsCSV(dataset))
|
||||
continue
|
||||
|
||||
const formattedData = formatter(dataset.data)
|
||||
const tokens = tokenize(formattedData)
|
||||
tokensByFormat[formatName] = tokens
|
||||
}
|
||||
|
||||
// Calculate savings vs TOON
|
||||
const toonTokens = tokensByFormat.toon!
|
||||
for (const [formatName, tokens] of Object.entries(tokensByFormat)) {
|
||||
const savings = tokens - toonTokens
|
||||
formatMetrics.push({
|
||||
name: formatName,
|
||||
tokens,
|
||||
savings,
|
||||
savingsPercent: formatName === 'toon' ? 0 : (savings / tokens) * 100,
|
||||
})
|
||||
}
|
||||
|
||||
results.push({
|
||||
dataset,
|
||||
formats: formatMetrics,
|
||||
})
|
||||
}
|
||||
|
||||
// Separate datasets by CSV support
|
||||
const mixedStructureDatasets = results.filter(r => !supportsCSV(r.dataset))
|
||||
const flatOnlyDatasets = results.filter(r => supportsCSV(r.dataset))
|
||||
|
||||
// Mixed-Structure Track (no CSV)
|
||||
const mixedCharts = mixedStructureDatasets
|
||||
.map(result => generateDatasetChart(result))
|
||||
.join('\n\n')
|
||||
|
||||
// Flat-Only Track (with CSV)
|
||||
const flatCharts = flatOnlyDatasets
|
||||
.map((result) => {
|
||||
const csv = result.formats.find(f => f.name === 'csv')
|
||||
const toon = result.formats.find(f => f.name === 'toon')!
|
||||
|
||||
if (!csv)
|
||||
return generateDatasetChart(result)
|
||||
|
||||
// Special handling to show CSV first with TOON overhead
|
||||
const { dataset } = result
|
||||
const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
|
||||
const eligibility = dataset.metadata.tabularEligibility
|
||||
const name = dataset.description
|
||||
|
||||
// CSV line
|
||||
const csvPercentage = Math.min(100, (csv.tokens / toon.tokens) * 100)
|
||||
const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
|
||||
const csvStr = csv.tokens.toLocaleString('en-US')
|
||||
|
||||
const line1 = `${emoji} ${name} ┊ Tabular: ${eligibility}%`
|
||||
const line2 = ` │`
|
||||
const line3 = ` CSV ${csvBar} ${csvStr.padStart(TOKEN_PADDING)} tokens`
|
||||
|
||||
const toonOverhead = toon.tokens - csv.tokens
|
||||
const toonOverheadPercent = (toonOverhead / csv.tokens) * 100
|
||||
const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
|
||||
const toonStr = toon.tokens.toLocaleString('en-US')
|
||||
const toonVsCSV = toonOverheadPercent >= 0
|
||||
? `(+${toonOverheadPercent.toFixed(1)}% vs CSV)`
|
||||
: `(${toonOverheadPercent.toFixed(1)}% vs CSV)`
|
||||
const toonLine = ` TOON ${toonBar} ${toonStr.padStart(TOKEN_PADDING)} tokens ${toonVsCSV}`
|
||||
|
||||
// Other format comparisons (vs TOON)
|
||||
const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
|
||||
const format = result.formats.find(f => f.name === formatName)
|
||||
if (!format)
|
||||
return undefined
|
||||
|
||||
return ` ${formatComparisonLine(format, index === array.length - 1)}`
|
||||
}).filter(Boolean)
|
||||
|
||||
return [line1, line2, line3, toonLine, ...comparisonLines].join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
|
||||
// Calculate totals for mixed structure
|
||||
const { totalToonTokens: totalToonTokensMixed, totals: mixedTotals } = calculateTotalMetrics(mixedStructureDatasets, COMPARISON_FORMAT_ORDER)
|
||||
const mixedTotalLines = generateTotalLines(totalToonTokensMixed, mixedTotals)
|
||||
|
||||
// Calculate totals for flat-only
|
||||
const { totalToonTokens: totalToonTokensFlat, totals: flatTotals } = calculateTotalMetrics(flatOnlyDatasets, COMPARISON_FORMAT_ORDER)
|
||||
const totalCSVTokensFlat = flatOnlyDatasets.reduce((sum, r) => {
|
||||
const csv = r.formats.find(f => f.name === 'csv')
|
||||
return sum + (csv?.tokens || 0)
|
||||
}, 0)
|
||||
const flatTotalLines = generateTotalLines(totalToonTokensFlat, flatTotals, { name: 'csv', tokens: totalCSVTokensFlat })
|
||||
|
||||
const barChartSection = `
|
||||
#### Mixed-Structure Track
|
||||
|
||||
Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
|
||||
|
||||
\`\`\`
|
||||
${mixedCharts}
|
||||
|
||||
${mixedTotalLines}
|
||||
\`\`\`
|
||||
|
||||
#### Flat-Only Track
|
||||
|
||||
Datasets with flat tabular structures where CSV is applicable.
|
||||
|
||||
\`\`\`
|
||||
${flatCharts}
|
||||
|
||||
${flatTotalLines}
|
||||
\`\`\`
|
||||
`.trim()
|
||||
|
||||
// Generate detailed examples (optional: show a few examples)
|
||||
const detailedExamples = results
|
||||
.filter(r => DETAILED_EXAMPLE_DATASETS.includes(r.dataset.name as any))
|
||||
.map((result, i, filtered) => {
|
||||
let displayData = result.dataset.data
|
||||
|
||||
// Truncate for display
|
||||
if (result.dataset.name === 'github') {
|
||||
displayData = {
|
||||
repositories: displayData.repositories.slice(0, GITHUB_REPO_LIMIT).map((repo: Record<string, any>) => ({
|
||||
...repo,
|
||||
description: repo.description?.slice(0, GITHUB_DESC_LIMIT) + (repo.description?.length > GITHUB_DESC_LIMIT ? '…' : ''),
|
||||
})),
|
||||
}
|
||||
}
|
||||
else if (result.dataset.name === 'analytics') {
|
||||
displayData = { metrics: displayData.metrics.slice(0, ANALYTICS_METRICS_LIMIT) }
|
||||
}
|
||||
|
||||
const emoji = DATASET_ICONS[result.dataset.name] || DEFAULT_DATASET_ICON
|
||||
const json = result.formats.find(f => f.name === 'json-pretty')!
|
||||
const toon = result.formats.find(f => f.name === 'toon')!
|
||||
const separator = i < filtered.length - 1 ? '---' : ''
|
||||
|
||||
return `
|
||||
#### ${emoji} ${result.dataset.description}
|
||||
|
||||
**Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent.toFixed(1)}% reduction vs JSON)
|
||||
|
||||
**JSON** (${json.tokens.toLocaleString('en-US')} tokens):
|
||||
|
||||
\`\`\`json
|
||||
${JSON.stringify(displayData, undefined, 2)}
|
||||
\`\`\`
|
||||
|
||||
**TOON** (${toon.tokens.toLocaleString('en-US')} tokens):
|
||||
|
||||
\`\`\`
|
||||
${encode(displayData)}
|
||||
\`\`\`
|
||||
|
||||
${separator}
|
||||
`.trim()
|
||||
})
|
||||
.join('\n\n')
|
||||
|
||||
const markdown = `
|
||||
${barChartSection}
|
||||
|
||||
<details>
|
||||
<summary><strong>Show detailed examples</strong></summary>
|
||||
|
||||
${detailedExamples}
|
||||
|
||||
</details>
|
||||
`.trimStart()
|
||||
|
||||
prompts.log.message(barChartSection)
|
||||
|
||||
const resultsDir = path.join(BENCHMARKS_DIR, 'results')
|
||||
await ensureDir(resultsDir)
|
||||
|
||||
const outputFilePath = path.join(resultsDir, 'token-efficiency.md')
|
||||
await fsp.writeFile(outputFilePath, markdown, 'utf-8')
|
||||
|
||||
prompts.log.success(`Report saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)
|
||||
@@ -0,0 +1,189 @@
|
||||
import process from 'node:process'
|
||||
import * as url from 'node:url'
|
||||
|
||||
export const ROOT_DIR: string = url.fileURLToPath(new URL('../../', import.meta.url))
|
||||
export const BENCHMARKS_DIR: string = url.fileURLToPath(new URL('../', import.meta.url))
|
||||
|
||||
/**
|
||||
* Default concurrency for parallel evaluations to prevent bursting
|
||||
*/
|
||||
export const DEFAULT_CONCURRENCY = 10
|
||||
|
||||
/**
|
||||
* Enable dry run mode for quick testing with limited AI requests
|
||||
*
|
||||
* @remarks
|
||||
* Set via environment variable: `DRY_RUN=true`.
|
||||
*/
|
||||
export const DRY_RUN: boolean = process.env.DRY_RUN === 'true'
|
||||
|
||||
/**
|
||||
* Limits applied during dry run mode
|
||||
*/
|
||||
export const DRY_RUN_LIMITS = {
|
||||
/** Maximum number of questions to evaluate */
|
||||
maxQuestions: 10,
|
||||
}
|
||||
|
||||
/**
|
||||
* Model-specific RPM (requests per minute) limits to handle API quotas
|
||||
*
|
||||
* @remarks
|
||||
* Set `undefined` for models without specific limits.
|
||||
*/
|
||||
/// keep-sorted
|
||||
export const MODEL_RPM_LIMITS: Record<string, number | undefined> = {
|
||||
'claude-haiku-4-5-20251001': 50,
|
||||
'gemini-3-flash-preview': 25,
|
||||
'gpt-5-nano': 50,
|
||||
'grok-4-1-fast-non-reasoning': 25,
|
||||
}
|
||||
|
||||
/**
|
||||
* Display names for data format types
|
||||
*/
|
||||
export const FORMATTER_DISPLAY_NAMES: Record<string, string> = {
|
||||
'json-pretty': 'JSON',
|
||||
'json-compact': 'JSON compact',
|
||||
'toon': 'TOON',
|
||||
'csv': 'CSV',
|
||||
'xml': 'XML',
|
||||
'yaml': 'YAML',
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Question type identifiers
|
||||
*/
|
||||
export const QUESTION_TYPES = [
|
||||
'field-retrieval',
|
||||
'retrieval',
|
||||
'aggregation',
|
||||
'filtering',
|
||||
'structure-awareness',
|
||||
'structural-validation',
|
||||
] as const
|
||||
|
||||
/**
|
||||
* Display names for question types
|
||||
*/
|
||||
export const QUESTION_TYPE_LABELS = {
|
||||
'field-retrieval': 'Field Retrieval',
|
||||
'retrieval': 'Retrieval',
|
||||
'aggregation': 'Aggregation',
|
||||
'filtering': 'Filtering',
|
||||
'structure-awareness': 'Structure Awareness',
|
||||
'structural-validation': 'Structural Validation',
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Dataset identifiers
|
||||
*/
|
||||
export const DATASET_NAMES = [
|
||||
'tabular',
|
||||
'nested',
|
||||
'analytics',
|
||||
'github',
|
||||
'event-logs',
|
||||
'nested-config',
|
||||
'large-uniform',
|
||||
'structural-validation-control',
|
||||
'structural-validation-truncated',
|
||||
'structural-validation-extra-rows',
|
||||
'structural-validation-width-mismatch',
|
||||
'structural-validation-missing-fields',
|
||||
] as const
|
||||
|
||||
/**
|
||||
* Structure class identifiers
|
||||
*/
|
||||
export const STRUCTURE_CLASSES = [
|
||||
'uniform',
|
||||
'semi-uniform',
|
||||
'nested',
|
||||
'deep',
|
||||
] as const
|
||||
|
||||
/**
|
||||
* Threshold values for filtering and aggregation questions
|
||||
*/
|
||||
export const QUESTION_THRESHOLDS = {
|
||||
tabular: {
|
||||
salaryRanges: [60000, 80000, 100000],
|
||||
experienceYears: [5, 10, 15, 20],
|
||||
departmentSalaryThreshold: 80000,
|
||||
departmentExperienceThreshold: 10,
|
||||
},
|
||||
nested: {
|
||||
highValueOrders: [200, 400, 600],
|
||||
statusValueThreshold: 300,
|
||||
itemCountThreshold: 3,
|
||||
totalThresholdsForItems: [300, 500],
|
||||
},
|
||||
analytics: {
|
||||
views: [6000],
|
||||
conversions: [20],
|
||||
viewsForFiltering: [6000, 7000],
|
||||
conversionsForFiltering: 15,
|
||||
revenueThresholds: [1000, 1500, 2000],
|
||||
viewsThresholdForRevenue: 6000,
|
||||
clicksForFiltering: [250, 400],
|
||||
conversionsForClickFiltering: 15,
|
||||
revenueForBounceRate: [1000, 1500],
|
||||
bounceRateThreshold: 0.5,
|
||||
},
|
||||
github: {
|
||||
stars: [100000, 150000, 200000],
|
||||
forks: [20000, 35000],
|
||||
watchers: [8000],
|
||||
starForkCombinations: [
|
||||
{ stars: 75000, forks: 15000 },
|
||||
{ stars: 100000, forks: 20000 },
|
||||
{ stars: 150000, forks: 30000 },
|
||||
{ stars: 200000, forks: 45000 },
|
||||
],
|
||||
starWatcherCombinations: [
|
||||
{ stars: 100000, watchers: 7000 },
|
||||
{ stars: 150000, watchers: 9000 },
|
||||
],
|
||||
},
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Question generation configuration
|
||||
*/
|
||||
export const QUESTION_LIMITS = {
|
||||
tabular: {
|
||||
fieldRetrieval: 12,
|
||||
aggregationDepartments: 3,
|
||||
filteringMultiConditionDepartments: 5,
|
||||
filteringExperience: 3,
|
||||
filteringDepartmentExp: 3,
|
||||
filteringDepartmentActive: 2,
|
||||
},
|
||||
nested: {
|
||||
fieldRetrievalOrders: 8,
|
||||
fieldRetrievalCustomers: 8,
|
||||
aggregationStatuses: 3,
|
||||
filteringStatusAndValue: 4,
|
||||
filteringStatusAndItems: 3,
|
||||
},
|
||||
analytics: {
|
||||
fieldRetrievalDates: 9,
|
||||
},
|
||||
github: {
|
||||
fieldRetrievalRepos: 11,
|
||||
aggregationBranches: 2,
|
||||
filteringStarsAndForks: 3,
|
||||
},
|
||||
eventLogs: {
|
||||
fieldRetrieval: 10,
|
||||
aggregationEndpoints: 2,
|
||||
filteringLevelAndStatus: 3,
|
||||
filteringEndpointAndStatus: 3,
|
||||
filteringEndpointRetryable: 2,
|
||||
},
|
||||
nestedConfig: {
|
||||
fieldRetrieval: 10,
|
||||
filteringComplex: 5,
|
||||
},
|
||||
} as const
|
||||
@@ -0,0 +1,753 @@
|
||||
import type { Dataset } from './types.ts'
|
||||
import { faker } from '@faker-js/faker'
|
||||
import githubRepos from '../data/github-repos.json' with { type: 'json' }
|
||||
|
||||
// Seed for reproducibility
|
||||
faker.seed(12345)
|
||||
|
||||
/**
|
||||
* Employee record structure for tabular dataset
|
||||
*/
|
||||
export interface Employee {
|
||||
id: number
|
||||
name: string
|
||||
email: string
|
||||
department: string
|
||||
salary: number
|
||||
yearsExperience: number
|
||||
active: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* E-commerce order structure for nested dataset
|
||||
*/
|
||||
export interface Order {
|
||||
orderId: string
|
||||
customer: {
|
||||
id: number
|
||||
name: string
|
||||
email: string
|
||||
phone: string
|
||||
}
|
||||
items: {
|
||||
sku: string
|
||||
name: string
|
||||
quantity: number
|
||||
price: number
|
||||
}[]
|
||||
subtotal: number
|
||||
tax: number
|
||||
total: number
|
||||
status: string
|
||||
orderDate?: string
|
||||
createdAt?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Analytics metric structure for time-series dataset
|
||||
*/
|
||||
export interface AnalyticsMetric {
|
||||
date: string
|
||||
views: number
|
||||
clicks: number
|
||||
conversions: number
|
||||
revenue: number
|
||||
bounceRate: number
|
||||
}
|
||||
|
||||
/**
|
||||
* GitHub repository structure for real-world dataset
|
||||
*/
|
||||
export interface Repository {
|
||||
id: number
|
||||
name: string
|
||||
repo: string
|
||||
description: string
|
||||
stars: number
|
||||
watchers: number
|
||||
forks: number
|
||||
defaultBranch: string
|
||||
createdAt: string
|
||||
updatedAt: string
|
||||
pushedAt: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Event log structure for semi-uniform dataset
|
||||
*/
|
||||
export interface EventLog {
|
||||
timestamp: string
|
||||
level: 'info' | 'warn' | 'error'
|
||||
endpoint: string
|
||||
statusCode: number
|
||||
responseTime: number
|
||||
userId: number
|
||||
error?: {
|
||||
message: string
|
||||
stack: string
|
||||
retryable: boolean
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Nested configuration structure for deeply nested dataset
|
||||
*/
|
||||
export interface NestedConfig {
|
||||
environment: string
|
||||
version: string
|
||||
database: {
|
||||
host: string
|
||||
port: number
|
||||
name: string
|
||||
pool: {
|
||||
min: number
|
||||
max: number
|
||||
idleTimeout: number
|
||||
}
|
||||
replicas: {
|
||||
host: string
|
||||
port: number
|
||||
priority: number
|
||||
}[]
|
||||
}
|
||||
features: Record<string, {
|
||||
enabled: boolean
|
||||
rollout: number
|
||||
variants: {
|
||||
name: string
|
||||
weight: number
|
||||
config: Record<string, any>
|
||||
}[]
|
||||
}>
|
||||
authentication: {
|
||||
providers: {
|
||||
name: string
|
||||
clientId: string
|
||||
scopes: string[]
|
||||
config: Record<string, any>
|
||||
}[]
|
||||
session: {
|
||||
secret: string
|
||||
duration: number
|
||||
refreshThreshold: number
|
||||
}
|
||||
}
|
||||
permissions: {
|
||||
roles: Record<string, {
|
||||
permissions: string[]
|
||||
inherits: string[]
|
||||
}>
|
||||
groups: Record<string, {
|
||||
members: string[]
|
||||
roles: string[]
|
||||
}>
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Product structure for large uniform arrays
|
||||
*/
|
||||
export interface Product {
|
||||
sku: string
|
||||
name: string
|
||||
category: string
|
||||
price: number
|
||||
qty: number
|
||||
lastUpdated: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal types for structural validation pattern generation
|
||||
*/
|
||||
type StructuralValidationType = 'truncated' | 'extra-rows' | 'width-mismatch' | 'missing-fields'
|
||||
|
||||
interface StructuralValidationFixture {
|
||||
type: StructuralValidationType
|
||||
description: string
|
||||
data: Record<string, unknown>
|
||||
isValid: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate analytics time-series data
|
||||
*/
|
||||
export function generateAnalyticsData(days: number, startDate = '2025-01-01'): {
|
||||
metrics: AnalyticsMetric[]
|
||||
} {
|
||||
const date = new Date(startDate)
|
||||
|
||||
return {
|
||||
metrics: Array.from({ length: days }, (_, i) => {
|
||||
const currentDate = new Date(date)
|
||||
currentDate.setDate(currentDate.getDate() + i)
|
||||
|
||||
// Simulate realistic web traffic with some variation
|
||||
const baseViews = 5000
|
||||
const weekendMultiplier = currentDate.getDay() === 0 || currentDate.getDay() === 6 ? 0.7 : 1.0
|
||||
const views = Math.round(baseViews * weekendMultiplier + faker.number.int({ min: -1000, max: 3000 }))
|
||||
const clicks = Math.round(views * faker.number.float({ min: 0.02, max: 0.08 }))
|
||||
const conversions = Math.round(clicks * faker.number.float({ min: 0.05, max: 0.15 }))
|
||||
const avgOrderValue = faker.number.float({ min: 49.99, max: 299.99 })
|
||||
const revenue = Number((conversions * avgOrderValue).toFixed(2))
|
||||
|
||||
return {
|
||||
date: currentDate.toISOString().split('T')[0]!,
|
||||
views,
|
||||
clicks,
|
||||
conversions,
|
||||
revenue,
|
||||
bounceRate: faker.number.float({ min: 0.3, max: 0.7, fractionDigits: 2 }),
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate employee data (uniform tabular structure)
|
||||
*/
|
||||
const departments = ['Engineering', 'Sales', 'Marketing', 'HR', 'Operations', 'Finance'] as const
|
||||
|
||||
function generateEmployees(count: number): { employees: Employee[] } {
|
||||
return {
|
||||
employees: Array.from({ length: count }, (_, i): Employee => {
|
||||
const yearsExp = faker.number.int({ min: 1, max: 25 })
|
||||
return {
|
||||
id: i + 1,
|
||||
name: faker.person.fullName(),
|
||||
email: faker.internet.email().toLowerCase(),
|
||||
department: departments[i % departments.length]!,
|
||||
salary: faker.number.int({ min: 45000, max: 150000 }),
|
||||
yearsExperience: yearsExp,
|
||||
active: faker.datatype.boolean(0.8), // 80% active
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tabular dataset: Uniform employee records
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's tabular array format.
|
||||
*/
|
||||
const tabularDataset: Dataset = {
|
||||
name: 'tabular',
|
||||
description: 'Uniform employee records',
|
||||
data: generateEmployees(100),
|
||||
metadata: {
|
||||
supportsCSV: true,
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100, // All arrays contain uniform objects with primitive values only
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate e-commerce orders (nested structure)
|
||||
*/
|
||||
const PRODUCT_NAMES = ['Wireless Mouse', 'USB Cable', 'Laptop Stand', 'Keyboard', 'Webcam', 'Headphones', 'Monitor', 'Desk Lamp'] as const
|
||||
const ORDER_STATUSES = ['pending', 'processing', 'shipped', 'delivered', 'cancelled'] as const
|
||||
|
||||
function generateOrders(count: number): { orders: Order[] } {
|
||||
return {
|
||||
orders: Array.from({ length: count }, (_, i) => {
|
||||
const customerId = (i % 20) + 1 // Rotate through 20 customers
|
||||
const itemCount = faker.number.int({ min: 1, max: 4 }) // 1-4 items per order
|
||||
|
||||
const items = Array.from({ length: itemCount }, (_, j) => {
|
||||
const price = faker.number.float({
|
||||
min: 9.99,
|
||||
max: 199.99,
|
||||
fractionDigits: 2,
|
||||
})
|
||||
const quantity = faker.number.int({ min: 1, max: 5 })
|
||||
return {
|
||||
sku: `SKU-${faker.string.alphanumeric({ length: 6 }).toUpperCase()}`,
|
||||
name: PRODUCT_NAMES[j % PRODUCT_NAMES.length]!,
|
||||
quantity,
|
||||
price,
|
||||
}
|
||||
})
|
||||
|
||||
const subtotal = Number(items.reduce((sum, item) => sum + (item.price * item.quantity), 0).toFixed(2))
|
||||
const tax = Number((subtotal * 0.08).toFixed(2)) // 8% tax rate
|
||||
const total = Number((subtotal + tax).toFixed(2))
|
||||
|
||||
return {
|
||||
orderId: `ORD-${String(i + 1).padStart(4, '0')}`,
|
||||
customer: {
|
||||
id: customerId,
|
||||
name: faker.person.fullName(),
|
||||
email: faker.internet.email().toLowerCase(),
|
||||
phone: faker.phone.number(),
|
||||
},
|
||||
items,
|
||||
subtotal,
|
||||
tax,
|
||||
total,
|
||||
status: ORDER_STATUSES[i % ORDER_STATUSES.length]!,
|
||||
orderDate: faker.date.recent({ days: 90 }).toISOString().split('T')[0],
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Nested dataset: E-commerce orders with nested structures
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's handling of complex nested objects.
|
||||
*/
|
||||
const nestedDataset: Dataset = {
|
||||
name: 'nested',
|
||||
description: 'E-commerce orders with nested structures',
|
||||
data: generateOrders(50),
|
||||
metadata: {
|
||||
supportsCSV: false,
|
||||
structureClass: 'nested',
|
||||
tabularEligibility: 33, // Top-level orders array has nested objects (not tabular), but nested items arrays are tabular
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Analytics dataset: Time-series metrics
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's handling of numeric data and date fields.
|
||||
*/
|
||||
const analyticsDataset: Dataset = {
|
||||
name: 'analytics',
|
||||
description: 'Time-series analytics data',
|
||||
data: generateAnalyticsData(60),
|
||||
metadata: {
|
||||
supportsCSV: true,
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100, // Uniform time-series records with consistent primitive fields
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Real-world dataset: Top 100 starred GitHub repositories
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's tabular format with real data.
|
||||
*/
|
||||
const githubDataset: Dataset = {
|
||||
name: 'github',
|
||||
description: 'Top 100 GitHub repositories',
|
||||
data: {
|
||||
repositories: githubRepos,
|
||||
},
|
||||
metadata: {
|
||||
supportsCSV: true,
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100, // Repository array contains uniform objects with primitive values
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a single e-commerce order with nested structure
|
||||
*
|
||||
* @remarks
|
||||
* Used for token efficiency benchmarks.
|
||||
*/
|
||||
export function generateOrderData(): Order {
|
||||
return {
|
||||
orderId: faker.string.alphanumeric({ length: 12, casing: 'upper' }),
|
||||
customer: {
|
||||
id: faker.number.int({ min: 1000, max: 9999 }),
|
||||
name: faker.person.fullName(),
|
||||
email: faker.internet.email(),
|
||||
phone: faker.phone.number(),
|
||||
},
|
||||
items: Array.from({ length: faker.number.int({ min: 2, max: 5 }) }, () => ({
|
||||
sku: faker.string.alphanumeric({ length: 8, casing: 'upper' }),
|
||||
name: faker.commerce.productName(),
|
||||
quantity: faker.number.int({ min: 1, max: 5 }),
|
||||
price: Number(faker.commerce.price({ min: 10, max: 200 })),
|
||||
})),
|
||||
subtotal: Number(faker.commerce.price({ min: 100, max: 500 })),
|
||||
tax: Number(faker.commerce.price({ min: 10, max: 50 })),
|
||||
total: Number(faker.commerce.price({ min: 110, max: 550 })),
|
||||
status: faker.helpers.arrayElement(['pending', 'processing', 'shipped', 'delivered']),
|
||||
createdAt: faker.date.recent({ days: 7 }).toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate event logs (semi-uniform structure)
|
||||
*
|
||||
* @remarks
|
||||
* Approximately 50% of logs include nested error objects, 50% are flat.
|
||||
* This creates ~45% tabular eligibility.
|
||||
*/
|
||||
export function generateEventLogs(count: number): { logs: EventLog[] } {
|
||||
const endpoints = ['/api/users', '/api/orders', '/api/products', '/api/auth', '/api/payments']
|
||||
const levels = ['info', 'warn', 'error'] as const
|
||||
|
||||
return {
|
||||
logs: Array.from({ length: count }, () => {
|
||||
const level = faker.helpers.arrayElement(levels)
|
||||
const hasError = level === 'error' || (level === 'warn' && faker.datatype.boolean(0.3))
|
||||
|
||||
const log: EventLog = {
|
||||
timestamp: faker.date.recent({ days: 7 }).toISOString(),
|
||||
level,
|
||||
endpoint: faker.helpers.arrayElement(endpoints),
|
||||
statusCode: hasError
|
||||
? faker.number.int({ min: 400, max: 599 })
|
||||
: faker.number.int({ min: 200, max: 299 }),
|
||||
responseTime: faker.number.int({ min: 10, max: 5000 }),
|
||||
userId: faker.number.int({ min: 1000, max: 9999 }),
|
||||
}
|
||||
|
||||
if (hasError) {
|
||||
log.error = {
|
||||
message: faker.helpers.arrayElement([
|
||||
'Database connection timeout',
|
||||
'Invalid authentication token',
|
||||
'Resource not found',
|
||||
'Internal server error',
|
||||
'Rate limit exceeded',
|
||||
]),
|
||||
stack: `Error: ${faker.lorem.sentence()}\n at ${faker.lorem.word()}\n at ${faker.lorem.word()}`,
|
||||
retryable: faker.datatype.boolean(0.6),
|
||||
}
|
||||
}
|
||||
|
||||
return log
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate deeply nested configuration
|
||||
*
|
||||
* @remarks
|
||||
* Creates a complex nested structure with minimal tabular eligibility (~0%).
|
||||
*/
|
||||
export function generateNestedConfig(): NestedConfig {
|
||||
return {
|
||||
environment: faker.helpers.arrayElement(['production', 'staging', 'development']),
|
||||
version: faker.system.semver(),
|
||||
database: {
|
||||
host: faker.internet.domainName(),
|
||||
port: 5432,
|
||||
name: faker.database.type(),
|
||||
pool: {
|
||||
min: 2,
|
||||
max: faker.number.int({ min: 10, max: 50 }),
|
||||
idleTimeout: 30000,
|
||||
},
|
||||
replicas: Array.from({ length: 3 }, (_, i) => ({
|
||||
host: `replica-${i + 1}.${faker.internet.domainName()}`,
|
||||
port: 5432,
|
||||
priority: i + 1,
|
||||
})),
|
||||
},
|
||||
features: {
|
||||
darkMode: {
|
||||
enabled: faker.datatype.boolean(),
|
||||
rollout: faker.number.int({ min: 0, max: 100 }),
|
||||
variants: [
|
||||
{
|
||||
name: 'default',
|
||||
weight: 70,
|
||||
config: { theme: 'dark', animations: true },
|
||||
},
|
||||
{
|
||||
name: 'minimal',
|
||||
weight: 30,
|
||||
config: { theme: 'dark', animations: false },
|
||||
},
|
||||
],
|
||||
},
|
||||
analytics: {
|
||||
enabled: faker.datatype.boolean(),
|
||||
rollout: faker.number.int({ min: 0, max: 100 }),
|
||||
variants: [
|
||||
{
|
||||
name: 'full',
|
||||
weight: 100,
|
||||
config: { tracking: 'all', sampling: 1.0 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
authentication: {
|
||||
providers: [
|
||||
{
|
||||
name: 'oauth2',
|
||||
clientId: faker.string.uuid(),
|
||||
scopes: ['read', 'write', 'admin'],
|
||||
config: {
|
||||
authUrl: faker.internet.url(),
|
||||
tokenUrl: faker.internet.url(),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'saml',
|
||||
clientId: faker.string.uuid(),
|
||||
scopes: ['read'],
|
||||
config: {
|
||||
entryPoint: faker.internet.url(),
|
||||
cert: faker.string.alphanumeric({ length: 64 }),
|
||||
},
|
||||
},
|
||||
],
|
||||
session: {
|
||||
secret: faker.string.alphanumeric({ length: 32 }),
|
||||
duration: 86400,
|
||||
refreshThreshold: 3600,
|
||||
},
|
||||
},
|
||||
permissions: {
|
||||
roles: {
|
||||
admin: {
|
||||
permissions: ['read', 'write', 'delete', 'manage_users', 'manage_roles'],
|
||||
inherits: [],
|
||||
},
|
||||
editor: {
|
||||
permissions: ['read', 'write'],
|
||||
inherits: ['viewer'],
|
||||
},
|
||||
viewer: {
|
||||
permissions: ['read'],
|
||||
inherits: [],
|
||||
},
|
||||
},
|
||||
groups: {
|
||||
engineering: {
|
||||
members: Array.from({ length: 5 }, () => faker.internet.email()),
|
||||
roles: ['admin', 'editor'],
|
||||
},
|
||||
support: {
|
||||
members: Array.from({ length: 3 }, () => faker.internet.email()),
|
||||
roles: ['viewer'],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate large uniform product array (5000+ rows)
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's token efficiency and structural reliability at scale.
|
||||
*/
|
||||
export function generateProducts(count: number): { products: Product[] } {
|
||||
const categories = ['Electronics', 'Clothing', 'Home & Garden', 'Sports', 'Books', 'Toys'] as const
|
||||
|
||||
return {
|
||||
products: Array.from({ length: count }, (_, i): Product => ({
|
||||
sku: `SKU-${String(i + 1).padStart(6, '0')}`,
|
||||
name: faker.commerce.productName(),
|
||||
category: categories[i % categories.length]!,
|
||||
price: Number(faker.commerce.price({ min: 5, max: 500 })),
|
||||
qty: faker.number.int({ min: 0, max: 1000 }),
|
||||
lastUpdated: faker.date.recent({ days: 30 }).toISOString().split('T')[0]!,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate structural validation fixtures from employee data
|
||||
*
|
||||
* @remarks
|
||||
* Creates deliberately corrupted datasets to test TOON's structural validation
|
||||
* capabilities via [N] length declarations and {fields} headers.
|
||||
* Internal function used to generate structural validation datasets.
|
||||
*/
|
||||
function generateStructuralValidationFixtures(): StructuralValidationFixture[] {
|
||||
const baseData = generateEmployees(20)
|
||||
|
||||
return [
|
||||
// Valid baseline
|
||||
{
|
||||
type: 'truncated' as const,
|
||||
description: 'Valid complete dataset (control)',
|
||||
data: { employees: baseData.employees },
|
||||
isValid: true,
|
||||
},
|
||||
// Truncated array (missing last 3 rows)
|
||||
{
|
||||
type: 'truncated' as const,
|
||||
description: 'Array truncated: 3 rows removed from end',
|
||||
data: { employees: baseData.employees.slice(0, -3) },
|
||||
isValid: false, // [N] won't match actual row count in TOON
|
||||
},
|
||||
// Extra rows (3 more than original)
|
||||
{
|
||||
type: 'extra-rows' as const,
|
||||
description: 'Extra rows added beyond declared length',
|
||||
data: {
|
||||
employees: [
|
||||
...baseData.employees,
|
||||
...generateEmployees(3).employees,
|
||||
],
|
||||
},
|
||||
isValid: false, // [N] won't match actual row count in TOON
|
||||
},
|
||||
// Width mismatch (inconsistent field count)
|
||||
{
|
||||
type: 'width-mismatch' as const,
|
||||
description: 'Inconsistent field count (missing salary in row 10)',
|
||||
data: {
|
||||
employees: baseData.employees.map((emp, i) => {
|
||||
if (i === 9) {
|
||||
// Row 10, missing salary field
|
||||
const { salary, ...rest } = emp
|
||||
return rest
|
||||
}
|
||||
return emp
|
||||
}),
|
||||
},
|
||||
isValid: false, // Not all objects have same fields (tabular requirement)
|
||||
},
|
||||
// Missing required fields
|
||||
{
|
||||
type: 'missing-fields' as const,
|
||||
description: 'Missing required fields (no email in multiple rows)',
|
||||
data: {
|
||||
employees: baseData.employees.map((emp, i) => {
|
||||
if (i % 5 === 0) {
|
||||
// Every 5th row, missing email
|
||||
const { email, ...rest } = emp
|
||||
return rest
|
||||
}
|
||||
return emp
|
||||
}),
|
||||
},
|
||||
isValid: false, // Not all objects have same fields (tabular requirement)
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Event logs dataset: Semi-uniform structure
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON with semi-uniform data (~50% flat, ~50% with nested errors).
|
||||
*/
|
||||
const eventLogsDataset: Dataset = {
|
||||
name: 'event-logs',
|
||||
description: 'Semi-uniform event logs',
|
||||
data: generateEventLogs(75),
|
||||
metadata: {
|
||||
supportsCSV: false,
|
||||
structureClass: 'semi-uniform',
|
||||
tabularEligibility: 50, // Top-level logs array is tabular, but ~50% have nested optional error objects
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Nested config dataset: Deeply nested structure
|
||||
*
|
||||
* @remarks
|
||||
* Tests TOON's worst-case scenario with deeply nested configuration.
|
||||
*/
|
||||
const nestedConfigDataset: Dataset = {
|
||||
name: 'nested-config',
|
||||
description: 'Deeply nested configuration',
|
||||
data: generateNestedConfig(),
|
||||
metadata: {
|
||||
supportsCSV: false,
|
||||
structureClass: 'deep',
|
||||
tabularEligibility: 0, // Deeply nested configuration with no tabular arrays
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural validation datasets: Tests ability to detect incomplete, truncated, or corrupted data
|
||||
*
|
||||
* @remarks
|
||||
* These datasets test TOON's structural validation advantages via [N] length declarations
|
||||
* and {fields} headers. CSV is included to demonstrate its lack of structural metadata.
|
||||
*/
|
||||
const structuralValidationDatasets: Dataset[] = generateStructuralValidationFixtures().map((fixture, index) => {
|
||||
const datasetNames = [
|
||||
'structural-validation-control',
|
||||
'structural-validation-truncated',
|
||||
'structural-validation-extra-rows',
|
||||
'structural-validation-width-mismatch',
|
||||
'structural-validation-missing-fields',
|
||||
] as const
|
||||
|
||||
return {
|
||||
name: datasetNames[index]!,
|
||||
description: fixture.description,
|
||||
data: fixture.data,
|
||||
metadata: {
|
||||
supportsCSV: true, // Include CSV to show it can't validate structure
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Datasets for accuracy benchmarks (smaller sizes for faster evaluation)
|
||||
*/
|
||||
export const ACCURACY_DATASETS: Dataset[] = [
|
||||
tabularDataset, // 100 employees
|
||||
nestedDataset, // 50 orders
|
||||
analyticsDataset, // 60 days
|
||||
githubDataset, // 100 repos
|
||||
eventLogsDataset, // 75 logs
|
||||
nestedConfigDataset, // 1 config
|
||||
...structuralValidationDatasets, // 5 validation fixtures
|
||||
]
|
||||
|
||||
/**
|
||||
* Datasets for token efficiency benchmarks (larger sizes to amplify token differences)
|
||||
*/
|
||||
export const TOKEN_EFFICIENCY_DATASETS: Dataset[] = [
|
||||
// Tabular: 2000 employees
|
||||
{
|
||||
name: 'tabular',
|
||||
description: 'Uniform employee records',
|
||||
data: generateEmployees(2000),
|
||||
metadata: {
|
||||
supportsCSV: true,
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100, // All arrays contain uniform objects with primitive values only
|
||||
},
|
||||
},
|
||||
// Nested: 500 orders
|
||||
{
|
||||
name: 'nested',
|
||||
description: 'E-commerce orders with nested structures',
|
||||
data: generateOrders(500),
|
||||
metadata: {
|
||||
supportsCSV: false,
|
||||
structureClass: 'nested',
|
||||
tabularEligibility: 33, // Top-level orders array has nested objects (not tabular), but nested items arrays are tabular
|
||||
},
|
||||
},
|
||||
// Analytics: 365 days
|
||||
{
|
||||
name: 'analytics',
|
||||
description: 'Time-series analytics data',
|
||||
data: generateAnalyticsData(365),
|
||||
metadata: {
|
||||
supportsCSV: true,
|
||||
structureClass: 'uniform',
|
||||
tabularEligibility: 100, // Uniform time-series records with consistent primitive fields
|
||||
},
|
||||
},
|
||||
// GitHub: 100 repos (same as accuracy)
|
||||
githubDataset,
|
||||
// Event logs: 2000 logs
|
||||
{
|
||||
name: 'event-logs',
|
||||
description: 'Semi-uniform event logs',
|
||||
data: generateEventLogs(2000),
|
||||
metadata: {
|
||||
supportsCSV: false,
|
||||
structureClass: 'semi-uniform',
|
||||
tabularEligibility: 50, // Top-level logs array is tabular, but ~50% have nested optional error objects
|
||||
},
|
||||
},
|
||||
// Nested config: 1 config (same as accuracy)
|
||||
nestedConfigDataset,
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { LanguageModelV3 } from '@ai-sdk/provider'
|
||||
import type { EvaluationResult, Question } from './types.ts'
|
||||
import { anthropic } from '@ai-sdk/anthropic'
|
||||
import { google } from '@ai-sdk/google'
|
||||
import { openai } from '@ai-sdk/openai'
|
||||
import { xai } from '@ai-sdk/xai'
|
||||
import { generateText } from 'ai'
|
||||
import { compareAnswers } from './normalize.ts'
|
||||
|
||||
/**
|
||||
* Models used for evaluation
|
||||
*/
|
||||
export const models: LanguageModelV3[] = [
|
||||
anthropic('claude-haiku-4-5-20251001'),
|
||||
google('gemini-3-flash-preview'),
|
||||
openai('gpt-5-nano'),
|
||||
xai('grok-4-1-fast-non-reasoning'),
|
||||
]
|
||||
|
||||
/**
|
||||
* Format primers
|
||||
*
|
||||
* @remarks
|
||||
* Neutral descriptions to help models parse each format.
|
||||
*/
|
||||
export const PRIMERS: Record<string, string> = {
|
||||
'toon': 'TOON: Indentation-based. Arrays declare length and fields (e.g., items[N]{f1,f2}:). Rows use single delimiter. Values may be quoted.',
|
||||
'json-pretty': 'JSON: Strict JSON objects/arrays with repeated keys per row.',
|
||||
'json-compact': 'JSON (compact): Strict JSON without extra whitespace.',
|
||||
'yaml': 'YAML: Indentation-based key/value and lists (- items).',
|
||||
'xml': 'XML: Tag-based tree structure with nested elements.',
|
||||
'csv': 'CSV: Header row, comma-separated values. First row contains field names.',
|
||||
}
|
||||
|
||||
/**
|
||||
* Code fence language tags for proper syntax highlighting
|
||||
*/
|
||||
export const FENCE: Record<string, string> = {
|
||||
'toon': 'toon',
|
||||
'json-pretty': 'json',
|
||||
'json-compact': 'json',
|
||||
'yaml': 'yaml',
|
||||
'xml': 'xml',
|
||||
'csv': 'csv',
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate a single question with a specific format and model
|
||||
*/
|
||||
export async function evaluateQuestion(
|
||||
{
|
||||
question,
|
||||
formatName,
|
||||
formattedData,
|
||||
model,
|
||||
}:
|
||||
{
|
||||
question: Question
|
||||
formatName: string
|
||||
formattedData: string
|
||||
model: LanguageModelV3
|
||||
},
|
||||
): Promise<EvaluationResult> {
|
||||
const primer = PRIMERS[formatName] ?? ''
|
||||
const fence = FENCE[formatName] ?? ''
|
||||
|
||||
const prompt = `
|
||||
${primer}
|
||||
|
||||
Given the following data in ${formatName} format:
|
||||
|
||||
\`\`\`${fence}
|
||||
${formattedData}
|
||||
\`\`\`
|
||||
|
||||
Question: ${question.prompt}
|
||||
|
||||
Answer format requirements:
|
||||
- Provide only the value itself, no explanation
|
||||
- For numbers: output digits only (no commas, currency symbols, or units)
|
||||
- For dates/field names: use the exact string from the data
|
||||
- For lists: output comma-separated values with no spaces
|
||||
|
||||
Answer:
|
||||
`.trim()
|
||||
|
||||
const startTime = performance.now()
|
||||
const { text, usage } = await generateText({ model, prompt })
|
||||
|
||||
const actual = text.trim()
|
||||
const latencyMs = performance.now() - startTime
|
||||
|
||||
const comparisonResult = compareAnswers(
|
||||
actual,
|
||||
question.groundTruth,
|
||||
question.answerType ?? 'string',
|
||||
question.normalizationOptions,
|
||||
)
|
||||
const isCorrect = comparisonResult.match
|
||||
|
||||
return {
|
||||
questionId: question.id,
|
||||
format: formatName,
|
||||
model: model.modelId,
|
||||
expected: question.groundTruth,
|
||||
actual,
|
||||
isCorrect,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
latencyMs,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { Dataset } from './types.ts'
|
||||
import { stringify as stringifyCSV } from 'csv-stringify/sync'
|
||||
import { XMLBuilder } from 'fast-xml-parser'
|
||||
import { stringify as stringifyYAML } from 'yaml'
|
||||
import { encode as encodeToon } from '../../packages/toon/src/index.ts'
|
||||
|
||||
/**
|
||||
* Format converters registry
|
||||
*
|
||||
* @remarks
|
||||
* All formatters attempt to preserve semantic equivalence with the source data,
|
||||
* meaning the converted data should represent the same information. However,
|
||||
* CSV has inherent limitations with nested structures (see `toCSV` docs).
|
||||
*/
|
||||
export const formatters: Record<string, (data: unknown) => string> = {
|
||||
'json-pretty': data => JSON.stringify(data, undefined, 2),
|
||||
'json-compact': data => JSON.stringify(data),
|
||||
'toon': data => encodeToon(data),
|
||||
'csv': data => toCSV(data),
|
||||
'xml': data => toXML(data),
|
||||
'yaml': data => stringifyYAML(data),
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert data to CSV format
|
||||
*
|
||||
* @remarks
|
||||
* Limitations: CSV is designed for flat tabular data only.
|
||||
*
|
||||
* This formatter:
|
||||
* - Only handles top-level objects with arrays of flat objects
|
||||
* - Cannot properly represent deeply nested structures (nested arrays/objects within rows)
|
||||
* - Loses nested structure information during conversion
|
||||
* - May produce misleading results for datasets with complex nesting (e.g., e-commerce orders with nested items)
|
||||
*
|
||||
* For datasets with nested structures, CSV comparisons may not be fair or representative
|
||||
* of how CSV would typically be used in practice.
|
||||
*/
|
||||
function toCSV(data: unknown): string {
|
||||
const sections: string[] = []
|
||||
|
||||
// Handle top-level object with arrays
|
||||
if (typeof data === 'object' && data !== null && !Array.isArray(data)) {
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
if (Array.isArray(value) && value.length > 0) {
|
||||
sections.push(`# ${key}`)
|
||||
sections.push(stringifyCSV(value, { header: true }))
|
||||
}
|
||||
}
|
||||
return sections.join('\n').trim()
|
||||
}
|
||||
|
||||
// Root-level array
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
return stringifyCSV(data, { header: true }).trim()
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert data to XML format
|
||||
*
|
||||
* @remarks
|
||||
* Uses `fast-xml-parser` to generate well-formatted XML with:
|
||||
* - 2-space indentation for readability
|
||||
* - Empty nodes suppressed
|
||||
* - Proper escaping of special characters
|
||||
*/
|
||||
function toXML(data: unknown): string {
|
||||
const builder = new XMLBuilder({
|
||||
format: true,
|
||||
indentBy: ' ',
|
||||
suppressEmptyNode: true,
|
||||
})
|
||||
|
||||
return builder.build(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a dataset supports CSV format
|
||||
*
|
||||
* @remarks
|
||||
* CSV is only suitable for flat tabular data. Datasets with nested structures
|
||||
* should not be compared using CSV as it cannot properly represent the data.
|
||||
*/
|
||||
export function supportsCSV(dataset: Dataset): boolean {
|
||||
return dataset.metadata.supportsCSV
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* Type of expected answer for deterministic comparison
|
||||
*/
|
||||
export type AnswerType
|
||||
= | 'integer'
|
||||
| 'number'
|
||||
| 'boolean'
|
||||
| 'date'
|
||||
| 'string'
|
||||
| 'csv-list-ordered'
|
||||
| 'csv-list-unordered'
|
||||
|
||||
/**
|
||||
* Options for answer normalization and comparison
|
||||
*/
|
||||
export interface NormalizationOptions {
|
||||
/**
|
||||
* Tolerance for floating-point number comparison (e.g., 1e-6).
|
||||
* @default 1e-6
|
||||
*/
|
||||
tolerance?: number
|
||||
|
||||
/**
|
||||
* Whether string comparison should be case-sensitive.
|
||||
* @default false
|
||||
*/
|
||||
caseSensitive?: boolean
|
||||
|
||||
/**
|
||||
* Allow currency symbols ($, €, etc.) in number extraction.
|
||||
* @default true
|
||||
*/
|
||||
allowCurrency?: boolean
|
||||
|
||||
/**
|
||||
* Allow percent signs (%) in number extraction (will divide by 100).
|
||||
* @default true
|
||||
*/
|
||||
allowPercent?: boolean
|
||||
|
||||
/**
|
||||
* Number of decimal places to round to for number comparison.
|
||||
* If specified, overrides tolerance-based comparison.
|
||||
*/
|
||||
decimalPlaces?: number
|
||||
}
|
||||
|
||||
interface NormalizedResult {
|
||||
success: boolean
|
||||
value?: unknown
|
||||
error?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Default normalization options
|
||||
*/
|
||||
const DEFAULT_OPTIONS: Required<NormalizationOptions> = {
|
||||
tolerance: 1e-6,
|
||||
caseSensitive: false,
|
||||
allowCurrency: true,
|
||||
allowPercent: true,
|
||||
decimalPlaces: undefined!,
|
||||
}
|
||||
|
||||
// Regex pattern constants
|
||||
const INTEGER_PATTERN_WITH_CURRENCY = /[$€£¥]?\s*-?\d[\d,]*/
|
||||
const INTEGER_PATTERN = /-?\d[\d,]*/
|
||||
const NUMBER_PATTERN_WITH_CURRENCY = /[$€£¥]?\s*-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?%?/i
|
||||
const NUMBER_PATTERN = /-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?%?/i
|
||||
const WRAPPING_QUOTES_PATTERN = /^["']|["']$/g
|
||||
const CODE_FENCE_PATTERN = /^```[\s\S]*?```$/g
|
||||
const LANGUAGE_IDENTIFIER_PATTERN = /^\w+\n/
|
||||
const CURRENCY_AND_FORMATTING_CHARS = /[$€£¥,\s]/g
|
||||
const NUMBER_CLEANUP_CHARS = /[$€£¥,%\s]/g
|
||||
|
||||
// Boolean value constants
|
||||
const TRUE_VALUES = new Set(['true', 'yes', 'y', '1'])
|
||||
const FALSE_VALUES = new Set(['false', 'no', 'n', '0'])
|
||||
|
||||
// Numeric constants
|
||||
const PERCENTAGE_DIVISOR = 100
|
||||
const DECIMAL_BASE = 10
|
||||
const MONTH_OFFSET = 1 // JavaScript months are 0-indexed
|
||||
const DATE_COMPONENT_WIDTH = 2
|
||||
const DATE_PAD_CHAR = '0'
|
||||
|
||||
// String constants
|
||||
const CSV_DELIMITER = ','
|
||||
|
||||
/**
|
||||
* Strip wrapping quotes from a string
|
||||
*/
|
||||
function stripWrappingQuotes(text: string): string {
|
||||
return text.trim().replace(WRAPPING_QUOTES_PATTERN, '')
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and normalize an integer from a string
|
||||
*
|
||||
* @remarks
|
||||
* Handles: "42", "1,234", "$5,678", " -99 ", "The answer is 42."
|
||||
*/
|
||||
function normalizeInteger(text: string, options: Required<NormalizationOptions>): NormalizedResult {
|
||||
// Strip common formatting, extract first integer-like token
|
||||
const pattern = options.allowCurrency
|
||||
? INTEGER_PATTERN_WITH_CURRENCY
|
||||
: INTEGER_PATTERN
|
||||
|
||||
const match = text.match(pattern)
|
||||
if (!match)
|
||||
return { success: false, error: `No integer found in: "${text}"` }
|
||||
|
||||
// Remove currency symbols, spaces, and thousand separators
|
||||
const normalizedValue = match[0].replace(CURRENCY_AND_FORMATTING_CHARS, '')
|
||||
const parsedNumber = Number.parseInt(normalizedValue, DECIMAL_BASE)
|
||||
|
||||
if (Number.isNaN(parsedNumber))
|
||||
return { success: false, error: `Failed to parse integer: "${match[0]}"` }
|
||||
|
||||
return { success: true, value: parsedNumber }
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and normalize a floating-point number from a string
|
||||
*
|
||||
* @remarks
|
||||
* Handles: "3.14", "1,234.56", "$5,678.90", "42%", "1.5e-3", "Price: $99.99"
|
||||
*/
|
||||
function normalizeNumber(text: string, options: Required<NormalizationOptions>): NormalizedResult {
|
||||
// Extract first number-like token (supports scientific notation)
|
||||
const pattern = options.allowCurrency
|
||||
? NUMBER_PATTERN_WITH_CURRENCY
|
||||
: NUMBER_PATTERN
|
||||
|
||||
const match = text.match(pattern)
|
||||
if (!match)
|
||||
return { success: false, error: `No number found in: "${text}"` }
|
||||
|
||||
const token = match[0]
|
||||
const hasPercentSign = options.allowPercent && token.endsWith('%')
|
||||
|
||||
// Remove currency, commas, spaces, and percent sign
|
||||
const normalizedToken = token.replace(NUMBER_CLEANUP_CHARS, '')
|
||||
let parsedNumber = Number.parseFloat(normalizedToken)
|
||||
|
||||
if (Number.isNaN(parsedNumber))
|
||||
return { success: false, error: `Failed to parse number: "${token}"` }
|
||||
|
||||
// Convert percentage to decimal if present
|
||||
if (hasPercentSign)
|
||||
parsedNumber = parsedNumber / PERCENTAGE_DIVISOR
|
||||
|
||||
// Round to specified decimal places if requested
|
||||
if (options.decimalPlaces !== undefined) {
|
||||
const factor = DECIMAL_BASE ** options.decimalPlaces
|
||||
parsedNumber = Math.round(parsedNumber * factor) / factor
|
||||
}
|
||||
|
||||
return { success: true, value: parsedNumber }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a boolean/yes-no answer
|
||||
*
|
||||
* @remarks
|
||||
* Handles: "true", "false", "yes", "no", "y", "n", "1", "0" (case-insensitive)
|
||||
*/
|
||||
function normalizeBoolean(text: string): NormalizedResult {
|
||||
const normalizedValue = text.trim().toLowerCase()
|
||||
|
||||
if (TRUE_VALUES.has(normalizedValue))
|
||||
return { success: true, value: true }
|
||||
|
||||
if (FALSE_VALUES.has(normalizedValue))
|
||||
return { success: true, value: false }
|
||||
|
||||
return { success: false, error: `Not a boolean: "${text}"` }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a date string to YYYY-MM-DD format
|
||||
*
|
||||
* @remarks
|
||||
* Handles: ISO dates, "Nov 1, 2025", "2025-11-01", RFC 2822, etc.
|
||||
*/
|
||||
function normalizeDate(text: string): NormalizedResult {
|
||||
const cleaned = stripWrappingQuotes(text)
|
||||
|
||||
// Try parsing as date
|
||||
const parsedDate = new Date(cleaned)
|
||||
if (Number.isNaN(parsedDate.getTime()))
|
||||
return { success: false, error: `Invalid date: "${text}"` }
|
||||
|
||||
// Normalize to YYYY-MM-DD (UTC)
|
||||
const year = parsedDate.getUTCFullYear()
|
||||
const monthPadded = String(parsedDate.getUTCMonth() + MONTH_OFFSET).padStart(DATE_COMPONENT_WIDTH, DATE_PAD_CHAR)
|
||||
const dayPadded = String(parsedDate.getUTCDate()).padStart(DATE_COMPONENT_WIDTH, DATE_PAD_CHAR)
|
||||
const normalized = `${year}-${monthPadded}-${dayPadded}`
|
||||
|
||||
return { success: true, value: normalized }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a string (trim, optionally case-insensitive)
|
||||
*
|
||||
* @remarks
|
||||
* Handles wrapping quotes and code fences.
|
||||
*/
|
||||
function normalizeString(text: string, options: Required<NormalizationOptions>): NormalizedResult {
|
||||
let trimmedText = text.trim()
|
||||
|
||||
// Strip wrapping quotes
|
||||
trimmedText = trimmedText.replace(WRAPPING_QUOTES_PATTERN, '')
|
||||
|
||||
// Strip code fences (```...```)
|
||||
trimmedText = trimmedText.replace(CODE_FENCE_PATTERN, (match) => {
|
||||
const inner = match.slice(3, -3).trim()
|
||||
// Remove language identifier if present (e.g., ```json)
|
||||
return inner.replace(LANGUAGE_IDENTIFIER_PATTERN, '')
|
||||
})
|
||||
|
||||
trimmedText = trimmedText.trim()
|
||||
|
||||
const value = options.caseSensitive ? trimmedText : trimmedText.toLowerCase()
|
||||
return { success: true, value }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a comma-separated list (ordered)
|
||||
*
|
||||
* @remarks
|
||||
* Handles: "a,b,c", "a, b, c", " a , b , c "
|
||||
*/
|
||||
function normalizeCsvListOrdered(text: string, options: Required<NormalizationOptions>): NormalizedResult {
|
||||
const strippedText = stripWrappingQuotes(text)
|
||||
const items = strippedText
|
||||
.split(CSV_DELIMITER)
|
||||
.map(item => item.trim())
|
||||
.filter(item => item.length > 0)
|
||||
|
||||
const normalizedItems = items.map(item =>
|
||||
options.caseSensitive ? item : item.toLowerCase(),
|
||||
)
|
||||
|
||||
return { success: true, value: normalizedItems }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a comma-separated list (unordered, compare as sets)
|
||||
*
|
||||
* @remarks
|
||||
* Handles: "c,a,b" equals "a,b,c"
|
||||
*/
|
||||
function normalizeCsvListUnordered(text: string, options: Required<NormalizationOptions>): NormalizedResult {
|
||||
const result = normalizeCsvListOrdered(text, options)
|
||||
if (!result.success)
|
||||
return result
|
||||
|
||||
// Type guard: ensure result.value is an array
|
||||
if (!Array.isArray(result.value))
|
||||
return { success: false, error: 'Expected array result from normalizeCsvListOrdered' }
|
||||
|
||||
// Sort for deterministic comparison
|
||||
const sorted = [...result.value].sort()
|
||||
return { success: true, value: sorted }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a value based on its expected kind
|
||||
*/
|
||||
export function normalizeAnswer(
|
||||
text: string,
|
||||
kind: AnswerType,
|
||||
options: Partial<NormalizationOptions> = {},
|
||||
): NormalizedResult {
|
||||
const resolvedOptions: Required<NormalizationOptions> = { ...DEFAULT_OPTIONS, ...options }
|
||||
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
return normalizeInteger(text, resolvedOptions)
|
||||
case 'number':
|
||||
return normalizeNumber(text, resolvedOptions)
|
||||
case 'boolean':
|
||||
return normalizeBoolean(text)
|
||||
case 'date':
|
||||
return normalizeDate(text)
|
||||
case 'string':
|
||||
return normalizeString(text, resolvedOptions)
|
||||
case 'csv-list-ordered':
|
||||
return normalizeCsvListOrdered(text, resolvedOptions)
|
||||
case 'csv-list-unordered':
|
||||
return normalizeCsvListUnordered(text, resolvedOptions)
|
||||
default:
|
||||
return { success: false, error: `Unknown answer kind: ${kind}` }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two normalized values based on answer kind
|
||||
*/
|
||||
function compareValues(
|
||||
actual: unknown,
|
||||
expected: unknown,
|
||||
kind: AnswerType,
|
||||
options: Required<NormalizationOptions>,
|
||||
): boolean {
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
case 'boolean':
|
||||
case 'date':
|
||||
case 'string':
|
||||
return actual === expected
|
||||
|
||||
case 'number':
|
||||
if (typeof actual !== 'number' || typeof expected !== 'number')
|
||||
return false
|
||||
|
||||
if (options.decimalPlaces !== undefined) {
|
||||
// Already rounded during normalization
|
||||
return actual === expected
|
||||
}
|
||||
return Math.abs(actual - expected) <= options.tolerance
|
||||
|
||||
case 'csv-list-ordered':
|
||||
if (!Array.isArray(actual) || !Array.isArray(expected))
|
||||
return false
|
||||
if (actual.length !== expected.length)
|
||||
return false
|
||||
return actual.every((item, i) => item === expected[i])
|
||||
|
||||
case 'csv-list-unordered':
|
||||
if (!Array.isArray(actual) || !Array.isArray(expected))
|
||||
return false
|
||||
if (actual.length !== expected.length)
|
||||
return false
|
||||
// Already sorted during normalization
|
||||
return actual.every((item, i) => item === expected[i])
|
||||
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare actual and expected answers with deterministic, type-aware normalization
|
||||
*
|
||||
* @remarks
|
||||
* Returns true if answers match within the specified tolerance/rules.
|
||||
*/
|
||||
export function compareAnswers(
|
||||
actual: string,
|
||||
expected: string,
|
||||
kind: AnswerType,
|
||||
options: Partial<NormalizationOptions> = {},
|
||||
): { match: boolean, details?: string } {
|
||||
const resolvedOptions: Required<NormalizationOptions> = { ...DEFAULT_OPTIONS, ...options }
|
||||
|
||||
// Normalize both answers
|
||||
const actualResult = normalizeAnswer(actual, kind, resolvedOptions)
|
||||
const expectedResult = normalizeAnswer(expected, kind, resolvedOptions)
|
||||
|
||||
// If either normalization failed, return false with details
|
||||
if (!actualResult.success) {
|
||||
return {
|
||||
match: false,
|
||||
details: `Failed to normalize actual answer: ${actualResult.error}`,
|
||||
}
|
||||
}
|
||||
|
||||
if (!expectedResult.success) {
|
||||
return {
|
||||
match: false,
|
||||
details: `Failed to normalize expected answer: ${expectedResult.error}`,
|
||||
}
|
||||
}
|
||||
|
||||
// Compare normalized values
|
||||
const match = compareValues(actualResult.value, expectedResult.value, kind, resolvedOptions)
|
||||
|
||||
return {
|
||||
match,
|
||||
details: match
|
||||
? undefined
|
||||
: `Mismatch: actual="${actualResult.value}" vs expected="${expectedResult.value}"`,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
import type { AnalyticsMetric } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
|
||||
import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate analytics (website metrics) questions
|
||||
*/
|
||||
export function generateAnalyticsQuestions(metrics: AnalyticsMetric[], getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Field retrieval: date-based metrics
|
||||
const metricFieldGenerators: Array<(metric: AnalyticsMetric, getId: () => string) => Question> = [
|
||||
(metric, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What are the views for ${metric.date}?`)
|
||||
.groundTruth(String(metric.views))
|
||||
.type('field-retrieval')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(metric, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the revenue for ${metric.date}?`)
|
||||
.groundTruth(String(metric.revenue))
|
||||
.type('field-retrieval')
|
||||
.dataset('analytics')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
(metric, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the bounce rate for ${metric.date}?`)
|
||||
.groundTruth(String(metric.bounceRate))
|
||||
.type('field-retrieval')
|
||||
.dataset('analytics')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
(metric, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many conversions were there on ${metric.date}?`)
|
||||
.groundTruth(String(metric.conversions))
|
||||
.type('field-retrieval')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
]
|
||||
|
||||
questions.push(...rotateQuestions(
|
||||
metrics,
|
||||
metricFieldGenerators,
|
||||
QUESTION_LIMITS.analytics.fieldRetrievalDates,
|
||||
SAMPLE_STRIDES.ANALYTICS_FIELD,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Aggregation: basic statistics
|
||||
const totalDays = metrics.length
|
||||
const totalViews = metrics.reduce((sum, m) => sum + m.views, 0)
|
||||
const totalConversions = metrics.reduce((sum, m) => sum + m.conversions, 0)
|
||||
const totalRevenue = metrics.reduce((sum, m) => sum + m.revenue, 0)
|
||||
const avgBounceRate = metrics.reduce((sum, m) => sum + m.bounceRate, 0) / metrics.length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many days of data are in the dataset?')
|
||||
.groundTruth(String(totalDays))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of views across all dates?')
|
||||
.groundTruth(String(totalViews))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of conversions across all dates?')
|
||||
.groundTruth(String(totalConversions))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total revenue across all dates?')
|
||||
.groundTruth(String(totalRevenue.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the average bounce rate?')
|
||||
.groundTruth(String(avgBounceRate.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: high views/conversions
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.views) {
|
||||
const count = metrics.filter(m => m.views > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had more than ${threshold} views?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.conversions) {
|
||||
const count = metrics.filter(m => m.conversions > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had more than ${threshold} conversions?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: multi-condition (views AND revenue)
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.viewsForFiltering) {
|
||||
const count = metrics.filter(
|
||||
m => m.views > threshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForFiltering,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had more than ${threshold} views and more than ${QUESTION_THRESHOLDS.analytics.conversionsForFiltering} conversions?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: revenue thresholds
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.revenueThresholds) {
|
||||
const count = metrics.filter(
|
||||
m => m.revenue > threshold && m.views > QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had revenue greater than ${threshold} with views above ${QUESTION_THRESHOLDS.analytics.viewsThresholdForRevenue}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: clicks and conversions
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.clicksForFiltering) {
|
||||
const count = metrics.filter(
|
||||
m => m.clicks > threshold && m.conversions > QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had more than ${threshold} clicks and more than ${QUESTION_THRESHOLDS.analytics.conversionsForClickFiltering} conversions?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: revenue and bounce rate
|
||||
for (const threshold of QUESTION_THRESHOLDS.analytics.revenueForBounceRate) {
|
||||
const count = metrics.filter(
|
||||
m => m.revenue > threshold && m.bounceRate < QUESTION_THRESHOLDS.analytics.bounceRateThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many days had revenue greater than ${threshold} with bounce rate below ${QUESTION_THRESHOLDS.analytics.bounceRateThreshold}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import type { EventLog } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS } from '../constants.ts'
|
||||
import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate event log questions
|
||||
*/
|
||||
export function generateEventLogsQuestions(logs: EventLog[], getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Field retrieval: log metadata
|
||||
const logFieldGenerators: Array<(log: EventLog, getId: () => string) => Question> = [
|
||||
(log, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the level of the log at ${log.timestamp}?`)
|
||||
.groundTruth(log.level)
|
||||
.type('field-retrieval')
|
||||
.dataset('event-logs')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(log, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the endpoint for the log at ${log.timestamp}?`)
|
||||
.groundTruth(log.endpoint)
|
||||
.type('field-retrieval')
|
||||
.dataset('event-logs')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(log, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the status code for the log at ${log.timestamp}?`)
|
||||
.groundTruth(String(log.statusCode))
|
||||
.type('field-retrieval')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(log, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the response time for the log at ${log.timestamp}?`)
|
||||
.groundTruth(String(log.responseTime))
|
||||
.type('field-retrieval')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
]
|
||||
|
||||
questions.push(...rotateQuestions(
|
||||
logs,
|
||||
logFieldGenerators,
|
||||
QUESTION_LIMITS.eventLogs.fieldRetrieval,
|
||||
SAMPLE_STRIDES.EVENT_LOG_FIELD,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Aggregation: basic statistics
|
||||
const totalLogs = logs.length
|
||||
const avgResponseTime = logs.reduce((sum, l) => sum + l.responseTime, 0) / logs.length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many log entries are in the dataset?')
|
||||
.groundTruth(String(totalLogs))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the average response time across all logs?')
|
||||
.groundTruth(String(avgResponseTime.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: by level
|
||||
const levels = [...new Set(logs.map(l => l.level))]
|
||||
for (const level of levels) {
|
||||
const count = logs.filter(l => l.level === level).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many log entries have level "${level}"?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: by endpoint
|
||||
const endpoints = [...new Set(logs.map(l => l.endpoint))]
|
||||
for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.aggregationEndpoints)) {
|
||||
const count = logs.filter(l => l.endpoint === endpoint).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many log entries are for endpoint "${endpoint}"?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: by status code range
|
||||
const errorCount = logs.filter(l => l.statusCode >= 400).length
|
||||
const successCount = logs.filter(l => l.statusCode >= 200 && l.statusCode < 300).length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many log entries have a status code indicating an error (>= 400)?')
|
||||
.groundTruth(String(errorCount))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many log entries have a successful status code (200-299)?')
|
||||
.groundTruth(String(successCount))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: retryable errors
|
||||
const retryableErrorCount = logs.filter(l => l.error?.retryable === true).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many log entries have a retryable error?')
|
||||
.groundTruth(String(retryableErrorCount))
|
||||
.type('aggregation')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Filtering: multi-condition (level AND status)
|
||||
for (const level of levels.slice(0, QUESTION_LIMITS.eventLogs.filteringLevelAndStatus)) {
|
||||
// Skip `info` level as it never has status >= 400 by design
|
||||
if (level === 'info')
|
||||
continue
|
||||
|
||||
const count = logs.filter(l => l.level === level && l.statusCode >= 400).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many log entries have level "${level}" and status code >= 400?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: endpoint AND status
|
||||
for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.filteringEndpointAndStatus)) {
|
||||
const count = logs.filter(l => l.endpoint === endpoint && l.statusCode >= 500).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many log entries are for endpoint "${endpoint}" with status code >= 500?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: endpoint AND retryable error
|
||||
for (const endpoint of endpoints.slice(0, QUESTION_LIMITS.eventLogs.filteringEndpointRetryable)) {
|
||||
const count = logs.filter(l => l.endpoint === endpoint && l.error?.retryable === true).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many log entries for endpoint "${endpoint}" have a retryable error?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { Repository } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
|
||||
import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate GitHub repository questions
|
||||
*/
|
||||
export function generateGithubQuestions(repos: Repository[], getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Field retrieval: repository metadata
|
||||
const repoFieldGenerators: Array<(repo: Repository, getId: () => string) => Question> = [
|
||||
(repo, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many stars does ${repo.repo} have?`)
|
||||
.groundTruth(String(repo.stars))
|
||||
.type('field-retrieval')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(repo, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many forks does ${repo.repo} have?`)
|
||||
.groundTruth(String(repo.forks))
|
||||
.type('field-retrieval')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(repo, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many watchers does ${repo.repo} have?`)
|
||||
.groundTruth(String(repo.watchers))
|
||||
.type('field-retrieval')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(repo, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the main branch of ${repo.repo}?`)
|
||||
.groundTruth(repo.defaultBranch)
|
||||
.type('field-retrieval')
|
||||
.dataset('github')
|
||||
.answerType('string')
|
||||
.normalize({ caseSensitive: true })
|
||||
.build(),
|
||||
]
|
||||
|
||||
questions.push(...rotateQuestions(
|
||||
repos,
|
||||
repoFieldGenerators,
|
||||
QUESTION_LIMITS.github.fieldRetrievalRepos,
|
||||
SAMPLE_STRIDES.REPO_FIELD,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Aggregation: basic statistics
|
||||
const totalRepos = repos.length
|
||||
const totalStars = repos.reduce((sum, r) => sum + r.stars, 0)
|
||||
const totalForks = repos.reduce((sum, r) => sum + r.forks, 0)
|
||||
const avgStars = totalStars / totalRepos
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many repositories are in the dataset?')
|
||||
.groundTruth(String(totalRepos))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of stars across all repositories?')
|
||||
.groundTruth(String(totalStars))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of forks across all repositories?')
|
||||
.groundTruth(String(totalForks))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the average number of stars per repository?')
|
||||
.groundTruth(String(Math.round(avgStars)))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: by default branch
|
||||
const branches = [...new Set(repos.map(r => r.defaultBranch))]
|
||||
for (const branch of branches.slice(0, QUESTION_LIMITS.github.aggregationBranches)) {
|
||||
const count = repos.filter(r => r.defaultBranch === branch).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories use "${branch}" as their default branch?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: high star counts
|
||||
for (const threshold of QUESTION_THRESHOLDS.github.stars) {
|
||||
const count = repos.filter(r => r.stars > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories have more than ${threshold} stars?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: high fork counts
|
||||
for (const threshold of QUESTION_THRESHOLDS.github.forks) {
|
||||
const count = repos.filter(r => r.forks > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories have more than ${threshold} forks?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: high watcher counts
|
||||
for (const threshold of QUESTION_THRESHOLDS.github.watchers) {
|
||||
const count = repos.filter(r => r.watchers > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories have more than ${threshold} watchers?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: multi-condition (stars AND forks)
|
||||
for (const combo of QUESTION_THRESHOLDS.github.starForkCombinations.slice(0, QUESTION_LIMITS.github.filteringStarsAndForks)) {
|
||||
const count = repos.filter(
|
||||
r => r.stars > combo.stars && r.forks > combo.forks,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories have more than ${combo.stars} stars and more than ${combo.forks} forks?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: stars AND watchers
|
||||
for (const combo of QUESTION_THRESHOLDS.github.starWatcherCombinations) {
|
||||
const count = repos.filter(
|
||||
r => r.stars > combo.stars && r.watchers > combo.watchers,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many repositories have more than ${combo.stars} stars and more than ${combo.watchers} watchers?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { AnalyticsMetric, Employee, EventLog, NestedConfig, Order, Repository } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { ACCURACY_DATASETS } from '../datasets.ts'
|
||||
import { generateAnalyticsQuestions } from './analytics.ts'
|
||||
import { generateEventLogsQuestions } from './event-logs.ts'
|
||||
import { generateGithubQuestions } from './github.ts'
|
||||
import { generateNestedConfigQuestions } from './nested-config.ts'
|
||||
import { generateNestedQuestions } from './nested.ts'
|
||||
import { generateStructuralValidationQuestions } from './structural-validation.ts'
|
||||
import { generateStructureQuestions } from './structure.ts'
|
||||
import { generateTabularQuestions } from './tabular.ts'
|
||||
import { createIdGenerator } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate questions from all datasets
|
||||
*
|
||||
* @remarks
|
||||
* - Field Retrieval: Direct field access with no computation
|
||||
* Examples: "What is X's salary?", "What is the status of order Y?"
|
||||
* - Aggregation: Counts, sums, averages, min/max operations (including single-condition filters)
|
||||
* Examples: "How many X?", "What is the total/average?", "How many X > threshold?"
|
||||
* - Filtering: Multi-condition queries requiring complex logical operations
|
||||
* Examples: "How many X WHERE condition1 AND condition2?"
|
||||
* - Structure Awareness: Tests format-native structural affordances (TOON's [N] and {fields}, CSV's header)
|
||||
* Examples: "How many records?", "List the field names", "What is the last record's field?"
|
||||
*/
|
||||
export function generateQuestions(): Question[] {
|
||||
const questions: Question[] = []
|
||||
const idGen = createIdGenerator()
|
||||
const getId = () => idGen.next().value
|
||||
|
||||
// Get datasets with proper typing
|
||||
const tabular = (ACCURACY_DATASETS.find(d => d.name === 'tabular')?.data.employees as Employee[]) ?? []
|
||||
const nested = (ACCURACY_DATASETS.find(d => d.name === 'nested')?.data.orders as Order[]) ?? []
|
||||
const analytics = (ACCURACY_DATASETS.find(d => d.name === 'analytics')?.data.metrics as AnalyticsMetric[]) ?? []
|
||||
const github = (ACCURACY_DATASETS.find(d => d.name === 'github')?.data.repositories as Repository[]) ?? []
|
||||
const eventLogs = (ACCURACY_DATASETS.find(d => d.name === 'event-logs')?.data.logs as EventLog[]) ?? []
|
||||
const nestedConfig = ACCURACY_DATASETS.find(d => d.name === 'nested-config')?.data as NestedConfig | undefined
|
||||
|
||||
// Generate questions for each dataset
|
||||
questions.push(...generateTabularQuestions(tabular, getId))
|
||||
questions.push(...generateNestedQuestions(nested, getId))
|
||||
questions.push(...generateAnalyticsQuestions(analytics, getId))
|
||||
questions.push(...generateGithubQuestions(github, getId))
|
||||
questions.push(...generateEventLogsQuestions(eventLogs, getId))
|
||||
questions.push(...generateNestedConfigQuestions(nestedConfig, getId))
|
||||
|
||||
// Generate structure-awareness questions (tests format-native affordances)
|
||||
questions.push(...generateStructureQuestions(tabular, nested, analytics, github, eventLogs, getId))
|
||||
|
||||
// Generate structural-validation questions (tests ability to detect corrupted data)
|
||||
questions.push(...generateStructuralValidationQuestions(getId))
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import type { NestedConfig } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS } from '../constants.ts'
|
||||
import { QuestionBuilder } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate nested configuration questions
|
||||
*/
|
||||
export function generateNestedConfigQuestions(config: NestedConfig | undefined, getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
if (!config)
|
||||
return questions
|
||||
|
||||
// Field retrieval: top-level config values
|
||||
const fieldRetrievalQuestions = [
|
||||
{
|
||||
prompt: 'What is the environment in the configuration?',
|
||||
groundTruth: config.environment,
|
||||
answerType: 'string' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the database host?',
|
||||
groundTruth: config.database.host,
|
||||
answerType: 'string' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the database port?',
|
||||
groundTruth: String(config.database.port),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the maximum connection pool size?',
|
||||
groundTruth: String(config.database.pool.max),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the session duration?',
|
||||
groundTruth: String(config.authentication.session.duration),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the minimum connection pool size?',
|
||||
groundTruth: String(config.database.pool.min),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the connection pool idle timeout?',
|
||||
groundTruth: String(config.database.pool.idleTimeout),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the database name?',
|
||||
groundTruth: config.database.name,
|
||||
answerType: 'string' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the session refresh threshold?',
|
||||
groundTruth: String(config.authentication.session.refreshThreshold),
|
||||
answerType: 'integer' as const,
|
||||
},
|
||||
{
|
||||
prompt: 'What is the version in the configuration?',
|
||||
groundTruth: config.version,
|
||||
answerType: 'string' as const,
|
||||
},
|
||||
]
|
||||
|
||||
for (const q of fieldRetrievalQuestions.slice(0, QUESTION_LIMITS.nestedConfig.fieldRetrieval)) {
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(q.prompt)
|
||||
.groundTruth(q.groundTruth)
|
||||
.type('field-retrieval')
|
||||
.dataset('nested-config')
|
||||
.answerType(q.answerType)
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: counts of nested structures
|
||||
const roleCount = Object.keys(config.permissions.roles).length
|
||||
const groupCount = Object.keys(config.permissions.groups).length
|
||||
const providerCount = config.authentication.providers.length
|
||||
const featureCount = Object.keys(config.features).length
|
||||
const replicaCount = config.database.replicas.length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many roles are defined in permissions?')
|
||||
.groundTruth(String(roleCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many groups are defined in permissions?')
|
||||
.groundTruth(String(groupCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many authentication providers are configured?')
|
||||
.groundTruth(String(providerCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many feature flags are defined?')
|
||||
.groundTruth(String(featureCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many database replicas are configured?')
|
||||
.groundTruth(String(replicaCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: providers with admin scope
|
||||
const adminScopeProviderCount = config.authentication.providers.filter(p => p.scopes.includes('admin')).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many authentication providers include the "admin" scope?')
|
||||
.groundTruth(String(adminScopeProviderCount))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: feature flag details
|
||||
const enabledFeatures = Object.entries(config.features).filter(([_, f]) => f.enabled).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many feature flags are enabled?')
|
||||
.groundTruth(String(enabledFeatures))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: role permissions
|
||||
const adminPermissions = config.permissions.roles.admin?.permissions.length ?? 0
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many permissions does the admin role have?')
|
||||
.groundTruth(String(adminPermissions))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: additional nested counts
|
||||
const totalPermissions = Object.values(config.permissions.roles).reduce((sum, role) => sum + role.permissions.length, 0)
|
||||
const distinctPermissions = new Set(Object.values(config.permissions.roles).flatMap(r => r.permissions)).size
|
||||
const totalVariants = Object.values(config.features).reduce((sum, f) => sum + f.variants.length, 0)
|
||||
const highPriorityReplicas = config.database.replicas.filter(r => r.priority > 2).length
|
||||
const featuresWithHighRollout = Object.values(config.features).filter(f => f.rollout > 50).length
|
||||
const groupsWithMultipleRoles = Object.values(config.permissions.groups).filter(g => g.roles.length > 1).length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of permissions across all roles?')
|
||||
.groundTruth(String(totalPermissions))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many distinct permissions are defined across all roles?')
|
||||
.groundTruth(String(distinctPermissions))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total number of variants across all feature flags?')
|
||||
.groundTruth(String(totalVariants))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many database replicas have a priority greater than 2?')
|
||||
.groundTruth(String(highPriorityReplicas))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many feature flags have a rollout percentage greater than 50?')
|
||||
.groundTruth(String(featuresWithHighRollout))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many groups have more than one role assigned?')
|
||||
.groundTruth(String(groupsWithMultipleRoles))
|
||||
.type('aggregation')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Filtering: complex multi-condition queries
|
||||
const filteringQuestions = [
|
||||
{
|
||||
prompt: 'How many feature flags are enabled with rollout greater than 50%?',
|
||||
groundTruth: String(Object.entries(config.features)
|
||||
.filter(([_, f]) => f.enabled && f.rollout > 50).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many groups have the admin role?',
|
||||
groundTruth: String(Object.entries(config.permissions.groups)
|
||||
.filter(([_, g]) => g.roles.includes('admin')).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many database replicas have priority greater than 2 and port 5432?',
|
||||
groundTruth: String(config.database.replicas
|
||||
.filter(r => r.priority > 2 && r.port === 5432).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many authentication providers have more than 2 scopes?',
|
||||
groundTruth: String(config.authentication.providers
|
||||
.filter(p => p.scopes.length > 2).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many roles have at least 5 permissions?',
|
||||
groundTruth: String(Object.values(config.permissions.roles)
|
||||
.filter(r => r.permissions.length >= 5).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many feature flags are disabled with rollout less than 25%?',
|
||||
groundTruth: String(Object.values(config.features)
|
||||
.filter(f => !f.enabled && f.rollout < 25).length),
|
||||
},
|
||||
{
|
||||
prompt: 'How many enabled features have at least 2 variants?',
|
||||
groundTruth: String(Object.values(config.features)
|
||||
.filter(f => f.enabled && f.variants.length >= 2).length),
|
||||
},
|
||||
]
|
||||
|
||||
for (const q of filteringQuestions.slice(0, QUESTION_LIMITS.nestedConfig.filteringComplex)) {
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(q.prompt)
|
||||
.groundTruth(q.groundTruth)
|
||||
.type('filtering')
|
||||
.dataset('nested-config')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
import type { Order } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
|
||||
import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate nested (orders) questions
|
||||
*/
|
||||
export function generateNestedQuestions(orders: Order[], getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Field retrieval: order totals and statuses
|
||||
const orderFieldGenerators: Array<(order: Order, getId: () => string) => Question> = [
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the total for order ${order.orderId}?`)
|
||||
.groundTruth(String(order.total))
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the status of order ${order.orderId}?`)
|
||||
.groundTruth(order.status)
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
]
|
||||
|
||||
questions.push(...rotateQuestions(
|
||||
orders,
|
||||
orderFieldGenerators,
|
||||
QUESTION_LIMITS.nested.fieldRetrievalOrders,
|
||||
SAMPLE_STRIDES.ORDER_FIELD,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Field retrieval: customer info and order dates
|
||||
const customerFieldGenerators: Array<(order: Order, getId: () => string) => Question> = [
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the customer name for order ${order.orderId}?`)
|
||||
.groundTruth(order.customer.name)
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the customer email for order ${order.orderId}?`)
|
||||
.groundTruth(order.customer.email)
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the order date for order ${order.orderId}?`)
|
||||
.groundTruth(order.orderDate || '')
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(order, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many items are in order ${order.orderId}?`)
|
||||
.groundTruth(String(order.items.length))
|
||||
.type('field-retrieval')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
]
|
||||
|
||||
// Use stride + 1 for customer fields to offset from order fields
|
||||
const customerOrders = orders.map((_, i) => orders[i * SAMPLE_STRIDES.CUSTOMER_FIELD + 1] || orders[i]).filter(Boolean) as Order[]
|
||||
questions.push(...rotateQuestions(
|
||||
customerOrders,
|
||||
customerFieldGenerators,
|
||||
QUESTION_LIMITS.nested.fieldRetrievalCustomers,
|
||||
1,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Aggregation: totals and averages
|
||||
const totalRevenue = orders.reduce((sum, o) => sum + o.total, 0)
|
||||
const avgOrderValue = totalRevenue / orders.length
|
||||
const totalOrders = orders.length
|
||||
const maxOrderValue = Math.max(...orders.map(o => o.total))
|
||||
|
||||
// Count by status
|
||||
const statuses = [...new Set(orders.map(o => o.status))]
|
||||
for (const status of statuses.slice(0, QUESTION_LIMITS.nested.aggregationStatuses)) {
|
||||
const count = orders.filter(o => o.status === status).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many orders have status "${status}"?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the total revenue across all orders?')
|
||||
.groundTruth(String(totalRevenue.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the average order value?')
|
||||
.groundTruth(String(avgOrderValue.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many orders are in the dataset?')
|
||||
.groundTruth(String(totalOrders))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the highest order total?')
|
||||
.groundTruth(String(maxOrderValue.toFixed(2)))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('number')
|
||||
.normalize({ decimalPlaces: 2 })
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Aggregation: high-value orders (single-condition filter)
|
||||
for (const threshold of QUESTION_THRESHOLDS.nested.highValueOrders) {
|
||||
const count = orders.filter(o => o.total > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many orders have a total greater than ${threshold}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: multi-condition queries (status AND value)
|
||||
const orderStatuses = [...new Set(orders.map(o => o.status))]
|
||||
for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndValue)) {
|
||||
const count = orders.filter(
|
||||
o => o.status === status && o.total > QUESTION_THRESHOLDS.nested.statusValueThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many orders have status "${status}" and total greater than ${QUESTION_THRESHOLDS.nested.statusValueThreshold}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: status AND items count (multi-condition)
|
||||
for (const status of orderStatuses.slice(0, QUESTION_LIMITS.nested.filteringStatusAndItems)) {
|
||||
const count = orders.filter(
|
||||
o => o.status === status && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many orders have status "${status}" and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: total AND items count (multi-condition)
|
||||
for (const threshold of QUESTION_THRESHOLDS.nested.totalThresholdsForItems) {
|
||||
const count = orders.filter(
|
||||
o => o.total > threshold && o.items.length >= QUESTION_THRESHOLDS.nested.itemCountThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many orders have a total greater than ${threshold} and at least ${QUESTION_THRESHOLDS.nested.itemCountThreshold} items?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { Question } from '../types.ts'
|
||||
import { QuestionBuilder } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate structural validation questions for all incompleteness fixtures
|
||||
*
|
||||
* These questions test the ability to detect incomplete, truncated, or corrupted data
|
||||
* by validating structural metadata (TOON's [N] length declarations and {fields} headers).
|
||||
*
|
||||
* @remarks
|
||||
* - TOON's advantage: Explicit [N] and {fields} enable validation
|
||||
* - CSV disadvantage: No structural metadata to validate against
|
||||
* - JSON/YAML disadvantage: Require manual counting and schema inference
|
||||
*/
|
||||
export function generateStructuralValidationQuestions(
|
||||
getId: () => string,
|
||||
): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Dataset names and their expected validity
|
||||
const validationFixtures = [
|
||||
{ dataset: 'structural-validation-control', isValid: true, description: 'Valid complete dataset (control)' },
|
||||
{ dataset: 'structural-validation-truncated', isValid: false, description: 'Array truncated: 3 rows removed from end' },
|
||||
{ dataset: 'structural-validation-extra-rows', isValid: false, description: 'Extra rows added beyond declared length' },
|
||||
{ dataset: 'structural-validation-width-mismatch', isValid: false, description: 'Inconsistent field count (missing salary in row 10)' },
|
||||
{ dataset: 'structural-validation-missing-fields', isValid: false, description: 'Missing required fields (no email in multiple rows)' },
|
||||
] as const
|
||||
|
||||
// Generate one validation question per fixture
|
||||
for (const fixture of validationFixtures) {
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('Is this data complete and valid? Answer only YES or NO.')
|
||||
.groundTruth(fixture.isValid ? 'YES' : 'NO')
|
||||
.type('structural-validation')
|
||||
.dataset(fixture.dataset)
|
||||
.answerType('boolean')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
import type { AnalyticsMetric, Employee, EventLog, Order, Repository } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QuestionBuilder } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate structure-awareness questions across all datasets
|
||||
*
|
||||
* These questions test format-native structural affordances:
|
||||
* - TOON's explicit array length [N] and field declarations {fields}
|
||||
* - CSV's header row (but no explicit length)
|
||||
* - JSON/YAML have neither unless the model counts manually
|
||||
*/
|
||||
export function generateStructureQuestions(
|
||||
employees: Employee[],
|
||||
orders: Order[],
|
||||
metrics: AnalyticsMetric[],
|
||||
repos: Repository[],
|
||||
logs: EventLog[],
|
||||
getId: () => string,
|
||||
): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// ========== TABULAR DATASET (Employees) ==========
|
||||
|
||||
// Count: Total employees (tests array length awareness)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many employees are in the dataset?')
|
||||
.groundTruth(String(employees.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field list: Employee fields (tests field name awareness)
|
||||
const employeeFields = 'id,name,email,department,salary,yearsExperience,active'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('List the field names for employees (comma-separated, in order).')
|
||||
.groundTruth(employeeFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Positional: Third field name for employees (tests TOON {fields} syntax)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the 3rd field name for employees?')
|
||||
.groundTruth('email')
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last employee's department (tests ability to find last row using length)
|
||||
const lastEmployee = employees.at(-1)!
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the department of the last employee in the dataset?')
|
||||
.groundTruth(lastEmployee.department)
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last employee's name
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the name of the last employee in the dataset?')
|
||||
.groundTruth(lastEmployee.name)
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field count: How many fields per employee (tests schema awareness)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many fields does each employee record have?')
|
||||
.groundTruth('7')
|
||||
.type('structure-awareness')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// ========== NESTED DATASET (Orders) ==========
|
||||
|
||||
// Count: Total orders
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many orders are in the dataset?')
|
||||
.groundTruth(String(orders.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field list: Order fields
|
||||
const orderFields = 'orderId,customer,items,subtotal,tax,total,status,orderDate'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('List the top-level field names for orders (comma-separated, in order).')
|
||||
.groundTruth(orderFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Nested count: Items in specific order
|
||||
const orderWithManyItems = orders.reduce((max, order) =>
|
||||
order.items.length > max.items.length ? order : max,
|
||||
)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many items are in order ${orderWithManyItems.orderId}?`)
|
||||
.groundTruth(String(orderWithManyItems.items.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Nested field list: Item fields
|
||||
const itemFields = 'sku,name,quantity,price'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What are the field names for items within orders (comma-separated, in order)?')
|
||||
.groundTruth(itemFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last order's status
|
||||
const lastOrder = orders.at(-1)!
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the status of the last order in the dataset?')
|
||||
.groundTruth(lastOrder.status)
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Customer field list
|
||||
const customerFields = 'id,name,email,phone'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What are the field names for customer objects within orders (comma-separated, in order)?')
|
||||
.groundTruth(customerFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('nested')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// ========== ANALYTICS DATASET (Metrics) ==========
|
||||
|
||||
// Count: Total metrics
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many metric records are in the dataset?')
|
||||
.groundTruth(String(metrics.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field list: Metric fields
|
||||
const metricFields = 'date,views,clicks,conversions,revenue,bounceRate'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('List the field names for metrics (comma-separated, in order).')
|
||||
.groundTruth(metricFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('analytics')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Positional: Fifth field name for metrics (tests TOON {fields} syntax)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the 5th field name for analytics metrics?')
|
||||
.groundTruth('revenue')
|
||||
.type('structure-awareness')
|
||||
.dataset('analytics')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last metric's date
|
||||
const lastMetric = metrics.at(-1)!
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the date of the last metric record in the dataset?')
|
||||
.groundTruth(lastMetric.date)
|
||||
.type('structure-awareness')
|
||||
.dataset('analytics')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field count: How many fields per metric
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many fields does each metric record have?')
|
||||
.groundTruth('6')
|
||||
.type('structure-awareness')
|
||||
.dataset('analytics')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// ========== GITHUB DATASET (Repositories) ==========
|
||||
|
||||
// Count: Total repositories
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many repositories are in the dataset?')
|
||||
.groundTruth(String(repos.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field list: Repository fields
|
||||
const repoFields = 'id,name,repo,description,stars,watchers,forks,defaultBranch,createdAt,updatedAt,pushedAt'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('List the field names for repositories (comma-separated, in order).')
|
||||
.groundTruth(repoFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('github')
|
||||
.answerType('csv-list-ordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Positional: Seventh field name for repos (tests TOON {fields} syntax)
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the 7th field name for GitHub repositories?')
|
||||
.groundTruth('forks')
|
||||
.type('structure-awareness')
|
||||
.dataset('github')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last repo's name
|
||||
const lastRepo = repos.at(-1)!
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the name of the last repository in the dataset?')
|
||||
.groundTruth(lastRepo.name)
|
||||
.type('structure-awareness')
|
||||
.dataset('github')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field count: How many fields per repository
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many fields does each repository record have?')
|
||||
.groundTruth('11')
|
||||
.type('structure-awareness')
|
||||
.dataset('github')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// ========== EVENT LOGS DATASET ==========
|
||||
|
||||
// Count: Total logs
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many log entries are in the dataset?')
|
||||
.groundTruth(String(logs.length))
|
||||
.type('structure-awareness')
|
||||
.dataset('event-logs')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Field list: Base log fields (including optional error)
|
||||
const logFields = 'timestamp,level,endpoint,statusCode,responseTime,userId,error'
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('List the field names for log entries (comma-separated, any order, including optional fields).')
|
||||
.groundTruth(logFields)
|
||||
.type('structure-awareness')
|
||||
.dataset('event-logs')
|
||||
.answerType('csv-list-unordered')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Last row: Last log's level
|
||||
const lastLog = logs.at(-1)!
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the level of the last log entry in the dataset?')
|
||||
.groundTruth(lastLog.level)
|
||||
.type('structure-awareness')
|
||||
.dataset('event-logs')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
)
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
import type { Employee } from '../datasets.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
import { QUESTION_LIMITS, QUESTION_THRESHOLDS } from '../constants.ts'
|
||||
import { QuestionBuilder, rotateQuestions, SAMPLE_STRIDES } from './utils.ts'
|
||||
|
||||
/**
|
||||
* Generate tabular (employee) questions
|
||||
*/
|
||||
export function generateTabularQuestions(employees: Employee[], getId: () => string): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
// Field retrieval: specific employees
|
||||
const fieldGenerators: Array<(emp: Employee, getId: () => string) => Question> = [
|
||||
(emp, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the salary of ${emp.name}?`)
|
||||
.groundTruth(String(emp.salary))
|
||||
.type('field-retrieval')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(emp, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What department does ${emp.name} work in?`)
|
||||
.groundTruth(emp.department)
|
||||
.type('field-retrieval')
|
||||
.dataset('tabular')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(emp, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`What is the email address of ${emp.name}?`)
|
||||
.groundTruth(emp.email)
|
||||
.type('field-retrieval')
|
||||
.dataset('tabular')
|
||||
.answerType('string')
|
||||
.build(),
|
||||
(emp, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many years of experience does ${emp.name} have?`)
|
||||
.groundTruth(String(emp.yearsExperience))
|
||||
.type('field-retrieval')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
(emp, getId) => new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`Is ${emp.name} an active employee?`)
|
||||
.groundTruth(emp.active ? 'yes' : 'no')
|
||||
.type('field-retrieval')
|
||||
.dataset('tabular')
|
||||
.answerType('boolean')
|
||||
.build(),
|
||||
]
|
||||
|
||||
questions.push(...rotateQuestions(
|
||||
employees,
|
||||
fieldGenerators,
|
||||
QUESTION_LIMITS.tabular.fieldRetrieval,
|
||||
SAMPLE_STRIDES.EMPLOYEE_FIELD,
|
||||
getId,
|
||||
))
|
||||
|
||||
// Aggregation: count by department
|
||||
const departments = [...new Set(employees.map(e => e.department))]
|
||||
for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.aggregationDepartments)) {
|
||||
const count = employees.filter(e => e.department === dept).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many employees work in ${dept}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: salary ranges (single-condition filters)
|
||||
for (const threshold of QUESTION_THRESHOLDS.tabular.salaryRanges) {
|
||||
const count = employees.filter(e => e.salary > threshold).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many employees have a salary greater than ${threshold}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Aggregation: totals and averages
|
||||
const totalEmployees = employees.length
|
||||
const avgSalary = Math.round(employees.reduce((sum, e) => sum + e.salary, 0) / totalEmployees)
|
||||
const activeCount = employees.filter(e => e.active).length
|
||||
const inactiveCount = employees.filter(e => !e.active).length
|
||||
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many employees are in the dataset?')
|
||||
.groundTruth(String(totalEmployees))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('What is the average salary across all employees?')
|
||||
.groundTruth(String(avgSalary))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many employees are active?')
|
||||
.groundTruth(String(activeCount))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt('How many employees are inactive?')
|
||||
.groundTruth(String(inactiveCount))
|
||||
.type('aggregation')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
|
||||
// Filtering: count by department with salary filter (multi-condition)
|
||||
for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringMultiConditionDepartments)) {
|
||||
const count = employees.filter(
|
||||
e => e.department === dept && e.salary > QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many employees in ${dept} have a salary greater than ${QUESTION_THRESHOLDS.tabular.departmentSalaryThreshold}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: active employees by experience (multi-condition)
|
||||
for (const exp of QUESTION_THRESHOLDS.tabular.experienceYears.slice(0, QUESTION_LIMITS.tabular.filteringExperience)) {
|
||||
const count = employees.filter(e => e.yearsExperience > exp && e.active).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many active employees have more than ${exp} years of experience?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: department by experience (multi-condition)
|
||||
for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentExp)) {
|
||||
const count = employees.filter(
|
||||
e => e.department === dept && e.yearsExperience > QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold,
|
||||
).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many employees in ${dept} have more than ${QUESTION_THRESHOLDS.tabular.departmentExperienceThreshold} years of experience?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
// Filtering: department by active status (multi-condition)
|
||||
for (const dept of departments.slice(0, QUESTION_LIMITS.tabular.filteringDepartmentActive)) {
|
||||
const count = employees.filter(e => e.department === dept && e.active).length
|
||||
questions.push(
|
||||
new QuestionBuilder()
|
||||
.id(getId())
|
||||
.prompt(`How many active employees work in ${dept}?`)
|
||||
.groundTruth(String(count))
|
||||
.type('filtering')
|
||||
.dataset('tabular')
|
||||
.answerType('integer')
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { AnswerType, NormalizationOptions } from '../normalize.ts'
|
||||
import type { Question } from '../types.ts'
|
||||
|
||||
// Constants for sampling strides
|
||||
export const SAMPLE_STRIDES = {
|
||||
EMPLOYEE_FIELD: 2,
|
||||
ORDER_FIELD: 2,
|
||||
CUSTOMER_FIELD: 2,
|
||||
ANALYTICS_FIELD: 3,
|
||||
METRIC_FIELD: 3,
|
||||
REPO_FIELD: 7,
|
||||
EVENT_LOG_FIELD: 5,
|
||||
} as const
|
||||
|
||||
/**
|
||||
* ID Generator
|
||||
*/
|
||||
export function* createIdGenerator(): Generator<string, never, never> {
|
||||
let id = 1
|
||||
while (true) {
|
||||
yield `q${id++}`
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Question Builder class for fluent question creation
|
||||
*/
|
||||
export class QuestionBuilder {
|
||||
private question: Partial<Question> = {}
|
||||
|
||||
id(id: string): this {
|
||||
this.question.id = id
|
||||
return this
|
||||
}
|
||||
|
||||
prompt(prompt: string): this {
|
||||
this.question.prompt = prompt
|
||||
return this
|
||||
}
|
||||
|
||||
groundTruth(groundTruth: string): this {
|
||||
this.question.groundTruth = groundTruth
|
||||
return this
|
||||
}
|
||||
|
||||
type(type: Question['type']): this {
|
||||
this.question.type = type
|
||||
return this
|
||||
}
|
||||
|
||||
dataset(dataset: Question['dataset']): this {
|
||||
this.question.dataset = dataset
|
||||
return this
|
||||
}
|
||||
|
||||
answerType(kind: AnswerType): this {
|
||||
this.question.answerType = kind
|
||||
return this
|
||||
}
|
||||
|
||||
normalize(options: Partial<NormalizationOptions>): this {
|
||||
this.question.normalizationOptions = options
|
||||
return this
|
||||
}
|
||||
|
||||
build(): Question {
|
||||
if (!this.question.id || !this.question.prompt || !this.question.groundTruth || !this.question.type || !this.question.dataset) {
|
||||
throw new Error('Incomplete question')
|
||||
}
|
||||
|
||||
return this.question as Question
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rotate through question generators
|
||||
*/
|
||||
export function rotateQuestions<T>(
|
||||
items: T[],
|
||||
generators: ((item: T, getId: () => string) => Question)[],
|
||||
limit: number,
|
||||
stride: number,
|
||||
getId: () => string,
|
||||
): Question[] {
|
||||
const questions: Question[] = []
|
||||
|
||||
for (let i = 0; i < Math.min(limit, items.length); i++) {
|
||||
const item = items[i * stride] || items[i]
|
||||
if (!item)
|
||||
continue
|
||||
|
||||
const generatorIndex = i % generators.length
|
||||
const generator = generators[generatorIndex]
|
||||
if (generator) {
|
||||
questions.push(generator(item, getId))
|
||||
}
|
||||
}
|
||||
|
||||
return questions
|
||||
}
|
||||
@@ -0,0 +1,641 @@
|
||||
import type { Dataset, EfficiencyRanking, EvaluationResult, FormatResult, Question } from './types.ts'
|
||||
import { FORMATTER_DISPLAY_NAMES, QUESTION_TYPE_LABELS, QUESTION_TYPES } from './constants.ts'
|
||||
import { ACCURACY_DATASETS } from './datasets.ts'
|
||||
import { models, PRIMERS } from './evaluate.ts'
|
||||
import { supportsCSV } from './formatters.ts'
|
||||
import { generateQuestions } from './questions/index.ts'
|
||||
import { createProgressBar, tokenize } from './utils.ts'
|
||||
|
||||
const EFFICIENCY_CHART_STYLE: 'vertical' | 'horizontal' = 'horizontal'
|
||||
|
||||
/**
|
||||
* Calculate token counts for all format+dataset combinations
|
||||
*
|
||||
* @remarks
|
||||
* Includes primer tokens for fairer comparison across formats
|
||||
*/
|
||||
export function calculateTokenCounts(
|
||||
formatters: Record<string, (data: unknown) => string>,
|
||||
): Record<string, number> {
|
||||
const tokenCounts: Record<string, number> = {}
|
||||
|
||||
for (const [formatName, formatter] of Object.entries(formatters)) {
|
||||
for (const dataset of ACCURACY_DATASETS) {
|
||||
// Skip CSV for datasets that don't support it
|
||||
if (formatName === 'csv' && !supportsCSV(dataset))
|
||||
continue
|
||||
|
||||
const formattedData = formatter(dataset.data)
|
||||
const primer = PRIMERS[formatName] ?? ''
|
||||
// Include primer in token count for fair comparison
|
||||
const fullPrompt = primer ? `${primer}\n\n${formattedData}` : formattedData
|
||||
const key = `${formatName}-${dataset.name}`
|
||||
tokenCounts[key] = tokenize(fullPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
return tokenCounts
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate per-format statistics from evaluation results
|
||||
*/
|
||||
export function calculateFormatResults(
|
||||
results: EvaluationResult[],
|
||||
tokenCounts: Record<string, number>,
|
||||
): FormatResult[] {
|
||||
const formatNames = [...new Set(results.map(r => r.format))]
|
||||
|
||||
return formatNames.map((formatName) => {
|
||||
const formatResults = results.filter(r => r.format === formatName)
|
||||
const correctCount = formatResults.filter(r => r.isCorrect).length
|
||||
const totalCount = formatResults.length
|
||||
const accuracy = correctCount / totalCount
|
||||
|
||||
// Calculate average tokens across all datasets for this format
|
||||
const formatTokenEntries = Object.entries(tokenCounts)
|
||||
.filter(([key]) => key.startsWith(`${formatName}-`))
|
||||
const avgTokens = formatTokenEntries.reduce((sum, [, tokens]) => sum + tokens, 0) / formatTokenEntries.length
|
||||
|
||||
const averageLatency = formatResults.reduce((sum, r) => sum + r.latencyMs, 0) / totalCount
|
||||
|
||||
return {
|
||||
format: formatName,
|
||||
accuracy,
|
||||
totalTokens: Math.round(avgTokens),
|
||||
averageLatency: Math.round(averageLatency),
|
||||
correctCount,
|
||||
totalCount,
|
||||
}
|
||||
}).sort((a, b) => b.accuracy - a.accuracy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate consolidated retrieval accuracy report
|
||||
*/
|
||||
export function generateAccuracyReport(
|
||||
results: EvaluationResult[],
|
||||
formatResults: FormatResult[],
|
||||
tokenCounts: Record<string, number>,
|
||||
): string {
|
||||
const questions = generateQuestions()
|
||||
const totalQuestions = [...new Set(results.map(r => r.questionId))].length
|
||||
const modelIds = models.map(m => m.modelId)
|
||||
const modelNames = modelIds.filter(id => results.some(r => r.model === id))
|
||||
|
||||
return `
|
||||
Benchmarks test LLM comprehension across different input formats using ${totalQuestions} data retrieval questions on ${modelNames.length} ${modelNames.length === 1 ? 'model' : 'models'}.
|
||||
|
||||
<details>
|
||||
<summary><strong>Show Dataset Catalog</strong></summary>
|
||||
|
||||
${generateDatasetCatalog(ACCURACY_DATASETS)}
|
||||
|
||||
</details>
|
||||
|
||||
#### Efficiency Ranking (Accuracy per 1K Tokens)
|
||||
|
||||
${generateEfficiencyRankingReport(formatResults, totalQuestions, modelNames.length)}
|
||||
|
||||
#### Per-Model Accuracy
|
||||
|
||||
${generateDetailedAccuracyReport(formatResults, results, questions, tokenCounts)}
|
||||
`.trimStart()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate dataset catalog section
|
||||
*/
|
||||
function generateDatasetCatalog(datasets: Dataset[]): string {
|
||||
const rows = datasets.map((dataset) => {
|
||||
const csvSupport = supportsCSV(dataset) ? '✓' : '✗'
|
||||
const rowCount = Object.values(dataset.data)[0]?.length ?? 1
|
||||
const structure = dataset.metadata.structureClass
|
||||
const eligibility = `${dataset.metadata.tabularEligibility}%`
|
||||
|
||||
return `| ${dataset.description} | ${rowCount} | ${structure} | ${csvSupport} | ${eligibility} |`
|
||||
}).join('\n')
|
||||
|
||||
return `
|
||||
#### Dataset Catalog
|
||||
|
||||
| Dataset | Rows | Structure | CSV Support | Eligibility |
|
||||
| ------- | ---- | --------- | ----------- | ----------- |
|
||||
${rows}
|
||||
|
||||
**Structure classes:**
|
||||
- **uniform**: All objects have identical fields with primitive values
|
||||
- **semi-uniform**: Mix of uniform and non-uniform structures
|
||||
- **nested**: Objects with nested structures (nested objects or arrays)
|
||||
- **deep**: Highly nested with minimal tabular eligibility
|
||||
|
||||
**CSV Support:** ✓ (supported), ✗ (not supported – would require lossy flattening)
|
||||
|
||||
**Eligibility:** Percentage of arrays that qualify for TOON's tabular format (uniform objects with primitive values)
|
||||
`.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate efficiency ranking report
|
||||
*/
|
||||
function generateEfficiencyRankingReport(
|
||||
formatResults: FormatResult[],
|
||||
totalQuestions: number,
|
||||
modelCount: number,
|
||||
): string {
|
||||
const toon = formatResults.find(r => r.format === 'toon')
|
||||
const json = formatResults.find(r => r.format === 'json-pretty')
|
||||
const csv = formatResults.find(r => r.format === 'csv')
|
||||
|
||||
// Build efficiency ranking (accuracy per 1k tokens)
|
||||
const efficiencyRanking = formatResults
|
||||
// Exclude CSV since it only supports a subset of datasets (~half the questions)
|
||||
.filter(fr => fr.format !== 'csv')
|
||||
.map((fr) => {
|
||||
const efficiency = (fr.accuracy * 100) / (fr.totalTokens / 1000)
|
||||
return {
|
||||
format: fr.format,
|
||||
efficiency,
|
||||
accuracy: fr.accuracy,
|
||||
tokens: fr.totalTokens,
|
||||
}
|
||||
})
|
||||
.sort((a, b) => b.efficiency - a.efficiency)
|
||||
|
||||
const efficiencyChart = EFFICIENCY_CHART_STYLE === 'vertical'
|
||||
? generateVerticalEfficiencyChart(efficiencyRanking)
|
||||
: generateHorizontalEfficiencyChart(efficiencyRanking)
|
||||
|
||||
// Build summary text
|
||||
let summary = ''
|
||||
if (toon && json) {
|
||||
const toonVsJson = `**${(toon.accuracy * 100).toFixed(1)}%** accuracy (vs JSON's ${(json.accuracy * 100).toFixed(1)}%)`
|
||||
const tokenSavings = `**${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens**`
|
||||
summary = `TOON achieves ${toonVsJson} while using ${tokenSavings}.`
|
||||
}
|
||||
|
||||
// Add CSV note if available
|
||||
let csvNote = ''
|
||||
if (csv) {
|
||||
// CSV totalCount is evaluations (questions × models), so divide by number of models to get question count
|
||||
const csvQuestionCount = csv.totalCount / modelCount
|
||||
csvNote = `**Note on CSV:** Excluded from ranking as it only supports ${csvQuestionCount} of ${totalQuestions} questions (flat tabular data only). While CSV is highly token-efficient for simple tabular data, it cannot represent nested structures that other formats handle.`
|
||||
}
|
||||
|
||||
return `
|
||||
Each format ranked by efficiency (accuracy percentage per 1,000 tokens):
|
||||
|
||||
\`\`\`
|
||||
${efficiencyChart}
|
||||
\`\`\`
|
||||
|
||||
*Efficiency score = (Accuracy % ÷ Tokens) × 1,000. Higher is better.*
|
||||
|
||||
> [!TIP]
|
||||
> ${summary}
|
||||
|
||||
${csvNote}
|
||||
`.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate detailed accuracy report with breakdowns and methodology
|
||||
*/
|
||||
function generateDetailedAccuracyReport(
|
||||
formatResults: FormatResult[],
|
||||
results: EvaluationResult[],
|
||||
questions: Question[],
|
||||
tokenCounts: Record<string, number>,
|
||||
): string {
|
||||
const toon = formatResults.find(r => r.format === 'toon')
|
||||
const json = formatResults.find(r => r.format === 'json-pretty')
|
||||
|
||||
const modelIds = models.map(m => m.modelId)
|
||||
const modelNames = modelIds.filter(id => results.some(r => r.model === id))
|
||||
|
||||
// Generate model breakdown section
|
||||
const modelBreakdown = generateModelBreakdown(formatResults, results, modelNames)
|
||||
|
||||
// Generate summary comparison
|
||||
const summaryComparison = generateSummaryComparison(toon, json)
|
||||
|
||||
// Generate performance by dataset
|
||||
const datasetBreakdown = generateDatasetBreakdown(formatResults, results, questions, tokenCounts)
|
||||
|
||||
// Generate performance by model
|
||||
const modelPerformance = generateModelPerformanceTable(formatResults, results, modelNames)
|
||||
|
||||
// Generate question type breakdown
|
||||
const questionTypeBreakdown = generateQuestionTypeBreakdown(formatResults, results, questions)
|
||||
const totalQuestions = [...new Set(results.map(r => r.questionId))].length
|
||||
|
||||
// Calculate question type distribution
|
||||
const fieldRetrievalCount = questions.filter(q => q.type === 'field-retrieval').length
|
||||
const aggregationCount = questions.filter(q => q.type === 'aggregation').length
|
||||
const filteringCount = questions.filter(q => q.type === 'filtering').length
|
||||
const structureAwarenessCount = questions.filter(q => q.type === 'structure-awareness').length
|
||||
const structuralValidationCount = questions.filter(q => q.type === 'structural-validation').length
|
||||
|
||||
const fieldRetrievalPercent = ((fieldRetrievalCount / totalQuestions) * 100).toFixed(0)
|
||||
const aggregationPercent = ((aggregationCount / totalQuestions) * 100).toFixed(0)
|
||||
const filteringPercent = ((filteringCount / totalQuestions) * 100).toFixed(0)
|
||||
const structureAwarenessPercent = ((structureAwarenessCount / totalQuestions) * 100).toFixed(0)
|
||||
const structuralValidationPercent = ((structuralValidationCount / totalQuestions) * 100).toFixed(0)
|
||||
|
||||
// Calculate dataset sizes
|
||||
const tabularSize = ACCURACY_DATASETS.find(d => d.name === 'tabular')?.data.employees?.length || 0
|
||||
const nestedSize = ACCURACY_DATASETS.find(d => d.name === 'nested')?.data.orders?.length || 0
|
||||
const analyticsSize = ACCURACY_DATASETS.find(d => d.name === 'analytics')?.data.metrics?.length || 0
|
||||
const githubSize = ACCURACY_DATASETS.find(d => d.name === 'github')?.data.repositories?.length || 0
|
||||
const eventLogsSize = ACCURACY_DATASETS.find(d => d.name === 'event-logs')?.data.logs?.length || 0
|
||||
const nestedConfigSize = 1 // Single config object
|
||||
|
||||
// Calculate number of formats and evaluations
|
||||
const formatCount = formatResults.length
|
||||
const totalEvaluations = totalQuestions * formatCount * modelNames.length
|
||||
|
||||
return `
|
||||
Accuracy across ${modelNames.length} ${modelNames.length === 1 ? 'LLM' : 'LLMs'} on ${totalQuestions} data retrieval questions:
|
||||
|
||||
\`\`\`
|
||||
${modelBreakdown}
|
||||
\`\`\`
|
||||
|
||||
${summaryComparison}
|
||||
|
||||
<details>
|
||||
<summary><strong>Performance by dataset, model, and question type</strong></summary>
|
||||
|
||||
#### Performance by Question Type
|
||||
|
||||
${questionTypeBreakdown}
|
||||
|
||||
#### Performance by Dataset
|
||||
|
||||
${datasetBreakdown}
|
||||
|
||||
#### Performance by Model
|
||||
|
||||
${modelPerformance}
|
||||
|
||||
</details>
|
||||
|
||||
#### What's Being Measured
|
||||
|
||||
This benchmark tests **LLM comprehension and data retrieval accuracy** across different input formats. Each LLM receives formatted data and must answer questions about it. This does **not** test the model's ability to generate TOON output – only to read and understand it.
|
||||
|
||||
#### Datasets Tested
|
||||
|
||||
Eleven datasets designed to test different structural patterns and validation capabilities:
|
||||
|
||||
**Primary datasets:**
|
||||
|
||||
1. **Tabular** (${tabularSize} employee records): Uniform objects with identical fields – optimal for TOON's tabular format.
|
||||
2. **Nested** (${nestedSize} e-commerce orders): Complex structures with nested customer objects and item arrays.
|
||||
3. **Analytics** (${analyticsSize} days of metrics): Time-series data with dates and numeric values.
|
||||
4. **GitHub** (${githubSize} repositories): Real-world data from top GitHub repos by stars.
|
||||
5. **Event Logs** (${eventLogsSize} logs): Semi-uniform data with ~50% flat logs and ~50% with nested error objects.
|
||||
6. **Nested Config** (${nestedConfigSize} configuration): Deeply nested configuration with minimal tabular eligibility.
|
||||
|
||||
**Structural validation datasets:**
|
||||
|
||||
7. **Control**: Valid complete dataset (baseline for validation)
|
||||
8. **Truncated**: Array with 3 rows removed from end (tests \`[N]\` length detection)
|
||||
9. **Extra rows**: Array with 3 additional rows beyond declared length
|
||||
10. **Width mismatch**: Inconsistent field count (missing salary in row 10)
|
||||
11. **Missing fields**: Systematic field omissions (no email in multiple rows)
|
||||
|
||||
#### Question Types
|
||||
|
||||
${totalQuestions} questions are generated dynamically across five categories:
|
||||
|
||||
- **Field retrieval (${fieldRetrievalPercent}%)**: Direct value lookups or values that can be read straight off a record (including booleans and simple counts such as array lengths)
|
||||
- Example: "What is Alice's salary?" → \`75000\`
|
||||
- Example: "How many items are in order ORD-0042?" → \`3\`
|
||||
- Example: "What is the customer name for order ORD-0042?" → \`John Doe\`
|
||||
|
||||
- **Aggregation (${aggregationPercent}%)**: Dataset-level totals and averages plus single-condition filters (counts, sums, min/max comparisons)
|
||||
- Example: "How many employees work in Engineering?" → \`17\`
|
||||
- Example: "What is the total revenue across all orders?" → \`45123.50\`
|
||||
- Example: "How many employees have salary > 80000?" → \`23\`
|
||||
|
||||
- **Filtering (${filteringPercent}%)**: Multi-condition queries requiring compound logic (AND constraints across fields)
|
||||
- Example: "How many employees in Sales have salary > 80000?" → \`5\`
|
||||
- Example: "How many active employees have more than 10 years of experience?" → \`8\`
|
||||
|
||||
- **Structure awareness (${structureAwarenessPercent}%)**: Tests format-native structural affordances (TOON's \`[N]\` count and \`{fields}\`, CSV's header row)
|
||||
- Example: "How many employees are in the dataset?" → \`100\`
|
||||
- Example: "List the field names for employees" → \`id, name, email, department, salary, yearsExperience, active\`
|
||||
- Example: "What is the department of the last employee?" → \`Sales\`
|
||||
|
||||
- **Structural validation (${structuralValidationPercent}%)**: Tests ability to detect incomplete, truncated, or corrupted data using structural metadata
|
||||
- Example: "Is this data complete and valid?" → \`YES\` (control dataset) or \`NO\` (corrupted datasets)
|
||||
- Tests TOON's \`[N]\` length validation and \`{fields}\` consistency checking
|
||||
- Demonstrates CSV's lack of structural validation capabilities
|
||||
|
||||
#### Evaluation Process
|
||||
|
||||
1. **Format conversion**: Each dataset is converted to all ${formatCount} formats (${formatResults.map(f => FORMATTER_DISPLAY_NAMES[f.format] || f.format).join(', ')}).
|
||||
2. **Query LLM**: Each model receives formatted data + question in a prompt and extracts the answer.
|
||||
3. **Validate deterministically**: Answers are validated using type-aware comparison (e.g., \`50000\` = \`$50,000\`, \`Engineering\` = \`engineering\`, \`2025-01-01\` = \`January 1, 2025\`) without requiring an LLM judge.
|
||||
|
||||
#### Models & Configuration
|
||||
|
||||
- **Models tested**: ${modelNames.map(m => `\`${m}\``).join(', ')}
|
||||
- **Token counting**: Using \`gpt-tokenizer\` with \`o200k_base\` encoding (GPT-5 tokenizer)
|
||||
- **Temperature**: Not set (models use their defaults)
|
||||
- **Total evaluations**: ${totalQuestions} questions × ${formatCount} formats × ${modelNames.length} models = ${totalEvaluations.toLocaleString('en-US')} LLM calls
|
||||
`.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate ASCII bar chart showing per-model accuracy across formats
|
||||
*/
|
||||
function generateModelBreakdown(
|
||||
formatResults: FormatResult[],
|
||||
results: EvaluationResult[],
|
||||
modelNames: string[],
|
||||
): string {
|
||||
const maxDisplayNameWidth = Math.max(
|
||||
...Object.values(FORMATTER_DISPLAY_NAMES).map(name => name.length),
|
||||
)
|
||||
const progressBarWidth = 20
|
||||
|
||||
return modelNames.map((modelName, i) => {
|
||||
const modelResults = formatResults.map((fr) => {
|
||||
const modelFormatResults = results.filter(r => r.model === modelName && r.format === fr.format)
|
||||
const correctCount = modelFormatResults.filter(r => r.isCorrect).length
|
||||
const totalCount = modelFormatResults.length
|
||||
const accuracy = totalCount > 0 ? correctCount / totalCount : 0
|
||||
|
||||
return {
|
||||
format: fr.format,
|
||||
accuracy,
|
||||
correctCount,
|
||||
totalCount,
|
||||
}
|
||||
}).sort((a, b) => b.accuracy - a.accuracy)
|
||||
|
||||
const formatLines = modelResults.map((result) => {
|
||||
const bar = createProgressBar(result.accuracy, 1, progressBarWidth)
|
||||
const accuracyString = `${(result.accuracy * 100).toFixed(1)}%`.padStart(6)
|
||||
const countString = `(${result.correctCount}/${result.totalCount})`
|
||||
const prefix = result.format === 'toon' ? '→ ' : ' '
|
||||
const displayName = FORMATTER_DISPLAY_NAMES[result.format] || result.format
|
||||
return `${prefix}${displayName.padEnd(maxDisplayNameWidth)} ${bar} ${accuracyString} ${countString}`
|
||||
}).join('\n')
|
||||
|
||||
// Add blank line before model name, except for first model
|
||||
return `${i > 0 ? '\n' : ''}${modelName}\n${formatLines}`
|
||||
}).join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate summary comparison between TOON and JSON formats
|
||||
*/
|
||||
function generateSummaryComparison(
|
||||
toon: FormatResult | undefined,
|
||||
json: FormatResult | undefined,
|
||||
): string {
|
||||
if (!toon || !json)
|
||||
return ''
|
||||
|
||||
return `
|
||||
> [!TIP]
|
||||
> TOON achieves **${(toon.accuracy * 100).toFixed(1)}% accuracy** (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using **${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens** on these datasets.
|
||||
`.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate per-dataset performance breakdown tables
|
||||
*/
|
||||
function generateDatasetBreakdown(
|
||||
formatResults: FormatResult[],
|
||||
results: EvaluationResult[],
|
||||
questions: Question[],
|
||||
tokenCounts: Record<string, number>,
|
||||
): string {
|
||||
// Build question ID to dataset mapping for O(1) lookups
|
||||
const questionDatasetMap = new Map(questions.map(q => [q.id, q.dataset]))
|
||||
|
||||
return ACCURACY_DATASETS.map((dataset) => {
|
||||
const datasetResults = formatResults.map((fr) => {
|
||||
const datasetFormatResults = results.filter(r => questionDatasetMap.get(r.questionId) === dataset.name)
|
||||
if (datasetFormatResults.length === 0)
|
||||
return undefined
|
||||
|
||||
const formatDatasetResults = datasetFormatResults.filter(r => r.format === fr.format)
|
||||
if (formatDatasetResults.length === 0)
|
||||
return undefined
|
||||
|
||||
const correctCount = formatDatasetResults.filter(r => r.isCorrect).length
|
||||
const totalCount = formatDatasetResults.length
|
||||
const accuracy = totalCount > 0 ? correctCount / totalCount : 0
|
||||
|
||||
// Get token count for this dataset+format
|
||||
const tokenKey = `${fr.format}-${dataset.name}`
|
||||
const tokens = tokenCounts[tokenKey] || fr.totalTokens
|
||||
|
||||
return {
|
||||
format: fr.format,
|
||||
accuracy,
|
||||
tokens,
|
||||
correctCount,
|
||||
totalCount,
|
||||
}
|
||||
}).filter(Boolean) as { format: string, accuracy: number, tokens: number, correctCount: number, totalCount: number }[]
|
||||
|
||||
if (datasetResults.length === 0)
|
||||
return ''
|
||||
|
||||
// Sort by efficiency
|
||||
datasetResults.sort((a, b) => {
|
||||
const effA = (a.accuracy ** 2) / (a.tokens / 1000)
|
||||
const effB = (b.accuracy ** 2) / (b.tokens / 1000)
|
||||
return effB - effA
|
||||
})
|
||||
|
||||
const tableRows = datasetResults.slice(0, 6).map(result =>
|
||||
`| \`${result.format}\` | ${(result.accuracy * 100).toFixed(1)}% | ${result.tokens.toLocaleString('en-US')} | ${result.correctCount}/${result.totalCount} |`,
|
||||
).join('\n')
|
||||
|
||||
return `
|
||||
##### ${dataset.description}
|
||||
|
||||
| Format | Accuracy | Tokens | Correct/Total |
|
||||
| ------ | -------- | ------ | ------------- |
|
||||
${tableRows}
|
||||
`.trimStart()
|
||||
}).filter(Boolean).join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate question type breakdown table
|
||||
*/
|
||||
function generateQuestionTypeBreakdown(
|
||||
formatResults: FormatResult[],
|
||||
results: EvaluationResult[],
|
||||
questions: Question[],
|
||||
): string {
|
||||
// Build header
|
||||
const formatNames = formatResults.map(fr => FORMATTER_DISPLAY_NAMES[fr.format] || fr.format)
|
||||
const header = `| Question Type | ${formatNames.join(' | ')} |`
|
||||
const separator = `| ------------- | ${formatNames.map(() => '----').join(' | ')} |`
|
||||
|
||||
// Build rows
|
||||
const rows = QUESTION_TYPES.map((type) => {
|
||||
const questionIds = questions.filter(q => q.type === type).map(q => q.id)
|
||||
const typeResults = results.filter(r => questionIds.includes(r.questionId))
|
||||
|
||||
if (typeResults.length === 0)
|
||||
return undefined
|
||||
|
||||
const accuracies = formatResults.map((fr) => {
|
||||
const formatTypeResults = typeResults.filter(r => r.format === fr.format)
|
||||
if (formatTypeResults.length === 0)
|
||||
return 'N/A'
|
||||
|
||||
const correctCount = formatTypeResults.filter(r => r.isCorrect).length
|
||||
const totalCount = formatTypeResults.length
|
||||
const accuracy = totalCount > 0 ? correctCount / totalCount : 0
|
||||
return `${(accuracy * 100).toFixed(1)}%`
|
||||
})
|
||||
|
||||
return `| ${QUESTION_TYPE_LABELS[type]} | ${accuracies.join(' | ')} |`
|
||||
}).filter(Boolean)
|
||||
|
||||
return `
|
||||
${header}
|
||||
${separator}
|
||||
${rows.join('\n')}
|
||||
`.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate per-model performance comparison tables
|
||||
*/
|
||||
function generateModelPerformanceTable(
|
||||
formatResults: FormatResult[],
|
||||
results: EvaluationResult[],
|
||||
modelNames: string[],
|
||||
): string {
|
||||
return modelNames.map((modelName) => {
|
||||
const modelResults = formatResults.map((fr) => {
|
||||
const modelFormatResults = results.filter(r => r.model === modelName && r.format === fr.format)
|
||||
const correctCount = modelFormatResults.filter(r => r.isCorrect).length
|
||||
const totalCount = modelFormatResults.length
|
||||
const accuracy = correctCount / totalCount
|
||||
|
||||
return {
|
||||
format: fr.format,
|
||||
accuracy,
|
||||
correctCount,
|
||||
totalCount,
|
||||
}
|
||||
}).sort((a, b) => b.accuracy - a.accuracy)
|
||||
|
||||
const tableRows = modelResults.map(result =>
|
||||
`| \`${result.format}\` | ${(result.accuracy * 100).toFixed(1)}% | ${result.correctCount}/${result.totalCount} |`,
|
||||
).join('\n')
|
||||
|
||||
return `
|
||||
##### ${modelName}
|
||||
|
||||
| Format | Accuracy | Correct/Total |
|
||||
| ------ | -------- | ------------- |
|
||||
${tableRows}
|
||||
`.trimStart()
|
||||
}).join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate horizontal bar chart for efficiency ranking
|
||||
*/
|
||||
function generateHorizontalEfficiencyChart(
|
||||
ranking: EfficiencyRanking[],
|
||||
): string {
|
||||
const barWidth = 20
|
||||
const maxEfficiency = Math.max(...ranking.map(r => r.efficiency))
|
||||
const maxFormatWidth = Math.max(...ranking.map((r) => {
|
||||
const displayName = FORMATTER_DISPLAY_NAMES[r.format] || r.format
|
||||
return displayName.length
|
||||
}))
|
||||
|
||||
return ranking
|
||||
.map((r) => {
|
||||
const normalizedValue = r.efficiency / maxEfficiency
|
||||
const bar = createProgressBar(normalizedValue, 1, barWidth)
|
||||
const displayName = FORMATTER_DISPLAY_NAMES[r.format] || r.format
|
||||
const formatName = displayName.padEnd(maxFormatWidth)
|
||||
const efficiency = r.efficiency.toFixed(1).padStart(4)
|
||||
const accuracy = `${(r.accuracy * 100).toFixed(1)}%`.padStart(5)
|
||||
const tokens = r.tokens.toLocaleString('en-US').padStart(5)
|
||||
|
||||
return `${formatName} ${bar} ${efficiency} acc%/1K tok │ ${accuracy} acc │ ${tokens} tokens`
|
||||
})
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate vertical bar chart for efficiency ranking
|
||||
*/
|
||||
function generateVerticalEfficiencyChart(
|
||||
ranking: EfficiencyRanking[],
|
||||
): string {
|
||||
const maxEfficiency = Math.max(...ranking.map(r => r.efficiency))
|
||||
const chartHeight = 8
|
||||
|
||||
// Generate rows from top to bottom
|
||||
const rows: string[] = []
|
||||
|
||||
// Y-axis and bars
|
||||
for (let i = chartHeight; i >= 0; i--) {
|
||||
const threshold = (i / chartHeight) * maxEfficiency
|
||||
const yLabel = i === chartHeight || i === Math.floor(chartHeight / 2) || i === 0
|
||||
? Math.round(threshold).toString().padStart(4)
|
||||
: ' '
|
||||
|
||||
const bars = ranking
|
||||
.map((r) => {
|
||||
const barHeight = (r.efficiency / maxEfficiency) * chartHeight
|
||||
let char = ' '
|
||||
if (barHeight >= i) {
|
||||
// Use different characters for visual distinction
|
||||
if (ranking.indexOf(r) === 0)
|
||||
char = '▓' // Top format
|
||||
else if (ranking.indexOf(r) <= 2)
|
||||
char = '▒' // Top 3
|
||||
else
|
||||
char = '░' // Rest
|
||||
}
|
||||
return char
|
||||
})
|
||||
.join(' ')
|
||||
|
||||
rows.push(`${yLabel}│ ${bars}`)
|
||||
}
|
||||
|
||||
// X-axis
|
||||
const axis = ` └──${ranking.map(() => '┴').join('────')}──`
|
||||
rows.push(axis)
|
||||
|
||||
// Format labels (split long names into multiple rows)
|
||||
const formatRow1 = ranking
|
||||
.map((r) => {
|
||||
const parts = r.format.split('-')
|
||||
return (parts[0] || '').padEnd(5).substring(0, 5)
|
||||
})
|
||||
.join('')
|
||||
rows.push(` ${formatRow1}`)
|
||||
|
||||
const formatRow2 = ranking
|
||||
.map((r) => {
|
||||
const parts = r.format.split('-')
|
||||
return (parts[1] || '').padEnd(5).substring(0, 5)
|
||||
})
|
||||
.join('')
|
||||
if (formatRow2.trim())
|
||||
rows.push(` ${formatRow2}`)
|
||||
|
||||
return rows.join('\n')
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { Storage, StorageValue } from 'unstorage'
|
||||
import type { EvaluationResult } from './types.ts'
|
||||
import * as path from 'node:path'
|
||||
import { createStorage } from 'unstorage'
|
||||
import fsDriver from 'unstorage/drivers/fs'
|
||||
import { BENCHMARKS_DIR } from './constants.ts'
|
||||
|
||||
/**
|
||||
* Storage instance for model results
|
||||
*
|
||||
* @remarks
|
||||
* Stores results in: `benchmarks/results/accuracy/models/`
|
||||
*/
|
||||
export const resultsStorage: Storage<StorageValue> = createStorage({
|
||||
driver: fsDriver({
|
||||
base: path.join(BENCHMARKS_DIR, 'results', 'accuracy', 'models'),
|
||||
}),
|
||||
})
|
||||
|
||||
export async function loadModelResults(modelId: string): Promise<EvaluationResult[] | undefined> {
|
||||
const data = await resultsStorage.getItem<EvaluationResult[]>(modelId)
|
||||
return data ?? undefined
|
||||
}
|
||||
|
||||
export async function saveModelResults(modelId: string, results: EvaluationResult[]): Promise<void> {
|
||||
await resultsStorage.setItem(modelId, results)
|
||||
}
|
||||
|
||||
export async function getAllModelResults(): Promise<Record<string, EvaluationResult[]>> {
|
||||
const keys = await resultsStorage.getKeys()
|
||||
const results: Record<string, EvaluationResult[]> = {}
|
||||
|
||||
await Promise.all(
|
||||
keys.map(async (modelId) => {
|
||||
const data = await resultsStorage.getItem<EvaluationResult[]>(modelId)
|
||||
if (data)
|
||||
results[modelId] = data
|
||||
}),
|
||||
)
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
export async function hasModelResults(modelId: string): Promise<boolean> {
|
||||
return await resultsStorage.hasItem(modelId)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import type { DATASET_NAMES, QUESTION_TYPES, STRUCTURE_CLASSES } from './constants.ts'
|
||||
import type { AnswerType, NormalizationOptions } from './normalize.ts'
|
||||
|
||||
export type QuestionType = typeof QUESTION_TYPES[number]
|
||||
export type DatasetName = typeof DATASET_NAMES[number]
|
||||
export type StructureClass = typeof STRUCTURE_CLASSES[number]
|
||||
|
||||
export interface DatasetMetadata {
|
||||
supportsCSV: boolean
|
||||
structureClass: StructureClass
|
||||
tabularEligibility: number
|
||||
}
|
||||
|
||||
export interface Dataset {
|
||||
name: DatasetName
|
||||
description: string
|
||||
data: Record<string, any>
|
||||
metadata: DatasetMetadata
|
||||
}
|
||||
|
||||
export interface Question {
|
||||
id: string
|
||||
prompt: string
|
||||
groundTruth: string
|
||||
type: QuestionType
|
||||
dataset: DatasetName
|
||||
/**
|
||||
* Expected answer kind for deterministic comparison.
|
||||
* @default 'string'
|
||||
*/
|
||||
answerType?: AnswerType
|
||||
/**
|
||||
* Options for answer normalization and comparison.
|
||||
*/
|
||||
normalizationOptions?: Partial<NormalizationOptions>
|
||||
}
|
||||
|
||||
export interface EvaluationResult {
|
||||
questionId: string
|
||||
format: string
|
||||
model: string
|
||||
expected: string
|
||||
actual: string
|
||||
isCorrect: boolean
|
||||
inputTokens?: number
|
||||
outputTokens?: number
|
||||
latencyMs: number
|
||||
}
|
||||
|
||||
export interface FormatResult {
|
||||
format: string
|
||||
accuracy: number
|
||||
totalTokens: number
|
||||
averageLatency: number
|
||||
correctCount: number
|
||||
totalCount: number
|
||||
}
|
||||
|
||||
export interface EfficiencyRanking {
|
||||
format: string
|
||||
efficiency: number
|
||||
accuracy: number
|
||||
tokens: number
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import { encode } from 'gpt-tokenizer'
|
||||
|
||||
/**
|
||||
* Generate visual progress bar using ASCII characters
|
||||
*
|
||||
* @param value - Current value
|
||||
* @param max - Maximum value
|
||||
* @param width - Width of the bar in characters (default: 25)
|
||||
* @param chars - Characters to use for filled and empty sections
|
||||
* @param chars.filled - Character for filled portion (default: '█')
|
||||
* @param chars.empty - Character for empty portion (default: '░')
|
||||
* @returns ASCII progress bar string
|
||||
*
|
||||
* @example
|
||||
* createProgressBar(75, 100, 20) // "███████████████░░░░░"
|
||||
* createProgressBar(0.5, 1, 10) // "█████░░░░░"
|
||||
* createProgressBar(0.75, 1, 20, { filled: '▓', empty: '░' }) // "▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░"
|
||||
*/
|
||||
export function createProgressBar(
|
||||
value: number,
|
||||
max: number,
|
||||
width = 25,
|
||||
chars: { filled: string, empty: string } = { filled: '█', empty: '░' },
|
||||
): string {
|
||||
const filled = Math.round((value / max) * width)
|
||||
const empty = width - filled
|
||||
return chars.filled.repeat(filled) + chars.empty.repeat(empty)
|
||||
}
|
||||
|
||||
/**
|
||||
* Count tokens in text using gpt-tokenizer (o200k_base encoding)
|
||||
*
|
||||
* @param text - Text to tokenize
|
||||
* @returns Number of tokens
|
||||
*
|
||||
* @example
|
||||
* tokenize("Hello, world!") // 4
|
||||
*/
|
||||
export function tokenize(text: string): number {
|
||||
return encode(text).length
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it recursively if needed
|
||||
*
|
||||
* @param dirPath - Directory path to ensure exists
|
||||
*/
|
||||
export async function ensureDir(dirPath: string): Promise<void> {
|
||||
await fsp.mkdir(dirPath, { recursive: true })
|
||||
}
|
||||
Reference in New Issue
Block a user