chore: import upstream snapshot with attribution

benchmarks/scripts/accuracy-benchmark.ts

@@ -0,0 +1,207 @@
+import type { Question } from '../src/types.ts'
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import process from 'node:process'
+import * as prompts from '@clack/prompts'
+import PQueue from 'p-queue'
+import { BENCHMARKS_DIR, DEFAULT_CONCURRENCY, DRY_RUN, DRY_RUN_LIMITS, MODEL_RPM_LIMITS, ROOT_DIR } from '../src/constants.ts'
+import { ACCURACY_DATASETS } from '../src/datasets.ts'
+import { evaluateQuestion, models } from '../src/evaluate.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { generateQuestions } from '../src/questions/index.ts'
+import { calculateFormatResults, calculateTokenCounts, generateAccuracyReport } from '../src/report.ts'
+import { getAllModelResults, hasModelResults, saveModelResults } from '../src/storage.ts'
+import { ensureDir } from '../src/utils.ts'
+
+// Constants
+const PROGRESS_UPDATE_INTERVAL = 10
+const RATE_LIMIT_INTERVAL_MS = 60_000
+
+prompts.intro('Retrieval Accuracy Benchmark')
+
+/**
+ * Generate evaluation tasks for a model
+ */
+function generateEvaluationTasks(questions: Question[]): { question: Question, formatName: string }[] {
+  const tasks: { question: Question, formatName: string }[] = []
+
+  for (const question of questions) {
+    for (const [formatName] of Object.entries(formatters)) {
+      // Skip CSV for datasets that don't support it
+      const dataset = ACCURACY_DATASETS.find(d => d.name === question.dataset)
+      if (formatName === 'csv' && dataset && !supportsCSV(dataset))
+        continue
+
+      tasks.push({ question, formatName })
+    }
+  }
+
+  return tasks
+}
+
+/**
+ * Check which models already have saved results
+ */
+async function checkExistingResults(activeModels: typeof models) {
+  const existingModelResults: Record<string, boolean> = {}
+
+  for (const model of activeModels) {
+    const existingResult = await hasModelResults(model.modelId)
+    if (existingResult)
+      existingModelResults[model.modelId] = existingResult
+  }
+
+  return existingModelResults
+}
+
+/**
+ * Create a progress updater function
+ */
+function createProgressUpdater(spinner: ReturnType<typeof prompts.spinner>, total: number) {
+  let completed = 0
+
+  return () => {
+    completed++
+    if (completed % PROGRESS_UPDATE_INTERVAL === 0 || completed === total) {
+      const percent = ((completed / total) * 100).toFixed(1)
+      spinner.message(`Progress: ${completed}/${total} (${percent}%)`)
+    }
+  }
+}
+
+/**
+ * Create a rate-limited queue for model evaluation
+ */
+function createEvaluationQueue(modelId: string) {
+  const rpmLimit = MODEL_RPM_LIMITS[modelId]
+
+  return new PQueue({
+    concurrency: DEFAULT_CONCURRENCY,
+    intervalCap: rpmLimit ?? Infinity,
+    interval: rpmLimit ? RATE_LIMIT_INTERVAL_MS : 0,
+  })
+}
+
+// Prompt user to select which models to benchmark
+const modelChoices = models.map(({ modelId }) => ({
+  value: modelId,
+  label: modelId,
+}))
+
+const selectedModels = await prompts.multiselect({
+  message: 'Select models to benchmark (Space to select, Enter to confirm)',
+  options: modelChoices,
+  required: true,
+})
+
+if (prompts.isCancel(selectedModels)) {
+  prompts.cancel('Benchmark cancelled')
+  process.exit(0)
+}
+
+const activeModels = models.filter(m => selectedModels.includes(m.modelId))
+
+prompts.log.info(`Selected ${activeModels.length} model(s): ${activeModels.map(m => m.modelId).join(', ')}`)
+
+// Check which models already have results
+const existingModelResults = await checkExistingResults(activeModels)
+
+if (Object.keys(existingModelResults).length > 0) {
+  prompts.log.info(`Found existing results for ${Object.keys(existingModelResults).length} model(s)`)
+}
+
+if (DRY_RUN) {
+  prompts.log.info('Limiting questions and models for dry run')
+}
+
+let questions = generateQuestions()
+
+// Apply dry run limits if enabled
+if (DRY_RUN && DRY_RUN_LIMITS.maxQuestions) {
+  questions = questions.slice(0, DRY_RUN_LIMITS.maxQuestions)
+}
+
+prompts.log.info(`Evaluating ${questions.length} questions`)
+prompts.log.info(`Testing ${Object.keys(formatters).length} formats`)
+
+// Evaluate each model separately and save results incrementally
+for (const model of activeModels) {
+  const modelId = model.modelId
+
+  // Skip if results already exist
+  if (existingModelResults[modelId]) {
+    prompts.log.info(`Skipping ${modelId} (results already exist)`)
+    continue
+  }
+
+  prompts.log.step(`Running benchmark for ${modelId}`)
+
+  // Generate evaluation tasks for this model
+  const tasks = generateEvaluationTasks(questions)
+
+  const total = tasks.length
+  const rpmLimit = MODEL_RPM_LIMITS[modelId]
+  const queue = createEvaluationQueue(modelId)
+
+  const evalSpinner = prompts.spinner()
+  evalSpinner.start(`Running ${total} evaluations (concurrency: ${DEFAULT_CONCURRENCY}, RPM limit: ${rpmLimit ?? 'unlimited'})`)
+
+  const updateProgress = createProgressUpdater(evalSpinner, total)
+
+  // Queue all tasks
+  const modelResultPromises = tasks.map(task =>
+    queue.add(async () => {
+      // Format data on-demand
+      const dataset = ACCURACY_DATASETS.find(d => d.name === task.question.dataset)!
+      const formatter = formatters[task.formatName]!
+      const formattedData = formatter(dataset.data)
+
+      const result = await evaluateQuestion({
+        question: task.question,
+        formatName: task.formatName,
+        formattedData,
+        model,
+      })
+
+      // Progress update after task completes
+      updateProgress()
+
+      return result
+    }),
+  )
+
+  // Wait for all tasks to complete
+  const modelResults = await Promise.all(modelResultPromises)
+
+  evalSpinner.stop(`Evaluation complete for ${modelId}`)
+
+  // Save results immediately for this model
+  await saveModelResults(modelId, modelResults)
+  prompts.log.success(`Saved results for ${modelId}`)
+}
+
+// Generate/regenerate markdown report from all available model results
+const reportSpinner = prompts.spinner()
+reportSpinner.start('Generating report from all model results')
+
+// Load all available model results (including any that were skipped)
+const allModelResults = await getAllModelResults()
+const allResults = Object.values(allModelResults).flat()
+
+if (allResults.length === 0) {
+  prompts.log.warn('No results available to generate report')
+  process.exit(0)
+}
+
+const tokenCounts = calculateTokenCounts(formatters)
+const formatResults = calculateFormatResults(allResults, tokenCounts)
+const accuracyReport = generateAccuracyReport(allResults, formatResults, tokenCounts)
+
+const resultsDir = path.join(BENCHMARKS_DIR, 'results')
+await ensureDir(resultsDir)
+
+const outputFilePath = path.join(resultsDir, 'retrieval-accuracy.md')
+await fsp.writeFile(outputFilePath, accuracyReport)
+
+reportSpinner.stop('Report generation complete!')
+prompts.log.info(`Report saved to: \`${path.relative(ROOT_DIR, outputFilePath)}\``)

benchmarks/scripts/fetch-github-repos.ts

@@ -0,0 +1,88 @@
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import process from 'node:process'
+import * as prompts from '@clack/prompts'
+import { ofetch } from 'ofetch'
+import pMap from 'p-map'
+import { BENCHMARKS_DIR } from '../src/constants.ts'
+import { ensureDir } from '../src/utils.ts'
+
+prompts.intro('GitHub Repositories Fetcher')
+
+try {
+  // Fetch top 100 repos from GitHub
+  const repoList = await searchTop100Repos()
+  const repos = await fetchRepoDetails(repoList)
+
+  if (repos.length === 0) {
+    prompts.log.error('No repositories fetched. Exiting.')
+    process.exit(1)
+  }
+
+  // Sort by stars descending
+  repos.sort((a, b) => b.stars - a.stars)
+
+  await saveRepos(repos)
+
+  prompts.log.success('Done!')
+}
+catch (error) {
+  prompts.log.error(String(error))
+  process.exit(1)
+}
+
+async function searchTop100Repos(): Promise<string[]> {
+  const s = prompts.spinner()
+  s.start('Fetching top 100 starred repositories')
+
+  const response = await ofetch<{ items: { full_name: string }[] }>(
+    'https://api.github.com/search/repositories',
+    {
+      query: {
+        q: 'stars:>1',
+        sort: 'stars',
+        order: 'desc',
+        per_page: 100,
+      },
+      headers: {
+        'Accept': 'application/vnd.github+json',
+        'X-GitHub-Api-Version': '2022-11-28',
+      },
+    },
+  )
+
+  s.stop('Fetched top 100 repositories')
+
+  return response.items.map(item => item.full_name)
+}
+
+async function fetchRepoDetails(repoList: string[]): Promise<Record<string, any>[]> {
+  const s = prompts.spinner()
+  s.start(`Fetching ${repoList.length} GitHub repositories`)
+
+  const repos = await pMap(
+    repoList,
+    async (repoPath, index) => {
+      s.message(`[${index + 1}/${repoList.length}] Fetching ${repoPath}`)
+      const { repo } = await ofetch(`https://ungh.cc/repos/${repoPath}`)
+      return repo
+    },
+    { concurrency: 5 },
+  )
+
+  s.stop(`Successfully fetched ${repos.length}/${repoList.length} repositories`)
+
+  return repos
+}
+
+async function saveRepos(repos: Record<string, any>[]): Promise<void> {
+  const outputDir = path.join(BENCHMARKS_DIR, 'data')
+  const outputFile = path.join(outputDir, 'github-repos.json')
+
+  await ensureDir(outputDir)
+  const jsonOutput = JSON.stringify(repos, undefined, 2)
+  await fsp.writeFile(outputFile, `${jsonOutput}\n`, 'utf-8')
+
+  const relativePath = path.relative(BENCHMARKS_DIR, outputFile)
+  prompts.log.info(`Result saved to \`${relativePath}\``)
+}

benchmarks/scripts/token-efficiency-benchmark.ts

@@ -0,0 +1,349 @@
+import type { Dataset } from '../src/types.ts'
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import * as prompts from '@clack/prompts'
+import { encode } from '../../packages/toon/src/index.ts'
+import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants.ts'
+import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets.ts'
+import { formatters, supportsCSV } from '../src/formatters.ts'
+import { createProgressBar, ensureDir, tokenize } from '../src/utils.ts'
+
+interface FormatMetrics {
+  name: string
+  tokens: number
+  savings: number
+  savingsPercent: number
+}
+
+interface BenchmarkResult {
+  dataset: Dataset
+  formats: FormatMetrics[]
+}
+
+// Constants
+const DATASET_ICONS: Record<string, string> = {
+  'tabular': '👥',
+  'nested': '🛒',
+  'analytics': '📈',
+  'github': '⭐',
+  'event-logs': '🧾',
+  'nested-config': '🧩',
+}
+
+const COMPARISON_FORMAT_ORDER = ['json-pretty', 'json-compact', 'yaml', 'xml'] as const
+
+const PROGRESS_BAR_WIDTH = 20
+const TOKEN_PADDING = 7
+
+const DEFAULT_DATASET_ICON = '📊'
+
+const DETAILED_EXAMPLE_DATASETS = ['github', 'analytics'] as const
+const GITHUB_REPO_LIMIT = 3
+const GITHUB_DESC_LIMIT = 80
+const ANALYTICS_METRICS_LIMIT = 5
+
+prompts.intro('Token Efficiency Benchmark')
+
+/**
+ * Format a comparison line showing savings vs TOON
+ */
+function formatComparisonLine(format: FormatMetrics, isLast: boolean = false): string {
+  const label = FORMATTER_DISPLAY_NAMES[format.name] || format.name.toUpperCase()
+  const signedPercent = format.savingsPercent >= 0
+    ? `−${format.savingsPercent.toFixed(1)}%`
+    : `+${Math.abs(format.savingsPercent).toFixed(1)}%`
+  const connector = isLast ? '└─' : '├─'
+  const tokenStr = format.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+  return `${connector} vs ${label.padEnd(13)} ${`(${signedPercent})`.padEnd(20)}   ${tokenStr} tokens`
+}
+
+/**
+ * Calculate total tokens and savings for a set of datasets
+ */
+function calculateTotalMetrics(datasets: BenchmarkResult[], formatNames: readonly string[]) {
+  const totalToonTokens = datasets.reduce((sum, r) => {
+    const toon = r.formats.find(f => f.name === 'toon')!
+    return sum + toon.tokens
+  }, 0)
+
+  const totals = formatNames.map((formatName) => {
+    const totalTokens = datasets.reduce((sum, r) => {
+      const format = r.formats.find(f => f.name === formatName)
+      return sum + (format?.tokens || 0)
+    }, 0)
+    const savings = totalTokens - totalToonTokens
+    const savingsPercent = (savings / totalTokens) * 100
+    return { name: formatName, tokens: totalTokens, savingsPercent }
+  })
+
+  return { totalToonTokens, totals }
+}
+
+/**
+ * Generate total lines for a track
+ */
+function generateTotalLines(
+  totalToonTokens: number,
+  totals: { name: string, tokens: number, savingsPercent: number }[],
+  baselineFormat?: { name: string, tokens: number },
+) {
+  const separatorHalf = '─'.repeat(36)
+  const lines = [`${separatorHalf} Total ${separatorHalf}`]
+
+  if (baselineFormat) {
+    // Flat-only track with CSV baseline
+    const csvPercentage = Math.min(100, (baselineFormat.tokens / totalToonTokens) * 100)
+    const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
+    const csvStr = baselineFormat.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   CSV                 ${csvBar}   ${csvStr} tokens`)
+
+    const overheadPercent = ((totalToonTokens - baselineFormat.tokens) / baselineFormat.tokens) * 100
+    const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
+    const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   TOON                ${toonBar}   ${toonStr} tokens   (+${overheadPercent.toFixed(1)}% vs CSV)`)
+  }
+  else {
+    // Mixed-structure track
+    const totalPercentage = Math.min(100, (totalToonTokens / totals[0]!.tokens) * 100)
+    const totalBar = createProgressBar(totalPercentage, 100, PROGRESS_BAR_WIDTH)
+    const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
+    lines.push(`   TOON                ${totalBar}   ${toonStr} tokens`)
+  }
+
+  // Add comparison lines
+  for (let i = 0; i < totals.length; i++) {
+    const format = totals[i]!
+    const isLast = i === totals.length - 1
+    lines.push(`   ${formatComparisonLine({
+      name: format.name,
+      tokens: format.tokens,
+      savings: 0, // Not used in this context
+      savingsPercent: format.savingsPercent,
+    }, isLast)}`)
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Generate bar chart for a dataset
+ */
+function generateDatasetChart(result: BenchmarkResult): string {
+  const { dataset, formats } = result
+  const toon = formats.find(f => f.name === 'toon')!
+  const jsonPretty = formats.find(f => f.name === 'json-pretty')!
+
+  const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
+  const eligibility = dataset.metadata.tabularEligibility
+  const name = dataset.description
+
+  const percentage = Math.min(100, 100 - jsonPretty.savingsPercent)
+  const bar = createProgressBar(percentage, 100, PROGRESS_BAR_WIDTH)
+  const toonStr = toon.tokens.toLocaleString('en-US')
+
+  const line1 = `${emoji} ${name}  ┊  Tabular: ${eligibility}%`
+  const line2 = `   │`
+  const line3 = `   TOON                ${bar}   ${toonStr.padStart(TOKEN_PADDING)} tokens`
+
+  const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
+    const format = formats.find(f => f.name === formatName)
+    if (!format)
+      return undefined
+
+    return `   ${formatComparisonLine(format, index === array.length - 1)}`
+  }).filter(Boolean)
+
+  return [line1, line2, line3, ...comparisonLines].join('\n')
+}
+
+const results: BenchmarkResult[] = []
+
+// Calculate token counts for all datasets
+for (const dataset of TOKEN_EFFICIENCY_DATASETS) {
+  const formatMetrics: FormatMetrics[] = []
+  const tokensByFormat: Record<string, number> = {}
+
+  // Calculate tokens for each format
+  for (const [formatName, formatter] of Object.entries(formatters)) {
+    // Skip CSV for datasets that don't support it
+    if (formatName === 'csv' && !supportsCSV(dataset))
+      continue
+
+    const formattedData = formatter(dataset.data)
+    const tokens = tokenize(formattedData)
+    tokensByFormat[formatName] = tokens
+  }
+
+  // Calculate savings vs TOON
+  const toonTokens = tokensByFormat.toon!
+  for (const [formatName, tokens] of Object.entries(tokensByFormat)) {
+    const savings = tokens - toonTokens
+    formatMetrics.push({
+      name: formatName,
+      tokens,
+      savings,
+      savingsPercent: formatName === 'toon' ? 0 : (savings / tokens) * 100,
+    })
+  }
+
+  results.push({
+    dataset,
+    formats: formatMetrics,
+  })
+}
+
+// Separate datasets by CSV support
+const mixedStructureDatasets = results.filter(r => !supportsCSV(r.dataset))
+const flatOnlyDatasets = results.filter(r => supportsCSV(r.dataset))
+
+// Mixed-Structure Track (no CSV)
+const mixedCharts = mixedStructureDatasets
+  .map(result => generateDatasetChart(result))
+  .join('\n\n')
+
+// Flat-Only Track (with CSV)
+const flatCharts = flatOnlyDatasets
+  .map((result) => {
+    const csv = result.formats.find(f => f.name === 'csv')
+    const toon = result.formats.find(f => f.name === 'toon')!
+
+    if (!csv)
+      return generateDatasetChart(result)
+
+    // Special handling to show CSV first with TOON overhead
+    const { dataset } = result
+    const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
+    const eligibility = dataset.metadata.tabularEligibility
+    const name = dataset.description
+
+    // CSV line
+    const csvPercentage = Math.min(100, (csv.tokens / toon.tokens) * 100)
+    const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH)
+    const csvStr = csv.tokens.toLocaleString('en-US')
+
+    const line1 = `${emoji} ${name}  ┊  Tabular: ${eligibility}%`
+    const line2 = `   │`
+    const line3 = `   CSV                 ${csvBar}   ${csvStr.padStart(TOKEN_PADDING)} tokens`
+
+    const toonOverhead = toon.tokens - csv.tokens
+    const toonOverheadPercent = (toonOverhead / csv.tokens) * 100
+    const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH)
+    const toonStr = toon.tokens.toLocaleString('en-US')
+    const toonVsCSV = toonOverheadPercent >= 0
+      ? `(+${toonOverheadPercent.toFixed(1)}% vs CSV)`
+      : `(${toonOverheadPercent.toFixed(1)}% vs CSV)`
+    const toonLine = `   TOON                ${toonBar}   ${toonStr.padStart(TOKEN_PADDING)} tokens   ${toonVsCSV}`
+
+    // Other format comparisons (vs TOON)
+    const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName, index, array) => {
+      const format = result.formats.find(f => f.name === formatName)
+      if (!format)
+        return undefined
+
+      return `   ${formatComparisonLine(format, index === array.length - 1)}`
+    }).filter(Boolean)
+
+    return [line1, line2, line3, toonLine, ...comparisonLines].join('\n')
+  })
+  .join('\n\n')
+
+// Calculate totals for mixed structure
+const { totalToonTokens: totalToonTokensMixed, totals: mixedTotals } = calculateTotalMetrics(mixedStructureDatasets, COMPARISON_FORMAT_ORDER)
+const mixedTotalLines = generateTotalLines(totalToonTokensMixed, mixedTotals)
+
+// Calculate totals for flat-only
+const { totalToonTokens: totalToonTokensFlat, totals: flatTotals } = calculateTotalMetrics(flatOnlyDatasets, COMPARISON_FORMAT_ORDER)
+const totalCSVTokensFlat = flatOnlyDatasets.reduce((sum, r) => {
+  const csv = r.formats.find(f => f.name === 'csv')
+  return sum + (csv?.tokens || 0)
+}, 0)
+const flatTotalLines = generateTotalLines(totalToonTokensFlat, flatTotals, { name: 'csv', tokens: totalCSVTokensFlat })
+
+const barChartSection = `
+#### Mixed-Structure Track
+
+Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
+
+\`\`\`
+${mixedCharts}
+
+${mixedTotalLines}
+\`\`\`
+
+#### Flat-Only Track
+
+Datasets with flat tabular structures where CSV is applicable.
+
+\`\`\`
+${flatCharts}
+
+${flatTotalLines}
+\`\`\`
+`.trim()
+
+// Generate detailed examples (optional: show a few examples)
+const detailedExamples = results
+  .filter(r => DETAILED_EXAMPLE_DATASETS.includes(r.dataset.name as any))
+  .map((result, i, filtered) => {
+    let displayData = result.dataset.data
+
+    // Truncate for display
+    if (result.dataset.name === 'github') {
+      displayData = {
+        repositories: displayData.repositories.slice(0, GITHUB_REPO_LIMIT).map((repo: Record<string, any>) => ({
+          ...repo,
+          description: repo.description?.slice(0, GITHUB_DESC_LIMIT) + (repo.description?.length > GITHUB_DESC_LIMIT ? '…' : ''),
+        })),
+      }
+    }
+    else if (result.dataset.name === 'analytics') {
+      displayData = { metrics: displayData.metrics.slice(0, ANALYTICS_METRICS_LIMIT) }
+    }
+
+    const emoji = DATASET_ICONS[result.dataset.name] || DEFAULT_DATASET_ICON
+    const json = result.formats.find(f => f.name === 'json-pretty')!
+    const toon = result.formats.find(f => f.name === 'toon')!
+    const separator = i < filtered.length - 1 ? '---' : ''
+
+    return `
+#### ${emoji} ${result.dataset.description}
+
+**Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent.toFixed(1)}% reduction vs JSON)
+
+**JSON** (${json.tokens.toLocaleString('en-US')} tokens):
+
+\`\`\`json
+${JSON.stringify(displayData, undefined, 2)}
+\`\`\`
+
+**TOON** (${toon.tokens.toLocaleString('en-US')} tokens):
+
+\`\`\`
+${encode(displayData)}
+\`\`\`
+
+${separator}
+`.trim()
+  })
+  .join('\n\n')
+
+const markdown = `
+${barChartSection}
+
+<details>
+<summary><strong>Show detailed examples</strong></summary>
+
+${detailedExamples}
+
+</details>
+`.trimStart()
+
+prompts.log.message(barChartSection)
+
+const resultsDir = path.join(BENCHMARKS_DIR, 'results')
+await ensureDir(resultsDir)
+
+const outputFilePath = path.join(resultsDir, 'token-efficiency.md')
+await fsp.writeFile(outputFilePath, markdown, 'utf-8')
+
+prompts.log.success(`Report saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)