vercel-labs--zerolang/scripts/bench.mts

#!/usr/bin/env -S node --experimental-strip-types --disable-warning=ExperimentalWarning
import { spawnSync } from "node:child_process";
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { performance } from "node:perf_hooks";
import { gzipSync } from "node:zlib";

const root = process.cwd();
const outDir = join(root, ".zero", "bench");
const runtimeOutDir = join(root, ".zero", "out");
mkdirSync(outDir, { recursive: true });
mkdirSync(runtimeOutDir, { recursive: true });
cleanStaleGeneratedCOutputs();
loadDotEnvFiles([".zero/bench/snapshot.env", ".env", ".env.local"]);

const args = process.argv.slice(2);
const workerMode = args.includes("--worker");
const modeArg = flagValue("--mode");
const mode = modeArg ?? process.env.ZERO_BENCH_MODE ?? "local";
const runCount = Number.parseInt(process.env.ZERO_BENCH_RUNS ?? "5", 10);
const budgetMode = process.env.ZERO_BENCH_BUDGET ?? "off";
const directZeroMinimum = 18;

const cases = [
  { name: "baseline-hello", expectedStdout: "baseline hello", languages: ["zero"], sources: { zero: "benchmarks/zero/baseline-hello.0" } },
  { name: "baseline-file-cli", expectedStdout: "cli file ok", languages: ["zero"], skipLanguages: ["zero"], sources: { zero: "examples/cli-file.0" }, args: ["write"], env: { ZERO_CLI_FILE_MODE: "quiet" }, skipReason: "hosted filesystem CLI direct lowering remains a tracked std.fs breadth gap" },
  { name: "baseline-memory-package", expectedStdout: "memory package ok", languages: ["zero"], sources: { zero: "examples/memory-package" } },
  { name: "hello", expectedStdout: "hello benchmark" },
  { name: "add", expectedStdout: "math works" },
  { name: "structs", expectedStdout: "42" },
  { name: "params", expectedStdout: "42" },
  { name: "buffers", expectedStdout: "buffer" },
  { name: "parser", expectedStdout: "ident" },
  { name: "fileio", expectedStdout: "fileio" },
  { name: "codec", expectedStdout: "codec" },
  { name: "parse", expectedStdout: "parse" },
  { name: "slices", expectedStdout: "slices" },
  { name: "arena", expectedStdout: "arena" },
  { name: "fallibility", expectedStdout: "fallibility" },
  { name: "readall", expectedStdout: "readall", skipLanguages: ["zero"] },
  { name: "branches", expectedStdout: "branches" },
  { name: "module-package", expectedStdout: "module package", languages: ["zero"], sources: { zero: "benchmarks/zero/module-package" } },
  { name: "owned-file", expectedStdout: "owned file", languages: ["zero"], skipLanguages: ["zero"] },
  { name: "rescue", expectedStdout: "rescue", languages: ["zero"] },
  { name: "fs-resource", expectedStdout: "fs resource", languages: ["zero"], skipLanguages: ["zero"] },
  { name: "mem-copy-fill", expectedStdout: "mem copy fill", languages: ["zero"] },
  { name: "zero-hash", expectedStdout: "zero-hash ok", languages: ["zero"] },
  { name: "json", expectedStdout: "json", languages: ["zero"] },
  { name: "networking", expectedStdout: "networking", languages: ["zero"], skipLanguages: ["zero"] },
];

const languages = [
  {
    name: "zero",
    tool: "bin/zero",
    build: (benchCase) => ["bin/zero", ["build", "--json", "--emit", "exe", "--target", directZeroTarget(), graphInputPath(sourcePath(benchCase, "zero", ".0")), "--out", `.zero/bench/zero-${benchCase.name}`]],
    exe: (benchCase) => `.zero/bench/zero-${benchCase.name}`,
  },
];

const languageByName = new Map(languages.map((language) => [language.name, language]));
const caseByName = new Map(cases.map((benchCase) => [benchCase.name, benchCase]));

function cleanStaleGeneratedCOutputs() {
  for (const entry of readdirSync(outDir)) {
    if (/^zero-.+\.c$/.test(entry)) rmSync(join(outDir, entry), { force: true });
  }
}

function loadDotEnvFiles(paths) {
  for (const path of paths) {
    const fullPath = join(root, path);
    if (!existsSync(fullPath)) continue;
    const lines = readFileSync(fullPath, "utf8").split(/\r?\n/);
    for (const line of lines) {
      const trimmed = line.trim();
      if (trimmed === "" || trimmed.startsWith("#")) continue;
      const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(trimmed);
      if (!match || process.env[match[1]] !== undefined) continue;
      process.env[match[1]] = parseDotEnvValue(match[2]);
    }
  }
}

function parseDotEnvValue(value) {
  const trimmed = value.trim();
  const quote = trimmed[0];
  if ((quote === "\"" || quote === "'") && trimmed.endsWith(quote)) {
    const inner = trimmed.slice(1, -1);
    return quote === "\"" ? inner.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "\t").replace(/\\"/g, "\"").replace(/\\\\/g, "\\") : inner;
  }
  const commentStart = trimmed.search(/\s#/);
  return commentStart === -1 ? trimmed : trimmed.slice(0, commentStart).trimEnd();
}

function flagValue(name) {
  const index = args.indexOf(name);
  if (index === -1) return null;
  return args[index + 1] ?? null;
}

function sourcePath(benchCase, language, extension) {
  if (benchCase.sources?.[language]) return benchCase.sources[language];
  return `benchmarks/${language}/${benchCase.name}${extension}`;
}

function graphInputPath(path) {
  if (!path.endsWith(".0")) return path;
  const graph = `${path.slice(0, -2)}.graph`;
  if (!existsSync(graph)) throw new Error(`${path}: missing graph sidecar ${graph}; benchmark compiler input must be graph-backed`);
  return graph;
}

function directZeroTarget() {
  if (process.platform === "darwin" && process.arch === "arm64") return "darwin-arm64";
  if (process.platform === "darwin" && process.arch === "x64") return "darwin-x64";
  if (process.platform === "linux" && process.arch === "x64") return "linux-musl-x64";
  if (process.platform === "linux" && process.arch === "arm64") return "linux-arm64";
  return null;
}

function zeroDirectCapable() {
  return directZeroTarget() !== null;
}

function hasTool(command) {
  if (command.includes("/")) {
    return spawnSync(command, ["--help"], { stdio: "ignore" }).status === 0 || spawnSync(command, [], { stdio: "ignore" }).status !== null;
  }
  return spawnSync("sh", ["-c", `command -v ${command}`], { stdio: "ignore" }).status === 0;
}

function timed(command, commandArgs) {
  const started = performance.now();
  const result = spawnSync(command, commandArgs, { encoding: "utf8" });
  const elapsedMs = performance.now() - started;
  return { result, elapsedMs: Number(elapsedMs.toFixed(3)) };
}

function parseJsonOutput(stdout) {
  try {
    return JSON.parse(stdout || "{}");
  } catch {
    return null;
  }
}

function toolVersion(command, commandArgs) {
  const result = spawnSync(command, commandArgs, { encoding: "utf8" });
  if (result.error || result.status !== 0) return null;
  return (result.stdout ?? "").trim().split("\n")[0] || null;
}

function timedRun(command, commandArgs = [], env = {}) {
  const runEnv = { ...process.env, ...env };
  if (process.platform === "darwin") {
    const started = performance.now();
    const result = spawnSync("/usr/bin/time", ["-l", command, ...commandArgs], { encoding: "utf8", env: runEnv });
    const elapsedMs = performance.now() - started;
    if (result.status !== 0 && result.stderr.includes("sysctl kern.clockrate")) {
      const fallback = timed(command, commandArgs);
      return {
        result: fallback.result,
        elapsedMs: fallback.elapsedMs,
        peakRssBytes: null,
        stdout: fallback.result.stdout.trim(),
        stderr: fallback.result.stderr.trim(),
      };
    }
    const rssMatch = /(\d+)\s+maximum resident set size/.exec(result.stderr);
    return {
      result,
      elapsedMs: Number(elapsedMs.toFixed(3)),
      peakRssBytes: rssMatch ? Number(rssMatch[1]) : null,
      stdout: result.stdout.trim(),
      stderr: result.stderr.replace(/\s*\d+\s+maximum resident set size[\s\S]*$/m, "").trim(),
    };
  }
  const run = timed(command, commandArgs);
  return {
    result: run.result,
    elapsedMs: run.elapsedMs,
    peakRssBytes: null,
    stdout: run.result.stdout.trim(),
    stderr: run.result.stderr.trim(),
  };
}

function fileSize(path) {
  try {
    return statSync(path).size;
  } catch {
    return null;
  }
}

function gzipFileSize(path) {
  try {
    return gzipSync(readFileSync(path)).length;
  } catch {
    return null;
  }
}

function median(values) {
  if (values.length === 0) return null;
  const sorted = [...values].sort((left, right) => left - right);
  return sorted[Math.floor(sorted.length / 2)];
}

function envNumber(name, fallback) {
  const raw = process.env[name];
  if (!raw) return fallback;
  const parsed = Number(raw);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}

function allJobs() {
  const jobs = [];
  for (const language of languages) {
    for (const benchCase of cases) jobs.push({ language: language.name, case: benchCase.name });
  }
  return jobs;
}

function isCaseDefinedForLanguage(benchCase, language) {
  if (benchCase.skipLanguages?.includes(language.name)) return false;
  return !benchCase.languages || benchCase.languages.includes(language.name);
}

function executableJobs() {
  return allJobs().filter((job) => {
    const language = languageByName.get(job.language);
    const benchCase = caseByName.get(job.case);
    return language && benchCase && isCaseDefinedForLanguage(benchCase, language);
  });
}

function skippedResults() {
  return allJobs().flatMap((job) => {
    const language = languageByName.get(job.language);
    const benchCase = caseByName.get(job.case);
    if (!language || !benchCase || isCaseDefinedForLanguage(benchCase, language)) return [];
    const reason = benchCase.skipLanguages?.includes(language.name)
      ? (benchCase.skipReason ?? "case is skipped for this benchmark target")
      : "case is not defined for this benchmark target";
    return [{ language: language.name, case: benchCase.name, status: "skipped", reason }];
  });
}

function sortResults(results) {
  const order = new Map(allJobs().map((job, index) => [`${job.language}:${job.case}`, index]));
  return [...results].sort((left, right) => {
    const leftOrder = order.get(`${left.language}:${left.case}`) ?? Number.MAX_SAFE_INTEGER;
    const rightOrder = order.get(`${right.language}:${right.case}`) ?? Number.MAX_SAFE_INTEGER;
    return leftOrder - rightOrder;
  });
}

function parseWorkerJobs() {
  const encoded = flagValue("--jobs");
  if (!encoded) return allJobs();
  return JSON.parse(Buffer.from(encoded, "base64url").toString("utf8"));
}

function runJobs(jobs) {
  const results = [];
  for (const job of jobs) {
    const language = languageByName.get(job.language);
    const benchCase = caseByName.get(job.case);
    if (!language || !benchCase) throw new Error(`unknown benchmark job ${JSON.stringify(job)}`);
    results.push(runCase(language, benchCase));
  }
  return results;
}

function runCase(language, benchCase) {
  const name = benchCase.name;
  if (!isCaseDefinedForLanguage(benchCase, language)) {
    return { language: language.name, case: name, status: "skipped", reason: "case is not defined for this language" };
  }
  if (!hasTool(language.tool)) {
    return { language: language.name, case: name, status: "missing", tool: language.tool };
  }

  const exe = language.exe(benchCase);
  if (language.name === "zero") {
    rmSync(exe, { force: true });
    rmSync(`${exe}.c`, { force: true });
  }

  const directZero = language.name === "zero" && zeroDirectCapable();
  const backendMode = directZero ? "direct" : "native-toolchain";
  let build;
  let buildBody = null;
  if (directZero) {
    const [buildCommand, buildArgs] = language.build(benchCase);
    build = timed(buildCommand, buildArgs);
    buildBody = parseJsonOutput(build.result.stdout);
  } else {
    const [buildCommand, buildArgs] = language.build(benchCase);
    build = timed(buildCommand, buildArgs);
  }
  if (build.result.status !== 0) {
    const diagnosticCode = buildBody?.diagnostics?.[0]?.code ?? null;
    if (language.name === "zero" && diagnosticCode === "BLD004") {
      return {
        language: language.name,
        case: name,
        status: "skipped",
        reason: `direct executable runner unavailable for this case on ${directZeroTarget()}`,
        backendMode,
        target: directZero ? directZeroTarget() : null,
        buildMs: build.elapsedMs,
        buildJson: buildBody,
      };
    }
    return {
      language: language.name,
      case: name,
      status: "build-failed",
      backendMode,
      buildMs: build.elapsedMs,
      stderr: build.result.stderr.trim(),
      stdout: build.result.stdout?.trim(),
      buildJson: buildBody,
    };
  }

  if (directZero) {
    try {
      chmodSync(exe, 0o755);
    } catch {
      // The direct linker normally writes executable mode; chmod is best-effort for sandbox file systems.
    }
  }
  const runs = [];
  for (let index = 0; index < Math.max(1, runCount); index++) runs.push(timedRun(exe, benchCase.args ?? [], benchCase.env ?? {}));
  const run = runs[runs.length - 1];
  const runTimes = runs.map((item) => item.elapsedMs);
  const peakRssValues = runs.map((item) => item.peakRssBytes).filter((value) => value !== null);
  const stdout = run.stdout || run.stderr;
  const expected = benchCase.expectedStdout ?? null;
  const outputMatches = expected === null ? true : stdout.trim() === expected;
  return {
    language: language.name,
    case: name,
    status: runs.every((item) => item.result.status === 0) && outputMatches ? "ok" : (outputMatches ? "run-failed" : "semantic-failed"),
    backendMode,
    target: directZero ? directZeroTarget() : null,
    compiler: buildBody?.compiler ?? null,
    objectEmissionPath: buildBody?.objectBackend?.objectEmission?.path ?? null,
    cBridgeFallback: buildBody?.selfHostRouting?.cBridge?.required ?? buildBody?.cBridgeFallback ?? (directZero ? false : null),
    buildMs: build.elapsedMs,
    emitCMs: null,
    cCompileMs: null,
    runMs: median(runTimes),
    runMedianMs: median(runTimes),
    runMinMs: Math.min(...runTimes),
    runRuns: runs.map((item) => item.elapsedMs),
    artifactBytes: fileSize(exe),
    compressedArtifactBytes: gzipFileSize(exe),
    generatedCBytes: language.name === "zero" ? buildBody?.generatedCBytes ?? 0 : null,
    stdout,
    expectedStdout: expected,
    outputMatches,
    peakRssBytes: peakRssValues.length > 0 ? Math.max(...peakRssValues) : null,
  };
}

function average(values) {
  if (values.length === 0) return null;
  const total = values.reduce((sum, value) => sum + value, 0);
  return Number((total / values.length).toFixed(3));
}

function countBy(items, keyFn) {
  const counts = {};
  for (const item of items) {
    const key = keyFn(item) ?? "none";
    counts[key] = (counts[key] ?? 0) + 1;
  }
  return Object.fromEntries(Object.entries(counts).sort(([left], [right]) => left.localeCompare(right)));
}

function buildSummary(results) {
  const summary = {};
  for (const language of languages) {
    const languageResults = results.filter((item) => item.language === language.name);
    const ok = languageResults.filter((item) => item.status === "ok");
    const counts = {};
    for (const result of languageResults) counts[result.status] = (counts[result.status] ?? 0) + 1;
    summary[language.name] = {
      counts,
      averageBuildMs: average(ok.map((item) => item.buildMs).filter((value) => typeof value === "number")),
      averageArtifactBytes: average(ok.map((item) => item.artifactBytes).filter((value) => typeof value === "number")),
      averageCompressedArtifactBytes: average(ok.map((item) => item.compressedArtifactBytes).filter((value) => typeof value === "number")),
      averageRunMedianMs: average(ok.map((item) => item.runMedianMs).filter((value) => typeof value === "number")),
      byBackendMode: countBy(languageResults, (item) => item.backendMode),
    };
  }
  return summary;
}

function zeroResult(results, name) {
  return results.find((item) => item.language === "zero" && item.case === name && item.status === "ok") ?? null;
}

function measurementFact(results, name, measurement, status = "measured") {
  const result = zeroResult(results, name);
  return {
    name,
    measurement,
    status: result ? status : "missing",
    buildMs: result?.buildMs ?? null,
    runMedianMs: result?.runMedianMs ?? null,
    runMinMs: result?.runMinMs ?? null,
    peakRssBytes: result?.peakRssBytes ?? null,
    artifactBytes: result?.artifactBytes ?? null,
    compressedArtifactBytes: result?.compressedArtifactBytes ?? null,
  };
}

function buildMeasurementCoverage(results) {
  return {
    startup: [
      measurementFact(results, "baseline-hello", "cli-startup"),
    ],
    memory: [
      measurementFact(results, "baseline-hello", "runtime-memory-floor"),
      measurementFact(results, "baseline-memory-package", "peak-rss"),
      { name: "allocation-counts", measurement: "allocation-counts", status: "instrumentation-planned", note: "stdlib allocation APIs expose explicit allocators; runtime allocation counters need allocator instrumentation" },
    ],
    operations: [
      measurementFact(results, "slices", "slices"),
      measurementFact(results, "parser", "parsing"),
      measurementFact(results, "fileio", "file-io"),
      measurementFact(results, "codec", "codec"),
      measurementFact(results, "parse", "parse-primitives"),
      measurementFact(results, "branches", "branches"),
      measurementFact(results, "rescue", "fallible-rescue"),
      measurementFact(results, "networking", "networking"),
      measurementFact(results, "json", "json"),
    ],
  };
}

function buildEnvironment(extra = {}) {
  return {
    os: process.platform,
    arch: process.arch,
    node: process.version,
    ...extra,
  };
}

function buildReport(results, environment, runner) {
  const report = {
    generatedAt: new Date().toISOString(),
    methodology: {
      buildMs: "wall-clock process time measured by Node performance.now",
      runMs: "median wall-clock process time across repeated executions",
      runCount: Math.max(1, runCount),
      runMinMs: "minimum wall-clock process time across repeated executions",
      peakRssBytes: "maximum resident set size from /usr/bin/time -l on macOS; null when unavailable",
      compressedArtifactBytes: "gzip-compressed executable size in bytes for deployable-size baselines",
      generatedCBytes: "expected to be 0 for Zero benchmark rows",
      backendMode: "direct or native-toolchain per benchmark row",
      emitCMs: "retained as a nullable historical field; Zero no longer emits C",
      cCompileMs: "retained as a nullable historical field; Zero no longer invokes a C compiler backend",
      semanticChecks: "benchmark stdout must match the per-case expected output",
      runner,
    },
    claims: {
      buildTime: "buildMs records end-to-end compiler process time",
      artifactSize: "artifactBytes records output executable size for successful builds",
      compressedArtifactSize: "compressedArtifactBytes records gzip-compressed executable size for successful builds",
      runMedian: "runMedianMs records median process runtime across ZERO_BENCH_RUNS executions",
      generatedCBytes: "generatedCBytes is zero for Zero benchmarks",
      directZeroBenchmarks: "sandbox Linux runs use direct Zero artifacts for Zero benchmark cases",
      runtimeMeasurements: "measurements groups startup, memory, and operation benchmark facts; metadata-only entries call out runners that are not implemented yet",
    },
    limitations: [
      "ZERO_BENCH_RUNS=1 is a smoke test, not a publishable benchmark",
      "process-startup-heavy workloads are not general runtime performance claims",
      "skipped Zero cases are listed explicitly with a reason",
    ],
    environment,
    summary: buildSummary(results),
    measurements: buildMeasurementCoverage(results),
    results,
  };
  const typedReport = report as any;
  typedReport.trendSummary = buildBenchmarkTrendSummary(report);
  typedReport.budgets = evaluateBenchmarkBudgets(typedReport);
  return typedReport;
}

function buildBenchmarkTrendSummary(report) {
  const zeroRows = report.results.filter((row) => row.language === "zero" && row.backendMode === "direct" && row.status === "ok");
  const artifactBytes = zeroRows.map((row) => row.artifactBytes).filter((value) => typeof value === "number");
  const compressedArtifactBytes = zeroRows.map((row) => row.compressedArtifactBytes).filter((value) => typeof value === "number");
  const buildMs = zeroRows.map((row) => row.buildMs).filter((value) => typeof value === "number");
  const runMedianMs = zeroRows.map((row) => row.runMedianMs).filter((value) => typeof value === "number");
  const startupRows = report.measurements.startup.filter((row) => row.status === "measured");
  const memoryRows = report.measurements.memory.filter((row) => row.status === "measured");
  const operationRows = report.measurements.operations.filter((row) => row.status === "measured");
  return {
    schemaVersion: 1,
    kind: "zero-benchmark-trend-summary",
    generatedAt: report.generatedAt,
    sourceReport: ".zero/bench/latest.json",
    runCount,
    runner: report.environment.runner,
    trackedMetrics: ["artifactBytes", "compressedArtifactBytes", "buildMs", "runMedianMs", "runMinMs", "peakRssBytes"],
    directZero: {
      rows: zeroRows.length,
      generatedCBytes: zeroRows.reduce((total, row) => total + (row.generatedCBytes ?? 0), 0),
      averageArtifactBytes: average(artifactBytes),
      averageCompressedArtifactBytes: average(compressedArtifactBytes),
      averageBuildMs: average(buildMs),
      averageRunMedianMs: average(runMedianMs),
      cases: zeroRows.map((row) => row.case).sort(),
    },
    coverage: {
      startup: startupRows.length,
      memory: memoryRows.length,
      operations: operationRows.length,
    },
    rows: zeroRows.map((row) => ({
      case: row.case,
      artifactBytes: row.artifactBytes ?? null,
      compressedArtifactBytes: row.compressedArtifactBytes ?? null,
      buildMs: row.buildMs ?? null,
      runMedianMs: row.runMedianMs ?? null,
      runMinMs: row.runMinMs ?? null,
      peakRssBytes: row.peakRssBytes ?? null,
      objectEmissionPath: row.objectEmissionPath ?? null,
    })),
  };
}

function benchmarkBudgets() {
  return [
    {
      name: "baseline startup median",
      case: "baseline-hello",
      metric: "runMedianMs",
      max: envNumber("ZERO_BENCH_BASELINE_STARTUP_MAX_MS", 250),
      unit: "ms",
    },
    {
      name: "hello startup median",
      case: "hello",
      metric: "runMedianMs",
      max: envNumber("ZERO_BENCH_HELLO_STARTUP_MAX_MS", 250),
      unit: "ms",
    },
    {
      name: "json startup median",
      case: "json",
      metric: "runMedianMs",
      max: envNumber("ZERO_BENCH_JSON_STARTUP_MAX_MS", 250),
      unit: "ms",
    },
    {
      name: "parse startup median",
      case: "parse",
      metric: "runMedianMs",
      max: envNumber("ZERO_BENCH_PARSE_STARTUP_MAX_MS", 500),
      unit: "ms",
    },
    {
      name: "baseline executable size",
      case: "baseline-hello",
      metric: "artifactBytes",
      max: envNumber("ZERO_BENCH_BASELINE_ARTIFACT_MAX_BYTES", 131072),
      unit: "bytes",
    },
    {
      name: "hello executable size",
      case: "hello",
      metric: "artifactBytes",
      max: envNumber("ZERO_BENCH_HELLO_ARTIFACT_MAX_BYTES", 131072),
      unit: "bytes",
    },
    {
      name: "parse executable size",
      case: "parse",
      metric: "artifactBytes",
      max: envNumber("ZERO_BENCH_PARSE_ARTIFACT_MAX_BYTES", 262144),
      unit: "bytes",
    },
    {
      name: "baseline compressed executable size",
      case: "baseline-hello",
      metric: "compressedArtifactBytes",
      max: envNumber("ZERO_BENCH_BASELINE_COMPRESSED_ARTIFACT_MAX_BYTES", 32768),
      unit: "bytes",
    },
    {
      name: "hello compressed executable size",
      case: "hello",
      metric: "compressedArtifactBytes",
      max: envNumber("ZERO_BENCH_HELLO_COMPRESSED_ARTIFACT_MAX_BYTES", 32768),
      unit: "bytes",
    },
    {
      name: "parse compressed executable size",
      case: "parse",
      metric: "compressedArtifactBytes",
      max: envNumber("ZERO_BENCH_PARSE_COMPRESSED_ARTIFACT_MAX_BYTES", 65536),
      unit: "bytes",
    },
  ];
}

function evaluateBenchmarkBudgets(report) {
  const mode = budgetMode === "fail" || budgetMode === "warn" || budgetMode === "off" ? budgetMode : "off";
  if (mode === "off") {
    return { schemaVersion: 1, mode, status: "off", ok: true, overCount: 0, skippedCount: 0, items: [] };
  }
  const items = benchmarkBudgets().map((budget) => {
    const result = report.results.find((item) => item.language === "zero" && item.case === budget.case);
    if (!result) return { ...budget, status: "skipped", actual: null, reason: "case not selected" };
    if (result.status !== "ok") return { ...budget, status: "skipped", actual: null, reason: `case status ${result.status}` };
    const actual = result[budget.metric];
    if (typeof actual !== "number") return { ...budget, status: "skipped", actual: null, reason: "metric missing" };
    const overBy = Number(Math.max(0, actual - budget.max).toFixed(3));
    const unitSuffix = budget.unit === "ms" ? "Ms" : "Bytes";
    return {
      ...budget,
      status: actual <= budget.max ? "ok" : "over",
      actual,
      [`actual${unitSuffix}`]: actual,
      [`max${unitSuffix}`]: budget.max,
      [`overBy${unitSuffix}`]: overBy,
      ratio: Number((actual / budget.max).toFixed(3)),
    };
  });
  const overCount = items.filter((item) => item.status === "over").length;
  const skippedCount = items.filter((item) => item.status === "skipped").length;
  const problemCount = overCount + skippedCount;
  return {
    schemaVersion: 1,
    mode,
    status: problemCount === 0 ? "ok" : mode === "fail" ? "fail" : "warn",
    ok: problemCount === 0,
    overCount,
    skippedCount,
    items,
  };
}

function writeBenchmarkBudgetMessages(budget) {
  if (!budget || budget.mode === "off") return;
  for (const item of budget.items ?? []) {
    if (item.status === "over") {
      process.stderr.write(`benchmark budget ${budget.status}: ${item.name} ${item.actual} ${item.unit} > ${item.max} ${item.unit} (${item.metric})\n`);
    } else if (item.status === "skipped") {
      process.stderr.write(`benchmark budget ${budget.status}: ${item.name} skipped (${item.reason})\n`);
    }
  }
}

function assertBenchmarkGuardrails(report) {
  if (report.environment?.runner !== "vercel-sandbox") return;
  const guarded = ["baseline-hello", "hello", "add", "params", "buffers", "fileio", "codec", "parse", "fallibility", "branches", "rescue"];
  for (const name of guarded) {
    const result = report.results.find((item) => item.language === "zero" && item.case === name);
    if (!result) throw new Error(`missing benchmark result for ${name}`);
    if (result.status !== "ok") throw new Error(`${name} benchmark status was ${result.status}\n${formatFailedResult(result)}`);
    if (result.outputMatches !== true) throw new Error(`${name} benchmark stdout did not match\n${formatFailedResult(result)}`);
    if (result.backendMode !== "direct") throw new Error(`${name} did not use the direct backend\n${formatFailedResult(result)}`);
    if (result.generatedCBytes !== 0) throw new Error(`${name} direct benchmark emitted C bytes\n${formatFailedResult(result)}`);
    if (result.cBridgeFallback) throw new Error(`${name} direct benchmark used C bridge fallback\n${formatFailedResult(result)}`);
    if (typeof result.artifactBytes !== "number" || result.artifactBytes <= 0) throw new Error(`${name} missing artifact size\n${formatFailedResult(result)}`);
    if (typeof result.compressedArtifactBytes !== "number" || result.compressedArtifactBytes <= 0) throw new Error(`${name} missing compressed artifact size\n${formatFailedResult(result)}`);
    if (typeof result.runMedianMs !== "number" || result.runMedianMs <= 0) throw new Error(`${name} missing startup timing\n${formatFailedResult(result)}`);
    if (!("peakRssBytes" in result)) throw new Error(`${name} missing peak RSS field\n${formatFailedResult(result)}`);
  }
  for (const language of languages) {
    if (!report.summary[language.name]?.counts) throw new Error(`missing summary counts for ${language.name}`);
  }
}

function assertDirectBenchmarks(report) {
  if (report.environment?.runner !== "vercel-sandbox") return;
  for (const result of report.results.filter((item) => item.language === "zero" && item.status !== "skipped")) {
    const name = result.case;
    if (!result) throw new Error(`missing direct Zero benchmark result for ${name}`);
    if (result.status !== "ok") throw new Error(`${name} direct benchmark status was ${result.status}\n${formatFailedResult(result)}`);
    if (result.backendMode !== "direct") throw new Error(`${name} did not use the direct backend in sandbox benchmarks\n${formatFailedResult(result)}`);
    if (result.generatedCBytes !== 0) throw new Error(`${name} direct benchmark emitted C bytes\n${formatFailedResult(result)}`);
    if (result.cBridgeFallback) throw new Error(`${name} direct benchmark required a C bridge\n${formatFailedResult(result)}`);
    if (!result.objectEmissionPath?.startsWith("direct-")) throw new Error(`${name} missing direct object emission path\n${formatFailedResult(result)}`);
  }
}

function assertBenchmarkCoverage(report) {
  if (report.environment?.runner !== "vercel-sandbox") return;
  const zeroRows = report.results.filter((item) => item.language === "zero");
  const directRows = zeroRows.filter((item) => item.status === "ok" && item.backendMode === "direct");
  if (directRows.length < directZeroMinimum) {
    throw new Error(`Zero benchmark coverage requires at least ${directZeroMinimum} direct rows, found ${directRows.length}`);
  }
  for (const result of directRows) {
    if (result.outputMatches !== true) throw new Error(`${result.case} benchmark stdout did not match\n${formatFailedResult(result)}`);
    if (result.generatedCBytes !== 0) throw new Error(`${result.case} direct benchmark emitted C bytes\n${formatFailedResult(result)}`);
    if (result.cBridgeFallback) throw new Error(`${result.case} direct benchmark required a C bridge\n${formatFailedResult(result)}`);
    if (!result.objectEmissionPath?.startsWith("direct-")) throw new Error(`${result.case} missing direct object emission path\n${formatFailedResult(result)}`);
  }
  const removedBackendSkips = zeroRows.filter((item) => item.status === "skipped" && /removed C backend|C backend output/.test(item.reason ?? ""));
  if (removedBackendSkips.length > 0) {
    throw new Error(`removed-backend skip reasons remain: ${removedBackendSkips.map((item) => item.case).join(", ")}`);
  }
}

function formatFailedResult(result) {
  return JSON.stringify(
    {
      language: result.language,
      case: result.case,
      status: result.status,
      backendMode: result.backendMode ?? null,
      target: result.target ?? null,
      compiler: result.compiler ?? null,
      objectEmissionPath: result.objectEmissionPath ?? null,
      generatedCBytes: result.generatedCBytes ?? null,
      cBridgeFallback: result.cBridgeFallback ?? null,
      buildMs: result.buildMs ?? null,
      emitCMs: result.emitCMs ?? null,
      cCompileMs: result.cCompileMs ?? null,
      stdout: result.stdout ?? null,
      stderr: result.stderr ?? null,
      expectedStdout: result.expectedStdout ?? null,
      outputMatches: result.outputMatches ?? null,
    },
    null,
    2,
  );
}

function assertMeasurementCoverage(report) {
  for (const category of ["startup", "memory", "operations"]) {
    if (!Array.isArray(report.measurements?.[category]) || report.measurements[category].length === 0) {
      throw new Error(`missing ${category} measurement coverage`);
    }
  }
  if (report.environment?.runner !== "vercel-sandbox") return;
  for (const name of ["fileio", "codec", "parse", "branches", "rescue"]) {
    const fact = report.measurements.operations.find((item) => item.name === name);
    if (!fact || fact.status !== "measured") throw new Error(`missing measured operation benchmark for ${name}`);
  }
}

function benchmarkGuardrailsMarkdownReport(report) {
  const rows = ["# Benchmark Guardrails", "", "Timing is intentionally treated as noisy. The checked signals are semantic stdout, JSON shape, direct-output bytes, and artifact presence.", ""];
  for (const name of ["fs-resource", "mem-copy-fill"]) {
    const result = report.results.find((item) => item.language === "zero" && item.case === name);
    rows.push(`- ${name}: status=${result.status}, generatedCBytes=${result.generatedCBytes}, artifactBytes=${result.artifactBytes}, buildMs=${result.buildMs}, runMedianMs=${result.runMedianMs}`);
  }
  const directRows = report.results.filter((item) => item.language === "zero" && item.backendMode === "direct");
  const removedBackendSkips = report.results.filter((item) => item.language === "zero" && item.status === "skipped" && /removed C backend|C backend output/.test(item.reason ?? ""));
  rows.push(`- direct zero rows: ${directRows.length}, generatedCBytes=${directRows.reduce((total, row) => total + (row.generatedCBytes ?? 0), 0)}`);
  rows.push(`- direct zero threshold: ${directRows.length}/${directZeroMinimum}`);
  rows.push(`- removed-backend skip reasons: ${removedBackendSkips.length}`);
  rows.push("");
  rows.push(`- zero summary counts: ${JSON.stringify(report.summary.zero.counts)}`);
  rows.push("- timings are recorded for context; semantic output and artifact facts are the regression signals.");
  rows.push("");
  return `${rows.join("\n")}\n`;
}

function baselineMarkdownReport(report) {
  const rows = ["# Baseline Metrics", "", "These baselines track binary size, compressed size, startup timing, and peak RSS when available.", ""];
  for (const name of ["baseline-hello", "baseline-file-cli", "baseline-memory-package"]) {
    const result = report.results.find((item) => item.language === "zero" && item.case === name);
    if (result.status === "skipped") {
      rows.push(`- ${name}: status=skipped, reason=${result.reason}`);
    } else {
      rows.push(`- ${name}: artifactBytes=${result.artifactBytes}, compressedArtifactBytes=${result.compressedArtifactBytes}, runMedianMs=${result.runMedianMs}, runMinMs=${result.runMinMs}, peakRssBytes=${result.peakRssBytes}`);
    }
  }
  rows.push("");
  return `${rows.join("\n")}\n`;
}

function trendMarkdownReport(summary) {
  const rows = ["# Benchmark Trend Summary", "", "This file is a compact trend artifact. It is separate from the one-run smoke report so CI can archive it across runs.", ""];
  rows.push(`- runner: ${summary.runner}`);
  rows.push(`- runCount: ${summary.runCount}`);
  rows.push(`- direct Zero rows: ${summary.directZero.rows}`);
  rows.push(`- generatedCBytes: ${summary.directZero.generatedCBytes}`);
  rows.push(`- averageArtifactBytes: ${summary.directZero.averageArtifactBytes}`);
  rows.push(`- averageCompressedArtifactBytes: ${summary.directZero.averageCompressedArtifactBytes}`);
  rows.push(`- averageBuildMs: ${summary.directZero.averageBuildMs}`);
  rows.push(`- averageRunMedianMs: ${summary.directZero.averageRunMedianMs}`);
  rows.push(`- coverage: startup=${summary.coverage.startup}, memory=${summary.coverage.memory}, operations=${summary.coverage.operations}`);
  rows.push("");
  return `${rows.join("\n")}\n`;
}

function assertBenchmarkTrendSummary(report) {
  const summary = report.trendSummary;
  if (!summary || summary.kind !== "zero-benchmark-trend-summary") throw new Error("missing benchmark trend summary");
  if (!Array.isArray(summary.trackedMetrics) || !summary.trackedMetrics.includes("compressedArtifactBytes")) throw new Error("trend summary missing tracked metrics");
  if (summary.directZero.generatedCBytes !== 0) throw new Error("trend summary direct Zero C-output bytes changed");
  if (summary.directZero.rows <= 0 || summary.rows.length !== summary.directZero.rows) throw new Error("trend summary missing direct Zero rows");
  if (summary.coverage.startup <= 0 || summary.coverage.memory <= 0 || summary.coverage.operations <= 0) throw new Error("trend summary measurement coverage incomplete");
}

function writeReport(report) {
  report.trendSummary = buildBenchmarkTrendSummary(report);
  report.budgets = evaluateBenchmarkBudgets(report);
  const output = JSON.stringify(report, null, 2);
  const outputPath = join(outDir, "latest.json");
  const trendDir = join(outDir, "trends");
  mkdirSync(trendDir, { recursive: true });
  writeFileSync(outputPath, `${output}\n`);
  writeFileSync(join(dirname(outputPath), "latest.pretty.json"), `${output}\n`);
  writeFileSync(join(dirname(outputPath), "benchmark-guardrails.md"), benchmarkGuardrailsMarkdownReport(report));
  writeFileSync(join(dirname(outputPath), "baseline-metrics.md"), baselineMarkdownReport(report));
  writeFileSync(join(trendDir, "latest.json"), `${JSON.stringify(report.trendSummary, null, 2)}\n`);
  writeFileSync(join(trendDir, "summary.md"), trendMarkdownReport(report.trendSummary));

  try {
    writeBenchmarkBudgetMessages(report.budgets);
    if (report.budgets?.status === "fail") throw new Error(`benchmark budgets failed: ${report.budgets.overCount} over budget`);
    assertBenchmarkGuardrails(report);
    assertDirectBenchmarks(report);
    assertBenchmarkCoverage(report);
    assertMeasurementCoverage(report);
    assertBenchmarkTrendSummary(report);
  } catch (error) {
    const detail = error instanceof Error ? error.message : String(error);
    throw new Error(`${detail}\nfull benchmark report written to ${outputPath}`);
  }

  console.log(output);
}

function encodeJobs(jobs) {
  return Buffer.from(JSON.stringify(jobs), "utf8").toString("base64url");
}

function benchmarkJobGroups(jobs) {
  const groups = [];
  for (const language of languages) {
    const languageJobs = jobs.filter((job) => job.language === language.name);
    if (languageJobs.length > 0) groups.push({ language: language.name, jobs: languageJobs });
  }
  return groups;
}

function parsePositiveInt(value, fallback) {
  const parsed = Number.parseInt(value ?? "", 10);
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}

function parseOptionalNonNegativeInt(value) {
  if (value === undefined || value === "") return undefined;
  const parsed = Number.parseInt(value, 10);
  if (!Number.isFinite(parsed) || parsed < 0) throw new Error(`expected a non-negative integer, got ${value}`);
  return parsed;
}

function sandboxEnv() {
  return {
    ZERO_BENCH_RUNS: String(Math.max(1, runCount)),
  };
}

function sandboxSetupEnv() {
  return {};
}

function sandboxProjectDir() {
  return process.env.ZERO_BENCH_SANDBOX_PROJECT_DIR || "/vercel/sandbox/zero-lang";
}

function sandboxRuntime() {
  return process.env.ZERO_BENCH_SANDBOX_RUNTIME ?? "node24";
}

function sandboxTimeout() {
  return parsePositiveInt(process.env.ZERO_BENCH_SANDBOX_TIMEOUT_MS, 10 * 60 * 1000);
}

function sandboxVcpus() {
  return parsePositiveInt(process.env.ZERO_BENCH_SANDBOX_VCPUS, 16);
}

function snapshotFromEnv() {
  if (["1", "true", "yes"].includes((process.env.ZERO_BENCH_SANDBOX_REFRESH ?? "").toLowerCase())) return null;
  const snapshotId = process.env.ZERO_BENCH_SANDBOX_SNAPSHOT_ID?.trim();
  if (!snapshotId) return null;
  return {
    snapshotId,
    projectDir: sandboxProjectDir(),
    timeout: sandboxTimeout(),
    vcpus: sandboxVcpus(),
    reused: true,
  };
}

function writeSnapshotEnv(snapshot) {
  const snapshotPath = join(outDir, "snapshot.env");
  const lines = [
    `ZERO_BENCH_SANDBOX_SNAPSHOT_ID=${snapshot.snapshotId}`,
    `ZERO_BENCH_SANDBOX_PROJECT_DIR=${snapshot.projectDir}`,
    "",
  ];
  writeFileSync(snapshotPath, lines.join("\n"));
  process.stderr.write(`Created Vercel Sandbox benchmark snapshot: ${snapshot.snapshotId}\n`);
  process.stderr.write(`Saved reusable snapshot env to ${snapshotPath}\n`);
  process.stderr.write("Future benchmark runs will reuse it automatically unless ZERO_BENCH_SANDBOX_REFRESH is set.\n");
}

function createSourceArchive() {
  const archivePath = join(outDir, "source.tar.gz");
  rmSync(archivePath, { force: true });
  const excludes = [
    ".git",
    ".cursor",
    ".env",
    ".env.*",
    ".vercel",
    ".zero",
    "node_modules",
    "extensions/*/node_modules",
    "docs/node_modules",
  ];
  const excludeArgs = excludes.flatMap((exclude) => [`--exclude=${exclude}`, `--exclude=./${exclude}`]);
  const result = spawnSync("tar", [...excludeArgs, "-czf", archivePath, "."], {
    cwd: root,
    encoding: "utf8",
    env: { ...process.env, COPYFILE_DISABLE: "1" },
  });
  if (result.status !== 0) {
    throw new Error(`failed to create benchmark source archive\n${result.stderr.trim()}`);
  }
  return archivePath;
}

async function runSandboxCommand(sandbox, params, label) {
  const result = await sandbox.runCommand(params);
  const [stdout, stderr] = await Promise.all([result.stdout(), result.stderr()]);
  if (result.exitCode !== 0) {
    throw new Error(`${label} failed with exit code ${result.exitCode}\n${stderr || stdout}`);
  }
  return { result, stdout, stderr };
}

function describeError(error) {
  if (!(error instanceof Error)) return String(error);
  const parts = [error.message];
  if ("json" in error && error.json) parts.push(JSON.stringify(error.json));
  if ("text" in error && error.text) parts.push(String(error.text));
  return parts.filter(Boolean).join("\n");
}

async function createPreparedSnapshot(Sandbox) {
  const runtime = sandboxRuntime();
  const timeout = sandboxTimeout();
  const vcpus = sandboxVcpus();
  const snapshotExpiration = parseOptionalNonNegativeInt(process.env.ZERO_BENCH_SANDBOX_SNAPSHOT_EXPIRATION_MS);
  const archivePath = createSourceArchive();
  const sandbox = await Sandbox.create({ runtime, timeout, resources: { vcpus } });
  const projectDir = sandboxProjectDir();

  try {
    await sandbox.writeFiles([{ path: "/tmp/zero-lang-source.tar.gz", content: readFileSync(archivePath) }]);
    await runSandboxCommand(
      sandbox,
      {
        cmd: "bash",
        args: [
          "-lc",
          [
            "set -euo pipefail",
            "as_root() { if command -v sudo >/dev/null 2>&1; then sudo \"$@\"; else \"$@\"; fi; }",
            "install_build_tools() {",
            "  if command -v make >/dev/null 2>&1 && { command -v cc >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || command -v clang >/dev/null 2>&1; }; then return; fi",
            "  if command -v apt-get >/dev/null 2>&1; then",
            "    as_root apt-get update",
            "    as_root env DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential ca-certificates curl git gzip make tar xz-utils",
            "  elif command -v apk >/dev/null 2>&1; then",
            "    as_root apk add --no-cache build-base ca-certificates curl git gzip make tar xz",
            "  elif command -v dnf >/dev/null 2>&1; then",
            "    # Vercel Sandbox currently runs on Amazon Linux, which exposes dnf/yum.",
            "    as_root dnf install -y ca-certificates gcc git glibc-devel gzip make tar xz",
            "  elif command -v yum >/dev/null 2>&1; then",
            "    # Older Amazon Linux images may expose yum instead of dnf.",
            "    as_root yum install -y ca-certificates gcc git glibc-devel gzip make tar xz",
            "  else",
            "    echo 'no supported package manager found to install make and a C compiler' >&2",
            "    exit 127",
            "  fi",
            "}",
            "install_build_tools",
            "command -v make >/dev/null 2>&1",
            "command -v cc >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || command -v clang >/dev/null 2>&1",
            `mkdir -p ${projectDir}`,
            `tar -xzf /tmp/zero-lang-source.tar.gz -C ${projectDir}`,
            `cd ${projectDir}`,
            "corepack enable",
            "corepack prepare pnpm@11.1.3 --activate",
            "pnpm install --frozen-lockfile",
            "make -C native/zero-c",
            "node --version",
          ].join("\n"),
        ],
        env: sandboxSetupEnv(),
      },
      "benchmark sandbox setup",
    );
    let snapshot;
    try {
      snapshot = await sandbox.snapshot(snapshotExpiration === undefined ? undefined : { expiration: snapshotExpiration });
    } catch (error) {
      throw new Error(`benchmark sandbox snapshot failed\n${describeError(error)}`);
    }
    const preparedSnapshot = { snapshotId: snapshot.snapshotId, projectDir, timeout, vcpus, reused: false };
    writeSnapshotEnv(preparedSnapshot);
    return preparedSnapshot;
  } catch (error) {
    await sandbox.stop({ blocking: true }).catch(() => {});
    throw error;
  }
}

async function runSandboxLanguageGroup(Sandbox, snapshot, group, index) {
  const sandbox = await Sandbox.create({
    source: { type: "snapshot", snapshotId: snapshot.snapshotId },
    timeout: snapshot.timeout,
    resources: { vcpus: snapshot.vcpus },
  });
  try {
    await runSandboxCommand(sandbox, { cmd: "true" }, `${group.language} benchmark sandbox warmup`);
    const command = {
      cmd: "node",
      args: [
        "--experimental-strip-types",
        "--disable-warning=ExperimentalWarning",
        "scripts/bench.mts",
        "--worker",
        "--jobs",
        encodeJobs(group.jobs),
      ],
      cwd: snapshot.projectDir,
      env: sandboxEnv(),
    };
    const { stdout } = await runSandboxCommand(sandbox, command, `${group.language} benchmark sandbox`);
    return JSON.parse(stdout);
  } finally {
    await sandbox.stop({ blocking: true }).catch(() => {});
  }
}

async function runSandboxBench() {
  const { Sandbox } = await import("@vercel/sandbox").catch((error) => {
    throw new Error(`sandbox benchmark mode requires @vercel/sandbox. Run pnpm install before benchmarking.\n${error.message}`);
  });

  const jobs = executableJobs();
  const skipped = skippedResults();
  const groups = benchmarkJobGroups(jobs);
  const resultSlots = jobs.length + skipped.length;
  const skippedNote = skipped.length === 0 ? "" : ` (${skipped.length} skipped result slots recorded locally)`;
  const reusableSnapshot = snapshotFromEnv();
  if (reusableSnapshot) {
    process.stderr.write(`Using Vercel Sandbox benchmark snapshot ${reusableSnapshot.snapshotId} for ${jobs.length} executable jobs in ${groups.length} benchmark sandboxes${skippedNote}...\n`);
  } else {
    if (process.env.ZERO_BENCH_SANDBOX_SNAPSHOT_ID) process.stderr.write("Refreshing Vercel Sandbox benchmark snapshot because ZERO_BENCH_SANDBOX_REFRESH is set...\n");
    process.stderr.write(`Preparing Vercel Sandbox benchmark snapshot for ${jobs.length} executable jobs in ${groups.length} benchmark sandboxes${skippedNote}...\n`);
  }
  const snapshot = reusableSnapshot ?? (await createPreparedSnapshot(Sandbox));
  process.stderr.write(`Running ${jobs.length} executable benchmark jobs across ${groups.length} Vercel sandboxes for ${resultSlots} report rows...\n`);

  const workerReports = await Promise.all(groups.map((group, index) => runSandboxLanguageGroup(Sandbox, snapshot, group, index)));
  const results = sortResults([...workerReports.flatMap((report) => report.results), ...skipped]);
  const environment = {
    ...(workerReports[0]?.environment ?? buildEnvironment()),
    runner: "vercel-sandbox",
    sandboxRuntime: sandboxRuntime(),
    sandboxConcurrency: groups.length,
    sandboxShardStrategy: "zero-benchmark",
    sandboxBenchmarkGroups: groups.map((group) => group.language),
    sandboxSnapshotId: snapshot.snapshotId,
    sandboxSnapshotReused: snapshot.reused,
  };
  return buildReport(results, environment, "vercel-sandbox");
}

function runLocalBench() {
  const results = runJobs(allJobs());
  return buildReport(results, buildEnvironment({ runner: "local" }), "local");
}

async function main() {
  if (workerMode) {
    const results = runJobs(parseWorkerJobs());
    console.log(JSON.stringify({ environment: buildEnvironment({ runner: "vercel-sandbox-worker" }), results }));
    return;
  }

  const report = mode === "local" ? runLocalBench() : await runSandboxBench();
  writeReport(report);
}

main().catch((error) => {
  console.error(error.stack ?? error.message);
  process.exitCode = 1;
});