#!/usr/bin/env -S node --experimental-strip-types --disable-warning=ExperimentalWarning import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; const nodeArgs = ["--experimental-strip-types", "--disable-warning=ExperimentalWarning"]; const reportDir = ".zero/validation"; type Phase = { name: string; command: string; args: string[]; setup?: boolean; }; type Suite = { setup: Phase[]; phases: Phase[]; defaultJobs?: number; }; type PhaseResult = { name: string; command: string; ok: boolean; code: number | null; signal: NodeJS.Signals | null; durationMs: number; stdout: string; stderr: string; }; const suites: Record = { conformance: { setup: [ { name: "native-build", command: "make", args: ["-C", "native/zero-c"], setup: true }, ], phases: [ { name: "graph-input-policy", command: process.execPath, args: [...nodeArgs, "scripts/graph-input-policy.mts"] }, { name: "native-contracts", command: process.execPath, args: [...nodeArgs, "scripts/native-contracts.mts"] }, { name: "provenance-guardrails", command: process.execPath, args: [...nodeArgs, "scripts/provenance-guardrails.mts"] }, { name: "type-core-smoke", command: process.execPath, args: [...nodeArgs, "scripts/type-core-smoke.mts"] }, { name: "mir-verifier-smoke", command: process.execPath, args: [...nodeArgs, "scripts/mir-verifier-smoke.mts"] }, { name: "program-graph-smoke", command: process.execPath, args: [...nodeArgs, "scripts/program-graph-smoke.mts"] }, { name: "program-graph-parity", command: process.execPath, args: [...nodeArgs, "scripts/program-graph-parity.mts"] }, { name: "canonical-text-smoke", command: process.execPath, args: [...nodeArgs, "scripts/canonical-text-smoke.mts"] }, { name: "examples-gate", command: process.execPath, args: [...nodeArgs, "scripts/examples-gate.mts"] }, { name: "conformance-run", command: process.execPath, args: ["conformance/run.mjs"] }, ], }, "command-contracts": { setup: [ { name: "native-build", command: "make", args: ["-C", "native/zero-c"], setup: true }, ], phases: [ { name: "graph-input-policy", command: process.execPath, args: [...nodeArgs, "scripts/graph-input-policy.mts"] }, { name: "snapshot-command-contracts", command: process.execPath, args: [...nodeArgs, "scripts/snapshot-command-contracts.mts"] }, ], }, }; function usage() { const suiteNames = Object.keys(suites).join("|"); console.log(`Run validation phases with aggregate failure reporting. Usage: node --experimental-strip-types --disable-warning=ExperimentalWarning scripts/validation-suite.mts <${suiteNames}> [options] Options: --list List phases and exit. --phases Run only the named comma-separated phases after setup. --shard Run only this 1-based phase shard after setup. --jobs Run selected phases concurrently. Defaults to 1. --fail-fast Stop after the first failing phase. Environment: ZERO_VALIDATION_PHASES Default phase allowlist, for example a,b,c. ZERO_VALIDATION_SHARD Default shard, for example 1/4. ZERO_VALIDATION_JOBS Default jobs count. ZERO_VALIDATION_FAIL_FAST=1 Stop after the first failing phase.`); } const rawArgs = process.argv.slice(2); if (rawArgs.includes("--help") || rawArgs.includes("-h")) { usage(); process.exit(0); } const suiteName = rawArgs.find((arg) => !arg.startsWith("--")); if (!suiteName || !suites[suiteName]) { usage(); process.exit(2); } function optionValue(name: string) { const index = rawArgs.indexOf(name); return index === -1 ? undefined : rawArgs[index + 1]; } function parsePositiveInt(value: string | undefined, fallback: number) { const parsed = Number.parseInt(value ?? "", 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } function parseShard(value: string | undefined) { const text = value || process.env.ZERO_VALIDATION_SHARD || ""; if (!text) return null; const match = /^([1-9][0-9]*)\/([1-9][0-9]*)$/.exec(text); if (!match) throw new Error(`invalid shard ${text}; expected index/count, for example 1/4`); const index = Number.parseInt(match[1], 10); const count = Number.parseInt(match[2], 10); if (index > count) throw new Error(`invalid shard ${text}; index must be <= count`); return { index, count, text }; } function parsePhaseFilter(value: string | undefined, suite: Suite) { const text = value || process.env.ZERO_VALIDATION_PHASES || ""; if (!text.trim()) return null; const wanted = new Set(text.split(",").map((part) => part.trim()).filter(Boolean)); const known = new Set(suite.phases.map((phase) => phase.name)); const unknown = [...wanted].filter((name) => !known.has(name)); if (unknown.length > 0) { throw new Error(`unknown validation phase(s): ${unknown.join(", ")}`); } return wanted; } const suite = suites[suiteName]; const listOnly = rawArgs.includes("--list"); const phaseFilter = parsePhaseFilter(optionValue("--phases"), suite); const shard = parseShard(optionValue("--shard")); const jobs = parsePositiveInt(optionValue("--jobs") ?? process.env.ZERO_VALIDATION_JOBS, suite.defaultJobs ?? 1); const failFast = rawArgs.includes("--fail-fast") || process.env.ZERO_VALIDATION_FAIL_FAST === "1"; const filteredPhases = phaseFilter ? suite.phases.filter((phase) => phaseFilter.has(phase.name)) : suite.phases; const selectedPhases = shard ? filteredPhases.filter((_, index) => index % shard.count === shard.index - 1) : filteredPhases; if (listOnly) { for (const phase of suite.setup) console.log(`${phase.name} setup`); for (const [index, phase] of suite.phases.entries()) { const selected = selectedPhases.includes(phase) ? "selected" : phaseFilter && !phaseFilter.has(phase.name) ? "filtered" : "skipped"; console.log(`${index + 1}. ${phase.name} ${selected}`); } process.exit(0); } function runPhase(phase: Phase): Promise { const startedAt = Date.now(); process.stderr.write(`validation phase start: ${phase.name}\n`); return new Promise((resolve) => { const child = spawn(phase.command, phase.args, { cwd: process.cwd(), env: process.env, stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; let stderr = ""; child.stdout.on("data", (chunk) => { const text = chunk.toString(); stdout += text; process.stdout.write(text); }); child.stderr.on("data", (chunk) => { const text = chunk.toString(); stderr += text; process.stderr.write(text); }); child.on("error", (error) => { stderr += `${error.message}\n`; }); child.on("close", (code, signal) => { const durationMs = Date.now() - startedAt; const ok = code === 0 && signal === null; process.stderr.write(`validation phase ${ok ? "ok" : "failed"}: ${phase.name} (${durationMs}ms)\n`); resolve({ name: phase.name, command: [phase.command, ...phase.args].join(" "), ok, code, signal, durationMs, stdout, stderr, }); }); }); } async function runSequential(phases: Phase[]) { const results: PhaseResult[] = []; for (const phase of phases) { const result = await runPhase(phase); results.push(result); if (!result.ok && failFast) break; } return results; } async function runConcurrent(phases: Phase[]) { const results: PhaseResult[] = []; let next = 0; let stop = false; const workerCount = Math.min(jobs, Math.max(1, phases.length)); async function worker() { for (;;) { if (stop) return; const index = next++; if (index >= phases.length) return; const result = await runPhase(phases[index]); results[index] = result; if (!result.ok && failFast) stop = true; } } await Promise.all(Array.from({ length: workerCount }, () => worker())); return results.filter(Boolean); } function printFailures(results: any[]) { const failures = results.filter((result) => !result.ok); if (failures.length === 0) return; process.stderr.write(`\n${suiteName} collected ${failures.length} failing phase(s):\n`); for (const [index, failure] of failures.entries()) { process.stderr.write(`\n${index + 1}. ${failure.name}\n`); process.stderr.write(` command: ${failure.command}\n`); process.stderr.write(` exit: ${failure.code ?? "signal"}${failure.signal ? ` signal ${failure.signal}` : ""}\n`); const detail = (failure.stderr || failure.stdout || "").trim().split("\n").slice(-40).join("\n"); if (detail) process.stderr.write(`${detail}\n`); } } async function main() { mkdirSync(reportDir, { recursive: true }); const startedAt = Date.now(); const setupResults = await runSequential(suite.setup); const setupFailed = setupResults.some((result) => !result.ok); const phaseResults = setupFailed && failFast ? [] : jobs > 1 ? await runConcurrent(selectedPhases) : await runSequential(selectedPhases); const results = [...setupResults, ...phaseResults]; const ok = results.every((result) => result.ok); const report = { suite: suiteName, ok, phases: phaseFilter ? [...phaseFilter] : null, shard: shard?.text ?? null, jobs, failFast, durationMs: Date.now() - startedAt, results, }; const reportPath = join(reportDir, `${suiteName}${shard ? `-shard-${shard.index}-of-${shard.count}` : ""}.json`); writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); printFailures(results); if (!ok) { process.stderr.write(`\nvalidation suite failed: ${suiteName}\nreport: ${reportPath}\n`); process.exit(1); } process.stderr.write(`validation suite ok: ${suiteName}\nreport: ${reportPath}\n`); } main().catch((error) => { console.error(error instanceof Error ? error.stack || error.message : String(error)); process.exit(1); });