import { readFileSync } from "node:fs"; import { setTimeout as sleep } from "node:timers/promises"; import { apiFetch } from "../api.mjs"; import { emit } from "../output.mjs"; import { t } from "../i18n.mjs"; function truncate(v, len = 30) { if (v == null) return "-"; const s = String(v); return s.length > len ? s.slice(0, len - 1) + "…" : s; } function fmtTs(v) { if (!v) return "-"; try { return new Date(v).toLocaleString(); } catch { return String(v); } } const suiteSchema = [ { key: "id", header: "Suite ID", width: 22 }, { key: "name", header: "Name", width: 30 }, { key: "samples", header: "Samples" }, { key: "rubric", header: "Rubric", width: 16 }, { key: "updatedAt", header: "Updated", formatter: fmtTs }, ]; const runSchema = [ { key: "id", header: "Run ID", width: 22 }, { key: "suiteId", header: "Suite", width: 18 }, { key: "status", header: "Status", width: 12 }, { key: "model", header: "Model", width: 25 }, { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(3) : "-") }, { key: "duration", header: "Duration", formatter: (v) => (v != null ? `${(v / 1000).toFixed(1)}s` : "-"), }, { key: "startedAt", header: "Started", formatter: fmtTs }, ]; const sampleSchema = [ { key: "id", header: "Sample", width: 14 }, { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(2) : "-") }, { key: "passed", header: "✓", formatter: (v) => (v ? "✓" : "✗") }, { key: "input", header: "Input", width: 30, formatter: truncate }, { key: "output", header: "Output", width: 30, formatter: truncate }, ]; async function confirm(q) { return new Promise((resolve) => { process.stdout.write(`${q} (yes/no) `); process.stdin.setEncoding("utf8"); process.stdin.once("data", (c) => resolve(c.toString().trim().toLowerCase().startsWith("y"))); }); } async function watchRun(runId, globalOpts) { let lastStatus = ""; while (true) { await sleep(3000); const res = await apiFetch(`/api/evals/${runId}`); if (!res.ok) continue; const r = await res.json(); if (r.status !== lastStatus) { const done = r.progress?.completed ?? 0; const total = r.progress?.total ?? "?"; process.stderr.write(`[${new Date().toISOString()}] ${r.status} — ${done}/${total}\n`); lastStatus = r.status; } if (["completed", "failed", "cancelled"].includes(r.status)) { emit(r, globalOpts, runSchema); return; } } } function renderScorecard(data) { const score = data.score ?? data.overallScore ?? null; const passed = data.passed ?? data.summary?.passed ?? null; const total = data.total ?? data.summary?.total ?? null; process.stdout.write("\n=== Scorecard ===\n"); if (score != null) process.stdout.write(`Overall score: ${(score * 100).toFixed(1)}%\n`); if (passed != null && total != null) { process.stdout.write(`Passed: ${passed}/${total}\n`); const bar = "█".repeat(Math.round((passed / total) * 20)).padEnd(20, "░"); process.stdout.write(`[${bar}] ${((passed / total) * 100).toFixed(0)}%\n`); } const metrics = data.metrics ?? data.breakdown ?? {}; for (const [k, v] of Object.entries(metrics)) { process.stdout.write(` ${k}: ${typeof v === "number" ? v.toFixed(3) : v}\n`); } process.stdout.write("\n"); } export async function runEvalSuitesList(opts, cmd) { const res = await apiFetch("/api/evals/suites"); if (!res.ok) { process.stderr.write(`Error: ${res.status}\n`); process.exit(1); } const data = await res.json(); emit(data.items ?? data, cmd.optsWithGlobals(), suiteSchema); } export async function runEvalSuitesGet(id, opts, cmd) { const res = await apiFetch(`/api/evals/suites/${id}`); if (!res.ok) { process.stderr.write(`Not found: ${id}\n`); process.exit(1); } emit(await res.json(), cmd.optsWithGlobals()); } export async function runEvalSuitesCreate(opts, cmd) { if (!opts.file) { process.stderr.write("--file required\n"); process.exit(2); } const body = JSON.parse(readFileSync(opts.file, "utf8")); const res = await apiFetch("/api/evals/suites", { method: "POST", body }); if (!res.ok) { process.stderr.write(`Error: ${res.status}\n`); process.exit(1); } emit(await res.json(), cmd.optsWithGlobals()); } export async function runEvalRun(suiteId, opts, cmd) { const globalOpts = cmd.optsWithGlobals(); const body = { suiteId, model: opts.model ?? "auto", ...(opts.combo ? { combo: opts.combo } : {}), concurrency: opts.concurrency ?? 4, ...(opts.tag ? { tag: opts.tag } : {}), }; const res = await apiFetch("/api/evals", { method: "POST", body }); if (!res.ok) { process.stderr.write(`Error: ${res.status}\n`); process.exit(1); } const run = await res.json(); emit(run, globalOpts, runSchema); if (opts.watch) { if (process.stdout.isTTY) { const { startEvalWatchTui } = await import("../tui/EvalWatch.jsx"); await startEvalWatchTui({ runId: run.id, suiteId: opts.suite, baseUrl: globalOpts.baseUrl ?? "http://localhost:20128", apiKey: globalOpts.apiKey ?? process.env.OMNIROUTE_API_KEY, }); } else { process.stderr.write("\nWatching run... (Ctrl+C to detach)\n"); await watchRun(run.id, globalOpts); } } } export async function runEvalList(opts, cmd) { const params = new URLSearchParams({ limit: String(opts.limit ?? 50) }); if (opts.suite) params.set("suiteId", opts.suite); if (opts.status) params.set("status", opts.status); if (opts.since) params.set("since", opts.since); const res = await apiFetch(`/api/evals?${params}`); if (!res.ok) { process.stderr.write(`Error: ${res.status}\n`); process.exit(1); } const data = await res.json(); emit(data.items ?? data, cmd.optsWithGlobals(), runSchema); } export async function runEvalGet(id, opts, cmd) { const res = await apiFetch(`/api/evals/${id}`); if (!res.ok) { process.stderr.write(`Not found: ${id}\n`); process.exit(1); } emit(await res.json(), cmd.optsWithGlobals()); } export async function runEvalResults(id, opts, cmd) { const params = new URLSearchParams(); if (opts.failed) params.set("filter", "failed"); const res = await apiFetch(`/api/evals/${id}?${params}`); if (!res.ok) { process.stderr.write(`Not found: ${id}\n`); process.exit(1); } const data = await res.json(); emit(data.samples ?? data.results ?? [], cmd.optsWithGlobals(), sampleSchema); } export async function runEvalCancel(id, opts, cmd) { if (!opts.yes) { const ok = await confirm(`Cancel run ${id}?`); if (!ok) return; } const res = await apiFetch(`/api/evals/${id}`, { method: "POST", body: { op: "cancel" } }); if (!res.ok) { process.stderr.write(`Error: ${res.status}\n`); process.exit(1); } process.stdout.write("Cancelled\n"); } export async function runEvalScorecard(id, opts, cmd) { const res = await apiFetch(`/api/evals/${id}?scorecard=true`); if (!res.ok) { process.stderr.write(`Not found: ${id}\n`); process.exit(1); } const data = await res.json(); const globalOpts = cmd.optsWithGlobals(); if (globalOpts.output === "json") { emit(data, globalOpts); } else { renderScorecard(data); } } export function registerEval(program) { const evalCmd = program.command("eval").description(t("eval.description")); const suites = evalCmd.command("suites").description(t("eval.suites.description")); suites.command("list").description(t("eval.suites.list.description")).action(runEvalSuitesList); suites .command("get ") .description(t("eval.suites.get.description")) .action(runEvalSuitesGet); suites .command("create") .description(t("eval.suites.create.description")) .option("--file ", t("eval.suites.create.file")) .action(runEvalSuitesCreate); evalCmd .command("run ") .description(t("eval.run.description")) .option("-m, --model ", t("eval.run.model"), "auto") .option("--combo ", t("eval.run.combo")) .option("--concurrency ", t("eval.run.concurrency"), parseInt, 4) .option("--tag ", t("eval.run.tag")) .option("--watch", t("eval.run.watch")) .action(runEvalRun); evalCmd .command("list") .description(t("eval.list.description")) .option("--suite ", t("eval.list.suite")) .option("--status ", t("eval.list.status")) .option("--since ", t("eval.list.since")) .option("--limit ", t("eval.list.limit"), parseInt, 50) .action(runEvalList); evalCmd.command("get ").description(t("eval.get.description")).action(runEvalGet); evalCmd .command("results ") .description(t("eval.results.description")) .option("--failed", t("eval.results.failed")) .action(runEvalResults); evalCmd .command("cancel ") .description(t("eval.cancel.description")) .option("--yes", t("eval.cancel.yes")) .action(runEvalCancel); evalCmd .command("scorecard ") .description(t("eval.scorecard.description")) .action(runEvalScorecard); }