wehub-resource-sync
95 lines
4.1 KiB
TypeScript
95 lines
4.1 KiB
TypeScript
import { describe, it } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { runEval } from "../../../open-sse/services/compression/eval/runner.ts";
|
|
import { CONTROL_PAIR } from "../../../open-sse/services/compression/eval/judge.ts";
|
|
import type { EvalCase, ModelClient } from "../../../open-sse/services/compression/eval/types.ts";
|
|
|
|
const corpus: EvalCase[] = [
|
|
{ id: "p1", kind: "prose", context: "the sky is blue", question: "what color is the sky?", gold: "blue" },
|
|
{ id: "c1", kind: "code", context: "function f(){return 7}", question: "what does f return?", gold: "7" },
|
|
];
|
|
|
|
/** Stub that answers questions, judges fidelity, grades gold — keyed off prompt content. */
|
|
function smartStub(opts: { answerModel: string; judgeModel: string }): ModelClient {
|
|
return {
|
|
async complete(model, messages) {
|
|
const sys = messages.find((m) => m.role === "system")?.content ?? "";
|
|
const user = messages.find((m) => m.role === "user")?.content ?? "";
|
|
// Self-test control pair (judge): rank degraded vs good correctly.
|
|
if (model === opts.judgeModel && user.includes(CONTROL_PAIR.degraded)) {
|
|
return { text: "VERDICT: MATERIALLY_DIFFERS", usdCost: 0.01 };
|
|
}
|
|
if (model === opts.judgeModel && /judge|materially/i.test(sys)) {
|
|
return { text: "VERDICT: SAME", usdCost: 0.01 };
|
|
}
|
|
// Gold grader: always CORRECT for this stub.
|
|
if (/grader|correct\|incorrect/i.test(sys)) return { text: "VERDICT: CORRECT", usdCost: 0.01 };
|
|
// Answer generation.
|
|
return { text: "blue and seven", usdCost: 0.01 };
|
|
},
|
|
};
|
|
}
|
|
|
|
const baseOpts = {
|
|
config: { enabled: true, defaultMode: "lite", enginesExplicit: false } as any,
|
|
comboId: null,
|
|
combos: {},
|
|
answerModel: "answer-model",
|
|
judgeModel: "judge-model",
|
|
provider: "test",
|
|
costCapUsd: 0,
|
|
sample: undefined as number | undefined,
|
|
};
|
|
|
|
describe("eval runner", () => {
|
|
it("aborts when the judge fails self-test (no scores emitted)", async () => {
|
|
const broken: ModelClient = { async complete() { return { text: "VERDICT: SAME" }; } };
|
|
const r = await runEval({ ...baseOpts, corpus, client: broken });
|
|
assert.equal(r.aborted, true);
|
|
assert.match(r.abortReason ?? "", /self-test/i);
|
|
assert.equal(r.report, null);
|
|
});
|
|
|
|
it("produces an aggregated report with a passing judge", async () => {
|
|
const r = await runEval({ ...baseOpts, corpus, client: smartStub({ answerModel: "answer-model", judgeModel: "judge-model" }) });
|
|
assert.equal(r.aborted, false);
|
|
assert.ok(r.report);
|
|
assert.equal(r.report!.overall.casesScored, 2);
|
|
assert.equal(r.report!.stamps.answerModel, "answer-model");
|
|
assert.match(r.report!.stamps.corpusHash, /^[0-9a-f]{64}$/);
|
|
});
|
|
|
|
it("an errored case (model throws) is excluded but counted", async () => {
|
|
let n = 0;
|
|
const flaky: ModelClient = {
|
|
async complete(model, messages) {
|
|
const user = messages.find((m) => m.role === "user")?.content ?? "";
|
|
if (model === "judge-model" && user.includes(CONTROL_PAIR.degraded)) return { text: "VERDICT: MATERIALLY_DIFFERS" };
|
|
if (model === "judge-model") return { text: "VERDICT: SAME" };
|
|
n += 1;
|
|
if (n === 2) throw new Error("upstream 500"); // fail the 2nd answer call
|
|
return { text: "ok", usdCost: 0.01 };
|
|
},
|
|
};
|
|
const r = await runEval({ ...baseOpts, corpus, client: flaky });
|
|
assert.equal(r.aborted, false);
|
|
assert.equal(r.report!.overall.casesErrored, 1);
|
|
});
|
|
|
|
it("the cost cap stops the run and flags partial", async () => {
|
|
const r = await runEval({
|
|
...baseOpts,
|
|
costCapUsd: 0.025, // self-test (2 calls) + a little; cap trips during the case loop
|
|
corpus,
|
|
client: smartStub({ answerModel: "answer-model", judgeModel: "judge-model" }),
|
|
});
|
|
assert.equal(r.aborted, false);
|
|
assert.equal(r.report!.partial, true);
|
|
});
|
|
|
|
it("--sample N limits the cases scored", async () => {
|
|
const r = await runEval({ ...baseOpts, sample: 1, corpus, client: smartStub({ answerModel: "answer-model", judgeModel: "judge-model" }) });
|
|
assert.equal(r.report!.overall.casesScored + r.report!.overall.casesErrored, 1);
|
|
});
|
|
});
|