wehub-resource-sync
69 lines
3.1 KiB
TypeScript
69 lines
3.1 KiB
TypeScript
import type { ChatTurn, JudgeVerdict, ModelClient } from "./types.ts";
|
|
|
|
/**
|
|
* Fidelity judge prompt (D-D2a). Asks the judge to decide whether the compressed-context
|
|
* answer MATERIALLY differs from the full-context answer — wording differences that do not
|
|
* change the substance are "SAME". The judge must end with a single `VERDICT:` line.
|
|
*/
|
|
export function buildJudgePrompt(fullAnswer: string, compressedAnswer: string): ChatTurn[] {
|
|
return [
|
|
{
|
|
role: "system",
|
|
content:
|
|
"You are a strict evaluation judge. You are given two answers to the same question: " +
|
|
"answer A produced from the full context, and answer B produced from a compressed context. " +
|
|
"Decide whether B MATERIALLY differs from A (a difference that changes the substance, " +
|
|
"correctness, or key facts — NOT mere wording/format). Reply with exactly one final line: " +
|
|
"`VERDICT: SAME` or `VERDICT: MATERIALLY_DIFFERS`.",
|
|
},
|
|
{
|
|
role: "user",
|
|
content: `Answer A (full context):\n${fullAnswer}\n\nAnswer B (compressed context):\n${compressedAnswer}`,
|
|
},
|
|
];
|
|
}
|
|
|
|
/** PURE verdict parser. Tolerant of case/format; unrecognized output => "unparseable" (never guessed). */
|
|
export function parseJudgeVerdict(raw: string): JudgeVerdict {
|
|
const text = raw.toLowerCase();
|
|
const differs = /materially[_\s-]*differs|differs[_\s]+materially|\bdiffers\b/.test(text);
|
|
const same = /verdict:\s*same|\bsame\b/.test(text);
|
|
if (differs) return "materially-differs";
|
|
if (same) return "same";
|
|
return "unparseable";
|
|
}
|
|
|
|
/**
|
|
* Control pair for the self-test (D-D3 / ponytail discipline). The judge must rank the
|
|
* KNOWN-DEGRADED answer as MATERIALLY_DIFFERS and the KNOWN-GOOD answer as SAME, both relative
|
|
* to the same reference. A judge that mis-ranks either is untrusted and aborts the run.
|
|
*/
|
|
export const CONTROL_PAIR = {
|
|
reference: "The function returns 3 because the input is clamped to the upper bound.",
|
|
good: "It returns 3 since the value is clamped to the maximum allowed.",
|
|
degraded: "It returns 0 because the value is set to zero.",
|
|
} as const;
|
|
|
|
export interface SelfTestResult { passed: boolean; detail: string; }
|
|
|
|
/**
|
|
* Run the control pair through the judge. PASS requires: degraded => materially-differs AND
|
|
* good => same. Any other outcome (including unparseable) FAILS, so the runner aborts before
|
|
* emitting untrusted scores.
|
|
*/
|
|
export async function runSelfTest(client: ModelClient, judgeModel: string): Promise<SelfTestResult> {
|
|
const goodVerdict = parseJudgeVerdict(
|
|
(await client.complete(judgeModel, buildJudgePrompt(CONTROL_PAIR.reference, CONTROL_PAIR.good))).text
|
|
);
|
|
const degradedVerdict = parseJudgeVerdict(
|
|
(await client.complete(judgeModel, buildJudgePrompt(CONTROL_PAIR.reference, CONTROL_PAIR.degraded))).text
|
|
);
|
|
if (degradedVerdict !== "materially-differs") {
|
|
return { passed: false, detail: `judge failed to flag the known-degraded control (got "${degradedVerdict}")` };
|
|
}
|
|
if (goodVerdict !== "same") {
|
|
return { passed: false, detail: `judge flagged the known-good control as "${goodVerdict}" (expected same)` };
|
|
}
|
|
return { passed: true, detail: "control pair ranked correctly" };
|
|
}
|