wehub-resource-sync
523 lines
19 KiB
TypeScript
523 lines
19 KiB
TypeScript
/**
|
||
* Agent Answer-Content Eval (#511 / #599 follow-up — the CONTENT layer)
|
||
*
|
||
* The director question-answering eval (answering-runner.ts) only checks that
|
||
* the DIRECTOR routes to the teacher. It never generates the teacher's reply,
|
||
* so it cannot catch the failure mode where the director routes correctly but
|
||
* the dispatched agent's FIRST sentence still drifts (greets / opens a lecture /
|
||
* pivots) and only a LATER turn answers — "first sentence drifts, second answers".
|
||
*
|
||
* This eval closes that gap. For each scenario it runs the REAL agent-generate
|
||
* inputs — buildStructuredPrompt() + convertMessagesToOpenAI() — against the
|
||
* model, parses the structured output with the app's runtime parser, and uses an
|
||
* LLM judge to decide:
|
||
* - leads_with_answer : did the FIRST sentence(s) address the literal ask?
|
||
* - answered_anywhere : did the reply address it AT ALL (even if late)?
|
||
* The gap (answered_anywhere && !leads_with_answer) quantifies the
|
||
* drift-then-answer pathology directly.
|
||
*
|
||
* Scenarios are synthetic and anonymized, authored from a real-world failure
|
||
* taxonomy (opening-lecture override, ignored format/capability/navigation
|
||
* request, ignored correction, frustration re-ask, adjacent pivot, vague
|
||
* clarify) plus clean controls. No real user data is included.
|
||
*
|
||
* A/B (mirrors answering-runner's rule-13 strip):
|
||
* - baseline : agent-system with the "Responding to the User's Turn" section stripped
|
||
* - with_rule : agent-system as-shipped
|
||
*
|
||
* Pass criterion: with_rule mean leads_with_answer rate >= EVAL_PASS_THRESHOLD
|
||
* (default 0.7). Δ vs baseline is reported as informational.
|
||
*
|
||
* Required env:
|
||
* EVAL_AGENT_MODEL Model used to generate the agent reply (or DEFAULT_MODEL)
|
||
* EVAL_JUDGE_MODEL Model used as the answer judge
|
||
* Optional env:
|
||
* EVAL_SAMPLES Samples per (scenario, variant). Default 3.
|
||
* EVAL_PASS_THRESHOLD Min with_rule leads rate per scenario. Default 0.7.
|
||
* EVAL_SCENARIO Filter to a single scenario by case_id.
|
||
*
|
||
* Output: eval/orchestration/results-answer-content/<model>/<timestamp>/report.md
|
||
*/
|
||
|
||
import fs from 'fs';
|
||
import path from 'path';
|
||
import { fileURLToPath } from 'url';
|
||
import { callLLM } from '@/lib/ai/llm';
|
||
import { buildStructuredPrompt } from '@/lib/orchestration/prompt-builder';
|
||
import { convertMessagesToOpenAI } from '@/lib/orchestration/summarizers/message-converter';
|
||
import {
|
||
createParserState,
|
||
parseStructuredChunk,
|
||
finalizeParser,
|
||
} from '@/lib/orchestration/stateless-generate';
|
||
import { type AgentConfig, getActionsForRole } from '@/lib/orchestration/registry/types';
|
||
import type { AgentTurnSummary } from '@/lib/orchestration/types';
|
||
import type { StatelessChatRequest } from '@/lib/types/chat';
|
||
import { resolveEvalModel } from '../shared/resolve-model';
|
||
import { createRunDir } from '../shared/run-dir';
|
||
import { judgeAnswer, type AnswerVerdict } from './answer-content-judge';
|
||
import type { ScenarioAgent } from './types';
|
||
|
||
const OUTPUT_DIR = 'eval/orchestration/results-answer-content';
|
||
|
||
// ==================== Types ====================
|
||
|
||
interface ScenarioAgentSpec extends ScenarioAgent {
|
||
persona: string;
|
||
}
|
||
|
||
interface ScenarioTurn {
|
||
role: 'user' | 'agent';
|
||
agentId?: string;
|
||
text: string;
|
||
}
|
||
|
||
interface ContentScenario {
|
||
case_id: string;
|
||
category: string;
|
||
description: string;
|
||
agents: ScenarioAgentSpec[];
|
||
teacherAgentId: string;
|
||
turns: ScenarioTurn[];
|
||
answerKey: string;
|
||
expectedPreFix?: string;
|
||
/** Classroom context so the assembled prompt matches the live shape/bulk:
|
||
* a topical slide scene, the stage (incl. languageDirective), and the
|
||
* student profile. Shapes mirror StatelessChatRequest.storeState. Use `scenes`
|
||
* + `currentSceneId` for a multi-slide deck (e.g. so "skip to next page" is
|
||
* well-posed); `scene` is the single-slide shorthand. */
|
||
scene?: unknown;
|
||
scenes?: unknown[];
|
||
currentSceneId?: string;
|
||
stage?: unknown;
|
||
userProfile?: { nickname?: string; bio?: string };
|
||
mode?: 'autonomous' | 'playback';
|
||
}
|
||
|
||
type Variant = 'baseline' | 'with_rule';
|
||
|
||
interface SampleResult {
|
||
variant: Variant;
|
||
leadText: string;
|
||
fullText: string;
|
||
verdict: AnswerVerdict;
|
||
error?: string;
|
||
}
|
||
|
||
interface VariantAgg {
|
||
samples: SampleResult[];
|
||
leadsRate: number;
|
||
answeredRate: number;
|
||
}
|
||
|
||
interface ScenarioResult {
|
||
case_id: string;
|
||
category: string;
|
||
description: string;
|
||
baseline: VariantAgg;
|
||
withRule: VariantAgg;
|
||
delta: number;
|
||
passes: boolean;
|
||
}
|
||
|
||
// ==================== Input construction ====================
|
||
|
||
/** Build a full AgentConfig for the teacher from the scenario spec. */
|
||
function buildTeacherConfig(scenario: ContentScenario): AgentConfig {
|
||
const spec = scenario.agents.find((a) => a.id === scenario.teacherAgentId) ?? scenario.agents[0];
|
||
return {
|
||
id: spec.id,
|
||
name: spec.name,
|
||
role: spec.role,
|
||
persona: spec.persona,
|
||
avatar: '🧑🏫',
|
||
color: '#6d28d9',
|
||
// Use the canonical role action set (incl. play_video + full whiteboard) so
|
||
// the assembled prompt's available-actions section matches production.
|
||
allowedActions: getActionsForRole(spec.role),
|
||
priority: spec.priority,
|
||
createdAt: new Date(0),
|
||
updatedAt: new Date(0),
|
||
isDefault: true,
|
||
};
|
||
}
|
||
|
||
/** Store state from the scenario's classroom context (slide scene + stage),
|
||
* so buildStructuredPrompt produces the same "# Current State" shape/bulk as
|
||
* the live agent-generate path. Falls back to an empty state if a scenario
|
||
* omits context. */
|
||
function buildStoreState(scenario: ContentScenario): StatelessChatRequest['storeState'] {
|
||
const scenes = scenario.scenes ?? (scenario.scene ? [scenario.scene] : []);
|
||
const currentSceneId =
|
||
scenario.currentSceneId ?? (scenes[0] ? ((scenes[0] as { id?: string }).id ?? null) : null);
|
||
return {
|
||
stage: scenario.stage ?? null,
|
||
scenes,
|
||
currentSceneId,
|
||
mode: scenario.mode ?? 'playback',
|
||
whiteboardOpen: false,
|
||
} as unknown as StatelessChatRequest['storeState'];
|
||
}
|
||
|
||
/** Turn list -> the UIMessage[] shape convertMessagesToOpenAI expects. */
|
||
function buildMessages(scenario: ContentScenario): StatelessChatRequest['messages'] {
|
||
const nameById = new Map(scenario.agents.map((a) => [a.id, a.name]));
|
||
const messages = scenario.turns.map((turn, i) => {
|
||
if (turn.role === 'user') {
|
||
return {
|
||
id: `user-${i}`,
|
||
role: 'user' as const,
|
||
parts: [{ type: 'text', text: turn.text }],
|
||
metadata: { senderName: 'You', originalRole: 'user', createdAt: i },
|
||
};
|
||
}
|
||
const agentId = turn.agentId ?? scenario.teacherAgentId;
|
||
return {
|
||
id: `assistant-${i}`,
|
||
role: 'assistant' as const,
|
||
parts: [{ type: 'text', text: turn.text }],
|
||
metadata: { agentId, senderName: nameById.get(agentId) ?? agentId, createdAt: i },
|
||
};
|
||
});
|
||
return messages as unknown as StatelessChatRequest['messages'];
|
||
}
|
||
|
||
/** Prior agent turns -> AgentTurnSummary[] for peer context in the prompt. */
|
||
function buildAgentResponses(scenario: ContentScenario): AgentTurnSummary[] {
|
||
const nameById = new Map(scenario.agents.map((a) => [a.id, a.name]));
|
||
return scenario.turns
|
||
.filter((t) => t.role === 'agent')
|
||
.map((t) => {
|
||
const agentId = t.agentId ?? scenario.teacherAgentId;
|
||
return {
|
||
agentId,
|
||
agentName: nameById.get(agentId) ?? agentId,
|
||
contentPreview: t.text.slice(0, 200),
|
||
actionCount: 0,
|
||
whiteboardActions: [],
|
||
};
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Remove the "# Responding to the User's Turn" section from an assembled agent
|
||
* system prompt to build the pre-fix baseline. Bounded by the next section header.
|
||
*/
|
||
function stripAnsweringSection(prompt: string): string {
|
||
const re = /\n# Responding to the User's Turn[\s\S]*?(?=\n# Current State)/;
|
||
if (!re.test(prompt)) {
|
||
throw new Error(
|
||
'answer-content-runner: "# Responding to the User\'s Turn" section not found in agent prompt; baseline cannot be constructed',
|
||
);
|
||
}
|
||
return prompt.replace(re, '\n');
|
||
}
|
||
|
||
/** Extract ordered text blocks from a structured agent response. */
|
||
function extractTexts(raw: string): string[] {
|
||
const state = createParserState();
|
||
const streamed = parseStructuredChunk(raw, state);
|
||
if (streamed.textChunks.length > 0) return streamed.textChunks;
|
||
// Recover only when the model emitted plain prose (never opened a JSON array) —
|
||
// finalizeParser then surfaces that prose as text. If an array WAS started but
|
||
// yielded no text (e.g. actions-only), the correct result is "no text"; we must
|
||
// not fall through to finalizeParser's raw-buffer fallback, which would surface
|
||
// action JSON as fake speech.
|
||
if (!state.jsonStarted) return finalizeParser(state).textChunks;
|
||
return [];
|
||
}
|
||
|
||
/** Split into sentences across both Latin and CJK terminators, keeping order. */
|
||
function splitSentences(s: string): string[] {
|
||
return s
|
||
.split(/(?<=[.!?。！？])\s*/)
|
||
.map((x) => x.trim())
|
||
.filter(Boolean);
|
||
}
|
||
|
||
/**
|
||
* The "opening" judged for leads_with_answer: the first ~2 speech sentences
|
||
* across ALL ordered text blocks (not just the first block — a reply may be
|
||
* several `type:"text"` items, e.g. [{"Sure."},{"The derivative is 2x."}]). Two
|
||
* sentences let the judge distinguish a brief acknowledgement-then-answer
|
||
* ("Sure. The derivative is 2x."), which the prompt allows, from a
|
||
* greeting/lecture preamble before the answer ("Welcome! …" / "Today we'll
|
||
* discuss parabolas. The formula is…"), which is the drift this eval catches.
|
||
*/
|
||
function leadFromTexts(texts: string[]): string {
|
||
const joined = texts.join(' ');
|
||
const sentences = splitSentences(joined);
|
||
if (sentences.length === 0) return joined;
|
||
return sentences.slice(0, 2).join(' ');
|
||
}
|
||
|
||
// ==================== Sampling ====================
|
||
|
||
function lastUserMessage(scenario: ContentScenario): string {
|
||
for (let i = scenario.turns.length - 1; i >= 0; i--) {
|
||
if (scenario.turns[i].role === 'user') return scenario.turns[i].text;
|
||
}
|
||
return '';
|
||
}
|
||
|
||
async function sampleVariant(
|
||
scenario: ContentScenario,
|
||
variant: Variant,
|
||
systemPrompt: string,
|
||
openaiMessages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
|
||
agentModel: Awaited<ReturnType<typeof resolveEvalModel>>['model'],
|
||
judgeModel: Awaited<ReturnType<typeof resolveEvalModel>>['model'],
|
||
studentMessage: string,
|
||
samples: number,
|
||
): Promise<SampleResult[]> {
|
||
const tasks = Array.from({ length: samples }, async (): Promise<SampleResult> => {
|
||
try {
|
||
const gen = await callLLM(
|
||
{
|
||
model: agentModel,
|
||
messages: [{ role: 'system', content: systemPrompt }, ...openaiMessages],
|
||
},
|
||
`eval-answer-content-${variant}`,
|
||
);
|
||
const texts = extractTexts(gen.text);
|
||
const leadText = leadFromTexts(texts);
|
||
const fullText = texts.join(' ');
|
||
const verdict = await judgeAnswer(
|
||
judgeModel,
|
||
studentMessage,
|
||
scenario.answerKey,
|
||
leadText,
|
||
fullText,
|
||
);
|
||
return { variant, leadText, fullText, verdict };
|
||
} catch (err) {
|
||
const msg = err instanceof Error ? err.message : String(err);
|
||
return {
|
||
variant,
|
||
leadText: '',
|
||
fullText: '',
|
||
verdict: { leads_with_answer: false, answered_anywhere: false, reason: msg, error: true },
|
||
error: msg,
|
||
};
|
||
}
|
||
});
|
||
return Promise.all(tasks);
|
||
}
|
||
|
||
function aggregate(samples: SampleResult[]): VariantAgg {
|
||
// Errors count as failures for an eval gate: the denominator is ALL requested
|
||
// samples, so a scenario cannot "pass" on one good sample while the rest error
|
||
// out. An errored sample (generation or judge failure) contributes 0.
|
||
const n = samples.length || 1;
|
||
const ok = (s: SampleResult) => !s.error && !s.verdict.error;
|
||
const leadsRate = samples.filter((s) => ok(s) && s.verdict.leads_with_answer).length / n;
|
||
const answeredRate = samples.filter((s) => ok(s) && s.verdict.answered_anywhere).length / n;
|
||
return { samples, leadsRate, answeredRate };
|
||
}
|
||
|
||
// ==================== Reporting ====================
|
||
|
||
function pct(x: number): string {
|
||
return `${Math.round(x * 100)}%`;
|
||
}
|
||
|
||
function writeReport(
|
||
runDir: string,
|
||
results: ScenarioResult[],
|
||
modelStr: string,
|
||
judgeStr: string,
|
||
samples: number,
|
||
threshold: number,
|
||
): string {
|
||
const lines: string[] = [];
|
||
const overallPass = results.every((r) => r.passes);
|
||
const meanBaseLeads = results.reduce((a, r) => a + r.baseline.leadsRate, 0) / results.length;
|
||
const meanRuleLeads = results.reduce((a, r) => a + r.withRule.leadsRate, 0) / results.length;
|
||
const meanRuleAnswered =
|
||
results.reduce((a, r) => a + r.withRule.answeredRate, 0) / results.length;
|
||
|
||
lines.push(`# Agent Answer-Content Eval`, ``);
|
||
lines.push(`- **Date**: ${new Date().toISOString()}`);
|
||
lines.push(`- **Agent model**: ${modelStr}`);
|
||
lines.push(`- **Judge model**: ${judgeStr}`);
|
||
lines.push(`- **Samples per (scenario, variant)**: ${samples}`);
|
||
lines.push(`- **with_rule leads-with-answer threshold**: ${pct(threshold)}`);
|
||
lines.push(``);
|
||
lines.push(`## Aggregate`, ``);
|
||
lines.push(`| Variant | Mean leads-with-answer | Mean answered-anywhere |`);
|
||
lines.push(`|---|---|---|`);
|
||
lines.push(`| baseline (no answering rule) | ${pct(meanBaseLeads)} | — |`);
|
||
lines.push(`| with_rule (as-shipped) | ${pct(meanRuleLeads)} | ${pct(meanRuleAnswered)} |`);
|
||
lines.push(``);
|
||
lines.push(
|
||
`**Drift-then-answer gap (with_rule)**: answered-anywhere ${pct(meanRuleAnswered)} − leads ${pct(meanRuleLeads)} = **${pct(Math.max(0, meanRuleAnswered - meanRuleLeads))}** of replies answered only AFTER a drifting opener.`,
|
||
);
|
||
lines.push(``);
|
||
lines.push(`Overall verdict: **${overallPass ? 'PASS' : 'FAIL'}**`, ``);
|
||
|
||
lines.push(`## Per scenario`, ``);
|
||
lines.push(
|
||
`| # | case_id | category | baseline leads | with_rule leads | with_rule answered | Δ leads | pass? |`,
|
||
);
|
||
lines.push(`|---|---|---|---|---|---|---|---|`);
|
||
results.forEach((r, i) => {
|
||
lines.push(
|
||
`| ${i + 1} | ${r.case_id} | ${r.category} | ${pct(r.baseline.leadsRate)} | ${pct(r.withRule.leadsRate)} | ${pct(r.withRule.answeredRate)} | ${pct(r.delta)} | ${r.passes ? '✓' : '✗'} |`,
|
||
);
|
||
});
|
||
lines.push(``);
|
||
|
||
lines.push(`## Detail`, ``);
|
||
for (const r of results) {
|
||
lines.push(`### ${r.case_id} ${r.passes ? '✓' : '✗'}`, ``);
|
||
lines.push(`- ${r.description}`);
|
||
lines.push(
|
||
`- baseline leads ${pct(r.baseline.leadsRate)}; with_rule leads ${pct(r.withRule.leadsRate)} / answered ${pct(r.withRule.answeredRate)}; Δ leads ${pct(r.delta)}`,
|
||
);
|
||
lines.push(``, `<details><summary>with_rule samples</summary>`, ``);
|
||
for (const s of r.withRule.samples) {
|
||
if (s.error) {
|
||
lines.push(`- ERROR: ${s.error}`);
|
||
continue;
|
||
}
|
||
const tag = s.verdict.leads_with_answer
|
||
? 'LEADS'
|
||
: s.verdict.answered_anywhere
|
||
? 'DRIFT→answered'
|
||
: 'DRIFT';
|
||
lines.push(`- **${tag}** — lead: "${s.leadText.slice(0, 140)}" — ${s.verdict.reason}`);
|
||
}
|
||
lines.push(``, `</details>`, ``);
|
||
}
|
||
|
||
const reportPath = path.join(runDir, 'report.md');
|
||
fs.writeFileSync(reportPath, lines.join('\n'));
|
||
return reportPath;
|
||
}
|
||
|
||
// ==================== Main ====================
|
||
|
||
function getCurrentDir(): string {
|
||
return typeof __dirname !== 'undefined'
|
||
? __dirname
|
||
: path.dirname(fileURLToPath(import.meta.url));
|
||
}
|
||
|
||
function loadScenarios(): ContentScenario[] {
|
||
const p = path.join(getCurrentDir(), 'scenarios/answer-content.json');
|
||
const scenarios = JSON.parse(fs.readFileSync(p, 'utf-8')) as ContentScenario[];
|
||
const filter = process.env.EVAL_SCENARIO;
|
||
return filter ? scenarios.filter((s) => s.case_id === filter) : scenarios;
|
||
}
|
||
|
||
async function main() {
|
||
const modelStr = process.env.EVAL_AGENT_MODEL || process.env.DEFAULT_MODEL;
|
||
const judgeStr = process.env.EVAL_JUDGE_MODEL;
|
||
if (!modelStr) {
|
||
console.error(
|
||
'Error: EVAL_AGENT_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_AGENT_MODEL=google:gemini-3-flash-preview',
|
||
);
|
||
process.exit(1);
|
||
}
|
||
if (!judgeStr) {
|
||
console.error(
|
||
'Error: EVAL_JUDGE_MODEL must be set. Example: EVAL_JUDGE_MODEL=anthropic:claude-haiku-4-5',
|
||
);
|
||
process.exit(1);
|
||
}
|
||
const samples = Number(process.env.EVAL_SAMPLES || '3');
|
||
const threshold = Number(process.env.EVAL_PASS_THRESHOLD || '0.7');
|
||
|
||
console.log('=== Agent Answer-Content Eval ===');
|
||
console.log(`Agent: ${modelStr} | Judge: ${judgeStr} | Samples/variant: ${samples}`);
|
||
|
||
const { model: agentModel } = await resolveEvalModel(
|
||
'EVAL_AGENT_MODEL',
|
||
process.env.DEFAULT_MODEL,
|
||
);
|
||
const { model: judgeModel } = await resolveEvalModel('EVAL_JUDGE_MODEL');
|
||
const scenarios = loadScenarios();
|
||
if (scenarios.length === 0) {
|
||
const filter = process.env.EVAL_SCENARIO;
|
||
console.error(
|
||
filter
|
||
? `No scenario matches EVAL_SCENARIO="${filter}".`
|
||
: 'No scenarios found in scenarios/answer-content.json.',
|
||
);
|
||
process.exit(1);
|
||
}
|
||
console.log(`Loaded ${scenarios.length} scenario(s)`);
|
||
const runDir = createRunDir(OUTPUT_DIR, modelStr);
|
||
console.log(`Output: ${runDir}`);
|
||
|
||
const results: ScenarioResult[] = [];
|
||
for (const sc of scenarios) {
|
||
process.stdout.write(` - ${sc.case_id} ... `);
|
||
|
||
const teacher = buildTeacherConfig(sc);
|
||
const storeState = buildStoreState(sc);
|
||
const agentResponses = buildAgentResponses(sc);
|
||
const withRulePrompt = buildStructuredPrompt(
|
||
teacher,
|
||
storeState,
|
||
undefined,
|
||
[],
|
||
sc.userProfile,
|
||
agentResponses,
|
||
);
|
||
const baselinePrompt = stripAnsweringSection(withRulePrompt);
|
||
const openaiMessages = convertMessagesToOpenAI(buildMessages(sc), sc.teacherAgentId);
|
||
const studentMessage = lastUserMessage(sc);
|
||
|
||
const [bs, ws] = await Promise.all([
|
||
sampleVariant(
|
||
sc,
|
||
'baseline',
|
||
baselinePrompt,
|
||
openaiMessages,
|
||
agentModel,
|
||
judgeModel,
|
||
studentMessage,
|
||
samples,
|
||
),
|
||
sampleVariant(
|
||
sc,
|
||
'with_rule',
|
||
withRulePrompt,
|
||
openaiMessages,
|
||
agentModel,
|
||
judgeModel,
|
||
studentMessage,
|
||
samples,
|
||
),
|
||
]);
|
||
const baseline = aggregate(bs);
|
||
const withRule = aggregate(ws);
|
||
const delta = withRule.leadsRate - baseline.leadsRate;
|
||
const passes = withRule.leadsRate >= threshold;
|
||
results.push({
|
||
case_id: sc.case_id,
|
||
category: sc.category,
|
||
description: sc.description,
|
||
baseline,
|
||
withRule,
|
||
delta,
|
||
passes,
|
||
});
|
||
console.log(
|
||
`baseline=${pct(baseline.leadsRate)} with_rule=${pct(withRule.leadsRate)} answered=${pct(withRule.answeredRate)} ${passes ? 'PASS' : 'FAIL'}`,
|
||
);
|
||
}
|
||
|
||
const reportPath = writeReport(runDir, results, modelStr, judgeStr, samples, threshold);
|
||
const overallPass = results.every((r) => r.passes);
|
||
console.log(`\nReport: ${reportPath}`);
|
||
console.log(`Verdict: ${overallPass ? 'PASS' : 'FAIL'}`);
|
||
process.exit(overallPass ? 0 : 1);
|
||
}
|
||
|
||
main().catch((err) => {
|
||
console.error('Fatal error:', err);
|
||
process.exit(1);
|
||
});
|