wehub-resource-sync
440 lines
18 KiB
TypeScript
440 lines
18 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
||
import { readFileSync } from "node:fs";
|
||
import path from "path";
|
||
import { globSync } from "glob";
|
||
import { loadFixtureFile, validateFixtures } from "@copilotkit/aimock";
|
||
import type { ValidationResult } from "@copilotkit/aimock";
|
||
|
||
const REPO_ROOT = path.resolve(__dirname, "..", "..", "..");
|
||
|
||
const fixtureFiles: string[] = [
|
||
...globSync("showcase/aimock/shared/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
}),
|
||
...globSync("showcase/aimock/d4/**/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
}),
|
||
...globSync("showcase/aimock/d6/**/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
}),
|
||
...globSync("examples/integrations/*/fixtures/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
}),
|
||
...globSync("scripts/doc-tests/fixtures/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
}),
|
||
];
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Raw fixture entry with context preserved (loadFixtureFile strips context,
|
||
// but we need it for collision detection).
|
||
// ---------------------------------------------------------------------------
|
||
interface RawFixtureEntry {
|
||
match: {
|
||
userMessage?: string;
|
||
toolCallId?: string;
|
||
toolName?: string;
|
||
model?: string;
|
||
hasToolResult?: boolean;
|
||
turnIndex?: number;
|
||
sequenceIndex?: number;
|
||
endpoint?: string;
|
||
context?: string;
|
||
[key: string]: unknown;
|
||
};
|
||
response: unknown;
|
||
[key: string]: unknown;
|
||
}
|
||
|
||
/**
|
||
* Build a deterministic match key from the match object. The key encodes
|
||
* every field that aimock uses for disambiguation so two fixtures with
|
||
* identical keys would always collide at runtime.
|
||
*/
|
||
function matchKey(match: RawFixtureEntry["match"]): string {
|
||
const parts: string[] = [];
|
||
// Alphabetical, stable order
|
||
if (match.endpoint != null) parts.push(`endpoint=${match.endpoint}`);
|
||
if (match.hasToolResult != null)
|
||
parts.push(`hasToolResult=${match.hasToolResult}`);
|
||
if (match.model != null) parts.push(`model=${match.model}`);
|
||
if (match.sequenceIndex != null)
|
||
parts.push(`sequenceIndex=${match.sequenceIndex}`);
|
||
if (match.toolCallId != null) parts.push(`toolCallId=${match.toolCallId}`);
|
||
if (match.toolName != null) parts.push(`toolName=${match.toolName}`);
|
||
if (match.turnIndex != null) parts.push(`turnIndex=${match.turnIndex}`);
|
||
if (match.userMessage != null) parts.push(`userMessage=${match.userMessage}`);
|
||
return parts.join("|");
|
||
}
|
||
|
||
/** Load raw fixture entries from a JSON file, preserving context. */
|
||
function loadRawFixtures(filePath: string): RawFixtureEntry[] {
|
||
try {
|
||
const data = JSON.parse(readFileSync(filePath, "utf-8"));
|
||
return (data.fixtures ?? []) as RawFixtureEntry[];
|
||
} catch {
|
||
return [];
|
||
}
|
||
}
|
||
|
||
interface TaggedFixture {
|
||
entry: RawFixtureEntry;
|
||
file: string; // relative path
|
||
index: number; // index within file
|
||
}
|
||
|
||
/** Scope key: the context value, or "__shared__" for context-less fixtures. */
|
||
const SHARED_SCOPE = "__shared__";
|
||
|
||
function scopeOf(entry: RawFixtureEntry): string {
|
||
return entry.match.context ?? SHARED_SCOPE;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Deployment scopes — d4 and d6 fixtures never coexist at runtime, so
|
||
// collision detection must check within each deployment's load set:
|
||
// d4 runtime loads: shared/ + d4/
|
||
// d6 runtime loads: shared/ + d6/
|
||
// ---------------------------------------------------------------------------
|
||
const sharedFiles = globSync("showcase/aimock/shared/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
});
|
||
const d4Files = globSync("showcase/aimock/d4/**/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
});
|
||
const d6Files = globSync("showcase/aimock/d6/**/*.json", {
|
||
cwd: REPO_ROOT,
|
||
absolute: true,
|
||
});
|
||
|
||
function tagFiles(files: string[]): TaggedFixture[] {
|
||
return files.flatMap((fp) => {
|
||
const rel = path.relative(REPO_ROOT, fp);
|
||
return loadRawFixtures(fp).map((entry, i) => ({
|
||
entry,
|
||
file: rel,
|
||
index: i,
|
||
}));
|
||
});
|
||
}
|
||
|
||
const sharedTagged = tagFiles(sharedFiles);
|
||
const d4Tagged = tagFiles(d4Files);
|
||
const d6Tagged = tagFiles(d6Files);
|
||
|
||
/** Build a Map<contextScope, TaggedFixture[]> for a deployment's fixture set. */
|
||
function groupByContext(
|
||
fixtures: TaggedFixture[],
|
||
): Map<string, TaggedFixture[]> {
|
||
const map = new Map<string, TaggedFixture[]>();
|
||
for (const t of fixtures) {
|
||
const scope = scopeOf(t.entry);
|
||
if (!map.has(scope)) map.set(scope, []);
|
||
map.get(scope)!.push(t);
|
||
}
|
||
return map;
|
||
}
|
||
|
||
// Two deployment scopes: shared+d4 and shared+d6
|
||
const deploymentScopes: {
|
||
name: string;
|
||
byContext: Map<string, TaggedFixture[]>;
|
||
}[] = [
|
||
{ name: "d4", byContext: groupByContext([...sharedTagged, ...d4Tagged]) },
|
||
{ name: "d6", byContext: groupByContext([...sharedTagged, ...d6Tagged]) },
|
||
];
|
||
|
||
describe("aimock fixtures across repo", () => {
|
||
it("discovers at least one fixture file", () => {
|
||
expect(
|
||
fixtureFiles.length,
|
||
"fixture discovery returned 0 files — misconfigured glob or missing fixtures",
|
||
).toBeGreaterThan(0);
|
||
});
|
||
|
||
for (const filePath of fixtureFiles) {
|
||
const relative = path.relative(REPO_ROOT, filePath);
|
||
it(`${relative} loads and validates with zero errors`, () => {
|
||
const fixtures = loadFixtureFile(filePath);
|
||
|
||
// If loadFixtureFile returns [], the file itself is broken (unreadable,
|
||
// invalid JSON, or missing "fixtures" array). Treat as fatal.
|
||
expect(
|
||
fixtures.length,
|
||
`${relative} produced 0 fixtures — file is unreadable or malformed`,
|
||
).toBeGreaterThan(0);
|
||
|
||
const results = validateFixtures(fixtures);
|
||
const errors = results.filter(
|
||
(r: ValidationResult) => r.severity === "error",
|
||
);
|
||
|
||
if (errors.length > 0) {
|
||
const detail = errors
|
||
.map((e: ValidationResult) => ` [${e.fixtureIndex}] ${e.message}`)
|
||
.join("\n");
|
||
throw new Error(
|
||
`${relative} has ${errors.length} fixture validation error(s):\n${detail}`,
|
||
);
|
||
}
|
||
|
||
expect(errors).toEqual([]);
|
||
});
|
||
}
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Fixture collision detection
|
||
//
|
||
// Each test iterates over deployment scopes (d4, d6) independently because
|
||
// d4 and d6 fixtures never coexist at runtime.
|
||
// ---------------------------------------------------------------------------
|
||
describe("fixture collision detection", () => {
|
||
it("no exact duplicate match keys within the same context scope", () => {
|
||
// Known baseline: 230 duplicates across D6 feature files where different
|
||
// demos share the same pill prompts and toolCallIds. Bumped from 11 → 230
|
||
// when D6 per-integration fixtures were added — each integration's new
|
||
// feature-type fixtures (gen-ui-declarative, multimodal, prebuilt-*,
|
||
// tool-rendering-*-catchall, etc.) naturally share match keys with the
|
||
// pre-existing demo fixtures (render-a2ui, agentic-chat, tool-rendering,
|
||
// gen-ui-tool-based) for the same integration. At runtime these are
|
||
// disambiguated by the active demo / probe path.
|
||
//
|
||
// Bumped 230 → 276 (+46) when the gen-ui-headless-complete.json alias was
|
||
// added across all 18 slugs. The gen-ui-headless-complete probe references
|
||
// a dedicated fixtureFile but drives the same 4 headless-complete pills
|
||
// (weather/stock/highlight/revenue), so its 8 fixtures per slug share
|
||
// match keys with the pre-existing headless-complete.json for that
|
||
// context. These are disambiguated at runtime by the probe's fixtureFile
|
||
// / demo route, exactly like the other cross-feature key overlaps above.
|
||
//
|
||
// Bumped 276 → 288 (+12) when the declarative-gen-ui demo moved to the
|
||
// CopilotKitMiddleware auto-A2UI path across the 3 langgraph integrations
|
||
// (langgraph-python / -typescript / -fastapi). The middleware's inner
|
||
// forced tool is `render_a2ui`, so each integration's gen-ui-declarative.json
|
||
// gained 4 `render_a2ui` fixtures (KPI dashboard / pie / bar / status) that
|
||
// share match keys with the pre-existing render_a2ui entries in that
|
||
// integration's render-a2ui.json (the a2ui_fixed demo). 4 pills × 3
|
||
// integrations = 12. Disambiguated at runtime by the probe's fixtureFile /
|
||
// demo route, like the other cross-feature overlaps above.
|
||
//
|
||
// Bumped 288 → 290 (+2) when the hitl / gen-ui-interrupt / threadid demos
|
||
// were ported to google-adk (W3 parity). The new per-demo google-adk
|
||
// fixtures reuse google-adk's standard prebuilt-probe pills ("hi from the
|
||
// popup/sidebar test"), so they share match keys with the pre-existing
|
||
// prebuilt-popup.json / prebuilt-sidebar.json entries for that context.
|
||
// Disambiguated at runtime by the probe's fixtureFile / demo route, like
|
||
// the other cross-feature overlaps above.
|
||
//
|
||
// Bumped 290 → 291 (+1) in #5427 when BIA tool-rendering.json's bare 'AAPL'
|
||
// matchers were tightened to 'current price of AAPL' to stop shadowing
|
||
// gen-ui-headless-complete.json's 'price of AAPL right now' headless pill.
|
||
// The tightened matchers share keys with tool-rendering-custom-catchall.json's
|
||
// pre-existing 'current price of AAPL' entries in the same BIA context (the
|
||
// hasToolResult:false emitter pair and the hasToolResult:true narration pair).
|
||
// Disambiguated at runtime by feature route (tool-rendering vs custom-catchall
|
||
// fixtureFile) plus the catchall's distinct first prompt ('check Tokyo weather
|
||
// forecast') that gates the multi-pill session before the AAPL pill fires.
|
||
//
|
||
// NOTE: the a2ui-recovery demos (langgraph python/fastapi/typescript +
|
||
// strands python/typescript) deliberately use UNIQUE recovery prompts per
|
||
// framework. Inner render_a2ui fixtures cannot be context-scoped (the in-graph
|
||
// render sub-agent's model client does not forward x-aimock-context), and
|
||
// aimock loads every framework's d6 dir into one process, so identical prompts
|
||
// would let the first-loaded framework's fixture hijack another's render calls.
|
||
// Unique prompts keep each framework's inner fixtures distinct → no new
|
||
// shared-scope duplicates, so this ceiling stays at the pre-recovery baseline.
|
||
//
|
||
// Bumped 291 → 297 (+6) for the Claude SDK demo parity port after
|
||
// de-duplicating avoidable no-context beautiful-chat fallbacks. The
|
||
// remaining new overlaps are context-scoped cross-demo fixture aliases
|
||
// (interrupt/gen-ui-interrupt, declarative/render_a2ui, and copied
|
||
// LangGraph headless/feature-parity routes) that are disambiguated by
|
||
// fixtureFile/demo route like the existing integration parity copies above.
|
||
const KNOWN_DUPLICATE_CEILING = 297;
|
||
|
||
const collisions: string[] = [];
|
||
|
||
for (const { name: deploy, byContext } of deploymentScopes) {
|
||
for (const [ctx, fixtures] of byContext) {
|
||
const seen = new Map<string, TaggedFixture>();
|
||
for (const t of fixtures) {
|
||
const key = matchKey(t.entry.match);
|
||
const prev = seen.get(key);
|
||
if (prev) {
|
||
collisions.push(
|
||
`[${deploy}] context="${ctx}" key="${key}"\n` +
|
||
` first: ${prev.file}[${prev.index}]\n` +
|
||
` second: ${t.file}[${t.index}]`,
|
||
);
|
||
} else {
|
||
seen.set(key, t);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
expect(
|
||
collisions.length,
|
||
`Exact duplicate count (${collisions.length}) exceeds ceiling (${KNOWN_DUPLICATE_CEILING}).\n` +
|
||
`Entries beyond the ceiling (iteration order — NOT necessarily the newly introduced ones; diff against the baseline to find the real offenders):\n${collisions.slice(KNOWN_DUPLICATE_CEILING).join("\n\n")}`,
|
||
).toBeLessThanOrEqual(KNOWN_DUPLICATE_CEILING);
|
||
});
|
||
|
||
it("no substring shadow collisions within the same context scope", () => {
|
||
// A "substring shadow" is when fixture A's userMessage is a substring
|
||
// of fixture B's userMessage, AND they share the same values for all
|
||
// other differentiating fields (toolName, toolCallId, hasToolResult,
|
||
// turnIndex, sequenceIndex, endpoint). In that case aimock's substring
|
||
// matching would cause A to shadow B (or vice versa depending on load
|
||
// order), leading to non-deterministic behavior.
|
||
//
|
||
// Known baseline: 128 pre-existing shadows across d4+d6 (tracked for
|
||
// cleanup). The D6 per-integration feature-type fixtures
|
||
// (tool-rendering-*-catchall, agent-config, gen-ui-interrupt) create
|
||
// expected substring overlaps with pre-existing fixtures in the same
|
||
// context (e.g. "What's the current price of AAPL?" vs "AAPL" in
|
||
// tool-rendering.json, or "tone:professional — ..." vs
|
||
// "tone:professional" in chat-css.json). These are disambiguated at
|
||
// runtime by other match fields (toolCallId, toolName, turnIndex).
|
||
// This test fails if the count INCREASES, preventing new shadows
|
||
// from being introduced. Ratchet down as shadows are cleaned up.
|
||
// Bumped 123→128 in #5412: 5 new substring overlaps in d6/{ag2,cst}
|
||
// gen-ui-declarative + cst/tool-rendering fixtures, runtime-disambiguated
|
||
// by toolCallId chunk boundaries and load-order ordering of inner-call
|
||
// mirrors before outer fixtures (see _meta._note in those files).
|
||
// Bumped 128→134 in #5427: 6 new substring overlaps in
|
||
// d6/built-in-agent/{tool-rendering, tool-rendering-reasoning-chain}
|
||
// fixtures from the BIA 5-tool D6 port (weather/flight/stock/d20/
|
||
// catchall pill variants), runtime-disambiguated by toolName +
|
||
// toolCallId.
|
||
//
|
||
// Ratcheted 134→132 (-2) in #5427 follow-up: BIA tool-rendering.json's
|
||
// bare 'AAPL' matchers were tightened to 'current price of AAPL' (no
|
||
// longer a substring of gen-ui-headless-complete's 'price of AAPL right
|
||
// now' pill), removing 2 pre-existing shadow pairs. The companion
|
||
// sequenceIndex-gated emitter + narration-fallback pairs in
|
||
// gen-ui-headless-complete.json do not introduce new shadows — the
|
||
// narration fallbacks share the same userMessage prefix as the emitters
|
||
// (which the shadow detector skips because identical strings are caught
|
||
// by the exact-duplicate test, not the shadow test).
|
||
// Bumped 132→137: +5 substring shadows from the strands-typescript D6 port
|
||
// (its per-integration fixtures mirror the Python strands sibling —
|
||
// calculator + tool-rendering pill variants), runtime-disambiguated by
|
||
// toolCallId / toolName / turnIndex like the other per-integration copies.
|
||
// Bumped 139→142 after this PR rebased against current main. The remaining
|
||
// counted shadows are pre-existing D4/D6 baseline overlaps (for example
|
||
// weather/AAPL/project-planning/calculator prompt variants), not Claude SDK
|
||
// local fallback aliases. Browser-local Claude demos now get an AIMock
|
||
// context header from server-side HttpAgent defaults instead of relying on
|
||
// context-less prompt aliases.
|
||
const KNOWN_SHADOW_CEILING = 142;
|
||
|
||
const shadows: string[] = [];
|
||
|
||
for (const { name: deploy, byContext } of deploymentScopes) {
|
||
for (const [ctx, fixtures] of byContext) {
|
||
// Only consider fixtures that have a userMessage
|
||
const withMsg = fixtures.filter(
|
||
(t) => typeof t.entry.match.userMessage === "string",
|
||
);
|
||
|
||
for (let i = 0; i < withMsg.length; i++) {
|
||
for (let j = i + 1; j < withMsg.length; j++) {
|
||
const a = withMsg[i];
|
||
const b = withMsg[j];
|
||
const msgA = a.entry.match.userMessage!;
|
||
const msgB = b.entry.match.userMessage!;
|
||
|
||
// Skip if messages are identical (caught by exact-duplicate test)
|
||
if (msgA === msgB) continue;
|
||
|
||
// Check substring relationship
|
||
const aInB = msgB.includes(msgA);
|
||
const bInA = msgA.includes(msgB);
|
||
if (!aInB && !bInA) continue;
|
||
|
||
// Check if other differentiating criteria are identical
|
||
const diffFields = [
|
||
"toolName",
|
||
"toolCallId",
|
||
"hasToolResult",
|
||
"turnIndex",
|
||
"sequenceIndex",
|
||
"endpoint",
|
||
] as const;
|
||
const sameOtherCriteria = diffFields.every(
|
||
(f) => a.entry.match[f] === b.entry.match[f],
|
||
);
|
||
if (!sameOtherCriteria) continue;
|
||
|
||
const shorter = aInB ? a : b;
|
||
const longer = aInB ? b : a;
|
||
shadows.push(
|
||
`[${deploy}] context="${ctx}"\n` +
|
||
` shorter: "${shorter.entry.match.userMessage}" ` +
|
||
`(${shorter.file}[${shorter.index}])\n` +
|
||
` longer: "${longer.entry.match.userMessage}" ` +
|
||
`(${longer.file}[${longer.index}])`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Ratchet: fail if new shadows are introduced; lower the ceiling as
|
||
// pre-existing shadows are cleaned up.
|
||
expect(
|
||
shadows.length,
|
||
`Substring shadow count (${shadows.length}) exceeds ceiling (${KNOWN_SHADOW_CEILING}).\n` +
|
||
`Entries beyond the ceiling (iteration order — NOT necessarily the newly introduced ones; diff against the baseline to find the real offenders):\n${shadows.slice(KNOWN_SHADOW_CEILING).join("\n\n")}`,
|
||
).toBeLessThanOrEqual(KNOWN_SHADOW_CEILING);
|
||
});
|
||
|
||
it("shared (no-context) fixtures have no exact userMessage collisions with scoped fixtures", () => {
|
||
// Shared fixtures (no context field) match ANY context at runtime.
|
||
// If a shared fixture has the same userMessage as a scoped fixture,
|
||
// the match is ambiguous — load order determines which wins.
|
||
const sharedWithMsg = sharedTagged.filter(
|
||
(t) => typeof t.entry.match.userMessage === "string",
|
||
);
|
||
|
||
if (sharedWithMsg.length === 0) return; // nothing to check
|
||
|
||
const collisions: string[] = [];
|
||
|
||
for (const { name: deploy, byContext } of deploymentScopes) {
|
||
for (const [ctx, fixtures] of byContext) {
|
||
if (ctx === SHARED_SCOPE) continue;
|
||
|
||
for (const s of sharedWithMsg) {
|
||
for (const t of fixtures) {
|
||
if (typeof t.entry.match.userMessage !== "string") continue;
|
||
if (s.entry.match.userMessage !== t.entry.match.userMessage)
|
||
continue;
|
||
|
||
collisions.push(
|
||
`[${deploy}] userMessage="${s.entry.match.userMessage}"\n` +
|
||
` shared: ${s.file}[${s.index}]\n` +
|
||
` scoped: ${t.file}[${t.index}] (context="${ctx}")`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (collisions.length > 0) {
|
||
throw new Error(
|
||
`Found ${collisions.length} shared-vs-scoped userMessage collision(s):\n\n${collisions.join("\n\n")}`,
|
||
);
|
||
}
|
||
});
|
||
});
|