chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
// tests/integration/combo-matrix/auto.test.ts
|
||||
//
|
||||
// Integration tests for `model: "auto"` and `model: "auto/<variant>"` routing.
|
||||
//
|
||||
// How auto routing works:
|
||||
// - model:"auto" → chat.ts detects isAutoRouting → calls createVirtualAutoCombo(undefined)
|
||||
// which reads active connections from DB and builds a virtual combo (name="auto", id="auto").
|
||||
// - The virtual combo's default routerStrategy is "lkgp".
|
||||
// - combo.ts fetches getLKGP(combo.name, combo.id) = getLKGP("auto", "auto") → key "auto:auto".
|
||||
// - LKGPStrategy: if a LKGP record exists for that provider AND the provider is in the
|
||||
// candidate pool, it pins that provider deterministically (no randomness).
|
||||
// - Without a LKGP record, falls back to "rules" 9-factor scorer.
|
||||
//
|
||||
// Test 1 strategy: single-request, deterministic.
|
||||
// We seed a LKGP record pointing to "openai" via settingsDb.setLKGP("auto","auto","openai").
|
||||
// The assertion h.providersSeen()[0] === "openai" is guaranteed by the LKGP pin
|
||||
// and will FAIL if the LKGP lookup or the LKGPStrategy routing is broken.
|
||||
//
|
||||
// Test 2 strategy: pool-resolution check (not provider-specific).
|
||||
// Send model:"auto/coding" → virtual combo for the "coding" variant (MODE_PACKS quality-first).
|
||||
// Assert status 200 AND a seeded provider was dispatched. The pool should resolve from the
|
||||
// DB connections; this test fails if the variant is unrecognised (400) or the pool is empty.
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-auto");
|
||||
const { BaseExecutor, handleChat, buildRequest, seedConnection, resetStorage, settingsDb } = h;
|
||||
|
||||
function body(model: string) {
|
||||
return { model, stream: false, messages: [{ role: "user", content: "hello" }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Test 1: auto picks the LKGP-biased provider ─────────────────────────────
|
||||
//
|
||||
// Strategy: single-request, deterministic via LKGP seed.
|
||||
// The virtual auto-combo for model:"auto" has name="auto" and id="auto".
|
||||
// getLKGP("auto","auto") reads key "auto:auto" in the key_value table.
|
||||
// LKGPStrategy pins the seeded provider when it is healthy + in the pool.
|
||||
// This assertion fails if LKGP lookup, LKGPStrategy, or candidate resolution breaks.
|
||||
test("auto: LKGP record biases dispatch toward the seeded provider", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-auto" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-auto" });
|
||||
|
||||
// Bias toward openai by writing a LKGP record before the request.
|
||||
await settingsDb.setLKGP("auto", "auto", "openai");
|
||||
|
||||
h.installRecordingFetch();
|
||||
const r = await handleChat(buildRequest({ body: body("auto") }));
|
||||
assert.equal(r.status, 200, `Expected 200, got ${r.status}`);
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.length > 0, "Expected at least one upstream dispatch");
|
||||
assert.equal(
|
||||
seen[0],
|
||||
"openai",
|
||||
`Expected openai to be dispatched first (LKGP-biased), got: ${JSON.stringify(seen)}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Test 2: auto/coding resolves a virtual pool from seeded connections ──────
|
||||
//
|
||||
// "auto/coding" is a recognized variant (VALID_AUTO_VARIANTS has "coding").
|
||||
// createVirtualAutoCombo("coding") applies MODE_PACKS["quality-first"] weights
|
||||
// and uses the same DB connections as candidates.
|
||||
// This test fails if:
|
||||
// - the variant is not recognized and returns a 400, or
|
||||
// - the virtual pool is empty (no candidates resolved from DB connections), or
|
||||
// - the dispatched provider is not one of the seeded ones.
|
||||
test("auto/coding: virtual pool resolves and dispatches a seeded provider", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-coding" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-coding" });
|
||||
|
||||
h.installRecordingFetch();
|
||||
const r = await handleChat(buildRequest({ body: body("auto/coding") }));
|
||||
assert.equal(r.status, 200, `Expected 200 from auto/coding, got ${r.status}`);
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.length > 0, "Expected at least one upstream dispatch for auto/coding");
|
||||
|
||||
const seededProviders = new Set(["openai", "claude"]);
|
||||
assert.ok(
|
||||
seededProviders.has(seen[0]),
|
||||
`Expected a seeded provider (openai or claude), got: ${seen[0]} (all: ${JSON.stringify(seen)})`
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,367 @@
|
||||
// tests/integration/combo-matrix/context-relay-codex.test.ts
|
||||
//
|
||||
// Deterministic in-process test for context-relay CODEX-SPECIFIC HANDOFF
|
||||
// (combo.ts ~2143-2183): the quota-aware handoff that fires after a successful
|
||||
// codex response when the quota exceeds the threshold.
|
||||
//
|
||||
// Code path:
|
||||
// if (strategy === "context-relay" && relayOptions?.sessionId && relayConfig &&
|
||||
// relayConfig.handoffProviders.includes(provider) && provider === "codex") {
|
||||
// const connectionId = getSessionConnection(relayOptions.sessionId);
|
||||
// if (connectionId) {
|
||||
// const quotaInfo = await fetchCodexQuota(connectionId).catch(() => null);
|
||||
// if (quotaInfo) { ... maybeGenerateHandoff({ ..., expiresAt: resetCandidates[0] }); }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// PRIMARY observable: getHandoff(sessionId, comboName).expiresAt === session-window
|
||||
// reset time from the codex quota response. This is the CODEX-SPECIFIC proof:
|
||||
// the universal handoff path does NOT set expiresAt from a codex quota fetch.
|
||||
//
|
||||
// SECONDARY observables:
|
||||
// - The codex usage URL was fetched (proves the quota path ran).
|
||||
// - The handoff record has a non-empty .summary.
|
||||
//
|
||||
// CONTROL: combo targeting openai (not codex) → codex block NEVER runs →
|
||||
// codex usage URL is never fetched → expiresAt from quota doesn't appear.
|
||||
//
|
||||
// Seams used:
|
||||
// - registerCodexConnection / unregisterCodexConnection (codexQuotaFetcher.ts)
|
||||
// - clearSessions (sessionManager.ts) — cleanup between tests
|
||||
// - buildCodexResponsesSse helper (mirrors codex-stream-false.test.ts)
|
||||
// - Custom fetch mock distinguishes quota URL / summary dispatch / main dispatch
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
import {
|
||||
registerCodexConnection,
|
||||
unregisterCodexConnection,
|
||||
} from "../../../open-sse/services/codexQuotaFetcher.ts";
|
||||
import { clearSessions } from "../../../open-sse/services/sessionManager.ts";
|
||||
|
||||
// ── Harness setup ─────────────────────────────────────────────────────────────
|
||||
// Each harness owns an isolated DATA_DIR so DB handles don't clash.
|
||||
const h = await createComboRoutingHarness("combo-relay-codex");
|
||||
const {
|
||||
BaseExecutor,
|
||||
combosDb,
|
||||
handleChat,
|
||||
buildRequest,
|
||||
seedConnection,
|
||||
resetStorage,
|
||||
buildOpenAIResponse,
|
||||
waitFor,
|
||||
toPlainHeaders,
|
||||
} = h;
|
||||
|
||||
// Import DB helpers AFTER harness creation so they share the same DB instance
|
||||
// (DATA_DIR is set by the harness before any import triggers DB init).
|
||||
const { getHandoff } = await import("../../../src/lib/db/contextHandoffs.ts");
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const CODEX_COMBO_NAME = "m-relay-codex-quota";
|
||||
const SESSION_HEADER_VALUE = "relay-codex-quota-001";
|
||||
const SESSION_ID = `ext:${SESSION_HEADER_VALUE}`;
|
||||
|
||||
// Codex endpoint URLs (must match codexQuotaFetcher.ts and the executor config).
|
||||
const CODEX_USAGE_URL = "https://chatgpt.com/backend-api/wham/usage";
|
||||
const CODEX_RESPONSES_HOST = "chatgpt.com/backend-api/codex/responses";
|
||||
|
||||
// Summary JSON that parseHandoffJSON will successfully parse.
|
||||
const CODEX_SUMMARY_JSON = JSON.stringify({
|
||||
summary:
|
||||
"User is implementing a TypeScript context-relay codex quota-handoff test using TDD.",
|
||||
keyDecisions: ["codex provider selected", "quota threshold at 90%"],
|
||||
taskProgress: "writing deterministic integration test for codex handoff",
|
||||
activeEntities: ["combo.ts", "codexQuotaFetcher.ts", "contextHandoff.ts"],
|
||||
});
|
||||
|
||||
// ── SSE builder (mirrors codex-stream-false.test.ts) ─────────────────────────
|
||||
|
||||
function buildCodexResponsesSse(text = "codex assistant reply") {
|
||||
return new Response(
|
||||
[
|
||||
"event: response.created",
|
||||
'data: {"type":"response.created","response":{"id":"resp_codex_1","model":"gpt-5.3-codex","status":"in_progress","output":[]}}',
|
||||
"",
|
||||
"event: response.output_text.delta",
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.output_text.delta",
|
||||
output_index: 0,
|
||||
delta: text,
|
||||
})}`,
|
||||
"",
|
||||
"event: response.completed",
|
||||
`data: ${JSON.stringify({
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_codex_1",
|
||||
object: "response",
|
||||
model: "gpt-5.3-codex",
|
||||
status: "completed",
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
role: "assistant",
|
||||
content: [{ type: "output_text", text }],
|
||||
},
|
||||
],
|
||||
usage: { input_tokens: 6, output_tokens: 4 },
|
||||
},
|
||||
})}`,
|
||||
"",
|
||||
"data: [DONE]",
|
||||
"",
|
||||
].join("\n"),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/event-stream" },
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// ── Quota response builder ────────────────────────────────────────────────────
|
||||
// Produces a codex usage API response that parseCodexUsageResponse accepts.
|
||||
// primary_window → session → percentUsed = 0.90 (90%) — triggers handoff (threshold 0.85)
|
||||
// secondary_window → weekly → percentUsed = 0.85 (85%) — below threshold for weekly
|
||||
// Both are below HANDOFF_EXHAUSTION_THRESHOLD (0.95) so maybeGenerateHandoff proceeds.
|
||||
|
||||
function buildCodexUsageBody(
|
||||
sessionResetAtUnix: number,
|
||||
weeklyResetAtUnix: number
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
rate_limit: {
|
||||
primary_window: { used_percent: 90, reset_at: sessionResetAtUnix },
|
||||
secondary_window: { used_percent: 85, reset_at: weeklyResetAtUnix },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Request builder ───────────────────────────────────────────────────────────
|
||||
|
||||
function codexRequest(withSessionId = true) {
|
||||
return buildRequest({
|
||||
headers: withSessionId ? { "x-session-id": SESSION_HEADER_VALUE } : {},
|
||||
body: {
|
||||
model: CODEX_COMBO_NAME,
|
||||
stream: false,
|
||||
messages: [{ role: "user", content: "Write a TypeScript hello world." }],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Fetch mock ────────────────────────────────────────────────────────────────
|
||||
// Handles three URL classes:
|
||||
// 1. CODEX_USAGE_URL → codex quota JSON
|
||||
// 2. CODEX_RESPONSES_HOST (call 1) → main request Responses API SSE
|
||||
// 3. CODEX_RESPONSES_HOST (call 2) → summary request Responses API SSE (with CODEX_SUMMARY_JSON)
|
||||
//
|
||||
// The second codex-responses call is the handoff summary generation, identified by
|
||||
// call count (the first is always the main request).
|
||||
|
||||
function installCodexHandoffFetch(
|
||||
sessionResetAtUnix: number,
|
||||
weeklyResetAtUnix: number,
|
||||
seenUrls: string[]
|
||||
) {
|
||||
let codexResponsesCallCount = 0;
|
||||
|
||||
globalThis.fetch = async (url: unknown, init: unknown = {}) => {
|
||||
const u = String(url);
|
||||
seenUrls.push(u);
|
||||
|
||||
const headers =
|
||||
init != null && typeof init === "object"
|
||||
? toPlainHeaders((init as Record<string, unknown>).headers)
|
||||
: {};
|
||||
void headers; // recorded for debugging; not needed in assertions here
|
||||
|
||||
if (u === CODEX_USAGE_URL) {
|
||||
// Codex quota endpoint — return structured usage JSON.
|
||||
return new Response(
|
||||
JSON.stringify(buildCodexUsageBody(sessionResetAtUnix, weeklyResetAtUnix)),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } }
|
||||
);
|
||||
}
|
||||
|
||||
if (u.includes(CODEX_RESPONSES_HOST)) {
|
||||
codexResponsesCallCount++;
|
||||
if (codexResponsesCallCount === 1) {
|
||||
// Main codex request — return a successful Responses API SSE.
|
||||
return buildCodexResponsesSse("codex assistant reply ok");
|
||||
}
|
||||
// Subsequent call(s) are from the handoff summary generation.
|
||||
// Return Responses API SSE whose output_text is CODEX_SUMMARY_JSON so that
|
||||
// generateHandoffAsync → parseHandoffJSON succeeds.
|
||||
return buildCodexResponsesSse(CODEX_SUMMARY_JSON);
|
||||
}
|
||||
|
||||
// Fallback — should not be reached in these tests.
|
||||
return new Response(JSON.stringify({ error: { message: "unexpected URL in test" } }), {
|
||||
status: 400,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// ── Lifecycle hooks ───────────────────────────────────────────────────────────
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
// Clear in-memory session state so test 2 doesn't see stale connections from test 1.
|
||||
clearSessions();
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Test 1 (PRIMARY): codex quota handoff fires ───────────────────────────────
|
||||
//
|
||||
// Flow:
|
||||
// 1. Seed a codex DB connection + register with accessToken.
|
||||
// 2. Create context-relay combo → codex.
|
||||
// 3. Send request with x-session-id header.
|
||||
// 4. During request processing, chat.ts calls touchSession(sessionId, codexConnId).
|
||||
// 5. After the successful codex response, combo.ts enters the codex block:
|
||||
// - getSessionConnection → codexConnId (set in step 4).
|
||||
// - fetchCodexQuota(codexConnId) → hits our mock → returns 90% usage.
|
||||
// - resetCandidates[0] = session-window resetAt ISO string.
|
||||
// - maybeGenerateHandoff({ ..., expiresAt: sessionResetISO }) fires via setImmediate.
|
||||
// 6. generateHandoffAsync dispatches a second codex fetch for the summary.
|
||||
// 7. upsertHandoff writes to DB with expiresAt = sessionResetISO.
|
||||
//
|
||||
// Assertion: handoff.expiresAt === sessionResetISO (codex-specific, not TTL-derived).
|
||||
|
||||
test("context-relay codex quota handoff: fires and expiresAt matches session-window reset from quota", async () => {
|
||||
// 1. Seed codex connection.
|
||||
const conn = await seedConnection("codex", { apiKey: "sk-codex-handoff-test-1" });
|
||||
const codexConnId = conn.id;
|
||||
|
||||
// 2. Register connection meta so fetchCodexQuota can find credentials.
|
||||
// chat.ts also calls registerCodexConnection during request processing for OAuth
|
||||
// tokens; for API-key connections the accessToken may not be set there, so we
|
||||
// pre-register with the API key as the token. This is the documented test seam.
|
||||
registerCodexConnection(codexConnId, { accessToken: "sk-codex-handoff-test-1" });
|
||||
|
||||
// 3. Create a context-relay combo targeting codex.
|
||||
// Default handoffProviders: ["codex"] and handoffThreshold: 0.85 apply.
|
||||
await combosDb.createCombo({
|
||||
name: CODEX_COMBO_NAME,
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
{ id: "rc-codex-1", kind: "model", providerId: "codex", model: "gpt-5.3-codex" },
|
||||
],
|
||||
});
|
||||
|
||||
// 4. Compute quota reset times (future timestamps).
|
||||
const sessionResetAtUnix = Math.floor((Date.now() + 5 * 60 * 60 * 1000) / 1000); // +5h
|
||||
const weeklyResetAtUnix = Math.floor((Date.now() + 7 * 24 * 60 * 60 * 1000) / 1000); // +7d
|
||||
|
||||
// Expected expiresAt: parseWindowReset converts reset_at (unix seconds) → ISO string.
|
||||
// resetCandidates[0] = earliest ISO string = session window reset (5h < 7d).
|
||||
const expectedExpiresAt = new Date(sessionResetAtUnix * 1000).toISOString();
|
||||
|
||||
// 5. Install fetch mock and send request.
|
||||
const seenUrls: string[] = [];
|
||||
installCodexHandoffFetch(sessionResetAtUnix, weeklyResetAtUnix, seenUrls);
|
||||
|
||||
const r = await handleChat(codexRequest(true));
|
||||
assert.equal(r.status, 200, "main codex request must return 200");
|
||||
|
||||
// 6. Wait for setImmediate → generateHandoffAsync → upsertHandoff (up to 3s).
|
||||
const handoff = await waitFor(() => getHandoff(SESSION_ID, CODEX_COMBO_NAME), 3000);
|
||||
|
||||
// ── Primary assertion: handoff record exists ──────────────────────────────
|
||||
assert.ok(
|
||||
handoff !== null,
|
||||
"codex quota handoff record must be written to DB when quota ≥ threshold"
|
||||
);
|
||||
|
||||
// ── CODEX-SPECIFIC proof: expiresAt == session-window reset from quota ────
|
||||
// The universal handoff path never calls fetchCodexQuota, so it cannot produce
|
||||
// this specific expiresAt value. The value below can only come from the codex block.
|
||||
assert.equal(
|
||||
handoff!.expiresAt,
|
||||
expectedExpiresAt,
|
||||
`handoff.expiresAt must equal the session-window reset from codex quota (${expectedExpiresAt}); ` +
|
||||
`got ${handoff!.expiresAt}`
|
||||
);
|
||||
|
||||
// ── Secondary assertion: codex usage URL was fetched ─────────────────────
|
||||
assert.ok(
|
||||
seenUrls.includes(CODEX_USAGE_URL),
|
||||
`codex usage URL must have been fetched to produce the expiresAt; seen URLs: ${JSON.stringify(seenUrls)}`
|
||||
);
|
||||
|
||||
// ── Secondary assertion: summary was generated (non-empty) ───────────────
|
||||
assert.ok(
|
||||
typeof handoff!.summary === "string" && handoff!.summary.length > 0,
|
||||
`handoff.summary must be non-empty; got ${JSON.stringify(handoff!.summary)}`
|
||||
);
|
||||
|
||||
// ── Cleanup ───────────────────────────────────────────────────────────────
|
||||
unregisterCodexConnection(codexConnId);
|
||||
});
|
||||
|
||||
// ── Test 2 (CONTROL): non-codex provider never triggers the codex block ───────
|
||||
//
|
||||
// The codex block gates on `provider === "codex"`. A combo targeting openai
|
||||
// will never enter it, so the codex usage URL is never fetched and there is no
|
||||
// quota-derived expiresAt in the DB.
|
||||
//
|
||||
// This proves the primary test is not trivially green (i.e., the block is
|
||||
// actually gated on `provider === "codex"`).
|
||||
|
||||
test("context-relay codex quota handoff: does NOT fire when provider is openai (control)", async () => {
|
||||
// Seed openai (not codex).
|
||||
await seedConnection("openai", { apiKey: "sk-openai-control-no-codex-block" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: CODEX_COMBO_NAME,
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
{ id: "rc-openai-ctrl", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
|
||||
],
|
||||
});
|
||||
|
||||
const seenUrls: string[] = [];
|
||||
globalThis.fetch = async (url: unknown, _init: unknown = {}) => {
|
||||
seenUrls.push(String(url));
|
||||
return buildOpenAIResponse("assistant reply ok");
|
||||
};
|
||||
|
||||
const r = await handleChat(codexRequest(true));
|
||||
assert.equal(r.status, 200, "openai request must return 200");
|
||||
|
||||
// Give setImmediate time to fire if the block were incorrectly entered.
|
||||
await new Promise((res) => setTimeout(res, 400));
|
||||
|
||||
// The codex block requires provider === "codex", so it never runs for openai.
|
||||
// The codex usage URL must NOT have been fetched.
|
||||
assert.ok(
|
||||
!seenUrls.includes(CODEX_USAGE_URL),
|
||||
`codex usage URL must NOT be fetched for openai provider; seen: ${JSON.stringify(seenUrls)}`
|
||||
);
|
||||
|
||||
// No codex quota handoff record in DB.
|
||||
// (The universal handoff also does not fire because no prior model is seeded,
|
||||
// so getLastSessionModel returns null → no model switch detected.)
|
||||
const handoff = getHandoff(SESSION_ID, CODEX_COMBO_NAME);
|
||||
assert.equal(
|
||||
handoff,
|
||||
null,
|
||||
"no handoff record must exist when provider is openai (codex block never entered)"
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
// tests/integration/combo-matrix/context-relay-handoff.test.ts
|
||||
//
|
||||
// Deterministic in-process tests for context-relay UNIVERSAL HANDOFF behavior:
|
||||
// the session-context transfer that fires when a model switch is detected,
|
||||
// regardless of provider (provider-agnostic). This is the distinguishing
|
||||
// behavior of context-relay that was deferred as TODO(phase-2) in the
|
||||
// original combo-matrix coverage.
|
||||
//
|
||||
// What is tested here:
|
||||
// 1. Universal handoff fires (extra summary dispatch + DB record) when:
|
||||
// - universalHandoffConfig.enabled = true (default)
|
||||
// - request carries x-omniroute-session-id header
|
||||
// - session_model_history records a DIFFERENT prior model than the combo target
|
||||
// 2. Control (no model switch): prevModel === currModel — handoff must NOT fire.
|
||||
// 3. Control (no session ID): no header passed — handoff must NOT fire.
|
||||
//
|
||||
// Codex-specific block (lines 2143-2183 in combo.ts):
|
||||
// Requires strategy === "context-relay" AND provider === "codex" AND a live
|
||||
// codex session connection + quota fetcher. The seams (getSessionConnection /
|
||||
// fetchCodexQuota) are NOT exported from contextHandoff.ts as testable hooks;
|
||||
// there is no in-process mechanism equivalent to registerQuotaFetcher for the
|
||||
// codex path. This block requires a real codex provider connection (VPS) and
|
||||
// is NOT covered here. The universal-handoff path above is the user-visible,
|
||||
// provider-agnostic behavior and is now fully covered.
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness, providerFromUrl } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-relay-handoff");
|
||||
const {
|
||||
BaseExecutor,
|
||||
combosDb,
|
||||
handleChat,
|
||||
buildRequest,
|
||||
seedConnection,
|
||||
resetStorage,
|
||||
buildOpenAIResponse,
|
||||
waitFor,
|
||||
toPlainHeaders,
|
||||
} = h;
|
||||
|
||||
// Import DB helpers AFTER harness so they share the same DB instance (DATA_DIR
|
||||
// is set by the harness before any import triggers DB init).
|
||||
const { recordSessionModelUsage, getHandoff } = await import(
|
||||
"../../../src/lib/db/contextHandoffs.ts"
|
||||
);
|
||||
|
||||
// A minimal but valid handoff-JSON blob that parseHandoffJSON will accept.
|
||||
// Must have at minimum a non-empty "summary" field.
|
||||
const SCRIPTED_SUMMARY_JSON = JSON.stringify({
|
||||
summary: "User initiated a context-relay handoff test and requested a model switch.",
|
||||
keyDecisions: ["switched from gemini to openai"],
|
||||
taskProgress: "in-progress",
|
||||
activeEntities: ["context-relay-test"],
|
||||
});
|
||||
|
||||
const COMBO_NAME = "m-relay-handoff";
|
||||
// The session ID must use the external-session format: extractExternalSessionId in chat.ts
|
||||
// reads "x-session-id" (not "x-omniroute-session-id") and prefixes the result with "ext:".
|
||||
// We must seed session_model_history with this exact "ext:"-prefixed ID so that
|
||||
// getLastSessionModel(relayOptions.sessionId, comboName) returns the prior model.
|
||||
const SESSION_HEADER_VALUE = "relay-handoff-session-001";
|
||||
const SESSION_ID = `ext:${SESSION_HEADER_VALUE}`; // matches what extractExternalSessionId produces
|
||||
const PREV_MODEL = "gemini/gemini-2.5-flash"; // the "old" model — must differ from combo target
|
||||
const CURR_MODEL = "openai/gpt-4o-mini"; // the combo target model (provider/model format)
|
||||
|
||||
// Build a Request with the session-id header.
|
||||
function relayRequest(withSessionId = true) {
|
||||
const headers: Record<string, string> = withSessionId
|
||||
? { "x-session-id": SESSION_HEADER_VALUE } // x-session-id → relayOptions.sessionId = "ext:..."
|
||||
: {};
|
||||
return buildRequest({
|
||||
headers,
|
||||
body: {
|
||||
model: COMBO_NAME,
|
||||
stream: false,
|
||||
messages: [
|
||||
{ role: "user", content: "Prior turn — building something important in the test session." },
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Install a recording fetch that:
|
||||
// • returns a valid handoff JSON (wrapped in an OpenAI completion) for the
|
||||
// internal summary request (identified by _omnirouteInternalRequest flag in body)
|
||||
// • returns a normal OpenAI response for every other call
|
||||
function installHandoffAwareFetch() {
|
||||
h.calls.length = 0;
|
||||
|
||||
globalThis.fetch = async (url: any, init: any = {}) => {
|
||||
const u = String(url);
|
||||
const provider = providerFromUrl(u);
|
||||
const headers = toPlainHeaders(init?.headers);
|
||||
|
||||
// Parse request body to detect the internal summary request
|
||||
let bodyObj: Record<string, unknown> = {};
|
||||
try {
|
||||
bodyObj =
|
||||
typeof init?.body === "string"
|
||||
? (JSON.parse(init.body) as Record<string, unknown>)
|
||||
: ((init?.body ?? {}) as Record<string, unknown>);
|
||||
} catch {
|
||||
bodyObj = {};
|
||||
}
|
||||
|
||||
const call = {
|
||||
index: h.calls.length,
|
||||
provider,
|
||||
url: u,
|
||||
authorization: headers.authorization,
|
||||
model: typeof bodyObj.model === "string" ? bodyObj.model : undefined,
|
||||
};
|
||||
h.calls.push(call);
|
||||
|
||||
// Return valid handoff JSON for the internal summary generation request
|
||||
if (bodyObj._omnirouteInternalRequest === "universal-handoff") {
|
||||
return buildOpenAIResponse(SCRIPTED_SUMMARY_JSON);
|
||||
}
|
||||
|
||||
// Normal success response for the real request
|
||||
return buildOpenAIResponse("assistant reply ok");
|
||||
};
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Test 1: universal handoff fires on model switch ────────────────────────────
|
||||
//
|
||||
// Seed session history so getLastSessionModel returns PREV_MODEL (gemini).
|
||||
// Combo target is CURR_MODEL (openai). The switch is detected and
|
||||
// maybeGenerateUniversalHandoff fires via setImmediate → calls handleSingleModel
|
||||
// (extra upstream dispatch) → parses response → upsertHandoff writes to DB.
|
||||
//
|
||||
// Observable 1: h.calls has ≥2 entries (main request + summary dispatch).
|
||||
// Observable 2: getHandoff(sessionId, comboName) returns a record with a
|
||||
// non-empty .summary (proves the FULL path executed, not just the
|
||||
// dispatch). The assertion FAILS if the handoff did not fire.
|
||||
test("context-relay universal handoff: fires and writes handoff record on model switch", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-handoff" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-handoff" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: COMBO_NAME,
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// Single target — openai/gpt-4o-mini — so modelStr = CURR_MODEL
|
||||
{ id: "rh-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
|
||||
],
|
||||
});
|
||||
|
||||
// Seed the prior model usage so getLastSessionModel returns PREV_MODEL.
|
||||
// universalHandoffConfig.enabled = true by DEFAULT_UNIVERSAL_HANDOFF_CONFIG.
|
||||
recordSessionModelUsage(SESSION_ID, COMBO_NAME, PREV_MODEL, "gemini", undefined);
|
||||
|
||||
installHandoffAwareFetch();
|
||||
|
||||
const r = await handleChat(relayRequest(/* withSessionId */ true));
|
||||
assert.equal(r.status, 200, "main request must succeed");
|
||||
|
||||
// Wait for the setImmediate + generateUniversalHandoffAsync to complete and
|
||||
// write the DB record. Poll for up to 2 s — typically resolves in <100 ms.
|
||||
const handoff = await waitFor(
|
||||
() => getHandoff(SESSION_ID, COMBO_NAME),
|
||||
2000
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
handoff !== null,
|
||||
"universal handoff record must be written to DB when a model switch is detected"
|
||||
);
|
||||
assert.ok(
|
||||
typeof handoff!.summary === "string" && handoff!.summary.length > 0,
|
||||
`handoff.summary must be non-empty; got: ${JSON.stringify(handoff!.summary)}`
|
||||
);
|
||||
assert.equal(
|
||||
handoff!.comboName,
|
||||
COMBO_NAME,
|
||||
"handoff must be keyed to the correct combo"
|
||||
);
|
||||
assert.equal(
|
||||
handoff!.sessionId,
|
||||
SESSION_ID,
|
||||
"handoff must be keyed to the correct session"
|
||||
);
|
||||
|
||||
// Extra dispatch observable: main (index 0) + summary (index ≥ 1).
|
||||
assert.ok(
|
||||
h.calls.length >= 2,
|
||||
`expected ≥2 upstream dispatches (main + summary); got ${h.calls.length}: ${JSON.stringify(h.calls.map((c) => ({ i: c.index, p: c.provider, m: c.model })))}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Test 2: control — no model switch, handoff must NOT fire ──────────────────
|
||||
//
|
||||
// DO NOT seed a prior model. getLastSessionModel returns null → prevModel is
|
||||
// null → the `if (prevModel && prevModel !== modelStr)` branch is skipped →
|
||||
// no handoff. The assertion FAILS if the code incorrectly fires a handoff.
|
||||
test("context-relay universal handoff: does NOT fire when no prior model is recorded (no switch)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-noswitch" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: COMBO_NAME,
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [{ id: "ns-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" }],
|
||||
});
|
||||
|
||||
// No recordSessionModelUsage call → getLastSessionModel returns null → no switch.
|
||||
installHandoffAwareFetch();
|
||||
|
||||
const r = await handleChat(relayRequest(true));
|
||||
assert.equal(r.status, 200);
|
||||
|
||||
// Give setImmediate time to fire if the bug were present.
|
||||
await new Promise((res) => setTimeout(res, 250));
|
||||
|
||||
const handoff = getHandoff(SESSION_ID, COMBO_NAME);
|
||||
assert.equal(
|
||||
handoff,
|
||||
null,
|
||||
"handoff must NOT be written when no prior model exists (prevModel is null)"
|
||||
);
|
||||
|
||||
// Only the main request — no extra summary dispatch.
|
||||
assert.equal(
|
||||
h.calls.length,
|
||||
1,
|
||||
`expected exactly 1 upstream dispatch (main only); got ${h.calls.length}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Test 3: control — no session ID, handoff must NOT fire ────────────────────
|
||||
//
|
||||
// Session ID gate: `relayOptions?.sessionId` must be truthy.
|
||||
// Without the x-omniroute-session-id header, sessionId = null → block is skipped.
|
||||
test("context-relay universal handoff: does NOT fire when x-omniroute-session-id header is absent", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-nosid" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: COMBO_NAME,
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [{ id: "sid-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" }],
|
||||
});
|
||||
|
||||
// Seed prior model — but without the header the session block won't fire.
|
||||
recordSessionModelUsage(SESSION_ID, COMBO_NAME, PREV_MODEL, "gemini", undefined);
|
||||
|
||||
installHandoffAwareFetch();
|
||||
|
||||
// Send WITHOUT the session header.
|
||||
const r = await handleChat(relayRequest(/* withSessionId */ false));
|
||||
assert.equal(r.status, 200);
|
||||
|
||||
await new Promise((res) => setTimeout(res, 250));
|
||||
|
||||
const handoff = getHandoff(SESSION_ID, COMBO_NAME);
|
||||
assert.equal(
|
||||
handoff,
|
||||
null,
|
||||
"handoff must NOT be written when x-omniroute-session-id header is absent (sessionId gate)"
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
h.calls.length,
|
||||
1,
|
||||
`expected exactly 1 upstream dispatch (main only, no summary); got ${h.calls.length}`
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
// tests/integration/combo-matrix/cost-and-context.test.ts
|
||||
//
|
||||
// Matrix tests for cost-optimized, context-optimized, and context-relay strategies.
|
||||
//
|
||||
// Pricing source: DEFAULT_PRICING (src/shared/constants/pricing/frontier-labs.ts)
|
||||
// openai / gpt-4o-mini → input $0.15 / M tokens (cheap)
|
||||
// gemini / gemini-2.5-pro → input $2.00 / M tokens (expensive)
|
||||
//
|
||||
// Context-window source: MODEL_SPECS (src/shared/constants/modelSpecs.ts)
|
||||
// gpt-4o-mini → contextWindow 128 000
|
||||
// gemini-2.5-flash → contextWindow 1 048 576
|
||||
//
|
||||
// context-relay: does NOT appear in the sorting if-else chain of combo.ts; targets are
|
||||
// dispatched in combo-definition order. Two extra behaviours exist:
|
||||
// 1. Universal handoff (provider-agnostic) — covered by context-relay-handoff.test.ts.
|
||||
// 2. Codex-specific handoff — requires provider="codex" (see context-relay-handoff.test.ts
|
||||
// for why the codex block is documented but not covered here).
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-cost-context");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
function body(model: string, content = "hi") {
|
||||
return { model, stream: false, messages: [{ role: "user", content }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Test 1: cost-optimized ─────────────────────────────────────────────────────
|
||||
// sortTargetsByCost reads pricing?.input from getPricingForModel (src/lib/db/settings.ts).
|
||||
// In a fresh test DB the default-pricing layer is used (no litellm/modelsDev overrides).
|
||||
// openai / gpt-4o-mini → input $0.15 / M (CHEAPER → should be dispatched first)
|
||||
// gemini / gemini-2.5-pro → input $2.00 / M (EXPENSIVE → should be dispatched second)
|
||||
test("cost-optimized: cheapest model (gpt-4o-mini $0.15) dispatched before expensive (gemini-2.5-pro $2.00)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-cost" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-cost" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-cost-optimized",
|
||||
strategy: "cost-optimized",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// Order here is intentionally expensive-first to prove the sorter reorders them.
|
||||
{ id: "cc-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-pro" },
|
||||
{ id: "cc-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-cost-optimized") }));
|
||||
assert.equal(r.status, 200);
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
|
||||
assert.equal(
|
||||
seen[0],
|
||||
"openai",
|
||||
`expected cheapest provider (openai/gpt-4o-mini $0.15/M) to be dispatched first, got ${seen[0]}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Test 2: context-optimized ──────────────────────────────────────────────────
|
||||
// sortTargetsByContextSize reads contextWindow from getModelContextLimit →
|
||||
// getResolvedModelCapabilities → MODEL_SPECS (src/shared/constants/modelSpecs.ts).
|
||||
// In a fresh test DB (no synced capabilities), MODEL_SPECS values apply directly:
|
||||
// openai / gpt-4o-mini → contextWindow 128 000
|
||||
// gemini / gemini-2.5-flash → contextWindow 1 048 576 (LARGER → dispatched first)
|
||||
test("context-optimized: largest-context model (gemini-2.5-flash 1048576) dispatched before small-context (gpt-4o-mini 128000)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-ctx" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-ctx" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-context-optimized",
|
||||
strategy: "context-optimized",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// Order intentionally small-context-first to prove the sorter reorders them.
|
||||
{ id: "cx-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
|
||||
{ id: "cx-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-flash" },
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-context-optimized") }));
|
||||
assert.equal(r.status, 200);
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
|
||||
assert.equal(
|
||||
seen[0],
|
||||
"gemini",
|
||||
`expected largest-context provider (gemini/gemini-2.5-flash 1048576) to be dispatched first, got ${seen[0]}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Test 3: context-relay ──────────────────────────────────────────────────────
|
||||
// context-relay does NOT appear in the sorting if-else chain in combo.ts (confirmed at
|
||||
// lines 1459–1586). Targets are dispatched in combo-definition order (same as priority).
|
||||
// The only extra behaviour is a codex handoff at line 2144:
|
||||
// strategy === "context-relay" && relayOptions?.sessionId && relayConfig &&
|
||||
// relayConfig.handoffProviders.includes(provider) && provider === "codex"
|
||||
// Assertion: context-relay preserves combo-definition order (first model dispatched first).
|
||||
// Universal handoff + codex block coverage: see context-relay-handoff.test.ts.
|
||||
test("context-relay: preserves combo-definition order (openai dispatched before gemini)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-relay" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-relay" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-context-relay",
|
||||
strategy: "context-relay",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai is listed first; context-relay should NOT reorder targets.
|
||||
{ id: "cr-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
|
||||
{ id: "cr-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-flash" },
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-context-relay") }));
|
||||
assert.equal(r.status, 200);
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
|
||||
assert.equal(
|
||||
seen[0],
|
||||
"openai",
|
||||
`expected context-relay to preserve definition order (openai first), got ${seen[0]}`
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
// tests/integration/combo-matrix/fusion.test.ts
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-fusion-matrix");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
function body(model: string) {
|
||||
return { model, stream: false, messages: [{ role: "user", content: "fuse" }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
test("fusion: fans out to the whole panel then routes a judge synthesis turn", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-fz" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-fz" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-fz" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-fusion",
|
||||
strategy: "fusion",
|
||||
config: { judgeModel: "openai/gpt-4o-mini", fusionTuning: { minPanel: 2 } },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-fusion") }));
|
||||
assert.equal(r.status, 200);
|
||||
// 3 panel calls + 1 judge call = 4 upstream dispatches.
|
||||
assert.equal(h.calls.length, 4, `expected 3 panel + 1 judge, got ${h.calls.length}`);
|
||||
const providers = h.providersSeen();
|
||||
assert.ok(providers.includes("claude") && providers.includes("gemini"), "panel must include all members");
|
||||
assert.equal(providers.filter((p) => p === "openai").length >= 1, true, "judge (openai) must run");
|
||||
});
|
||||
|
||||
test("fusion: returns 503 when the whole panel fails", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-fz0" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-fz0" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-fusion-dead",
|
||||
strategy: "fusion",
|
||||
config: { judgeModel: "openai/gpt-4o-mini" },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
// Every panel call fails → fusion has nothing to synthesize → 503.
|
||||
h.installRecordingFetch(() => h.failure(503));
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-fusion-dead") }));
|
||||
assert.equal(r.status, 503);
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
// tests/integration/combo-matrix/ordered.test.ts
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-ordered");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
function body(model: string) {
|
||||
return { model, stream: false, messages: [{ role: "user", content: `route ${model}` }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
test("priority: always dispatches the first healthy target", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-p" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-p" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-priority",
|
||||
strategy: "priority",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-priority") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
|
||||
});
|
||||
|
||||
test("priority: falls back to the next target when the first fails", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-pf" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-pf" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-priority-fail",
|
||||
strategy: "priority",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
// First upstream call (openai) fails → must fall over to claude.
|
||||
h.installRecordingFetch((call) => (call.index === 0 ? h.failure(503) : undefined));
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-priority-fail") }));
|
||||
assert.equal(r.status, 200);
|
||||
assert.deepEqual(h.providersSeen(), ["openai", "claude"]);
|
||||
});
|
||||
|
||||
test("fill-first: keeps using the first target until it fails, then moves on", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-ff" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-ff" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-fill-first",
|
||||
strategy: "fill-first",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
// Healthy openai → all requests hit openai (fill-first preserves priority order).
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-fill-first") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
|
||||
});
|
||||
|
||||
test("round-robin: cycles through targets in batches (sticky limit = 3 default)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-rr" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-rr" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-rr" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-rr",
|
||||
strategy: "round-robin",
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
for (let i = 0; i < 9; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-rr") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
assert.deepEqual(h.providersSeen(), [
|
||||
"openai", "openai", "openai",
|
||||
"claude", "claude", "claude",
|
||||
"gemini", "gemini", "gemini",
|
||||
]);
|
||||
});
|
||||
|
||||
test("least-used: prefers the target with the fewest recent uses", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-lu" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-lu" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-least-used",
|
||||
strategy: "least-used",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
// Over an even number of calls each target should be picked roughly equally;
|
||||
// least-used must not pin to a single provider.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-least-used") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
const seen = h.providersSeen();
|
||||
assert.ok(seen.includes("openai"), "least-used must reach openai");
|
||||
assert.ok(seen.includes("claude"), "least-used must reach claude");
|
||||
});
|
||||
|
||||
test("random: over many calls reaches every target", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-rnd" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-rnd" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-rnd" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-random",
|
||||
strategy: "random",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-random") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
const unique = new Set(h.providersSeen());
|
||||
assert.equal(unique.size, 3, "random must reach all three providers over 30 calls");
|
||||
});
|
||||
|
||||
test("strict-random: uses every target once before repeating (deck, no early repeats)", async () => {
|
||||
const { _resetAllDecks } = await import("../../../src/shared/utils/shuffleDeck.ts");
|
||||
_resetAllDecks();
|
||||
await seedConnection("openai", { apiKey: "sk-openai-sr" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-sr" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-sr" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-strict-random",
|
||||
strategy: "strict-random",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-strict-random") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
// First full deck cycle: each provider exactly once (order varies, set is full).
|
||||
assert.deepEqual(new Set(h.providersSeen()), new Set(["openai", "claude", "gemini"]));
|
||||
});
|
||||
|
||||
test("p2c: spreads load across targets over many calls (no single-target pin)", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-p2c" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-p2c" });
|
||||
await seedConnection("gemini", { apiKey: "sk-gemini-p2c" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-p2c",
|
||||
strategy: "p2c",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-p2c") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
assert.ok(new Set(h.providersSeen()).size >= 2, "p2c must spread across at least two targets");
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
// tests/integration/combo-matrix/quota-aware.test.ts
|
||||
//
|
||||
// E2E matrix tests for the four quota-aware routing strategies:
|
||||
// reset-aware, reset-window, headroom, lkgp
|
||||
//
|
||||
// Each test engineers a specific non-default provider ordering so that a pass
|
||||
// proves the selector actually reordered targets, not merely preserved definition
|
||||
// order. The preferred target is always the SECOND one in the combo definition.
|
||||
//
|
||||
// Strategy → state source → seeded via:
|
||||
// reset-aware → registerQuotaFetcher + scoreResetAwareQuota
|
||||
// reset-window → registerQuotaFetcher + getResetWindowTimestampMs
|
||||
// headroom → __setHeadroomSaturationFetcherForTests
|
||||
// lkgp → setLKGP (key_value table, namespace='lkgp')
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-quota-aware");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
// Import quota / headroom seam hooks — must occur after the harness initialises
|
||||
// the DB so the module-level singletons inside quotaStrategies.ts are already live.
|
||||
const { registerQuotaFetcher } = await import("../../../open-sse/services/quotaPreflight.ts");
|
||||
const { __setHeadroomSaturationFetcherForTests } = await import(
|
||||
"../../../open-sse/services/combo/quotaStrategies.ts"
|
||||
);
|
||||
|
||||
function body(model: string) {
|
||||
return { model, stream: false, messages: [{ role: "user", content: "quota-aware route" }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
// Always restore the headroom fetcher override so it doesn't bleed into other tests.
|
||||
__setHeadroomSaturationFetcherForTests(null);
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
__setHeadroomSaturationFetcherForTests(null);
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Strategy 1: reset-aware ────────────────────────────────────────────────────
|
||||
//
|
||||
// Selector: orderTargetsByResetAwareQuota (open-sse/services/combo/quotaStrategies.ts)
|
||||
// State source: registerQuotaFetcher → scoreResetAwareQuota (quotaScoring.ts)
|
||||
//
|
||||
// Scoring: limitReached:true → score -Infinity. No fetcher → score 0.5 (neutral).
|
||||
//
|
||||
// Engineered state:
|
||||
// openai (1st in definition) → fetcher returns { limitReached: true } → score -Infinity
|
||||
// claude (2nd in definition) → no fetcher registered → score 0.5
|
||||
//
|
||||
// Expected: claude dispatched first (score 0.5 > -Infinity despite being second in combo).
|
||||
test("reset-aware: exhausted connection (limitReached) demoted — second target dispatched first", async () => {
|
||||
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-raware-exhausted" });
|
||||
const claudeConn = await seedConnection("claude", { apiKey: "sk-claude-raware-fresh" });
|
||||
|
||||
// Register AFTER seedConnection so connection objects are available.
|
||||
// The fetcher is keyed on provider name; connectionId is passed but we return
|
||||
// the same bad quota regardless so any openai connection is deprioritised.
|
||||
registerQuotaFetcher("openai", async (_connId) => ({ limitReached: true }));
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-reset-aware",
|
||||
strategy: "reset-aware",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST in definition — must be demoted by reset-aware (limitReached → -Infinity)
|
||||
{
|
||||
id: "ra-openai",
|
||||
kind: "model",
|
||||
providerId: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
connectionId: openaiConn.id,
|
||||
},
|
||||
// claude SECOND — must win (no fetcher → neutral score 0.5 > -Infinity)
|
||||
{
|
||||
id: "ra-claude",
|
||||
kind: "model",
|
||||
providerId: "claude",
|
||||
model: "claude-3-5-sonnet-20241022",
|
||||
connectionId: claudeConn.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-reset-aware") }));
|
||||
assert.equal(r.status, 200, "request must succeed via fallback to fresh claude target");
|
||||
|
||||
const first = h.providersSeen()[0];
|
||||
assert.equal(
|
||||
first,
|
||||
"claude",
|
||||
`reset-aware: openai has limitReached quota (score -Infinity), so claude (score 0.5) ` +
|
||||
`must be dispatched first even though it is second in the combo definition. Got: ${first}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Strategy 2: reset-window ───────────────────────────────────────────────────
|
||||
//
|
||||
// Selector: orderTargetsByResetWindow (quotaStrategies.ts)
|
||||
// State source: registerQuotaFetcher → getResetWindowTimestampMs (quotaScoring.ts)
|
||||
//
|
||||
// Sorting: ascending resetMs. No fetcher → resetMs Infinity → sorted last.
|
||||
//
|
||||
// Engineered state:
|
||||
// openai (1st in definition) → no fetcher → resetMs Infinity (sorted last)
|
||||
// gemini (2nd in definition) → fetcher returns a reset ~1 hour away
|
||||
// → resetMs ≈ now + 3600s (sorted first)
|
||||
//
|
||||
// The openai "limitReached" fetcher from test 1 persists in the registry but
|
||||
// limitReached → getResetWindowTimestampMs returns Infinity (same as no fetcher).
|
||||
//
|
||||
// Expected: gemini dispatched first (smallest resetMs).
|
||||
test("reset-window: target with nearest quota-reset dispatched first despite being second in definition", async () => {
|
||||
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-rwindow" });
|
||||
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-rwindow-soon" });
|
||||
|
||||
// Quota that resets in 1 hour → much sooner than openai (Infinity / limitReached).
|
||||
const soonResetAt = new Date(Date.now() + 60 * 60 * 1000).toISOString();
|
||||
registerQuotaFetcher("gemini", async (_connId) => ({
|
||||
percentUsed: 0.5,
|
||||
window7d: { percentUsed: 0.5, resetAt: soonResetAt },
|
||||
}));
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-reset-window",
|
||||
strategy: "reset-window",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST — must be demoted (no finite reset date → resetMs Infinity)
|
||||
{
|
||||
id: "rw-openai",
|
||||
kind: "model",
|
||||
providerId: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
connectionId: openaiConn.id,
|
||||
},
|
||||
// gemini SECOND — must win (earliest reset → smallest resetMs)
|
||||
{
|
||||
id: "rw-gemini",
|
||||
kind: "model",
|
||||
providerId: "gemini",
|
||||
model: "gemini-2.5-flash",
|
||||
connectionId: geminiConn.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-reset-window") }));
|
||||
assert.equal(r.status, 200, "request must succeed via gemini (nearest-reset) target");
|
||||
|
||||
const first = h.providersSeen()[0];
|
||||
assert.equal(
|
||||
first,
|
||||
"gemini",
|
||||
`reset-window: gemini quota resets in ~1h (resetMs finite), openai has no reset date ` +
|
||||
`(resetMs Infinity), so gemini must be dispatched first despite being second in combo. Got: ${first}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Strategy 3: headroom ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Selector: orderTargetsByHeadroom (quotaStrategies.ts)
|
||||
// State source: getSaturation — injectable via __setHeadroomSaturationFetcherForTests
|
||||
//
|
||||
// Ranking: headroom = 1 − max(util_5h, util_7d). Higher headroom → dispatched first.
|
||||
//
|
||||
// Engineered state:
|
||||
// openai (1st in definition) → saturation 0.9 → headroom 0.1 (near capacity)
|
||||
// claude (2nd in definition) → saturation 0.1 → headroom 0.9 (mostly free)
|
||||
//
|
||||
// The connectionId is embedded in each model target so getQuotaAwareConnectionsForTarget
|
||||
// correctly expands to the specific connection even when connections=[] (no quota fetcher
|
||||
// needed for headroom — only the saturation signal matters).
|
||||
//
|
||||
// Expected: claude dispatched first (headroom 0.9 > headroom 0.1).
|
||||
test("headroom: target with most free capacity dispatched first despite being second in definition", async () => {
|
||||
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-headroom-saturated" });
|
||||
const claudeConn = await seedConnection("claude", { apiKey: "sk-claude-headroom-free" });
|
||||
|
||||
__setHeadroomSaturationFetcherForTests(async (connectionId, _provider, _dim) => {
|
||||
if (connectionId === openaiConn.id) return 0.9; // saturated → headroom 0.1
|
||||
if (connectionId === claudeConn.id) return 0.1; // mostly free → headroom 0.9
|
||||
return 0; // unknown → treat as full headroom (fail-open)
|
||||
});
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-headroom",
|
||||
strategy: "headroom",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST in definition — must be demoted (saturation 0.9 → headroom 0.1)
|
||||
{
|
||||
id: "hr-openai",
|
||||
kind: "model",
|
||||
providerId: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
connectionId: openaiConn.id,
|
||||
},
|
||||
// claude SECOND — must win (saturation 0.1 → headroom 0.9, the most free capacity)
|
||||
{
|
||||
id: "hr-claude",
|
||||
kind: "model",
|
||||
providerId: "claude",
|
||||
model: "claude-3-5-sonnet-20241022",
|
||||
connectionId: claudeConn.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-headroom") }));
|
||||
assert.equal(r.status, 200, "request must succeed via claude (most headroom) target");
|
||||
|
||||
const first = h.providersSeen()[0];
|
||||
assert.equal(
|
||||
first,
|
||||
"claude",
|
||||
`headroom: openai saturation=0.9 → headroom=0.1, claude saturation=0.1 → headroom=0.9, ` +
|
||||
`so claude must be dispatched first despite being second in combo definition. Got: ${first}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Strategy 4: lkgp ──────────────────────────────────────────────────────────
|
||||
//
|
||||
// Selector: getLKGP(combo.name, combo.id || combo.name) (src/lib/db/settings.ts)
|
||||
// → reads key_value WHERE namespace='lkgp' AND key='<comboName>:<comboId>'
|
||||
// → moves the matched provider to front of orderedTargets
|
||||
//
|
||||
// Engineered state:
|
||||
// Pre-seed: setLKGP(comboId, comboId, "claude") → LKGP record says claude was best
|
||||
// Combo definition order: openai (1st), claude (2nd)
|
||||
//
|
||||
// Expected: claude moved to front despite being second in combo definition.
|
||||
test("lkgp: last-known-good provider is prioritised above definition order", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-lkgp-test" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-lkgp-test" });
|
||||
|
||||
// Explicit combo id keeps the LKGP key (`comboName:comboId`) predictable.
|
||||
const COMBO_ID = "m-lkgp-qa-001";
|
||||
|
||||
// Import and write the LKGP record into the fresh DB (after resetStorage in beforeEach).
|
||||
const { setLKGP } = await import("../../../src/lib/db/settings.ts");
|
||||
// Key written: "m-lkgp-qa-001:m-lkgp-qa-001" (comboName:comboId)
|
||||
// combo.ts reads: getLKGP(combo.name, combo.id || combo.name)
|
||||
await setLKGP(COMBO_ID, COMBO_ID, "claude");
|
||||
|
||||
await combosDb.createCombo({
|
||||
id: COMBO_ID,
|
||||
name: COMBO_ID,
|
||||
strategy: "lkgp",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST in definition — must be deprioritised (LKGP record points to claude)
|
||||
"openai/gpt-4o-mini",
|
||||
// claude SECOND — must be moved to front by the LKGP record
|
||||
"claude/claude-3-5-sonnet-20241022",
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body(COMBO_ID) }));
|
||||
assert.equal(r.status, 200, "request must succeed via claude (LKGP-preferred) target");
|
||||
|
||||
const first = h.providersSeen()[0];
|
||||
assert.equal(
|
||||
first,
|
||||
"claude",
|
||||
`lkgp: LKGP record names "claude" as last-known-good, so claude must be dispatched ` +
|
||||
`first despite being second in the combo definition. Got: ${first}`
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
// tests/integration/combo-matrix/quota-share.test.ts
|
||||
//
|
||||
// E2E matrix test for the INTERNAL quota-share routing strategy (DRR / deficit
|
||||
// round-robin over auto-minted qtSd/ pool combos).
|
||||
//
|
||||
// Drive path: handleChat → chatCore → handleComboChat (strategy "quota-share")
|
||||
// → selectQuotaShareTarget (DRR + P2C + bucket-gating) → executor dispatch.
|
||||
//
|
||||
// Two tests:
|
||||
// 1. DRR fairness — 2 equal-weight connections, 6 requests; both must be
|
||||
// selected and balanced, proving DRR ran end-to-end (not a single-winner
|
||||
// or definition-order dispatch).
|
||||
// 2. Saturation deprioritization — one connection marked saturated via
|
||||
// recordUsage (5h window); a single request must dispatch to the clean
|
||||
// connection first, proving the bucket-gating gate runs through the real
|
||||
// wire.
|
||||
//
|
||||
// Drive path chosen: Option A — direct `combosDb.createCombo({ strategy:
|
||||
// "quota-share", ... })`. The `handleComboChat` dispatch at combo.ts:1573
|
||||
// branches on `strategy === "quota-share"` verbatim; no qtSd/ pool plumbing is
|
||||
// required to reach `selectQuotaShareTarget`.
|
||||
//
|
||||
// The preferred target is always the SECOND one in the combo definition so that
|
||||
// a pass proves the selector actually reordered targets (same discipline as the
|
||||
// 17-strategy quota-aware.test.ts matrix).
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-quota-share");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
// DRR + bucket state seams — import AFTER harness initialises the DB so the
|
||||
// module-level in-process singletons are already live.
|
||||
const { _clearDrrStateForTest } = await import(
|
||||
"../../../open-sse/services/combo/quotaShareStrategy.ts"
|
||||
);
|
||||
const { recordUsage, _clearBucketsForTest } = await import(
|
||||
"../../../src/lib/quota/accountBuckets.ts"
|
||||
);
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function body(model: string, suffix = "") {
|
||||
return {
|
||||
model,
|
||||
stream: false,
|
||||
messages: [{ role: "user", content: `quota-share route${suffix}` }],
|
||||
};
|
||||
}
|
||||
|
||||
// ── Lifecycle ─────────────────────────────────────────────────────────────────
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
_clearDrrStateForTest();
|
||||
_clearBucketsForTest();
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
_clearDrrStateForTest();
|
||||
_clearBucketsForTest();
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
_clearDrrStateForTest();
|
||||
_clearBucketsForTest();
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
// ── Strategy: quota-share — DRR fairness ─────────────────────────────────────
|
||||
//
|
||||
// Mechanism: DRR (deficit round robin, quantum = weight / totalWeight).
|
||||
//
|
||||
// Engineered state:
|
||||
// openai (1st in definition) — equal weight 100
|
||||
// gemini (2nd in definition) — equal weight 100
|
||||
//
|
||||
// With 2 equal-weight targets DRR alternates: openai, gemini, openai, gemini, …
|
||||
// Over 6 requests both must be selected at least 2 times (proven to be exactly
|
||||
// 3 each by the deterministic math, but asserting >= 2 avoids brittleness if
|
||||
// P2C ever nudges a tie differently). The assertion fails if the pipeline
|
||||
// pin-picks the first candidate on every call — i.e. if DRR is bypassed.
|
||||
test("DRR fairness: 2 equal-weight connections alternate across 6 requests through real pipeline", async () => {
|
||||
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-qs-drr-1" });
|
||||
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-qs-drr-1" });
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-qs-drr",
|
||||
strategy: "quota-share",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST in definition
|
||||
{
|
||||
id: "qs-drr-openai",
|
||||
kind: "model",
|
||||
providerId: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
connectionId: openaiConn.id,
|
||||
},
|
||||
// gemini SECOND in definition — DRR must pick it on alternate rounds
|
||||
{
|
||||
id: "qs-drr-gemini",
|
||||
kind: "model",
|
||||
providerId: "gemini",
|
||||
model: "gemini-2.5-flash",
|
||||
connectionId: geminiConn.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
// Use a unique message suffix per request so session stickiness (message-hash
|
||||
// based) does not pin every call to the first-round winner. In real traffic
|
||||
// each conversation has distinct content; the DRR distributes across them.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-qs-drr", ` #${i}`) }));
|
||||
assert.equal(r.status, 200, `request ${i + 1} must succeed`);
|
||||
}
|
||||
|
||||
const seen = h.providersSeen();
|
||||
assert.equal(seen.length, 6, "all 6 requests must reach an upstream provider");
|
||||
|
||||
const openaiCount = seen.filter((p) => p === "openai").length;
|
||||
const geminiCount = seen.filter((p) => p === "gemini").length;
|
||||
|
||||
assert.ok(
|
||||
openaiCount >= 2,
|
||||
`DRR fairness: openai must be selected at least 2 times out of 6; got ${openaiCount}. ` +
|
||||
`Full sequence: [${seen.join(", ")}]. If openai was never selected, DRR was bypassed.`
|
||||
);
|
||||
assert.ok(
|
||||
geminiCount >= 2,
|
||||
`DRR fairness: gemini must be selected at least 2 times out of 6; got ${geminiCount}. ` +
|
||||
`Full sequence: [${seen.join(", ")}]. If gemini was never selected, DRR was bypassed.`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Strategy: quota-share — saturation deprioritization ──────────────────────
|
||||
//
|
||||
// Mechanism: filterEligibleBySaturation (isBucketSaturated, 5h window).
|
||||
//
|
||||
// Engineered state:
|
||||
// openai (1st in definition) — 5h bucket seeded at 100 % usage via
|
||||
// recordUsage → isBucketSaturated returns true
|
||||
// → filterEligibleBySaturation demotes it
|
||||
// gemini (2nd in definition) — no bucket recorded → eligible (clean)
|
||||
//
|
||||
// Expected: gemini dispatched first despite being second in the combo
|
||||
// definition. Proves the bucket-gating runs through the real pipeline wire
|
||||
// (not merely unit-tested in isolation).
|
||||
test("saturation deprioritization: saturated connection demoted — clean second target dispatched first", async () => {
|
||||
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-qs-sat-1" });
|
||||
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-qs-sat-1" });
|
||||
|
||||
// Seed openai connection as saturated (5h window, 100 % usage, reset in 1 h).
|
||||
// recordUsage writes the in-process _buckets store; isBucketSaturated reads it.
|
||||
// The resetAt must be in the FUTURE so the lazy-reset guard keeps the entry live.
|
||||
const futureResetAt = new Date(Date.now() + 3_600_000).toISOString();
|
||||
recordUsage(openaiConn.id, "5h", 100, futureResetAt);
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "m-qs-sat",
|
||||
strategy: "quota-share",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
|
||||
models: [
|
||||
// openai FIRST in definition — must be demoted (5h bucket saturated)
|
||||
{
|
||||
id: "qs-sat-openai",
|
||||
kind: "model",
|
||||
providerId: "openai",
|
||||
model: "gpt-4o-mini",
|
||||
connectionId: openaiConn.id,
|
||||
},
|
||||
// gemini SECOND — must win (no saturation recorded → eligible)
|
||||
{
|
||||
id: "qs-sat-gemini",
|
||||
kind: "model",
|
||||
providerId: "gemini",
|
||||
model: "gemini-2.5-flash",
|
||||
connectionId: geminiConn.id,
|
||||
},
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const r = await handleChat(buildRequest({ body: body("m-qs-sat") }));
|
||||
assert.equal(r.status, 200, "request must succeed via clean gemini target");
|
||||
|
||||
const first = h.providersSeen()[0];
|
||||
assert.equal(
|
||||
first,
|
||||
"gemini",
|
||||
`saturation deprioritization: openai 5h bucket is at 100 % (saturated), so ` +
|
||||
`filterEligibleBySaturation must demote it and dispatch gemini first despite it ` +
|
||||
`being second in the combo definition. Got: ${first}`
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
// tests/integration/combo-matrix/weighted.test.ts
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
|
||||
|
||||
const h = await createComboRoutingHarness("combo-weighted");
|
||||
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
|
||||
|
||||
function body(model: string) {
|
||||
return { model, stream: false, messages: [{ role: "user", content: "w" }] };
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = 0;
|
||||
await resetStorage();
|
||||
});
|
||||
test.afterEach(async () => {
|
||||
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
|
||||
await resetStorage();
|
||||
});
|
||||
test.after(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
test("weighted: 70/30 weights produce roughly proportional distribution", async () => {
|
||||
await seedConnection("openai", { apiKey: "sk-openai-w" });
|
||||
await seedConnection("claude", { apiKey: "sk-claude-w" });
|
||||
await combosDb.createCombo({
|
||||
name: "m-weighted",
|
||||
strategy: "weighted",
|
||||
config: { maxRetries: 0, retryDelayMs: 0, stickyWeightedLimit: 1 },
|
||||
models: [
|
||||
{ id: "w-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini", weight: 70 },
|
||||
{ id: "w-claude", kind: "model", providerId: "claude", model: "claude-3-5-sonnet-20241022", weight: 30 },
|
||||
],
|
||||
});
|
||||
h.installRecordingFetch();
|
||||
|
||||
const N = 200;
|
||||
for (let i = 0; i < N; i++) {
|
||||
const r = await handleChat(buildRequest({ body: body("m-weighted") }));
|
||||
assert.equal(r.status, 200);
|
||||
}
|
||||
const seen = h.providersSeen();
|
||||
const openaiShare = seen.filter((p) => p === "openai").length / N;
|
||||
// Tolerance ±0.12 absorbs sampling noise at N=200 while still proving the split.
|
||||
assert.ok(openaiShare > 0.58 && openaiShare < 0.82, `openai share ${openaiShare} not ~0.70`);
|
||||
assert.ok(seen.includes("claude"), "weighted must still reach the 30% target");
|
||||
});
|
||||
Reference in New Issue
Block a user