chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:12 +08:00
commit d8dcd5f6d1
8604 changed files with 2479390 additions and 0 deletions
@@ -0,0 +1,99 @@
// tests/integration/combo-matrix/auto.test.ts
//
// Integration tests for `model: "auto"` and `model: "auto/<variant>"` routing.
//
// How auto routing works:
// - model:"auto" → chat.ts detects isAutoRouting → calls createVirtualAutoCombo(undefined)
// which reads active connections from DB and builds a virtual combo (name="auto", id="auto").
// - The virtual combo's default routerStrategy is "lkgp".
// - combo.ts fetches getLKGP(combo.name, combo.id) = getLKGP("auto", "auto") → key "auto:auto".
// - LKGPStrategy: if a LKGP record exists for that provider AND the provider is in the
// candidate pool, it pins that provider deterministically (no randomness).
// - Without a LKGP record, falls back to "rules" 9-factor scorer.
//
// Test 1 strategy: single-request, deterministic.
// We seed a LKGP record pointing to "openai" via settingsDb.setLKGP("auto","auto","openai").
// The assertion h.providersSeen()[0] === "openai" is guaranteed by the LKGP pin
// and will FAIL if the LKGP lookup or the LKGPStrategy routing is broken.
//
// Test 2 strategy: pool-resolution check (not provider-specific).
// Send model:"auto/coding" → virtual combo for the "coding" variant (MODE_PACKS quality-first).
// Assert status 200 AND a seeded provider was dispatched. The pool should resolve from the
// DB connections; this test fails if the variant is unrecognised (400) or the pool is empty.
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-auto");
const { BaseExecutor, handleChat, buildRequest, seedConnection, resetStorage, settingsDb } = h;
function body(model: string) {
return { model, stream: false, messages: [{ role: "user", content: "hello" }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
// ── Test 1: auto picks the LKGP-biased provider ─────────────────────────────
//
// Strategy: single-request, deterministic via LKGP seed.
// The virtual auto-combo for model:"auto" has name="auto" and id="auto".
// getLKGP("auto","auto") reads key "auto:auto" in the key_value table.
// LKGPStrategy pins the seeded provider when it is healthy + in the pool.
// This assertion fails if LKGP lookup, LKGPStrategy, or candidate resolution breaks.
test("auto: LKGP record biases dispatch toward the seeded provider", async () => {
await seedConnection("openai", { apiKey: "sk-openai-auto" });
await seedConnection("claude", { apiKey: "sk-claude-auto" });
// Bias toward openai by writing a LKGP record before the request.
await settingsDb.setLKGP("auto", "auto", "openai");
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("auto") }));
assert.equal(r.status, 200, `Expected 200, got ${r.status}`);
const seen = h.providersSeen();
assert.ok(seen.length > 0, "Expected at least one upstream dispatch");
assert.equal(
seen[0],
"openai",
`Expected openai to be dispatched first (LKGP-biased), got: ${JSON.stringify(seen)}`
);
});
// ── Test 2: auto/coding resolves a virtual pool from seeded connections ──────
//
// "auto/coding" is a recognized variant (VALID_AUTO_VARIANTS has "coding").
// createVirtualAutoCombo("coding") applies MODE_PACKS["quality-first"] weights
// and uses the same DB connections as candidates.
// This test fails if:
// - the variant is not recognized and returns a 400, or
// - the virtual pool is empty (no candidates resolved from DB connections), or
// - the dispatched provider is not one of the seeded ones.
test("auto/coding: virtual pool resolves and dispatches a seeded provider", async () => {
await seedConnection("openai", { apiKey: "sk-openai-coding" });
await seedConnection("claude", { apiKey: "sk-claude-coding" });
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("auto/coding") }));
assert.equal(r.status, 200, `Expected 200 from auto/coding, got ${r.status}`);
const seen = h.providersSeen();
assert.ok(seen.length > 0, "Expected at least one upstream dispatch for auto/coding");
const seededProviders = new Set(["openai", "claude"]);
assert.ok(
seededProviders.has(seen[0]),
`Expected a seeded provider (openai or claude), got: ${seen[0]} (all: ${JSON.stringify(seen)})`
);
});
@@ -0,0 +1,367 @@
// tests/integration/combo-matrix/context-relay-codex.test.ts
//
// Deterministic in-process test for context-relay CODEX-SPECIFIC HANDOFF
// (combo.ts ~2143-2183): the quota-aware handoff that fires after a successful
// codex response when the quota exceeds the threshold.
//
// Code path:
// if (strategy === "context-relay" && relayOptions?.sessionId && relayConfig &&
// relayConfig.handoffProviders.includes(provider) && provider === "codex") {
// const connectionId = getSessionConnection(relayOptions.sessionId);
// if (connectionId) {
// const quotaInfo = await fetchCodexQuota(connectionId).catch(() => null);
// if (quotaInfo) { ... maybeGenerateHandoff({ ..., expiresAt: resetCandidates[0] }); }
// }
// }
//
// PRIMARY observable: getHandoff(sessionId, comboName).expiresAt === session-window
// reset time from the codex quota response. This is the CODEX-SPECIFIC proof:
// the universal handoff path does NOT set expiresAt from a codex quota fetch.
//
// SECONDARY observables:
// - The codex usage URL was fetched (proves the quota path ran).
// - The handoff record has a non-empty .summary.
//
// CONTROL: combo targeting openai (not codex) → codex block NEVER runs →
// codex usage URL is never fetched → expiresAt from quota doesn't appear.
//
// Seams used:
// - registerCodexConnection / unregisterCodexConnection (codexQuotaFetcher.ts)
// - clearSessions (sessionManager.ts) — cleanup between tests
// - buildCodexResponsesSse helper (mirrors codex-stream-false.test.ts)
// - Custom fetch mock distinguishes quota URL / summary dispatch / main dispatch
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
import {
registerCodexConnection,
unregisterCodexConnection,
} from "../../../open-sse/services/codexQuotaFetcher.ts";
import { clearSessions } from "../../../open-sse/services/sessionManager.ts";
// ── Harness setup ─────────────────────────────────────────────────────────────
// Each harness owns an isolated DATA_DIR so DB handles don't clash.
const h = await createComboRoutingHarness("combo-relay-codex");
const {
BaseExecutor,
combosDb,
handleChat,
buildRequest,
seedConnection,
resetStorage,
buildOpenAIResponse,
waitFor,
toPlainHeaders,
} = h;
// Import DB helpers AFTER harness creation so they share the same DB instance
// (DATA_DIR is set by the harness before any import triggers DB init).
const { getHandoff } = await import("../../../src/lib/db/contextHandoffs.ts");
// ── Constants ─────────────────────────────────────────────────────────────────
const CODEX_COMBO_NAME = "m-relay-codex-quota";
const SESSION_HEADER_VALUE = "relay-codex-quota-001";
const SESSION_ID = `ext:${SESSION_HEADER_VALUE}`;
// Codex endpoint URLs (must match codexQuotaFetcher.ts and the executor config).
const CODEX_USAGE_URL = "https://chatgpt.com/backend-api/wham/usage";
const CODEX_RESPONSES_HOST = "chatgpt.com/backend-api/codex/responses";
// Summary JSON that parseHandoffJSON will successfully parse.
const CODEX_SUMMARY_JSON = JSON.stringify({
summary:
"User is implementing a TypeScript context-relay codex quota-handoff test using TDD.",
keyDecisions: ["codex provider selected", "quota threshold at 90%"],
taskProgress: "writing deterministic integration test for codex handoff",
activeEntities: ["combo.ts", "codexQuotaFetcher.ts", "contextHandoff.ts"],
});
// ── SSE builder (mirrors codex-stream-false.test.ts) ─────────────────────────
function buildCodexResponsesSse(text = "codex assistant reply") {
return new Response(
[
"event: response.created",
'data: {"type":"response.created","response":{"id":"resp_codex_1","model":"gpt-5.3-codex","status":"in_progress","output":[]}}',
"",
"event: response.output_text.delta",
`data: ${JSON.stringify({
type: "response.output_text.delta",
output_index: 0,
delta: text,
})}`,
"",
"event: response.completed",
`data: ${JSON.stringify({
type: "response.completed",
response: {
id: "resp_codex_1",
object: "response",
model: "gpt-5.3-codex",
status: "completed",
output: [
{
type: "message",
role: "assistant",
content: [{ type: "output_text", text }],
},
],
usage: { input_tokens: 6, output_tokens: 4 },
},
})}`,
"",
"data: [DONE]",
"",
].join("\n"),
{
status: 200,
headers: { "Content-Type": "text/event-stream" },
}
);
}
// ── Quota response builder ────────────────────────────────────────────────────
// Produces a codex usage API response that parseCodexUsageResponse accepts.
// primary_window → session → percentUsed = 0.90 (90%) — triggers handoff (threshold 0.85)
// secondary_window → weekly → percentUsed = 0.85 (85%) — below threshold for weekly
// Both are below HANDOFF_EXHAUSTION_THRESHOLD (0.95) so maybeGenerateHandoff proceeds.
function buildCodexUsageBody(
sessionResetAtUnix: number,
weeklyResetAtUnix: number
): Record<string, unknown> {
return {
rate_limit: {
primary_window: { used_percent: 90, reset_at: sessionResetAtUnix },
secondary_window: { used_percent: 85, reset_at: weeklyResetAtUnix },
},
};
}
// ── Request builder ───────────────────────────────────────────────────────────
function codexRequest(withSessionId = true) {
return buildRequest({
headers: withSessionId ? { "x-session-id": SESSION_HEADER_VALUE } : {},
body: {
model: CODEX_COMBO_NAME,
stream: false,
messages: [{ role: "user", content: "Write a TypeScript hello world." }],
},
});
}
// ── Fetch mock ────────────────────────────────────────────────────────────────
// Handles three URL classes:
// 1. CODEX_USAGE_URL → codex quota JSON
// 2. CODEX_RESPONSES_HOST (call 1) → main request Responses API SSE
// 3. CODEX_RESPONSES_HOST (call 2) → summary request Responses API SSE (with CODEX_SUMMARY_JSON)
//
// The second codex-responses call is the handoff summary generation, identified by
// call count (the first is always the main request).
function installCodexHandoffFetch(
sessionResetAtUnix: number,
weeklyResetAtUnix: number,
seenUrls: string[]
) {
let codexResponsesCallCount = 0;
globalThis.fetch = async (url: unknown, init: unknown = {}) => {
const u = String(url);
seenUrls.push(u);
const headers =
init != null && typeof init === "object"
? toPlainHeaders((init as Record<string, unknown>).headers)
: {};
void headers; // recorded for debugging; not needed in assertions here
if (u === CODEX_USAGE_URL) {
// Codex quota endpoint — return structured usage JSON.
return new Response(
JSON.stringify(buildCodexUsageBody(sessionResetAtUnix, weeklyResetAtUnix)),
{ status: 200, headers: { "Content-Type": "application/json" } }
);
}
if (u.includes(CODEX_RESPONSES_HOST)) {
codexResponsesCallCount++;
if (codexResponsesCallCount === 1) {
// Main codex request — return a successful Responses API SSE.
return buildCodexResponsesSse("codex assistant reply ok");
}
// Subsequent call(s) are from the handoff summary generation.
// Return Responses API SSE whose output_text is CODEX_SUMMARY_JSON so that
// generateHandoffAsync → parseHandoffJSON succeeds.
return buildCodexResponsesSse(CODEX_SUMMARY_JSON);
}
// Fallback — should not be reached in these tests.
return new Response(JSON.stringify({ error: { message: "unexpected URL in test" } }), {
status: 400,
headers: { "Content-Type": "application/json" },
});
};
}
// ── Lifecycle hooks ───────────────────────────────────────────────────────────
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
// Clear in-memory session state so test 2 doesn't see stale connections from test 1.
clearSessions();
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
// ── Test 1 (PRIMARY): codex quota handoff fires ───────────────────────────────
//
// Flow:
// 1. Seed a codex DB connection + register with accessToken.
// 2. Create context-relay combo → codex.
// 3. Send request with x-session-id header.
// 4. During request processing, chat.ts calls touchSession(sessionId, codexConnId).
// 5. After the successful codex response, combo.ts enters the codex block:
// - getSessionConnection → codexConnId (set in step 4).
// - fetchCodexQuota(codexConnId) → hits our mock → returns 90% usage.
// - resetCandidates[0] = session-window resetAt ISO string.
// - maybeGenerateHandoff({ ..., expiresAt: sessionResetISO }) fires via setImmediate.
// 6. generateHandoffAsync dispatches a second codex fetch for the summary.
// 7. upsertHandoff writes to DB with expiresAt = sessionResetISO.
//
// Assertion: handoff.expiresAt === sessionResetISO (codex-specific, not TTL-derived).
test("context-relay codex quota handoff: fires and expiresAt matches session-window reset from quota", async () => {
// 1. Seed codex connection.
const conn = await seedConnection("codex", { apiKey: "sk-codex-handoff-test-1" });
const codexConnId = conn.id;
// 2. Register connection meta so fetchCodexQuota can find credentials.
// chat.ts also calls registerCodexConnection during request processing for OAuth
// tokens; for API-key connections the accessToken may not be set there, so we
// pre-register with the API key as the token. This is the documented test seam.
registerCodexConnection(codexConnId, { accessToken: "sk-codex-handoff-test-1" });
// 3. Create a context-relay combo targeting codex.
// Default handoffProviders: ["codex"] and handoffThreshold: 0.85 apply.
await combosDb.createCombo({
name: CODEX_COMBO_NAME,
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
{ id: "rc-codex-1", kind: "model", providerId: "codex", model: "gpt-5.3-codex" },
],
});
// 4. Compute quota reset times (future timestamps).
const sessionResetAtUnix = Math.floor((Date.now() + 5 * 60 * 60 * 1000) / 1000); // +5h
const weeklyResetAtUnix = Math.floor((Date.now() + 7 * 24 * 60 * 60 * 1000) / 1000); // +7d
// Expected expiresAt: parseWindowReset converts reset_at (unix seconds) → ISO string.
// resetCandidates[0] = earliest ISO string = session window reset (5h < 7d).
const expectedExpiresAt = new Date(sessionResetAtUnix * 1000).toISOString();
// 5. Install fetch mock and send request.
const seenUrls: string[] = [];
installCodexHandoffFetch(sessionResetAtUnix, weeklyResetAtUnix, seenUrls);
const r = await handleChat(codexRequest(true));
assert.equal(r.status, 200, "main codex request must return 200");
// 6. Wait for setImmediate → generateHandoffAsync → upsertHandoff (up to 3s).
const handoff = await waitFor(() => getHandoff(SESSION_ID, CODEX_COMBO_NAME), 3000);
// ── Primary assertion: handoff record exists ──────────────────────────────
assert.ok(
handoff !== null,
"codex quota handoff record must be written to DB when quota ≥ threshold"
);
// ── CODEX-SPECIFIC proof: expiresAt == session-window reset from quota ────
// The universal handoff path never calls fetchCodexQuota, so it cannot produce
// this specific expiresAt value. The value below can only come from the codex block.
assert.equal(
handoff!.expiresAt,
expectedExpiresAt,
`handoff.expiresAt must equal the session-window reset from codex quota (${expectedExpiresAt}); ` +
`got ${handoff!.expiresAt}`
);
// ── Secondary assertion: codex usage URL was fetched ─────────────────────
assert.ok(
seenUrls.includes(CODEX_USAGE_URL),
`codex usage URL must have been fetched to produce the expiresAt; seen URLs: ${JSON.stringify(seenUrls)}`
);
// ── Secondary assertion: summary was generated (non-empty) ───────────────
assert.ok(
typeof handoff!.summary === "string" && handoff!.summary.length > 0,
`handoff.summary must be non-empty; got ${JSON.stringify(handoff!.summary)}`
);
// ── Cleanup ───────────────────────────────────────────────────────────────
unregisterCodexConnection(codexConnId);
});
// ── Test 2 (CONTROL): non-codex provider never triggers the codex block ───────
//
// The codex block gates on `provider === "codex"`. A combo targeting openai
// will never enter it, so the codex usage URL is never fetched and there is no
// quota-derived expiresAt in the DB.
//
// This proves the primary test is not trivially green (i.e., the block is
// actually gated on `provider === "codex"`).
test("context-relay codex quota handoff: does NOT fire when provider is openai (control)", async () => {
// Seed openai (not codex).
await seedConnection("openai", { apiKey: "sk-openai-control-no-codex-block" });
await combosDb.createCombo({
name: CODEX_COMBO_NAME,
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
{ id: "rc-openai-ctrl", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
],
});
const seenUrls: string[] = [];
globalThis.fetch = async (url: unknown, _init: unknown = {}) => {
seenUrls.push(String(url));
return buildOpenAIResponse("assistant reply ok");
};
const r = await handleChat(codexRequest(true));
assert.equal(r.status, 200, "openai request must return 200");
// Give setImmediate time to fire if the block were incorrectly entered.
await new Promise((res) => setTimeout(res, 400));
// The codex block requires provider === "codex", so it never runs for openai.
// The codex usage URL must NOT have been fetched.
assert.ok(
!seenUrls.includes(CODEX_USAGE_URL),
`codex usage URL must NOT be fetched for openai provider; seen: ${JSON.stringify(seenUrls)}`
);
// No codex quota handoff record in DB.
// (The universal handoff also does not fire because no prior model is seeded,
// so getLastSessionModel returns null → no model switch detected.)
const handoff = getHandoff(SESSION_ID, CODEX_COMBO_NAME);
assert.equal(
handoff,
null,
"no handoff record must exist when provider is openai (codex block never entered)"
);
});
@@ -0,0 +1,282 @@
// tests/integration/combo-matrix/context-relay-handoff.test.ts
//
// Deterministic in-process tests for context-relay UNIVERSAL HANDOFF behavior:
// the session-context transfer that fires when a model switch is detected,
// regardless of provider (provider-agnostic). This is the distinguishing
// behavior of context-relay that was deferred as TODO(phase-2) in the
// original combo-matrix coverage.
//
// What is tested here:
// 1. Universal handoff fires (extra summary dispatch + DB record) when:
// - universalHandoffConfig.enabled = true (default)
// - request carries x-omniroute-session-id header
// - session_model_history records a DIFFERENT prior model than the combo target
// 2. Control (no model switch): prevModel === currModel — handoff must NOT fire.
// 3. Control (no session ID): no header passed — handoff must NOT fire.
//
// Codex-specific block (lines 2143-2183 in combo.ts):
// Requires strategy === "context-relay" AND provider === "codex" AND a live
// codex session connection + quota fetcher. The seams (getSessionConnection /
// fetchCodexQuota) are NOT exported from contextHandoff.ts as testable hooks;
// there is no in-process mechanism equivalent to registerQuotaFetcher for the
// codex path. This block requires a real codex provider connection (VPS) and
// is NOT covered here. The universal-handoff path above is the user-visible,
// provider-agnostic behavior and is now fully covered.
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness, providerFromUrl } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-relay-handoff");
const {
BaseExecutor,
combosDb,
handleChat,
buildRequest,
seedConnection,
resetStorage,
buildOpenAIResponse,
waitFor,
toPlainHeaders,
} = h;
// Import DB helpers AFTER harness so they share the same DB instance (DATA_DIR
// is set by the harness before any import triggers DB init).
const { recordSessionModelUsage, getHandoff } = await import(
"../../../src/lib/db/contextHandoffs.ts"
);
// A minimal but valid handoff-JSON blob that parseHandoffJSON will accept.
// Must have at minimum a non-empty "summary" field.
const SCRIPTED_SUMMARY_JSON = JSON.stringify({
summary: "User initiated a context-relay handoff test and requested a model switch.",
keyDecisions: ["switched from gemini to openai"],
taskProgress: "in-progress",
activeEntities: ["context-relay-test"],
});
const COMBO_NAME = "m-relay-handoff";
// The session ID must use the external-session format: extractExternalSessionId in chat.ts
// reads "x-session-id" (not "x-omniroute-session-id") and prefixes the result with "ext:".
// We must seed session_model_history with this exact "ext:"-prefixed ID so that
// getLastSessionModel(relayOptions.sessionId, comboName) returns the prior model.
const SESSION_HEADER_VALUE = "relay-handoff-session-001";
const SESSION_ID = `ext:${SESSION_HEADER_VALUE}`; // matches what extractExternalSessionId produces
const PREV_MODEL = "gemini/gemini-2.5-flash"; // the "old" model — must differ from combo target
const CURR_MODEL = "openai/gpt-4o-mini"; // the combo target model (provider/model format)
// Build a Request with the session-id header.
function relayRequest(withSessionId = true) {
const headers: Record<string, string> = withSessionId
? { "x-session-id": SESSION_HEADER_VALUE } // x-session-id → relayOptions.sessionId = "ext:..."
: {};
return buildRequest({
headers,
body: {
model: COMBO_NAME,
stream: false,
messages: [
{ role: "user", content: "Prior turn — building something important in the test session." },
],
},
});
}
// Install a recording fetch that:
// • returns a valid handoff JSON (wrapped in an OpenAI completion) for the
// internal summary request (identified by _omnirouteInternalRequest flag in body)
// • returns a normal OpenAI response for every other call
function installHandoffAwareFetch() {
h.calls.length = 0;
globalThis.fetch = async (url: any, init: any = {}) => {
const u = String(url);
const provider = providerFromUrl(u);
const headers = toPlainHeaders(init?.headers);
// Parse request body to detect the internal summary request
let bodyObj: Record<string, unknown> = {};
try {
bodyObj =
typeof init?.body === "string"
? (JSON.parse(init.body) as Record<string, unknown>)
: ((init?.body ?? {}) as Record<string, unknown>);
} catch {
bodyObj = {};
}
const call = {
index: h.calls.length,
provider,
url: u,
authorization: headers.authorization,
model: typeof bodyObj.model === "string" ? bodyObj.model : undefined,
};
h.calls.push(call);
// Return valid handoff JSON for the internal summary generation request
if (bodyObj._omnirouteInternalRequest === "universal-handoff") {
return buildOpenAIResponse(SCRIPTED_SUMMARY_JSON);
}
// Normal success response for the real request
return buildOpenAIResponse("assistant reply ok");
};
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
// ── Test 1: universal handoff fires on model switch ────────────────────────────
//
// Seed session history so getLastSessionModel returns PREV_MODEL (gemini).
// Combo target is CURR_MODEL (openai). The switch is detected and
// maybeGenerateUniversalHandoff fires via setImmediate → calls handleSingleModel
// (extra upstream dispatch) → parses response → upsertHandoff writes to DB.
//
// Observable 1: h.calls has ≥2 entries (main request + summary dispatch).
// Observable 2: getHandoff(sessionId, comboName) returns a record with a
// non-empty .summary (proves the FULL path executed, not just the
// dispatch). The assertion FAILS if the handoff did not fire.
test("context-relay universal handoff: fires and writes handoff record on model switch", async () => {
await seedConnection("openai", { apiKey: "sk-openai-handoff" });
await seedConnection("gemini", { apiKey: "sk-gemini-handoff" });
await combosDb.createCombo({
name: COMBO_NAME,
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// Single target — openai/gpt-4o-mini — so modelStr = CURR_MODEL
{ id: "rh-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
],
});
// Seed the prior model usage so getLastSessionModel returns PREV_MODEL.
// universalHandoffConfig.enabled = true by DEFAULT_UNIVERSAL_HANDOFF_CONFIG.
recordSessionModelUsage(SESSION_ID, COMBO_NAME, PREV_MODEL, "gemini", undefined);
installHandoffAwareFetch();
const r = await handleChat(relayRequest(/* withSessionId */ true));
assert.equal(r.status, 200, "main request must succeed");
// Wait for the setImmediate + generateUniversalHandoffAsync to complete and
// write the DB record. Poll for up to 2 s — typically resolves in <100 ms.
const handoff = await waitFor(
() => getHandoff(SESSION_ID, COMBO_NAME),
2000
);
assert.ok(
handoff !== null,
"universal handoff record must be written to DB when a model switch is detected"
);
assert.ok(
typeof handoff!.summary === "string" && handoff!.summary.length > 0,
`handoff.summary must be non-empty; got: ${JSON.stringify(handoff!.summary)}`
);
assert.equal(
handoff!.comboName,
COMBO_NAME,
"handoff must be keyed to the correct combo"
);
assert.equal(
handoff!.sessionId,
SESSION_ID,
"handoff must be keyed to the correct session"
);
// Extra dispatch observable: main (index 0) + summary (index ≥ 1).
assert.ok(
h.calls.length >= 2,
`expected ≥2 upstream dispatches (main + summary); got ${h.calls.length}: ${JSON.stringify(h.calls.map((c) => ({ i: c.index, p: c.provider, m: c.model })))}`
);
});
// ── Test 2: control — no model switch, handoff must NOT fire ──────────────────
//
// DO NOT seed a prior model. getLastSessionModel returns null → prevModel is
// null → the `if (prevModel && prevModel !== modelStr)` branch is skipped →
// no handoff. The assertion FAILS if the code incorrectly fires a handoff.
test("context-relay universal handoff: does NOT fire when no prior model is recorded (no switch)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-noswitch" });
await combosDb.createCombo({
name: COMBO_NAME,
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [{ id: "ns-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" }],
});
// No recordSessionModelUsage call → getLastSessionModel returns null → no switch.
installHandoffAwareFetch();
const r = await handleChat(relayRequest(true));
assert.equal(r.status, 200);
// Give setImmediate time to fire if the bug were present.
await new Promise((res) => setTimeout(res, 250));
const handoff = getHandoff(SESSION_ID, COMBO_NAME);
assert.equal(
handoff,
null,
"handoff must NOT be written when no prior model exists (prevModel is null)"
);
// Only the main request — no extra summary dispatch.
assert.equal(
h.calls.length,
1,
`expected exactly 1 upstream dispatch (main only); got ${h.calls.length}`
);
});
// ── Test 3: control — no session ID, handoff must NOT fire ────────────────────
//
// Session ID gate: `relayOptions?.sessionId` must be truthy.
// Without the x-omniroute-session-id header, sessionId = null → block is skipped.
test("context-relay universal handoff: does NOT fire when x-omniroute-session-id header is absent", async () => {
await seedConnection("openai", { apiKey: "sk-openai-nosid" });
await combosDb.createCombo({
name: COMBO_NAME,
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [{ id: "sid-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" }],
});
// Seed prior model — but without the header the session block won't fire.
recordSessionModelUsage(SESSION_ID, COMBO_NAME, PREV_MODEL, "gemini", undefined);
installHandoffAwareFetch();
// Send WITHOUT the session header.
const r = await handleChat(relayRequest(/* withSessionId */ false));
assert.equal(r.status, 200);
await new Promise((res) => setTimeout(res, 250));
const handoff = getHandoff(SESSION_ID, COMBO_NAME);
assert.equal(
handoff,
null,
"handoff must NOT be written when x-omniroute-session-id header is absent (sessionId gate)"
);
assert.equal(
h.calls.length,
1,
`expected exactly 1 upstream dispatch (main only, no summary); got ${h.calls.length}`
);
});
@@ -0,0 +1,142 @@
// tests/integration/combo-matrix/cost-and-context.test.ts
//
// Matrix tests for cost-optimized, context-optimized, and context-relay strategies.
//
// Pricing source: DEFAULT_PRICING (src/shared/constants/pricing/frontier-labs.ts)
// openai / gpt-4o-mini → input $0.15 / M tokens (cheap)
// gemini / gemini-2.5-pro → input $2.00 / M tokens (expensive)
//
// Context-window source: MODEL_SPECS (src/shared/constants/modelSpecs.ts)
// gpt-4o-mini → contextWindow 128 000
// gemini-2.5-flash → contextWindow 1 048 576
//
// context-relay: does NOT appear in the sorting if-else chain of combo.ts; targets are
// dispatched in combo-definition order. Two extra behaviours exist:
// 1. Universal handoff (provider-agnostic) — covered by context-relay-handoff.test.ts.
// 2. Codex-specific handoff — requires provider="codex" (see context-relay-handoff.test.ts
// for why the codex block is documented but not covered here).
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-cost-context");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
function body(model: string, content = "hi") {
return { model, stream: false, messages: [{ role: "user", content }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
// ── Test 1: cost-optimized ─────────────────────────────────────────────────────
// sortTargetsByCost reads pricing?.input from getPricingForModel (src/lib/db/settings.ts).
// In a fresh test DB the default-pricing layer is used (no litellm/modelsDev overrides).
// openai / gpt-4o-mini → input $0.15 / M (CHEAPER → should be dispatched first)
// gemini / gemini-2.5-pro → input $2.00 / M (EXPENSIVE → should be dispatched second)
test("cost-optimized: cheapest model (gpt-4o-mini $0.15) dispatched before expensive (gemini-2.5-pro $2.00)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-cost" });
await seedConnection("gemini", { apiKey: "sk-gemini-cost" });
await combosDb.createCombo({
name: "m-cost-optimized",
strategy: "cost-optimized",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// Order here is intentionally expensive-first to prove the sorter reorders them.
{ id: "cc-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-pro" },
{ id: "cc-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-cost-optimized") }));
assert.equal(r.status, 200);
const seen = h.providersSeen();
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
assert.equal(
seen[0],
"openai",
`expected cheapest provider (openai/gpt-4o-mini $0.15/M) to be dispatched first, got ${seen[0]}`
);
});
// ── Test 2: context-optimized ──────────────────────────────────────────────────
// sortTargetsByContextSize reads contextWindow from getModelContextLimit →
// getResolvedModelCapabilities → MODEL_SPECS (src/shared/constants/modelSpecs.ts).
// In a fresh test DB (no synced capabilities), MODEL_SPECS values apply directly:
// openai / gpt-4o-mini → contextWindow 128 000
// gemini / gemini-2.5-flash → contextWindow 1 048 576 (LARGER → dispatched first)
test("context-optimized: largest-context model (gemini-2.5-flash 1048576) dispatched before small-context (gpt-4o-mini 128000)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-ctx" });
await seedConnection("gemini", { apiKey: "sk-gemini-ctx" });
await combosDb.createCombo({
name: "m-context-optimized",
strategy: "context-optimized",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// Order intentionally small-context-first to prove the sorter reorders them.
{ id: "cx-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
{ id: "cx-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-flash" },
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-context-optimized") }));
assert.equal(r.status, 200);
const seen = h.providersSeen();
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
assert.equal(
seen[0],
"gemini",
`expected largest-context provider (gemini/gemini-2.5-flash 1048576) to be dispatched first, got ${seen[0]}`
);
});
// ── Test 3: context-relay ──────────────────────────────────────────────────────
// context-relay does NOT appear in the sorting if-else chain in combo.ts (confirmed at
// lines 14591586). Targets are dispatched in combo-definition order (same as priority).
// The only extra behaviour is a codex handoff at line 2144:
// strategy === "context-relay" && relayOptions?.sessionId && relayConfig &&
// relayConfig.handoffProviders.includes(provider) && provider === "codex"
// Assertion: context-relay preserves combo-definition order (first model dispatched first).
// Universal handoff + codex block coverage: see context-relay-handoff.test.ts.
test("context-relay: preserves combo-definition order (openai dispatched before gemini)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-relay" });
await seedConnection("gemini", { apiKey: "sk-gemini-relay" });
await combosDb.createCombo({
name: "m-context-relay",
strategy: "context-relay",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai is listed first; context-relay should NOT reorder targets.
{ id: "cr-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini" },
{ id: "cr-gemini", kind: "model", providerId: "gemini", model: "gemini-2.5-flash" },
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-context-relay") }));
assert.equal(r.status, 200);
const seen = h.providersSeen();
assert.ok(seen.length > 0, "at least one upstream dispatch expected");
assert.equal(
seen[0],
"openai",
`expected context-relay to preserve definition order (openai first), got ${seen[0]}`
);
});
@@ -0,0 +1,60 @@
// tests/integration/combo-matrix/fusion.test.ts
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-fusion-matrix");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
function body(model: string) {
return { model, stream: false, messages: [{ role: "user", content: "fuse" }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
test("fusion: fans out to the whole panel then routes a judge synthesis turn", async () => {
await seedConnection("openai", { apiKey: "sk-openai-fz" });
await seedConnection("claude", { apiKey: "sk-claude-fz" });
await seedConnection("gemini", { apiKey: "sk-gemini-fz" });
await combosDb.createCombo({
name: "m-fusion",
strategy: "fusion",
config: { judgeModel: "openai/gpt-4o-mini", fusionTuning: { minPanel: 2 } },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-fusion") }));
assert.equal(r.status, 200);
// 3 panel calls + 1 judge call = 4 upstream dispatches.
assert.equal(h.calls.length, 4, `expected 3 panel + 1 judge, got ${h.calls.length}`);
const providers = h.providersSeen();
assert.ok(providers.includes("claude") && providers.includes("gemini"), "panel must include all members");
assert.equal(providers.filter((p) => p === "openai").length >= 1, true, "judge (openai) must run");
});
test("fusion: returns 503 when the whole panel fails", async () => {
await seedConnection("openai", { apiKey: "sk-openai-fz0" });
await seedConnection("claude", { apiKey: "sk-claude-fz0" });
await combosDb.createCombo({
name: "m-fusion-dead",
strategy: "fusion",
config: { judgeModel: "openai/gpt-4o-mini" },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
});
// Every panel call fails → fusion has nothing to synthesize → 503.
h.installRecordingFetch(() => h.failure(503));
const r = await handleChat(buildRequest({ body: body("m-fusion-dead") }));
assert.equal(r.status, 503);
});
@@ -0,0 +1,183 @@
// tests/integration/combo-matrix/ordered.test.ts
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-ordered");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
function body(model: string) {
return { model, stream: false, messages: [{ role: "user", content: `route ${model}` }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
test("priority: always dispatches the first healthy target", async () => {
await seedConnection("openai", { apiKey: "sk-openai-p" });
await seedConnection("claude", { apiKey: "sk-claude-p" });
await combosDb.createCombo({
name: "m-priority",
strategy: "priority",
config: { maxRetries: 0, retryDelayMs: 0 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
});
h.installRecordingFetch();
for (let i = 0; i < 3; i++) {
const r = await handleChat(buildRequest({ body: body("m-priority") }));
assert.equal(r.status, 200);
}
assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
});
test("priority: falls back to the next target when the first fails", async () => {
await seedConnection("openai", { apiKey: "sk-openai-pf" });
await seedConnection("claude", { apiKey: "sk-claude-pf" });
await combosDb.createCombo({
name: "m-priority-fail",
strategy: "priority",
config: { maxRetries: 0, retryDelayMs: 0 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
});
// First upstream call (openai) fails → must fall over to claude.
h.installRecordingFetch((call) => (call.index === 0 ? h.failure(503) : undefined));
const r = await handleChat(buildRequest({ body: body("m-priority-fail") }));
assert.equal(r.status, 200);
assert.deepEqual(h.providersSeen(), ["openai", "claude"]);
});
test("fill-first: keeps using the first target until it fails, then moves on", async () => {
await seedConnection("openai", { apiKey: "sk-openai-ff" });
await seedConnection("claude", { apiKey: "sk-claude-ff" });
await combosDb.createCombo({
name: "m-fill-first",
strategy: "fill-first",
config: { maxRetries: 0, retryDelayMs: 0 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
});
h.installRecordingFetch();
// Healthy openai → all requests hit openai (fill-first preserves priority order).
for (let i = 0; i < 3; i++) {
const r = await handleChat(buildRequest({ body: body("m-fill-first") }));
assert.equal(r.status, 200);
}
assert.deepEqual(h.providersSeen(), ["openai", "openai", "openai"]);
});
test("round-robin: cycles through targets in batches (sticky limit = 3 default)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-rr" });
await seedConnection("claude", { apiKey: "sk-claude-rr" });
await seedConnection("gemini", { apiKey: "sk-gemini-rr" });
await combosDb.createCombo({
name: "m-rr",
strategy: "round-robin",
config: { maxRetries: 0, retryDelayMs: 0 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
});
h.installRecordingFetch();
for (let i = 0; i < 9; i++) {
const r = await handleChat(buildRequest({ body: body("m-rr") }));
assert.equal(r.status, 200);
}
assert.deepEqual(h.providersSeen(), [
"openai", "openai", "openai",
"claude", "claude", "claude",
"gemini", "gemini", "gemini",
]);
});
test("least-used: prefers the target with the fewest recent uses", async () => {
await seedConnection("openai", { apiKey: "sk-openai-lu" });
await seedConnection("claude", { apiKey: "sk-claude-lu" });
await combosDb.createCombo({
name: "m-least-used",
strategy: "least-used",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022"],
});
h.installRecordingFetch();
// Over an even number of calls each target should be picked roughly equally;
// least-used must not pin to a single provider.
for (let i = 0; i < 6; i++) {
const r = await handleChat(buildRequest({ body: body("m-least-used") }));
assert.equal(r.status, 200);
}
const seen = h.providersSeen();
assert.ok(seen.includes("openai"), "least-used must reach openai");
assert.ok(seen.includes("claude"), "least-used must reach claude");
});
test("random: over many calls reaches every target", async () => {
await seedConnection("openai", { apiKey: "sk-openai-rnd" });
await seedConnection("claude", { apiKey: "sk-claude-rnd" });
await seedConnection("gemini", { apiKey: "sk-gemini-rnd" });
await combosDb.createCombo({
name: "m-random",
strategy: "random",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
});
h.installRecordingFetch();
for (let i = 0; i < 30; i++) {
const r = await handleChat(buildRequest({ body: body("m-random") }));
assert.equal(r.status, 200);
}
const unique = new Set(h.providersSeen());
assert.equal(unique.size, 3, "random must reach all three providers over 30 calls");
});
test("strict-random: uses every target once before repeating (deck, no early repeats)", async () => {
const { _resetAllDecks } = await import("../../../src/shared/utils/shuffleDeck.ts");
_resetAllDecks();
await seedConnection("openai", { apiKey: "sk-openai-sr" });
await seedConnection("claude", { apiKey: "sk-claude-sr" });
await seedConnection("gemini", { apiKey: "sk-gemini-sr" });
await combosDb.createCombo({
name: "m-strict-random",
strategy: "strict-random",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
});
h.installRecordingFetch();
for (let i = 0; i < 3; i++) {
const r = await handleChat(buildRequest({ body: body("m-strict-random") }));
assert.equal(r.status, 200);
}
// First full deck cycle: each provider exactly once (order varies, set is full).
assert.deepEqual(new Set(h.providersSeen()), new Set(["openai", "claude", "gemini"]));
});
test("p2c: spreads load across targets over many calls (no single-target pin)", async () => {
await seedConnection("openai", { apiKey: "sk-openai-p2c" });
await seedConnection("claude", { apiKey: "sk-claude-p2c" });
await seedConnection("gemini", { apiKey: "sk-gemini-p2c" });
await combosDb.createCombo({
name: "m-p2c",
strategy: "p2c",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: ["openai/gpt-4o-mini", "claude/claude-3-5-sonnet-20241022", "gemini/gemini-2.5-flash"],
});
h.installRecordingFetch();
for (let i = 0; i < 30; i++) {
const r = await handleChat(buildRequest({ body: body("m-p2c") }));
assert.equal(r.status, 200);
}
assert.ok(new Set(h.providersSeen()).size >= 2, "p2c must spread across at least two targets");
});
@@ -0,0 +1,282 @@
// tests/integration/combo-matrix/quota-aware.test.ts
//
// E2E matrix tests for the four quota-aware routing strategies:
// reset-aware, reset-window, headroom, lkgp
//
// Each test engineers a specific non-default provider ordering so that a pass
// proves the selector actually reordered targets, not merely preserved definition
// order. The preferred target is always the SECOND one in the combo definition.
//
// Strategy → state source → seeded via:
// reset-aware → registerQuotaFetcher + scoreResetAwareQuota
// reset-window → registerQuotaFetcher + getResetWindowTimestampMs
// headroom → __setHeadroomSaturationFetcherForTests
// lkgp → setLKGP (key_value table, namespace='lkgp')
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-quota-aware");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
// Import quota / headroom seam hooks — must occur after the harness initialises
// the DB so the module-level singletons inside quotaStrategies.ts are already live.
const { registerQuotaFetcher } = await import("../../../open-sse/services/quotaPreflight.ts");
const { __setHeadroomSaturationFetcherForTests } = await import(
"../../../open-sse/services/combo/quotaStrategies.ts"
);
function body(model: string) {
return { model, stream: false, messages: [{ role: "user", content: "quota-aware route" }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
// Always restore the headroom fetcher override so it doesn't bleed into other tests.
__setHeadroomSaturationFetcherForTests(null);
await resetStorage();
});
test.after(async () => {
__setHeadroomSaturationFetcherForTests(null);
await h.cleanup();
});
// ── Strategy 1: reset-aware ────────────────────────────────────────────────────
//
// Selector: orderTargetsByResetAwareQuota (open-sse/services/combo/quotaStrategies.ts)
// State source: registerQuotaFetcher → scoreResetAwareQuota (quotaScoring.ts)
//
// Scoring: limitReached:true → score -Infinity. No fetcher → score 0.5 (neutral).
//
// Engineered state:
// openai (1st in definition) → fetcher returns { limitReached: true } → score -Infinity
// claude (2nd in definition) → no fetcher registered → score 0.5
//
// Expected: claude dispatched first (score 0.5 > -Infinity despite being second in combo).
test("reset-aware: exhausted connection (limitReached) demoted — second target dispatched first", async () => {
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-raware-exhausted" });
const claudeConn = await seedConnection("claude", { apiKey: "sk-claude-raware-fresh" });
// Register AFTER seedConnection so connection objects are available.
// The fetcher is keyed on provider name; connectionId is passed but we return
// the same bad quota regardless so any openai connection is deprioritised.
registerQuotaFetcher("openai", async (_connId) => ({ limitReached: true }));
await combosDb.createCombo({
name: "m-reset-aware",
strategy: "reset-aware",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST in definition — must be demoted by reset-aware (limitReached → -Infinity)
{
id: "ra-openai",
kind: "model",
providerId: "openai",
model: "gpt-4o-mini",
connectionId: openaiConn.id,
},
// claude SECOND — must win (no fetcher → neutral score 0.5 > -Infinity)
{
id: "ra-claude",
kind: "model",
providerId: "claude",
model: "claude-3-5-sonnet-20241022",
connectionId: claudeConn.id,
},
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-reset-aware") }));
assert.equal(r.status, 200, "request must succeed via fallback to fresh claude target");
const first = h.providersSeen()[0];
assert.equal(
first,
"claude",
`reset-aware: openai has limitReached quota (score -Infinity), so claude (score 0.5) ` +
`must be dispatched first even though it is second in the combo definition. Got: ${first}`
);
});
// ── Strategy 2: reset-window ───────────────────────────────────────────────────
//
// Selector: orderTargetsByResetWindow (quotaStrategies.ts)
// State source: registerQuotaFetcher → getResetWindowTimestampMs (quotaScoring.ts)
//
// Sorting: ascending resetMs. No fetcher → resetMs Infinity → sorted last.
//
// Engineered state:
// openai (1st in definition) → no fetcher → resetMs Infinity (sorted last)
// gemini (2nd in definition) → fetcher returns a reset ~1 hour away
// → resetMs ≈ now + 3600s (sorted first)
//
// The openai "limitReached" fetcher from test 1 persists in the registry but
// limitReached → getResetWindowTimestampMs returns Infinity (same as no fetcher).
//
// Expected: gemini dispatched first (smallest resetMs).
test("reset-window: target with nearest quota-reset dispatched first despite being second in definition", async () => {
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-rwindow" });
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-rwindow-soon" });
// Quota that resets in 1 hour → much sooner than openai (Infinity / limitReached).
const soonResetAt = new Date(Date.now() + 60 * 60 * 1000).toISOString();
registerQuotaFetcher("gemini", async (_connId) => ({
percentUsed: 0.5,
window7d: { percentUsed: 0.5, resetAt: soonResetAt },
}));
await combosDb.createCombo({
name: "m-reset-window",
strategy: "reset-window",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST — must be demoted (no finite reset date → resetMs Infinity)
{
id: "rw-openai",
kind: "model",
providerId: "openai",
model: "gpt-4o-mini",
connectionId: openaiConn.id,
},
// gemini SECOND — must win (earliest reset → smallest resetMs)
{
id: "rw-gemini",
kind: "model",
providerId: "gemini",
model: "gemini-2.5-flash",
connectionId: geminiConn.id,
},
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-reset-window") }));
assert.equal(r.status, 200, "request must succeed via gemini (nearest-reset) target");
const first = h.providersSeen()[0];
assert.equal(
first,
"gemini",
`reset-window: gemini quota resets in ~1h (resetMs finite), openai has no reset date ` +
`(resetMs Infinity), so gemini must be dispatched first despite being second in combo. Got: ${first}`
);
});
// ── Strategy 3: headroom ───────────────────────────────────────────────────────
//
// Selector: orderTargetsByHeadroom (quotaStrategies.ts)
// State source: getSaturation — injectable via __setHeadroomSaturationFetcherForTests
//
// Ranking: headroom = 1 max(util_5h, util_7d). Higher headroom → dispatched first.
//
// Engineered state:
// openai (1st in definition) → saturation 0.9 → headroom 0.1 (near capacity)
// claude (2nd in definition) → saturation 0.1 → headroom 0.9 (mostly free)
//
// The connectionId is embedded in each model target so getQuotaAwareConnectionsForTarget
// correctly expands to the specific connection even when connections=[] (no quota fetcher
// needed for headroom — only the saturation signal matters).
//
// Expected: claude dispatched first (headroom 0.9 > headroom 0.1).
test("headroom: target with most free capacity dispatched first despite being second in definition", async () => {
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-headroom-saturated" });
const claudeConn = await seedConnection("claude", { apiKey: "sk-claude-headroom-free" });
__setHeadroomSaturationFetcherForTests(async (connectionId, _provider, _dim) => {
if (connectionId === openaiConn.id) return 0.9; // saturated → headroom 0.1
if (connectionId === claudeConn.id) return 0.1; // mostly free → headroom 0.9
return 0; // unknown → treat as full headroom (fail-open)
});
await combosDb.createCombo({
name: "m-headroom",
strategy: "headroom",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST in definition — must be demoted (saturation 0.9 → headroom 0.1)
{
id: "hr-openai",
kind: "model",
providerId: "openai",
model: "gpt-4o-mini",
connectionId: openaiConn.id,
},
// claude SECOND — must win (saturation 0.1 → headroom 0.9, the most free capacity)
{
id: "hr-claude",
kind: "model",
providerId: "claude",
model: "claude-3-5-sonnet-20241022",
connectionId: claudeConn.id,
},
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-headroom") }));
assert.equal(r.status, 200, "request must succeed via claude (most headroom) target");
const first = h.providersSeen()[0];
assert.equal(
first,
"claude",
`headroom: openai saturation=0.9 → headroom=0.1, claude saturation=0.1 → headroom=0.9, ` +
`so claude must be dispatched first despite being second in combo definition. Got: ${first}`
);
});
// ── Strategy 4: lkgp ──────────────────────────────────────────────────────────
//
// Selector: getLKGP(combo.name, combo.id || combo.name) (src/lib/db/settings.ts)
// → reads key_value WHERE namespace='lkgp' AND key='<comboName>:<comboId>'
// → moves the matched provider to front of orderedTargets
//
// Engineered state:
// Pre-seed: setLKGP(comboId, comboId, "claude") → LKGP record says claude was best
// Combo definition order: openai (1st), claude (2nd)
//
// Expected: claude moved to front despite being second in combo definition.
test("lkgp: last-known-good provider is prioritised above definition order", async () => {
await seedConnection("openai", { apiKey: "sk-openai-lkgp-test" });
await seedConnection("claude", { apiKey: "sk-claude-lkgp-test" });
// Explicit combo id keeps the LKGP key (`comboName:comboId`) predictable.
const COMBO_ID = "m-lkgp-qa-001";
// Import and write the LKGP record into the fresh DB (after resetStorage in beforeEach).
const { setLKGP } = await import("../../../src/lib/db/settings.ts");
// Key written: "m-lkgp-qa-001:m-lkgp-qa-001" (comboName:comboId)
// combo.ts reads: getLKGP(combo.name, combo.id || combo.name)
await setLKGP(COMBO_ID, COMBO_ID, "claude");
await combosDb.createCombo({
id: COMBO_ID,
name: COMBO_ID,
strategy: "lkgp",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST in definition — must be deprioritised (LKGP record points to claude)
"openai/gpt-4o-mini",
// claude SECOND — must be moved to front by the LKGP record
"claude/claude-3-5-sonnet-20241022",
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body(COMBO_ID) }));
assert.equal(r.status, 200, "request must succeed via claude (LKGP-preferred) target");
const first = h.providersSeen()[0];
assert.equal(
first,
"claude",
`lkgp: LKGP record names "claude" as last-known-good, so claude must be dispatched ` +
`first despite being second in the combo definition. Got: ${first}`
);
});
@@ -0,0 +1,202 @@
// tests/integration/combo-matrix/quota-share.test.ts
//
// E2E matrix test for the INTERNAL quota-share routing strategy (DRR / deficit
// round-robin over auto-minted qtSd/ pool combos).
//
// Drive path: handleChat → chatCore → handleComboChat (strategy "quota-share")
// → selectQuotaShareTarget (DRR + P2C + bucket-gating) → executor dispatch.
//
// Two tests:
// 1. DRR fairness — 2 equal-weight connections, 6 requests; both must be
// selected and balanced, proving DRR ran end-to-end (not a single-winner
// or definition-order dispatch).
// 2. Saturation deprioritization — one connection marked saturated via
// recordUsage (5h window); a single request must dispatch to the clean
// connection first, proving the bucket-gating gate runs through the real
// wire.
//
// Drive path chosen: Option A — direct `combosDb.createCombo({ strategy:
// "quota-share", ... })`. The `handleComboChat` dispatch at combo.ts:1573
// branches on `strategy === "quota-share"` verbatim; no qtSd/ pool plumbing is
// required to reach `selectQuotaShareTarget`.
//
// The preferred target is always the SECOND one in the combo definition so that
// a pass proves the selector actually reordered targets (same discipline as the
// 17-strategy quota-aware.test.ts matrix).
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-quota-share");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
// DRR + bucket state seams — import AFTER harness initialises the DB so the
// module-level in-process singletons are already live.
const { _clearDrrStateForTest } = await import(
"../../../open-sse/services/combo/quotaShareStrategy.ts"
);
const { recordUsage, _clearBucketsForTest } = await import(
"../../../src/lib/quota/accountBuckets.ts"
);
// ── Helpers ───────────────────────────────────────────────────────────────────
function body(model: string, suffix = "") {
return {
model,
stream: false,
messages: [{ role: "user", content: `quota-share route${suffix}` }],
};
}
// ── Lifecycle ─────────────────────────────────────────────────────────────────
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
_clearDrrStateForTest();
_clearBucketsForTest();
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
_clearDrrStateForTest();
_clearBucketsForTest();
await resetStorage();
});
test.after(async () => {
_clearDrrStateForTest();
_clearBucketsForTest();
await h.cleanup();
});
// ── Strategy: quota-share — DRR fairness ─────────────────────────────────────
//
// Mechanism: DRR (deficit round robin, quantum = weight / totalWeight).
//
// Engineered state:
// openai (1st in definition) — equal weight 100
// gemini (2nd in definition) — equal weight 100
//
// With 2 equal-weight targets DRR alternates: openai, gemini, openai, gemini, …
// Over 6 requests both must be selected at least 2 times (proven to be exactly
// 3 each by the deterministic math, but asserting >= 2 avoids brittleness if
// P2C ever nudges a tie differently). The assertion fails if the pipeline
// pin-picks the first candidate on every call — i.e. if DRR is bypassed.
test("DRR fairness: 2 equal-weight connections alternate across 6 requests through real pipeline", async () => {
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-qs-drr-1" });
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-qs-drr-1" });
await combosDb.createCombo({
name: "m-qs-drr",
strategy: "quota-share",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST in definition
{
id: "qs-drr-openai",
kind: "model",
providerId: "openai",
model: "gpt-4o-mini",
connectionId: openaiConn.id,
},
// gemini SECOND in definition — DRR must pick it on alternate rounds
{
id: "qs-drr-gemini",
kind: "model",
providerId: "gemini",
model: "gemini-2.5-flash",
connectionId: geminiConn.id,
},
],
});
h.installRecordingFetch();
// Use a unique message suffix per request so session stickiness (message-hash
// based) does not pin every call to the first-round winner. In real traffic
// each conversation has distinct content; the DRR distributes across them.
for (let i = 0; i < 6; i++) {
const r = await handleChat(buildRequest({ body: body("m-qs-drr", ` #${i}`) }));
assert.equal(r.status, 200, `request ${i + 1} must succeed`);
}
const seen = h.providersSeen();
assert.equal(seen.length, 6, "all 6 requests must reach an upstream provider");
const openaiCount = seen.filter((p) => p === "openai").length;
const geminiCount = seen.filter((p) => p === "gemini").length;
assert.ok(
openaiCount >= 2,
`DRR fairness: openai must be selected at least 2 times out of 6; got ${openaiCount}. ` +
`Full sequence: [${seen.join(", ")}]. If openai was never selected, DRR was bypassed.`
);
assert.ok(
geminiCount >= 2,
`DRR fairness: gemini must be selected at least 2 times out of 6; got ${geminiCount}. ` +
`Full sequence: [${seen.join(", ")}]. If gemini was never selected, DRR was bypassed.`
);
});
// ── Strategy: quota-share — saturation deprioritization ──────────────────────
//
// Mechanism: filterEligibleBySaturation (isBucketSaturated, 5h window).
//
// Engineered state:
// openai (1st in definition) — 5h bucket seeded at 100 % usage via
// recordUsage → isBucketSaturated returns true
// → filterEligibleBySaturation demotes it
// gemini (2nd in definition) — no bucket recorded → eligible (clean)
//
// Expected: gemini dispatched first despite being second in the combo
// definition. Proves the bucket-gating runs through the real pipeline wire
// (not merely unit-tested in isolation).
test("saturation deprioritization: saturated connection demoted — clean second target dispatched first", async () => {
const openaiConn = await seedConnection("openai", { apiKey: "sk-openai-qs-sat-1" });
const geminiConn = await seedConnection("gemini", { apiKey: "sk-gemini-qs-sat-1" });
// Seed openai connection as saturated (5h window, 100 % usage, reset in 1 h).
// recordUsage writes the in-process _buckets store; isBucketSaturated reads it.
// The resetAt must be in the FUTURE so the lazy-reset guard keeps the entry live.
const futureResetAt = new Date(Date.now() + 3_600_000).toISOString();
recordUsage(openaiConn.id, "5h", 100, futureResetAt);
await combosDb.createCombo({
name: "m-qs-sat",
strategy: "quota-share",
config: { maxRetries: 0, retryDelayMs: 0, stickyRoundRobinLimit: 1 },
models: [
// openai FIRST in definition — must be demoted (5h bucket saturated)
{
id: "qs-sat-openai",
kind: "model",
providerId: "openai",
model: "gpt-4o-mini",
connectionId: openaiConn.id,
},
// gemini SECOND — must win (no saturation recorded → eligible)
{
id: "qs-sat-gemini",
kind: "model",
providerId: "gemini",
model: "gemini-2.5-flash",
connectionId: geminiConn.id,
},
],
});
h.installRecordingFetch();
const r = await handleChat(buildRequest({ body: body("m-qs-sat") }));
assert.equal(r.status, 200, "request must succeed via clean gemini target");
const first = h.providersSeen()[0];
assert.equal(
first,
"gemini",
`saturation deprioritization: openai 5h bucket is at 100 % (saturated), so ` +
`filterEligibleBySaturation must demote it and dispatch gemini first despite it ` +
`being second in the combo definition. Got: ${first}`
);
});
@@ -0,0 +1,49 @@
// tests/integration/combo-matrix/weighted.test.ts
import test from "node:test";
import assert from "node:assert/strict";
import { createComboRoutingHarness } from "../_comboRoutingHarness.ts";
const h = await createComboRoutingHarness("combo-weighted");
const { BaseExecutor, combosDb, handleChat, buildRequest, seedConnection, resetStorage } = h;
function body(model: string) {
return { model, stream: false, messages: [{ role: "user", content: "w" }] };
}
test.beforeEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = 0;
await resetStorage();
});
test.afterEach(async () => {
BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs;
await resetStorage();
});
test.after(async () => {
await h.cleanup();
});
test("weighted: 70/30 weights produce roughly proportional distribution", async () => {
await seedConnection("openai", { apiKey: "sk-openai-w" });
await seedConnection("claude", { apiKey: "sk-claude-w" });
await combosDb.createCombo({
name: "m-weighted",
strategy: "weighted",
config: { maxRetries: 0, retryDelayMs: 0, stickyWeightedLimit: 1 },
models: [
{ id: "w-openai", kind: "model", providerId: "openai", model: "gpt-4o-mini", weight: 70 },
{ id: "w-claude", kind: "model", providerId: "claude", model: "claude-3-5-sonnet-20241022", weight: 30 },
],
});
h.installRecordingFetch();
const N = 200;
for (let i = 0; i < N; i++) {
const r = await handleChat(buildRequest({ body: body("m-weighted") }));
assert.equal(r.status, 200);
}
const seen = h.providersSeen();
const openaiShare = seen.filter((p) => p === "openai").length / N;
// Tolerance ±0.12 absorbs sampling noise at N=200 while still proving the split.
assert.ok(openaiShare > 0.58 && openaiShare < 0.82, `openai share ${openaiShare} not ~0.70`);
assert.ok(seen.includes("claude"), "weighted must still reach the 30% target");
});