// tests/integration/combo-matrix/auto.test.ts // // Integration tests for `model: "auto"` and `model: "auto/"` routing. // // How auto routing works: // - model:"auto" → chat.ts detects isAutoRouting → calls createVirtualAutoCombo(undefined) // which reads active connections from DB and builds a virtual combo (name="auto", id="auto"). // - The virtual combo's default routerStrategy is "lkgp". // - combo.ts fetches getLKGP(combo.name, combo.id) = getLKGP("auto", "auto") → key "auto:auto". // - LKGPStrategy: if a LKGP record exists for that provider AND the provider is in the // candidate pool, it pins that provider deterministically (no randomness). // - Without a LKGP record, falls back to "rules" 9-factor scorer. // // Test 1 strategy: single-request, deterministic. // We seed a LKGP record pointing to "openai" via settingsDb.setLKGP("auto","auto","openai"). // The assertion h.providersSeen()[0] === "openai" is guaranteed by the LKGP pin // and will FAIL if the LKGP lookup or the LKGPStrategy routing is broken. // // Test 2 strategy: pool-resolution check (not provider-specific). // Send model:"auto/coding" → virtual combo for the "coding" variant (MODE_PACKS quality-first). // Assert status 200 AND a seeded provider was dispatched. The pool should resolve from the // DB connections; this test fails if the variant is unrecognised (400) or the pool is empty. import test from "node:test"; import assert from "node:assert/strict"; import { createComboRoutingHarness } from "../_comboRoutingHarness.ts"; const h = await createComboRoutingHarness("combo-auto"); const { BaseExecutor, handleChat, buildRequest, seedConnection, resetStorage, settingsDb } = h; function body(model: string) { return { model, stream: false, messages: [{ role: "user", content: "hello" }] }; } test.beforeEach(async () => { BaseExecutor.RETRY_CONFIG.delayMs = 0; await resetStorage(); }); test.afterEach(async () => { BaseExecutor.RETRY_CONFIG.delayMs = h.originalRetryDelayMs; await resetStorage(); }); test.after(async () => { await h.cleanup(); }); // ── Test 1: auto picks the LKGP-biased provider ───────────────────────────── // // Strategy: single-request, deterministic via LKGP seed. // The virtual auto-combo for model:"auto" has name="auto" and id="auto". // getLKGP("auto","auto") reads key "auto:auto" in the key_value table. // LKGPStrategy pins the seeded provider when it is healthy + in the pool. // This assertion fails if LKGP lookup, LKGPStrategy, or candidate resolution breaks. test("auto: LKGP record biases dispatch toward the seeded provider", async () => { await seedConnection("openai", { apiKey: "sk-openai-auto" }); await seedConnection("claude", { apiKey: "sk-claude-auto" }); // Bias toward openai by writing a LKGP record before the request. await settingsDb.setLKGP("auto", "auto", "openai"); h.installRecordingFetch(); const r = await handleChat(buildRequest({ body: body("auto") })); assert.equal(r.status, 200, `Expected 200, got ${r.status}`); const seen = h.providersSeen(); assert.ok(seen.length > 0, "Expected at least one upstream dispatch"); assert.equal( seen[0], "openai", `Expected openai to be dispatched first (LKGP-biased), got: ${JSON.stringify(seen)}` ); }); // ── Test 2: auto/coding resolves a virtual pool from seeded connections ────── // // "auto/coding" is a recognized variant (VALID_AUTO_VARIANTS has "coding"). // createVirtualAutoCombo("coding") applies MODE_PACKS["quality-first"] weights // and uses the same DB connections as candidates. // This test fails if: // - the variant is not recognized and returns a 400, or // - the virtual pool is empty (no candidates resolved from DB connections), or // - the dispatched provider is not one of the seeded ones. test("auto/coding: virtual pool resolves and dispatches a seeded provider", async () => { await seedConnection("openai", { apiKey: "sk-openai-coding" }); await seedConnection("claude", { apiKey: "sk-claude-coding" }); h.installRecordingFetch(); const r = await handleChat(buildRequest({ body: body("auto/coding") })); assert.equal(r.status, 200, `Expected 200 from auto/coding, got ${r.status}`); const seen = h.providersSeen(); assert.ok(seen.length > 0, "Expected at least one upstream dispatch for auto/coding"); const seededProviders = new Set(["openai", "claude"]); assert.ok( seededProviders.has(seen[0]), `Expected a seeded provider (openai or claude), got: ${seen[0]} (all: ${JSON.stringify(seen)})` ); });