/** * Tests for ScoreTierRotator and connectionDensity factor. * Verifies that multi-connection providers surface in ranked candidates * and that tiered rotation distributes traffic fairly. */ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { selectProvider, type AutoComboConfig } from "../../../open-sse/services/autoCombo/engine"; import { calculateFactors, calculateScore, DEFAULT_WEIGHTS, scorePool, type ProviderCandidate, type ScoredProvider, } from "../../../open-sse/services/autoCombo/scoring"; import { getTaskFitness } from "../../../open-sse/services/autoCombo/taskFitness"; import { resetDiversity } from "../../../open-sse/services/autoCombo/providerDiversity"; // Stub DB calls introduced by PR #3660 (Arena ELO / models.dev intelligence scoring). // This file tests rotation logic, not intelligence scoring — the DB shouldn't be // initialized during these unit tests, and the stub makes them fast and isolated. vi.mock("../../../src/lib/db/modelIntelligence.ts", () => ({ getModelIntelligenceBySource: vi.fn(() => null), setUserFitnessOverrideEntry: vi.fn(), deleteUserFitnessOverrideEntry: vi.fn(), })); function makeCandidate(overrides: Partial): ProviderCandidate { return { provider: "unknown", model: "unknown-model", quotaRemaining: 100, quotaTotal: 100, circuitBreakerState: "CLOSED", costPer1MTokens: 1, p95LatencyMs: 1000, latencyStdDev: 100, errorRate: 0.01, ...overrides, }; } function makeConfig(name: string): AutoComboConfig { return { id: `test-${name}`, name, type: "auto", candidatePool: [], weights: { ...DEFAULT_WEIGHTS }, explorationRate: 0, routerStrategy: "rules", }; } describe("Connection Density Factor", () => { const baseCandidate = makeCandidate({ provider: "cerebras", model: "llama-70b" }); it("multi-connection provider scores higher than single-connection at same quality", () => { const multiConn = makeCandidate({ provider: "cerebras", model: "llama-70b", connectionPoolSize: 43 }); const singleConn = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1 }); const pool = [multiConn, singleConn]; const multiFactors = calculateFactors(multiConn, pool, "coding", getTaskFitness); const singleFactors = calculateFactors(singleConn, pool, "coding", getTaskFitness); const multiScore = calculateScore(multiFactors, DEFAULT_WEIGHTS); const singleScore = calculateScore(singleFactors, DEFAULT_WEIGHTS); expect(multiFactors.connectionDensity).toBe(1.0); expect(singleFactors.connectionDensity).toBe(0.0); expect(multiScore).toBeGreaterThan(singleScore); }); it("density scales linearly from 0 to 10 connections, caps at 10+", () => { const make = (size: number) => makeCandidate({ connectionPoolSize: size }); const sizes = [1, 2, 5, 10, 20, 43]; const densities = sizes.map((s) => { const c = make(s); const pool = [c]; return calculateFactors(c, pool, "coding", getTaskFitness).connectionDensity; }); expect(densities[0]).toBeCloseTo(0.0, 5); expect(densities[1]).toBeCloseTo(0.1, 5); expect(densities[2]).toBeCloseTo(0.4, 5); expect(densities[3]).toBeCloseTo(0.9, 5); expect(densities[4]).toBe(1.0); expect(densities[5]).toBe(1.0); }); it("missing connectionPoolSize defaults to 1 (backward compat)", () => { const candidate = makeCandidate({ provider: "x" }); const pool = [candidate]; const factors = calculateFactors(candidate, pool, "coding", getTaskFitness); expect(factors.connectionDensity).toBe(0.0); }); it("DEFAULT_WEIGHTS still sum to 1.0 after adding density", () => { const sum = Object.values(DEFAULT_WEIGHTS).reduce((a, b) => a + b, 0); expect(Math.abs(sum - 1.0)).toBeLessThan(0.01); }); }); describe("Tiered Rotation in selectProvider", () => { beforeEach(() => { resetDiversity(); }); it("smart combo rotates within top tier across many requests", () => { const topA = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 95 }); const topB = makeCandidate({ provider: "anthropic", model: "claude-opus", quotaRemaining: 90 }); const topC = makeCandidate({ provider: "google", model: "gemini-ultra", quotaRemaining: 88 }); const mid = makeCandidate({ provider: "mistral", model: "mistral-large", quotaRemaining: 70 }); const pool = [topA, topB, topC, mid]; const config = makeConfig("smart"); const seen = new Set(); for (let i = 0; i < 50; i++) { const result = selectProvider(config, pool, "coding"); seen.add(`${result.provider}/${result.model}`); } expect(seen.size).toBeGreaterThanOrEqual(2); expect(seen.has("openai/gpt-4o") || seen.has("anthropic/claude-opus")).toBe(true); }); it("cheap combo pulls from rest tier (lower scores) more often than smart", () => { const top = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 100 }); const rest = makeCandidate({ provider: "cheap-provider", model: "cheap-model", quotaRemaining: 100, costPer1MTokens: 0, p95LatencyMs: 5000, }); const pool = [top, rest]; const config = makeConfig("cheap"); const counts: Record = {}; for (let i = 0; i < 200; i++) { const result = selectProvider(config, pool, "coding"); counts[result.provider] = (counts[result.provider] ?? 0) + 1; } expect(counts["cheap-provider"]).toBeGreaterThan(0); }); it("single-candidate pool always returns the same candidate", () => { const only = makeCandidate({ provider: "only", model: "only-model" }); const config = makeConfig("smart"); for (let i = 0; i < 10; i++) { const result = selectProvider(config, [only], "coding"); expect(result.provider).toBe("only"); expect(result.model).toBe("only-model"); } }); }); describe("scorePool with connectionDensity", () => { it("Cerebras with 43 keys ranks above single-connection providers of similar quality", () => { const cerebras = makeCandidate({ provider: "cerebras", model: "llama-3.1-70b", connectionPoolSize: 43, quotaRemaining: 100, }); const anthropic = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1, quotaRemaining: 100, }); const pool = [cerebras, anthropic]; const scored = scorePool(pool, "coding", DEFAULT_WEIGHTS, getTaskFitness); expect(scored[0].provider).toBe("cerebras"); }); }); describe("Per-Connection Rotation", () => { it("rotates across all 43 Cerebras connection IDs, not just one", () => { const cerebrasCandidates: ProviderCandidate[] = Array.from({ length: 43 }, (_, i) => makeCandidate({ provider: "cerebras", model: "llama-3.1-70b", connectionId: `cerebras-conn-${i + 1}`, }) ); const config = makeConfig("smart"); const seenConnections = new Set(); for (let i = 0; i < 200; i++) { const result = selectProvider(config, cerebrasCandidates, "coding"); if (result.connectionId) seenConnections.add(result.connectionId); } expect(seenConnections.size).toBeGreaterThanOrEqual(10); }); it("different combos maintain independent round-robin state", () => { const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) => makeCandidate({ provider: "p", model: "m", connectionId: `c-${i}` }) ); const smartConfig = makeConfig("smart-A"); const fastConfig = makeConfig("fast-B"); for (let i = 0; i < 5; i++) { selectProvider(smartConfig, candidates, "coding"); } const smartResults: string[] = []; for (let i = 0; i < 5; i++) { const r = selectProvider(smartConfig, candidates, "coding"); if (r.connectionId) smartResults.push(r.connectionId); } for (let i = 0; i < 5; i++) { selectProvider(fastConfig, candidates, "coding"); } const fastResults: string[] = []; for (let i = 0; i < 5; i++) { const r = selectProvider(fastConfig, candidates, "coding"); if (r.connectionId) fastResults.push(r.connectionId); } expect(smartResults.length).toBe(5); expect(fastResults.length).toBe(5); expect(new Set(smartResults).size).toBeGreaterThan(1); expect(new Set(fastResults).size).toBeGreaterThan(1); }); it("tied-score candidates from same provider+model are all reachable", () => { const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) => makeCandidate({ provider: "free", model: "free-model", connectionId: `key-${i}` }) ); const config = makeConfig("smart"); const visited = new Set(); for (let i = 0; i < 20; i++) { const result = selectProvider(config, candidates, "coding"); if (result.connectionId) visited.add(result.connectionId); } expect(visited.size).toBeGreaterThan(1); }); });