chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* #6512 (follow-up to #6328 / #6495) — regression guard for excluding paid-only
|
||||
* backends from the `auto/*` candidate pool when `hidePaidModels` is on.
|
||||
*
|
||||
* Tests the pure `filterPaidOnlyCandidates` helper wired into
|
||||
* `open-sse/services/autoCombo/virtualFactory.ts::createVirtualAutoCombo`.
|
||||
*/
|
||||
import { test } from "vitest";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { filterPaidOnlyCandidates } from "../../../open-sse/services/autoCombo/paidModelFilter.ts";
|
||||
|
||||
// `agentrouter/claude-opus-4-6` is a documented free model (FREE_MODEL_BUDGETS);
|
||||
// `openai/gpt-4o` is paid (openai has no documented free models).
|
||||
const FREE = { provider: "agentrouter", model: "claude-opus-4-6" };
|
||||
const PAID = { provider: "openai", model: "gpt-4o" };
|
||||
|
||||
test("hidePaidModels OFF (default) returns the pool UNCHANGED (identity, regression guard)", () => {
|
||||
const pool = [FREE, PAID];
|
||||
const result = filterPaidOnlyCandidates(pool, false);
|
||||
assert.equal(result, pool, "must return the exact same array reference when opt-in is off");
|
||||
assert.deepEqual(result, [FREE, PAID], "paid models must pass through when the flag is off");
|
||||
});
|
||||
|
||||
test("hidePaidModels ON drops the paid-only backend, keeps the free one", () => {
|
||||
const result = filterPaidOnlyCandidates([FREE, PAID], true);
|
||||
assert.deepEqual(
|
||||
result,
|
||||
[FREE],
|
||||
"openai/gpt-4o (paid) must be excluded; agentrouter/claude-opus-4-6 (free) kept"
|
||||
);
|
||||
});
|
||||
|
||||
test("hidePaidModels ON with an all-paid pool degrades to an empty pool", () => {
|
||||
const result = filterPaidOnlyCandidates([PAID, { provider: "openai", model: "gpt-4.1" }], true);
|
||||
assert.deepEqual(result, [], "an all-paid pool becomes empty — the graceful empty-pool path");
|
||||
});
|
||||
|
||||
test("hidePaidModels ON preserves extra candidate fields on kept entries", () => {
|
||||
const enriched = { provider: "agentrouter", model: "claude-opus-4-6", connectionId: "abc", extra: 1 };
|
||||
const result = filterPaidOnlyCandidates([enriched, PAID], true);
|
||||
assert.deepEqual(result, [enriched], "generic <T> filter must not strip candidate fields");
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* #6453 — Provider-family auto combos (`auto/glm`, `auto/minimax`, `auto/zai`,
|
||||
* `auto/mimo`, `auto/gemma`, `auto/llama`, `auto/gemini`).
|
||||
*
|
||||
* See: `open-sse/services/autoCombo/modelFamily.ts` (pure family detection) and
|
||||
* `open-sse/services/autoCombo/builtinCatalog.ts` (recognition + materialization).
|
||||
*
|
||||
* NOTE: tests/unit/autoCombo/ is a vitest-only scope (see vitest.mcp.config.ts);
|
||||
* the node:test runner does not walk this dir.
|
||||
*/
|
||||
import { describe, it, beforeEach, afterAll, vi } from "vitest";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
detectModelFamily,
|
||||
isValidModelFamily,
|
||||
AUTO_FAMILY_IDS,
|
||||
} from "../../../open-sse/services/autoCombo/modelFamily";
|
||||
|
||||
// First-touch DB migrations run once per worker and can exceed vitest's 5s
|
||||
// default in a cold thread; the DB-backed materialization tests below need it.
|
||||
vi.setConfig({ testTimeout: 20_000 });
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-family-combo-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const core = await import("../../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../../src/lib/db/providers.ts");
|
||||
const builtinCatalog = await import("../../../open-sse/services/autoCombo/builtinCatalog.ts");
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await resetStorage();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
if (ORIGINAL_DATA_DIR === undefined) {
|
||||
delete process.env.DATA_DIR;
|
||||
} else {
|
||||
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
||||
}
|
||||
});
|
||||
|
||||
describe("detectModelFamily (pure)", () => {
|
||||
it("recognizes glm model ids", () => {
|
||||
assert.equal(detectModelFamily("glm-5.2"), "glm");
|
||||
assert.equal(detectModelFamily("zai/glm-5.2"), "glm");
|
||||
});
|
||||
|
||||
it("recognizes minimax, mimo, gemma, llama, gemini model ids", () => {
|
||||
assert.equal(detectModelFamily("minimax-m3"), "minimax");
|
||||
assert.equal(detectModelFamily("mimo-v2.5"), "mimo");
|
||||
assert.equal(detectModelFamily("gemma-3-27b"), "gemma");
|
||||
assert.equal(detectModelFamily("llama-3.3-70b"), "llama");
|
||||
assert.equal(detectModelFamily("gemini-3-pro"), "gemini");
|
||||
});
|
||||
|
||||
it("returns null for unrelated model ids", () => {
|
||||
assert.equal(detectModelFamily("gpt-4o"), null);
|
||||
assert.equal(detectModelFamily(""), null);
|
||||
assert.equal(detectModelFamily(null), null);
|
||||
});
|
||||
|
||||
it("never detects zai from a model id (provider-override family)", () => {
|
||||
// zai's own models are named glm-*, not zai-*; auto/zai is resolved by
|
||||
// provider id, not by a model-name prefix (see modelFamily.ts comment).
|
||||
assert.equal(detectModelFamily("zai-glm-5.2"), null);
|
||||
});
|
||||
|
||||
it("isValidModelFamily accepts exactly the 7 advertised families", () => {
|
||||
for (const family of ["glm", "minimax", "mimo", "zai", "gemma", "llama", "gemini"]) {
|
||||
assert.equal(isValidModelFamily(family), true);
|
||||
}
|
||||
assert.equal(isValidModelFamily("gpt"), false);
|
||||
assert.equal(isValidModelFamily(undefined), false);
|
||||
});
|
||||
|
||||
it("advertises exactly one auto/<family> catalog id per family", () => {
|
||||
assert.deepEqual(
|
||||
[...AUTO_FAMILY_IDS].sort(),
|
||||
["auto/gemini", "auto/gemma", "auto/glm", "auto/llama", "auto/mimo", "auto/minimax", "auto/zai"]
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("auto/<family> materialization (#6453)", () => {
|
||||
it("resolves auto/glm to a virtual combo spanning every connected GLM backend", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "glm",
|
||||
authType: "apikey",
|
||||
name: "GLM direct",
|
||||
apiKey: "sk-test-glm",
|
||||
defaultModel: "glm-5.2",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "zai",
|
||||
authType: "apikey",
|
||||
name: "z.ai",
|
||||
apiKey: "sk-test-zai",
|
||||
defaultModel: "glm-5.2",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "OpenAI",
|
||||
apiKey: "sk-test-openai",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
});
|
||||
|
||||
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/glm", "glm");
|
||||
|
||||
assert.equal(combo.id, "auto/glm");
|
||||
assert.equal(combo.strategy, "auto");
|
||||
const providerIds = combo.models.map((m) => m.providerId).sort();
|
||||
assert.deepEqual(providerIds, ["glm", "zai"]);
|
||||
assert.ok(combo.models.every((m) => m.model.endsWith("glm-5.2")));
|
||||
});
|
||||
|
||||
it("resolves auto/zai to ONLY the zai-provider connection (provider-override family)", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "glm",
|
||||
authType: "apikey",
|
||||
name: "GLM direct",
|
||||
apiKey: "sk-test-glm",
|
||||
defaultModel: "glm-5.2",
|
||||
});
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "zai",
|
||||
authType: "apikey",
|
||||
name: "z.ai",
|
||||
apiKey: "sk-test-zai",
|
||||
defaultModel: "glm-5.2",
|
||||
});
|
||||
|
||||
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/zai", "zai");
|
||||
|
||||
assert.equal(combo.id, "auto/zai");
|
||||
const providerIds = combo.models.map((m) => m.providerId);
|
||||
assert.deepEqual(providerIds, ["zai"]);
|
||||
});
|
||||
|
||||
it("degrades gracefully to the family subset, excluding connected-but-unrelated providers", async () => {
|
||||
// Free/noAuth backends may still expose a family model (e.g. opencode serves
|
||||
// minimax under its own catalog) — the family combo is expected to include
|
||||
// those, but MUST exclude a connected provider whose model is a different
|
||||
// family entirely. This is the "subset available" degrade path (#6453).
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "OpenAI",
|
||||
apiKey: "sk-test-openai",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
});
|
||||
|
||||
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/minimax", "minimax");
|
||||
|
||||
assert.equal(combo.id, "auto/minimax");
|
||||
assert.ok(
|
||||
combo.models.every((m) => m.providerId !== "openai"),
|
||||
"auto/minimax must not include the connected openai/gpt-4o-mini candidate"
|
||||
);
|
||||
assert.ok(
|
||||
combo.models.every((m) => detectModelFamily(m.model) === "minimax"),
|
||||
"every candidate in auto/minimax must actually be a minimax model"
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
it("rejects auto/<unknownfamily> with the same clean error as any unknown combo", async () => {
|
||||
await assert.rejects(
|
||||
() => builtinCatalog.createBuiltinAutoCombo("auto/unknownfam", "unknownfam"),
|
||||
/Unknown built-in auto combo/
|
||||
);
|
||||
});
|
||||
|
||||
it("isRecognizedBuiltinAuto recognizes every auto/<family> id", () => {
|
||||
for (const family of ["glm", "minimax", "mimo", "zai", "gemma", "llama", "gemini"]) {
|
||||
assert.equal(builtinCatalog.isRecognizedBuiltinAuto(`auto/${family}`, family), true);
|
||||
}
|
||||
assert.equal(builtinCatalog.isRecognizedBuiltinAuto("auto/unknownfam", "unknownfam"), false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Unit tests for the `auto/<category>:<tier>` suffix composition filter.
|
||||
*
|
||||
* See: `open-sse/services/autoCombo/suffixComposition.ts`
|
||||
*
|
||||
* Focus: the `:free` tier filter. Regression test for the bug where
|
||||
* opencode (noAuth, free) and mimocode (noAuth, free) were NOT being
|
||||
* included in the free pool because the legacy `freeProviders` list
|
||||
* only contained paid-API-key providers with free tiers (kiro, qoder, ...).
|
||||
*
|
||||
* #4517.
|
||||
*/
|
||||
// NOTE: tests/unit/autoCombo/ is a vitest-only scope (the node:test runner in
|
||||
// test:unit:ci does not walk this dir), so this suite uses the vitest API even
|
||||
// though it asserts via node:assert. (#4753 originally landed it with node:test
|
||||
// imports → vitest reported "No test suite found" and the test ran nowhere.)
|
||||
import { describe, it, beforeAll, afterAll, beforeEach } from "vitest";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
buildAutoCandidateFilter,
|
||||
parseAutoSuffix,
|
||||
} from "../../../open-sse/services/autoCombo/suffixComposition";
|
||||
|
||||
describe("suffixComposition :free tier (#4517)", () => {
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
|
||||
beforeAll(() => {
|
||||
// Snapshot env so we can restore it after each test.
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset env to a known state so OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL
|
||||
// doesn't leak between cases.
|
||||
process.env = { ...ORIGINAL_ENV };
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
process.env = ORIGINAL_ENV;
|
||||
});
|
||||
|
||||
it("parseAutoSuffix recognizes coding:free", () => {
|
||||
assert.deepEqual(parseAutoSuffix("coding:free"), {
|
||||
valid: true,
|
||||
category: "coding",
|
||||
tier: "free",
|
||||
});
|
||||
});
|
||||
|
||||
it("buildAutoCandidateFilter keeps noAuth free providers", () => {
|
||||
// Regression: opencode and mimocode are noAuth and free, but the
|
||||
// pre-fix `freeProviders` list omitted them, so the filter rejected
|
||||
// their candidates even though they ARE free upstream.
|
||||
const filter = buildAutoCandidateFilter("coding", "free");
|
||||
assert.notEqual(filter, null);
|
||||
|
||||
assert.equal(filter!({ provider: "opencode", model: "big-pickle" }), true);
|
||||
assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true);
|
||||
assert.equal(filter!({ provider: "mimocode", model: "mimo-auto" }), true);
|
||||
assert.equal(filter!({ provider: "duckduckgo-web", model: "gpt-4o-mini" }), true);
|
||||
});
|
||||
|
||||
it("buildAutoCandidateFilter keeps legacy free providers", () => {
|
||||
// Don't break the existing list — kiro, qoder, groq, etc. must still pass.
|
||||
const filter = buildAutoCandidateFilter("coding", "free");
|
||||
assert.equal(filter!({ provider: "kiro", model: "claude-sonnet-4-5" }), true);
|
||||
assert.equal(filter!({ provider: "groq", model: "llama-3.3-70b" }), true);
|
||||
assert.equal(filter!({ provider: "qoder", model: "qwen3-coder-plus" }), true);
|
||||
});
|
||||
|
||||
it("buildAutoCandidateFilter rejects paid models under :free", () => {
|
||||
// The bug: opencode-go/glm-5.1 was being picked because the filter
|
||||
// fell back to the full pool when no free candidate was found.
|
||||
// After the fix, glm-5.1 must be rejected by the :free filter.
|
||||
const filter = buildAutoCandidateFilter("coding", "free");
|
||||
assert.equal(filter!({ provider: "opencode-go", model: "glm-5.1" }), false);
|
||||
assert.equal(filter!({ provider: "openai", model: "gpt-4o" }), false);
|
||||
assert.equal(filter!({ provider: "anthropic", model: "claude-sonnet-4-6" }), false);
|
||||
assert.equal(filter!({ provider: "deepseek", model: "deepseek-chat" }), false);
|
||||
});
|
||||
|
||||
it("buildAutoCandidateFilter returns null for category-only (no tier)", () => {
|
||||
// "coding" with no tier must NOT filter by tier — pass-through.
|
||||
const filter = buildAutoCandidateFilter("coding", undefined);
|
||||
assert.equal(filter, null);
|
||||
});
|
||||
|
||||
it("buildAutoCandidateFilter keeps free candidates alongside capability checks", () => {
|
||||
// The category and tier checks are AND-combined. For "coding:free" the
|
||||
// category check is a pass-through (no vision/reasoning filter), so
|
||||
// any free model should be kept.
|
||||
const filter = buildAutoCandidateFilter("coding", "free");
|
||||
assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true);
|
||||
// The "reasoning" category also pairs with ":free" and keeps free models.
|
||||
const reasoningFilter = buildAutoCandidateFilter("reasoning", "free");
|
||||
// big-pickle (model_capabilities: reasoning=1) should pass the reasoning check.
|
||||
assert.equal(reasoningFilter!({ provider: "opencode", model: "big-pickle" }), true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,239 @@
|
||||
/**
|
||||
* Tests for ScoreTierRotator and connectionDensity factor.
|
||||
* Verifies that multi-connection providers surface in ranked candidates
|
||||
* and that tiered rotation distributes traffic fairly.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import { selectProvider, type AutoComboConfig } from "../../../open-sse/services/autoCombo/engine";
|
||||
import {
|
||||
calculateFactors,
|
||||
calculateScore,
|
||||
DEFAULT_WEIGHTS,
|
||||
scorePool,
|
||||
type ProviderCandidate,
|
||||
type ScoredProvider,
|
||||
} from "../../../open-sse/services/autoCombo/scoring";
|
||||
import { getTaskFitness } from "../../../open-sse/services/autoCombo/taskFitness";
|
||||
import { resetDiversity } from "../../../open-sse/services/autoCombo/providerDiversity";
|
||||
|
||||
// Stub DB calls introduced by PR #3660 (Arena ELO / models.dev intelligence scoring).
|
||||
// This file tests rotation logic, not intelligence scoring — the DB shouldn't be
|
||||
// initialized during these unit tests, and the stub makes them fast and isolated.
|
||||
vi.mock("../../../src/lib/db/modelIntelligence.ts", () => ({
|
||||
getModelIntelligenceBySource: vi.fn(() => null),
|
||||
setUserFitnessOverrideEntry: vi.fn(),
|
||||
deleteUserFitnessOverrideEntry: vi.fn(),
|
||||
}));
|
||||
|
||||
function makeCandidate(overrides: Partial<ProviderCandidate>): ProviderCandidate {
|
||||
return {
|
||||
provider: "unknown",
|
||||
model: "unknown-model",
|
||||
quotaRemaining: 100,
|
||||
quotaTotal: 100,
|
||||
circuitBreakerState: "CLOSED",
|
||||
costPer1MTokens: 1,
|
||||
p95LatencyMs: 1000,
|
||||
latencyStdDev: 100,
|
||||
errorRate: 0.01,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeConfig(name: string): AutoComboConfig {
|
||||
return {
|
||||
id: `test-${name}`,
|
||||
name,
|
||||
type: "auto",
|
||||
candidatePool: [],
|
||||
weights: { ...DEFAULT_WEIGHTS },
|
||||
explorationRate: 0,
|
||||
routerStrategy: "rules",
|
||||
};
|
||||
}
|
||||
|
||||
describe("Connection Density Factor", () => {
|
||||
const baseCandidate = makeCandidate({ provider: "cerebras", model: "llama-70b" });
|
||||
|
||||
it("multi-connection provider scores higher than single-connection at same quality", () => {
|
||||
const multiConn = makeCandidate({ provider: "cerebras", model: "llama-70b", connectionPoolSize: 43 });
|
||||
const singleConn = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1 });
|
||||
const pool = [multiConn, singleConn];
|
||||
|
||||
const multiFactors = calculateFactors(multiConn, pool, "coding", getTaskFitness);
|
||||
const singleFactors = calculateFactors(singleConn, pool, "coding", getTaskFitness);
|
||||
const multiScore = calculateScore(multiFactors, DEFAULT_WEIGHTS);
|
||||
const singleScore = calculateScore(singleFactors, DEFAULT_WEIGHTS);
|
||||
|
||||
expect(multiFactors.connectionDensity).toBe(1.0);
|
||||
expect(singleFactors.connectionDensity).toBe(0.0);
|
||||
expect(multiScore).toBeGreaterThan(singleScore);
|
||||
});
|
||||
|
||||
it("density scales linearly from 0 to 10 connections, caps at 10+", () => {
|
||||
const make = (size: number) => makeCandidate({ connectionPoolSize: size });
|
||||
const sizes = [1, 2, 5, 10, 20, 43];
|
||||
const densities = sizes.map((s) => {
|
||||
const c = make(s);
|
||||
const pool = [c];
|
||||
return calculateFactors(c, pool, "coding", getTaskFitness).connectionDensity;
|
||||
});
|
||||
|
||||
expect(densities[0]).toBeCloseTo(0.0, 5);
|
||||
expect(densities[1]).toBeCloseTo(0.1, 5);
|
||||
expect(densities[2]).toBeCloseTo(0.4, 5);
|
||||
expect(densities[3]).toBeCloseTo(0.9, 5);
|
||||
expect(densities[4]).toBe(1.0);
|
||||
expect(densities[5]).toBe(1.0);
|
||||
});
|
||||
|
||||
it("missing connectionPoolSize defaults to 1 (backward compat)", () => {
|
||||
const candidate = makeCandidate({ provider: "x" });
|
||||
const pool = [candidate];
|
||||
const factors = calculateFactors(candidate, pool, "coding", getTaskFitness);
|
||||
expect(factors.connectionDensity).toBe(0.0);
|
||||
});
|
||||
|
||||
it("DEFAULT_WEIGHTS still sum to 1.0 after adding density", () => {
|
||||
const sum = Object.values(DEFAULT_WEIGHTS).reduce((a, b) => a + b, 0);
|
||||
expect(Math.abs(sum - 1.0)).toBeLessThan(0.01);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Tiered Rotation in selectProvider", () => {
|
||||
beforeEach(() => {
|
||||
resetDiversity();
|
||||
});
|
||||
|
||||
it("smart combo rotates within top tier across many requests", () => {
|
||||
const topA = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 95 });
|
||||
const topB = makeCandidate({ provider: "anthropic", model: "claude-opus", quotaRemaining: 90 });
|
||||
const topC = makeCandidate({ provider: "google", model: "gemini-ultra", quotaRemaining: 88 });
|
||||
const mid = makeCandidate({ provider: "mistral", model: "mistral-large", quotaRemaining: 70 });
|
||||
const pool = [topA, topB, topC, mid];
|
||||
|
||||
const config = makeConfig("smart");
|
||||
const seen = new Set<string>();
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const result = selectProvider(config, pool, "coding");
|
||||
seen.add(`${result.provider}/${result.model}`);
|
||||
}
|
||||
expect(seen.size).toBeGreaterThanOrEqual(2);
|
||||
expect(seen.has("openai/gpt-4o") || seen.has("anthropic/claude-opus")).toBe(true);
|
||||
});
|
||||
|
||||
it("cheap combo pulls from rest tier (lower scores) more often than smart", () => {
|
||||
const top = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 100 });
|
||||
const rest = makeCandidate({
|
||||
provider: "cheap-provider",
|
||||
model: "cheap-model",
|
||||
quotaRemaining: 100,
|
||||
costPer1MTokens: 0,
|
||||
p95LatencyMs: 5000,
|
||||
});
|
||||
const pool = [top, rest];
|
||||
|
||||
const config = makeConfig("cheap");
|
||||
const counts: Record<string, number> = {};
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const result = selectProvider(config, pool, "coding");
|
||||
counts[result.provider] = (counts[result.provider] ?? 0) + 1;
|
||||
}
|
||||
expect(counts["cheap-provider"]).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("single-candidate pool always returns the same candidate", () => {
|
||||
const only = makeCandidate({ provider: "only", model: "only-model" });
|
||||
const config = makeConfig("smart");
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const result = selectProvider(config, [only], "coding");
|
||||
expect(result.provider).toBe("only");
|
||||
expect(result.model).toBe("only-model");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("scorePool with connectionDensity", () => {
|
||||
it("Cerebras with 43 keys ranks above single-connection providers of similar quality", () => {
|
||||
const cerebras = makeCandidate({
|
||||
provider: "cerebras",
|
||||
model: "llama-3.1-70b",
|
||||
connectionPoolSize: 43,
|
||||
quotaRemaining: 100,
|
||||
});
|
||||
const anthropic = makeCandidate({
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet",
|
||||
connectionPoolSize: 1,
|
||||
quotaRemaining: 100,
|
||||
});
|
||||
const pool = [cerebras, anthropic];
|
||||
const scored = scorePool(pool, "coding", DEFAULT_WEIGHTS, getTaskFitness);
|
||||
expect(scored[0].provider).toBe("cerebras");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Per-Connection Rotation", () => {
|
||||
it("rotates across all 43 Cerebras connection IDs, not just one", () => {
|
||||
const cerebrasCandidates: ProviderCandidate[] = Array.from({ length: 43 }, (_, i) =>
|
||||
makeCandidate({
|
||||
provider: "cerebras",
|
||||
model: "llama-3.1-70b",
|
||||
connectionId: `cerebras-conn-${i + 1}`,
|
||||
})
|
||||
);
|
||||
const config = makeConfig("smart");
|
||||
|
||||
const seenConnections = new Set<string>();
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const result = selectProvider(config, cerebrasCandidates, "coding");
|
||||
if (result.connectionId) seenConnections.add(result.connectionId);
|
||||
}
|
||||
expect(seenConnections.size).toBeGreaterThanOrEqual(10);
|
||||
});
|
||||
|
||||
it("different combos maintain independent round-robin state", () => {
|
||||
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
|
||||
makeCandidate({ provider: "p", model: "m", connectionId: `c-${i}` })
|
||||
);
|
||||
const smartConfig = makeConfig("smart-A");
|
||||
const fastConfig = makeConfig("fast-B");
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
selectProvider(smartConfig, candidates, "coding");
|
||||
}
|
||||
const smartResults: string[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const r = selectProvider(smartConfig, candidates, "coding");
|
||||
if (r.connectionId) smartResults.push(r.connectionId);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
selectProvider(fastConfig, candidates, "coding");
|
||||
}
|
||||
const fastResults: string[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const r = selectProvider(fastConfig, candidates, "coding");
|
||||
if (r.connectionId) fastResults.push(r.connectionId);
|
||||
}
|
||||
|
||||
expect(smartResults.length).toBe(5);
|
||||
expect(fastResults.length).toBe(5);
|
||||
expect(new Set(smartResults).size).toBeGreaterThan(1);
|
||||
expect(new Set(fastResults).size).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("tied-score candidates from same provider+model are all reachable", () => {
|
||||
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
|
||||
makeCandidate({ provider: "free", model: "free-model", connectionId: `key-${i}` })
|
||||
);
|
||||
const config = makeConfig("smart");
|
||||
const visited = new Set<string>();
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const result = selectProvider(config, candidates, "coding");
|
||||
if (result.connectionId) visited.add(result.connectionId);
|
||||
}
|
||||
expect(visited.size).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user