chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:12 +08:00
commit d8dcd5f6d1
8604 changed files with 2479390 additions and 0 deletions
@@ -0,0 +1,43 @@
/**
* #6512 (follow-up to #6328 / #6495) — regression guard for excluding paid-only
* backends from the `auto/*` candidate pool when `hidePaidModels` is on.
*
* Tests the pure `filterPaidOnlyCandidates` helper wired into
* `open-sse/services/autoCombo/virtualFactory.ts::createVirtualAutoCombo`.
*/
import { test } from "vitest";
import assert from "node:assert/strict";
import { filterPaidOnlyCandidates } from "../../../open-sse/services/autoCombo/paidModelFilter.ts";
// `agentrouter/claude-opus-4-6` is a documented free model (FREE_MODEL_BUDGETS);
// `openai/gpt-4o` is paid (openai has no documented free models).
const FREE = { provider: "agentrouter", model: "claude-opus-4-6" };
const PAID = { provider: "openai", model: "gpt-4o" };
test("hidePaidModels OFF (default) returns the pool UNCHANGED (identity, regression guard)", () => {
const pool = [FREE, PAID];
const result = filterPaidOnlyCandidates(pool, false);
assert.equal(result, pool, "must return the exact same array reference when opt-in is off");
assert.deepEqual(result, [FREE, PAID], "paid models must pass through when the flag is off");
});
test("hidePaidModels ON drops the paid-only backend, keeps the free one", () => {
const result = filterPaidOnlyCandidates([FREE, PAID], true);
assert.deepEqual(
result,
[FREE],
"openai/gpt-4o (paid) must be excluded; agentrouter/claude-opus-4-6 (free) kept"
);
});
test("hidePaidModels ON with an all-paid pool degrades to an empty pool", () => {
const result = filterPaidOnlyCandidates([PAID, { provider: "openai", model: "gpt-4.1" }], true);
assert.deepEqual(result, [], "an all-paid pool becomes empty — the graceful empty-pool path");
});
test("hidePaidModels ON preserves extra candidate fields on kept entries", () => {
const enriched = { provider: "agentrouter", model: "claude-opus-4-6", connectionId: "abc", extra: 1 };
const result = filterPaidOnlyCandidates([enriched, PAID], true);
assert.deepEqual(result, [enriched], "generic <T> filter must not strip candidate fields");
});
@@ -0,0 +1,193 @@
/**
* #6453 — Provider-family auto combos (`auto/glm`, `auto/minimax`, `auto/zai`,
* `auto/mimo`, `auto/gemma`, `auto/llama`, `auto/gemini`).
*
* See: `open-sse/services/autoCombo/modelFamily.ts` (pure family detection) and
* `open-sse/services/autoCombo/builtinCatalog.ts` (recognition + materialization).
*
* NOTE: tests/unit/autoCombo/ is a vitest-only scope (see vitest.mcp.config.ts);
* the node:test runner does not walk this dir.
*/
import { describe, it, beforeEach, afterAll, vi } from "vitest";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
detectModelFamily,
isValidModelFamily,
AUTO_FAMILY_IDS,
} from "../../../open-sse/services/autoCombo/modelFamily";
// First-touch DB migrations run once per worker and can exceed vitest's 5s
// default in a cold thread; the DB-backed materialization tests below need it.
vi.setConfig({ testTimeout: 20_000 });
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-family-combo-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const core = await import("../../../src/lib/db/core.ts");
const providersDb = await import("../../../src/lib/db/providers.ts");
const builtinCatalog = await import("../../../open-sse/services/autoCombo/builtinCatalog.ts");
async function resetStorage() {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
beforeEach(async () => {
await resetStorage();
});
afterAll(async () => {
await resetStorage();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) {
delete process.env.DATA_DIR;
} else {
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
}
});
describe("detectModelFamily (pure)", () => {
it("recognizes glm model ids", () => {
assert.equal(detectModelFamily("glm-5.2"), "glm");
assert.equal(detectModelFamily("zai/glm-5.2"), "glm");
});
it("recognizes minimax, mimo, gemma, llama, gemini model ids", () => {
assert.equal(detectModelFamily("minimax-m3"), "minimax");
assert.equal(detectModelFamily("mimo-v2.5"), "mimo");
assert.equal(detectModelFamily("gemma-3-27b"), "gemma");
assert.equal(detectModelFamily("llama-3.3-70b"), "llama");
assert.equal(detectModelFamily("gemini-3-pro"), "gemini");
});
it("returns null for unrelated model ids", () => {
assert.equal(detectModelFamily("gpt-4o"), null);
assert.equal(detectModelFamily(""), null);
assert.equal(detectModelFamily(null), null);
});
it("never detects zai from a model id (provider-override family)", () => {
// zai's own models are named glm-*, not zai-*; auto/zai is resolved by
// provider id, not by a model-name prefix (see modelFamily.ts comment).
assert.equal(detectModelFamily("zai-glm-5.2"), null);
});
it("isValidModelFamily accepts exactly the 7 advertised families", () => {
for (const family of ["glm", "minimax", "mimo", "zai", "gemma", "llama", "gemini"]) {
assert.equal(isValidModelFamily(family), true);
}
assert.equal(isValidModelFamily("gpt"), false);
assert.equal(isValidModelFamily(undefined), false);
});
it("advertises exactly one auto/<family> catalog id per family", () => {
assert.deepEqual(
[...AUTO_FAMILY_IDS].sort(),
["auto/gemini", "auto/gemma", "auto/glm", "auto/llama", "auto/mimo", "auto/minimax", "auto/zai"]
);
});
});
describe("auto/<family> materialization (#6453)", () => {
it("resolves auto/glm to a virtual combo spanning every connected GLM backend", async () => {
await providersDb.createProviderConnection({
provider: "glm",
authType: "apikey",
name: "GLM direct",
apiKey: "sk-test-glm",
defaultModel: "glm-5.2",
});
await providersDb.createProviderConnection({
provider: "zai",
authType: "apikey",
name: "z.ai",
apiKey: "sk-test-zai",
defaultModel: "glm-5.2",
});
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "OpenAI",
apiKey: "sk-test-openai",
defaultModel: "gpt-4o-mini",
});
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/glm", "glm");
assert.equal(combo.id, "auto/glm");
assert.equal(combo.strategy, "auto");
const providerIds = combo.models.map((m) => m.providerId).sort();
assert.deepEqual(providerIds, ["glm", "zai"]);
assert.ok(combo.models.every((m) => m.model.endsWith("glm-5.2")));
});
it("resolves auto/zai to ONLY the zai-provider connection (provider-override family)", async () => {
await providersDb.createProviderConnection({
provider: "glm",
authType: "apikey",
name: "GLM direct",
apiKey: "sk-test-glm",
defaultModel: "glm-5.2",
});
await providersDb.createProviderConnection({
provider: "zai",
authType: "apikey",
name: "z.ai",
apiKey: "sk-test-zai",
defaultModel: "glm-5.2",
});
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/zai", "zai");
assert.equal(combo.id, "auto/zai");
const providerIds = combo.models.map((m) => m.providerId);
assert.deepEqual(providerIds, ["zai"]);
});
it("degrades gracefully to the family subset, excluding connected-but-unrelated providers", async () => {
// Free/noAuth backends may still expose a family model (e.g. opencode serves
// minimax under its own catalog) — the family combo is expected to include
// those, but MUST exclude a connected provider whose model is a different
// family entirely. This is the "subset available" degrade path (#6453).
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "OpenAI",
apiKey: "sk-test-openai",
defaultModel: "gpt-4o-mini",
});
const combo = await builtinCatalog.createBuiltinAutoCombo("auto/minimax", "minimax");
assert.equal(combo.id, "auto/minimax");
assert.ok(
combo.models.every((m) => m.providerId !== "openai"),
"auto/minimax must not include the connected openai/gpt-4o-mini candidate"
);
assert.ok(
combo.models.every((m) => detectModelFamily(m.model) === "minimax"),
"every candidate in auto/minimax must actually be a minimax model"
);
});
it("rejects auto/<unknownfamily> with the same clean error as any unknown combo", async () => {
await assert.rejects(
() => builtinCatalog.createBuiltinAutoCombo("auto/unknownfam", "unknownfam"),
/Unknown built-in auto combo/
);
});
it("isRecognizedBuiltinAuto recognizes every auto/<family> id", () => {
for (const family of ["glm", "minimax", "mimo", "zai", "gemma", "llama", "gemini"]) {
assert.equal(builtinCatalog.isRecognizedBuiltinAuto(`auto/${family}`, family), true);
}
assert.equal(builtinCatalog.isRecognizedBuiltinAuto("auto/unknownfam", "unknownfam"), false);
});
});
@@ -0,0 +1,98 @@
/**
* Unit tests for the `auto/<category>:<tier>` suffix composition filter.
*
* See: `open-sse/services/autoCombo/suffixComposition.ts`
*
* Focus: the `:free` tier filter. Regression test for the bug where
* opencode (noAuth, free) and mimocode (noAuth, free) were NOT being
* included in the free pool because the legacy `freeProviders` list
* only contained paid-API-key providers with free tiers (kiro, qoder, ...).
*
* #4517.
*/
// NOTE: tests/unit/autoCombo/ is a vitest-only scope (the node:test runner in
// test:unit:ci does not walk this dir), so this suite uses the vitest API even
// though it asserts via node:assert. (#4753 originally landed it with node:test
// imports → vitest reported "No test suite found" and the test ran nowhere.)
import { describe, it, beforeAll, afterAll, beforeEach } from "vitest";
import assert from "node:assert/strict";
import {
buildAutoCandidateFilter,
parseAutoSuffix,
} from "../../../open-sse/services/autoCombo/suffixComposition";
describe("suffixComposition :free tier (#4517)", () => {
const ORIGINAL_ENV = { ...process.env };
beforeAll(() => {
// Snapshot env so we can restore it after each test.
});
beforeEach(() => {
// Reset env to a known state so OMNIROUTE_AUTO_FREE_FALLBACK_TO_FULL_POOL
// doesn't leak between cases.
process.env = { ...ORIGINAL_ENV };
});
afterAll(() => {
process.env = ORIGINAL_ENV;
});
it("parseAutoSuffix recognizes coding:free", () => {
assert.deepEqual(parseAutoSuffix("coding:free"), {
valid: true,
category: "coding",
tier: "free",
});
});
it("buildAutoCandidateFilter keeps noAuth free providers", () => {
// Regression: opencode and mimocode are noAuth and free, but the
// pre-fix `freeProviders` list omitted them, so the filter rejected
// their candidates even though they ARE free upstream.
const filter = buildAutoCandidateFilter("coding", "free");
assert.notEqual(filter, null);
assert.equal(filter!({ provider: "opencode", model: "big-pickle" }), true);
assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true);
assert.equal(filter!({ provider: "mimocode", model: "mimo-auto" }), true);
assert.equal(filter!({ provider: "duckduckgo-web", model: "gpt-4o-mini" }), true);
});
it("buildAutoCandidateFilter keeps legacy free providers", () => {
// Don't break the existing list — kiro, qoder, groq, etc. must still pass.
const filter = buildAutoCandidateFilter("coding", "free");
assert.equal(filter!({ provider: "kiro", model: "claude-sonnet-4-5" }), true);
assert.equal(filter!({ provider: "groq", model: "llama-3.3-70b" }), true);
assert.equal(filter!({ provider: "qoder", model: "qwen3-coder-plus" }), true);
});
it("buildAutoCandidateFilter rejects paid models under :free", () => {
// The bug: opencode-go/glm-5.1 was being picked because the filter
// fell back to the full pool when no free candidate was found.
// After the fix, glm-5.1 must be rejected by the :free filter.
const filter = buildAutoCandidateFilter("coding", "free");
assert.equal(filter!({ provider: "opencode-go", model: "glm-5.1" }), false);
assert.equal(filter!({ provider: "openai", model: "gpt-4o" }), false);
assert.equal(filter!({ provider: "anthropic", model: "claude-sonnet-4-6" }), false);
assert.equal(filter!({ provider: "deepseek", model: "deepseek-chat" }), false);
});
it("buildAutoCandidateFilter returns null for category-only (no tier)", () => {
// "coding" with no tier must NOT filter by tier — pass-through.
const filter = buildAutoCandidateFilter("coding", undefined);
assert.equal(filter, null);
});
it("buildAutoCandidateFilter keeps free candidates alongside capability checks", () => {
// The category and tier checks are AND-combined. For "coding:free" the
// category check is a pass-through (no vision/reasoning filter), so
// any free model should be kept.
const filter = buildAutoCandidateFilter("coding", "free");
assert.equal(filter!({ provider: "opencode", model: "minimax-m3-free" }), true);
// The "reasoning" category also pairs with ":free" and keeps free models.
const reasoningFilter = buildAutoCandidateFilter("reasoning", "free");
// big-pickle (model_capabilities: reasoning=1) should pass the reasoning check.
assert.equal(reasoningFilter!({ provider: "opencode", model: "big-pickle" }), true);
});
});
+239
View File
@@ -0,0 +1,239 @@
/**
* Tests for ScoreTierRotator and connectionDensity factor.
* Verifies that multi-connection providers surface in ranked candidates
* and that tiered rotation distributes traffic fairly.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { selectProvider, type AutoComboConfig } from "../../../open-sse/services/autoCombo/engine";
import {
calculateFactors,
calculateScore,
DEFAULT_WEIGHTS,
scorePool,
type ProviderCandidate,
type ScoredProvider,
} from "../../../open-sse/services/autoCombo/scoring";
import { getTaskFitness } from "../../../open-sse/services/autoCombo/taskFitness";
import { resetDiversity } from "../../../open-sse/services/autoCombo/providerDiversity";
// Stub DB calls introduced by PR #3660 (Arena ELO / models.dev intelligence scoring).
// This file tests rotation logic, not intelligence scoring — the DB shouldn't be
// initialized during these unit tests, and the stub makes them fast and isolated.
vi.mock("../../../src/lib/db/modelIntelligence.ts", () => ({
getModelIntelligenceBySource: vi.fn(() => null),
setUserFitnessOverrideEntry: vi.fn(),
deleteUserFitnessOverrideEntry: vi.fn(),
}));
function makeCandidate(overrides: Partial<ProviderCandidate>): ProviderCandidate {
return {
provider: "unknown",
model: "unknown-model",
quotaRemaining: 100,
quotaTotal: 100,
circuitBreakerState: "CLOSED",
costPer1MTokens: 1,
p95LatencyMs: 1000,
latencyStdDev: 100,
errorRate: 0.01,
...overrides,
};
}
function makeConfig(name: string): AutoComboConfig {
return {
id: `test-${name}`,
name,
type: "auto",
candidatePool: [],
weights: { ...DEFAULT_WEIGHTS },
explorationRate: 0,
routerStrategy: "rules",
};
}
describe("Connection Density Factor", () => {
const baseCandidate = makeCandidate({ provider: "cerebras", model: "llama-70b" });
it("multi-connection provider scores higher than single-connection at same quality", () => {
const multiConn = makeCandidate({ provider: "cerebras", model: "llama-70b", connectionPoolSize: 43 });
const singleConn = makeCandidate({ provider: "anthropic", model: "claude-sonnet", connectionPoolSize: 1 });
const pool = [multiConn, singleConn];
const multiFactors = calculateFactors(multiConn, pool, "coding", getTaskFitness);
const singleFactors = calculateFactors(singleConn, pool, "coding", getTaskFitness);
const multiScore = calculateScore(multiFactors, DEFAULT_WEIGHTS);
const singleScore = calculateScore(singleFactors, DEFAULT_WEIGHTS);
expect(multiFactors.connectionDensity).toBe(1.0);
expect(singleFactors.connectionDensity).toBe(0.0);
expect(multiScore).toBeGreaterThan(singleScore);
});
it("density scales linearly from 0 to 10 connections, caps at 10+", () => {
const make = (size: number) => makeCandidate({ connectionPoolSize: size });
const sizes = [1, 2, 5, 10, 20, 43];
const densities = sizes.map((s) => {
const c = make(s);
const pool = [c];
return calculateFactors(c, pool, "coding", getTaskFitness).connectionDensity;
});
expect(densities[0]).toBeCloseTo(0.0, 5);
expect(densities[1]).toBeCloseTo(0.1, 5);
expect(densities[2]).toBeCloseTo(0.4, 5);
expect(densities[3]).toBeCloseTo(0.9, 5);
expect(densities[4]).toBe(1.0);
expect(densities[5]).toBe(1.0);
});
it("missing connectionPoolSize defaults to 1 (backward compat)", () => {
const candidate = makeCandidate({ provider: "x" });
const pool = [candidate];
const factors = calculateFactors(candidate, pool, "coding", getTaskFitness);
expect(factors.connectionDensity).toBe(0.0);
});
it("DEFAULT_WEIGHTS still sum to 1.0 after adding density", () => {
const sum = Object.values(DEFAULT_WEIGHTS).reduce((a, b) => a + b, 0);
expect(Math.abs(sum - 1.0)).toBeLessThan(0.01);
});
});
describe("Tiered Rotation in selectProvider", () => {
beforeEach(() => {
resetDiversity();
});
it("smart combo rotates within top tier across many requests", () => {
const topA = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 95 });
const topB = makeCandidate({ provider: "anthropic", model: "claude-opus", quotaRemaining: 90 });
const topC = makeCandidate({ provider: "google", model: "gemini-ultra", quotaRemaining: 88 });
const mid = makeCandidate({ provider: "mistral", model: "mistral-large", quotaRemaining: 70 });
const pool = [topA, topB, topC, mid];
const config = makeConfig("smart");
const seen = new Set<string>();
for (let i = 0; i < 50; i++) {
const result = selectProvider(config, pool, "coding");
seen.add(`${result.provider}/${result.model}`);
}
expect(seen.size).toBeGreaterThanOrEqual(2);
expect(seen.has("openai/gpt-4o") || seen.has("anthropic/claude-opus")).toBe(true);
});
it("cheap combo pulls from rest tier (lower scores) more often than smart", () => {
const top = makeCandidate({ provider: "openai", model: "gpt-4o", quotaRemaining: 100 });
const rest = makeCandidate({
provider: "cheap-provider",
model: "cheap-model",
quotaRemaining: 100,
costPer1MTokens: 0,
p95LatencyMs: 5000,
});
const pool = [top, rest];
const config = makeConfig("cheap");
const counts: Record<string, number> = {};
for (let i = 0; i < 200; i++) {
const result = selectProvider(config, pool, "coding");
counts[result.provider] = (counts[result.provider] ?? 0) + 1;
}
expect(counts["cheap-provider"]).toBeGreaterThan(0);
});
it("single-candidate pool always returns the same candidate", () => {
const only = makeCandidate({ provider: "only", model: "only-model" });
const config = makeConfig("smart");
for (let i = 0; i < 10; i++) {
const result = selectProvider(config, [only], "coding");
expect(result.provider).toBe("only");
expect(result.model).toBe("only-model");
}
});
});
describe("scorePool with connectionDensity", () => {
it("Cerebras with 43 keys ranks above single-connection providers of similar quality", () => {
const cerebras = makeCandidate({
provider: "cerebras",
model: "llama-3.1-70b",
connectionPoolSize: 43,
quotaRemaining: 100,
});
const anthropic = makeCandidate({
provider: "anthropic",
model: "claude-sonnet",
connectionPoolSize: 1,
quotaRemaining: 100,
});
const pool = [cerebras, anthropic];
const scored = scorePool(pool, "coding", DEFAULT_WEIGHTS, getTaskFitness);
expect(scored[0].provider).toBe("cerebras");
});
});
describe("Per-Connection Rotation", () => {
it("rotates across all 43 Cerebras connection IDs, not just one", () => {
const cerebrasCandidates: ProviderCandidate[] = Array.from({ length: 43 }, (_, i) =>
makeCandidate({
provider: "cerebras",
model: "llama-3.1-70b",
connectionId: `cerebras-conn-${i + 1}`,
})
);
const config = makeConfig("smart");
const seenConnections = new Set<string>();
for (let i = 0; i < 200; i++) {
const result = selectProvider(config, cerebrasCandidates, "coding");
if (result.connectionId) seenConnections.add(result.connectionId);
}
expect(seenConnections.size).toBeGreaterThanOrEqual(10);
});
it("different combos maintain independent round-robin state", () => {
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
makeCandidate({ provider: "p", model: "m", connectionId: `c-${i}` })
);
const smartConfig = makeConfig("smart-A");
const fastConfig = makeConfig("fast-B");
for (let i = 0; i < 5; i++) {
selectProvider(smartConfig, candidates, "coding");
}
const smartResults: string[] = [];
for (let i = 0; i < 5; i++) {
const r = selectProvider(smartConfig, candidates, "coding");
if (r.connectionId) smartResults.push(r.connectionId);
}
for (let i = 0; i < 5; i++) {
selectProvider(fastConfig, candidates, "coding");
}
const fastResults: string[] = [];
for (let i = 0; i < 5; i++) {
const r = selectProvider(fastConfig, candidates, "coding");
if (r.connectionId) fastResults.push(r.connectionId);
}
expect(smartResults.length).toBe(5);
expect(fastResults.length).toBe(5);
expect(new Set(smartResults).size).toBeGreaterThan(1);
expect(new Set(fastResults).size).toBeGreaterThan(1);
});
it("tied-score candidates from same provider+model are all reachable", () => {
const candidates: ProviderCandidate[] = Array.from({ length: 5 }, (_, i) =>
makeCandidate({ provider: "free", model: "free-model", connectionId: `key-${i}` })
);
const config = makeConfig("smart");
const visited = new Set<string>();
for (let i = 0; i < 20; i++) {
const result = selectProvider(config, candidates, "coding");
if (result.connectionId) visited.add(result.connectionId);
}
expect(visited.size).toBeGreaterThan(1);
});
});