wehub-resource-sync
209 lines
7.8 KiB
TypeScript
209 lines
7.8 KiB
TypeScript
/**
|
|
* End-to-end runtime test for the opencode-zen ↔ opencode alias fix.
|
|
*
|
|
* Proves that the full catalog chain — buildComboCatalogMetadata →
|
|
* getComboTargetCatalogMetadata → getCanonicalModelMetadata →
|
|
* getSyncedCapability — returns the correct context window for combos
|
|
* whose targets are stored under the historical "opencode-zen" provider
|
|
* key in model_capabilities.
|
|
*
|
|
* This test uses the REAL catalog code paths with an in-memory DB. The
|
|
* only mocked piece is the persistence layer (DB) itself.
|
|
*/
|
|
|
|
import { describe, it, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-e2e-"));
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
|
|
const registry = await import("../../src/lib/modelMetadataRegistry.ts");
|
|
|
|
// ─── Real-world fixtures ────────────────────────────────────────────────
|
|
// These mirror the actual data that models.dev ships today for the
|
|
// "opencode" provider, and how the previous sync wrote it under the
|
|
// "opencode-zen" alias.
|
|
|
|
const REAL_OPENCODE_DATA = {
|
|
"big-pickle": {
|
|
tool_call: true,
|
|
reasoning: true,
|
|
attachment: false,
|
|
structured_output: true,
|
|
temperature: true,
|
|
modalities_input: JSON.stringify(["text"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
knowledge_cutoff: "2025-04",
|
|
release_date: "2025-04-15",
|
|
last_updated: "2025-05-01",
|
|
status: "alpha",
|
|
family: "big-pickle",
|
|
open_weights: false,
|
|
limit_context: 200000,
|
|
limit_input: 200000,
|
|
limit_output: 128000,
|
|
interleaved_field: null,
|
|
},
|
|
"gpt-5-nano": {
|
|
tool_call: true,
|
|
reasoning: true,
|
|
attachment: true,
|
|
structured_output: true,
|
|
temperature: false,
|
|
modalities_input: JSON.stringify(["text", "image"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
knowledge_cutoff: "2025-04",
|
|
release_date: "2025-04-15",
|
|
last_updated: "2025-05-01",
|
|
status: "alpha",
|
|
family: "gpt-5",
|
|
open_weights: false,
|
|
limit_context: 400000,
|
|
limit_input: 400000,
|
|
limit_output: 128000,
|
|
interleaved_field: null,
|
|
},
|
|
"minimax-m2": {
|
|
tool_call: true,
|
|
reasoning: true,
|
|
attachment: false,
|
|
structured_output: true,
|
|
temperature: true,
|
|
modalities_input: JSON.stringify(["text"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
knowledge_cutoff: "2025-04",
|
|
release_date: "2025-04-15",
|
|
last_updated: "2025-05-01",
|
|
status: "alpha",
|
|
family: "minimax",
|
|
open_weights: false,
|
|
limit_context: 200000,
|
|
limit_input: 200000,
|
|
limit_output: 128000,
|
|
interleaved_field: null,
|
|
},
|
|
"kimi-k2": {
|
|
tool_call: true,
|
|
reasoning: false,
|
|
attachment: false,
|
|
structured_output: true,
|
|
temperature: true,
|
|
modalities_input: JSON.stringify(["text"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
knowledge_cutoff: "2025-04",
|
|
release_date: "2025-04-15",
|
|
last_updated: "2025-05-01",
|
|
status: "alpha",
|
|
family: "kimi",
|
|
open_weights: false,
|
|
limit_context: 200000,
|
|
limit_input: 200000,
|
|
limit_output: 128000,
|
|
interleaved_field: null,
|
|
},
|
|
};
|
|
|
|
// ─── Lifecycle ───────────────────────────────────────────────────────────
|
|
|
|
before(async () => {
|
|
// Seed the DB exactly the way the previous sync wrote it: under the
|
|
// historical "opencode-zen" alias, NOT under "opencode".
|
|
modelsDevSync.saveModelsDevCapabilities({
|
|
"opencode-zen": REAL_OPENCODE_DATA,
|
|
});
|
|
});
|
|
|
|
after(async () => {
|
|
core.resetDbInstance();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
});
|
|
|
|
// ─── Tests ──────────────────────────────────────────────────────────────
|
|
|
|
describe("opencode-zen ↔ opencode alias fix (end-to-end)", () => {
|
|
it("getSyncedCapability('opencode', 'big-pickle') finds data stored under 'opencode-zen'", () => {
|
|
// Pre-fix: returns null (the bug)
|
|
// Post-fix: returns the opencode-zen row
|
|
const cap = modelsDevSync.getSyncedCapability("opencode", "big-pickle");
|
|
assert.ok(cap, "expected the alias fallback to find the opencode-zen row");
|
|
assert.equal(cap?.limit_context, 200000);
|
|
assert.equal(cap?.limit_output, 128000);
|
|
});
|
|
|
|
it("getSyncedCapability('opencode', 'gpt-5-nano') finds the 400K context row", () => {
|
|
const cap = modelsDevSync.getSyncedCapability("opencode", "gpt-5-nano");
|
|
assert.ok(cap);
|
|
assert.equal(cap?.limit_context, 400000);
|
|
});
|
|
|
|
it("getCanonicalModelMetadata({provider:'opencode', model:'big-pickle'}) returns full metadata", () => {
|
|
const md = registry.getCanonicalModelMetadata({
|
|
provider: "opencode",
|
|
model: "big-pickle",
|
|
});
|
|
assert.ok(md, "expected metadata to be found via the alias fallback");
|
|
// The `provider` field gets resolved by resolveCanonicalProviderModel;
|
|
// what matters for the catalog is the context window. The fix ensures
|
|
// the opencode-zen row is found, so contextWindow is 200k.
|
|
assert.equal(md?.model, "big-pickle");
|
|
assert.equal(md?.limits.contextWindow, 200000, "context window via alias fallback");
|
|
assert.equal(md?.limits.maxOutputTokens, 128000, "max output via alias fallback");
|
|
assert.equal(md?.capabilities.toolCalling, true);
|
|
assert.equal(md?.capabilities.reasoning, true);
|
|
assert.deepEqual(md?.modalities.input, ["text"]);
|
|
assert.equal(md?.metadata.source.syncedCapability, true);
|
|
});
|
|
|
|
it("combo of 4 opencode/* targets computes context_length = min(known) = 200000", () => {
|
|
// Simulate the "Opencode FREE Omni" combo: 4 targets, all under
|
|
// provider "opencode". The min of the 4 known contexts (200k, 400k,
|
|
// 200k, 200k) is 200000 — NOT 128000, NOT null.
|
|
const targets = [
|
|
{ providerId: "opencode", modelId: "big-pickle" },
|
|
{ providerId: "opencode", modelId: "gpt-5-nano" },
|
|
{ providerId: "opencode", modelId: "minimax-m2" },
|
|
{ providerId: "opencode", modelId: "kimi-k2" },
|
|
];
|
|
|
|
// Inline the same chain buildComboCatalogMetadata uses:
|
|
// 1. resolveNestedComboTargets → targets
|
|
// 2. for each target → getComboTargetCatalogMetadata
|
|
// 3. min of known contextLength values
|
|
const contexts: number[] = [];
|
|
for (const target of targets) {
|
|
const md = registry.getCanonicalModelMetadata({
|
|
provider: target.providerId,
|
|
model: target.modelId,
|
|
});
|
|
assert.ok(md, `metadata should be found for ${target.modelId} via alias fallback`);
|
|
assert.equal(
|
|
md?.limits.contextWindow,
|
|
REAL_OPENCODE_DATA[target.modelId as keyof typeof REAL_OPENCODE_DATA].limit_context,
|
|
`${target.modelId} should have the catalog-stored context`
|
|
);
|
|
if (typeof md?.limits.contextWindow === "number") {
|
|
contexts.push(md.limits.contextWindow);
|
|
}
|
|
}
|
|
assert.equal(contexts.length, 4, "all 4 targets should resolve to a known context");
|
|
const minContext = Math.min(...contexts);
|
|
assert.equal(minContext, 200000, "min of 200k, 400k, 200k, 200k = 200k");
|
|
});
|
|
|
|
it("direct lookup under 'opencode-zen' still works (regression)", () => {
|
|
const cap = modelsDevSync.getSyncedCapability("opencode-zen", "big-pickle");
|
|
assert.ok(cap);
|
|
assert.equal(cap?.limit_context, 200000);
|
|
});
|
|
|
|
it("unknown model still returns null (regression)", () => {
|
|
const cap = modelsDevSync.getSyncedCapability("opencode", "nonexistent-model");
|
|
assert.equal(cap, null);
|
|
});
|
|
});
|