/** * End-to-end runtime test for the opencode-zen ↔ opencode alias fix. * * Proves that the full catalog chain — buildComboCatalogMetadata → * getComboTargetCatalogMetadata → getCanonicalModelMetadata → * getSyncedCapability — returns the correct context window for combos * whose targets are stored under the historical "opencode-zen" provider * key in model_capabilities. * * This test uses the REAL catalog code paths with an in-memory DB. The * only mocked piece is the persistence layer (DB) itself. */ import { describe, it, before, after } from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-e2e-")); process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const modelsDevSync = await import("../../src/lib/modelsDevSync.ts"); const registry = await import("../../src/lib/modelMetadataRegistry.ts"); // ─── Real-world fixtures ──────────────────────────────────────────────── // These mirror the actual data that models.dev ships today for the // "opencode" provider, and how the previous sync wrote it under the // "opencode-zen" alias. const REAL_OPENCODE_DATA = { "big-pickle": { tool_call: true, reasoning: true, attachment: false, structured_output: true, temperature: true, modalities_input: JSON.stringify(["text"]), modalities_output: JSON.stringify(["text"]), knowledge_cutoff: "2025-04", release_date: "2025-04-15", last_updated: "2025-05-01", status: "alpha", family: "big-pickle", open_weights: false, limit_context: 200000, limit_input: 200000, limit_output: 128000, interleaved_field: null, }, "gpt-5-nano": { tool_call: true, reasoning: true, attachment: true, structured_output: true, temperature: false, modalities_input: JSON.stringify(["text", "image"]), modalities_output: JSON.stringify(["text"]), knowledge_cutoff: "2025-04", release_date: "2025-04-15", last_updated: "2025-05-01", status: "alpha", family: "gpt-5", open_weights: false, limit_context: 400000, limit_input: 400000, limit_output: 128000, interleaved_field: null, }, "minimax-m2": { tool_call: true, reasoning: true, attachment: false, structured_output: true, temperature: true, modalities_input: JSON.stringify(["text"]), modalities_output: JSON.stringify(["text"]), knowledge_cutoff: "2025-04", release_date: "2025-04-15", last_updated: "2025-05-01", status: "alpha", family: "minimax", open_weights: false, limit_context: 200000, limit_input: 200000, limit_output: 128000, interleaved_field: null, }, "kimi-k2": { tool_call: true, reasoning: false, attachment: false, structured_output: true, temperature: true, modalities_input: JSON.stringify(["text"]), modalities_output: JSON.stringify(["text"]), knowledge_cutoff: "2025-04", release_date: "2025-04-15", last_updated: "2025-05-01", status: "alpha", family: "kimi", open_weights: false, limit_context: 200000, limit_input: 200000, limit_output: 128000, interleaved_field: null, }, }; // ─── Lifecycle ─────────────────────────────────────────────────────────── before(async () => { // Seed the DB exactly the way the previous sync wrote it: under the // historical "opencode-zen" alias, NOT under "opencode". modelsDevSync.saveModelsDevCapabilities({ "opencode-zen": REAL_OPENCODE_DATA, }); }); after(async () => { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); // ─── Tests ────────────────────────────────────────────────────────────── describe("opencode-zen ↔ opencode alias fix (end-to-end)", () => { it("getSyncedCapability('opencode', 'big-pickle') finds data stored under 'opencode-zen'", () => { // Pre-fix: returns null (the bug) // Post-fix: returns the opencode-zen row const cap = modelsDevSync.getSyncedCapability("opencode", "big-pickle"); assert.ok(cap, "expected the alias fallback to find the opencode-zen row"); assert.equal(cap?.limit_context, 200000); assert.equal(cap?.limit_output, 128000); }); it("getSyncedCapability('opencode', 'gpt-5-nano') finds the 400K context row", () => { const cap = modelsDevSync.getSyncedCapability("opencode", "gpt-5-nano"); assert.ok(cap); assert.equal(cap?.limit_context, 400000); }); it("getCanonicalModelMetadata({provider:'opencode', model:'big-pickle'}) returns full metadata", () => { const md = registry.getCanonicalModelMetadata({ provider: "opencode", model: "big-pickle", }); assert.ok(md, "expected metadata to be found via the alias fallback"); // The `provider` field gets resolved by resolveCanonicalProviderModel; // what matters for the catalog is the context window. The fix ensures // the opencode-zen row is found, so contextWindow is 200k. assert.equal(md?.model, "big-pickle"); assert.equal(md?.limits.contextWindow, 200000, "context window via alias fallback"); assert.equal(md?.limits.maxOutputTokens, 128000, "max output via alias fallback"); assert.equal(md?.capabilities.toolCalling, true); assert.equal(md?.capabilities.reasoning, true); assert.deepEqual(md?.modalities.input, ["text"]); assert.equal(md?.metadata.source.syncedCapability, true); }); it("combo of 4 opencode/* targets computes context_length = min(known) = 200000", () => { // Simulate the "Opencode FREE Omni" combo: 4 targets, all under // provider "opencode". The min of the 4 known contexts (200k, 400k, // 200k, 200k) is 200000 — NOT 128000, NOT null. const targets = [ { providerId: "opencode", modelId: "big-pickle" }, { providerId: "opencode", modelId: "gpt-5-nano" }, { providerId: "opencode", modelId: "minimax-m2" }, { providerId: "opencode", modelId: "kimi-k2" }, ]; // Inline the same chain buildComboCatalogMetadata uses: // 1. resolveNestedComboTargets → targets // 2. for each target → getComboTargetCatalogMetadata // 3. min of known contextLength values const contexts: number[] = []; for (const target of targets) { const md = registry.getCanonicalModelMetadata({ provider: target.providerId, model: target.modelId, }); assert.ok(md, `metadata should be found for ${target.modelId} via alias fallback`); assert.equal( md?.limits.contextWindow, REAL_OPENCODE_DATA[target.modelId as keyof typeof REAL_OPENCODE_DATA].limit_context, `${target.modelId} should have the catalog-stored context` ); if (typeof md?.limits.contextWindow === "number") { contexts.push(md.limits.contextWindow); } } assert.equal(contexts.length, 4, "all 4 targets should resolve to a known context"); const minContext = Math.min(...contexts); assert.equal(minContext, 200000, "min of 200k, 400k, 200k, 200k = 200k"); }); it("direct lookup under 'opencode-zen' still works (regression)", () => { const cap = modelsDevSync.getSyncedCapability("opencode-zen", "big-pickle"); assert.ok(cap); assert.equal(cap?.limit_context, 200000); }); it("unknown model still returns null (regression)", () => { const cap = modelsDevSync.getSyncedCapability("opencode", "nonexistent-model"); assert.equal(cap, null); }); });