wehub-resource-sync
261 lines
9.6 KiB
TypeScript
261 lines
9.6 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-capabilities-"));
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
|
|
const modelCapabilities = await import("../../src/lib/modelCapabilities.ts");
|
|
|
|
function buildCapability(overrides = {}) {
|
|
return {
|
|
tool_call: null,
|
|
reasoning: null,
|
|
attachment: null,
|
|
structured_output: null,
|
|
temperature: null,
|
|
modalities_input: "[]",
|
|
modalities_output: "[]",
|
|
knowledge_cutoff: null,
|
|
release_date: null,
|
|
last_updated: null,
|
|
status: null,
|
|
family: null,
|
|
open_weights: null,
|
|
limit_context: null,
|
|
limit_input: null,
|
|
limit_output: null,
|
|
interleaved_field: null,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function resetStorage() {
|
|
core.resetDbInstance();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
|
}
|
|
|
|
test.beforeEach(() => {
|
|
resetStorage();
|
|
});
|
|
|
|
test.after(() => {
|
|
core.resetDbInstance();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
});
|
|
|
|
test("canonical model capability resolver lets exact synced metadata override global specs", () => {
|
|
modelsDevSync.saveModelsDevCapabilities({
|
|
openai: {
|
|
"gpt-4o-2024-11-20": buildCapability({
|
|
tool_call: false,
|
|
reasoning: false,
|
|
attachment: true,
|
|
structured_output: true,
|
|
temperature: true,
|
|
modalities_input: JSON.stringify(["text", "image"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
family: "gpt-4",
|
|
status: "stable",
|
|
limit_context: 256000,
|
|
limit_input: 256000,
|
|
limit_output: 12345,
|
|
}),
|
|
},
|
|
antigravity: {
|
|
// Since #3229, ANTIGRAVITY_MODEL_ALIASES maps both "gemini-3.1-pro-high" and
|
|
// "gemini-3.1-pro-low" → "gemini-3.1-pro", so the capability resolver looks
|
|
// synced metadata up under the canonical "gemini-3.1-pro" key. Save it there.
|
|
"gemini-3.1-pro": buildCapability({
|
|
tool_call: false,
|
|
reasoning: false,
|
|
modalities_input: JSON.stringify(["text"]),
|
|
modalities_output: JSON.stringify(["text"]),
|
|
limit_context: 1024,
|
|
limit_output: 9999,
|
|
}),
|
|
},
|
|
});
|
|
|
|
const gpt4o = modelCapabilities.getResolvedModelCapabilities("openai/gpt-4o-2024-11-20");
|
|
assert.equal(gpt4o.toolCalling, false);
|
|
assert.equal(gpt4o.reasoning, false);
|
|
assert.equal(gpt4o.supportsVision, true);
|
|
assert.equal(gpt4o.contextWindow, 256000);
|
|
assert.equal(gpt4o.maxInputTokens, 256000);
|
|
assert.equal(gpt4o.maxOutputTokens, 12345);
|
|
assert.equal(modelCapabilities.getModelContextLimit("openai", "gpt-4o-2024-11-20"), 256000);
|
|
assert.equal(modelCapabilities.capMaxOutputTokens("openai/gpt-4o-2024-11-20", 999999), 12345);
|
|
|
|
const geminiHigh = modelCapabilities.getResolvedModelCapabilities(
|
|
"antigravity/gemini-3.1-pro-high"
|
|
);
|
|
assert.equal(geminiHigh.toolCalling, false);
|
|
assert.equal(geminiHigh.reasoning, false);
|
|
assert.equal(geminiHigh.supportsThinking, false);
|
|
assert.equal(geminiHigh.contextWindow, 1024);
|
|
assert.equal(geminiHigh.maxOutputTokens, 9999);
|
|
assert.equal(geminiHigh.defaultThinkingBudget, 24576);
|
|
assert.equal(
|
|
modelCapabilities.capThinkingBudget("antigravity/gemini-3.1-pro-high", 40000),
|
|
32768
|
|
);
|
|
|
|
const codexGpt55 = modelCapabilities.getResolvedModelCapabilities("codex/gpt-5.5");
|
|
assert.equal(codexGpt55.contextWindow, 400000);
|
|
// #6191: max_input_tokens is a distinct, smaller cap than the context window.
|
|
assert.equal(codexGpt55.maxInputTokens, 272000);
|
|
assert.equal(codexGpt55.maxOutputTokens, 128000);
|
|
assert.equal(codexGpt55.supportsThinking, true);
|
|
assert.equal(codexGpt55.supportsVision, true);
|
|
|
|
const bedrockSonnet46 = modelCapabilities.getResolvedModelCapabilities(
|
|
"bedrock/eu.anthropic.claude-sonnet-4-6"
|
|
);
|
|
assert.equal(bedrockSonnet46.contextWindow, 1000000);
|
|
assert.equal(bedrockSonnet46.maxInputTokens, 1000000);
|
|
assert.equal(bedrockSonnet46.maxOutputTokens, 64000);
|
|
assert.equal(bedrockSonnet46.supportsVision, true);
|
|
|
|
const bedrockSonnet45 = modelCapabilities.getResolvedModelCapabilities(
|
|
"bedrock/anthropic.claude-sonnet-4-5"
|
|
);
|
|
assert.equal(bedrockSonnet45.contextWindow, 200000);
|
|
assert.equal(bedrockSonnet45.maxOutputTokens, 64000);
|
|
|
|
const bedrockOpus46 = modelCapabilities.getResolvedModelCapabilities(
|
|
"bedrock/anthropic.claude-opus-4-6"
|
|
);
|
|
assert.equal(bedrockOpus46.contextWindow, 1000000);
|
|
assert.equal(bedrockOpus46.maxOutputTokens, 128000);
|
|
|
|
const bareGpt55 = modelCapabilities.getResolvedModelCapabilities("gpt-5.5");
|
|
assert.equal(bareGpt55.contextWindow, 1050000);
|
|
});
|
|
|
|
test("unknown models keep maxOutputTokens null instead of using a generic default", () => {
|
|
const unknown = modelCapabilities.getResolvedModelCapabilities(
|
|
"openai-compatible-local/custom-large-output-model"
|
|
);
|
|
|
|
assert.equal(unknown.contextWindow, null);
|
|
assert.equal(unknown.maxInputTokens, null);
|
|
assert.equal(unknown.maxOutputTokens, null);
|
|
assert.equal(
|
|
modelCapabilities.capMaxOutputTokens(
|
|
"openai-compatible-local/custom-large-output-model",
|
|
32000
|
|
),
|
|
32000
|
|
);
|
|
assert.equal(
|
|
modelCapabilities.capMaxOutputTokens("openai-compatible-local/custom-large-output-model"),
|
|
null
|
|
);
|
|
});
|
|
|
|
test("GPT OSS and DeepSeek Reasoner models support tool calling", () => {
|
|
// GPT OSS models should not be blocked by the heuristic
|
|
assert.equal(modelCapabilities.supportsToolCalling("fake-provider/gpt-oss-120b"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("gpt-oss-120b"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("nvidia/openai/gpt-oss-20b"), false); // in registry
|
|
|
|
// DeepSeek Reasoner supports tool calling
|
|
assert.equal(modelCapabilities.supportsToolCalling("deepseek-reasoner"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("deepseek/deepseek-r1"), true);
|
|
|
|
// Full capability resolution
|
|
const gptOss = modelCapabilities.getResolvedModelCapabilities("fake-provider/gpt-oss-120b");
|
|
assert.equal(gptOss.toolCalling, true);
|
|
const deepseek = modelCapabilities.getResolvedModelCapabilities("deepseek/deepseek-reasoner");
|
|
assert.equal(deepseek.toolCalling, true);
|
|
});
|
|
|
|
test("Kimi K2.6 supports vision capability", () => {
|
|
const kimi = modelCapabilities.getResolvedModelCapabilities("kimi-k2.6");
|
|
assert.equal(kimi.supportsVision, true);
|
|
assert.equal(kimi.supportsThinking, true);
|
|
assert.equal(kimi.supportsTools, true);
|
|
assert.equal(kimi.contextWindow, 262144);
|
|
assert.equal(kimi.maxOutputTokens, 262144);
|
|
|
|
// Also test via alias
|
|
const kimiThinking = modelCapabilities.getResolvedModelCapabilities("kimi-k2.6-thinking");
|
|
assert.equal(kimiThinking.supportsVision, true);
|
|
});
|
|
|
|
test("Kimi K2.7 Code resolves full capabilities instead of the degraded import defaults (#3761)", () => {
|
|
// Spec-driven, so it works for any provider serving the model.
|
|
const kimi = modelCapabilities.getResolvedModelCapabilities("kimi-k2.7-code");
|
|
assert.equal(kimi.contextWindow, 262144);
|
|
assert.equal(kimi.maxOutputTokens, 262144);
|
|
assert.equal(kimi.supportsVision, true);
|
|
assert.equal(kimi.supportsThinking, true);
|
|
assert.equal(kimi.supportsTools, true);
|
|
|
|
// The reported case: imported via Ollama Cloud's "import from /models". Before the
|
|
// fix this had no spec/registry entry, so context fell back to the 128000 default
|
|
// and max output to 8192, with vision dropped.
|
|
const ollama = modelCapabilities.getResolvedModelCapabilities("ollama-cloud/kimi-k2.7-code");
|
|
assert.equal(ollama.contextWindow, 262144);
|
|
assert.equal(ollama.maxOutputTokens, 262144);
|
|
assert.equal(ollama.supportsVision, true);
|
|
assert.notEqual(ollama.contextWindow, 128000);
|
|
assert.notEqual(ollama.maxOutputTokens, 8192);
|
|
});
|
|
|
|
test("GLM-5.2 context limits respect provider-hosted caps", () => {
|
|
modelsDevSync.saveModelsDevCapabilities({
|
|
huggingface: {
|
|
"zai-org/GLM-5.2": buildCapability({
|
|
limit_context: 128000,
|
|
limit_input: 128000,
|
|
limit_output: 128000,
|
|
}),
|
|
},
|
|
"cloudflare-ai": {
|
|
"@cf/zai-org/glm-5.2": buildCapability({
|
|
limit_context: 128000,
|
|
limit_input: 128000,
|
|
limit_output: 128000,
|
|
}),
|
|
},
|
|
zenmux: {
|
|
"z-ai/glm-5.2": buildCapability({
|
|
limit_context: 128000,
|
|
limit_input: 128000,
|
|
limit_output: 128000,
|
|
}),
|
|
},
|
|
});
|
|
|
|
for (const modelId of ["glm-5.2", "opencode-go/glm-5.2", "opencode/glm-5.2", "oc/glm-5.2"]) {
|
|
const capabilities = modelCapabilities.getResolvedModelCapabilities(modelId);
|
|
assert.equal(capabilities.contextWindow, 1000000, modelId);
|
|
assert.equal(capabilities.maxInputTokens, 1000000, modelId);
|
|
}
|
|
|
|
for (const modelId of ["zenmux/z-ai/glm-5.2", "zenmux/z-ai/glm-5.2-free"]) {
|
|
const capabilities = modelCapabilities.getResolvedModelCapabilities(modelId);
|
|
assert.equal(capabilities.contextWindow, 1000000, modelId);
|
|
assert.equal(capabilities.maxInputTokens, 1000000, modelId);
|
|
}
|
|
|
|
for (const modelId of ["cloudflare-ai/@cf/zai-org/glm-5.2", "cf/@cf/zai-org/glm-5.2"]) {
|
|
const capabilities = modelCapabilities.getResolvedModelCapabilities(modelId);
|
|
assert.equal(capabilities.contextWindow, 262144, modelId);
|
|
assert.equal(capabilities.maxInputTokens, 262144, modelId);
|
|
}
|
|
|
|
for (const modelId of ["huggingface/zai-org/GLM-5.2", "hf/zai-org/GLM-5.2"]) {
|
|
const capabilities = modelCapabilities.getResolvedModelCapabilities(modelId);
|
|
assert.equal(capabilities.contextWindow, 262144, modelId);
|
|
assert.equal(capabilities.maxInputTokens, 262144, modelId);
|
|
}
|
|
});
|