wehub-resource-sync
455 lines
17 KiB
TypeScript
455 lines
17 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-services-hardening-"));
|
|
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
|
|
if (!process.env.API_KEY_SECRET) {
|
|
process.env.API_KEY_SECRET = "test-services-hardening-secret-" + Date.now();
|
|
}
|
|
|
|
test.after(() => {
|
|
if (ORIGINAL_DATA_DIR === undefined) {
|
|
delete process.env.DATA_DIR;
|
|
} else {
|
|
process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
|
}
|
|
try {
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
} catch {
|
|
// Best effort cleanup
|
|
}
|
|
});
|
|
|
|
const signatureStore = await import("../../open-sse/services/geminiThoughtSignatureStore.ts");
|
|
const modelCapabilities = await import("../../open-sse/services/modelCapabilities.ts");
|
|
const comboAgentMiddleware = await import("../../open-sse/services/comboAgentMiddleware.ts");
|
|
const rateLimitSemaphore = await import("../../open-sse/services/rateLimitSemaphore.ts");
|
|
const comboMetrics = await import("../../open-sse/services/comboMetrics.ts");
|
|
const modelService = await import("../../open-sse/services/model.ts");
|
|
const errorClassifier = await import("../../open-sse/services/errorClassifier.ts");
|
|
const providerModels = await import("../../open-sse/config/providerModels.ts");
|
|
|
|
const originalDateNow = Date.now;
|
|
const originalConsoleWarn = console.warn;
|
|
const originalSyntheticAlias = providerModels.PROVIDER_ID_TO_ALIAS.synthetic;
|
|
const originalSyntheticModels = providerModels.PROVIDER_MODELS.synthetic;
|
|
|
|
function wait(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
test.afterEach(() => {
|
|
Date.now = originalDateNow;
|
|
console.warn = originalConsoleWarn;
|
|
if (originalSyntheticAlias === undefined) {
|
|
delete providerModels.PROVIDER_ID_TO_ALIAS.synthetic;
|
|
} else {
|
|
providerModels.PROVIDER_ID_TO_ALIAS.synthetic = originalSyntheticAlias;
|
|
}
|
|
if (originalSyntheticModels === undefined) {
|
|
delete providerModels.PROVIDER_MODELS.synthetic;
|
|
} else {
|
|
providerModels.PROVIDER_MODELS.synthetic = originalSyntheticModels;
|
|
}
|
|
rateLimitSemaphore.resetAll();
|
|
comboMetrics.resetAllComboMetrics();
|
|
});
|
|
|
|
test("gemini thought signature store handles invalid input, memory TTL and max-size pruning", () => {
|
|
signatureStore.storeGeminiThoughtSignature("", "sig");
|
|
signatureStore.storeGeminiThoughtSignature("call-empty", "");
|
|
assert.equal(signatureStore.getGeminiThoughtSignature(""), null);
|
|
assert.equal(signatureStore.getGeminiThoughtSignature("call-empty"), null);
|
|
|
|
let now = 1_000;
|
|
Date.now = () => now;
|
|
|
|
signatureStore.storeGeminiThoughtSignature("call-live", "sig-live");
|
|
assert.equal(signatureStore.getGeminiThoughtSignature("call-live"), "sig-live");
|
|
|
|
now += 60 * 60 * 1000 + 1;
|
|
assert.equal(signatureStore.getGeminiThoughtSignatureMemorySizeForTests(), 0);
|
|
assert.equal(signatureStore.getGeminiThoughtSignature("call-live"), "sig-live");
|
|
|
|
now = 10_000;
|
|
for (let index = 0; index < 1002; index++) {
|
|
signatureStore.storeGeminiThoughtSignature(`call-${index}`, `sig-${index}`);
|
|
now += 1;
|
|
}
|
|
|
|
assert.equal(signatureStore.getGeminiThoughtSignatureMemorySizeForTests(), 1000);
|
|
assert.equal(signatureStore.getGeminiThoughtSignature("call-0"), "sig-0");
|
|
assert.equal(signatureStore.getGeminiThoughtSignature("call-1001"), "sig-1001");
|
|
});
|
|
|
|
test("model capability helpers cover denylist, empty input and default-safe paths", () => {
|
|
providerModels.PROVIDER_ID_TO_ALIAS.synthetic = "synthetic";
|
|
providerModels.PROVIDER_MODELS.synthetic = [
|
|
{ id: "tool-safe", toolCalling: true, supportsReasoning: true },
|
|
{ id: "tool-blocked", toolCalling: false, supportsReasoning: false },
|
|
{ id: "tool-unknown" },
|
|
];
|
|
|
|
assert.equal(modelCapabilities.supportsToolCalling("synthetic/tool-safe"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("synthetic/tool-blocked"), false);
|
|
assert.equal(modelCapabilities.supportsReasoning("synthetic/tool-safe"), true);
|
|
assert.equal(modelCapabilities.supportsReasoning("synthetic/tool-blocked"), false);
|
|
assert.equal(modelCapabilities.supportsToolCalling("synthetic/tool-unknown"), true);
|
|
assert.equal(modelCapabilities.supportsReasoning("synthetic/tool-unknown"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("missing-provider/tool"), true);
|
|
assert.equal(modelCapabilities.supportsReasoning("missing-provider/tool"), true);
|
|
|
|
assert.equal(modelCapabilities.supportsToolCalling(""), false);
|
|
assert.equal(modelCapabilities.supportsToolCalling("openai/gpt-oss-120b"), true);
|
|
assert.equal(modelCapabilities.supportsToolCalling("deepseek-reasoner"), true);
|
|
assert.equal(
|
|
modelCapabilities.supportsToolCalling("openai/nonexistent-default-safe-model"),
|
|
true
|
|
);
|
|
|
|
assert.equal(modelCapabilities.supportsReasoning(""), true);
|
|
assert.equal(modelCapabilities.supportsReasoning("antigravity/claude-sonnet-4-6"), false);
|
|
assert.equal(modelCapabilities.supportsReasoning("antigravity/claude-sonnet-4"), false);
|
|
assert.equal(modelCapabilities.supportsReasoning("openai/nonexistent-default-safe-model"), true);
|
|
});
|
|
|
|
test("combo agent middleware covers system override, tool filtering, tag stripping and pin propagation", () => {
|
|
const originalTools = [
|
|
{ type: "function", function: { name: "keep_me" } },
|
|
{ name: "drop_me" },
|
|
{ type: "function", function: { name: "keep_tool" } },
|
|
];
|
|
const body = {
|
|
model: "combo/default",
|
|
messages: [
|
|
{ role: "system", content: "old system" },
|
|
{ role: "user", content: "hello" },
|
|
{ role: "assistant", content: "cached\n<omniModel>anthropic/claude-sonnet-4-6</omniModel>" },
|
|
],
|
|
tools: originalTools,
|
|
};
|
|
|
|
const filtered = comboAgentMiddleware.applyToolFilter(originalTools, "^keep");
|
|
assert.deepEqual(filtered, [
|
|
{ type: "function", function: { name: "keep_me" } },
|
|
{ type: "function", function: { name: "keep_tool" } },
|
|
]);
|
|
assert.equal(comboAgentMiddleware.applyToolFilter(undefined, "^keep"), undefined);
|
|
assert.equal(comboAgentMiddleware.applyToolFilter(originalTools, null), originalTools);
|
|
|
|
const warnMessages = [];
|
|
console.warn = (message) => warnMessages.push(String(message));
|
|
assert.equal(comboAgentMiddleware.applyToolFilter(originalTools, "["), originalTools);
|
|
assert.equal(warnMessages.length, 1);
|
|
|
|
const stripped = comboAgentMiddleware.stripModelTags(body.messages);
|
|
assert.deepEqual(stripped, [
|
|
{ role: "system", content: "old system" },
|
|
{ role: "user", content: "hello" },
|
|
{ role: "assistant", content: "cached" },
|
|
]);
|
|
|
|
const overridden = comboAgentMiddleware.applySystemMessageOverride(body.messages, "new system");
|
|
assert.deepEqual(overridden[0], { role: "system", content: "new system" });
|
|
assert.equal(overridden.filter((message) => message.role === "system").length, 1);
|
|
|
|
const result = comboAgentMiddleware.applyComboAgentMiddleware(
|
|
body,
|
|
{
|
|
context_cache_protection: true,
|
|
system_message: "combo system",
|
|
tool_filter_regex: "^keep",
|
|
},
|
|
"openai/gpt-4o"
|
|
);
|
|
|
|
// PR #3399: server-side session pinning replaced <omniModel> tag extraction;
|
|
// pinnedModel is now always null from applyComboAgentMiddleware.
|
|
assert.equal(result.pinnedModel, null);
|
|
assert.equal(result.body.model, "combo/default");
|
|
assert.deepEqual(result.body.messages, [
|
|
{ role: "system", content: "combo system" },
|
|
{ role: "user", content: "hello" },
|
|
{ role: "assistant", content: "cached" },
|
|
]);
|
|
assert.deepEqual(result.body.tools, filtered);
|
|
|
|
const passthrough = comboAgentMiddleware.applyComboAgentMiddleware(body, null, "openai/gpt-4o");
|
|
assert.equal(passthrough.pinnedModel, null);
|
|
assert.equal(passthrough.body, body);
|
|
});
|
|
|
|
test("rate limit semaphore covers immediate acquire, timeout, cooldown drain and reset", async () => {
|
|
const release = await rateLimitSemaphore.acquire("model-a", { maxConcurrency: 1 });
|
|
assert.deepEqual(rateLimitSemaphore.getStats()["model-a"], {
|
|
running: 1,
|
|
queued: 0,
|
|
max: 1,
|
|
rateLimitedUntil: null,
|
|
});
|
|
|
|
release();
|
|
release();
|
|
// #2903 (perf-ram) prunes idle gates when they reach zero running/queued, so the
|
|
// entry may be absent here — assert "no running slots" without assuming it persists.
|
|
assert.equal(rateLimitSemaphore.getStats()["model-a"]?.running ?? 0, 0);
|
|
|
|
const heldRelease = await rateLimitSemaphore.acquire("model-b", { maxConcurrency: 1 });
|
|
const timeoutPromise = rateLimitSemaphore.acquire("model-b", {
|
|
maxConcurrency: 1,
|
|
timeoutMs: 15,
|
|
});
|
|
await assert.rejects(timeoutPromise, (error) => (error as any).code === "SEMAPHORE_TIMEOUT");
|
|
heldRelease();
|
|
|
|
const firstRelease = await rateLimitSemaphore.acquire("model-c", { maxConcurrency: 1 });
|
|
const secondPromise = rateLimitSemaphore.acquire("model-c", {
|
|
maxConcurrency: 1,
|
|
timeoutMs: 500,
|
|
});
|
|
rateLimitSemaphore.markRateLimited("model-c", 20);
|
|
firstRelease();
|
|
await wait(10);
|
|
assert.equal(rateLimitSemaphore.getStats()["model-c"].queued, 1);
|
|
const secondRelease = await secondPromise;
|
|
secondRelease();
|
|
assert.equal(rateLimitSemaphore.getStats()["model-c"]?.queued ?? 0, 0);
|
|
|
|
const blockingRelease = await rateLimitSemaphore.acquire("model-d", { maxConcurrency: 1 });
|
|
const queuedPromise = rateLimitSemaphore.acquire("model-d", {
|
|
maxConcurrency: 1,
|
|
timeoutMs: 500,
|
|
});
|
|
rateLimitSemaphore.resetAll();
|
|
await assert.rejects(queuedPromise, /Semaphore reset/);
|
|
blockingRelease();
|
|
});
|
|
|
|
test("combo metrics cover empty reads, intent tracking, per-model stats and resets", () => {
|
|
assert.equal(comboMetrics.getComboMetrics("missing"), null);
|
|
|
|
comboMetrics.recordComboShadowRequest("shadow-only", "openai/shadow", {
|
|
success: true,
|
|
latencyMs: 40,
|
|
});
|
|
const shadowOnlyMetrics = comboMetrics.getComboMetrics("shadow-only");
|
|
assert.equal(shadowOnlyMetrics.productionTraffic, false);
|
|
assert.equal(shadowOnlyMetrics.totalRequests, 0);
|
|
assert.equal(shadowOnlyMetrics.shadow.totalRequests, 1);
|
|
comboMetrics.resetComboMetrics("shadow-only");
|
|
|
|
comboMetrics.recordComboIntent("idle", "chat");
|
|
const idleMetrics = comboMetrics.getComboMetrics("idle");
|
|
assert.equal(idleMetrics.productionTraffic, false);
|
|
assert.equal(idleMetrics.avgLatencyMs, 0);
|
|
assert.equal(idleMetrics.successRate, 0);
|
|
assert.equal(idleMetrics.fallbackRate, 0);
|
|
assert.deepEqual(idleMetrics.intentCounts, { chat: 1 });
|
|
|
|
comboMetrics.recordComboIntent("writer", "");
|
|
comboMetrics.recordComboIntent("writer", "chat");
|
|
comboMetrics.recordComboRequest("writer", "openai/gpt-4o", {
|
|
success: true,
|
|
latencyMs: 120,
|
|
fallbackCount: 1,
|
|
strategy: "priority",
|
|
target: {
|
|
executionKey: "writer>step-openai-a",
|
|
stepId: "step-openai-a",
|
|
provider: "openai",
|
|
connectionId: "conn-openai-a",
|
|
label: "Primary OpenAI",
|
|
},
|
|
});
|
|
comboMetrics.recordComboRequest("writer", "openai/gpt-4o", {
|
|
success: false,
|
|
latencyMs: 80,
|
|
fallbackCount: 0,
|
|
strategy: "least-used",
|
|
target: {
|
|
executionKey: "writer>step-openai-a",
|
|
stepId: "step-openai-a",
|
|
provider: "openai",
|
|
connectionId: "conn-openai-a",
|
|
label: "Primary OpenAI",
|
|
},
|
|
});
|
|
comboMetrics.recordComboRequest("writer", null, {
|
|
success: true,
|
|
latencyMs: 50,
|
|
fallbackCount: 0,
|
|
strategy: "least-used",
|
|
});
|
|
|
|
const writer = comboMetrics.getComboMetrics("writer");
|
|
assert.equal(writer.productionTraffic, true);
|
|
assert.equal(writer.strategy, "least-used");
|
|
assert.equal(writer.totalRequests, 3);
|
|
assert.equal(writer.totalSuccesses, 2);
|
|
assert.equal(writer.totalFailures, 1);
|
|
assert.equal(writer.totalFallbacks, 1);
|
|
assert.equal(writer.avgLatencyMs, 83);
|
|
assert.equal(writer.successRate, 67);
|
|
assert.equal(writer.fallbackRate, 33);
|
|
assert.deepEqual(writer.intentCounts, { unknown: 1, chat: 1 });
|
|
assert.equal(writer.byModel["openai/gpt-4o"].requests, 2);
|
|
assert.equal(writer.byModel["openai/gpt-4o"].successRate, 50);
|
|
assert.equal(writer.byModel["openai/gpt-4o"].avgLatencyMs, 100);
|
|
assert.equal(writer.byTarget["writer>step-openai-a"].requests, 2);
|
|
assert.equal(writer.byTarget["writer>step-openai-a"].successRate, 50);
|
|
assert.equal(writer.byTarget["writer>step-openai-a"].connectionId, "conn-openai-a");
|
|
assert.equal(writer.byTarget["writer>step-openai-a"].label, "Primary OpenAI");
|
|
|
|
const allMetrics = comboMetrics.getAllComboMetrics();
|
|
assert.equal(Object.keys(allMetrics).length, 2);
|
|
assert.ok(allMetrics.idle);
|
|
assert.ok(allMetrics.writer);
|
|
|
|
comboMetrics.resetComboMetrics("writer");
|
|
assert.equal(comboMetrics.getComboMetrics("writer"), null);
|
|
|
|
comboMetrics.recordComboRequest("reader", "anthropic/claude-sonnet-4-6", {
|
|
success: true,
|
|
latencyMs: 30,
|
|
});
|
|
comboMetrics.resetAllComboMetrics();
|
|
assert.deepEqual(comboMetrics.getAllComboMetrics(), {});
|
|
});
|
|
|
|
test("model helpers cover malformed input, alias maps, wildcard aliases, ambiguity and provider inference", async () => {
|
|
assert.equal(modelService.resolveProviderAlias("gh"), "github");
|
|
assert.equal(modelService.resolveProviderAlias("openai"), "openai");
|
|
|
|
assert.deepEqual(modelService.parseModel("../escape"), {
|
|
provider: null,
|
|
model: null,
|
|
isAlias: false,
|
|
providerAlias: null,
|
|
extendedContext: false,
|
|
});
|
|
assert.deepEqual(modelService.parseModel("bad\u0000model"), {
|
|
provider: null,
|
|
model: null,
|
|
isAlias: false,
|
|
providerAlias: null,
|
|
extendedContext: false,
|
|
});
|
|
|
|
assert.deepEqual(
|
|
modelService.resolveModelAliasFromMap("alias-a", {
|
|
"alias-a": "gh/gemini-3-pro",
|
|
}),
|
|
{
|
|
provider: "github",
|
|
model: "gemini-3-pro",
|
|
}
|
|
);
|
|
assert.deepEqual(
|
|
modelService.resolveModelAliasFromMap("alias-b", {
|
|
"alias-b": { provider: "cx", model: "gpt-5.2-codex" },
|
|
}),
|
|
{
|
|
provider: "codex",
|
|
model: "gpt-5.2-codex",
|
|
}
|
|
);
|
|
assert.equal(modelService.resolveModelAliasFromMap("missing", { other: "x/y" }), null);
|
|
assert.equal(modelService.resolveModelAliasFromMap("broken", { broken: "no-slash" }), null);
|
|
|
|
const exactAlias = await modelService.getModelInfoCore("alias-exact[1m]", async () => ({
|
|
"alias-exact": "gh/gemini-3-pro",
|
|
}));
|
|
assert.deepEqual(exactAlias, {
|
|
provider: "github",
|
|
model: "gemini-3.1-pro-preview",
|
|
extendedContext: true,
|
|
});
|
|
|
|
const wildcardAlias = await modelService.getModelInfoCore("claude-sonnet-special", {
|
|
"claude-sonnet-*": "anthropic/claude-sonnet-4-5-20250929",
|
|
});
|
|
assert.equal(wildcardAlias.provider, "anthropic");
|
|
assert.equal(wildcardAlias.model, "claude-sonnet-4-5-20250929");
|
|
assert.equal(wildcardAlias.wildcardPattern, "claude-sonnet-*");
|
|
|
|
const ambiguous = await modelService.getModelInfoCore("claude-haiku-4.5", {});
|
|
assert.equal(ambiguous.provider, null);
|
|
assert.equal(ambiguous.errorType, "ambiguous_model");
|
|
assert.ok(ambiguous.errorMessage.includes("provider/model"));
|
|
assert.ok(Array.isArray(ambiguous.candidateProviders));
|
|
assert.ok(ambiguous.candidateProviders.length >= 2);
|
|
|
|
assert.deepEqual(await modelService.getModelInfoCore("claude-unknown", {}), {
|
|
provider: "anthropic",
|
|
model: "claude-unknown",
|
|
extendedContext: false,
|
|
});
|
|
assert.deepEqual(await modelService.getModelInfoCore("gemini-custom", {}), {
|
|
provider: "gemini",
|
|
model: "gemini-custom",
|
|
extendedContext: false,
|
|
});
|
|
const unknownModel = await modelService.getModelInfoCore("made-up-model", {});
|
|
assert.equal(unknownModel.provider, null);
|
|
assert.equal(unknownModel.errorType, "model_not_found");
|
|
assert.ok(unknownModel.errorMessage.includes("made-up-model"));
|
|
assert.ok(unknownModel.errorMessage.includes("provider/model prefix"));
|
|
});
|
|
|
|
test("error classifier covers empty-content helpers, context overflow and remaining error classes", () => {
|
|
assert.equal(errorClassifier.isEmptyContentResponse(null), false);
|
|
assert.equal(errorClassifier.isEmptyContentResponse({ choices: [] }), true);
|
|
assert.equal(
|
|
errorClassifier.isEmptyContentResponse({
|
|
choices: [{ message: { content: "", tool_calls: [{ id: "call_1" }] } }],
|
|
}),
|
|
false
|
|
);
|
|
assert.equal(errorClassifier.isEmptyContentResponse({ content: [] }), true);
|
|
assert.equal(errorClassifier.isEmptyContentResponse({ text: " " }), true);
|
|
assert.equal(errorClassifier.isEmptyContentResponse({ content: null }), true);
|
|
assert.equal(errorClassifier.isEmptyContentResponse({ content: "hello" }), false);
|
|
|
|
assert.equal(errorClassifier.isContextOverflow("request exceeds context window"), true);
|
|
assert.equal(errorClassifier.isContextOverflow("plain auth error"), false);
|
|
|
|
assert.equal(
|
|
errorClassifier.classifyProviderError(
|
|
429,
|
|
JSON.stringify({ error: { message: "insufficient_quota: exceeded your current quota" } })
|
|
),
|
|
errorClassifier.PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED
|
|
);
|
|
assert.equal(
|
|
errorClassifier.classifyProviderError(
|
|
401,
|
|
JSON.stringify({ error: { message: "invalid authentication credentials" } })
|
|
),
|
|
errorClassifier.PROVIDER_ERROR_TYPES.OAUTH_INVALID_TOKEN
|
|
);
|
|
assert.equal(
|
|
errorClassifier.classifyProviderError(
|
|
403,
|
|
JSON.stringify({ error: { message: "account_deactivated due to billing issue" } })
|
|
),
|
|
errorClassifier.PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED
|
|
);
|
|
assert.equal(
|
|
errorClassifier.classifyProviderError(500, { error: { message: "upstream exploded" } }),
|
|
errorClassifier.PROVIDER_ERROR_TYPES.SERVER_ERROR
|
|
);
|
|
assert.equal(
|
|
errorClassifier.classifyProviderError(400, { error: { message: "input exceeds context" } }),
|
|
errorClassifier.PROVIDER_ERROR_TYPES.CONTEXT_OVERFLOW
|
|
);
|
|
|
|
const circular = {};
|
|
circular.self = circular;
|
|
assert.equal(errorClassifier.classifyProviderError(418, circular), null);
|
|
});
|