chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,319 @@
|
||||
/**
|
||||
* scripts/test/_vpsClient.mjs
|
||||
*
|
||||
* Reusable Phase-3 VPS HTTP client for OmniRoute combo live-smoke tests.
|
||||
* NOT a test file — intentionally placed in scripts/test/ so check:test-discovery
|
||||
* does not scan it.
|
||||
*
|
||||
* Combo create/delete mechanism: SSH-sqlite fallback.
|
||||
* /api/combos requires management auth (returns 401 unauthenticated).
|
||||
* We insert/delete rows directly via:
|
||||
* execFileSync("ssh", ["root@192.168.0.15", "sqlite3", "/root/.omniroute/storage.sqlite", SQL])
|
||||
* Values are static test-scoped data — no untrusted interpolation.
|
||||
*
|
||||
* combos table schema (PRAGMA table_info):
|
||||
* id TEXT PK, name TEXT NOT NULL, data TEXT NOT NULL (JSON blob),
|
||||
* created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
|
||||
* system_message TEXT, tool_filter_regex TEXT,
|
||||
* context_cache_protection INTEGER DEFAULT 0, sort_order INTEGER NOT NULL DEFAULT 0
|
||||
*
|
||||
* The `data` column stores the full combo as JSON (name, models[], strategy, config,
|
||||
* id, createdAt, updatedAt, version, sortOrder).
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config
|
||||
// ---------------------------------------------------------------------------
|
||||
const BASE_URL = process.env.COMBO_LIVE_BASE_URL ?? "http://192.168.0.15:20128";
|
||||
const API_KEY = process.env.COMBO_LIVE_API_KEY ?? null;
|
||||
const VPS_SSH_HOST = "root@192.168.0.15";
|
||||
const VPS_DB_PATH = "/root/.omniroute/storage.sqlite";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Nonce counter — increments per call so semantic cache cannot serve stale
|
||||
// ---------------------------------------------------------------------------
|
||||
let _nonceCounter = 0;
|
||||
export function nonce() {
|
||||
return ++_nonceCounter;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared fetch helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
function authHeaders() {
|
||||
const h = { "Content-Type": "application/json" };
|
||||
if (API_KEY) h["Authorization"] = `Bearer ${API_KEY}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
async function fetchJson(path, options = {}) {
|
||||
const url = `${BASE_URL}${path}`;
|
||||
const res = await fetch(url, { ...options, headers: { ...authHeaders(), ...(options.headers ?? {}) } });
|
||||
let body;
|
||||
try {
|
||||
body = await res.json();
|
||||
} catch {
|
||||
body = null;
|
||||
}
|
||||
return { status: res.status, ok: res.ok, body };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// health() — GET /api/monitoring/health
|
||||
// ---------------------------------------------------------------------------
|
||||
export async function health() {
|
||||
const { status, body } = await fetchJson("/api/monitoring/health");
|
||||
return {
|
||||
status,
|
||||
version: body?.version ?? null,
|
||||
uptime: body?.uptime ?? null,
|
||||
raw: body,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// chat() — POST /v1/chat/completions (non-streaming)
|
||||
// ---------------------------------------------------------------------------
|
||||
export async function chat(model, { maxTokens = 16, content } = {}) {
|
||||
const n = nonce();
|
||||
const userContent = content ?? `ping ${n}`;
|
||||
const payload = {
|
||||
model,
|
||||
stream: false,
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0,
|
||||
messages: [{ role: "user", content: userContent }],
|
||||
};
|
||||
const { status, body } = await fetchJson("/v1/chat/completions", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const text = body?.choices?.[0]?.message?.content ?? null;
|
||||
return {
|
||||
status,
|
||||
model: body?.model ?? model,
|
||||
text,
|
||||
raw: body,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Combo create/delete via SSH sqlite
|
||||
// ---------------------------------------------------------------------------
|
||||
function sshSqlite(sql) {
|
||||
// Pipe SQL via stdin to avoid shell quoting issues with complex SQL statements.
|
||||
// ssh + sqlite3 reads from stdin when no trailing SQL arg is given.
|
||||
return execFileSync("ssh", [VPS_SSH_HOST, "sqlite3", VPS_DB_PATH], {
|
||||
input: sql,
|
||||
encoding: "utf8",
|
||||
timeout: 15_000,
|
||||
}).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* createCombo(def) — inserts a combo row via SSH sqlite.
|
||||
*
|
||||
* def shape:
|
||||
* { name, strategy, models, config }
|
||||
*
|
||||
* models: array of "providerId/model" strings OR
|
||||
* array of { providerId, model, connectionId?, weight? } objects.
|
||||
*
|
||||
* config: optional object (judgeModel, fusionTuning, etc.)
|
||||
*
|
||||
* Returns the combo id string.
|
||||
*/
|
||||
export function createCombo(def) {
|
||||
const id = randomUUID();
|
||||
const now = new Date().toISOString();
|
||||
const strategy = def.strategy ?? "priority";
|
||||
const config = def.config ?? {};
|
||||
const name = def.name;
|
||||
|
||||
// Normalise models to the shape the DB expects
|
||||
const rawModels = def.models ?? [];
|
||||
const models = rawModels.map((m, idx) => {
|
||||
if (typeof m === "string") {
|
||||
// "providerId/model" shorthand
|
||||
const slashIdx = m.indexOf("/");
|
||||
const providerId = slashIdx >= 0 ? m.slice(0, slashIdx) : m;
|
||||
const model = slashIdx >= 0 ? m.slice(slashIdx + 1) : m;
|
||||
const slugName = name.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
const slugModel = model.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
return {
|
||||
id: `${slugName}-model-${idx + 1}-${providerId}-${slugModel}-${randomUUID()}`,
|
||||
kind: "model",
|
||||
model: `${providerId}/${model}`,
|
||||
providerId,
|
||||
weight: 1,
|
||||
};
|
||||
}
|
||||
// Already an object
|
||||
const providerId = m.providerId;
|
||||
const model = m.model;
|
||||
const slugName = name.toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
const slugModel = (model ?? "").toLowerCase().replace(/[^a-z0-9]/g, "-");
|
||||
return {
|
||||
id: m.id ?? `${slugName}-model-${idx + 1}-${providerId}-${slugModel}-${randomUUID()}`,
|
||||
kind: "model",
|
||||
model: model.includes("/") ? model : `${providerId}/${model}`,
|
||||
providerId,
|
||||
...(m.connectionId ? { connectionId: m.connectionId } : {}),
|
||||
weight: m.weight ?? 1,
|
||||
};
|
||||
});
|
||||
|
||||
const dataObj = {
|
||||
name,
|
||||
models,
|
||||
strategy,
|
||||
config,
|
||||
id,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
version: 2,
|
||||
sortOrder: 9999,
|
||||
};
|
||||
|
||||
const dataJson = JSON.stringify(dataObj).replace(/'/g, "''");
|
||||
const nameSafe = name.replace(/'/g, "''");
|
||||
const sql = `INSERT INTO combos (id, name, data, created_at, updated_at, sort_order) VALUES ('${id}', '${nameSafe}', '${dataJson}', '${now}', '${now}', 9999);`;
|
||||
sshSqlite(sql);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* deleteCombo(nameOrId) — deletes a combo by name or id via SSH sqlite.
|
||||
* Only deletes __live_test__* prefixed combos as a safety guard.
|
||||
*/
|
||||
export function deleteCombo(nameOrId) {
|
||||
if (!nameOrId.startsWith("__live_test__")) {
|
||||
throw new Error(`deleteCombo safety guard: refusing to delete '${nameOrId}' — only __live_test__* combos allowed.`);
|
||||
}
|
||||
const safe = nameOrId.replace(/'/g, "''");
|
||||
// Try delete by name first, then by id
|
||||
sshSqlite(`DELETE FROM combos WHERE name='${safe}' OR id='${safe}';`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// listHealthyProviders(candidates) — probe each "provider/model" candidate
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* For each "provider/model" string, fire a minimal chat() and keep those
|
||||
* returning HTTP 200 with non-empty text.
|
||||
*
|
||||
* @param {string[]} candidates - array of "provider/model" strings
|
||||
* @returns {Promise<string[]>} healthy candidates
|
||||
*/
|
||||
export async function listHealthyProviders(candidates) {
|
||||
const results = await Promise.allSettled(
|
||||
candidates.map(async (c) => {
|
||||
const r = await chat(c, { maxTokens: 16 });
|
||||
return r.status === 200 && r.text ? c : null;
|
||||
})
|
||||
);
|
||||
return results
|
||||
.map((r) => (r.status === "fulfilled" ? r.value : null))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Preflight self-check (run directly: node scripts/test/_vpsClient.mjs)
|
||||
// ---------------------------------------------------------------------------
|
||||
const isMain = process.argv[1]?.endsWith("_vpsClient.mjs") ||
|
||||
import.meta.url === `file://${process.argv[1]}`;
|
||||
|
||||
if (isMain) {
|
||||
(async () => {
|
||||
console.log("=== OmniRoute VPS Phase-3 Preflight ===");
|
||||
console.log(`Base URL: ${BASE_URL}`);
|
||||
console.log(`API key: ${API_KEY ? "set (Bearer)" : "not set (REQUIRE_API_KEY=false)"}`);
|
||||
console.log();
|
||||
|
||||
// 1. Health
|
||||
console.log("--- health() ---");
|
||||
try {
|
||||
const h = await health();
|
||||
console.log(` status: ${h.status}`);
|
||||
console.log(` version: ${h.version}`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// 2. Combo mechanism probe
|
||||
console.log();
|
||||
console.log("--- combo create/delete mechanism ---");
|
||||
console.log(" /api/combos GET (unauthenticated):", (() => {
|
||||
try {
|
||||
const r = execFileSync("curl", ["-s", "-o", "/dev/null", "-w", "%{http_code}", `${BASE_URL}/api/combos`], { encoding: "utf8", timeout: 5000 });
|
||||
return r.trim();
|
||||
} catch { return "error"; }
|
||||
})());
|
||||
console.log(" Mechanism: SSH sqlite fallback (management API requires auth)");
|
||||
console.log(" SSH host:", VPS_SSH_HOST);
|
||||
console.log(" DB path:", VPS_DB_PATH);
|
||||
|
||||
// 3. Create + delete a probe combo via SSH sqlite
|
||||
console.log();
|
||||
console.log("--- createCombo / deleteCombo (SSH sqlite) ---");
|
||||
const probeName = "__live_test__probe";
|
||||
let probeId;
|
||||
try {
|
||||
probeId = createCombo({
|
||||
name: probeName,
|
||||
strategy: "priority",
|
||||
models: ["groq/llama-3.1-8b-instant"],
|
||||
config: {},
|
||||
});
|
||||
console.log(` created id: ${probeId}`);
|
||||
deleteCombo(probeName);
|
||||
console.log(` deleted OK`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
// Attempt cleanup on error
|
||||
if (probeId) {
|
||||
try { deleteCombo(probeName); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. ollama-cloud chat probe
|
||||
console.log();
|
||||
console.log("--- chat probe: ollama-cloud/glm-5.2 ---");
|
||||
try {
|
||||
const r = await chat("ollama-cloud/glm-5.2", { maxTokens: 16 });
|
||||
console.log(` status: ${r.status}`);
|
||||
console.log(` model: ${r.model}`);
|
||||
console.log(` text: ${r.text ? r.text.slice(0, 80) : "(empty)"}`);
|
||||
if (r.status !== 200 || !r.text) {
|
||||
console.log(" NOTE: ollama-cloud/glm-5.2 did not return a valid response.");
|
||||
console.log(" raw error:", JSON.stringify(r.raw?.error ?? r.raw).slice(0, 200));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// 5. Quick healthy provider scan over a small candidate set
|
||||
console.log();
|
||||
console.log("--- listHealthyProviders (subset scan) ---");
|
||||
const candidates = [
|
||||
"groq/llama-3.1-8b-instant",
|
||||
"gemini/gemini-2.0-flash",
|
||||
"deepseek/deepseek-chat",
|
||||
"cerebras/llama3.1-8b",
|
||||
"ollama-cloud/glm-5.2",
|
||||
];
|
||||
try {
|
||||
const healthy = await listHealthyProviders(candidates);
|
||||
console.log(` tested: ${candidates.join(", ")}`);
|
||||
console.log(` healthy (${healthy.length}/${candidates.length}): ${healthy.join(", ") || "(none)"}`);
|
||||
} catch (e) {
|
||||
console.error(` ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log("=== Preflight complete ===");
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,656 @@
|
||||
/**
|
||||
* scripts/test/combo-live-vps.mjs
|
||||
*
|
||||
* Phase-3 VPS HTTP scenario driver for OmniRoute combo routing.
|
||||
* Exercises 6 strategies (priority / round-robin / weighted / cost-optimized /
|
||||
* fusion / auto) against the live server at 192.168.0.15:20128.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/test/combo-live-vps.mjs
|
||||
* node scripts/test/combo-live-vps.mjs --only=round-robin
|
||||
* node scripts/test/combo-live-vps.mjs --failover # runs all 7 base scenarios + real failover
|
||||
*
|
||||
* Safety rules:
|
||||
* - Only creates/deletes __live_test__* combos
|
||||
* - Always cleans up in `finally` blocks
|
||||
* - Never stops services or touches other data
|
||||
*
|
||||
* Exit code: 0 if all scenarios PASS or SKIP; non-zero only on real FAIL.
|
||||
* Task 8 (--failover) can be appended after the main() call at the bottom.
|
||||
*/
|
||||
|
||||
import { chat, createCombo, deleteCombo, listHealthyProviders, nonce } from "./_vpsClient.mjs";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Candidate list — broad to maximise coverage across volatile provider health.
|
||||
// listHealthyProviders() probes each with a real chat() call.
|
||||
// ---------------------------------------------------------------------------
|
||||
const BROAD_CANDIDATES = [
|
||||
"groq/llama-3.1-8b-instant",
|
||||
"groq/llama-3.3-70b-versatile",
|
||||
"minimax/MiniMax-M3",
|
||||
"minimax/minimax-m3",
|
||||
"kimi-coding-apikey/moonshot-v1-8k",
|
||||
"openrouter/openai/gpt-3.5-turbo",
|
||||
"cerebras/llama-3.3-70b",
|
||||
"cerebras/llama3.1-8b",
|
||||
"deepseek/deepseek-chat",
|
||||
"ollama-cloud/glm-5.2",
|
||||
"glm/glm-4-flash",
|
||||
"gemini/gemini-2.0-flash",
|
||||
];
|
||||
|
||||
// Known approximate input cost ($/M tokens) from OmniRoute's default-pricing constants.
|
||||
// Used only to identify cheap vs pricey pairs for cost-optimized scenario.
|
||||
// Models absent from this map are treated as unknown cost (Infinity in the server's
|
||||
// sortModelsByCost, i.e. sorted last — effectively "most expensive").
|
||||
const KNOWN_INPUT_COST = {
|
||||
"groq/llama-3.1-8b-instant": 0, // inference-hosts.ts: price=0 (free tier)
|
||||
"groq/llama-3.3-70b-versatile": 0, // inference-hosts.ts: price=0 (free tier)
|
||||
"cerebras/llama3.1-8b": 0, // inference-hosts.ts: price=0
|
||||
"cerebras/llama-3.3-70b": 0, // inference-hosts.ts: price=0
|
||||
"deepseek/deepseek-chat": 0, // inference-hosts.ts: price=0
|
||||
"minimax/MiniMax-M3": 0.5, // regional.ts: $0.5/M input
|
||||
"minimax/minimax-m3": 0.5, // regional.ts: $0.5/M input
|
||||
"kimi-coding-apikey/moonshot-v1-8k": 1, // not in pricing table → Infinity on server → treated pricey
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI args
|
||||
// ---------------------------------------------------------------------------
|
||||
const onlyArg = process.argv.find((a) => a.startsWith("--only="));
|
||||
const onlyScenario = onlyArg ? onlyArg.slice(7) : null;
|
||||
// --failover: opt-in flag that appends a real-failover scenario (broken primary → healthy fallback)
|
||||
const failoverFlag = process.argv.includes("--failover");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result tracking
|
||||
// ---------------------------------------------------------------------------
|
||||
let exitCode = 0;
|
||||
const summary = [];
|
||||
|
||||
function pass(name, detail = "") {
|
||||
summary.push({ name, result: "PASS" });
|
||||
console.log(`PASS [${name}]${detail ? ": " + detail : ""}`);
|
||||
}
|
||||
|
||||
function skip(name, reason) {
|
||||
summary.push({ name, result: "SKIP" });
|
||||
console.log(`SKIP [${name}]: ${reason}`);
|
||||
}
|
||||
|
||||
function fail(name, reason, err = null) {
|
||||
summary.push({ name, result: "FAIL" });
|
||||
console.error(`FAIL [${name}]: ${reason}`);
|
||||
if (err) console.error(" caused by:", err?.message ?? String(err));
|
||||
exitCode = 1;
|
||||
}
|
||||
|
||||
async function runScenario(name, fn) {
|
||||
if (onlyScenario && onlyScenario !== name) return;
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
fail(name, `unexpected error: ${err?.message ?? String(err)}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: split "provider/model" → { providerId, modelPart }
|
||||
// For multi-segment paths like "openrouter/openai/gpt-3.5-turbo":
|
||||
// providerId = "openrouter", modelPart = "openai/gpt-3.5-turbo"
|
||||
// ---------------------------------------------------------------------------
|
||||
function splitProviderModel(full) {
|
||||
const idx = full.indexOf("/");
|
||||
if (idx < 0) return { providerId: full, modelPart: full };
|
||||
return { providerId: full.slice(0, idx), modelPart: full.slice(idx + 1) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STEP 0: Cache probe
|
||||
// Verifies that a sqlite-inserted combo is immediately routable.
|
||||
// getComboByName() in combos.ts does a direct DB read with no TTL cache,
|
||||
// so the combo should be visible as soon as the SSH INSERT commits.
|
||||
// If not (unexpected caching behaviour), we poll up to 12s before blocking.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function step0CacheProbe(healthy) {
|
||||
console.log("\n=== STEP 0: sqlite → routable cache probe ===");
|
||||
const probe = healthy[0];
|
||||
const probeName = "__live_test__probe";
|
||||
let id;
|
||||
let blocked = false;
|
||||
|
||||
try {
|
||||
id = createCombo({ name: probeName, strategy: "priority", models: [probe] });
|
||||
console.log(` created combo id=${id} model=${probe}`);
|
||||
|
||||
// Attempt immediate chat — expect instant visibility (getComboByName bypasses cache)
|
||||
let r = await chat(probeName, { maxTokens: 4 });
|
||||
if (r.status === 200 && r.text) {
|
||||
console.log(
|
||||
` chat() → ${r.status} model=${r.model} text="${r.text.slice(0, 40)}"`
|
||||
);
|
||||
console.log(" PROBE RESULT: immediately routable — getComboByName bypasses in-memory cache");
|
||||
} else {
|
||||
console.log(
|
||||
` Immediate chat → status=${r.status} text=${r.text ?? "(empty)"}`
|
||||
);
|
||||
console.log(" Polling up to 12 s (TTL cache unexpectedly active)...");
|
||||
let resolved = false;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await new Promise((res) => setTimeout(res, 2000));
|
||||
r = await chat(probeName, { maxTokens: 4 });
|
||||
if (r.status === 200 && r.text) {
|
||||
console.log(` Resolved after ~${(i + 1) * 2}s: model=${r.model}`);
|
||||
console.log(" PROBE RESULT: visible after TTL expiry");
|
||||
resolved = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!resolved) {
|
||||
console.error(" PROBE RESULT: BLOCKER — combo not routable within 12s");
|
||||
blocked = true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (id) {
|
||||
try {
|
||||
deleteCombo(probeName);
|
||||
console.log(` cleanup: ${probeName} deleted`);
|
||||
} catch (e) {
|
||||
console.error(` cleanup error: ${e?.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blocked) {
|
||||
throw new Error("BLOCKER: sqlite-inserted combo not routable within 12s — cannot run scenarios");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 1: priority
|
||||
// Two healthy models. Single chat() call → 200 + non-empty text.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioPriority(healthy) {
|
||||
const name = "__live_test__priority";
|
||||
if (healthy.length < 1) {
|
||||
skip("priority", "no healthy providers");
|
||||
return;
|
||||
}
|
||||
const models = healthy.slice(0, 2);
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({ name, strategy: "priority", models });
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("priority", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("priority", "response text is empty");
|
||||
return;
|
||||
}
|
||||
pass("priority", `status=200 model=${r.model} text="${r.text.slice(0, 40)}"`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 2: round-robin
|
||||
// ≥2 healthy models. 5 calls with unique nonces → at least 2 distinct
|
||||
// response.model values (each call must return 200).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioRoundRobin(healthy) {
|
||||
const name = "__live_test__round-robin";
|
||||
if (healthy.length < 2) {
|
||||
skip("round-robin", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
const models = healthy.slice(0, Math.min(3, healthy.length));
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({ name, strategy: "round-robin", models });
|
||||
const served = new Set();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("round-robin", `call ${i + 1} returned status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
served.add(r.model);
|
||||
}
|
||||
if (served.size < 2) {
|
||||
fail(
|
||||
"round-robin",
|
||||
`only 1 distinct model across 5 calls: [${[...served].join(", ")}] — round-robin not distributing`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass("round-robin", `${served.size} distinct models across 5 calls: [${[...served].join(", ")}]`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 3: weighted
|
||||
// Two healthy models with equal weight (50/50). 8 calls → both appear at
|
||||
// least once (loose statistical check — P(only 1 model in 8 calls) ≈ 0.8%).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioWeighted(healthy) {
|
||||
const name = "__live_test__weighted";
|
||||
if (healthy.length < 2) {
|
||||
skip("weighted", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
const [m1, m2] = healthy.slice(0, 2);
|
||||
const { providerId: p1, modelPart: mp1 } = splitProviderModel(m1);
|
||||
const { providerId: p2, modelPart: mp2 } = splitProviderModel(m2);
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "weighted",
|
||||
models: [
|
||||
{ providerId: p1, model: mp1, weight: 50 },
|
||||
{ providerId: p2, model: mp2, weight: 50 },
|
||||
],
|
||||
});
|
||||
const tally = {};
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("weighted", `call ${i + 1} returned status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
tally[r.model] = (tally[r.model] ?? 0) + 1;
|
||||
}
|
||||
const distinct = Object.keys(tally);
|
||||
if (distinct.length < 2) {
|
||||
fail(
|
||||
"weighted",
|
||||
`only 1 distinct model across 8 calls: ${JSON.stringify(tally)} — weighted routing not distributing`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass("weighted", `8 calls distribution: ${JSON.stringify(tally)}`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 4: cost-optimized
|
||||
// Find a cheap+pricey healthy pair (based on KNOWN_INPUT_COST).
|
||||
// Insert pricey first (position 0) so cost-optimized must reorder.
|
||||
// Assert: the served model matches the cheap provider.
|
||||
//
|
||||
// OmniRoute's sortModelsByCost uses getPricingForModel which merges
|
||||
// default-pricing constants. groq models have price=0; minimax M3 has $0.5.
|
||||
// Cost-optimized sorts ascending → groq (0) before minimax (0.5).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioCostOptimized(healthy) {
|
||||
const name = "__live_test__cost-optimized";
|
||||
|
||||
// Partition healthy into cheap (price=0) and pricey (price>0 or unknown>0)
|
||||
const cheap = healthy.filter(
|
||||
(m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] === 0
|
||||
);
|
||||
const pricey = healthy.filter(
|
||||
(m) => KNOWN_INPUT_COST[m] !== undefined && KNOWN_INPUT_COST[m] > 0
|
||||
);
|
||||
|
||||
if (cheap.length === 0 || pricey.length === 0) {
|
||||
skip(
|
||||
"cost-optimized",
|
||||
`no distinguishable cheap+pricey pair among healthy=[${healthy.join(", ")}]`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const cheapModel = cheap[0];
|
||||
const priceyModel = pricey[0];
|
||||
let id;
|
||||
try {
|
||||
// Insert pricey first — cost-optimized should reorder to serve cheapModel first
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "cost-optimized",
|
||||
models: [priceyModel, cheapModel],
|
||||
});
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("cost-optimized", `status=${r.status}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("cost-optimized", "empty response text");
|
||||
return;
|
||||
}
|
||||
// Verify the cheap model was served (response.model contains cheap provider's model name)
|
||||
const cheapProvider = splitProviderModel(cheapModel).providerId;
|
||||
// Both direct match and provider-substring match are accepted since OmniRoute
|
||||
// returns the raw upstream model name (e.g. "llama-3.1-8b-instant" not "groq/...")
|
||||
const cheapModelPart = splitProviderModel(cheapModel).modelPart.toLowerCase();
|
||||
const servedModel = (r.model ?? "").toLowerCase();
|
||||
const isChapModel =
|
||||
servedModel === cheapModelPart ||
|
||||
servedModel.includes(cheapModelPart) ||
|
||||
servedModel === cheapModel.toLowerCase() ||
|
||||
// fallback: check provider header if available (response.model could be bare name)
|
||||
servedModel.includes(cheapProvider.toLowerCase());
|
||||
|
||||
if (!isChapModel) {
|
||||
fail(
|
||||
"cost-optimized",
|
||||
`expected cheaper model (${cheapModel}, price=${KNOWN_INPUT_COST[cheapModel]}) but got ${r.model} — cost-optimized may not have reordered`
|
||||
);
|
||||
return;
|
||||
}
|
||||
pass(
|
||||
"cost-optimized",
|
||||
`cheaper model served: ${r.model} (cheap=${cheapModel}@$${KNOWN_INPUT_COST[cheapModel]}, pricey=${priceyModel}@$${KNOWN_INPUT_COST[priceyModel]})`
|
||||
);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 5: fusion
|
||||
// Panel of 2-3 healthy models fan out in parallel; judge synthesizes 1 answer.
|
||||
// Cost guard: panel ≤3, max_tokens=16, one call.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioFusion(healthy) {
|
||||
const name = "__live_test__fusion";
|
||||
if (healthy.length < 2) {
|
||||
skip("fusion", `need ≥2 healthy providers, found ${healthy.length}`);
|
||||
return;
|
||||
}
|
||||
// Panel: up to 3 distinct models
|
||||
const panelModels = healthy.slice(0, Math.min(3, healthy.length));
|
||||
// Judge: reuse first healthy model (cheap, already warmed up)
|
||||
const judgeModel = panelModels[0];
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "fusion",
|
||||
models: panelModels,
|
||||
config: {
|
||||
judgeModel,
|
||||
fusionTuning: { minPanel: 2 },
|
||||
},
|
||||
});
|
||||
// Use a unique nonce in content to defeat semantic cache
|
||||
const r = await chat(name, {
|
||||
maxTokens: 16,
|
||||
content: `hi ${nonce()} answer in one word`,
|
||||
});
|
||||
if (r.status !== 200) {
|
||||
fail("fusion", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("fusion", "judge returned empty synthesized text");
|
||||
return;
|
||||
}
|
||||
pass("fusion", `synthesized text="${r.text.slice(0, 60)}" served-model=${r.model}`);
|
||||
} finally {
|
||||
if (id) try { deleteCombo(name); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 6: auto
|
||||
// model="auto" → virtual auto-combo bypasses DB lookup entirely.
|
||||
// Assert: status=200, non-empty text, response.model is a real model (not "auto").
|
||||
// Also test "auto/fast" variant (skip if it 400s as unknown).
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioAuto() {
|
||||
// 6a: bare "auto"
|
||||
const r = await chat("auto", { maxTokens: 16 });
|
||||
if (r.status !== 200) {
|
||||
fail("auto", `status=${r.status} raw=${JSON.stringify(r.raw)?.slice(0, 120)}`);
|
||||
return;
|
||||
}
|
||||
if (!r.text) {
|
||||
fail("auto", "empty response text");
|
||||
return;
|
||||
}
|
||||
const servedModel = r.model ?? "";
|
||||
if (servedModel === "auto" || !servedModel) {
|
||||
fail("auto", `response.model is still "auto" — pool was not resolved`);
|
||||
return;
|
||||
}
|
||||
pass("auto", `status=200 resolved-model=${servedModel} text="${r.text.slice(0, 40)}"`);
|
||||
|
||||
// 6b: "auto/fast" variant
|
||||
const r2 = await chat("auto/fast", { maxTokens: 16 });
|
||||
if (r2.status === 400 || r2.status === 404) {
|
||||
skip("auto/fast", `variant not recognised (${r2.status})`);
|
||||
} else if (r2.status !== 200 || !r2.text) {
|
||||
fail("auto/fast", `status=${r2.status} text=${r2.text ?? "(empty)"}`);
|
||||
} else {
|
||||
pass("auto/fast", `status=200 model=${r2.model} text="${r2.text.slice(0, 40)}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 7: failover (opt-in via --failover)
|
||||
//
|
||||
// Approach: CROSS-PROVIDER BOGUS-MODEL (deterministic, no SSH crypto required).
|
||||
//
|
||||
// Why cross-provider?
|
||||
// A same-provider bogus-model fails silently: when target[0] (bogus) returns a
|
||||
// 404, OmniRoute calls recordProviderCooldown(providerA, undefined) — a provider-
|
||||
// wide key. The combo then pre-screens target[1] (same providerA, real model) and
|
||||
// finds providerA in cooldown → skips it → combo returns the 404 from target[0].
|
||||
// Using DIFFERENT providers avoids this: target[0] puts providerA in cooldown,
|
||||
// target[1] on providerB is unaffected, providerB serves the healthy response.
|
||||
//
|
||||
// Mechanism:
|
||||
// - Target[0]: <providerA>/__nonexistent_model_xyz__ → upstream 404 → providerA cooldown
|
||||
// - Target[1]: <providerB>/<realModel> → not in cooldown → 200 served
|
||||
// - config.maxRetries:0, retryDelayMs:0 → immediate fallover, no retry delay
|
||||
// - Bogus model can never return 200 → any 200 proves fallover to the real target
|
||||
//
|
||||
// If only 1 distinct provider is healthy: SKIP (BROKEN-CONNECTION approach needed; see
|
||||
// comment below for how to implement it using encrypted wrong API key via SSH sqlite).
|
||||
//
|
||||
// BROKEN-CONNECTION approach (for future reference or if cross-provider is unavailable):
|
||||
// 1. SSH to read STORAGE_ENCRYPTION_KEY from /root/.omniroute/.env
|
||||
// 2. Encrypt a wrong API key using scryptSync(key,"omniroute-field-encryption-v1",32)+AES-256-GCM
|
||||
// 3. INSERT broken provider_connection row into provider_connections table via SSH sqlite
|
||||
// 4. Combo: [{providerId:glm, model:glm/glm-4-flash, connectionId:brokenConnId}, realModel]
|
||||
// 5. Broken conn → 401 → recordProviderCooldown("glm",brokenConnId) → key "glm:brokenConnId"
|
||||
// 6. Target[1] on different provider → unaffected → 200
|
||||
// 7. Finally: DELETE combo AND broken connection
|
||||
// ---------------------------------------------------------------------------
|
||||
async function scenarioFailover(healthy) {
|
||||
const name = "__live_test__failover";
|
||||
|
||||
if (healthy.length < 1) {
|
||||
skip("failover", "no healthy providers — cannot run failover scenario");
|
||||
return;
|
||||
}
|
||||
|
||||
// Group healthy providers by providerId to find two distinct providers
|
||||
const byProvider = new Map();
|
||||
for (const m of healthy) {
|
||||
const { providerId } = splitProviderModel(m);
|
||||
if (!byProvider.has(providerId)) byProvider.set(providerId, []);
|
||||
byProvider.get(providerId).push(m);
|
||||
}
|
||||
const distinctProviders = [...byProvider.keys()];
|
||||
|
||||
if (distinctProviders.length < 2) {
|
||||
// Cannot use cross-provider approach with only one provider.
|
||||
// Same-provider bogus-model fails: 404 → recordProviderCooldown(provider, undefined) →
|
||||
// provider-wide cooldown blocks target[1] on the same provider.
|
||||
skip(
|
||||
"failover",
|
||||
`need ≥2 distinct healthy providers for cross-provider approach; ` +
|
||||
`found only 1 [${distinctProviders.join(", ")}]. ` +
|
||||
`Implement BROKEN-CONNECTION approach to run failover with a single provider.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// CROSS-PROVIDER BOGUS-MODEL:
|
||||
// target[0] = <providerA>/__nonexistent_model_xyz__ (will get 404, puts providerA in cooldown)
|
||||
// target[1] = <providerB>/<realHealthyModel> (different provider, not in cooldown)
|
||||
const bogusProvider = distinctProviders[0];
|
||||
const realModel = byProvider.get(distinctProviders[1])[0];
|
||||
const bogusModel = `${bogusProvider}/__nonexistent_model_xyz__`;
|
||||
const { modelPart: realModelPart } = splitProviderModel(realModel);
|
||||
|
||||
let id;
|
||||
try {
|
||||
id = createCombo({
|
||||
name,
|
||||
strategy: "priority",
|
||||
models: [bogusModel, realModel],
|
||||
config: { maxRetries: 0, retryDelayMs: 0 },
|
||||
});
|
||||
|
||||
console.log(
|
||||
` failover combo (CROSS-PROVIDER BOGUS-MODEL):\n` +
|
||||
` [0] ${bogusModel} ← broken primary (will 404)\n` +
|
||||
` [1] ${realModel} ← healthy fallback (different provider)\n` +
|
||||
` strategy=priority, maxRetries=0`
|
||||
);
|
||||
|
||||
const r = await chat(name, { maxTokens: 16 });
|
||||
|
||||
// Assertions:
|
||||
// 1. status=200: the combo succeeded — ONLY possible if it fell over to target[1],
|
||||
// because target[0] (bogus model) can NEVER return 200 from the upstream.
|
||||
// 2. Non-empty text: real LLM content was returned (not an empty error body).
|
||||
// 3. Served model is NOT the bogus one (belt-and-suspenders; 200 already proves it).
|
||||
// 4. Served model matches the real healthy target (positive proof of which model served).
|
||||
|
||||
if (r.status !== 200) {
|
||||
fail(
|
||||
"failover",
|
||||
`expected status=200 after fallover (broken ${bogusModel} → ${realModel}), ` +
|
||||
`got status=${r.status}. ` +
|
||||
`raw=${JSON.stringify(r.raw)?.slice(0, 200)}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!r.text) {
|
||||
fail("failover", `status=200 but empty text — fallover may have served a no-content response`);
|
||||
return;
|
||||
}
|
||||
|
||||
const bogusModelPart = "__nonexistent_model_xyz__";
|
||||
const servedModel = (r.model ?? "").toLowerCase();
|
||||
|
||||
// Negative proof: bogus model did NOT serve (should never happen, but guard anyway)
|
||||
if (servedModel.includes(bogusModelPart.toLowerCase())) {
|
||||
fail(
|
||||
"failover",
|
||||
`bogus model string "${bogusModelPart}" appears in served model field "${r.model}" — ` +
|
||||
`impossible 200 from a non-existent model; something is wrong`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Positive proof: served model matches the real healthy target
|
||||
const realModelLower = realModelPart.toLowerCase();
|
||||
const servedMatchesReal =
|
||||
servedModel === realModelLower ||
|
||||
servedModel.includes(realModelLower) ||
|
||||
servedModel === realModel.toLowerCase();
|
||||
|
||||
const proofNote = servedMatchesReal
|
||||
? ""
|
||||
: ` [NOTE: served="${r.model}" ≠ expected="${realModelPart}" ` +
|
||||
`— upstream alias likely; 200+text from bogus-primary is the failover proof]`;
|
||||
|
||||
pass(
|
||||
"failover",
|
||||
`CROSS-PROVIDER BOGUS-MODEL: broken primary (${bogusModel}) → ` +
|
||||
`fallover → served ${r.model} (real target: ${realModel}) ` +
|
||||
`text="${r.text.slice(0, 40)}"${proofNote}`
|
||||
);
|
||||
} finally {
|
||||
if (id) {
|
||||
try {
|
||||
deleteCombo(name);
|
||||
console.log(` cleanup: ${name} deleted`);
|
||||
} catch (e) {
|
||||
console.error(` cleanup error: ${e?.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
async function main() {
|
||||
console.log("=== OmniRoute combo-live-vps scenario driver ===");
|
||||
if (onlyScenario) console.log(`Filtering to scenario: ${onlyScenario}`);
|
||||
console.log();
|
||||
|
||||
// --- Health probe ---
|
||||
console.log("Probing healthy providers (broad candidate list)...");
|
||||
const healthy = await listHealthyProviders(BROAD_CANDIDATES);
|
||||
console.log(` healthy (${healthy.length}/${BROAD_CANDIDATES.length}): [${healthy.join(", ")}]`);
|
||||
|
||||
if (healthy.length === 0) {
|
||||
console.error("FATAL: no healthy providers — cannot run any scenarios");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// --- STEP 0: cache probe ---
|
||||
if (!onlyScenario) {
|
||||
// Run STEP 0 unconditionally unless --only is passed (targeted run skips housekeeping)
|
||||
try {
|
||||
await step0CacheProbe(healthy);
|
||||
} catch (err) {
|
||||
console.error(`\nBLOCKER: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n=== Scenarios ===\n");
|
||||
|
||||
await runScenario("priority", () => scenarioPriority(healthy));
|
||||
await runScenario("round-robin", () => scenarioRoundRobin(healthy));
|
||||
await runScenario("weighted", () => scenarioWeighted(healthy));
|
||||
await runScenario("cost-optimized", () => scenarioCostOptimized(healthy));
|
||||
await runScenario("fusion", () => scenarioFusion(healthy));
|
||||
await runScenario("auto", () => scenarioAuto());
|
||||
|
||||
// --- Scenario 7: failover (opt-in) ---
|
||||
// Only runs when --failover is passed. Uses a bogus-model primary to force a real
|
||||
// combo failover to the healthy secondary, proving the priority fallover path works
|
||||
// against the live server without stopping any service.
|
||||
if (failoverFlag) {
|
||||
console.log("\n=== Failover scenario (--failover) ===\n");
|
||||
await runScenario("failover", () => scenarioFailover(healthy));
|
||||
}
|
||||
|
||||
// --- Summary ---
|
||||
console.log("\n=== Summary ===");
|
||||
for (const { name, result } of summary) {
|
||||
console.log(` ${result.padEnd(4)} [${name}]`);
|
||||
}
|
||||
const passed = summary.filter((s) => s.result === "PASS").length;
|
||||
const skipped = summary.filter((s) => s.result === "SKIP").length;
|
||||
const failed = summary.filter((s) => s.result === "FAIL").length;
|
||||
console.log(`\n ${passed} PASS ${skipped} SKIP ${failed} FAIL`);
|
||||
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("FATAL:", err?.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user