wehub-resource-sync
444 lines
14 KiB
TypeScript
444 lines
14 KiB
TypeScript
/**
|
|
* Unit tests for src/lib/db/modelIntelligence.ts
|
|
*
|
|
* Uses the project's own DB infrastructure (core.ts getDbInstance)
|
|
* with a temp DATA_DIR. Follows the Node.js native test runner pattern.
|
|
*/
|
|
|
|
import { describe, it, beforeEach } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(
|
|
path.join(os.tmpdir(), "omniroute-mi-test-"),
|
|
);
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const mi = await import("../../src/lib/db/modelIntelligence.ts");
|
|
|
|
function resetStorage(): void {
|
|
core.resetDbInstance();
|
|
try {
|
|
if (fs.existsSync(TEST_DATA_DIR)) {
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
}
|
|
} catch { /* EBUSY — ignore */ }
|
|
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
|
}
|
|
|
|
function insertEntry(
|
|
model: string,
|
|
source: string,
|
|
category: string,
|
|
score: number,
|
|
opts: {
|
|
eloRaw?: number | null;
|
|
confidence?: string | null;
|
|
expiresAt?: string | null;
|
|
} = {},
|
|
): void {
|
|
const db = core.getDbInstance();
|
|
db.prepare(
|
|
`INSERT OR REPLACE INTO model_intelligence
|
|
(model, source, category, score, elo_raw, confidence, synced_at, expires_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)`,
|
|
).run(
|
|
model,
|
|
source,
|
|
category,
|
|
score,
|
|
opts.eloRaw ?? null,
|
|
opts.confidence ?? null,
|
|
opts.expiresAt ?? null,
|
|
);
|
|
}
|
|
|
|
// ─── Tests ───────────────────────────────────────────────
|
|
|
|
describe("upsertModelIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("inserts a new entry", () => {
|
|
mi.upsertModelIntelligence({
|
|
model: "claude-sonnet",
|
|
source: "arena_elo",
|
|
category: "coding",
|
|
score: 0.92,
|
|
eloRaw: 1350,
|
|
confidence: "high",
|
|
expiresAt: null,
|
|
});
|
|
|
|
const entry = mi.getModelIntelligenceBySource("claude-sonnet", "arena_elo", "coding");
|
|
assert.ok(entry, "Row should exist after upsert");
|
|
assert.strictEqual(entry.model, "claude-sonnet");
|
|
assert.strictEqual(entry.source, "arena_elo");
|
|
assert.strictEqual(entry.category, "coding");
|
|
assert.strictEqual(entry.score, 0.92);
|
|
assert.strictEqual(entry.eloRaw, 1350);
|
|
assert.strictEqual(entry.confidence, "high");
|
|
assert.ok(entry.syncedAt, "synced_at should be auto-populated");
|
|
});
|
|
|
|
it("updates an existing entry (INSERT OR REPLACE)", () => {
|
|
insertEntry("gpt-4o", "arena_elo", "coding", 0.85);
|
|
|
|
mi.upsertModelIntelligence({
|
|
model: "gpt-4o",
|
|
source: "arena_elo",
|
|
category: "coding",
|
|
score: 0.90,
|
|
eloRaw: 1400,
|
|
confidence: "high",
|
|
expiresAt: null,
|
|
});
|
|
|
|
const entry = mi.getModelIntelligenceBySource("gpt-4o", "arena_elo", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.score, 0.90);
|
|
assert.strictEqual(entry.eloRaw, 1400);
|
|
});
|
|
});
|
|
|
|
describe("getModelIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("returns user_override when all three sources exist (highest priority)", () => {
|
|
insertEntry("claude-sonnet", "models_dev_tier", "coding", 0.75);
|
|
insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
|
|
insertEntry("claude-sonnet", "user_override", "coding", 0.99);
|
|
|
|
const entry = mi.getModelIntelligence("claude-sonnet", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.source, "user_override");
|
|
assert.strictEqual(entry.score, 0.99);
|
|
});
|
|
|
|
it("returns arena_elo when no user_override exists", () => {
|
|
insertEntry("gpt-4o", "arena_elo", "coding", 0.87);
|
|
insertEntry("gpt-4o", "models_dev_tier", "coding", 0.70);
|
|
|
|
const entry = mi.getModelIntelligence("gpt-4o", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.source, "arena_elo");
|
|
assert.strictEqual(entry.score, 0.87);
|
|
});
|
|
|
|
it("returns models_dev_tier when only that source exists", () => {
|
|
insertEntry("llama-4", "models_dev_tier", "coding", 0.65);
|
|
|
|
const entry = mi.getModelIntelligence("llama-4", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.source, "models_dev_tier");
|
|
assert.strictEqual(entry.score, 0.65);
|
|
});
|
|
|
|
it("returns null when no entry exists", () => {
|
|
const entry = mi.getModelIntelligence("nonexistent-model", "coding");
|
|
assert.strictEqual(entry, null);
|
|
});
|
|
|
|
it("skips expired entries and falls through to next source", () => {
|
|
insertEntry("gemini-pro", "arena_elo", "coding", 0.82, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
insertEntry("gemini-pro", "models_dev_tier", "coding", 0.70);
|
|
|
|
const entry = mi.getModelIntelligence("gemini-pro", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.source, "models_dev_tier");
|
|
});
|
|
|
|
it("returns null when all entries for a model+category are expired", () => {
|
|
insertEntry("expired-model", "arena_elo", "coding", 0.80, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
|
|
const entry = mi.getModelIntelligence("expired-model", "coding");
|
|
assert.strictEqual(entry, null);
|
|
});
|
|
|
|
it("model names require exact match (case-sensitive in DB)", () => {
|
|
insertEntry("Claude-Sonnet", "arena_elo", "coding", 0.90);
|
|
|
|
const exact = mi.getModelIntelligence("Claude-Sonnet", "coding");
|
|
assert.ok(exact);
|
|
|
|
const different = mi.getModelIntelligence("claude-sonnet", "coding");
|
|
assert.strictEqual(different, null);
|
|
});
|
|
});
|
|
|
|
describe("getModelIntelligenceBySource", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("returns a specific source entry", () => {
|
|
insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
|
|
insertEntry("claude-sonnet", "user_override", "coding", 0.99);
|
|
|
|
const entry = mi.getModelIntelligenceBySource("claude-sonnet", "arena_elo", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.source, "arena_elo");
|
|
assert.strictEqual(entry.score, 0.88);
|
|
});
|
|
|
|
it("returns null when the specific source does not exist", () => {
|
|
insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
|
|
|
|
const entry = mi.getModelIntelligenceBySource("claude-sonnet", "user_override", "coding");
|
|
assert.strictEqual(entry, null);
|
|
});
|
|
|
|
it("returns null when no entries exist at all", () => {
|
|
const entry = mi.getModelIntelligenceBySource("nonexistent", "arena_elo", "coding");
|
|
assert.strictEqual(entry, null);
|
|
});
|
|
});
|
|
|
|
describe("deleteModelIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("deletes an entry and returns true", () => {
|
|
insertEntry("gpt-4o", "arena_elo", "coding", 0.87);
|
|
|
|
const deleted = mi.deleteModelIntelligence("gpt-4o", "arena_elo", "coding");
|
|
assert.strictEqual(deleted, true);
|
|
|
|
const entry = mi.getModelIntelligenceBySource("gpt-4o", "arena_elo", "coding");
|
|
assert.strictEqual(entry, null);
|
|
});
|
|
|
|
it("returns false when entry does not exist", () => {
|
|
const deleted = mi.deleteModelIntelligence("nonexistent", "arena_elo", "coding");
|
|
assert.strictEqual(deleted, false);
|
|
});
|
|
});
|
|
|
|
describe("deleteExpiredIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("deletes only expired entries leaving valid ones", () => {
|
|
insertEntry("old-model", "arena_elo", "coding", 0.7, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
insertEntry("fresh-model", "arena_elo", "coding", 0.9, {
|
|
expiresAt: "2099-12-31T23:59:59Z",
|
|
});
|
|
insertEntry("permanent-model", "user_override", "coding", 0.95);
|
|
|
|
const deleted = mi.deleteExpiredIntelligence();
|
|
assert.strictEqual(deleted, 1);
|
|
|
|
const remaining = mi.listModelIntelligence();
|
|
assert.strictEqual(remaining.length, 2);
|
|
});
|
|
|
|
it("deletes expired entries for a specific source only", () => {
|
|
insertEntry("old-arena", "arena_elo", "coding", 0.7, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
insertEntry("old-tier", "models_dev_tier", "coding", 0.5, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
|
|
const deleted = mi.deleteExpiredIntelligence("arena_elo");
|
|
assert.strictEqual(deleted, 1);
|
|
|
|
const remaining = mi.listModelIntelligence();
|
|
assert.strictEqual(remaining.length, 1);
|
|
assert.strictEqual(remaining[0].source, "models_dev_tier");
|
|
});
|
|
|
|
it("returns 0 when no expired entries exist", () => {
|
|
insertEntry("fresh-model", "arena_elo", "coding", 0.9, {
|
|
expiresAt: "2099-12-31T23:59:59Z",
|
|
});
|
|
|
|
const deleted = mi.deleteExpiredIntelligence();
|
|
assert.strictEqual(deleted, 0);
|
|
});
|
|
});
|
|
|
|
describe("listModelIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("lists all entries when no filters provided", () => {
|
|
insertEntry("model-a", "arena_elo", "coding", 0.8);
|
|
insertEntry("model-b", "arena_elo", "review", 0.75);
|
|
insertEntry("model-c", "user_override", "coding", 0.95);
|
|
|
|
const entries = mi.listModelIntelligence();
|
|
assert.strictEqual(entries.length, 3);
|
|
});
|
|
|
|
it("filters by source", () => {
|
|
insertEntry("model-a", "arena_elo", "coding", 0.8);
|
|
insertEntry("model-b", "user_override", "coding", 0.95);
|
|
insertEntry("model-c", "models_dev_tier", "coding", 0.6);
|
|
|
|
const entries = mi.listModelIntelligence({ source: "arena_elo" });
|
|
assert.strictEqual(entries.length, 1);
|
|
assert.strictEqual(entries[0].source, "arena_elo");
|
|
});
|
|
|
|
it("filters by category", () => {
|
|
insertEntry("model-a", "arena_elo", "coding", 0.8);
|
|
insertEntry("model-b", "arena_elo", "review", 0.75);
|
|
insertEntry("model-c", "arena_elo", "documentation", 0.7);
|
|
|
|
const entries = mi.listModelIntelligence({ category: "review" });
|
|
assert.strictEqual(entries.length, 1);
|
|
assert.strictEqual(entries[0].category, "review");
|
|
});
|
|
|
|
it("filters by both source and category", () => {
|
|
insertEntry("model-a", "arena_elo", "coding", 0.8);
|
|
insertEntry("model-b", "arena_elo", "review", 0.75);
|
|
insertEntry("model-c", "user_override", "coding", 0.95);
|
|
|
|
const entries = mi.listModelIntelligence({ source: "arena_elo", category: "coding" });
|
|
assert.strictEqual(entries.length, 1);
|
|
assert.strictEqual(entries[0].model, "model-a");
|
|
});
|
|
|
|
it("returns empty array for empty table", () => {
|
|
const entries = mi.listModelIntelligence();
|
|
assert.ok(Array.isArray(entries));
|
|
assert.strictEqual(entries.length, 0);
|
|
});
|
|
});
|
|
|
|
describe("bulkUpsertModelIntelligence", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("bulk inserts multiple entries", () => {
|
|
const count = mi.bulkUpsertModelIntelligence([
|
|
{ model: "model-a", source: "arena_elo", category: "coding", score: 0.80, eloRaw: 1300, confidence: "high", expiresAt: "2099-12-31T23:59:59Z" },
|
|
{ model: "model-b", source: "arena_elo", category: "coding", score: 0.70, eloRaw: 1200, confidence: "medium", expiresAt: "2099-12-31T23:59:59Z" },
|
|
{ model: "model-c", source: "arena_elo", category: "review", score: 0.85, eloRaw: 1350, confidence: "high", expiresAt: "2099-12-31T23:59:59Z" },
|
|
]);
|
|
|
|
assert.strictEqual(count, 3);
|
|
const entries = mi.listModelIntelligence();
|
|
assert.strictEqual(entries.length, 3);
|
|
});
|
|
|
|
it("returns 0 for empty input array", () => {
|
|
const count = mi.bulkUpsertModelIntelligence([]);
|
|
assert.strictEqual(count, 0);
|
|
});
|
|
|
|
it("replaces existing entries on conflict (INSERT OR REPLACE)", () => {
|
|
insertEntry("model-a", "arena_elo", "coding", 0.70);
|
|
|
|
mi.bulkUpsertModelIntelligence([
|
|
{ model: "model-a", source: "arena_elo", category: "coding", score: 0.90, eloRaw: 1450, confidence: "high", expiresAt: null },
|
|
]);
|
|
|
|
const entry = mi.getModelIntelligenceBySource("model-a", "arena_elo", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.score, 0.90);
|
|
assert.strictEqual(entry.eloRaw, 1450);
|
|
});
|
|
});
|
|
|
|
describe("getResolvedTaskFitness", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("returns user_override score when all sources exist", () => {
|
|
insertEntry("claude-sonnet", "models_dev_tier", "coding", 0.75);
|
|
insertEntry("claude-sonnet", "arena_elo", "coding", 0.88);
|
|
insertEntry("claude-sonnet", "user_override", "coding", 0.99);
|
|
|
|
const score = mi.getResolvedTaskFitness("claude-sonnet", "coding");
|
|
assert.strictEqual(score, 0.99);
|
|
});
|
|
|
|
it("returns arena_elo score when no user_override exists", () => {
|
|
insertEntry("gpt-4o", "arena_elo", "coding", 0.87);
|
|
insertEntry("gpt-4o", "models_dev_tier", "coding", 0.70);
|
|
|
|
const score = mi.getResolvedTaskFitness("gpt-4o", "coding");
|
|
assert.strictEqual(score, 0.87);
|
|
});
|
|
|
|
it("returns models_dev_tier score when only that source exists", () => {
|
|
insertEntry("llama-4", "models_dev_tier", "coding", 0.65);
|
|
|
|
const score = mi.getResolvedTaskFitness("llama-4", "coding");
|
|
assert.strictEqual(score, 0.65);
|
|
});
|
|
|
|
it("returns null when nothing exists", () => {
|
|
const score = mi.getResolvedTaskFitness("nonexistent", "coding");
|
|
assert.strictEqual(score, null);
|
|
});
|
|
|
|
it("skips expired entries in the resolution chain", () => {
|
|
insertEntry("gemini-pro", "arena_elo", "coding", 0.82, {
|
|
expiresAt: "2000-01-01T00:00:00Z",
|
|
});
|
|
insertEntry("gemini-pro", "models_dev_tier", "coding", 0.70);
|
|
|
|
const score = mi.getResolvedTaskFitness("gemini-pro", "coding");
|
|
assert.strictEqual(score, 0.70);
|
|
});
|
|
});
|
|
|
|
describe("edge cases", () => {
|
|
beforeEach(() => { resetStorage(); });
|
|
|
|
it("score values are stored and retrieved with float precision", () => {
|
|
mi.upsertModelIntelligence({
|
|
model: "precise-model",
|
|
source: "arena_elo",
|
|
category: "coding",
|
|
score: 0.1234,
|
|
eloRaw: null,
|
|
confidence: null,
|
|
expiresAt: null,
|
|
});
|
|
|
|
const entry = mi.getModelIntelligence("precise-model", "coding");
|
|
assert.ok(entry);
|
|
assert.ok(Math.abs(entry.score - 0.1234) < 0.0001);
|
|
});
|
|
|
|
it("null eloRaw and confidence are handled correctly", () => {
|
|
mi.upsertModelIntelligence({
|
|
model: "minimal-model",
|
|
source: "models_dev_tier",
|
|
category: "default",
|
|
score: 0.6,
|
|
eloRaw: null,
|
|
confidence: null,
|
|
expiresAt: null,
|
|
});
|
|
|
|
const entry = mi.getModelIntelligence("minimal-model", "default");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.eloRaw, null);
|
|
assert.strictEqual(entry.confidence, null);
|
|
});
|
|
|
|
it("NULL expires_at means never expires", () => {
|
|
mi.upsertModelIntelligence({
|
|
model: "permanent-model",
|
|
source: "user_override",
|
|
category: "coding",
|
|
score: 0.95,
|
|
eloRaw: null,
|
|
confidence: null,
|
|
expiresAt: null,
|
|
});
|
|
|
|
const entry = mi.getModelIntelligence("permanent-model", "coding");
|
|
assert.ok(entry);
|
|
assert.strictEqual(entry.expiresAt, null);
|
|
assert.strictEqual(entry.score, 0.95);
|
|
});
|
|
});
|