wehub-resource-sync
433 lines
13 KiB
TypeScript
433 lines
13 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-db-health-check-"));
|
|
process.env.DATA_DIR = TEST_DATA_DIR;
|
|
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "task-303-api-key-secret";
|
|
|
|
const core = await import("../../src/lib/db/core.ts");
|
|
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
|
|
const providersDb = await import("../../src/lib/db/providers.ts");
|
|
const healthCheckDb = await import("../../src/lib/db/healthCheck.ts");
|
|
|
|
async function resetStorage() {
|
|
core.resetDbInstance();
|
|
apiKeysDb.resetApiKeyState();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
|
}
|
|
|
|
test.beforeEach(async () => {
|
|
await resetStorage();
|
|
});
|
|
|
|
test.after(async () => {
|
|
core.resetDbInstance();
|
|
apiKeysDb.resetApiKeyState();
|
|
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
|
});
|
|
|
|
function insertBrokenRows(db) {
|
|
db.prepare(
|
|
`INSERT INTO quota_snapshots
|
|
(provider, connection_id, window_key, remaining_percentage, is_exhausted, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
).run("openai", "missing-conn", "monthly", 75, 0, new Date().toISOString());
|
|
|
|
db.prepare(
|
|
`INSERT INTO quota_snapshots
|
|
(provider, connection_id, window_key, remaining_percentage, is_exhausted, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
).run("openai", "missing-conn-2", "monthly", 55, 0, "not-a-timestamp");
|
|
|
|
db.prepare(
|
|
"INSERT INTO domain_budgets (api_key_id, daily_limit_usd, monthly_limit_usd, warning_threshold) VALUES (?, ?, ?, ?)"
|
|
).run("missing-key", 10, 100, 0.8);
|
|
db.prepare("INSERT INTO domain_cost_history (api_key_id, cost, timestamp) VALUES (?, ?, ?)").run(
|
|
"missing-key",
|
|
1.5,
|
|
Date.now()
|
|
);
|
|
db.prepare("INSERT INTO domain_fallback_chains (model, chain) VALUES (?, ?)").run(
|
|
"broken-model",
|
|
"{invalid"
|
|
);
|
|
db.prepare(
|
|
"INSERT INTO domain_lockout_state (identifier, attempts, locked_until) VALUES (?, ?, ?)"
|
|
).run("broken-lockout", "{invalid", null);
|
|
db.prepare(
|
|
"INSERT INTO domain_circuit_breakers (name, state, failure_count, last_failure_time, options) VALUES (?, ?, ?, ?, ?)"
|
|
).run("broken-breaker", "OPEN", 3, Date.now(), "{invalid");
|
|
}
|
|
|
|
test("runDbHealthCheck reports issues without mutating when autoRepair is disabled", async () => {
|
|
const db = core.getDbInstance();
|
|
insertBrokenRows(db);
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, { autoRepair: false });
|
|
|
|
assert.equal(result.isHealthy, false);
|
|
assert.equal(result.repairedCount, 0);
|
|
assert.equal(result.issues.length, 6);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM quota_snapshots").get() as any).count, 2);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM domain_budgets").get() as any).count, 1);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_fallback_chains").get() as any).count,
|
|
1
|
|
);
|
|
});
|
|
|
|
test("runDbHealthCheck tolerates databases without a combos table", async () => {
|
|
const db = core.getDbInstance();
|
|
db.exec("DROP TABLE combos");
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, { autoRepair: false });
|
|
|
|
assert.equal(result.isHealthy, true);
|
|
assert.equal(
|
|
result.issues.some((issue) => issue.table === "combos"),
|
|
false
|
|
);
|
|
});
|
|
|
|
test("runDbHealthCheck auto-repairs orphan rows and invalid JSON payloads", async () => {
|
|
const db = core.getDbInstance();
|
|
insertBrokenRows(db);
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, {
|
|
autoRepair: true,
|
|
createBackupBeforeRepair: () => true,
|
|
});
|
|
|
|
assert.equal(result.isHealthy, false);
|
|
assert.equal(result.backupCreated, true);
|
|
assert.equal(result.repairedCount, 7);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM quota_snapshots").get() as any).count, 0);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM domain_budgets").get() as any).count, 0);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_cost_history").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_fallback_chains").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_lockout_state").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(
|
|
db
|
|
.prepare("SELECT options FROM domain_circuit_breakers WHERE name = ?")
|
|
.get("broken-breaker") as any
|
|
).options,
|
|
null
|
|
);
|
|
});
|
|
|
|
test("runDbHealthCheck repairs broken combo payloads, combo refs and stale connection pins", async () => {
|
|
const db = core.getDbInstance();
|
|
const now = new Date().toISOString();
|
|
const activeConnection = await providersDb.createProviderConnection({
|
|
provider: "openai",
|
|
authType: "apikey",
|
|
name: "Healthy Connection",
|
|
apiKey: "sk-healthy",
|
|
});
|
|
db.prepare(
|
|
"INSERT INTO combos (id, name, data, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
|
|
).run(
|
|
"combo-child",
|
|
"combo-child",
|
|
JSON.stringify({
|
|
id: "combo-child",
|
|
name: "combo-child",
|
|
strategy: "priority",
|
|
models: ["openai/gpt-4o-mini"],
|
|
config: {},
|
|
isActive: true,
|
|
createdAt: now,
|
|
updatedAt: now,
|
|
}),
|
|
0,
|
|
now,
|
|
now
|
|
);
|
|
|
|
db.prepare(
|
|
"INSERT INTO combos (id, name, data, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
|
|
).run("combo-invalid", "combo-invalid", "{invalid", 1, now, now);
|
|
|
|
db.prepare(
|
|
"INSERT INTO combos (id, name, data, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
|
|
).run(
|
|
"combo-broken",
|
|
"combo-broken",
|
|
JSON.stringify({
|
|
id: "combo-broken",
|
|
name: "combo-broken",
|
|
strategy: "priority",
|
|
models: [
|
|
"combo-child",
|
|
"combo-broken",
|
|
{ id: "ref-missing", kind: "combo-ref", comboName: "missing-child", weight: 0 },
|
|
{
|
|
id: "model-pinned",
|
|
kind: "model",
|
|
providerId: "openai",
|
|
model: "openai/gpt-4o-mini",
|
|
connectionId: "missing-connection",
|
|
weight: 0,
|
|
},
|
|
{
|
|
id: "model-healthy",
|
|
kind: "model",
|
|
providerId: "openai",
|
|
model: "openai/gpt-4o-mini",
|
|
connectionId: activeConnection.id,
|
|
weight: 0,
|
|
},
|
|
],
|
|
config: {},
|
|
isActive: true,
|
|
createdAt: now,
|
|
updatedAt: now,
|
|
}),
|
|
2,
|
|
now,
|
|
now
|
|
);
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, {
|
|
autoRepair: true,
|
|
createBackupBeforeRepair: () => false,
|
|
});
|
|
const invalidCombo = JSON.parse(
|
|
(db.prepare("SELECT data FROM combos WHERE id = ?").get("combo-invalid") as any).data
|
|
);
|
|
const repairedCombo = JSON.parse(
|
|
(db.prepare("SELECT data FROM combos WHERE id = ?").get("combo-broken") as any).data
|
|
);
|
|
|
|
assert.equal(
|
|
result.issues.some((issue) => issue.table === "combos"),
|
|
true
|
|
);
|
|
assert.equal(result.repairedCount, 5);
|
|
assert.equal(invalidCombo.isActive, false);
|
|
assert.match(invalidCombo.repairNote, /invalid JSON/i);
|
|
assert.equal(repairedCombo.models.length, 3);
|
|
assert.equal(
|
|
repairedCombo.models.some(
|
|
(step) => step.kind === "combo-ref" && step.comboName === "combo-child"
|
|
),
|
|
true
|
|
);
|
|
assert.equal("connectionId" in repairedCombo.models[1], false);
|
|
assert.equal(repairedCombo.models[2].connectionId, activeConnection.id);
|
|
assert.match(repairedCombo.repairNote, /broken combo step/i);
|
|
assert.match(repairedCombo.repairNote, /legacy combo ref/i);
|
|
});
|
|
|
|
test("runDbHealthCheck diagnosis does not request backups for combo-only issues", async () => {
|
|
const db = core.getDbInstance();
|
|
const now = new Date().toISOString();
|
|
let backupAttempts = 0;
|
|
|
|
db.prepare(
|
|
"INSERT INTO combos (id, name, data, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
|
|
).run(
|
|
"combo-broken",
|
|
"combo-broken",
|
|
JSON.stringify({
|
|
id: "combo-broken",
|
|
name: "combo-broken",
|
|
strategy: "priority",
|
|
models: [{ id: "ref-missing", kind: "combo-ref", comboName: "missing-child", weight: 0 }],
|
|
config: {},
|
|
isActive: true,
|
|
createdAt: now,
|
|
updatedAt: now,
|
|
}),
|
|
1,
|
|
now,
|
|
now
|
|
);
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, {
|
|
autoRepair: false,
|
|
createBackupBeforeRepair: () => {
|
|
backupAttempts += 1;
|
|
return true;
|
|
},
|
|
});
|
|
|
|
assert.equal(result.repairedCount, 0);
|
|
assert.equal(result.backupCreated, false);
|
|
assert.equal(backupAttempts, 0);
|
|
});
|
|
|
|
test("getDbInstance can auto-repair persisted broken rows when startup repair is forced", async () => {
|
|
let db = core.getDbInstance();
|
|
insertBrokenRows(db);
|
|
core.resetDbInstance();
|
|
|
|
const previousForce = process.env.OMNIROUTE_FORCE_DB_HEALTHCHECK;
|
|
process.env.OMNIROUTE_FORCE_DB_HEALTHCHECK = "1";
|
|
try {
|
|
db = core.getDbInstance();
|
|
} finally {
|
|
if (previousForce === undefined) {
|
|
delete process.env.OMNIROUTE_FORCE_DB_HEALTHCHECK;
|
|
} else {
|
|
process.env.OMNIROUTE_FORCE_DB_HEALTHCHECK = previousForce;
|
|
}
|
|
}
|
|
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM quota_snapshots").get() as any).count, 0);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM domain_budgets").get() as any).count, 0);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_cost_history").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_fallback_chains").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_lockout_state").get() as any).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(
|
|
db
|
|
.prepare("SELECT options FROM domain_circuit_breakers WHERE name = ?")
|
|
.get("broken-breaker") as any
|
|
).options,
|
|
null
|
|
);
|
|
});
|
|
|
|
test("getDbInstance skips automatic startup repair during tests unless forced", async () => {
|
|
let db = core.getDbInstance();
|
|
insertBrokenRows(db);
|
|
core.resetDbInstance();
|
|
|
|
db = core.getDbInstance();
|
|
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM quota_snapshots").get() as any).count, 2);
|
|
assert.equal((db.prepare("SELECT COUNT(*) AS count FROM domain_budgets").get() as any).count, 1);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_cost_history").get() as any).count,
|
|
1
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_fallback_chains").get() as any).count,
|
|
1
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT COUNT(*) AS count FROM domain_lockout_state").get() as any).count,
|
|
1
|
|
);
|
|
});
|
|
|
|
test("runDbHealthCheck repairs a drifted db_meta schema version", async () => {
|
|
const db = core.getDbInstance();
|
|
db.prepare("UPDATE db_meta SET value = ? WHERE key = 'schema_version'").run("0");
|
|
|
|
const result = healthCheckDb.runDbHealthCheck(db, {
|
|
autoRepair: true,
|
|
createBackupBeforeRepair: () => false,
|
|
});
|
|
|
|
assert.equal(
|
|
result.issues.some((issue) => issue.table === "db_meta"),
|
|
true
|
|
);
|
|
assert.equal(
|
|
(db.prepare("SELECT value FROM db_meta WHERE key = 'schema_version'").get() as any).value,
|
|
"1"
|
|
);
|
|
});
|
|
|
|
test("deleteApiKey removes domain budget and cost history rows for that key", async () => {
|
|
const created = await apiKeysDb.createApiKey("Cleanup Key", "machine-health");
|
|
const db = core.getDbInstance();
|
|
|
|
db.prepare(
|
|
"INSERT INTO domain_budgets (api_key_id, daily_limit_usd, monthly_limit_usd, warning_threshold) VALUES (?, ?, ?, ?)"
|
|
).run(created.id, 5, 50, 0.9);
|
|
db.prepare("INSERT INTO domain_cost_history (api_key_id, cost, timestamp) VALUES (?, ?, ?)").run(
|
|
created.id,
|
|
0.5,
|
|
Date.now()
|
|
);
|
|
|
|
assert.equal(await apiKeysDb.deleteApiKey(created.id), true);
|
|
assert.equal(
|
|
(
|
|
db
|
|
.prepare("SELECT COUNT(*) AS count FROM domain_budgets WHERE api_key_id = ?")
|
|
.get(created.id) as any
|
|
).count,
|
|
0
|
|
);
|
|
assert.equal(
|
|
(
|
|
db
|
|
.prepare("SELECT COUNT(*) AS count FROM domain_cost_history WHERE api_key_id = ?")
|
|
.get((created as any).id) as any
|
|
).count,
|
|
0
|
|
);
|
|
});
|
|
|
|
test("deleteProviderConnection and bulk delete remove related quota snapshots", async () => {
|
|
const first = await providersDb.createProviderConnection({
|
|
provider: "openai",
|
|
authType: "apikey",
|
|
name: "Primary",
|
|
apiKey: "sk-primary",
|
|
});
|
|
const second = await providersDb.createProviderConnection({
|
|
provider: "openai",
|
|
authType: "apikey",
|
|
name: "Secondary",
|
|
apiKey: "sk-secondary",
|
|
});
|
|
const db = core.getDbInstance();
|
|
|
|
db.prepare(
|
|
`INSERT INTO quota_snapshots
|
|
(provider, connection_id, window_key, remaining_percentage, is_exhausted, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
).run("openai", first.id, "monthly", 80, 0, new Date().toISOString());
|
|
db.prepare(
|
|
`INSERT INTO quota_snapshots
|
|
(provider, connection_id, window_key, remaining_percentage, is_exhausted, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
).run("openai", second.id, "monthly", 40, 0, new Date().toISOString());
|
|
|
|
assert.equal(await providersDb.deleteProviderConnection((first as any).id), true);
|
|
assert.equal(
|
|
(
|
|
db.prepare("SELECT COUNT(*) AS count FROM quota_snapshots WHERE connection_id = ?") as any
|
|
).get(first.id).count,
|
|
0
|
|
);
|
|
|
|
await providersDb.deleteProviderConnectionsByProvider("openai");
|
|
assert.equal(
|
|
(
|
|
db
|
|
.prepare("SELECT COUNT(*) AS count FROM quota_snapshots WHERE connection_id = ?")
|
|
.get(second.id) as any
|
|
).count,
|
|
0
|
|
);
|
|
});
|