diegosouzapw--omniroute/tests/unit/modelsDevSync-extended.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";

const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-models-dev-"));
process.env.DATA_DIR = TEST_DATA_DIR;

const core = await import("../../src/lib/db/core.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");

const modulePath = path.join(process.cwd(), "src/lib/modelsDevSync.ts");
const originalFetch = globalThis.fetch;
const originalEnv = { ...process.env };
const loadedModules = new Set();

const MOCK_MODELS_DEV_DATA = {
  openai: {
    id: "openai",
    models: {
      "gpt-4o": {
        id: "gpt-4o",
        name: "GPT-4o",
        family: "gpt-4",
        attachment: true,
        reasoning: false,
        tool_call: true,
        structured_output: true,
        temperature: true,
        knowledge: "2024-10",
        release_date: "2024-05-13",
        last_updated: "2024-10-01",
        open_weights: false,
        cost: {
          input: 2.5,
          output: 10,
          cache_read: 1.25,
          cache_write: 2.5,
        },
        limit: {
          context: 128000,
          input: 128000,
          output: 16384,
        },
        modalities: {
          input: ["text", "image"],
          output: ["text"],
        },
      },
    },
  },
  anthropic: {
    id: "anthropic",
    models: {
      "claude-sonnet-4-20250514": {
        id: "claude-sonnet-4-20250514",
        name: "Claude Sonnet 4",
        tool_call: true,
        reasoning: false,
        attachment: true,
        structured_output: true,
        temperature: true,
        release_date: "2025-05-14",
        last_updated: "2025-05-14",
        open_weights: false,
        cost: {
          input: 3,
          output: 15,
          cache_read: 0.3,
        },
        limit: {
          context: 200000,
          output: 64000,
        },
        modalities: {
          input: ["text", "image"],
          output: ["text"],
        },
      },
    },
  },
};

async function importFresh(label) {
  const mod = await import(
    `${pathToFileURL(modulePath).href}?case=${label}-${Date.now()}-${Math.random()}`
  );
  loadedModules.add(mod);
  return mod;
}

function restoreEnv() {
  for (const key of Object.keys(process.env)) {
    if (!(key in originalEnv)) delete process.env[key];
  }
  Object.assign(process.env, originalEnv);
  process.env.DATA_DIR = TEST_DATA_DIR;
}

function resetStorage() {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}

async function waitFor(predicate, timeoutMs = 200) {
  const start = Date.now();
  while (Date.now() - start < timeoutMs) {
    const value = predicate();
    if (value) return value;
    await new Promise((resolve) => setTimeout(resolve, 10));
  }
  return null;
}

function mockFetchWith(body, status = 200, statusText = "OK") {
  globalThis.fetch = async () =>
    new Response(typeof body === "string" ? body : JSON.stringify(body), {
      status,
      statusText,
      headers: { "content-type": "application/json" },
    });
}

test.beforeEach(async () => {
  resetStorage();
});

test.afterEach(async () => {
  for (const mod of loadedModules) {
    if (typeof (mod as any).stopPeriodicSync === "function") {
      (mod as any).stopPeriodicSync();
    }
  }
  loadedModules.clear();
  globalThis.fetch = originalFetch;
  restoreEnv();
  core.resetDbInstance();
});

test.after(async () => {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
});

test("fetchModelsDev caches successful responses and rejects invalid JSON or non-ok responses", async () => {
  const modelsDev = await importFresh("fetch-cache");
  let calls = 0;
  globalThis.fetch = async () => {
    calls += 1;
    return new Response(JSON.stringify(MOCK_MODELS_DEV_DATA), {
      status: 200,
      headers: { "content-type": "application/json" },
    });
  };

  const first = await modelsDev.fetchModelsDev();
  const second = await modelsDev.fetchModelsDev();

  assert.strictEqual(first, second);
  assert.equal(calls, 1);

  const invalid = await importFresh("fetch-invalid-json");
  mockFetchWith("not-json");
  await assert.rejects(() => invalid.fetchModelsDev(), /invalid JSON/);

  const nonOk = await importFresh("fetch-non-ok");
  mockFetchWith({ error: "boom" }, 503, "Service Unavailable");
  await assert.rejects(() => nonOk.fetchModelsDev(), /models\.dev fetch failed \[503\]/);
});

test("modelsDev interval falls back to the default when env values are invalid or non-positive", async () => {
  process.env.MODELS_DEV_SYNC_INTERVAL = "0";
  const zeroInterval = await importFresh("interval-zero");
  zeroInterval.startPeriodicSync();
  assert.equal(zeroInterval.getSyncStatus().intervalMs, 86400 * 1000);
  zeroInterval.stopPeriodicSync();

  process.env.MODELS_DEV_SYNC_INTERVAL = "not-a-number";
  const invalidInterval = await importFresh("interval-invalid");
  invalidInterval.startPeriodicSync();
  assert.equal(invalidInterval.getSyncStatus().intervalMs, 86400 * 1000);
  invalidInterval.stopPeriodicSync();
});

test("transform helpers skip incomplete pricing entries and preserve partial capability defaults", async () => {
  const modelsDev = await importFresh("transform-edge-cases");
  const raw = {
    sparse: {
      id: "sparse",
      models: {
        "missing-cost": {
          id: "missing-cost",
          name: "Missing Cost",
        },
        "missing-input": {
          id: "missing-input",
          name: "Missing Input",
          cost: { output: 4.2 },
        },
        complete: {
          id: "complete",
          name: "Complete",
          cost: { input: 1.5 },
          interleaved: { field: "" },
        },
      },
    },
    nomodels: {
      id: "nomodels",
    },
  };

  const pricing = modelsDev.transformModelsDevToPricing(raw);
  assert.deepEqual(pricing.sparse, {
    complete: {
      input: 1.5,
      output: 0,
    },
  });
  assert.equal(pricing.nomodels, undefined);

  const capabilities = modelsDev.transformModelsDevToCapabilities(raw);
  assert.equal(capabilities.sparse.complete.tool_call, null);
  assert.equal(capabilities.sparse.complete.reasoning, null);
  assert.equal(capabilities.sparse.complete.attachment, null);
  assert.equal(capabilities.sparse.complete.structured_output, null);
  assert.equal(capabilities.sparse.complete.temperature, null);
  assert.equal(capabilities.sparse.complete.modalities_input, "[]");
  assert.equal(capabilities.sparse.complete.modalities_output, "[]");
  assert.equal(capabilities.sparse.complete.limit_context, null);
  assert.equal(capabilities.sparse.complete.limit_input, null);
  assert.equal(capabilities.sparse.complete.limit_output, null);
  assert.equal(capabilities.sparse.complete.interleaved_field, null);
  assert.equal(capabilities.nomodels, undefined);
});

test("modelsDev pricing helpers persist records, skip corrupted rows, and clear the namespace", async () => {
  const modelsDev = await importFresh("pricing-storage");
  const pricing = modelsDev.transformModelsDevToPricing(MOCK_MODELS_DEV_DATA);

  modelsDev.saveModelsDevPricing(pricing);
  const saved = modelsDev.getModelsDevPricing();
  assert.equal(saved.openai["gpt-4o"].input, 2.5);
  assert.equal(saved.cx["gpt-4o"].cache_creation, 2.5);

  const db = core.getDbInstance();
  db.prepare("INSERT INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
    "models_dev_pricing",
    "corrupted",
    "{oops"
  );

  const withCorruption = modelsDev.getModelsDevPricing();
  assert.equal(withCorruption.corrupted, undefined);

  modelsDev.clearModelsDevPricing();
  assert.deepEqual(modelsDev.getModelsDevPricing(), {});
});

test("modelsDev capabilities helpers create the table, persist rows, filter by provider/model, and expose context limits", async () => {
  const modelsDev = await importFresh("capabilities-storage");
  const capabilities = modelsDev.transformModelsDevToCapabilities(MOCK_MODELS_DEV_DATA);

  modelsDev.ensureCapabilitiesTable();
  modelsDev.saveModelsDevCapabilities(capabilities);

  const allCaps = modelsDev.getSyncedCapabilities();
  const openaiOnly = modelsDev.getSyncedCapabilities("openai", "gpt-4o");

  assert.equal(allCaps.openai["gpt-4o"].tool_call, true);
  assert.equal(allCaps.anthropic["claude-sonnet-4-20250514"].attachment, true);
  assert.deepEqual(Object.keys(openaiOnly), ["openai"]);
  assert.equal(openaiOnly.openai["gpt-4o"].limit_context, 128000);
  assert.equal("getModelContextLimit" in modelsDev, false);

  modelsDev.clearModelsDevCapabilities();
  assert.deepEqual(modelsDev.getSyncedCapabilities(), {});
});

test("modelsDev capability helpers coerce false/null values and ignore malformed rows", async () => {
  const modelsDev = await importFresh("capabilities-malformed");
  const db = core.getDbInstance();
  const originalPrepare = db.prepare.bind(db);
  db.prepare = (sql) => {
    if (String(sql).includes("SELECT * FROM model_capabilities")) {
      return {
        all: () => [
          123,
          { provider: null, model_id: "missing-provider" },
          {
            provider: "openai",
            model_id: "coerced-model",
            tool_call: 0,
            reasoning: null,
            attachment: 0,
            structured_output: 0,
            temperature: 0,
            modalities_input: null,
            modalities_output: 42,
            knowledge_cutoff: 77,
            release_date: 88,
            last_updated: 99,
            status: 123,
            family: 456,
            open_weights: null,
            limit_context: "bad",
            limit_input: 4096,
            limit_output: "nope",
            interleaved_field: 321,
          },
        ],
      };
    }
    return originalPrepare(sql);
  };

  try {
    const openai = modelsDev.getSyncedCapabilities("openai");
    assert.deepEqual(openai.openai["coerced-model"], {
      tool_call: false,
      reasoning: null,
      attachment: false,
      structured_output: false,
      temperature: false,
      modalities_input: "[]",
      modalities_output: "[]",
      knowledge_cutoff: null,
      release_date: null,
      last_updated: null,
      status: null,
      family: null,
      open_weights: null,
      limit_context: null,
      limit_input: 4096,
      limit_output: null,
      interleaved_field: null,
    });

    const all = modelsDev.getSyncedCapabilities();
    assert.equal(all["7"], undefined);
    assert.equal(all.openai["missing-provider"], undefined);
  } finally {
    db.prepare = originalPrepare;
  }
});

test("modelsDev pricing helpers ignore malformed sqlite rows without crashing", async () => {
  const modelsDev = await importFresh("pricing-malformed");
  const db = core.getDbInstance();
  const originalPrepare = db.prepare.bind(db);
  db.prepare = (sql) => {
    if (String(sql).includes("SELECT key, value FROM key_value")) {
      return {
        all: () => [
          123,
          { key: 123, value: JSON.stringify({ ignored: true }) },
          { key: "missing-value", value: 456 },
          { key: "broken", value: "{oops" },
          { key: "openai", value: JSON.stringify({ "gpt-4o": { input: 2.5, output: 10 } }) },
        ],
      };
    }
    return originalPrepare(sql);
  };

  try {
    assert.deepEqual(modelsDev.getModelsDevPricing(), {
      openai: {
        "gpt-4o": {
          input: 2.5,
          output: 10,
        },
      },
    });
  } finally {
    db.prepare = originalPrepare;
  }
});

test("saveModelsDevCapabilities round-trips false and null booleans", async () => {
  const modelsDev = await importFresh("capabilities-roundtrip-falsey");
  modelsDev.saveModelsDevCapabilities({
    openai: {
      "gpt-falsey": {
        tool_call: false,
        reasoning: null,
        attachment: false,
        structured_output: false,
        temperature: null,
        modalities_input: "[]",
        modalities_output: '["text"]',
        knowledge_cutoff: null,
        release_date: null,
        last_updated: null,
        status: null,
        family: null,
        open_weights: false,
        limit_context: null,
        limit_input: null,
        limit_output: 1024,
        interleaved_field: null,
      },
    },
  });

  assert.deepEqual(modelsDev.getSyncedCapabilities("openai", "gpt-falsey"), {
    openai: {
      "gpt-falsey": {
        tool_call: false,
        reasoning: null,
        attachment: false,
        structured_output: false,
        temperature: null,
        modalities_input: "[]",
        modalities_output: '["text"]',
        knowledge_cutoff: null,
        release_date: null,
        last_updated: null,
        status: null,
        family: null,
        open_weights: false,
        limit_context: null,
        limit_input: null,
        limit_output: 1024,
        interleaved_field: null,
      },
    },
  });
});

test("syncModelsDev supports dry-run mode, persistence, capability toggles, and failure reporting", async () => {
  const modelsDev = await importFresh("sync-main");
  mockFetchWith(MOCK_MODELS_DEV_DATA);

  const dryRun = await modelsDev.syncModelsDev({ dryRun: true, syncCapabilities: false });
  assert.equal(dryRun.success, true);
  assert.equal(dryRun.dryRun, true);
  assert.equal(dryRun.capabilityCount, 0);
  assert.ok(dryRun.data.pricing.openai["gpt-4o"]);
  assert.deepEqual(modelsDev.getModelsDevPricing(), {});

  const persisted = await modelsDev.syncModelsDev();
  assert.equal(persisted.success, true);
  assert.equal(persisted.dryRun, false);
  assert.ok(persisted.modelCount > 0);
  assert.ok(modelsDev.getModelsDevPricing().anthropic["claude-sonnet-4-20250514"]);
  assert.ok(modelsDev.getSyncedCapabilities().openai["gpt-4o"]);
  assert.ok(modelsDev.getSyncStatus().lastSync);
  assert.ok(modelsDev.getSyncStatus().lastSyncModelCount > 0);

  const failing = await importFresh("sync-failure");
  globalThis.fetch = async () => {
    throw new Error("network down");
  };
  const failed = await failing.syncModelsDev();
  assert.equal(failed.success, false);
  assert.match(failed.error, /network down/);
});

test("syncModelsDev string failures are normalized into an error payload", async () => {
  const modelsDev = await importFresh("sync-string-error");
  globalThis.fetch = async () => {
    throw "hard fail";
  };

  const failed = await modelsDev.syncModelsDev({ dryRun: true });
  assert.equal(failed.success, false);
  assert.equal(failed.error, "hard fail");
  assert.equal(failed.dryRun, true);
});

test("syncModelsDev honors abort signals during retry backoff", async () => {
  const modelsDev = await importFresh("sync-abort");
  const warnings = [];
  const originalWarn = console.warn;
  console.warn = (...args) => warnings.push(args.map((arg) => String(arg)).join(" "));
  globalThis.fetch = async () => {
    throw new Error("network down");
  };

  try {
    const controller = new AbortController();
    const pending = modelsDev.syncModelsDev({ signal: controller.signal, maxRetries: 3 });
    const warned = await waitFor(() => warnings.length > 0, 100);
    assert.ok(warned, "expected the first retry warning before aborting");

    controller.abort();
    const aborted = await pending;
    assert.equal(aborted.success, false);
    assert.equal(aborted.error, "aborted");
    assert.equal(warnings.length, 1);
  } finally {
    console.warn = originalWarn;
  }
});

test("startPeriodicSync, stopPeriodicSync, getSyncStatus, and initModelsDevSync honor settings and avoid duplicate timers", async () => {
  const modelsDev = await importFresh("periodic-sync");
  mockFetchWith(MOCK_MODELS_DEV_DATA);

  modelsDev.startPeriodicSync(25);
  const started = modelsDev.getSyncStatus();
  assert.equal(started.enabled, true);
  assert.equal(started.intervalMs, 25);

  modelsDev.startPeriodicSync(99);
  assert.equal(modelsDev.getSyncStatus().intervalMs, 25);

  const syncedAt = await waitFor(() => modelsDev.getSyncStatus().lastSync, 2000);
  assert.ok(syncedAt, "expected initial periodic sync to complete");
  assert.ok(modelsDev.getSyncStatus().nextSync);

  modelsDev.stopPeriodicSync();
  const stopped = modelsDev.getSyncStatus();
  assert.equal(stopped.enabled, false);
  assert.equal(stopped.nextSync, null);

  await settingsDb.updateSettings({
    modelsDevSyncEnabled: false,
    modelsDevSyncInterval: 15,
  });
  const disabled = await importFresh("init-disabled");
  await disabled.initModelsDevSync();
  assert.equal(disabled.getSyncStatus().enabled, false);

  await settingsDb.updateSettings({
    modelsDevSyncEnabled: true,
    modelsDevSyncInterval: 15,
  });
  const enabled = await importFresh("init-enabled");
  mockFetchWith(MOCK_MODELS_DEV_DATA);
  await enabled.initModelsDevSync();
  assert.equal(enabled.getSyncStatus().enabled, true);
  assert.equal(enabled.getSyncStatus().intervalMs, 15);
  await waitFor(() => enabled.getSyncStatus().lastSync, 2000);
});

test("stopPeriodicSync aborts the in-flight initial sync", async () => {
  const modelsDev = await importFresh("periodic-stop-abort");
  let aborted = false;

  globalThis.fetch = async (_url, init) =>
    await new Promise((_resolve, reject) => {
      const signal = init?.signal;
      const onAbort = () => {
        aborted = true;
        const error = new Error("aborted");
        error.name = "AbortError";
        reject(error);
      };

      signal?.addEventListener("abort", onAbort, { once: true });
    });

  modelsDev.startPeriodicSync(25);
  await waitFor(() => modelsDev.getSyncStatus().enabled, 50);
  modelsDev.stopPeriodicSync();

  const stopped = await waitFor(() => aborted, 200);
  assert.equal(stopped, true);
  assert.equal(modelsDev.getSyncStatus().enabled, false);
  assert.equal(modelsDev.getSyncStatus().lastSync, null);
});