diegosouzapw--omniroute/tests/unit/provider-sweep-live-discovery.test.ts

/**
 * TDD regression for the provider-model-sweep (2026-06-19): seven providers were
 * classified as "fixed-official" in the registry (their `models[]` is a small
 * hardcoded seed), yet each exposes a real OpenAI-style `/models` endpoint that
 * serves a much larger, constantly-changing live catalog. Hardcoding the full
 * catalog would re-introduce the very staleness the sweep set out to fix, so the
 * correct fix mirrors #4249 (vercel-ai-gateway), #4202 (zenmux) and #3976
 * (llm7/byteplus): add the provider to NAMED_OPENAI_STYLE_PROVIDERS so the
 * import route does a live `<baseUrl>/models` fetch, keeping the small registry
 * seed only as the offline fallback.
 *
 * Root cause (shared with #4249): a keyed `format: "openai"` provider that is not
 * `openai-compatible-*`, not self-hosted, and not in NAMED_OPENAI_STYLE_PROVIDERS
 * never probes upstream `/models`, so the route serves the tiny hardcoded seed.
 *
 * Each case below pins the exact `/models` URL the route derives after stripping
 * `/chat/completions` (and a trailing `/v1`) from the registry baseUrl.
 */
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sweep-live-"));
process.env.DATA_DIR = TEST_DATA_DIR;

const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const modelsRoute = await import("../../src/app/api/providers/[id]/models/route.ts");

async function resetStorage() {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}

test.after(() => {
  core.resetDbInstance();
  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
});

interface ModelsBody {
  provider: string;
  connectionId: string;
  models: Array<{ id: string }>;
  source?: string;
}

// provider → the upstream /models URL the route resolves from its registry baseUrl.
const LIVE_CASES: Array<{ provider: string; liveUrl: string }> = [
  { provider: "venice", liveUrl: "https://api.venice.ai/api/v1/models" },
  { provider: "deepinfra", liveUrl: "https://api.deepinfra.com/v1/openai/models" },
  { provider: "wandb", liveUrl: "https://api.inference.wandb.ai/v1/models" },
  { provider: "pollinations", liveUrl: "https://gen.pollinations.ai/v1/models" },
  { provider: "nscale", liveUrl: "https://inference.api.nscale.com/v1/models" },
  { provider: "inference-net", liveUrl: "https://api.inference.net/v1/models" },
  { provider: "moonshot", liveUrl: "https://api.moonshot.ai/v1/models" },
  // GPU-cloud / aggregator marketplaces (sweep cont.).
  { provider: "crof", liveUrl: "https://crof.ai/v1/models" },
  { provider: "featherless-ai", liveUrl: "https://api.featherless.ai/v1/models" },
  { provider: "ovhcloud", liveUrl: "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1/models" },
  { provider: "sambanova", liveUrl: "https://api.sambanova.ai/v1/models" },
  { provider: "orcarouter", liveUrl: "https://api.orcarouter.ai/v1/models" },
  { provider: "uncloseai", liveUrl: "https://hermes.ai.unturf.com/v1/models" },
  { provider: "opencode-go", liveUrl: "https://opencode.ai/zen/go/v1/models" },
  { provider: "baseten", liveUrl: "https://inference.baseten.co/v1/models" },
  { provider: "hyperbolic", liveUrl: "https://api.hyperbolic.xyz/v1/models" },
  { provider: "nebius", liveUrl: "https://api.tokenfactory.nebius.com/v1/models" },
  { provider: "scaleway", liveUrl: "https://api.scaleway.ai/v1/models" },
  { provider: "together", liveUrl: "https://api.together.xyz/v1/models" },
  // escalated cmqlvxg4o: api-airforce carries a real live `https://api.airforce/v1/models`
  // catalog but was left out of the sweep, so it served its stale hardcoded seed
  // (grok-3, grok-2-1212, claude-3.7-sonnet …) — models that no longer exist upstream,
  // so chat failed even though the connection test passed. Same class as the rows above.
  { provider: "api-airforce", liveUrl: "https://api.airforce/v1/models" },
];

for (const { provider, liveUrl } of LIVE_CASES) {
  test(`sweep: ${provider} import fetches the live /models catalog`, async () => {
    await resetStorage();
    const connection = await providersDb.createProviderConnection({
      provider,
      authType: "apikey",
      name: `${provider}-live`,
      apiKey: "sweep-key",
    });

    let fetched = false;
    const originalFetch = globalThis.fetch;
    globalThis.fetch = async (url) => {
      if (String(url) === liveUrl) {
        fetched = true;
        return Response.json({
          object: "list",
          data: [{ id: `${provider}-live-a` }, { id: `${provider}-live-b` }],
        });
      }
      // Bogus probe variants (…/v1/v1/models, …/chat/completions/models) → 404.
      return new Response("not found", { status: 404 });
    };

    try {
      const response = await modelsRoute.GET(
        new Request(`http://localhost/api/providers/${connection.id}/models?refresh=true`),
        { params: { id: connection.id } }
      );
      assert.equal(response.status, 200);
      const body = (await response.json()) as ModelsBody;
      assert.equal(body.provider, provider);
      assert.ok(fetched, `should have probed ${liveUrl}`);
      assert.equal(body.source, "api", "should serve the live upstream catalog, not local_catalog");
      const ids = body.models.map((m) => m.id);
      assert.ok(
        ids.includes(`${provider}-live-a`) && ids.includes(`${provider}-live-b`),
        `live models missing from catalog: ${ids.join(",")}`
      );
    } finally {
      globalThis.fetch = originalFetch;
    }
  });
}

test("sweep: live-discovery providers fall back to the local seed when upstream is down", async () => {
  await resetStorage();
  const connection = await providersDb.createProviderConnection({
    provider: "venice",
    authType: "apikey",
    name: "venice-fallback",
    apiKey: "sweep-key-2",
  });

  const originalFetch = globalThis.fetch;
  globalThis.fetch = async () => new Response("bad gateway", { status: 502 });

  try {
    const response = await modelsRoute.GET(
      new Request(`http://localhost/api/providers/${connection.id}/models?refresh=true`),
      { params: { id: connection.id } }
    );
    assert.equal(response.status, 200);
    const body = (await response.json()) as ModelsBody;
    assert.equal(body.provider, "venice");
    assert.equal(body.source, "local_catalog", "import must not break when upstream is down");
    assert.ok(body.models.length > 0, "fallback seed should be non-empty");
  } finally {
    globalThis.fetch = originalFetch;
  }
});