#!/usr/bin/env bun import path from "node:path"; import { cp, mkdir, rm, writeFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { mergeDeep } from "remeda"; import { z } from "zod"; import { generate } from "../src/generate.js"; import { AuthoredModel, AuthoredModelShape, Model, Provider } from "../src/schema.js"; const root = path.join(import.meta.dirname, "..", "..", ".."); const providersPath = path.join(root, "providers"); const modelsPath = path.join(root, "models"); const LegacyExtendsModel = AuthoredModelShape .partial() .extend({ extends: z .object({ from: z.string(), omit: z.array(z.string()).optional(), }) .strict(), }) .strict(); const diffOutput = await Bun.$`git diff --name-only HEAD -- providers`.cwd(root).text(); const changedProviderPaths = diffOutput .split("\n") .filter(Boolean) .filter((filePath) => /^providers\/[^/]+\/models\/.+\.toml$/.test(filePath)); if (changedProviderPaths.length === 0) { process.exit(0); } const baselineRoot = path.join(tmpdir(), `models-dev-compare-${Date.now()}`); await mkdir(baselineRoot, { recursive: true }); try { const baselineProvidersPath = path.join(baselineRoot, "providers"); await cp(providersPath, baselineProvidersPath, { recursive: true }); const baselineModelsPath = path.join(baselineRoot, "models"); await cp(modelsPath, baselineModelsPath, { recursive: true }); await installModelNamespaceAliases(baselineModelsPath); for (const filePath of changedProviderPaths) { const tempFilePath = path.join(baselineRoot, filePath); const show = Bun.spawn(["git", "show", `HEAD:${filePath}`], { cwd: root, stdout: "pipe", stderr: "pipe", }); const exitCode = await show.exited; if (exitCode !== 0) { await rm(tempFilePath, { force: true }); continue; } const contents = await new Response(show.stdout).text(); await mkdir(path.dirname(tempFilePath), { recursive: true }); await writeFile(tempFilePath, contents); } const before = await generateForComparison(baselineProvidersPath); const after = await generate(providersPath); for (const filePath of changedProviderPaths) { const match = /^providers\/([^/]+)\/models\/(.+)\.toml$/.exec(filePath); if (!match) continue; const [, providerID, modelID] = match; if (providerID === undefined || modelID === undefined) continue; const beforeModel = before[providerID]?.models[modelID]; const afterModel = after[providerID]?.models[modelID]; const beforeJson = sortedJson(beforeModel); const afterJson = sortedJson(afterModel); if (beforeJson === afterJson) { continue; } const beforeFilePath = path.join(baselineRoot, "before.json"); const afterFilePath = path.join(baselineRoot, "after.json"); await writeFile(beforeFilePath, `${beforeJson}\n`); await writeFile(afterFilePath, `${afterJson}\n`); const diff = Bun.spawn( [ "diff", "-u", "-L", `${filePath} (before)`, "-L", `${filePath} (after)`, beforeFilePath, afterFilePath, ], { stdout: "pipe", stderr: "pipe", }, ); const output = await new Response(diff.stdout).text(); process.stdout.write(output); } } finally { await rm(baselineRoot, { recursive: true, force: true }); } async function installModelNamespaceAliases(directory: string) { await copyModelAlias( directory, "deepseek/deepseek-r1", "amazon-bedrock/deepseek.r1-v1:0", ); await copyModelAlias( directory, "meta/llama-4-maverick-17b-instruct", "amazon-bedrock/meta.llama4-maverick-17b-instruct-v1:0", ); await copyModelAlias( directory, "meta/llama-4-scout-17b-instruct", "amazon-bedrock/meta.llama4-scout-17b-instruct-v1:0", ); await copyModelAlias( directory, "meta/llama-3.3-70b-instruct", "llama/llama-3.3-70b-instruct", ); await copyModelAliasWithReplacements( directory, "openai/gpt-5.5-pro", "opencode/gpt-5.5-pro", [ [/release_date = "2026-04-23"/, 'release_date = "2026-04-24"'], [/last_updated = "2026-04-23"/, 'last_updated = "2026-04-24"'], [/structured_output = true/, "structured_output = false"], ], ); await copyModelAlias( directory, "tencent/hy3-preview", "tencent-tokenhub/hy3-preview", ); } async function copyModelAlias(directory: string, from: string, to: string) { return copyModelAliasWithReplacements(directory, from, to, []); } async function copyModelAliasWithReplacements( directory: string, from: string, to: string, replacements: Array<[RegExp, string]>, ) { const source = path.join(directory, `${from}.toml`); const target = path.join(directory, `${to}.toml`); if (!existsSync(source) || existsSync(target)) return; await mkdir(path.dirname(target), { recursive: true }); if (replacements.length === 0) { await cp(source, target); return; } let text = await Bun.file(source).text(); for (const [pattern, replacement] of replacements) { text = text.replace(pattern, replacement); } await writeFile(target, text); } async function generateForComparison(directory: string) { for await (const file of new Bun.Glob("**/*.toml").scan({ cwd: directory })) { const text = await Bun.file(path.join(directory, file)).text(); if (/^\[extends\]/m.test(text)) { return generateLegacyExtends(directory); } } return generate(directory); } async function generateLegacyExtends(directory: string) { const result: Record = {}; const pendingModels: Array<{ providerID: string; modelID: string; modelPath: string; model: z.infer; }> = []; for await (const providerPath of new Bun.Glob("*/provider.toml").scan({ cwd: directory, absolute: true, })) { const providerID = path.basename(path.dirname(providerPath)); const toml = await import(providerPath, { with: { type: "toml" } }).then( (mod) => mod.default, ); toml.id = providerID; toml.models = {}; const provider = Provider.safeParse(toml); if (!provider.success) { provider.error.cause = { providerPath, toml }; throw provider.error; } const modelsPath = path.join(directory, providerID, "models"); for await (const modelPath of new Bun.Glob("**/*.toml").scan({ cwd: modelsPath, absolute: true, followSymlinks: true, })) { const modelID = path.relative(modelsPath, modelPath).slice(0, -5); const toml = await import(modelPath, { with: { type: "toml" } }).then( (mod) => mod.default, ); toml.id = modelID; if (toml.extends !== undefined) { const model = LegacyExtendsModel.safeParse(toml); if (!model.success) { model.error.cause = { modelPath, toml }; throw model.error; } pendingModels.push({ providerID, modelID, modelPath, model: model.data, }); continue; } const model = AuthoredModel.safeParse(toml); if (!model.success) { model.error.cause = { modelPath, toml }; throw model.error; } provider.data.models[modelID] = normalizeModelCost(model.data); } result[providerID] = provider.data; } const nameToProviderID = new Map(); for (const provider of Object.values(result)) { const nameKey = provider.name.toLowerCase(); const existingID = nameToProviderID.get(nameKey); if (existingID !== undefined) { throw new Error( `Duplicate provider name "${provider.name}" used by both "${existingID}" and "${provider.id}". Provider names must be unique.`, ); } nameToProviderID.set(nameKey, provider.id); } for (const pendingModel of pendingModels) { const [providerID, ...modelParts] = pendingModel.model.extends.from.split("/"); const modelID = modelParts.join("/"); if (providerID === undefined) { throw new Error(`Invalid legacy extends.from: ${pendingModel.model.extends.from}`); } const baseModel = result[providerID]?.models[modelID]; if (baseModel === undefined) { throw new Error(`Unable to resolve legacy extends.from: ${pendingModel.model.extends.from}`, { cause: { modelPath: pendingModel.modelPath, toml: pendingModel.model }, }); } const { extends: extendsConfig, ...overrides } = pendingModel.model; const { reasoning_options: _reasoningOptions, ...inherited } = baseModel; const merged: Record = structuredClone( mergeDeep(inherited, overrides), ); applyOmit(merged, extendsConfig.omit ?? []); const model = Model.safeParse(normalizeCost(merged)); if (!model.success) { model.error.cause = { modelPath: pendingModel.modelPath, toml: merged }; throw model.error; } result[pendingModel.providerID]!.models[pendingModel.modelID] = model.data; } return result; } function normalizeModelCost(model: z.infer): Model { return normalizeCost(model) as Model; } function normalizeCost(model: Record) { const cost = model.cost; if (cost === undefined || cost === null || typeof cost !== "object" || Array.isArray(cost)) { return model; } const tiers = (cost as { tiers?: unknown }).tiers; if (!Array.isArray(tiers) || tiers.length !== 1) { return model; } const contextOver200k = tiers.find((tier) => { if (tier === null || typeof tier !== "object" || Array.isArray(tier)) return false; const tierConfig = (tier as { tier?: unknown }).tier; if (tierConfig === null || typeof tierConfig !== "object" || Array.isArray(tierConfig)) return false; const type = (tierConfig as { type?: unknown }).type; const size = (tierConfig as { size?: unknown }).size; return ( (type === undefined || type === "context") && typeof size === "number" && size >= 200_000 ); }); if (contextOver200k === undefined) { return model; } const { tier: _tier, ...legacyCost } = contextOver200k as Record; return { ...model, cost: { ...(cost as Record), context_over_200k: legacyCost, }, }; } function applyOmit(target: Record, paths: string[]) { omitLoop: for (const omit of paths) { const parts = omit.split("."); const parents: Array<{ value: Record; key: string; }> = []; let current = target; for (const part of parts.slice(0, -1)) { const next = current[part]; if ( next === undefined || next === null || typeof next !== "object" || Array.isArray(next) ) { continue omitLoop; } parents.push({ value: current, key: part }); current = next as Record; } const lastPart = parts.at(-1); if (lastPart === undefined || !(lastPart in current)) { continue; } delete current[lastPart]; for (let index = parents.length - 1; index >= 0; index--) { const parent = parents[index]; if (parent === undefined) continue; const value = parent.value[parent.key]; if ( value === null || value === undefined || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length > 0 ) { break; } delete parent.value[parent.key]; } } } function sortedJson(value: unknown) { return JSON.stringify(sortJson(value), null, 2); } function sortJson(value: unknown): unknown { if (Array.isArray(value)) { return value.map(sortJson); } if (value !== null && typeof value === "object") { return Object.fromEntries( Object.entries(value) .sort(([a], [b]) => a.localeCompare(b)) .map(([key, item]) => [key, sortJson(item)]), ); } return value; }