chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,15 @@
|
||||
import { expect, test } from "bun:test";
|
||||
|
||||
import { inferKimiFamily } from "../src/family.js";
|
||||
|
||||
test("Kimi family inference ignores K2 versions", () => {
|
||||
expect(inferKimiFamily("moonshotai/kimi-k2.5")).toBe("kimi-k2");
|
||||
expect(inferKimiFamily("moonshotai/kimi-k2.7-code")).toBe("kimi-k2");
|
||||
expect(inferKimiFamily("Kimi K2.6")).toBe("kimi-k2");
|
||||
});
|
||||
|
||||
test("Kimi family inference preserves thinking variants", () => {
|
||||
expect(inferKimiFamily("moonshotai/kimi-k2-thinking")).toBe("kimi-thinking");
|
||||
expect(inferKimiFamily("Kimi K2.5 Thinking")).toBe("kimi-thinking");
|
||||
expect(inferKimiFamily("moonshotai/kimi-k2.6:thinking")).toBe("kimi-thinking");
|
||||
});
|
||||
@@ -0,0 +1,380 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import path from "node:path";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { generate, generateCatalog } from "../src/index.js";
|
||||
|
||||
async function withFixture<T>(callback: (root: string) => Promise<T>) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "models-dev-test-"));
|
||||
try {
|
||||
return await callback(root);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function write(root: string, file: string, content: string) {
|
||||
const filePath = path.join(root, file);
|
||||
await mkdir(path.dirname(filePath), { recursive: true });
|
||||
await Bun.write(filePath, content);
|
||||
}
|
||||
|
||||
function stable(value: unknown): string {
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map(stable).join(",")}]`;
|
||||
}
|
||||
if (value !== null && typeof value === "object") {
|
||||
return `{${Object.entries(value)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, item]) => `${JSON.stringify(key)}:${stable(item)}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
describe("catalog generation", () => {
|
||||
test("base_model can factor metadata without changing provider JSON", async () => {
|
||||
await withFixture(async (root) => {
|
||||
await write(root, "providers/direct/provider.toml", providerToml("Direct"));
|
||||
await write(root, "providers/factored/provider.toml", providerToml("Factored"));
|
||||
await write(root, "models/lab/model.toml", modelMetadataToml());
|
||||
await write(
|
||||
root,
|
||||
"providers/direct/models/model.toml",
|
||||
`${providerFieldsToml()}
|
||||
|
||||
[cost]
|
||||
input = 1.25
|
||||
output = 2.50
|
||||
cache_read = 0.125
|
||||
`,
|
||||
);
|
||||
await write(
|
||||
root,
|
||||
"providers/factored/models/model.toml",
|
||||
`base_model = "lab/model"
|
||||
reasoning_options = []
|
||||
|
||||
[cost]
|
||||
input = 1.25
|
||||
output = 2.50
|
||||
cache_read = 0.125
|
||||
`,
|
||||
);
|
||||
|
||||
const catalog = await generateCatalog(root);
|
||||
|
||||
expect(catalog.models["lab/model"]?.benchmarks).toEqual([
|
||||
{
|
||||
name: "SWE-Bench Verified",
|
||||
score: 71.2,
|
||||
metric: "resolved",
|
||||
harness: "Example Harness",
|
||||
variant: "high",
|
||||
dataset: "verified",
|
||||
version: "1",
|
||||
source: "https://example.com/benchmarks",
|
||||
},
|
||||
]);
|
||||
expect(catalog.models["lab/model"]?.weights).toEqual([
|
||||
{
|
||||
label: "Weights",
|
||||
url: "https://huggingface.co/lab/model",
|
||||
format: "safetensors",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(catalog.providers.factored?.models.model).toEqual(
|
||||
catalog.providers.direct?.models.model,
|
||||
);
|
||||
expect(catalog.providers.factored?.models.model).not.toHaveProperty(
|
||||
"base_model",
|
||||
);
|
||||
expect(catalog.providers.factored?.models.model).not.toHaveProperty(
|
||||
"benchmarks",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("base_model_omit removes inherited metadata fields", async () => {
|
||||
await withFixture(async (root) => {
|
||||
await write(root, "providers/provider/provider.toml", providerToml("Provider"));
|
||||
await write(root, "models/lab/model.toml", modelMetadataToml());
|
||||
await write(
|
||||
root,
|
||||
"providers/provider/models/model.toml",
|
||||
`base_model = "lab/model"
|
||||
base_model_omit = ["limit.input", "structured_output"]
|
||||
reasoning_options = []
|
||||
|
||||
[cost]
|
||||
input = 1.25
|
||||
output = 2.50
|
||||
|
||||
[limit]
|
||||
context = 200_000
|
||||
output = 32_000
|
||||
`,
|
||||
);
|
||||
|
||||
const providers = await generate(path.join(root, "providers"));
|
||||
const model = providers.provider?.models.model;
|
||||
|
||||
expect(model?.structured_output).toBeUndefined();
|
||||
expect(model?.limit).toEqual({
|
||||
context: 200_000,
|
||||
output: 32_000,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("base_model can inherit sibling fields from partial object overrides", async () => {
|
||||
await withFixture(async (root) => {
|
||||
await write(root, "providers/provider/provider.toml", providerToml("Provider"));
|
||||
await write(root, "models/lab/model.toml", modelMetadataToml());
|
||||
await write(
|
||||
root,
|
||||
"providers/provider/models/model.toml",
|
||||
`base_model = "lab/model"
|
||||
open_weights = true
|
||||
reasoning_options = []
|
||||
|
||||
[cost]
|
||||
input = 1.25
|
||||
output = 2.50
|
||||
|
||||
[limit]
|
||||
context = 200_000
|
||||
|
||||
[modalities]
|
||||
input = ["text"]
|
||||
`,
|
||||
);
|
||||
|
||||
const providers = await generate(path.join(root, "providers"));
|
||||
const model = providers.provider?.models.model;
|
||||
|
||||
expect(model?.open_weights).toBe(true);
|
||||
expect(model?.limit).toEqual({
|
||||
context: 200_000,
|
||||
input: 272_000,
|
||||
output: 128_000,
|
||||
});
|
||||
expect(model?.modalities).toEqual({
|
||||
input: ["text"],
|
||||
output: ["text"],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("repository provider TOMLs do not use legacy extends tables", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const matches: string[] = [];
|
||||
|
||||
for await (const file of new Bun.Glob("providers/**/*.toml").scan({
|
||||
cwd: root,
|
||||
})) {
|
||||
const text = await Bun.file(path.join(root, file)).text();
|
||||
if (/^\[extends\]/m.test(text)) matches.push(file);
|
||||
}
|
||||
|
||||
expect(matches).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository provider JSON strips authored metadata pointers", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const providers = await generate(path.join(root, "providers"));
|
||||
const leaked: string[] = [];
|
||||
|
||||
for (const [providerID, provider] of Object.entries(providers)) {
|
||||
for (const [modelID, model] of Object.entries(provider.models)) {
|
||||
const encoded = stable(model);
|
||||
if (encoded.includes("base_model") || encoded.includes("base_model_omit")) {
|
||||
leaked.push(`${providerID}/${modelID}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(leaked).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository provider JSON excludes model-only metadata", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const providers = await generate(path.join(root, "providers"));
|
||||
const modelOnlyFields = ["benchmarks", "license", "links", "weights"];
|
||||
const leaked: string[] = [];
|
||||
|
||||
for (const [providerID, provider] of Object.entries(providers)) {
|
||||
for (const [modelID, model] of Object.entries(provider.models)) {
|
||||
const leakedFields = modelOnlyFields.filter((field) => field in model);
|
||||
if (leakedFields.length > 0) {
|
||||
leaked.push(`${providerID}/${modelID}: ${leakedFields.join(", ")}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(leaked).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository model metadata avoids provider-only namespaces", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const providerNamespaces = [
|
||||
"amazon-bedrock",
|
||||
"llama",
|
||||
"opencode",
|
||||
"tencent-tokenhub",
|
||||
"zai",
|
||||
];
|
||||
const namespaceDirs = providerNamespaces.filter((namespace) =>
|
||||
existsSync(path.join(root, "models", namespace))
|
||||
);
|
||||
const baseModelRefs: string[] = [];
|
||||
|
||||
for await (const file of new Bun.Glob("providers/**/*.toml").scan({
|
||||
cwd: root,
|
||||
})) {
|
||||
const text = await Bun.file(path.join(root, file)).text();
|
||||
const match = /^base_model = "([^/"]+)\//m.exec(text);
|
||||
if (match?.[1] !== undefined && providerNamespaces.includes(match[1])) {
|
||||
baseModelRefs.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
expect(namespaceDirs).toEqual([]);
|
||||
expect(baseModelRefs).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository open-weight model metadata includes weights links", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const catalog = await generateCatalog(root);
|
||||
const missingWeights: string[] = [];
|
||||
const closedWithWeights: string[] = [];
|
||||
|
||||
for (const [modelID, model] of Object.entries(catalog.models)) {
|
||||
const hasWeights = (model.weights?.length ?? 0) > 0;
|
||||
if (model.open_weights === true && !hasWeights) {
|
||||
missingWeights.push(modelID);
|
||||
}
|
||||
if (model.open_weights !== true && hasWeights) {
|
||||
closedWithWeights.push(modelID);
|
||||
}
|
||||
}
|
||||
|
||||
expect(missingWeights).toEqual([]);
|
||||
expect(closedWithWeights).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository benchmark metadata is sourced", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const catalog = await generateCatalog(root);
|
||||
const unsourced: string[] = [];
|
||||
|
||||
for (const [modelID, model] of Object.entries(catalog.models)) {
|
||||
for (const benchmark of model.benchmarks ?? []) {
|
||||
if (benchmark.source === undefined) {
|
||||
unsourced.push(`${modelID}: ${benchmark.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(unsourced).toEqual([]);
|
||||
});
|
||||
|
||||
test("repository benchmark names are normalized", async () => {
|
||||
const root = path.join(import.meta.dirname, "..", "..", "..");
|
||||
const catalog = await generateCatalog(root);
|
||||
const qualifiedNames: string[] = [];
|
||||
|
||||
for (const [modelID, model] of Object.entries(catalog.models)) {
|
||||
for (const benchmark of model.benchmarks ?? []) {
|
||||
if (/\s\([^)]+\)$/.test(benchmark.name)) {
|
||||
qualifiedNames.push(`${modelID}: ${benchmark.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(qualifiedNames).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
function providerToml(name: string) {
|
||||
return `name = "${name}"
|
||||
npm = "@ai-sdk/openai"
|
||||
env = ["API_KEY"]
|
||||
doc = "https://example.com/models"
|
||||
`;
|
||||
}
|
||||
|
||||
function modelMetadataToml() {
|
||||
return `name = "Lab Model"
|
||||
description = "Example model for catalog generation and inheritance tests"
|
||||
family = "gpt"
|
||||
release_date = "2026-01-02"
|
||||
last_updated = "2026-01-03"
|
||||
attachment = true
|
||||
reasoning = true
|
||||
temperature = false
|
||||
tool_call = true
|
||||
structured_output = true
|
||||
knowledge = "2025-12"
|
||||
open_weights = true
|
||||
license = "Example License"
|
||||
|
||||
[limit]
|
||||
context = 400_000
|
||||
input = 272_000
|
||||
output = 128_000
|
||||
|
||||
[modalities]
|
||||
input = ["text", "image"]
|
||||
output = ["text"]
|
||||
|
||||
[[links]]
|
||||
label = "Model card"
|
||||
url = "https://example.com/model"
|
||||
type = "model_card"
|
||||
|
||||
[[weights]]
|
||||
label = "Weights"
|
||||
url = "https://huggingface.co/lab/model"
|
||||
format = "safetensors"
|
||||
|
||||
[[benchmarks]]
|
||||
name = "SWE-Bench Verified"
|
||||
score = 71.2
|
||||
metric = "resolved"
|
||||
harness = "Example Harness"
|
||||
variant = "high"
|
||||
dataset = "verified"
|
||||
version = "1"
|
||||
source = "https://example.com/benchmarks"
|
||||
`;
|
||||
}
|
||||
|
||||
function providerFieldsToml() {
|
||||
return `name = "Lab Model"
|
||||
description = "Example model for catalog generation and inheritance tests"
|
||||
family = "gpt"
|
||||
release_date = "2026-01-02"
|
||||
last_updated = "2026-01-03"
|
||||
attachment = true
|
||||
reasoning = true
|
||||
reasoning_options = []
|
||||
temperature = false
|
||||
tool_call = true
|
||||
structured_output = true
|
||||
knowledge = "2025-12"
|
||||
open_weights = true
|
||||
|
||||
[limit]
|
||||
context = 400_000
|
||||
input = 272_000
|
||||
output = 128_000
|
||||
|
||||
[modalities]
|
||||
input = ["text", "image"]
|
||||
output = ["text"]
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { z } from "zod";
|
||||
|
||||
import { AuthoredModel } from "../src/index.js";
|
||||
|
||||
type AuthoredModelData = z.infer<typeof AuthoredModel>;
|
||||
|
||||
describe("model schema", () => {
|
||||
test("requires reasoning_options when reasoning is true", () => {
|
||||
const model = baseModel({ reasoning: true });
|
||||
|
||||
expect(AuthoredModel.safeParse(model).success).toBe(false);
|
||||
});
|
||||
|
||||
test("accepts empty reasoning_options when reasoning is true", () => {
|
||||
const model = baseModel({
|
||||
reasoning: true,
|
||||
reasoning_options: [],
|
||||
});
|
||||
|
||||
expect(AuthoredModel.safeParse(model).success).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects reasoning_options when reasoning is false", () => {
|
||||
const model = baseModel({
|
||||
reasoning: false,
|
||||
reasoning_options: [],
|
||||
});
|
||||
|
||||
expect(AuthoredModel.safeParse(model).success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
function baseModel(overrides: Partial<AuthoredModelData>) {
|
||||
return {
|
||||
id: "example/model",
|
||||
name: "Example Model",
|
||||
description: "Example model for schema validation and regression tests",
|
||||
attachment: false,
|
||||
reasoning: false,
|
||||
tool_call: true,
|
||||
release_date: "2026-01-01",
|
||||
last_updated: "2026-01-01",
|
||||
modalities: {
|
||||
input: ["text"],
|
||||
output: ["text"],
|
||||
},
|
||||
open_weights: false,
|
||||
limit: {
|
||||
context: 1_000,
|
||||
output: 100,
|
||||
},
|
||||
cost: {
|
||||
input: 1,
|
||||
output: 2,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user