chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@internal/llm-model-catalog",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./src/index.ts",
|
||||
"types": "./src/index.ts",
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"@trigger.dev/database": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internal/testcontainers": "workspace:*",
|
||||
"vitest": "4.1.7"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"generate": "node scripts/generate.mjs",
|
||||
"sync-prices": "bash scripts/sync-model-prices.sh && node scripts/generate.mjs",
|
||||
"sync-prices:check": "bash scripts/sync-model-prices.sh --check",
|
||||
"generate-catalog": "bash scripts/generate-model-catalog.sh && node scripts/generate.mjs",
|
||||
"generate-catalog:dry-run": "bash scripts/generate-model-catalog.sh --dry-run"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
logs/
|
||||
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Generate model-catalog.json by researching each unique base model using Claude Code CLI.
|
||||
# Usage: ./scripts/generate-model-catalog.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --dry-run Print models that would be researched without running Claude
|
||||
# --filter <pattern> Only research models matching this ERE pattern (e.g. "gpt-4o|claude")
|
||||
# --max <n> Maximum number of models to research (useful for testing)
|
||||
# --stale-days <n> Re-research models older than N days (default: 7)
|
||||
# --force Re-research all models regardless of resolvedAt timestamp
|
||||
# --skip-hidden Skip models already marked as hidden/deprecated (saves time)
|
||||
# --concurrency <n> Number of models to research in parallel (default: 5)
|
||||
#
|
||||
# The script:
|
||||
# 1. Extracts all modelNames from defaultPrices.ts
|
||||
# 2. Groups dated variants to their base model
|
||||
# 3. Runs research-model.sh for each base model (in parallel)
|
||||
# 4. Writes results incrementally to model-catalog.json
|
||||
#
|
||||
# Logs are written to scripts/logs/ for debugging failures.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PACKAGE_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
DEFAULTS_FILE="$PACKAGE_DIR/src/defaultPrices.ts"
|
||||
CATALOG_FILE="$PACKAGE_DIR/src/model-catalog.json"
|
||||
RESEARCH_SCRIPT="$SCRIPT_DIR/research-model.sh"
|
||||
|
||||
LOG_DIR="$SCRIPT_DIR/logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
DRY_RUN=false
|
||||
FILTER=""
|
||||
MAX_MODELS=0
|
||||
STALE_DAYS=7
|
||||
FORCE=false
|
||||
SKIP_HIDDEN=false
|
||||
CONCURRENCY=5
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--filter) FILTER="$2"; shift 2 ;;
|
||||
--max) MAX_MODELS="$2"; shift 2 ;;
|
||||
--stale-days) STALE_DAYS="$2"; shift 2 ;;
|
||||
--force) FORCE=true; shift ;;
|
||||
--skip-hidden) SKIP_HIDDEN=true; shift ;;
|
||||
--concurrency) CONCURRENCY="$2"; shift 2 ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Extract all model names from defaultPrices.ts
|
||||
ALL_MODELS=$(grep -o '"modelName": "[^"]*"' "$DEFAULTS_FILE" | sed 's/"modelName": "//;s/"//' | sort -u)
|
||||
|
||||
# Skip embedding, legacy completion, and fine-tuned models
|
||||
SKIP_PATTERNS="^text-embedding|^textembedding|^text-ada|^text-babbage|^text-curie|^text-davinci|^text-bison|^text-unicorn|^code-bison|^code-gecko|^codechat-bison|^chat-bison|^babbage-002|^davinci-002|^ft:|^gemini-live"
|
||||
|
||||
FILTERED_MODELS=$(echo "$ALL_MODELS" | grep -vE "$SKIP_PATTERNS")
|
||||
|
||||
if [[ -n "$FILTER" ]]; then
|
||||
FILTERED_MODELS=$(echo "$FILTERED_MODELS" | grep -E "$FILTER" || true)
|
||||
fi
|
||||
|
||||
# Group dated variants to base models
|
||||
declare -A BASE_TO_VARIANTS
|
||||
declare -A MODEL_TO_BASE
|
||||
|
||||
for model in $FILTERED_MODELS; do
|
||||
base=$(echo "$model" | sed -E 's/-[0-9]{4}-?[0-9]{2}-?[0-9]{2}$//')
|
||||
base_no_latest=$(echo "$base" | sed -E 's/-latest$//')
|
||||
if [[ ${#base_no_latest} -lt ${#base} ]]; then
|
||||
base="$base_no_latest"
|
||||
fi
|
||||
|
||||
MODEL_TO_BASE["$model"]="$base"
|
||||
|
||||
if [[ -n "${BASE_TO_VARIANTS[$base]:-}" ]]; then
|
||||
BASE_TO_VARIANTS["$base"]="${BASE_TO_VARIANTS[$base]} $model"
|
||||
else
|
||||
BASE_TO_VARIANTS["$base"]="$model"
|
||||
fi
|
||||
done
|
||||
|
||||
BASE_MODELS=$(printf '%s\n' "${!BASE_TO_VARIANTS[@]}" | sort -u)
|
||||
TOTAL=$(echo "$BASE_MODELS" | wc -l | tr -d ' ')
|
||||
|
||||
if [[ "$MAX_MODELS" -gt 0 ]]; then
|
||||
BASE_MODELS=$(echo "$BASE_MODELS" | head -n "$MAX_MODELS")
|
||||
TOTAL=$(echo "$BASE_MODELS" | wc -l | tr -d ' ')
|
||||
fi
|
||||
|
||||
echo "Found $TOTAL unique base models (concurrency: $CONCURRENCY)"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo ""
|
||||
echo "Base models and their variants:"
|
||||
for base in $BASE_MODELS; do
|
||||
echo " $base → ${BASE_TO_VARIANTS[$base]}"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Load existing catalog
|
||||
if [[ -f "$CATALOG_FILE" ]]; then
|
||||
EXISTING_CATALOG=$(cat "$CATALOG_FILE")
|
||||
else
|
||||
EXISTING_CATALOG="{}"
|
||||
fi
|
||||
|
||||
# Lock file for thread-safe catalog writes
|
||||
LOCK_FILE="$LOG_DIR/.catalog.lock"
|
||||
RESULTS_DIR="$LOG_DIR/results"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
ERRORS=0
|
||||
FAILED_MODELS=""
|
||||
SKIPPED=0
|
||||
RESEARCHED=0
|
||||
CHANGED=0
|
||||
|
||||
# --- Determine which models need research ---
|
||||
|
||||
MODELS_TO_RESEARCH=""
|
||||
COUNT=0
|
||||
|
||||
for base in $BASE_MODELS; do
|
||||
COUNT=$((COUNT + 1))
|
||||
|
||||
SKIP_REASON=$(echo "$EXISTING_CATALOG" | node -e "
|
||||
const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
const entry = data['$base'];
|
||||
if (!entry) { process.stdout.write('missing'); return; }
|
||||
if ($FORCE) { process.stdout.write('force'); return; }
|
||||
if ($SKIP_HIDDEN && entry.isHidden) { process.stdout.write('hidden'); return; }
|
||||
const resolvedAt = entry.resolvedAt ? new Date(entry.resolvedAt) : null;
|
||||
if (!resolvedAt) { process.stdout.write('no_timestamp'); return; }
|
||||
const staleMs = $STALE_DAYS * 24 * 60 * 60 * 1000;
|
||||
if (Date.now() - resolvedAt.getTime() > staleMs) { process.stdout.write('stale'); return; }
|
||||
process.stdout.write('fresh');
|
||||
" 2>/dev/null || echo "missing")
|
||||
|
||||
case "$SKIP_REASON" in
|
||||
fresh)
|
||||
RESOLVED_DATE=$(echo "$EXISTING_CATALOG" | node -e "const d=JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));const r=d['$base']?.resolvedAt;console.log(r?r.split('T')[0]:'?')" 2>/dev/null)
|
||||
echo "[$COUNT/$TOTAL] Skipping $base (resolved $RESOLVED_DATE)"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
;;
|
||||
hidden)
|
||||
echo "[$COUNT/$TOTAL] Skipping $base (hidden/deprecated)"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
;;
|
||||
*)
|
||||
MODELS_TO_RESEARCH="$MODELS_TO_RESEARCH $base"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
RESEARCH_COUNT=$(echo "$MODELS_TO_RESEARCH" | wc -w | tr -d ' ')
|
||||
echo ""
|
||||
echo "Researching $RESEARCH_COUNT models, skipped $SKIPPED"
|
||||
echo ""
|
||||
|
||||
if [[ "$RESEARCH_COUNT" -eq 0 ]]; then
|
||||
echo "Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Research function (called per model, may run in parallel) ---
|
||||
|
||||
research_model() {
|
||||
local base="$1"
|
||||
local idx="$2"
|
||||
local total="$3"
|
||||
local model_log="$LOG_DIR/$base.log"
|
||||
local result_file="$RESULTS_DIR/$base.json"
|
||||
|
||||
echo "[$idx/$total] Researching $base..."
|
||||
|
||||
local raw
|
||||
raw=$("$RESEARCH_SCRIPT" "$base" 3 2>&1) || {
|
||||
echo "$raw" > "$model_log"
|
||||
echo " ERROR: Failed to research $base (after retries). Log: $model_log" >&2
|
||||
echo '{"error":true}' > "$result_file"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "$raw" > "$model_log"
|
||||
|
||||
local entry
|
||||
entry=$(echo "$raw" | node -e "
|
||||
try {
|
||||
const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
let text = (typeof d.result === 'string' ? d.result : JSON.stringify(d)).trim();
|
||||
text = text.replace(/^\`\`\`(?:json)?\s*/i, '').replace(/\s*\`\`\`\s*$/, '').trim();
|
||||
const jsonMatch = text.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) text = jsonMatch[0];
|
||||
const r = JSON.parse(text);
|
||||
if (!r.provider) throw new Error('missing provider field');
|
||||
process.stdout.write(JSON.stringify({
|
||||
provider: r.provider,
|
||||
description: r.description || '',
|
||||
contextWindow: r.contextWindow || null,
|
||||
maxOutputTokens: r.maxOutputTokens || null,
|
||||
capabilities: r.capabilities || [],
|
||||
releaseDate: r.releaseDate || null,
|
||||
isHidden: r.isHidden === true,
|
||||
supportsStructuredOutput: r.supportsStructuredOutput === true,
|
||||
supportsParallelToolCalls: r.supportsParallelToolCalls === true,
|
||||
supportsStreamingToolCalls: r.supportsStreamingToolCalls === true,
|
||||
deprecationDate: r.deprecationDate || null,
|
||||
knowledgeCutoff: r.knowledgeCutoff || null,
|
||||
resolvedAt: new Date().toISOString()
|
||||
}));
|
||||
} catch(e) {
|
||||
process.stderr.write(e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
" 2>"$LOG_DIR/$base.parse-error") || {
|
||||
local parse_err
|
||||
parse_err=$(cat "$LOG_DIR/$base.parse-error" 2>/dev/null)
|
||||
echo " ERROR: Failed to parse response for $base: $parse_err" >&2
|
||||
echo " Raw response saved to: $model_log" >&2
|
||||
echo '{"error":true}' > "$result_file"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "$entry" > "$result_file"
|
||||
echo " OK: $(echo "$entry" | node -e "const d=JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));console.log(d.provider + ' / ' + (d.contextWindow||'?') + ' ctx / ' + d.capabilities.length + ' caps')" 2>/dev/null)"
|
||||
}
|
||||
|
||||
export -f research_model
|
||||
export RESEARCH_SCRIPT LOG_DIR RESULTS_DIR
|
||||
|
||||
# --- Run research in parallel ---
|
||||
|
||||
IDX=0
|
||||
PIDS=()
|
||||
MODEL_LIST=($MODELS_TO_RESEARCH)
|
||||
|
||||
for base in "${MODEL_LIST[@]}"; do
|
||||
IDX=$((IDX + 1))
|
||||
|
||||
research_model "$base" "$IDX" "$RESEARCH_COUNT" &
|
||||
PIDS+=($!)
|
||||
|
||||
# Throttle concurrency
|
||||
if [[ ${#PIDS[@]} -ge $CONCURRENCY ]]; then
|
||||
wait "${PIDS[0]}" 2>/dev/null || true
|
||||
PIDS=("${PIDS[@]:1}")
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for remaining
|
||||
for pid in "${PIDS[@]}"; do
|
||||
wait "$pid" 2>/dev/null || true
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Research complete. Merging results..."
|
||||
|
||||
# --- Merge results into catalog ---
|
||||
|
||||
CATALOG="$EXISTING_CATALOG"
|
||||
|
||||
for base in "${MODEL_LIST[@]}"; do
|
||||
RESULT_FILE="$RESULTS_DIR/$base.json"
|
||||
|
||||
if [[ ! -f "$RESULT_FILE" ]]; then
|
||||
ERRORS=$((ERRORS + 1))
|
||||
FAILED_MODELS="$FAILED_MODELS $base"
|
||||
continue
|
||||
fi
|
||||
|
||||
ENTRY=$(cat "$RESULT_FILE")
|
||||
|
||||
# Check for error marker
|
||||
if echo "$ENTRY" | node -e "const d=JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));process.exit(d.error?0:1)" 2>/dev/null; then
|
||||
ERRORS=$((ERRORS + 1))
|
||||
FAILED_MODELS="$FAILED_MODELS $base"
|
||||
continue
|
||||
fi
|
||||
|
||||
RESEARCHED=$((RESEARCHED + 1))
|
||||
|
||||
# Diff detection: compare with existing entry
|
||||
OLD_ENTRY=$(echo "$EXISTING_CATALOG" | node -e "
|
||||
const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
const e = d['$base'];
|
||||
if (e) { delete e.resolvedAt; process.stdout.write(JSON.stringify(e)); }
|
||||
else process.stdout.write('null');
|
||||
" 2>/dev/null)
|
||||
|
||||
NEW_FOR_DIFF=$(echo "$ENTRY" | node -e "
|
||||
const d = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
delete d.resolvedAt;
|
||||
process.stdout.write(JSON.stringify(d));
|
||||
" 2>/dev/null)
|
||||
|
||||
if [[ "$OLD_ENTRY" != "null" && "$OLD_ENTRY" != "$NEW_FOR_DIFF" ]]; then
|
||||
CHANGED=$((CHANGED + 1))
|
||||
# Log what changed
|
||||
node -e "
|
||||
const old = JSON.parse('$OLD_ENTRY');
|
||||
const cur = JSON.parse('$NEW_FOR_DIFF');
|
||||
const changes = [];
|
||||
for (const k of new Set([...Object.keys(old), ...Object.keys(cur)])) {
|
||||
const o = JSON.stringify(old[k]); const n = JSON.stringify(cur[k]);
|
||||
if (o !== n) changes.push(k + ': ' + o + ' → ' + n);
|
||||
}
|
||||
if (changes.length) console.log(' CHANGED: ' + changes.join(', '));
|
||||
" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Apply to all variants of this base model
|
||||
for variant in ${BASE_TO_VARIANTS[$base]}; do
|
||||
CATALOG=$(echo "$CATALOG" | node -e "
|
||||
const catalog = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
catalog['$variant'] = $ENTRY;
|
||||
process.stdout.write(JSON.stringify(catalog));
|
||||
")
|
||||
done
|
||||
done
|
||||
|
||||
# Write final catalog
|
||||
echo "$CATALOG" | node -e "
|
||||
const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf-8'));
|
||||
const sorted = Object.keys(data).sort().reduce((acc, k) => { acc[k] = data[k]; return acc; }, {});
|
||||
process.stdout.write(JSON.stringify(sorted, null, 2) + '\n');
|
||||
" > "$CATALOG_FILE"
|
||||
|
||||
# Cleanup results
|
||||
rm -rf "$RESULTS_DIR"
|
||||
|
||||
FINAL_COUNT=$(node -e "console.log(Object.keys(JSON.parse(require('fs').readFileSync('$CATALOG_FILE','utf-8'))).length)")
|
||||
echo ""
|
||||
echo "Done! $FINAL_COUNT entries in catalog"
|
||||
echo " Researched: $RESEARCHED | Changed: $CHANGED | Skipped: $SKIPPED | Errors: $ERRORS"
|
||||
|
||||
if [[ "$ERRORS" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "Failed models:$FAILED_MODELS"
|
||||
RETRY_PATTERN=$(echo "$FAILED_MODELS" | tr ' ' '\n' | grep -v '^$' | sed 's/\./\\./g; s/^/^/; s/$/$/' | paste -sd '|' -)
|
||||
echo "Retry with: $0 --filter \"$RETRY_PATTERN\""
|
||||
fi
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Cross-platform generation script for the llm-pricing package.
|
||||
// Generates TypeScript modules from JSON data files:
|
||||
// 1. defaultPrices.ts ← default-model-prices.json (synced from Langfuse)
|
||||
// 2. modelCatalog.ts ← model-catalog.json (our maintained catalog metadata)
|
||||
//
|
||||
// Usage: node scripts/generate.mjs
|
||||
//
|
||||
// To update the source JSON files:
|
||||
// - Pricing: pnpm run sync-prices (fetches from Langfuse, requires curl)
|
||||
// - Catalog: pnpm run generate-catalog (uses Claude CLI to research models)
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const srcDir = join(__dirname, "..", "src");
|
||||
|
||||
// --- 1. Generate defaultPrices.ts from default-model-prices.json ---
|
||||
|
||||
const pricesJsonPath = join(srcDir, "default-model-prices.json");
|
||||
|
||||
if (existsSync(pricesJsonPath)) {
|
||||
const raw = JSON.parse(readFileSync(pricesJsonPath, "utf-8"));
|
||||
const stripped = raw.map((e) => ({
|
||||
modelName: e.modelName.trim(),
|
||||
matchPattern: e.matchPattern,
|
||||
startDate: e.createdAt,
|
||||
pricingTiers: e.pricingTiers.map((t) => ({
|
||||
name: t.name,
|
||||
isDefault: t.isDefault,
|
||||
priority: t.priority,
|
||||
conditions: t.conditions.map((c) => ({
|
||||
usageDetailPattern: c.usageDetailPattern,
|
||||
operator: c.operator,
|
||||
value: c.value,
|
||||
})),
|
||||
prices: t.prices,
|
||||
})),
|
||||
}));
|
||||
|
||||
let out = 'import type { DefaultModelDefinition } from "./types.js";\n\n';
|
||||
out += "// Auto-generated from default-model-prices.json — do not edit manually.\n";
|
||||
out +=
|
||||
"// Run `pnpm run sync-prices` to update the JSON, then `pnpm run generate` to regenerate.\n";
|
||||
out += "// Source: https://github.com/langfuse/langfuse\n\n";
|
||||
out += "export const defaultModelPrices: DefaultModelDefinition[] = ";
|
||||
out += JSON.stringify(stripped, null, 2) + ";\n";
|
||||
|
||||
writeFileSync(join(srcDir, "defaultPrices.ts"), out);
|
||||
console.log(`Generated defaultPrices.ts (${stripped.length} models)`);
|
||||
} else {
|
||||
console.log("Skipping defaultPrices.ts — default-model-prices.json not found");
|
||||
}
|
||||
|
||||
// --- 2. Generate modelCatalog.ts from model-catalog.json ---
|
||||
|
||||
const catalogJsonPath = join(srcDir, "model-catalog.json");
|
||||
|
||||
if (existsSync(catalogJsonPath)) {
|
||||
const data = JSON.parse(readFileSync(catalogJsonPath, "utf-8"));
|
||||
|
||||
// Backfill missing fields for old entries
|
||||
for (const key of Object.keys(data)) {
|
||||
if (data[key].releaseDate === undefined) data[key].releaseDate = null;
|
||||
if (data[key].isHidden === undefined) data[key].isHidden = false;
|
||||
if (data[key].supportsStructuredOutput === undefined)
|
||||
data[key].supportsStructuredOutput = false;
|
||||
if (data[key].supportsParallelToolCalls === undefined)
|
||||
data[key].supportsParallelToolCalls = false;
|
||||
if (data[key].supportsStreamingToolCalls === undefined)
|
||||
data[key].supportsStreamingToolCalls = false;
|
||||
if (data[key].deprecationDate === undefined) data[key].deprecationDate = null;
|
||||
if (data[key].knowledgeCutoff === undefined) data[key].knowledgeCutoff = null;
|
||||
if (data[key].resolvedAt === undefined) data[key].resolvedAt = new Date().toISOString();
|
||||
{
|
||||
// Always recompute base model name (don't trust existing values)
|
||||
// Strip trailing date (-YYYYMMDD or -YYYY-MM-DD) and -latest suffix
|
||||
// Keep original naming (dots, etc.) — don't normalize
|
||||
let base = key.replace(/-\d{4}-?\d{2}-?\d{2}$/, "").replace(/-latest$/, "");
|
||||
data[key].baseModelName = base !== key ? base : null;
|
||||
}
|
||||
}
|
||||
|
||||
let out = 'import type { ModelCatalogEntry } from "./types.js";\n\n';
|
||||
out += "// Auto-generated from model-catalog.json — do not edit manually.\n";
|
||||
out +=
|
||||
"// Run `pnpm run generate-catalog` to update the JSON, then `pnpm run generate` to regenerate.\n\n";
|
||||
out += "export const modelCatalog: Record<string, ModelCatalogEntry> = ";
|
||||
out += JSON.stringify(data, null, 2) + ";\n";
|
||||
|
||||
writeFileSync(join(srcDir, "modelCatalog.ts"), out);
|
||||
console.log(`Generated modelCatalog.ts (${Object.keys(data).length} entries)`);
|
||||
} else {
|
||||
console.log("Skipping modelCatalog.ts — model-catalog.json not found");
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Research a single LLM model using Claude Code CLI and output structured JSON.
|
||||
# Usage: ./scripts/research-model.sh <model-name>
|
||||
#
|
||||
# Example:
|
||||
# ./scripts/research-model.sh gpt-4o
|
||||
# → {"provider":"openai","description":"...","contextWindow":128000,...}
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 <model-name>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MODEL_NAME="$1"
|
||||
MAX_RETRIES="${2:-3}"
|
||||
|
||||
PROMPT="Research the LLM model '${MODEL_NAME}' and return ONLY a valid JSON object (no markdown, no explanation, no code fences) with these exact fields:
|
||||
|
||||
{
|
||||
\"provider\": \"<provider-id>\",
|
||||
\"description\": \"<1-2 sentence description of the model>\",
|
||||
\"contextWindow\": <max input context window in tokens, or null if unknown>,
|
||||
\"maxOutputTokens\": <max output tokens, or null if unknown>,
|
||||
\"capabilities\": [<list of capability strings>],
|
||||
\"releaseDate\": \"<YYYY-MM-DD or null if unknown>\",
|
||||
\"isHidden\": <true if deprecated, false otherwise>,
|
||||
\"supportsStructuredOutput\": <true/false>,
|
||||
\"supportsParallelToolCalls\": <true/false>,
|
||||
\"supportsStreamingToolCalls\": <true/false>,
|
||||
\"deprecationDate\": \"<YYYY-MM-DD or null if no known sunset date>\",
|
||||
\"knowledgeCutoff\": \"<YYYY-MM-DD or null if unknown>\"
|
||||
}
|
||||
|
||||
Rules:
|
||||
- provider must be one of: \"openai\", \"anthropic\", \"google\", \"meta\", \"mistral\", \"cohere\", \"ai21\", \"amazon\", \"xai\", \"deepseek\", \"qwen\", \"perplexity\" or the correct provider lowercase id
|
||||
- description should be concise and factual (what the model is good at, its position in the provider's lineup)
|
||||
- contextWindow is the maximum input context in tokens (e.g. 128000 for GPT-4o, 200000 for Claude Sonnet 4)
|
||||
- maxOutputTokens is the maximum output the model can generate in a single response
|
||||
- capabilities should be drawn from: \"vision\", \"tool_use\", \"streaming\", \"json_mode\", \"extended_thinking\", \"code_execution\", \"image_generation\", \"audio_input\", \"audio_output\", \"embedding\", \"fine_tunable\"
|
||||
- Only include capabilities you are confident the model supports
|
||||
- releaseDate is when the model was first publicly available (API launch date), in YYYY-MM-DD format. Use null if unknown. If the model is a dated variant (e.g. gpt-4o-2024-08-06), the date in the name IS the release date.
|
||||
- isHidden should be true if the model is deprecated, discontinued, no longer available via API, or superseded by a newer version. Examples: gpt-3.5-turbo, claude-1.x, claude-2.x, text-davinci-003, gpt-4-0314 are hidden. Current/active models like gpt-4o, claude-sonnet-4-6, gemini-2.5-flash are NOT hidden.
|
||||
- supportsStructuredOutput: true if the model reliably follows JSON schemas / structured output mode (e.g. OpenAI's response_format, Anthropic's tool_use for structured output). false for older models that don't support it well.
|
||||
- supportsParallelToolCalls: true if the model can call multiple tools in a single assistant turn. Most modern models support this.
|
||||
- supportsStreamingToolCalls: true if the model supports streaming partial tool call arguments as they're generated.
|
||||
- deprecationDate: the date the provider has announced the model will be sunset/removed from their API, in YYYY-MM-DD format. Use null if no deprecation date has been announced. Only use dates that have been officially published by the provider.
|
||||
- knowledgeCutoff: the date when the model's training data ends, in YYYY-MM-DD format. Use null if unknown. This is different from releaseDate — it's when the training data was cut off, not when the model launched.
|
||||
- Output ONLY the JSON object, nothing else"
|
||||
|
||||
for attempt in $(seq 1 "$MAX_RETRIES"); do
|
||||
RESULT=$(claude -p "$PROMPT" --model opus --output-format json --permission-mode bypassPermissions --tools WebSearch,WebFetch 2>/dev/null) && {
|
||||
echo "$RESULT"
|
||||
exit 0
|
||||
}
|
||||
if [[ "$attempt" -lt "$MAX_RETRIES" ]]; then
|
||||
echo " Retry $attempt/$MAX_RETRIES for $MODEL_NAME..." >&2
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
|
||||
echo " Failed after $MAX_RETRIES attempts for $MODEL_NAME" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Sync default model prices from Langfuse's repository and generate the TS module.
|
||||
# Usage: ./scripts/sync-model-prices.sh [--check]
|
||||
# --check: Exit 1 if prices are outdated (for CI)
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PACKAGE_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
JSON_TARGET="$PACKAGE_DIR/src/default-model-prices.json"
|
||||
SOURCE_URL="https://raw.githubusercontent.com/langfuse/langfuse/main/worker/src/constants/default-model-prices.json"
|
||||
|
||||
CHECK_MODE=false
|
||||
if [[ "${1:-}" == "--check" ]]; then
|
||||
CHECK_MODE=true
|
||||
fi
|
||||
|
||||
echo "Fetching latest model prices from Langfuse..."
|
||||
TMPFILE=$(mktemp)
|
||||
trap 'rm -f "$TMPFILE"' EXIT
|
||||
|
||||
if ! curl -fsSL "$SOURCE_URL" -o "$TMPFILE"; then
|
||||
echo "ERROR: Failed to fetch from $SOURCE_URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate it's valid JSON with at least some models
|
||||
MODEL_COUNT=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$TMPFILE','utf-8')).length)" 2>/dev/null || echo "0")
|
||||
if [[ "$MODEL_COUNT" -lt 10 ]]; then
|
||||
echo "ERROR: Downloaded file has only $MODEL_COUNT models (expected 100+). Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if $CHECK_MODE; then
|
||||
if diff -q "$JSON_TARGET" "$TMPFILE" > /dev/null 2>&1; then
|
||||
echo "Model prices are up to date ($MODEL_COUNT models)"
|
||||
exit 0
|
||||
else
|
||||
echo "Model prices are OUTDATED. Run 'pnpm run sync-prices' in @internal/llm-model-catalog to update."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
cp "$TMPFILE" "$JSON_TARGET"
|
||||
echo "Updated default-model-prices.json ($MODEL_COUNT models)"
|
||||
echo "Run 'pnpm run generate' to regenerate defaultPrices.ts"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
||||
export { ModelPricingRegistry } from "./registry.js";
|
||||
export { seedLlmPricing } from "./seed.js";
|
||||
export { syncLlmCatalog } from "./sync.js";
|
||||
export { defaultModelPrices } from "./defaultPrices.js";
|
||||
export { modelCatalog } from "./modelCatalog.js";
|
||||
export type {
|
||||
LlmModelWithPricing,
|
||||
LlmCostResult,
|
||||
LlmPricingTierWithPrices,
|
||||
LlmPriceEntry,
|
||||
PricingCondition,
|
||||
DefaultModelDefinition,
|
||||
ModelCatalogEntry,
|
||||
} from "./types.js";
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,521 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { ModelPricingRegistry } from "./registry.js";
|
||||
import { defaultModelPrices } from "./defaultPrices.js";
|
||||
import type { LlmModelWithPricing } from "./types.js";
|
||||
|
||||
// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag
|
||||
function compilePattern(pattern: string): RegExp {
|
||||
if (pattern.startsWith("(?i)")) {
|
||||
return new RegExp(pattern.slice(4), "i");
|
||||
}
|
||||
return new RegExp(pattern);
|
||||
}
|
||||
|
||||
// Create a mock registry that we can load with test data without Prisma
|
||||
class TestableRegistry extends ModelPricingRegistry {
|
||||
loadPatterns(models: LlmModelWithPricing[]) {
|
||||
// Access private fields via any cast for testing
|
||||
const self = this as any;
|
||||
self._patterns = models.map((model) => ({
|
||||
regex: compilePattern(model.matchPattern),
|
||||
model,
|
||||
}));
|
||||
self._exactMatchCache = new Map();
|
||||
self._loaded = true;
|
||||
}
|
||||
}
|
||||
|
||||
const gpt4o: LlmModelWithPricing = {
|
||||
id: "model-gpt4o",
|
||||
friendlyId: "llm_model_gpt4o",
|
||||
modelName: "gpt-4o",
|
||||
matchPattern: "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$",
|
||||
startDate: null,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-gpt4o-standard",
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.0000025 },
|
||||
{ usageType: "output", price: 0.00001 },
|
||||
{ usageType: "input_cached_tokens", price: 0.00000125 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const claudeSonnet: LlmModelWithPricing = {
|
||||
id: "model-claude-sonnet",
|
||||
friendlyId: "llm_model_claude_sonnet",
|
||||
modelName: "claude-sonnet-4-0",
|
||||
matchPattern: "^claude-sonnet-4-0(-\\d{8})?$",
|
||||
startDate: null,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-claude-sonnet-standard",
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.000003 },
|
||||
{ usageType: "output", price: 0.000015 },
|
||||
{ usageType: "input_cached_tokens", price: 0.0000015 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Prices cache reads under the Anthropic-style alias `cache_read_input_tokens` (not
|
||||
// `input_cached_tokens`) plus a cache-creation price, to exercise alias resolution.
|
||||
const claudeWithCache: LlmModelWithPricing = {
|
||||
id: "model-claude-with-cache",
|
||||
friendlyId: "llm_model_claude_with_cache",
|
||||
modelName: "claude-with-cache",
|
||||
matchPattern: "^claude-with-cache$",
|
||||
startDate: null,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-claude-with-cache",
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.000003 },
|
||||
{ usageType: "output", price: 0.000015 },
|
||||
{ usageType: "cache_read_input_tokens", price: 0.0000003 },
|
||||
{ usageType: "cache_creation_input_tokens", price: 0.00000375 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// No cache prices at all — cached tokens should fall back to the input price.
|
||||
const noCachePrice: LlmModelWithPricing = {
|
||||
id: "model-no-cache-price",
|
||||
friendlyId: "llm_model_no_cache_price",
|
||||
modelName: "no-cache-price",
|
||||
matchPattern: "^no-cache-price$",
|
||||
startDate: null,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-no-cache-price",
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.000003 },
|
||||
{ usageType: "output", price: 0.000015 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("ModelPricingRegistry", () => {
|
||||
let registry: TestableRegistry;
|
||||
|
||||
beforeEach(() => {
|
||||
registry = new TestableRegistry(null as any);
|
||||
registry.loadPatterns([gpt4o, claudeSonnet, claudeWithCache, noCachePrice]);
|
||||
});
|
||||
|
||||
describe("match", () => {
|
||||
it("should match exact model name", () => {
|
||||
const result = registry.match("gpt-4o");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.modelName).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("should match model with date suffix", () => {
|
||||
const result = registry.match("gpt-4o-2024-08-06");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.modelName).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("should match claude model", () => {
|
||||
const result = registry.match("claude-sonnet-4-0-20250514");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.modelName).toBe("claude-sonnet-4-0");
|
||||
});
|
||||
|
||||
it("should return null for unknown model", () => {
|
||||
const result = registry.match("unknown-model-xyz");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("should cache exact matches", () => {
|
||||
registry.match("gpt-4o");
|
||||
registry.match("gpt-4o");
|
||||
// Second call should use cache - no way to verify without mocking, but it shouldn't error
|
||||
expect(registry.match("gpt-4o")!.modelName).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("should cache misses", () => {
|
||||
expect(registry.match("unknown")).toBeNull();
|
||||
expect(registry.match("unknown")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("calculateCost", () => {
|
||||
it("should calculate cost for input and output tokens", () => {
|
||||
const result = registry.calculateCost("gpt-4o", {
|
||||
input: 1000,
|
||||
output: 100,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.matchedModelName).toBe("gpt-4o");
|
||||
expect(result!.pricingTierName).toBe("Standard");
|
||||
expect(result!.inputCost).toBeCloseTo(0.0025); // 1000 * 0.0000025
|
||||
expect(result!.outputCost).toBeCloseTo(0.001); // 100 * 0.00001
|
||||
expect(result!.totalCost).toBeCloseTo(0.0035);
|
||||
});
|
||||
|
||||
it("should include cached token costs and charge input only on the fresh portion", () => {
|
||||
// input_tokens (500) is inclusive of the 200 cached read tokens, so the input price
|
||||
// applies to the 300 fresh tokens and the cache price to the 200 cached tokens — the
|
||||
// cached tokens must not be billed twice.
|
||||
const result = registry.calculateCost("gpt-4o", {
|
||||
input: 500,
|
||||
output: 50,
|
||||
input_cached_tokens: 200,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.costDetails["input"]).toBeCloseTo(0.00075); // (500 - 200) * 0.0000025
|
||||
expect(result!.costDetails["output"]).toBeCloseTo(0.0005); // 50 * 0.00001
|
||||
expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.00025); // 200 * 0.00000125
|
||||
expect(result!.totalCost).toBeCloseTo(0.0015);
|
||||
});
|
||||
|
||||
it("should not double-charge cache creation tokens (subset of input)", () => {
|
||||
// input (1000) is inclusive of both the 400 cache-read and 300 cache-creation tokens.
|
||||
const result = registry.calculateCost("claude-with-cache", {
|
||||
input: 1000,
|
||||
output: 100,
|
||||
input_cached_tokens: 400,
|
||||
cache_creation_input_tokens: 300,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
// fresh input = 1000 - 400 - 300 = 300
|
||||
expect(result!.costDetails["input"]).toBeCloseTo(0.0009); // 300 * 0.000003
|
||||
expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.00012); // 400 * 0.0000003
|
||||
expect(result!.costDetails["cache_creation_input_tokens"]).toBeCloseTo(0.001125); // 300 * 0.00000375
|
||||
expect(result!.costDetails["output"]).toBeCloseTo(0.0015); // 100 * 0.000015
|
||||
// 0.0009 + 0.00012 + 0.001125 + 0.0015
|
||||
expect(result!.totalCost).toBeCloseTo(0.003645);
|
||||
});
|
||||
|
||||
it("should apply the cache-read discount when priced under a provider alias key", () => {
|
||||
// The usage is normalized to `input_cached_tokens` but this model prices cache reads
|
||||
// under `cache_read_input_tokens` — the discount must still apply.
|
||||
const result = registry.calculateCost("claude-with-cache", {
|
||||
input: 1000,
|
||||
input_cached_tokens: 400,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.costDetails["input"]).toBeCloseTo(0.0018); // (1000 - 400) * 0.000003
|
||||
expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.00012); // 400 * 0.0000003
|
||||
expect(result!.totalCost).toBeCloseTo(0.00192);
|
||||
});
|
||||
|
||||
it("should fall back to the input price for cache tokens when no cache price exists", () => {
|
||||
// no-cache-price model has only input/output prices; cached tokens must still be billed
|
||||
// (at the input price) — never free, never double-charged. Total equals input * price.
|
||||
const result = registry.calculateCost("no-cache-price", {
|
||||
input: 1000,
|
||||
input_cached_tokens: 400,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.costDetails["input"]).toBeCloseTo(0.0018); // (1000 - 400) * 0.000003
|
||||
expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.0012); // 400 * 0.000003
|
||||
expect(result!.totalCost).toBeCloseTo(0.003); // 1000 * 0.000003 — unchanged from no-cache behavior
|
||||
});
|
||||
|
||||
it("should return null for unknown model", () => {
|
||||
const result = registry.calculateCost("unknown-model", { input: 100, output: 50 });
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("should handle zero tokens", () => {
|
||||
const result = registry.calculateCost("gpt-4o", { input: 0, output: 0 });
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.totalCost).toBe(0);
|
||||
});
|
||||
|
||||
it("should handle missing usage types gracefully", () => {
|
||||
const result = registry.calculateCost("gpt-4o", { input: 100 });
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.inputCost).toBeCloseTo(0.00025);
|
||||
expect(result!.outputCost).toBe(0); // No output tokens
|
||||
expect(result!.totalCost).toBeCloseTo(0.00025);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isLoaded", () => {
|
||||
it("should return false before loading", () => {
|
||||
const freshRegistry = new TestableRegistry(null as any);
|
||||
expect(freshRegistry.isLoaded).toBe(false);
|
||||
});
|
||||
|
||||
it("should return true after loading", () => {
|
||||
expect(registry.isLoaded).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefix stripping", () => {
|
||||
it("should match gateway-prefixed model names", () => {
|
||||
const result = registry.match("openai/gpt-4o");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.modelName).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("should match openrouter-prefixed model names with date suffix", () => {
|
||||
const result = registry.match("openai/gpt-4o-2024-08-06");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.modelName).toBe("gpt-4o");
|
||||
});
|
||||
|
||||
it("should return null for prefixed unknown model", () => {
|
||||
const result = registry.match("xai/unknown-model");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("tier matching", () => {
|
||||
const multiTierModel: LlmModelWithPricing = {
|
||||
id: "model-gemini-pro",
|
||||
friendlyId: "llm_model_gemini_pro",
|
||||
modelName: "gemini-2.5-pro",
|
||||
matchPattern: "^gemini-2\\.5-pro$",
|
||||
startDate: null,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-large-context",
|
||||
name: "Large Context",
|
||||
isDefault: false,
|
||||
priority: 0,
|
||||
conditions: [{ usageDetailPattern: "input", operator: "gt" as const, value: 200000 }],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.0000025 },
|
||||
{ usageType: "output", price: 0.00001 },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "tier-standard",
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 1,
|
||||
conditions: [],
|
||||
prices: [
|
||||
{ usageType: "input", price: 0.00000125 },
|
||||
{ usageType: "output", price: 0.000005 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
it("should use conditional tier when conditions match", () => {
|
||||
const tieredRegistry = new TestableRegistry(null as any);
|
||||
tieredRegistry.loadPatterns([multiTierModel]);
|
||||
|
||||
const result = tieredRegistry.calculateCost("gemini-2.5-pro", {
|
||||
input: 250000,
|
||||
output: 1000,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.pricingTierName).toBe("Large Context");
|
||||
expect(result!.inputCost).toBeCloseTo(0.625); // 250000 * 0.0000025
|
||||
});
|
||||
|
||||
it("should fall back to default tier when conditions do not match", () => {
|
||||
const tieredRegistry = new TestableRegistry(null as any);
|
||||
tieredRegistry.loadPatterns([multiTierModel]);
|
||||
|
||||
const result = tieredRegistry.calculateCost("gemini-2.5-pro", {
|
||||
input: 1000,
|
||||
output: 100,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.pricingTierName).toBe("Standard");
|
||||
expect(result!.inputCost).toBeCloseTo(0.00125); // 1000 * 0.00000125
|
||||
});
|
||||
|
||||
it("should not let unconditional tier win over conditional match", () => {
|
||||
// Model where unconditional tier has lower priority than conditional
|
||||
const model: LlmModelWithPricing = {
|
||||
...multiTierModel,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-unconditional",
|
||||
name: "Unconditional",
|
||||
isDefault: false,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: [{ usageType: "input", price: 0.001 }],
|
||||
},
|
||||
{
|
||||
id: "tier-conditional",
|
||||
name: "Conditional",
|
||||
isDefault: false,
|
||||
priority: 1,
|
||||
conditions: [{ usageDetailPattern: "input", operator: "gt" as const, value: 100 }],
|
||||
prices: [{ usageType: "input", price: 0.0001 }],
|
||||
},
|
||||
{
|
||||
id: "tier-default",
|
||||
name: "Default",
|
||||
isDefault: true,
|
||||
priority: 2,
|
||||
conditions: [],
|
||||
prices: [{ usageType: "input", price: 0.01 }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const tieredRegistry = new TestableRegistry(null as any);
|
||||
tieredRegistry.loadPatterns([model]);
|
||||
|
||||
// Condition matches — conditional tier should win, not the unconditional one
|
||||
const result = tieredRegistry.calculateCost("gemini-2.5-pro", { input: 500 });
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.pricingTierName).toBe("Conditional");
|
||||
});
|
||||
|
||||
it("should fall back to isDefault tier when no conditions match", () => {
|
||||
const model: LlmModelWithPricing = {
|
||||
...multiTierModel,
|
||||
pricingTiers: [
|
||||
{
|
||||
id: "tier-conditional",
|
||||
name: "Conditional",
|
||||
isDefault: false,
|
||||
priority: 0,
|
||||
conditions: [{ usageDetailPattern: "input", operator: "gt" as const, value: 999999 }],
|
||||
prices: [{ usageType: "input", price: 0.001 }],
|
||||
},
|
||||
{
|
||||
id: "tier-default",
|
||||
name: "Default",
|
||||
isDefault: true,
|
||||
priority: 1,
|
||||
conditions: [],
|
||||
prices: [{ usageType: "input", price: 0.0001 }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const tieredRegistry = new TestableRegistry(null as any);
|
||||
tieredRegistry.loadPatterns([model]);
|
||||
|
||||
const result = tieredRegistry.calculateCost("gemini-2.5-pro", { input: 100 });
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.pricingTierName).toBe("Default");
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultModelPrices (Langfuse JSON)", () => {
|
||||
it("should load all models from the JSON file", () => {
|
||||
expect(defaultModelPrices.length).toBeGreaterThan(100);
|
||||
});
|
||||
|
||||
it("should compile all match patterns without errors", () => {
|
||||
const langfuseRegistry = new TestableRegistry(null as any);
|
||||
const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({
|
||||
id: `test-${i}`,
|
||||
friendlyId: `llm_model_test${i}`,
|
||||
modelName: def.modelName,
|
||||
matchPattern: def.matchPattern,
|
||||
startDate: def.startDate ? new Date(def.startDate) : null,
|
||||
pricingTiers: def.pricingTiers.map((tier, j) => ({
|
||||
id: `tier-${i}-${j}`,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: tier.conditions,
|
||||
prices: Object.entries(tier.prices).map(([usageType, price]) => ({
|
||||
usageType,
|
||||
price,
|
||||
})),
|
||||
})),
|
||||
}));
|
||||
|
||||
// This should not throw — all 141 patterns should compile
|
||||
expect(() => langfuseRegistry.loadPatterns(models)).not.toThrow();
|
||||
expect(langfuseRegistry.isLoaded).toBe(true);
|
||||
});
|
||||
|
||||
it("should match real-world model names from Langfuse patterns", () => {
|
||||
const langfuseRegistry = new TestableRegistry(null as any);
|
||||
const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({
|
||||
id: `test-${i}`,
|
||||
friendlyId: `llm_model_test${i}`,
|
||||
modelName: def.modelName,
|
||||
matchPattern: def.matchPattern,
|
||||
startDate: null,
|
||||
pricingTiers: def.pricingTiers.map((tier, j) => ({
|
||||
id: `tier-${i}-${j}`,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: tier.conditions,
|
||||
prices: Object.entries(tier.prices).map(([usageType, price]) => ({
|
||||
usageType,
|
||||
price,
|
||||
})),
|
||||
})),
|
||||
}));
|
||||
langfuseRegistry.loadPatterns(models);
|
||||
|
||||
// Test real model strings that SDKs send
|
||||
expect(langfuseRegistry.match("gpt-4o")).not.toBeNull();
|
||||
expect(langfuseRegistry.match("gpt-4o-mini")).not.toBeNull();
|
||||
expect(langfuseRegistry.match("claude-sonnet-4-5-20250929")).not.toBeNull();
|
||||
expect(langfuseRegistry.match("claude-sonnet-4-20250514")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadFromModels / toSerializable (worker in-memory load)", () => {
|
||||
it("matches and prices from in-memory models without a DB", () => {
|
||||
const reg = new ModelPricingRegistry();
|
||||
reg.loadFromModels([gpt4o, claudeSonnet]);
|
||||
|
||||
expect(reg.isLoaded).toBe(true);
|
||||
expect(reg.match("gpt-4o")).not.toBeNull();
|
||||
expect(reg.match("gpt-4o-2024-08-06")).not.toBeNull();
|
||||
|
||||
const cost = reg.calculateCost("gpt-4o", { input: 1000, output: 500 });
|
||||
expect(cost).not.toBeNull();
|
||||
expect(cost!.totalCost).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("round-trips through toSerializable (the main->worker broadcast shape)", () => {
|
||||
const source = new ModelPricingRegistry();
|
||||
source.loadFromModels([gpt4o, claudeSonnet]);
|
||||
|
||||
const worker = new ModelPricingRegistry();
|
||||
worker.loadFromModels(source.toSerializable());
|
||||
|
||||
for (const m of ["gpt-4o", "gpt-4o-2024-08-06", "claude-sonnet-4-0", "unknown-model"]) {
|
||||
expect(worker.match(m)).toEqual(source.match(m));
|
||||
}
|
||||
const usage = { input: 1234, output: 567 };
|
||||
expect(worker.calculateCost("gpt-4o", usage)).toEqual(source.calculateCost("gpt-4o", usage));
|
||||
});
|
||||
|
||||
it("throws if loadFromDatabase is called without a prisma client", async () => {
|
||||
const reg = new ModelPricingRegistry();
|
||||
await expect(reg.loadFromDatabase()).rejects.toThrow(/requires a prisma client/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database";
|
||||
import type {
|
||||
LlmModelWithPricing,
|
||||
LlmCostResult,
|
||||
LlmPricingTierWithPrices,
|
||||
PricingCondition,
|
||||
} from "./types.js";
|
||||
|
||||
type CompiledPattern = {
|
||||
regex: RegExp;
|
||||
model: LlmModelWithPricing;
|
||||
};
|
||||
|
||||
// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag
|
||||
function compilePattern(pattern: string): RegExp {
|
||||
if (pattern.startsWith("(?i)")) {
|
||||
return new RegExp(pattern.slice(4), "i");
|
||||
}
|
||||
return new RegExp(pattern);
|
||||
}
|
||||
|
||||
export class ModelPricingRegistry {
|
||||
private _prisma?: PrismaClient | PrismaReplicaClient;
|
||||
private _patterns: CompiledPattern[] = [];
|
||||
// TODO: When we add project-based models (users adding their own), this cache grows unbounded
|
||||
// between reloads. Fine-tuned model IDs (e.g. "ft:gpt-3.5-turbo:org:name:id") create unique
|
||||
// entries per model string. Consider adding an LRU cap or size limit at that point.
|
||||
private _exactMatchCache: Map<string, LlmModelWithPricing | null> = new Map();
|
||||
private _loaded = false;
|
||||
private _readyResolve!: () => void;
|
||||
|
||||
/** Resolves once the initial `loadFromDatabase()` completes successfully. */
|
||||
readonly isReady: Promise<void>;
|
||||
|
||||
constructor(prisma?: PrismaClient | PrismaReplicaClient) {
|
||||
this._prisma = prisma;
|
||||
this.isReady = new Promise<void>((resolve) => {
|
||||
this._readyResolve = resolve;
|
||||
});
|
||||
}
|
||||
|
||||
get isLoaded(): boolean {
|
||||
return this._loaded;
|
||||
}
|
||||
|
||||
async loadFromDatabase(): Promise<void> {
|
||||
if (!this._prisma) {
|
||||
throw new Error("loadFromDatabase requires a prisma client; use loadFromModels in a worker");
|
||||
}
|
||||
|
||||
const models = await this._prisma.llmModel.findMany({
|
||||
where: { projectId: null },
|
||||
include: {
|
||||
pricingTiers: {
|
||||
include: { prices: true },
|
||||
orderBy: { priority: "asc" },
|
||||
},
|
||||
},
|
||||
orderBy: [{ startDate: "desc" }],
|
||||
});
|
||||
|
||||
const compiled: CompiledPattern[] = [];
|
||||
|
||||
for (const model of models) {
|
||||
try {
|
||||
const regex = compilePattern(model.matchPattern);
|
||||
const tiers: LlmPricingTierWithPrices[] = model.pricingTiers.map((tier) => ({
|
||||
id: tier.id,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: (tier.conditions as PricingCondition[]) ?? [],
|
||||
prices: tier.prices.map((p) => ({
|
||||
usageType: p.usageType,
|
||||
price: Number(p.price),
|
||||
})),
|
||||
}));
|
||||
|
||||
compiled.push({
|
||||
regex,
|
||||
model: {
|
||||
id: model.id,
|
||||
friendlyId: model.friendlyId,
|
||||
modelName: model.modelName,
|
||||
matchPattern: model.matchPattern,
|
||||
startDate: model.startDate,
|
||||
pricingTiers: tiers,
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Skip models with invalid regex patterns
|
||||
console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.applyPatterns(compiled);
|
||||
}
|
||||
|
||||
// Build the registry from already-loaded model rows (e.g. broadcast to a worker), no DB.
|
||||
loadFromModels(models: LlmModelWithPricing[]): void {
|
||||
const compiled: CompiledPattern[] = [];
|
||||
|
||||
for (const model of models) {
|
||||
try {
|
||||
compiled.push({ regex: compilePattern(model.matchPattern), model });
|
||||
} catch {
|
||||
console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.applyPatterns(compiled);
|
||||
}
|
||||
|
||||
// Serializable snapshot of the loaded models (safe to postMessage to a worker).
|
||||
toSerializable(): LlmModelWithPricing[] {
|
||||
return this._patterns.map((p) => p.model);
|
||||
}
|
||||
|
||||
private applyPatterns(compiled: CompiledPattern[]): void {
|
||||
this._patterns = compiled;
|
||||
this._exactMatchCache.clear();
|
||||
|
||||
if (!this._loaded) {
|
||||
this._loaded = true;
|
||||
this._readyResolve();
|
||||
}
|
||||
}
|
||||
|
||||
async reload(): Promise<void> {
|
||||
await this.loadFromDatabase();
|
||||
}
|
||||
|
||||
match(responseModel: string): LlmModelWithPricing | null {
|
||||
if (!this._loaded) return null;
|
||||
|
||||
// Check exact match cache
|
||||
const cached = this._exactMatchCache.get(responseModel);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
// Iterate compiled regex patterns
|
||||
for (const { regex, model } of this._patterns) {
|
||||
if (regex.test(responseModel)) {
|
||||
this._exactMatchCache.set(responseModel, model);
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: strip provider prefix (e.g. "mistral/mistral-large-3" → "mistral-large-3")
|
||||
// Gateway and OpenRouter prepend the provider to the model name.
|
||||
if (responseModel.includes("/")) {
|
||||
const stripped = responseModel.split("/").slice(1).join("/");
|
||||
for (const { regex, model } of this._patterns) {
|
||||
if (regex.test(stripped)) {
|
||||
this._exactMatchCache.set(responseModel, model);
|
||||
return model;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cache miss
|
||||
this._exactMatchCache.set(responseModel, null);
|
||||
return null;
|
||||
}
|
||||
|
||||
calculateCost(responseModel: string, usageDetails: Record<string, number>): LlmCostResult | null {
|
||||
const model = this.match(responseModel);
|
||||
if (!model) return null;
|
||||
|
||||
const tier = this._matchPricingTier(model.pricingTiers, usageDetails);
|
||||
if (!tier) return null;
|
||||
|
||||
const costDetails: Record<string, number> = {};
|
||||
let totalCost = 0;
|
||||
|
||||
// `input_tokens` (the "input" usage value) is the TOTAL prompt token count and is
|
||||
// inclusive of cache-read and cache-creation tokens — providers report it that way and
|
||||
// the AI SDK passes it through (verified: total_tokens == input + output, never the
|
||||
// sum of the decomposed parts). Cache reads/writes are therefore a SUBSET of input, not
|
||||
// additional to it. Charging the full input count at the input price AND charging a
|
||||
// separate cache line double-counts those tokens, so the input price must apply only to
|
||||
// the fresh (non-cached) remainder.
|
||||
const priceByType = new Map(tier.prices.map((p) => [p.usageType, p.price]));
|
||||
const resolvePrice = (aliases: string[]): number | undefined => {
|
||||
for (const alias of aliases) {
|
||||
const price = priceByType.get(alias);
|
||||
if (price !== undefined) return price;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const inputPrice = resolvePrice(["input", "input_tokens"]) ?? 0;
|
||||
const cacheReadTokens = usageDetails["input_cached_tokens"] ?? 0;
|
||||
const cacheCreationTokens = usageDetails["cache_creation_input_tokens"] ?? 0;
|
||||
|
||||
// Providers price cache reads/writes under provider-specific keys, but our usage details
|
||||
// normalize them to `input_cached_tokens` / `cache_creation_input_tokens`. Resolve the
|
||||
// matching price across the known aliases, falling back to the input price so cache tokens
|
||||
// are never billed for free and never dropped when a model lacks a dedicated cache price.
|
||||
const cacheReadPrice =
|
||||
resolvePrice(["input_cached_tokens", "input_cache_read", "cache_read_input_tokens"]) ??
|
||||
inputPrice;
|
||||
const cacheCreationPrice =
|
||||
resolvePrice([
|
||||
"cache_creation_input_tokens",
|
||||
"input_cache_creation",
|
||||
"input_cache_creation_5m",
|
||||
"input_cache_creation_1h",
|
||||
]) ?? inputPrice;
|
||||
|
||||
const totalInputTokens = usageDetails["input"] ?? usageDetails["input_tokens"] ?? 0;
|
||||
const freshInputTokens = Math.max(0, totalInputTokens - cacheReadTokens - cacheCreationTokens);
|
||||
|
||||
const addCost = (usageType: string, tokenCount: number, price: number) => {
|
||||
if (tokenCount <= 0 || price <= 0) return;
|
||||
const cost = tokenCount * price;
|
||||
costDetails[usageType] = (costDetails[usageType] ?? 0) + cost;
|
||||
totalCost += cost;
|
||||
};
|
||||
|
||||
addCost("input", freshInputTokens, inputPrice);
|
||||
addCost("input_cached_tokens", cacheReadTokens, cacheReadPrice);
|
||||
addCost("cache_creation_input_tokens", cacheCreationTokens, cacheCreationPrice);
|
||||
|
||||
// Charge every remaining usage type generically. The input + cache types are handled
|
||||
// above (and their alias keys skipped here) so they are never charged twice.
|
||||
const handledUsageTypes = new Set([
|
||||
"input",
|
||||
"input_tokens",
|
||||
"input_cached_tokens",
|
||||
"input_cache_read",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation_input_tokens",
|
||||
"input_cache_creation",
|
||||
"input_cache_creation_5m",
|
||||
"input_cache_creation_1h",
|
||||
]);
|
||||
for (const priceEntry of tier.prices) {
|
||||
if (handledUsageTypes.has(priceEntry.usageType)) continue;
|
||||
const tokenCount = usageDetails[priceEntry.usageType] ?? 0;
|
||||
if (tokenCount === 0) continue;
|
||||
const cost = tokenCount * priceEntry.price;
|
||||
costDetails[priceEntry.usageType] = cost;
|
||||
totalCost += cost;
|
||||
}
|
||||
|
||||
const inputCost = costDetails["input"] ?? 0;
|
||||
const outputCost = costDetails["output"] ?? 0;
|
||||
|
||||
return {
|
||||
matchedModelId: model.friendlyId,
|
||||
matchedModelName: model.modelName,
|
||||
pricingTierId: tier.id,
|
||||
pricingTierName: tier.name,
|
||||
inputCost,
|
||||
outputCost,
|
||||
totalCost,
|
||||
costDetails,
|
||||
};
|
||||
}
|
||||
|
||||
private _matchPricingTier(
|
||||
tiers: LlmPricingTierWithPrices[],
|
||||
usageDetails: Record<string, number>
|
||||
): LlmPricingTierWithPrices | null {
|
||||
if (tiers.length === 0) return null;
|
||||
|
||||
// Tiers are sorted by priority ascending (lowest first).
|
||||
// First pass: evaluate tiers that have conditions — first match wins.
|
||||
for (const tier of tiers) {
|
||||
if (tier.conditions.length > 0 && this._evaluateConditions(tier.conditions, usageDetails)) {
|
||||
return tier;
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: fall back to the default tier, or first tier with no conditions
|
||||
const defaultTier = tiers.find((t) => t.isDefault);
|
||||
if (defaultTier) return defaultTier;
|
||||
|
||||
const unconditional = tiers.find((t) => t.conditions.length === 0);
|
||||
return unconditional ?? tiers[0] ?? null;
|
||||
}
|
||||
|
||||
private _evaluateConditions(
|
||||
conditions: PricingCondition[],
|
||||
usageDetails: Record<string, number>
|
||||
): boolean {
|
||||
return conditions.every((condition) => {
|
||||
// Find matching usage detail key
|
||||
const regex = new RegExp(condition.usageDetailPattern);
|
||||
const matchingValue = Object.entries(usageDetails).find(([key]) => regex.test(key));
|
||||
const value = matchingValue?.[1] ?? 0;
|
||||
|
||||
switch (condition.operator) {
|
||||
case "gt":
|
||||
return value > condition.value;
|
||||
case "gte":
|
||||
return value >= condition.value;
|
||||
case "lt":
|
||||
return value < condition.value;
|
||||
case "lte":
|
||||
return value <= condition.value;
|
||||
case "eq":
|
||||
return value === condition.value;
|
||||
case "neq":
|
||||
return value !== condition.value;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { defaultModelPrices } from "./defaultPrices.js";
|
||||
import { modelCatalog } from "./modelCatalog.js";
|
||||
import { syncLlmCatalog } from "./sync.js";
|
||||
|
||||
export async function seedLlmPricing(prisma: PrismaClient): Promise<{
|
||||
modelsCreated: number;
|
||||
modelsSkipped: number;
|
||||
modelsUpdated: number;
|
||||
}> {
|
||||
let modelsCreated = 0;
|
||||
let modelsSkipped = 0;
|
||||
|
||||
for (const modelDef of defaultModelPrices) {
|
||||
// Check if this model already exists (don't overwrite admin changes)
|
||||
const existing = await prisma.llmModel.findFirst({
|
||||
where: {
|
||||
projectId: null,
|
||||
modelName: modelDef.modelName,
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
modelsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look up catalog metadata for this model
|
||||
const catalog = modelCatalog[modelDef.modelName];
|
||||
|
||||
// Create model + tiers atomically so partial models can't be left behind
|
||||
await prisma.$transaction(async (tx) => {
|
||||
const model = await tx.llmModel.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("llm_model"),
|
||||
modelName: modelDef.modelName.trim(),
|
||||
matchPattern: modelDef.matchPattern,
|
||||
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
|
||||
source: "default",
|
||||
// Catalog metadata (from model-catalog.json)
|
||||
provider: catalog?.provider ?? null,
|
||||
description: catalog?.description ?? null,
|
||||
contextWindow: catalog?.contextWindow ?? null,
|
||||
maxOutputTokens: catalog?.maxOutputTokens ?? null,
|
||||
capabilities: catalog?.capabilities ?? [],
|
||||
isHidden: catalog?.isHidden ?? false,
|
||||
baseModelName: catalog?.baseModelName ?? null,
|
||||
pricingUnit: "tokens",
|
||||
},
|
||||
});
|
||||
|
||||
for (const tier of modelDef.pricingTiers) {
|
||||
await tx.llmPricingTier.create({
|
||||
data: {
|
||||
modelId: model.id,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: tier.conditions,
|
||||
prices: {
|
||||
create: Object.entries(tier.prices).map(([usageType, price]) => ({
|
||||
modelId: model.id,
|
||||
usageType,
|
||||
price,
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
modelsCreated++;
|
||||
}
|
||||
|
||||
// Sync catalog metadata on existing default models
|
||||
const syncResult = await syncLlmCatalog(prisma);
|
||||
|
||||
return { modelsCreated, modelsSkipped, modelsUpdated: syncResult.modelsUpdated };
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { postgresTest } from "@internal/testcontainers";
|
||||
import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { describe, expect } from "vitest";
|
||||
import { defaultModelPrices } from "./defaultPrices.js";
|
||||
import { modelCatalog } from "./modelCatalog.js";
|
||||
import { syncLlmCatalog } from "./sync.js";
|
||||
|
||||
function getGpt4oDefinition() {
|
||||
const def = defaultModelPrices.find((m) => m.modelName === "gpt-4o");
|
||||
if (def === undefined) {
|
||||
throw new Error("expected gpt-4o in defaultModelPrices");
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
const gpt4oDef = getGpt4oDefinition();
|
||||
|
||||
function getGeminiProDefinition() {
|
||||
const def = defaultModelPrices.find((m) => m.modelName === "gemini-pro");
|
||||
if (def === undefined) {
|
||||
throw new Error("expected gemini-pro in defaultModelPrices");
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
const geminiProDef = getGeminiProDefinition();
|
||||
|
||||
/** If sync used `catalog?.baseModelName ?? existing.baseModelName`, sync would keep this string instead of clearing to null. */
|
||||
const STALE_BASE_MODEL_NAME = "wrong-base-model-sentinel";
|
||||
|
||||
const STALE_INPUT_PRICE = 0.099;
|
||||
const STALE_OUTPUT_PRICE = 0.088;
|
||||
|
||||
async function createGpt4oWithStalePricing(prisma: PrismaClient, source: "default" | "admin") {
|
||||
const model = await prisma.llmModel.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("llm_model"),
|
||||
projectId: null,
|
||||
modelName: gpt4oDef.modelName,
|
||||
matchPattern: "^stale-pattern$",
|
||||
startDate: gpt4oDef.startDate ? new Date(gpt4oDef.startDate) : null,
|
||||
source,
|
||||
provider: "stale-provider",
|
||||
description: "stale description",
|
||||
contextWindow: 111,
|
||||
maxOutputTokens: 222,
|
||||
capabilities: ["stale-cap"],
|
||||
isHidden: true,
|
||||
baseModelName: "stale-base",
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.llmPricingTier.create({
|
||||
data: {
|
||||
modelId: model.id,
|
||||
name: "Standard",
|
||||
isDefault: true,
|
||||
priority: 0,
|
||||
conditions: [],
|
||||
prices: {
|
||||
create: [
|
||||
{ modelId: model.id, usageType: "input", price: STALE_INPUT_PRICE },
|
||||
{ modelId: model.id, usageType: "output", price: STALE_OUTPUT_PRICE },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
async function createGeminiProWithStaleBaseModelName(prisma: PrismaClient) {
|
||||
const catalogEntry = modelCatalog[geminiProDef.modelName];
|
||||
expect(catalogEntry).toBeDefined();
|
||||
expect(catalogEntry.baseModelName).toBeNull();
|
||||
|
||||
const model = await prisma.llmModel.create({
|
||||
data: {
|
||||
friendlyId: generateFriendlyId("llm_model"),
|
||||
projectId: null,
|
||||
modelName: geminiProDef.modelName,
|
||||
matchPattern: "^stale-gemini-pattern$",
|
||||
startDate: geminiProDef.startDate ? new Date(geminiProDef.startDate) : null,
|
||||
source: "default",
|
||||
provider: "stale-provider",
|
||||
description: "stale description",
|
||||
contextWindow: 111,
|
||||
maxOutputTokens: 222,
|
||||
capabilities: ["stale-cap"],
|
||||
isHidden: true,
|
||||
baseModelName: STALE_BASE_MODEL_NAME,
|
||||
},
|
||||
});
|
||||
|
||||
const tier = geminiProDef.pricingTiers[0];
|
||||
await prisma.llmPricingTier.create({
|
||||
data: {
|
||||
modelId: model.id,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: tier.conditions,
|
||||
prices: {
|
||||
create: Object.entries(tier.prices).map(([usageType, price]) => ({
|
||||
modelId: model.id,
|
||||
usageType,
|
||||
price,
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
async function loadGpt4oWithTiers(prisma: PrismaClient) {
|
||||
return prisma.llmModel.findFirst({
|
||||
where: { projectId: null, modelName: gpt4oDef.modelName },
|
||||
include: {
|
||||
pricingTiers: {
|
||||
include: { prices: true },
|
||||
orderBy: { priority: "asc" },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function expectBundledGpt4oPricing(
|
||||
model: NonNullable<Awaited<ReturnType<typeof loadGpt4oWithTiers>>>
|
||||
) {
|
||||
expect(model.matchPattern).toBe(gpt4oDef.matchPattern);
|
||||
expect(model.pricingTiers).toHaveLength(gpt4oDef.pricingTiers.length);
|
||||
|
||||
const dbTier = model.pricingTiers[0];
|
||||
const defTier = gpt4oDef.pricingTiers[0];
|
||||
expect(dbTier.name).toBe(defTier.name);
|
||||
expect(dbTier.isDefault).toBe(defTier.isDefault);
|
||||
expect(dbTier.priority).toBe(defTier.priority);
|
||||
|
||||
const priceByType = new Map(dbTier.prices.map((p) => [p.usageType, Number(p.price)]));
|
||||
for (const [usageType, expected] of Object.entries(defTier.prices)) {
|
||||
expect(priceByType.get(usageType)).toBeCloseTo(expected, 12);
|
||||
}
|
||||
expect(priceByType.size).toBe(Object.keys(defTier.prices).length);
|
||||
}
|
||||
|
||||
describe("syncLlmCatalog", () => {
|
||||
postgresTest(
|
||||
"rebuilds gpt-4o pricing tiers from bundled defaults when source is default",
|
||||
async ({ prisma }) => {
|
||||
await createGpt4oWithStalePricing(prisma, "default");
|
||||
|
||||
const result = await syncLlmCatalog(prisma);
|
||||
|
||||
expect(result.modelsUpdated).toBe(1);
|
||||
expect(result.modelsSkipped).toBe(defaultModelPrices.length - 1);
|
||||
|
||||
const after = await loadGpt4oWithTiers(prisma);
|
||||
expect(after).not.toBeNull();
|
||||
expectBundledGpt4oPricing(after!);
|
||||
}
|
||||
);
|
||||
|
||||
postgresTest(
|
||||
"does not replace pricing tiers when model source is not default",
|
||||
async ({ prisma }) => {
|
||||
await createGpt4oWithStalePricing(prisma, "admin");
|
||||
|
||||
const result = await syncLlmCatalog(prisma);
|
||||
|
||||
expect(result.modelsUpdated).toBe(0);
|
||||
expect(result.modelsSkipped).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const after = await loadGpt4oWithTiers(prisma);
|
||||
expect(after).not.toBeNull();
|
||||
expect(after!.matchPattern).toBe("^stale-pattern$");
|
||||
expect(after!.pricingTiers).toHaveLength(1);
|
||||
const prices = after!.pricingTiers[0].prices;
|
||||
const input = prices.find((p) => p.usageType === "input");
|
||||
const output = prices.find((p) => p.usageType === "output");
|
||||
expect(Number(input?.price)).toBeCloseTo(STALE_INPUT_PRICE, 12);
|
||||
expect(Number(output?.price)).toBeCloseTo(STALE_OUTPUT_PRICE, 12);
|
||||
expect(prices).toHaveLength(2);
|
||||
}
|
||||
);
|
||||
|
||||
postgresTest(
|
||||
"clears baseModelName when bundled catalog has null (regression for nullish-coalescing merge)",
|
||||
async ({ prisma }) => {
|
||||
await createGeminiProWithStaleBaseModelName(prisma);
|
||||
|
||||
await syncLlmCatalog(prisma);
|
||||
|
||||
const after = await prisma.llmModel.findFirst({
|
||||
where: { projectId: null, modelName: geminiProDef.modelName },
|
||||
});
|
||||
expect(after).not.toBeNull();
|
||||
expect(after!.baseModelName).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { Prisma, PrismaClient } from "@trigger.dev/database";
|
||||
import { defaultModelPrices } from "./defaultPrices.js";
|
||||
import { modelCatalog } from "./modelCatalog.js";
|
||||
import type { DefaultModelDefinition } from "./types.js";
|
||||
|
||||
function pricingTierCreateData(
|
||||
modelId: string,
|
||||
tier: DefaultModelDefinition["pricingTiers"][number]
|
||||
): Prisma.LlmPricingTierUncheckedCreateInput {
|
||||
return {
|
||||
modelId,
|
||||
name: tier.name,
|
||||
isDefault: tier.isDefault,
|
||||
priority: tier.priority,
|
||||
conditions: tier.conditions,
|
||||
prices: {
|
||||
create: Object.entries(tier.prices).map(([usageType, price]) => ({
|
||||
modelId,
|
||||
usageType,
|
||||
price,
|
||||
})),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function syncLlmCatalog(prisma: PrismaClient): Promise<{
|
||||
modelsUpdated: number;
|
||||
modelsSkipped: number;
|
||||
}> {
|
||||
let modelsUpdated = 0;
|
||||
let modelsSkipped = 0;
|
||||
|
||||
for (const modelDef of defaultModelPrices) {
|
||||
const existing = await prisma.llmModel.findFirst({
|
||||
where: {
|
||||
projectId: null,
|
||||
modelName: modelDef.modelName,
|
||||
},
|
||||
});
|
||||
|
||||
// Skip if model doesn't exist yet (seed handles creation)
|
||||
if (!existing) {
|
||||
modelsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Don't overwrite admin-edited models
|
||||
if (existing.source !== "default") {
|
||||
modelsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const catalog = modelCatalog[modelDef.modelName];
|
||||
|
||||
const applied = await prisma.$transaction(async (tx) => {
|
||||
const updateResult = await tx.llmModel.updateMany({
|
||||
where: { id: existing.id, source: "default" },
|
||||
data: {
|
||||
// Update match pattern and start date from Langfuse (may have changed)
|
||||
matchPattern: modelDef.matchPattern,
|
||||
startDate: modelDef.startDate ? new Date(modelDef.startDate) : null,
|
||||
// Update catalog metadata
|
||||
provider: catalog?.provider ?? existing.provider,
|
||||
description: catalog?.description ?? existing.description,
|
||||
contextWindow:
|
||||
catalog?.contextWindow === undefined ? existing.contextWindow : catalog.contextWindow,
|
||||
maxOutputTokens:
|
||||
catalog?.maxOutputTokens === undefined
|
||||
? existing.maxOutputTokens
|
||||
: catalog.maxOutputTokens,
|
||||
capabilities: catalog?.capabilities ?? existing.capabilities,
|
||||
isHidden: catalog?.isHidden ?? existing.isHidden,
|
||||
baseModelName:
|
||||
catalog?.baseModelName === undefined ? existing.baseModelName : catalog.baseModelName,
|
||||
pricingUnit: existing.pricingUnit ?? "tokens",
|
||||
},
|
||||
});
|
||||
|
||||
if (updateResult.count !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await tx.llmPricingTier.deleteMany({ where: { modelId: existing.id } });
|
||||
|
||||
for (const tier of modelDef.pricingTiers) {
|
||||
await tx.llmPricingTier.create({
|
||||
data: pricingTierCreateData(existing.id, tier),
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
if (applied) {
|
||||
modelsUpdated++;
|
||||
}
|
||||
}
|
||||
|
||||
return { modelsUpdated, modelsSkipped };
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
export type PricingCondition = {
|
||||
usageDetailPattern: string;
|
||||
operator: "gt" | "gte" | "lt" | "lte" | "eq" | "neq";
|
||||
value: number;
|
||||
};
|
||||
|
||||
export type LlmPriceEntry = {
|
||||
usageType: string;
|
||||
price: number;
|
||||
};
|
||||
|
||||
export type LlmPricingTierWithPrices = {
|
||||
id: string;
|
||||
name: string;
|
||||
isDefault: boolean;
|
||||
priority: number;
|
||||
conditions: PricingCondition[];
|
||||
prices: LlmPriceEntry[];
|
||||
};
|
||||
|
||||
export type LlmModelWithPricing = {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
modelName: string;
|
||||
matchPattern: string;
|
||||
startDate: Date | null;
|
||||
pricingTiers: LlmPricingTierWithPrices[];
|
||||
};
|
||||
|
||||
export type LlmCostResult = {
|
||||
matchedModelId: string;
|
||||
matchedModelName: string;
|
||||
pricingTierId: string;
|
||||
pricingTierName: string;
|
||||
inputCost: number;
|
||||
outputCost: number;
|
||||
totalCost: number;
|
||||
costDetails: Record<string, number>;
|
||||
};
|
||||
|
||||
export type ModelCatalogEntry = {
|
||||
provider: string;
|
||||
description: string;
|
||||
contextWindow: number | null;
|
||||
maxOutputTokens: number | null;
|
||||
capabilities: string[];
|
||||
/** ISO date string of when the model was publicly released (e.g. "2025-06-15"). */
|
||||
releaseDate: string | null;
|
||||
/** Whether the model is deprecated/legacy and should be hidden from the registry by default. */
|
||||
isHidden: boolean;
|
||||
/** Whether the model supports reliable structured JSON output (schema adherence). */
|
||||
supportsStructuredOutput: boolean;
|
||||
/** Whether the model can call multiple tools in a single turn. */
|
||||
supportsParallelToolCalls: boolean;
|
||||
/** Whether the model supports streaming partial tool call results. */
|
||||
supportsStreamingToolCalls: boolean;
|
||||
/** ISO date string of when the model will be deprecated/sunset, if known. */
|
||||
deprecationDate: string | null;
|
||||
/** ISO date string of the model's training data cutoff (e.g. "2024-10-01"). */
|
||||
knowledgeCutoff: string | null;
|
||||
/** ISO timestamp of when this entry was last researched/resolved. */
|
||||
resolvedAt: string;
|
||||
/** The base model this is a variant of, or null if this IS the base model. */
|
||||
baseModelName: string | null;
|
||||
};
|
||||
|
||||
export type DefaultModelDefinition = {
|
||||
modelName: string;
|
||||
matchPattern: string;
|
||||
startDate?: string;
|
||||
// Catalog metadata (merged from model-catalog.json during seed)
|
||||
provider?: string;
|
||||
description?: string;
|
||||
contextWindow?: number | null;
|
||||
maxOutputTokens?: number | null;
|
||||
capabilities?: string[];
|
||||
isHidden?: boolean;
|
||||
pricingTiers: Array<{
|
||||
name: string;
|
||||
isDefault: boolean;
|
||||
priority: number;
|
||||
conditions: PricingCondition[];
|
||||
prices: Record<string, number>;
|
||||
}>;
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2019",
|
||||
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"strict": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"exclude": ["node_modules", "**/*.test.ts"]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
sequence: { sequencer: DurationShardingSequencer },
|
||||
include: ["**/*.test.ts"],
|
||||
globals: true,
|
||||
isolate: true,
|
||||
fileParallelism: false,
|
||||
testTimeout: 120_000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user