wehub-resource-sync
181 lines
6.1 KiB
Bash
181 lines
6.1 KiB
Bash
# Shared utilities for Cloudflare AI Gateway scripts
|
|
# Source this file: source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
|
|
|
|
# =============================================================================
|
|
# CONFIGURATION: Providers and models to include
|
|
# =============================================================================
|
|
|
|
# Providers to include ALL models from (generated with defaults)
|
|
INCLUDE_ALL_PROVIDERS="workers-ai"
|
|
|
|
# Namespaces to skip entirely (provider/namespace format)
|
|
SKIP_NAMESPACES="replicate/replicate-internal"
|
|
|
|
# Specific models to skip (exact model IDs)
|
|
SKIP_MODELS="aura-1 whisper"
|
|
|
|
# Providers to cross-reference from source provider files
|
|
CROSS_REFERENCE_PROVIDERS="openai anthropic"
|
|
|
|
# For cross-referenced providers, only include these well-known models (regex patterns)
|
|
# Format: "provider/model-pattern"
|
|
# Use $ anchor for exact matches to avoid dated versions and variants
|
|
# NOTE: These patterns match the Cloudflare API format (which uses BOTH dots and hyphens)
|
|
# The filename conversion to hyphens happens in 03_generate_model_toml.sh
|
|
WELL_KNOWN_MODELS=(
|
|
# OpenAI - canonical names only, no dated versions
|
|
# API uses BOTH dots and hyphens: gpt-5.1 AND gpt-5-1, gpt-3.5-turbo AND gpt-3-5-turbo
|
|
"openai/gpt-5[\.-]2$"
|
|
"openai/gpt-5[\.-]1$"
|
|
"openai/gpt-5[\.-]1-codex$"
|
|
"openai/gpt-4o$"
|
|
"openai/gpt-4o-mini$"
|
|
"openai/gpt-4-turbo$"
|
|
"openai/gpt-4$"
|
|
"openai/gpt-3[\.-]5-turbo$"
|
|
"openai/o1$"
|
|
"openai/o3$"
|
|
"openai/o3-mini$"
|
|
"openai/o3-pro$"
|
|
"openai/o4-mini$"
|
|
|
|
# Anthropic - canonical names only, no dated versions or duplicates
|
|
# API uses BOTH dots and hyphens: claude-3.5-sonnet AND claude-3-5-sonnet
|
|
"anthropic/claude-sonnet-4-5$"
|
|
"anthropic/claude-opus-4-6$"
|
|
"anthropic/claude-opus-4-5$"
|
|
"anthropic/claude-haiku-4-5$"
|
|
"anthropic/claude-opus-4-1$"
|
|
"anthropic/claude-sonnet-4$"
|
|
"anthropic/claude-opus-4$"
|
|
"anthropic/claude-3[\.-]5-sonnet$"
|
|
"anthropic/claude-3[\.-]5-haiku$"
|
|
"anthropic/claude-3-opus$"
|
|
"anthropic/claude-3-sonnet$"
|
|
"anthropic/claude-3-haiku$"
|
|
)
|
|
|
|
# =============================================================================
|
|
# Helper function to get mapped model name for source file lookup
|
|
# =============================================================================
|
|
# This function maps Cloudflare API model names to source provider file names.
|
|
# Input: model name from API (may have dots, e.g., "claude-3.5-sonnet")
|
|
# Output: source file name (uses hyphens, e.g., "claude-3-5-sonnet-20241022")
|
|
get_mapped_name() {
|
|
local model_name="$1"
|
|
# First convert dots to hyphens for consistent lookup
|
|
local normalized_name=$(echo "${model_name}" | sed 's/\./-/g')
|
|
|
|
case "${normalized_name}" in
|
|
# Anthropic mappings - map to source file names with date suffixes
|
|
"claude-sonnet-4-5") echo "claude-sonnet-4-5" ;;
|
|
"claude-opus-4-6") echo "claude-opus-4-6" ;;
|
|
"claude-opus-4-5") echo "claude-opus-4-5" ;;
|
|
"claude-haiku-4-5") echo "claude-haiku-4-5" ;;
|
|
"claude-opus-4-1") echo "claude-opus-4-1" ;;
|
|
"claude-sonnet-4") echo "claude-sonnet-4-0" ;;
|
|
"claude-opus-4") echo "claude-opus-4-0" ;;
|
|
"claude-3-5-sonnet") echo "claude-3-5-sonnet-20241022" ;;
|
|
"claude-3-5-haiku") echo "claude-3-5-haiku-latest" ;;
|
|
"claude-3-opus") echo "claude-3-opus-20240229" ;;
|
|
"claude-3-sonnet") echo "claude-3-sonnet-20240229" ;;
|
|
"claude-3-haiku") echo "claude-3-haiku-20240307" ;;
|
|
# OpenAI mappings - source files use dots, but we look up with hyphens
|
|
"gpt-5-1") echo "gpt-5.1" ;;
|
|
"gpt-5-2") echo "gpt-5.2" ;;
|
|
"gpt-5-1-codex") echo "gpt-5.1-codex" ;;
|
|
"gpt-3-5-turbo") echo "gpt-3.5-turbo" ;;
|
|
*) echo "${normalized_name}" ;;
|
|
esac
|
|
}
|
|
|
|
# =============================================================================
|
|
# Helper function to check if a model should be included
|
|
# =============================================================================
|
|
should_include_model() {
|
|
local model_id="$1"
|
|
local provider
|
|
|
|
# Extract provider from model ID (first path segment)
|
|
provider=$(echo "${model_id}" | cut -d'/' -f1)
|
|
|
|
# Check if model matches any skip pattern (partial match on model name)
|
|
for skip in ${SKIP_MODELS}; do
|
|
if [[ "${model_id}" == *"${skip}"* ]]; then
|
|
return 1 # Exclude
|
|
fi
|
|
done
|
|
|
|
# Check if model is in a skipped namespace
|
|
for ns in ${SKIP_NAMESPACES}; do
|
|
if [[ "${model_id}" == ${ns}/* ]]; then
|
|
return 1 # Exclude
|
|
fi
|
|
done
|
|
|
|
# Check if provider is in the "include all" list
|
|
for p in ${INCLUDE_ALL_PROVIDERS}; do
|
|
if [[ "${provider}" == "${p}" ]]; then
|
|
# Special case for workers-ai: only include models with workers-ai/@cf/ prefix
|
|
if [[ "${provider}" == "workers-ai" && "${model_id}" != "workers-ai/@cf/"* ]]; then
|
|
return 1 # Exclude workers-ai models that don't have @cf/ namespace
|
|
fi
|
|
return 0 # Include
|
|
fi
|
|
done
|
|
|
|
# Check if model matches any well-known pattern
|
|
for pattern in "${WELL_KNOWN_MODELS[@]}"; do
|
|
if echo "${model_id}" | grep -qE "^${pattern}"; then
|
|
return 0 # Include
|
|
fi
|
|
done
|
|
|
|
# Skip
|
|
return 1
|
|
}
|
|
|
|
# =============================================================================
|
|
# Helper function to find source file for cross-referenced models
|
|
# =============================================================================
|
|
find_source_file() {
|
|
local provider="$1"
|
|
local model_name="$2"
|
|
local providers_dir="$3"
|
|
|
|
# Check if provider is in cross-reference list
|
|
local is_cross_ref=false
|
|
for p in ${CROSS_REFERENCE_PROVIDERS}; do
|
|
if [[ "${provider}" == "${p}" ]]; then
|
|
is_cross_ref=true
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ "${is_cross_ref}" != "true" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
# Get mapped name
|
|
local mapped_name
|
|
mapped_name=$(get_mapped_name "${model_name}")
|
|
|
|
local source_file="${providers_dir}/${provider}/models/${mapped_name}.toml"
|
|
|
|
if [[ -f "${source_file}" ]]; then
|
|
echo "${source_file}"
|
|
return 0
|
|
fi
|
|
|
|
# Try original name if mapping didn't work
|
|
if [[ "${mapped_name}" != "${model_name}" ]]; then
|
|
source_file="${providers_dir}/${provider}/models/${model_name}.toml"
|
|
if [[ -f "${source_file}" ]]; then
|
|
echo "${source_file}"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
return 1
|
|
}
|