import { BACKOFF_STEPS_MS, PROVIDER_PROFILES, RateLimitReason, HTTP_STATUS, } from "../config/constants.ts"; import { BACKOFF_CONFIG, COOLDOWN_MS, calculateBackoffCooldown, findMatchingErrorRule, matchErrorRuleByText, matchErrorRuleByStatus, } from "../config/errorConfig.ts"; import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts"; import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts"; import { DEFAULT_RESILIENCE_SETTINGS, resolveResilienceSettings, } from "../../src/lib/resilience/settings"; import { resolveModelLockoutSettings } from "../../src/lib/resilience/modelLockoutSettings"; import { getAllCircuitBreakerStatuses, getCircuitBreaker, } from "../../src/shared/utils/circuitBreaker"; import { classify429FromError, looksLikeQuotaExhausted, type FailureKind, } from "../../src/shared/utils/classify429"; import { resolveProviderId } from "../../src/shared/constants/providers"; import { resolveUseUpstream429BreakerHints } from "../../src/shared/utils/providerHints"; import { getCodexModelScope } from "../config/codexQuotaScopes.ts"; import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts"; import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts"; import { setConnectionRateLimitUntil } from "@/lib/db/providers"; import { parseRetryHintFromJsonBody } from "./retryAfterJson.ts"; export type ProviderProfile = { baseCooldownMs: number; useUpstreamRetryHints: boolean; useUpstream429BreakerHints?: boolean; maxCooldownMs: number; maxBackoffSteps: number; failureThreshold: number; resetTimeoutMs: number; transientCooldown: number; rateLimitCooldown: number; maxBackoffLevel: number; circuitBreakerThreshold: number; circuitBreakerReset: number; // Adaptive circuit breaker fields degradationThreshold?: number; // Provider-level cooldown fields providerFailureThreshold: number; providerFailureWindowMs: number; providerCooldownMs: number; maxBackoffMultiplier?: number; backoffEscalationCount?: number; }; type JsonRecord = Record; type RateLimitReasonValue = (typeof RateLimitReason)[keyof typeof RateLimitReason]; type ModelLockoutEntry = { reason: string; until: number; lockedAt: number; failureCount: number; lastFailureAt: number; resetAfterMs: number; }; type ModelFailureState = { failureCount: number; lastFailureAt: number; resetAfterMs: number; /** Cooldown applied on the last failure — extends the escalation window so a * model that fails again right after its lockout expires keeps escalating. */ lastCooldownMs?: number; }; type AccountState = JsonRecord & { id?: string | null; rateLimitedUntil?: string | null; backoffLevel?: number | null; lastError?: unknown; status?: string; }; function toJsonRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } // Provider-level failure tracking for circuit breaker behavior // Error codes that count toward provider-level failure threshold. // 429 is included: per-error-type cooldowns (rate_limit: 60s, quota_exhausted: 1h) // prevent cascading provider trips at scale (Issue #1846 concern addressed), // while still allowing the circuit breaker to open on sustained 429s and // prevent infinite combo retries (Issue #3200). const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 429, 500, 502, 503, 504]); // Per-connection failure deduplication: prevents rapid-fire failures from the // same connection from counting multiple times toward the provider breaker. const CONNECTION_FAILURE_DEDUP_MS = 5000; const MAX_CONNECTION_FAILURE_DEDUP_ENTRIES = 10_000; const lastConnectionFailure = new Map(); function pruneConnectionFailureDedupeEntries(): void { while (lastConnectionFailure.size > MAX_CONNECTION_FAILURE_DEDUP_ENTRIES) { const oldestKey = lastConnectionFailure.keys().next().value; if (typeof oldestKey !== "string") return; lastConnectionFailure.delete(oldestKey); } } const _connectionFailureSweep = setInterval(() => { const now = Date.now(); for (const [key, ts] of lastConnectionFailure) { if (now - ts > CONNECTION_FAILURE_DEDUP_MS) lastConnectionFailure.delete(key); } }, 60_000); if (typeof _connectionFailureSweep === "object" && "unref" in _connectionFailureSweep) { (_connectionFailureSweep as { unref?: () => void }).unref?.(); } // T06 (sub2api PR #1037): Signals that indicate permanent account deactivation. // When a 401 body contains these strings, the account is permanently dead // and should NOT be retried after token refresh. export const ACCOUNT_DEACTIVATED_SIGNALS = [ "account_deactivated", "account has been deactivated", "account has been disabled", "your account has been suspended", "this account is deactivated", // AG (Antigravity/Google Cloud Code) permanent ban signals "verify your account to continue", "this service has been disabled in this account for violation", "this service has been disabled in this account", ]; // Custom banned signals — loaded from DB settings at runtime. // Combined with ACCOUNT_DEACTIVATED_SIGNALS in isAccountDeactivated(). let _customBannedSignals: string[] = []; export function setCustomBannedSignals(signals: string[]): void { _customBannedSignals = signals; } export function getMergedBannedSignals(): string[] { if (_customBannedSignals.length === 0) return ACCOUNT_DEACTIVATED_SIGNALS; return [...ACCOUNT_DEACTIVATED_SIGNALS, ..._customBannedSignals]; } // T10 (sub2api PR #1169): Signals that indicate billing credits are exhausted. // Distinct from rate-limit 429 — the account won't recover until credits are added. export const CREDITS_EXHAUSTED_SIGNALS = [ "insufficient_quota", "billing_hard_limit_reached", "exceeded your current quota", "exceeded your current usage quota", "credit_balance_too_low", "your credit balance is too low", "credits exhausted", "out of credits", "payment required", "free tier of the model has been exhausted", // #5239: providers (e.g. DeepSeek/GLM-style) return "Insufficient account balance" // on a depleted key. 402 is already terminalized by status, but catch non-402 // out-of-credit bodies here too. "insufficient balance", "insufficient_balance", "insufficient account balance", ]; // T11: Signals that indicate OAuth token is invalid/expired (not permanent deactivation) export const OAUTH_INVALID_TOKEN_SIGNALS = [ "invalid authentication credentials", "oauth 2", "login cookie", "valid authentication credential", "invalid credentials", ]; // Context overflow patterns — the prompt exceeds the model's maximum context length. // Different providers phrase this differently. Used to decide whether a 400 error // should trigger combo fallback (a different model may have a larger context window). const CONTEXT_OVERFLOW_PATTERNS = [ /\binput is too long\b/i, /\binput too long\b/i, /\bcontext.*(too long|exceeded|overflow|limit)/i, /\btoo many tokens\b/i, /\bprompt is too long\b/i, /\bcontext window/i, /\bmaximum context/i, /\bmax.*token/i, /\btoken limit/i, /\brequest too large\b/i, ]; // Structured error codes that reliably indicate model access denied // (more reliable than regex on human-readable messages). // OpenAI: { error: { code: "model_not_found", ... } } // Anthropic: { error: { type: "not_found_error", ... } } const MODEL_ACCESS_DENIED_CODES = new Set([ "model_not_found", // OpenAI, OpenAI-compatible (Kiro, Together, Fireworks, etc.) "deployment_not_found", // Azure OpenAI ]); const MODEL_ACCESS_DENIED_TYPES = new Set([ "not_found_error", // Anthropic: model doesn't exist — reliably model-scoped ]); // Anthropic's permission_error is NOT exclusively model-access related: it also // covers API-key scope, organization restrictions and feature gating. Treating it // as model-access-denied unconditionally would make a genuinely auth-restricted key // silently exhaust every combo target and hide the real error from the caller. // So it only counts when the message text confirms it refers to the model. const MODEL_ACCESS_AMBIGUOUS_TYPES = new Set([ "permission_error", // Anthropic: could be model access OR key/org/feature scope ]); // Model access patterns — the account does not have access to the requested model // but a different account (e.g. PRO vs free tier) may support it. const MODEL_ACCESS_DENIED_PATTERNS = [ /\binvalid model\b/i, /\bmodel.*not.*(?:available|found|supported|accessible)\b/i, /\bmodel.*(?:does not exist|doesn't exist)\b/i, /\baccess.*denied.*model\b/i, /\bmodel.*access.*denied\b/i, /\bplease select a different model\b/i, // "...access to the requested model" / "model ... access" — bounded lookahead // (no nested quantifiers) so it stays ReDoS-safe while requiring BOTH an // access/permission word and "model" so a pure auth error never matches. /\b(?:access|permission)[\s\S]{0,60}?\bmodel\b/i, /\bmodel[\s\S]{0,60}?\b(?:access|permission)\b/i, ]; // Pure credential/authentication failures — the key or token itself is bad, which // is NOT a model-availability problem. Some providers phrase these as a 400 that // also mentions the model (e.g. "invalid api key for model X"), which would // otherwise trip MODEL_ACCESS_DENIED_PATTERNS above and trigger combo fallback // across every target, masking the real "fix your credential" error. When the // text clearly indicates a bad credential, the regex-based model-access detection // is suppressed (structured codes/types like model_not_found are unaffected). const AUTH_CREDENTIAL_ERROR_PATTERNS = [ /\b(?:invalid|incorrect|expired|missing|revoked)\s+api[\s_-]?key\b/i, /\bapi[\s_-]?key\s+(?:is\s+)?(?:invalid|incorrect|expired|missing|revoked|not\s+valid)\b/i, /\bauthentication\s+(?:failed|error|required)\b/i, /\b(?:invalid|expired|missing|revoked)\s+(?:token|credentials?|bearer)\b/i, /\bunauthorized\b/i, /\bnot\s+authenticated\b/i, ]; // Malformed request patterns — the model rejected the message format but a different // provider/model in the combo may accept it. const MALFORMED_REQUEST_PATTERNS = [ /\bimproperly formed request\b/i, /\binvalid.*message.*format/i, /\bmessages must alternate\b/i, /\bempty (message|content)\b/i, // Tool call function name errors /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, /function.*name.*(?:blank|empty|missing)/i, /tool_call.*name.*(?:blank|empty|missing)/i, ]; // Rate-limit text on a 400 — some providers (e.g. MiMoCode) signal throttling with a // non-standard 400 status whose body carries rate-limit semantics instead of a 429 // (#4976). When detected, the request is fallback-worthy at connection-cooldown scope // (NOT a whole-provider breaker) so combo routing can fail over to another free target. // Bounded, non-overlapping patterns only (ReDoS-safe — no nested quantifiers). const RATE_LIMIT_TEXT_PATTERNS = [ /high.?frequency/i, /non-compliant/i, /too many requests/i, /rate.?limit/i, /频繁/, // "frequent" (zh) — high-frequency request throttling /频率/, // "frequency" (zh) — request-frequency throttling ]; // Parameter validation errors — model-specific constraints (different models = different limits) const PARAM_VALIDATION_PATTERNS = [ /max_tokens.*illegal/i, /max_tokens.*must be/i, /max_tokens.*range/i, /parameter is illegal/i, /is illegal.*range/i, ]; /** * T06: Returns true if response body indicates the account is permanently deactivated. */ export function isAccountDeactivated(errorText: string): boolean { const lower = String(errorText || "").toLowerCase(); return getMergedBannedSignals().some((sig) => lower.includes(sig)); } /** * T10: Returns true if response body indicates credits/quota are permanently exhausted. */ export function isCreditsExhausted(errorText: string): boolean { const lower = String(errorText || "").toLowerCase(); return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig)); } /** * T11: Returns true if response body indicates OAuth token is invalid/expired. * This is different from permanent account deactivation - token refresh can recover. */ export function isOAuthInvalidToken(errorText: string): boolean { const lower = String(errorText || "").toLowerCase(); return OAUTH_INVALID_TOKEN_SIGNALS.some((sig) => lower.includes(sig)); } // ─── Resilience Profile Helper ────────────────────────────────────────────── function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } function isCompatibleProvider(provider: string | null | undefined): boolean { return ( typeof provider === "string" && (provider.startsWith("openai-compatible-") || provider.startsWith("anthropic-compatible-")) ); } function buildProviderProfile( category: "oauth" | "apikey", settings?: Record | null ) { const resilience = settings ? resolveResilienceSettings(settings) : DEFAULT_RESILIENCE_SETTINGS; const connectionCooldown = resilience.connectionCooldown[category]; const providerBreaker = resilience.providerBreaker[category]; return { baseCooldownMs: connectionCooldown.baseCooldownMs, useUpstreamRetryHints: connectionCooldown.useUpstreamRetryHints, useUpstream429BreakerHints: connectionCooldown.useUpstream429BreakerHints, maxCooldownMs: resolveModelLockoutSettings(settings).maxCooldownMs, maxBackoffSteps: connectionCooldown.maxBackoffSteps, failureThreshold: providerBreaker.failureThreshold, resetTimeoutMs: providerBreaker.resetTimeoutMs, transientCooldown: connectionCooldown.baseCooldownMs, rateLimitCooldown: connectionCooldown.useUpstreamRetryHints ? 0 : connectionCooldown.baseCooldownMs, maxBackoffLevel: connectionCooldown.maxBackoffSteps, circuitBreakerThreshold: providerBreaker.failureThreshold, circuitBreakerReset: providerBreaker.resetTimeoutMs, degradationThreshold: providerBreaker.degradationThreshold, // Provider-level cooldown fields are not exposed in resilience settings yet. providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold, providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs, maxBackoffMultiplier: PROVIDER_PROFILES[category].maxBackoffMultiplier, backoffEscalationCount: PROVIDER_PROFILES[category].backoffEscalationCount, providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs, } satisfies ProviderProfile; } /** * Get the resilience profile for a provider (oauth or apikey). * @param {string} provider - Provider ID or alias */ export function getProviderProfile(provider: string): ProviderProfile { const category = getProviderCategory(provider); return buildProviderProfile(category); } function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean { if (!provider) return true; return getProviderCategory(provider) === "oauth"; } export async function getRuntimeProviderProfile(provider: string | null | undefined) { try { const { getCachedSettings } = await import("@/lib/db/readCache"); const settings = await getCachedSettings(); const category = getProviderCategory(provider || ""); return buildProviderProfile(category, settings); } catch { return getProviderProfile(provider || ""); } } // ─── Per-Model Lockout Tracking ───────────────────────────────────────────── // In-memory map: "provider:connectionId:model" → { reason, until, lockedAt } const modelLockouts = new Map(); const modelFailureState = new Map(); // Aliases (e.g. "cx" → "codex") must share lockout state with their canonical // provider, otherwise a model locked via one spelling stays routable via the other. const canonicalProviderCache = new Map(); function getCanonicalLockProvider(provider: string): string { let canonical = canonicalProviderCache.get(provider); if (!canonical) { canonical = resolveProviderId(provider); canonicalProviderCache.set(provider, canonical); } return canonical; } function getModelLockKey(provider: string, connectionId: string, model: string) { const canonicalProvider = getCanonicalLockProvider(provider); const lockModel = canonicalProvider === "codex" ? getCodexModelScope(model) : getQuotaScopedModelForProvider(canonicalProvider, model) || model; return `${canonicalProvider}:${connectionId}:${lockModel}`; } function getFailureWindowMs(profile: ProviderProfile | null = null, fallbackMs = 30 * 60 * 1000) { const configured = profile?.resetTimeoutMs; return typeof configured === "number" && configured > 0 ? configured : fallbackMs; } function cleanupModelLockKey(key: string, now = Date.now()) { const entry = modelLockouts.get(key); if (entry && now > entry.until) { modelLockouts.delete(key); } const failure = modelFailureState.get(key); if (!failure) return; // The escalation window extends past the applied cooldown: a model that fails // again right after its lockout expires must keep escalating, not reset to 1. if (now - failure.lastFailureAt <= failure.resetAfterMs + (failure.lastCooldownMs ?? 0)) return; if (modelLockouts.has(key)) return; modelFailureState.delete(key); } function getModelLockBaseCooldown( status: number, fallbackCooldownMs: number, profile: ProviderProfile | null = null ) { if (Number.isFinite(fallbackCooldownMs) && fallbackCooldownMs > 0) { return fallbackCooldownMs; } if (typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0) { return profile.baseCooldownMs; } return status === HTTP_STATUS.RATE_LIMITED ? getQuotaCooldown(0) : COOLDOWN_MS.transientInitial; } function getScaledCooldown( baseCooldownMs: number, failureCount: number, maxBackoffLevel = BACKOFF_CONFIG.maxLevel ) { const safeBase = Number.isFinite(baseCooldownMs) && baseCooldownMs > 0 ? baseCooldownMs : 1000; const exponent = Math.min(Math.max(0, failureCount - 1), Math.max(0, maxBackoffLevel)); return safeBase * Math.pow(2, exponent); } // Auto-cleanup expired lockouts every 15 seconds (lazy init for Cloudflare Workers compatibility) let _cleanupTimer: ReturnType | null = null; function ensureCleanupTimer() { if (_cleanupTimer) return; try { _cleanupTimer = setInterval(() => { const now = Date.now(); for (const key of modelLockouts.keys()) cleanupModelLockKey(key, now); for (const key of modelFailureState.keys()) cleanupModelLockKey(key, now); }, 15_000); if (typeof _cleanupTimer === "object" && "unref" in _cleanupTimer) { (_cleanupTimer as { unref?: () => void }).unref?.(); // Don't prevent process exit (Node.js only) } } catch { // Cloudflare Workers may not support setInterval outside handlers — skip cleanup timer } } /** * Lock a specific model on a specific account * @param {string} provider * @param {string} connectionId * @param {string} model * @param {string} reason - from RateLimitReason * @param {number} cooldownMs */ export function lockModel( provider: string, connectionId: string, model: string | null | undefined, reason: string, cooldownMs: number, metadata: Partial = {} ): void { if (!model) return; // No model → skip model-level locking ensureCleanupTimer(); const key = getModelLockKey(provider, connectionId, model); cleanupModelLockKey(key); const newUntil = Date.now() + cooldownMs; // Preserve the longer cooldown if an existing lock has more time remaining. // Safe without a mutex: no await between get/set, so this runs atomically // within Node.js's single-threaded event loop. const existing = modelLockouts.get(key); if (existing && existing.until > newUntil) { if (metadata.failureCount && metadata.failureCount > existing.failureCount) { existing.failureCount = metadata.failureCount; existing.lastFailureAt = metadata.lastFailureAt ?? existing.lastFailureAt; existing.resetAfterMs = metadata.resetAfterMs ?? existing.resetAfterMs; modelLockouts.set(key, existing); } return; } const now = Date.now(); modelLockouts.set(key, { reason, until: newUntil, lockedAt: now, failureCount: metadata.failureCount ?? existing?.failureCount ?? 1, lastFailureAt: metadata.lastFailureAt ?? now, resetAfterMs: metadata.resetAfterMs ?? existing?.resetAfterMs ?? 0, }); } /** * Pick the `exactCooldownMs` to apply to a model lockout (#1308). * * When the upstream response carried an explicit reset longer than the base * cooldown — e.g. Antigravity "Resets in 160h", a `Retry-After` header, or a * parseable reset text already extracted by `checkFallbackError`/`parseRetryFromErrorText` * into `parsedCooldownMs` — honor it exactly so an exhausted model is not retried * again within minutes. Otherwise preserve the previous behavior: return `0` to let * `recordModelLockoutFailure` apply its exponential backoff, or the base cooldown when * backoff is disabled. */ export function selectLockoutCooldownMs( parsedCooldownMs: number, settings: { baseCooldownMs: number; useExponentialBackoff: boolean } ): number { if (typeof parsedCooldownMs === "number" && parsedCooldownMs > settings.baseCooldownMs) { return parsedCooldownMs; } return settings.useExponentialBackoff ? 0 : settings.baseCooldownMs; } export function recordModelLockoutFailure( provider: string, connectionId: string, model: string, reason: string, status: number, fallbackCooldownMs: number, profile: ProviderProfile | null = null, options: { exactCooldownMs?: number | null; maxCooldownMs?: number } = {} ) { ensureCleanupTimer(); const key = getModelLockKey(provider, connectionId, model); const now = Date.now(); cleanupModelLockKey(key, now); // For daily quota exhaustion (quota_exhausted), set cooldown until tomorrow 00:00 // Use exactCooldownMs to bypass exponential backoff, ensuring precise lock until midnight if (reason === "quota_exhausted" && typeof options.exactCooldownMs !== "number") { options = { ...options, exactCooldownMs: getMsUntilTomorrow() }; } const resetAfterMs = getFailureWindowMs(profile); const previous = modelFailureState.get(key); // Escalation window extends past the previously applied cooldown so a model // that fails again right after its lockout expires keeps escalating. const withinWindow = previous && now - previous.lastFailureAt <= previous.resetAfterMs + (previous.lastCooldownMs ?? 0); const failureCount = withinWindow ? previous.failureCount + 1 : 1; const baseCooldownMs = getModelLockBaseCooldown(status, fallbackCooldownMs, profile); // Cap exponential backoff so repeated failures cannot produce absurdly long // lockouts; exact cooldowns (e.g. daily-quota until-midnight) are not capped. const maxCooldownMs = typeof options.maxCooldownMs === "number" && options.maxCooldownMs > 0 ? options.maxCooldownMs : BACKOFF_CONFIG.max; const cooldownMs = typeof options.exactCooldownMs === "number" && options.exactCooldownMs > 0 ? options.exactCooldownMs : Math.min( getScaledCooldown( baseCooldownMs, failureCount, profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel ), maxCooldownMs ); modelFailureState.set(key, { failureCount, lastFailureAt: now, resetAfterMs, lastCooldownMs: cooldownMs, }); lockModel(provider, connectionId, model, reason, cooldownMs, { failureCount, lastFailureAt: now, resetAfterMs, }); return { cooldownMs, failureCount, resetAfterMs, }; } export function clearModelLock( provider: string, connectionId: string, model: string | null | undefined ): boolean { if (!model) return false; const key = getModelLockKey(provider, connectionId, model); const hadLock = modelLockouts.delete(key); const hadFailureState = modelFailureState.delete(key); return hadLock || hadFailureState; } /** * Whether a provider should use per-model lockouts instead of connection-wide cooldowns. * Compatible and passthrough providers multiplex multiple upstream models behind one * connection, so transient 404/429 responses should stay model-scoped instead of * poisoning the whole connection. * * @param provider - Provider ID * @param _model - Model ID (reserved for future use) * @param connectionPassthroughModels - Optional per-connection override from providerSpecificData. * When provided, takes precedence over registry/provider-level logic. */ export function hasPerModelQuota( provider: string | null | undefined, _model: string | null | undefined = null, connectionPassthroughModels?: boolean ): boolean { // Connection-level override takes precedence (e.g., user-configured ModelScope) if (typeof connectionPassthroughModels === "boolean") { return connectionPassthroughModels; } if (!provider) return false; if (getCanonicalLockProvider(provider) === "codex") return true; if (provider === "gemini" || provider === "github") return true; if (getPassthroughProviders().has(provider)) return true; if (isCompatibleProvider(provider)) return true; return false; } /** * Lock a model (not connection) for a provider with per-model quotas. * No-ops for providers that don't use per-model lockouts. */ export function lockModelIfPerModelQuota( provider: string, connectionId: string, model: string | null, reason: string, cooldownMs: number, connectionPassthroughModels?: boolean ): boolean { if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false; // Skip model-level lock if the entire provider is in circuit-breaker cooldown. // The provider cooldown already prevents all requests, so a model lock is redundant. if (isProviderInCooldown(provider)) return false; lockModel(provider, connectionId, model, reason, cooldownMs); return true; } export function shouldMarkAccountExhaustedFrom429( provider: string | null | undefined, model: string | null | undefined = null, connectionPassthroughModels?: boolean, failureKind?: FailureKind ): boolean { // A plain 429 means transient rate limiting / high traffic for many OAuth providers. // Only connection-poison the quota cache when the upstream body explicitly says // the long-window quota is exhausted; otherwise fallback should try another account // without making this one look quota-depleted for 5 minutes. if (failureKind === "rate_limit" || failureKind === "transient") return false; return ( shouldPreserveQuotaSignalsFor429(provider) && !hasPerModelQuota(provider, model, connectionPassthroughModels) ); } export function classifyLockoutReason(status: number): string { if (status === 429) return "rate_limit"; if (status === 403) return "quota_exhausted"; return "unknown"; } export type DecayResult = { cleared: boolean; newFailureCount: number }; export function decayModelFailureCount( provider: string, connectionId: string, model: string ): DecayResult { const key = getModelLockKey(provider, connectionId, model); const failure = modelFailureState.get(key); if (!failure) return { cleared: false, newFailureCount: 0 }; const newFailureCount = Math.floor(failure.failureCount / 2); if (newFailureCount === 0) { modelFailureState.delete(key); return { cleared: true, newFailureCount: 0 }; } else { modelFailureState.set(key, { ...failure, failureCount: newFailureCount, }); return { cleared: false, newFailureCount }; } } /** * Clear all in-memory model lockouts and failure state (for tests / full reset). */ export function clearAllModelLockouts(): void { modelLockouts.clear(); modelFailureState.clear(); } /** * Check if a specific model on a specific account is locked * @returns {boolean} */ export function isModelLocked( provider: string, connectionId: string, model: string | null | undefined ): boolean { if (!model) return false; const key = getModelLockKey(provider, connectionId, model); cleanupModelLockKey(key); const entry = modelLockouts.get(key); return Boolean(entry); } /** * Get model lockout info (for debugging/dashboard) */ export function getModelLockoutInfo( provider: string, connectionId: string, model: string | null | undefined ) { if (!model) return null; const key = getModelLockKey(provider, connectionId, model); cleanupModelLockKey(key); const entry = modelLockouts.get(key); if (!entry) return null; return { reason: entry.reason, remainingMs: entry.until - Date.now(), lockedAt: new Date(entry.lockedAt).toISOString(), failureCount: entry.failureCount, }; } export type ModelLockoutInfo = { provider: string; connectionId: string; model: string; reason: string; remainingMs: number; failureCount: number; lockedAt: string; until: number; }; /** * Get all active model lockouts (for dashboard) */ export function getAllModelLockouts(): ModelLockoutInfo[] { const now = Date.now(); const active: ModelLockoutInfo[] = []; for (const key of modelLockouts.keys()) { cleanupModelLockKey(key, now); } for (const [key, entry] of modelLockouts) { const [provider, connectionId, ...modelParts] = key.split(":"); const model = modelParts.join(":"); active.push({ provider, connectionId, model, reason: entry.reason, remainingMs: entry.until - now, failureCount: entry.failureCount, lockedAt: new Date(entry.lockedAt).toISOString(), until: entry.until, }); } return active; } // ─── Provider Breaker Compatibility Wrappers ──────────────────────────────── // Legacy helpers now delegate to the shared provider circuit breaker. type ProviderBreakerProfile = { failureThreshold?: number; degradationThreshold?: number; resetTimeoutMs?: number; circuitBreakerThreshold?: number; circuitBreakerReset?: number; }; function getProviderBreaker(provider: string | null | undefined) { return provider ? getCircuitBreaker(provider) : null; } function configureProviderBreaker( provider: string | null | undefined, profile?: ProviderBreakerProfile | null ) { if (!provider) return null; const resolvedProfile = { ...getProviderProfile(provider), ...profile }; // Issue #2100 follow-up: resolve useUpstream429BreakerHints from the // provider profile (stored override) or fall back to per-provider default. // Stored value type is `boolean | undefined` — never `null` after PATCH. const userValue = resolvedProfile.useUpstream429BreakerHints; const useHints = resolveUseUpstream429BreakerHints(provider, userValue); return getCircuitBreaker(provider, { failureThreshold: resolvedProfile.failureThreshold ?? resolvedProfile.circuitBreakerThreshold, resetTimeout: resolvedProfile.resetTimeoutMs ?? resolvedProfile.circuitBreakerReset, ...(useHints ? { cooldownByKind: { rate_limit: 60_000, quota_exhausted: 3_600_000, } satisfies Partial>, classifyError: classify429FromError, } : {}), degradationThreshold: resolvedProfile.degradationThreshold, maxBackoffMultiplier: resolvedProfile.maxBackoffMultiplier, backoffEscalationCount: resolvedProfile.backoffEscalationCount, }); } /** * Check if a provider is currently blocked by the shared circuit breaker. */ export function isProviderInCooldown(provider: string | null | undefined): boolean { const breaker = getProviderBreaker(provider); return breaker ? !breaker.canExecute() : false; } /** * Get remaining retry-after time for a provider breaker. */ export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null { const breaker = getProviderBreaker(provider); if (!breaker || breaker.canExecute()) return null; const remaining = breaker.getRetryAfterMs(); return remaining > 0 ? remaining : null; } export function getProviderBreakerState(provider: string | null | undefined) { const breaker = getProviderBreaker(provider); return breaker?.getStatus?.() ?? null; } /** * Record a provider failure against the shared circuit breaker. * Delegates to the existing CircuitBreaker utility which handles * failure counting, threshold detection, and state transitions. * * IMPORTANT: If the breaker is already OPEN (in cooldown), we skip * recording the failure to prevent resetting the cooldown timer. * This matches the original behavior where failures during cooldown * were ignored to avoid indefinite lockout. */ export function recordProviderFailure( provider: string | null | undefined, log?: { warn?: (...args: unknown[]) => void }, connectionId?: string | null, profile?: ProviderBreakerProfile | null ): void { if (!provider) return; // Deduplicate rapid-fire failures from the same connection if (connectionId) { const dedupKey = `${provider}:${connectionId}`; const now = Date.now(); const lastFailure = lastConnectionFailure.get(dedupKey); if (lastFailure && now - lastFailure < CONNECTION_FAILURE_DEDUP_MS) { return; } lastConnectionFailure.delete(dedupKey); lastConnectionFailure.set(dedupKey, now); pruneConnectionFailureDedupeEntries(); } const breaker = configureProviderBreaker(provider, profile); if (!breaker) return; if (!breaker.canExecute()) return; breaker._onFailure(); if (!breaker.canExecute()) { log?.warn?.(`[ProviderFailure] ${provider}: circuit breaker opened after repeated failures`); } } /** * Reset the shared provider breaker. */ export function clearProviderFailure(provider: string | null | undefined): void { const breaker = getProviderBreaker(provider); breaker?.reset(); } /** * Get all providers currently blocked by the shared breaker. */ export function getProvidersInCooldown(): Array<{ provider: string; failureCount: number; cooldownRemainingMs: number | null; lastFailureAt: number | null; }> { return getAllCircuitBreakerStatuses() .filter((status) => { const breaker = getProviderBreaker(status.name); return Boolean(breaker && !breaker.canExecute()); }) .map((status) => ({ provider: status.name, failureCount: status.failureCount, cooldownRemainingMs: status.retryAfterMs || null, lastFailureAt: status.lastFailureTime, })); } /** * Check if a status code should be counted toward provider failure threshold */ export function isProviderFailureCode(status: number): boolean { return PROVIDER_FAILURE_ERROR_CODES.has(status); } /** * Returns true when a checkFallbackError result signals that the entire provider * quota is exhausted for this request, so the combo router can skip remaining * targets from the same provider (#1731). * * Covers: * - reason === "quota_exhausted" (subscription, daily, credits) * - creditsExhausted flag * - dailyQuotaExhausted flag */ export function isProviderExhaustedReason(result: { reason?: string; creditsExhausted?: boolean; dailyQuotaExhausted?: boolean; }): boolean { if (result.creditsExhausted || result.dailyQuotaExhausted) return true; return result.reason === RateLimitReason.QUOTA_EXHAUSTED; } // ─── Retry-After Parsing ──────────────────────────────────────────────────── /** * Parse retry-after information from JSON error response bodies. * Providers embed retry info in different formats. * * @param {string|object} responseBody - Raw response body or parsed JSON * @returns {{ retryAfterMs: number|null, reason: string }} */ export function parseRetryAfterFromBody(responseBody: unknown): { retryAfterMs: number | null; reason: RateLimitReasonValue; } { let body: JsonRecord; try { body = toJsonRecord(typeof responseBody === "string" ? JSON.parse(responseBody) : responseBody); } catch { return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN }; } if (Object.keys(body).length === 0) { return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN }; } // Gemini: { error: { details: [{ retryDelay: "33s" }] } } const error = toJsonRecord(body.error); const details = error.details || body.details || []; for (const detail of Array.isArray(details) ? details : []) { const detailRecord = toJsonRecord(detail); if (detailRecord.retryDelay) { return { retryAfterMs: parseDelayString(detailRecord.retryDelay), reason: RateLimitReason.RATE_LIMIT_EXCEEDED, }; } } // OpenAI: "Please retry after 20s" in message const msg = String(error.message || body.message || ""); const retryMatch = /retry\s+after\s+(\d+)\s*s/i.exec(msg); if (retryMatch) { return { retryAfterMs: Number.parseInt(retryMatch[1], 10) * 1000, reason: RateLimitReason.RATE_LIMIT_EXCEEDED, }; } // Anthropic: error type classification const errorType = String(error.type || body.type || ""); if (errorType === "rate_limit_error") { return { retryAfterMs: null, reason: RateLimitReason.RATE_LIMIT_EXCEEDED }; } // Classify by error message keywords const reason = classifyErrorText(msg || errorType); return { retryAfterMs: null, reason }; } /** * Parse delay strings like "33s", "2m", "1h", "1500ms" */ function parseDelayString(value: unknown): number | null { if (!value) return null; const str = String(value).trim(); const msMatch = /^(\d+)\s*ms$/i.exec(str); if (msMatch) return Number.parseInt(msMatch[1], 10); const secMatch = /^(\d+)\s*s$/i.exec(str); if (secMatch) return Number.parseInt(secMatch[1], 10) * 1000; const minMatch = /^(\d+)\s*m$/i.exec(str); if (minMatch) return Number.parseInt(minMatch[1], 10) * 60 * 1000; const hrMatch = /^(\d+)\s*h$/i.exec(str); if (hrMatch) return Number.parseInt(hrMatch[1], 10) * 3600 * 1000; // Bare number → seconds const num = Number.parseInt(str, 10); return Number.isNaN(num) ? null : num * 1000; } // T07: parse retry time from error text body with combined "XhYmZs" format. export function parseRetryFromErrorText(errorText: unknown): number | null { if (!errorText || typeof errorText !== "string") return null; const msg: string = String(errorText); const bodyHintMs = parseRetryHintFromJsonBody(msg, MAX_PROVIDER_COOLDOWN_MS); if (bodyHintMs !== null) return bodyHintMs; // Issue #2321: parse embedded absolute ISO retry timestamps. const isoMatch = /\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i.exec( msg ); if (isoMatch) { const parsedTs = Date.parse(isoMatch[1]); if (Number.isFinite(parsedTs)) { const waitMs = parsedTs - Date.now(); if (waitMs > 0) return waitMs; } } const match = /reset after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg); if (match?.[1] || match?.[2] || match?.[3]) return computeDurationMs(match); // Variant without "reset after": "will reset after XhYmZs" const altMatch = /will reset after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg); if (altMatch?.[1] || altMatch?.[2] || altMatch?.[3]) return computeDurationMs(altMatch); // Antigravity / Cloud Code phrasing: "Resets in 164h27m24s". const resetsInMatch = /resets? in (\d+h)?(\d+m)?(\d+s)?/i.exec(msg); if (resetsInMatch?.[1] || resetsInMatch?.[2] || resetsInMatch?.[3]) { return computeDurationMs(resetsInMatch); } return null; } /** * Compute total milliseconds from regex match groups (Xh)(Ym)(Zs) * Capped at 30 days to prevent adversarial/buggy upstream from locking indefinitely. */ const MAX_PROVIDER_COOLDOWN_MS = 30 * 24 * 60 * 60 * 1000; // 30 days function computeDurationMs(match: RegExpMatchArray): number | null { let totalMs = 0; if (match[1]) totalMs += Number.parseInt(match[1], 10) * 3600 * 1000; // hours if (match[2]) totalMs += Number.parseInt(match[2], 10) * 60 * 1000; // minutes if (match[3]) totalMs += Number.parseInt(match[3], 10) * 1000; // seconds return totalMs > 0 ? Math.min(totalMs, MAX_PROVIDER_COOLDOWN_MS) : null; } function isSubscriptionQuotaText(lower: string): boolean { return ( lower.includes("usage limit reached") || lower.includes("usage limit has been") || lower.includes("claude pro usage limit") || lower.includes("you've reached your usage limit") || lower.includes("you have reached your usage limit") ); } // ─── Error Classification ─────────────────────────────────────────────────── /** * Classify error text into RateLimitReason */ export function classifyErrorText(errorText: unknown): RateLimitReasonValue { if (!errorText) return RateLimitReason.UNKNOWN; const lower = String(errorText).toLowerCase(); if ( lower.includes("quota exceeded") || lower.includes("quota depleted") || lower.includes("quota will reset") || lower.includes("your quota will reset") || lower.includes("quota has been exceeded") || lower.includes("hour quota") || lower.includes("billing") || looksLikeQuotaExhausted(lower) || // Issue #2321: Anthropic OAuth (Claude Code Pro/Team) 429 bodies surface // the subscription quota with phrases that contain neither "quota" nor // "billing". Without these patterns the error was classified as a // transient RATE_LIMIT_EXCEEDED (~5s base cooldown), which cascades all // Pro accounts into a tight retry loop until the 5h window resets. isSubscriptionQuotaText(lower) ) { return RateLimitReason.QUOTA_EXHAUSTED; } // T10: credits_exhausted signals if (isCreditsExhausted(lower)) { return RateLimitReason.QUOTA_EXHAUSTED; } // T06: account_deactivated signals if (isAccountDeactivated(lower)) { return RateLimitReason.AUTH_ERROR; } const configuredRule = matchErrorRuleByText(errorText); if (configuredRule?.reason) return configuredRule.reason; if (lower.includes("rate_limit")) return RateLimitReason.RATE_LIMIT_EXCEEDED; if (lower.includes("resource exhausted") || lower.includes("high demand")) return RateLimitReason.MODEL_CAPACITY; if ( lower.includes("unauthorized") || lower.includes("invalid api key") || lower.includes("authentication") ) { return RateLimitReason.AUTH_ERROR; } if (lower.includes("server error") || lower.includes("internal error")) { return RateLimitReason.SERVER_ERROR; } return RateLimitReason.UNKNOWN; } /** * Classify HTTP status + error text into RateLimitReason * * If context (provider, headers, body) is supplied, provider-specific rules * are evaluated FIRST. A provider like Opencode can signal account-wide quota * exhaustion via `x-ratelimit-remaining-requests: 0` even when the body says * "rate limit" — without context, classifyError falls through to the global * text rules and misclassifies as RATE_LIMIT_EXCEEDED. With context, the * provider rule takes precedence. */ export function classifyError( status: number, errorText: unknown, context?: { provider?: string | null; headers?: Record | null; body?: unknown } ): RateLimitReasonValue { // Provider-specific rules take priority — they have the most accurate signal // (e.g. `x-ratelimit-remaining-requests: 0` is irrefutable account exhaustion). if (context?.provider) { const match = getProviderErrorRuleMatch( context.provider, status, context.headers ?? null, context.body ); if (match) return match.reason; } // Text classification takes priority (more specific) const textReason = classifyErrorText(errorText); if (textReason !== RateLimitReason.UNKNOWN) return textReason; // Fall back to status code if (status === HTTP_STATUS.UNAUTHORIZED || status === HTTP_STATUS.FORBIDDEN) { return RateLimitReason.AUTH_ERROR; } if (status === HTTP_STATUS.PAYMENT_REQUIRED) { return RateLimitReason.QUOTA_EXHAUSTED; } if (status === HTTP_STATUS.RATE_LIMITED) { return RateLimitReason.RATE_LIMIT_EXCEEDED; } if (status === HTTP_STATUS.SERVICE_UNAVAILABLE || status === 529) { return RateLimitReason.MODEL_CAPACITY; } if (status >= 500) { return RateLimitReason.SERVER_ERROR; } return RateLimitReason.UNKNOWN; } // ─── Daily Quota Helpers ──────────────────────────────────────────────────── /** * Calculate milliseconds from now until tomorrow at midnight (00:00:00). * Used to lock a model until the next day when daily quota is exhausted. * @returns {number} Milliseconds until tomorrow */ export function getMsUntilTomorrow(): number { const nowMs = Date.now(); const tomorrow = new Date(nowMs); tomorrow.setDate(tomorrow.getDate() + 1); tomorrow.setHours(0, 0, 0, 0); const ms = tomorrow.getTime() - nowMs; // Guard against DST edge cases: if ms is negative (shouldn't happen) or // unreasonably large (>25h due to spring-forward), cap at 24 hours. return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000; } /** * Check if error text indicates daily quota exhaustion (as opposed to rate limiting). * Daily quota errors typically mention "today's quota" or "try again tomorrow". * @param {string} errorText - Error message text * @returns {boolean} True if daily quota is exhausted */ export function isDailyQuotaExhausted(errorText: string): boolean { if (!errorText) return false; const lower = errorText.toLowerCase(); return ( lower.includes("today's quota") || lower.includes("daily quota") || lower.includes("try again tomorrow") ); } // ─── Configurable Backoff ─────────────────────────────────────────────────── /** * Get backoff duration from configurable steps. * @param {number} failureCount - Number of consecutive failures * @returns {number} Duration in ms */ export function getBackoffDuration(failureCount: number): number { const idx = Math.min(failureCount, BACKOFF_STEPS_MS.length - 1); return BACKOFF_STEPS_MS[idx]; } // ─── Original API (Backward Compatible) ──────────────────────────────────── /** * Calculate exponential backoff cooldown for rate limits (429) * Level 0: 1s, Level 1: 2s, Level 2: 4s... → max 2 min * @param {number} backoffLevel - Current backoff level * @returns {number} Cooldown in milliseconds */ export function getQuotaCooldown(backoffLevel = 0) { return calculateBackoffCooldown(backoffLevel); } /** * Check if error should trigger account fallback (switch to next account) * @param {number} status - HTTP status code * @param {string} errorText - Error message text * @param {number} backoffLevel - Current backoff level for exponential backoff * @param {string} [model] - Optional model name for model-level lockout * @param {string} [provider] - Provider ID for profile-aware cooldowns * @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number, reason?: string }} */ export function checkFallbackError( status: number, errorText: string | null, backoffLevel: number = 0, _model: string | null = null, provider: string | null = null, headers: Headers | Record | null = null, profileOverride: ProviderProfile | null = null, structuredError?: { code?: string | null; type?: string | null } | null ): { shouldFallback: boolean; cooldownMs: number; baseCooldownMs?: number; newBackoffLevel?: number; usedUpstreamRetryHint?: boolean; reason?: string; permanent?: boolean; creditsExhausted?: boolean; dailyQuotaExhausted?: boolean; /** G-02: true when the error originates from an embedded service supervisor (not the upstream AI * provider itself). Callers should apply connection cooldown only — do NOT record a provider * circuit-breaker failure when this flag is set. */ skipProviderBreaker?: boolean; quotaResetHintMs?: number; /** #6061: the provider-configured cooldown (ms) before backoff scaling, surfaced so the * caller can persist an explicit reset window instead of the engine's scaled cooldown. */ configuredCooldownMs?: number; } { // G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown). // These are NOT upstream AI provider failures — they are local supervisor state changes. // Apply a short 5s connection cooldown without tripping the provider circuit breaker. if (status === 503 && headers) { const hintValue = typeof (headers as Headers).get === "function" ? (headers as Headers).get("x-omni-fallback-hint") : (headers as Record)["x-omni-fallback-hint"] || (headers as Record)["X-Omni-Fallback-Hint"]; if (typeof hintValue === "string" && hintValue.toLowerCase() === "connection_cooldown") { return { shouldFallback: true, cooldownMs: 5_000, baseCooldownMs: 5_000, newBackoffLevel: 0, reason: "service_not_running", skipProviderBreaker: true, }; } } const errorStr = (errorText || "").toString(); const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null); const maxBackoffSteps = profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel; const retryableStatuses = new Set([ HTTP_STATUS.REQUEST_TIMEOUT, HTTP_STATUS.RATE_LIMITED, HTTP_STATUS.SERVER_ERROR, HTTP_STATUS.BAD_GATEWAY, HTTP_STATUS.SERVICE_UNAVAILABLE, HTTP_STATUS.GATEWAY_TIMEOUT, ]); function parseResetFromHeaders(headers: Headers | Record | null): number | null { if (!headers) return null; const recordHeaders = headers as Record; // Retry-After header const retryAfter = typeof (headers as Headers).get === "function" ? (headers as Headers).get("retry-after") : recordHeaders["retry-after"] || recordHeaders["Retry-After"]; if (retryAfter) { const seconds = Number.parseInt(retryAfter, 10); if (!Number.isNaN(seconds) && String(seconds) === String(retryAfter).trim()) { return Date.now() + seconds * 1000; } const date = new Date(retryAfter); if (!Number.isNaN(date.getTime())) return date.getTime(); } // X-RateLimit-Reset const rlReset = typeof (headers as Headers).get === "function" ? (headers as Headers).get("x-ratelimit-reset") : recordHeaders["x-ratelimit-reset"] || recordHeaders["X-RateLimit-Reset"]; if (rlReset) { const ts = Number.parseInt(rlReset, 10); if (!Number.isNaN(ts)) { return ts > 10000000000 ? ts : ts * 1000; } } return null; } function getUpstreamRetryHintMs() { if (!profile?.useUpstreamRetryHints) return null; const resetTime = parseResetFromHeaders(headers); if (resetTime) { const waitMs = Math.max(resetTime - Date.now(), 0); if (waitMs > 0) return waitMs; } const retryFromErrorText = parseRetryFromErrorText(errorStr); if (retryFromErrorText && retryFromErrorText > 0) { return retryFromErrorText; } return null; } function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) { void reason; const baseCooldownMs = typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0 ? profile.baseCooldownMs : COOLDOWN_MS.transientInitial; return { baseCooldownMs, cooldownMs: getScaledCooldown(baseCooldownMs, level + 1, maxBackoffSteps), newBackoffLevel: Math.min(level + 1, maxBackoffSteps), }; } function buildRetryableFallback(reason: RateLimitReasonValue) { const upstreamRetryHintMs = getUpstreamRetryHintMs(); if (typeof upstreamRetryHintMs === "number" && upstreamRetryHintMs > 0) { return { shouldFallback: true, cooldownMs: upstreamRetryHintMs, baseCooldownMs: upstreamRetryHintMs, newBackoffLevel: 0, usedUpstreamRetryHint: true, reason, }; } const scaled = getScaledBaseCooldown(reason, backoffLevel); return { shouldFallback: true, cooldownMs: scaled.cooldownMs, baseCooldownMs: scaled.baseCooldownMs, newBackoffLevel: scaled.newBackoffLevel, usedUpstreamRetryHint: false, reason, }; } const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED; const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider); const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429; // Check error message FIRST - specific patterns take priority over status codes if (errorText) { // T06 (sub2api #1037): Permanent account deactivation — do NOT retry, mark as permanent failure if (isAccountDeactivated(errorStr)) { return { shouldFallback: true, cooldownMs: 365 * 24 * 60 * 60 * 1000, // 1 year = effectively permanent reason: RateLimitReason.AUTH_ERROR, permanent: true, }; } // T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) { return { shouldFallback: true, cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown reason: RateLimitReason.QUOTA_EXHAUSTED, creditsExhausted: true, }; } // Daily quota exhausted — lock model until tomorrow if (shouldUseQuotaSignal && isDailyQuotaExhausted(errorStr)) { const msUntilTomorrow = getMsUntilTomorrow(); // Cap at 24 hours to handle timezone edge cases const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000); return { shouldFallback: true, cooldownMs, reason: RateLimitReason.QUOTA_EXHAUSTED, dailyQuotaExhausted: true, }; } // Issue #2321: Anthropic OAuth (Claude Pro/Team) returns 429 with // "Usage Limit Reached" for the 5-hour subscription quota. The // pattern-based classifier now flags these as QUOTA_EXHAUSTED, but // without a dedicated branch the request would still fall through to // the generic 429 retry path (~5s base cooldown). Honor upstream // Retry-After / reset hints only when the profile enables them; // otherwise apply a local 1h cooldown so all Pro accounts on the same // subscription tier stop cycling through tight retries without letting // upstream-provided windows bypass the operator setting. (We // deliberately do not use COOLDOWN_MS.paymentRequired here — that // constant is 2 minutes, which is shorter than the recovery time of a // subscription quota.) if ( shouldUseQuotaSignal && !isCreditsExhausted(errorStr) && !isDailyQuotaExhausted(errorStr) && isSubscriptionQuotaText(errorStr.toLowerCase()) ) { // getUpstreamRetryHintMs() gates both headers and body reset text on // profile.useUpstreamRetryHints. const hintMs = getUpstreamRetryHintMs(); const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour const bodyHint = parseRetryFromErrorText(errorStr); return { shouldFallback: true, cooldownMs: hintMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS, reason: RateLimitReason.QUOTA_EXHAUSTED, usedUpstreamRetryHint: Boolean(hintMs), quotaResetHintMs: bodyHint ?? undefined, }; } const quotaResetHintMs = parseRetryFromErrorText(errorStr); if ( shouldUseQuotaSignal && quotaResetHintMs && classifyErrorText(errorStr) === RateLimitReason.QUOTA_EXHAUSTED ) { const fallbackResult = buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED); return { ...fallbackResult, quotaResetHintMs, }; } // #2929: A route-restriction 403 (e.g. Fireworks Fire Pass keys returning // "Fire Pass API keys are not authorized for this route." on the /models // endpoint) means the key is valid but lacks access to THIS route — it still // serves chat. It must NOT cool down the connection or be classified as an // auth error, otherwise a single model-listing 403 marks the key unavailable. if ( status === HTTP_STATUS.FORBIDDEN && errorStr.toLowerCase().includes("not authorized for this route") ) { return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; } if ( status === HTTP_STATUS.FORBIDDEN && provider && getProviderCategory(provider) === "apikey" && !errorStr.toLowerCase().includes("has not been used in project") && !errorStr.toLowerCase().includes("hour quota") && !errorStr.toLowerCase().includes("quota has been exceeded") ) { return buildRetryableFallback(RateLimitReason.AUTH_ERROR); } } // Gemini-specific: use known published RPM/RPD limits to distinguish 429 types. // Gemini returns the same error body for both, so we use per-model request // counters to decide: if daily count >= RPD → quota_exhausted (midnight lockout); // if minute count >= RPM → rate_limit_exceeded (exponential backoff). if (provider === "gemini" && status === HTTP_STATUS.RATE_LIMITED && _model) { if (isRpdExhausted(_model)) { return buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED); } if (isRpmExhausted(_model)) { return buildRetryableFallback(RateLimitReason.RATE_LIMIT_EXCEEDED); } } const configuredRule = isRateLimitStatus && !preserveQuota429 ? matchErrorRuleByStatus(status) : findMatchingErrorRule(status, errorStr); if (configuredRule) { if (configuredRule.backoff) { // Provider-specific rules in `providerRuleRegistry` are MORE SPECIFIC // than the configured (global) rule, so we check them first. If a // provider rule matches, it overrides the configured rule's reason // (e.g. Opencode's `x-ratelimit-remaining-requests: 0` overrides // 429 → RATE_LIMIT_EXCEEDED). We do NOT call the full `classifyError` // here because its global status fallback would otherwise override // specific configured reasons (e.g. 503 → SERVER_ERROR would be // shadowed by 503 → MODEL_CAPACITY). const providerMatch = provider ? getProviderErrorRuleMatch(provider, status, headers, structuredError ?? null) : null; const reason = providerMatch ? providerMatch.reason : (configuredRule.reason ?? RateLimitReason.UNKNOWN); // Fix C: thread `providerMatch.cooldownMs` through so a configured rule // like the "Monthly usage limit reached. Resets in N days." matcher can // declare an explicit cooldown (e.g. 13 days) and have it win over the // scaled backoff default returned by `buildRetryableFallback`. Without // this, the rule's reason is used but its cooldownMs is silently // dropped — which is exactly the user-visible bug where a 13-day // upstream quota reset was being treated as ~60s. const providerCooldownMs = providerMatch?.cooldownMs !== undefined && providerMatch.cooldownMs > 0 ? providerMatch.cooldownMs : undefined; const fallback = buildRetryableFallback(reason); if (providerCooldownMs !== undefined) { return { ...fallback, cooldownMs: providerCooldownMs, baseCooldownMs: providerCooldownMs, configuredCooldownMs: providerCooldownMs, }; } return fallback; } const cooldownMs = configuredRule.cooldownMs ?? 0; return { shouldFallback: true, cooldownMs, baseCooldownMs: cooldownMs, configuredCooldownMs: cooldownMs, reason: configuredRule.reason ?? RateLimitReason.UNKNOWN, }; } if (status === HTTP_STATUS.NOT_ACCEPTABLE || retryableStatuses.has(status)) { return buildRetryableFallback(RateLimitReason.SERVER_ERROR); } // 400 — context overflow / malformed request / model access denied if (status === HTTP_STATUS.BAD_REQUEST) { // Check structured error codes first (more reliable, no false positives) // OpenAI: error.code === "model_not_found" // Anthropic: error.type === "not_found_error" / "permission_error" const structuredCode = typeof structuredError?.code === "string" ? structuredError.code.toLowerCase() : ""; const structuredType = typeof structuredError?.type === "string" ? structuredError.type.toLowerCase() : ""; // A clear bad-credential error must never be reclassified as model-access // (which would silently exhaust every combo target). Structured detection // below still catches genuine model_not_found / not_found_error codes. const looksLikeAuthCredentialError = AUTH_CREDENTIAL_ERROR_PATTERNS.some((p) => p.test(errorStr) ); const matchesModelAccessPattern = !looksLikeAuthCredentialError && MODEL_ACCESS_DENIED_PATTERNS.some((p) => p.test(errorStr)); const isModelAccessDeniedStructured = !!structuredError && (MODEL_ACCESS_DENIED_CODES.has(structuredCode) || MODEL_ACCESS_DENIED_TYPES.has(structuredType) || // Ambiguous types (e.g. Anthropic permission_error) only count as a model // access denial when the message text confirms it is about the model. (MODEL_ACCESS_AMBIGUOUS_TYPES.has(structuredType) && matchesModelAccessPattern)); const isOverflow = CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(errorStr)); const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr)); const isParamValidation = PARAM_VALIDATION_PATTERNS.some((p) => p.test(errorStr)); const isModelAccessDenied = isModelAccessDeniedStructured || matchesModelAccessPattern; if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied) { return { shouldFallback: true, cooldownMs: 0, reason: RateLimitReason.MODEL_CAPACITY, }; } // Some providers (e.g. MiMoCode) signal throttling with a non-standard 400 whose // body carries rate-limit semantics ("Detected high-frequency non-compliant // requests from you.") instead of a 429. Detected here (AFTER malformed/overflow // detection above, so a genuinely malformed 400 still wins and keeps its #2101 // zero-cooldown MODEL_CAPACITY classification), it is fallback-worthy at // connection-cooldown scope so combo can fail over to another target (#4976). if (RATE_LIMIT_TEXT_PATTERNS.some((p) => p.test(errorStr))) { return buildRetryableFallback(RateLimitReason.RATE_LIMIT_EXCEEDED); } // Generic 400 is not account-fallback-worthy. Combo routing may still try a // different provider/model because combo fallback is target-level orchestration. return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; } // All other errors - fallback with transient cooldown return { shouldFallback: true, cooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient, baseCooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient, reason: RateLimitReason.UNKNOWN, }; } // ─── Account State Management ─────────────────────────────────────────────── /** * Normalize a stored cooldown timestamp to epoch milliseconds. * * `rate_limited_until` is a TEXT column, but some write paths persist a raw * epoch NUMBER (e.g. `setConnectionRateLimitUntil` on the Antigravity full-quota * path). SQLite TEXT affinity coerces it to a numeric string like * "1781696905131.0", which `new Date(...)` cannot parse (→ NaN). Accept numeric * epoch strings/numbers as well as ISO strings and Date objects (#3954). */ export function cooldownUntilMs(value: string | number | Date | null | undefined): number { if (value === null || value === undefined || value === "") return NaN; if (value instanceof Date) return value.getTime(); if (typeof value === "number") return value; const raw = value.trim(); if (/^\d+(\.\d+)?$/.test(raw)) return Number(raw); return new Date(raw).getTime(); } /** * Check if account is currently unavailable (cooldown not expired) */ export function isAccountUnavailable(unavailableUntil: string | Date | null | undefined): boolean { if (!unavailableUntil) return false; const ms = cooldownUntilMs(unavailableUntil); return Number.isFinite(ms) && ms > Date.now(); } /** * Calculate unavailable until timestamp */ export function getUnavailableUntil(cooldownMs: number): string { return new Date(Date.now() + cooldownMs).toISOString(); } /** * Get the earliest rateLimitedUntil from a list of accounts */ export function getEarliestRateLimitedUntil( accounts: Array<{ rateLimitedUntil?: string | null }> ): string | null { let earliest: number | null = null; const now = Date.now(); for (const acc of accounts) { if (!acc.rateLimitedUntil) continue; const until = cooldownUntilMs(acc.rateLimitedUntil); if (!Number.isFinite(until) || until <= now) continue; if (!earliest || until < earliest) earliest = until; } if (!earliest) return null; return new Date(earliest).toISOString(); } /** * Format rateLimitedUntil to human-readable "reset after Xm Ys" */ export function formatRetryAfter( rateLimitedUntil: string | number | Date | null | undefined ): string { if (!rateLimitedUntil) return ""; const diffMs = new Date(rateLimitedUntil).getTime() - Date.now(); if (diffMs <= 0) return "reset after 0s"; const totalSec = Math.ceil(diffMs / 1000); const h = Math.floor(totalSec / 3600); const m = Math.floor((totalSec % 3600) / 60); const s = totalSec % 60; const parts: string[] = []; if (h > 0) parts.push(`${h}h`); if (m > 0) parts.push(`${m}m`); if (s > 0 || parts.length === 0) parts.push(`${s}s`); return `reset after ${parts.join(" ")}`; } /** * Filter available accounts (not in cooldown) */ export function filterAvailableAccounts( accounts: T[], excludeId: string | null = null ): T[] { const now = Date.now(); return accounts.filter((acc) => { if (excludeId && acc.id === excludeId) return false; if (acc.rateLimitedUntil) { const until = cooldownUntilMs(acc.rateLimitedUntil); if (Number.isFinite(until) && until > now) return false; } return true; }); } /** * Reset account state when request succeeds */ export function resetAccountState( account: T ): T | AccountState { if (!account) return account; // Persist the cooldown clear so a successfully-retried connection is no longer // marked as rate-limited in `provider_connections.rate_limited_until`. Mirrors // Fix A: the in-memory AccountState and the DB row must agree so the cascade // survives the request boundary and `clearStaleCrashCooldowns` doesn't have // to rediscover what we already know is healthy. // Best-effort: a DB write failure must not crash the request path. const connId = (account as AccountState | null | undefined)?.id; if (typeof connId === "string" && connId.length > 0) { try { setConnectionRateLimitUntil(connId, null); } catch { // ignore — best effort } } return { ...account, rateLimitedUntil: null, backoffLevel: 0, lastError: null, status: "active", }; } /** * Apply error state to account */ export function applyErrorState( account: T, status: number, errorText: string | null, provider: string | null = null ): T | AccountState { if (!account) return account; const backoffLevel = account.backoffLevel || 0; const fallbackDecision = checkFallbackError(status, errorText, backoffLevel, null, provider); const { cooldownMs, reason } = fallbackDecision; const newBackoffLevel = "newBackoffLevel" in fallbackDecision ? fallbackDecision.newBackoffLevel : undefined; // Cooldown may be overridden by a configured provider rule (see // `accountFallback.ts:1511-1540` provider-match branch + the // `providerMatch.cooldownMs` thread-through added by Fix C). When the // configured rule sets an explicit cooldownMs, it wins over the scaled // backoff default that `checkFallbackError` returned. const configuredCooldownMs = "configuredCooldownMs" in fallbackDecision ? (fallbackDecision as { configuredCooldownMs?: number }).configuredCooldownMs : undefined; const effectiveCooldownMs = typeof configuredCooldownMs === "number" && configuredCooldownMs > 0 ? configuredCooldownMs : cooldownMs; const nextState: T | AccountState = { ...account, rateLimitedUntil: effectiveCooldownMs > 0 ? getUnavailableUntil(effectiveCooldownMs) : null, backoffLevel: newBackoffLevel ?? backoffLevel, lastError: { status, message: errorText, timestamp: new Date().toISOString(), reason }, status: "error", }; // Persist the cooldown to `provider_connections.rate_limited_until` so the // cascade survives the request boundary. Before Fix A the cooldown only // lived in the in-memory AccountState object returned here, which was // discarded the moment the request ended — the same exhausted key was // re-picked on the very next request. // Best-effort try/catch mirrors `open-sse/executors/antigravity.ts:343` // (`markConnectionQuotaExhausted`) so a DB failure can never crash the // chat path. See issue #1 (per-account 429 cascade not persisting). const connId = (account as AccountState | null | undefined)?.id; if ( typeof connId === "string" && connId.length > 0 && effectiveCooldownMs > 0 && nextState.rateLimitedUntil ) { try { const untilMs = cooldownUntilMs(nextState.rateLimitedUntil); if (Number.isFinite(untilMs) && untilMs > Date.now()) { setConnectionRateLimitUntil(connId, untilMs); } } catch { // ignore — best effort } } return nextState; } /** * Get account health score (0-100) for P2C selection (Phase 9) * @param {object} account * @returns {number} score 0 = unhealthy, 100 = perfectly healthy */ export function getAccountHealth( account: AccountState | null | undefined, model?: unknown ): number { if (!account) return 0; let score = 100; score -= (account.backoffLevel || 0) * 10; if (account.lastError) score -= 20; if (account.rateLimitedUntil && isAccountUnavailable(account.rateLimitedUntil)) score -= 30; return Math.max(0, score); }