wehub-resource-sync
1864 lines
70 KiB
TypeScript
1864 lines
70 KiB
TypeScript
import {
|
|
BACKOFF_STEPS_MS,
|
|
PROVIDER_PROFILES,
|
|
RateLimitReason,
|
|
HTTP_STATUS,
|
|
} from "../config/constants.ts";
|
|
import {
|
|
BACKOFF_CONFIG,
|
|
COOLDOWN_MS,
|
|
calculateBackoffCooldown,
|
|
findMatchingErrorRule,
|
|
matchErrorRuleByText,
|
|
matchErrorRuleByStatus,
|
|
} from "../config/errorConfig.ts";
|
|
import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts";
|
|
import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts";
|
|
import {
|
|
DEFAULT_RESILIENCE_SETTINGS,
|
|
resolveResilienceSettings,
|
|
} from "../../src/lib/resilience/settings";
|
|
import { resolveModelLockoutSettings } from "../../src/lib/resilience/modelLockoutSettings";
|
|
import {
|
|
getAllCircuitBreakerStatuses,
|
|
getCircuitBreaker,
|
|
} from "../../src/shared/utils/circuitBreaker";
|
|
import {
|
|
classify429FromError,
|
|
looksLikeQuotaExhausted,
|
|
type FailureKind,
|
|
} from "../../src/shared/utils/classify429";
|
|
import { resolveProviderId } from "../../src/shared/constants/providers";
|
|
import { resolveUseUpstream429BreakerHints } from "../../src/shared/utils/providerHints";
|
|
import { getCodexModelScope } from "../config/codexQuotaScopes.ts";
|
|
import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts";
|
|
import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts";
|
|
import { setConnectionRateLimitUntil } from "@/lib/db/providers";
|
|
import { parseRetryHintFromJsonBody } from "./retryAfterJson.ts";
|
|
|
|
export type ProviderProfile = {
|
|
baseCooldownMs: number;
|
|
useUpstreamRetryHints: boolean;
|
|
useUpstream429BreakerHints?: boolean;
|
|
maxCooldownMs: number;
|
|
maxBackoffSteps: number;
|
|
failureThreshold: number;
|
|
resetTimeoutMs: number;
|
|
transientCooldown: number;
|
|
rateLimitCooldown: number;
|
|
maxBackoffLevel: number;
|
|
circuitBreakerThreshold: number;
|
|
circuitBreakerReset: number;
|
|
// Adaptive circuit breaker fields
|
|
degradationThreshold?: number;
|
|
// Provider-level cooldown fields
|
|
providerFailureThreshold: number;
|
|
providerFailureWindowMs: number;
|
|
providerCooldownMs: number;
|
|
maxBackoffMultiplier?: number;
|
|
backoffEscalationCount?: number;
|
|
};
|
|
type JsonRecord = Record<string, unknown>;
|
|
type RateLimitReasonValue = (typeof RateLimitReason)[keyof typeof RateLimitReason];
|
|
type ModelLockoutEntry = {
|
|
reason: string;
|
|
until: number;
|
|
lockedAt: number;
|
|
failureCount: number;
|
|
lastFailureAt: number;
|
|
resetAfterMs: number;
|
|
};
|
|
type ModelFailureState = {
|
|
failureCount: number;
|
|
lastFailureAt: number;
|
|
resetAfterMs: number;
|
|
/** Cooldown applied on the last failure — extends the escalation window so a
|
|
* model that fails again right after its lockout expires keeps escalating. */
|
|
lastCooldownMs?: number;
|
|
};
|
|
type AccountState = JsonRecord & {
|
|
id?: string | null;
|
|
rateLimitedUntil?: string | null;
|
|
backoffLevel?: number | null;
|
|
lastError?: unknown;
|
|
status?: string;
|
|
};
|
|
|
|
function toJsonRecord(value: unknown): JsonRecord {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
|
}
|
|
|
|
// Provider-level failure tracking for circuit breaker behavior
|
|
// Error codes that count toward provider-level failure threshold.
|
|
// 429 is included: per-error-type cooldowns (rate_limit: 60s, quota_exhausted: 1h)
|
|
// prevent cascading provider trips at scale (Issue #1846 concern addressed),
|
|
// while still allowing the circuit breaker to open on sustained 429s and
|
|
// prevent infinite combo retries (Issue #3200).
|
|
const PROVIDER_FAILURE_ERROR_CODES = new Set([408, 429, 500, 502, 503, 504]);
|
|
|
|
// Per-connection failure deduplication: prevents rapid-fire failures from the
|
|
// same connection from counting multiple times toward the provider breaker.
|
|
const CONNECTION_FAILURE_DEDUP_MS = 5000;
|
|
const MAX_CONNECTION_FAILURE_DEDUP_ENTRIES = 10_000;
|
|
const lastConnectionFailure = new Map<string, number>();
|
|
|
|
function pruneConnectionFailureDedupeEntries(): void {
|
|
while (lastConnectionFailure.size > MAX_CONNECTION_FAILURE_DEDUP_ENTRIES) {
|
|
const oldestKey = lastConnectionFailure.keys().next().value;
|
|
if (typeof oldestKey !== "string") return;
|
|
lastConnectionFailure.delete(oldestKey);
|
|
}
|
|
}
|
|
|
|
const _connectionFailureSweep = setInterval(() => {
|
|
const now = Date.now();
|
|
for (const [key, ts] of lastConnectionFailure) {
|
|
if (now - ts > CONNECTION_FAILURE_DEDUP_MS) lastConnectionFailure.delete(key);
|
|
}
|
|
}, 60_000);
|
|
if (typeof _connectionFailureSweep === "object" && "unref" in _connectionFailureSweep) {
|
|
(_connectionFailureSweep as { unref?: () => void }).unref?.();
|
|
}
|
|
|
|
// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
|
|
// When a 401 body contains these strings, the account is permanently dead
|
|
// and should NOT be retried after token refresh.
|
|
export const ACCOUNT_DEACTIVATED_SIGNALS = [
|
|
"account_deactivated",
|
|
"account has been deactivated",
|
|
"account has been disabled",
|
|
"your account has been suspended",
|
|
"this account is deactivated",
|
|
// AG (Antigravity/Google Cloud Code) permanent ban signals
|
|
"verify your account to continue",
|
|
"this service has been disabled in this account for violation",
|
|
"this service has been disabled in this account",
|
|
];
|
|
|
|
// Custom banned signals — loaded from DB settings at runtime.
|
|
// Combined with ACCOUNT_DEACTIVATED_SIGNALS in isAccountDeactivated().
|
|
let _customBannedSignals: string[] = [];
|
|
|
|
export function setCustomBannedSignals(signals: string[]): void {
|
|
_customBannedSignals = signals;
|
|
}
|
|
|
|
export function getMergedBannedSignals(): string[] {
|
|
if (_customBannedSignals.length === 0) return ACCOUNT_DEACTIVATED_SIGNALS;
|
|
return [...ACCOUNT_DEACTIVATED_SIGNALS, ..._customBannedSignals];
|
|
}
|
|
|
|
// T10 (sub2api PR #1169): Signals that indicate billing credits are exhausted.
|
|
// Distinct from rate-limit 429 — the account won't recover until credits are added.
|
|
export const CREDITS_EXHAUSTED_SIGNALS = [
|
|
"insufficient_quota",
|
|
"billing_hard_limit_reached",
|
|
"exceeded your current quota",
|
|
"exceeded your current usage quota",
|
|
"credit_balance_too_low",
|
|
"your credit balance is too low",
|
|
"credits exhausted",
|
|
"out of credits",
|
|
"payment required",
|
|
"free tier of the model has been exhausted",
|
|
// #5239: providers (e.g. DeepSeek/GLM-style) return "Insufficient account balance"
|
|
// on a depleted key. 402 is already terminalized by status, but catch non-402
|
|
// out-of-credit bodies here too.
|
|
"insufficient balance",
|
|
"insufficient_balance",
|
|
"insufficient account balance",
|
|
];
|
|
|
|
// T11: Signals that indicate OAuth token is invalid/expired (not permanent deactivation)
|
|
export const OAUTH_INVALID_TOKEN_SIGNALS = [
|
|
"invalid authentication credentials",
|
|
"oauth 2",
|
|
"login cookie",
|
|
"valid authentication credential",
|
|
"invalid credentials",
|
|
];
|
|
|
|
// Context overflow patterns — the prompt exceeds the model's maximum context length.
|
|
// Different providers phrase this differently. Used to decide whether a 400 error
|
|
// should trigger combo fallback (a different model may have a larger context window).
|
|
const CONTEXT_OVERFLOW_PATTERNS = [
|
|
/\binput is too long\b/i,
|
|
/\binput too long\b/i,
|
|
/\bcontext.*(too long|exceeded|overflow|limit)/i,
|
|
/\btoo many tokens\b/i,
|
|
/\bprompt is too long\b/i,
|
|
/\bcontext window/i,
|
|
/\bmaximum context/i,
|
|
/\bmax.*token/i,
|
|
/\btoken limit/i,
|
|
/\brequest too large\b/i,
|
|
];
|
|
|
|
// Structured error codes that reliably indicate model access denied
|
|
// (more reliable than regex on human-readable messages).
|
|
// OpenAI: { error: { code: "model_not_found", ... } }
|
|
// Anthropic: { error: { type: "not_found_error", ... } }
|
|
const MODEL_ACCESS_DENIED_CODES = new Set([
|
|
"model_not_found", // OpenAI, OpenAI-compatible (Kiro, Together, Fireworks, etc.)
|
|
"deployment_not_found", // Azure OpenAI
|
|
]);
|
|
|
|
const MODEL_ACCESS_DENIED_TYPES = new Set([
|
|
"not_found_error", // Anthropic: model doesn't exist — reliably model-scoped
|
|
]);
|
|
|
|
// Anthropic's permission_error is NOT exclusively model-access related: it also
|
|
// covers API-key scope, organization restrictions and feature gating. Treating it
|
|
// as model-access-denied unconditionally would make a genuinely auth-restricted key
|
|
// silently exhaust every combo target and hide the real error from the caller.
|
|
// So it only counts when the message text confirms it refers to the model.
|
|
const MODEL_ACCESS_AMBIGUOUS_TYPES = new Set([
|
|
"permission_error", // Anthropic: could be model access OR key/org/feature scope
|
|
]);
|
|
|
|
// Model access patterns — the account does not have access to the requested model
|
|
// but a different account (e.g. PRO vs free tier) may support it.
|
|
const MODEL_ACCESS_DENIED_PATTERNS = [
|
|
/\binvalid model\b/i,
|
|
/\bmodel.*not.*(?:available|found|supported|accessible)\b/i,
|
|
/\bmodel.*(?:does not exist|doesn't exist)\b/i,
|
|
/\baccess.*denied.*model\b/i,
|
|
/\bmodel.*access.*denied\b/i,
|
|
/\bplease select a different model\b/i,
|
|
// "...access to the requested model" / "model ... access" — bounded lookahead
|
|
// (no nested quantifiers) so it stays ReDoS-safe while requiring BOTH an
|
|
// access/permission word and "model" so a pure auth error never matches.
|
|
/\b(?:access|permission)[\s\S]{0,60}?\bmodel\b/i,
|
|
/\bmodel[\s\S]{0,60}?\b(?:access|permission)\b/i,
|
|
];
|
|
|
|
// Pure credential/authentication failures — the key or token itself is bad, which
|
|
// is NOT a model-availability problem. Some providers phrase these as a 400 that
|
|
// also mentions the model (e.g. "invalid api key for model X"), which would
|
|
// otherwise trip MODEL_ACCESS_DENIED_PATTERNS above and trigger combo fallback
|
|
// across every target, masking the real "fix your credential" error. When the
|
|
// text clearly indicates a bad credential, the regex-based model-access detection
|
|
// is suppressed (structured codes/types like model_not_found are unaffected).
|
|
const AUTH_CREDENTIAL_ERROR_PATTERNS = [
|
|
/\b(?:invalid|incorrect|expired|missing|revoked)\s+api[\s_-]?key\b/i,
|
|
/\bapi[\s_-]?key\s+(?:is\s+)?(?:invalid|incorrect|expired|missing|revoked|not\s+valid)\b/i,
|
|
/\bauthentication\s+(?:failed|error|required)\b/i,
|
|
/\b(?:invalid|expired|missing|revoked)\s+(?:token|credentials?|bearer)\b/i,
|
|
/\bunauthorized\b/i,
|
|
/\bnot\s+authenticated\b/i,
|
|
];
|
|
|
|
// Malformed request patterns — the model rejected the message format but a different
|
|
// provider/model in the combo may accept it.
|
|
const MALFORMED_REQUEST_PATTERNS = [
|
|
/\bimproperly formed request\b/i,
|
|
/\binvalid.*message.*format/i,
|
|
/\bmessages must alternate\b/i,
|
|
/\bempty (message|content)\b/i,
|
|
// Tool call function name errors
|
|
/\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
|
|
/function.*name.*(?:blank|empty|missing)/i,
|
|
/tool_call.*name.*(?:blank|empty|missing)/i,
|
|
];
|
|
|
|
// Rate-limit text on a 400 — some providers (e.g. MiMoCode) signal throttling with a
|
|
// non-standard 400 status whose body carries rate-limit semantics instead of a 429
|
|
// (#4976). When detected, the request is fallback-worthy at connection-cooldown scope
|
|
// (NOT a whole-provider breaker) so combo routing can fail over to another free target.
|
|
// Bounded, non-overlapping patterns only (ReDoS-safe — no nested quantifiers).
|
|
const RATE_LIMIT_TEXT_PATTERNS = [
|
|
/high.?frequency/i,
|
|
/non-compliant/i,
|
|
/too many requests/i,
|
|
/rate.?limit/i,
|
|
/频繁/, // "frequent" (zh) — high-frequency request throttling
|
|
/频率/, // "frequency" (zh) — request-frequency throttling
|
|
];
|
|
|
|
// Parameter validation errors — model-specific constraints (different models = different limits)
|
|
const PARAM_VALIDATION_PATTERNS = [
|
|
/max_tokens.*illegal/i,
|
|
/max_tokens.*must be/i,
|
|
/max_tokens.*range/i,
|
|
/parameter is illegal/i,
|
|
/is illegal.*range/i,
|
|
];
|
|
|
|
/**
|
|
* T06: Returns true if response body indicates the account is permanently deactivated.
|
|
*/
|
|
export function isAccountDeactivated(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return getMergedBannedSignals().some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
/**
|
|
* T10: Returns true if response body indicates credits/quota are permanently exhausted.
|
|
*/
|
|
export function isCreditsExhausted(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
/**
|
|
* T11: Returns true if response body indicates OAuth token is invalid/expired.
|
|
* This is different from permanent account deactivation - token refresh can recover.
|
|
*/
|
|
export function isOAuthInvalidToken(errorText: string): boolean {
|
|
const lower = String(errorText || "").toLowerCase();
|
|
return OAUTH_INVALID_TOKEN_SIGNALS.some((sig) => lower.includes(sig));
|
|
}
|
|
|
|
// ─── Resilience Profile Helper ──────────────────────────────────────────────
|
|
|
|
function asRecord(value: unknown): JsonRecord {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
|
}
|
|
|
|
function isCompatibleProvider(provider: string | null | undefined): boolean {
|
|
return (
|
|
typeof provider === "string" &&
|
|
(provider.startsWith("openai-compatible-") || provider.startsWith("anthropic-compatible-"))
|
|
);
|
|
}
|
|
|
|
function buildProviderProfile(
|
|
category: "oauth" | "apikey",
|
|
settings?: Record<string, unknown> | null
|
|
) {
|
|
const resilience = settings ? resolveResilienceSettings(settings) : DEFAULT_RESILIENCE_SETTINGS;
|
|
const connectionCooldown = resilience.connectionCooldown[category];
|
|
const providerBreaker = resilience.providerBreaker[category];
|
|
|
|
return {
|
|
baseCooldownMs: connectionCooldown.baseCooldownMs,
|
|
useUpstreamRetryHints: connectionCooldown.useUpstreamRetryHints,
|
|
useUpstream429BreakerHints: connectionCooldown.useUpstream429BreakerHints,
|
|
maxCooldownMs: resolveModelLockoutSettings(settings).maxCooldownMs,
|
|
maxBackoffSteps: connectionCooldown.maxBackoffSteps,
|
|
failureThreshold: providerBreaker.failureThreshold,
|
|
resetTimeoutMs: providerBreaker.resetTimeoutMs,
|
|
transientCooldown: connectionCooldown.baseCooldownMs,
|
|
rateLimitCooldown: connectionCooldown.useUpstreamRetryHints
|
|
? 0
|
|
: connectionCooldown.baseCooldownMs,
|
|
maxBackoffLevel: connectionCooldown.maxBackoffSteps,
|
|
circuitBreakerThreshold: providerBreaker.failureThreshold,
|
|
circuitBreakerReset: providerBreaker.resetTimeoutMs,
|
|
degradationThreshold: providerBreaker.degradationThreshold,
|
|
// Provider-level cooldown fields are not exposed in resilience settings yet.
|
|
providerFailureThreshold: PROVIDER_PROFILES[category].providerFailureThreshold,
|
|
providerFailureWindowMs: PROVIDER_PROFILES[category].providerFailureWindowMs,
|
|
maxBackoffMultiplier: PROVIDER_PROFILES[category].maxBackoffMultiplier,
|
|
backoffEscalationCount: PROVIDER_PROFILES[category].backoffEscalationCount,
|
|
providerCooldownMs: PROVIDER_PROFILES[category].providerCooldownMs,
|
|
} satisfies ProviderProfile;
|
|
}
|
|
|
|
/**
|
|
* Get the resilience profile for a provider (oauth or apikey).
|
|
* @param {string} provider - Provider ID or alias
|
|
*/
|
|
export function getProviderProfile(provider: string): ProviderProfile {
|
|
const category = getProviderCategory(provider);
|
|
return buildProviderProfile(category);
|
|
}
|
|
|
|
function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean {
|
|
if (!provider) return true;
|
|
return getProviderCategory(provider) === "oauth";
|
|
}
|
|
|
|
export async function getRuntimeProviderProfile(provider: string | null | undefined) {
|
|
try {
|
|
const { getCachedSettings } = await import("@/lib/db/readCache");
|
|
const settings = await getCachedSettings();
|
|
const category = getProviderCategory(provider || "");
|
|
return buildProviderProfile(category, settings);
|
|
} catch {
|
|
return getProviderProfile(provider || "");
|
|
}
|
|
}
|
|
|
|
// ─── Per-Model Lockout Tracking ─────────────────────────────────────────────
|
|
// In-memory map: "provider:connectionId:model" → { reason, until, lockedAt }
|
|
const modelLockouts = new Map<string, ModelLockoutEntry>();
|
|
const modelFailureState = new Map<string, ModelFailureState>();
|
|
|
|
// Aliases (e.g. "cx" → "codex") must share lockout state with their canonical
|
|
// provider, otherwise a model locked via one spelling stays routable via the other.
|
|
const canonicalProviderCache = new Map<string, string>();
|
|
function getCanonicalLockProvider(provider: string): string {
|
|
let canonical = canonicalProviderCache.get(provider);
|
|
if (!canonical) {
|
|
canonical = resolveProviderId(provider);
|
|
canonicalProviderCache.set(provider, canonical);
|
|
}
|
|
return canonical;
|
|
}
|
|
|
|
function getModelLockKey(provider: string, connectionId: string, model: string) {
|
|
const canonicalProvider = getCanonicalLockProvider(provider);
|
|
const lockModel =
|
|
canonicalProvider === "codex"
|
|
? getCodexModelScope(model)
|
|
: getQuotaScopedModelForProvider(canonicalProvider, model) || model;
|
|
return `${canonicalProvider}:${connectionId}:${lockModel}`;
|
|
}
|
|
|
|
function getFailureWindowMs(profile: ProviderProfile | null = null, fallbackMs = 30 * 60 * 1000) {
|
|
const configured = profile?.resetTimeoutMs;
|
|
return typeof configured === "number" && configured > 0 ? configured : fallbackMs;
|
|
}
|
|
|
|
function cleanupModelLockKey(key: string, now = Date.now()) {
|
|
const entry = modelLockouts.get(key);
|
|
if (entry && now > entry.until) {
|
|
modelLockouts.delete(key);
|
|
}
|
|
|
|
const failure = modelFailureState.get(key);
|
|
if (!failure) return;
|
|
// The escalation window extends past the applied cooldown: a model that fails
|
|
// again right after its lockout expires must keep escalating, not reset to 1.
|
|
if (now - failure.lastFailureAt <= failure.resetAfterMs + (failure.lastCooldownMs ?? 0)) return;
|
|
if (modelLockouts.has(key)) return;
|
|
modelFailureState.delete(key);
|
|
}
|
|
|
|
function getModelLockBaseCooldown(
|
|
status: number,
|
|
fallbackCooldownMs: number,
|
|
profile: ProviderProfile | null = null
|
|
) {
|
|
if (Number.isFinite(fallbackCooldownMs) && fallbackCooldownMs > 0) {
|
|
return fallbackCooldownMs;
|
|
}
|
|
if (typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0) {
|
|
return profile.baseCooldownMs;
|
|
}
|
|
return status === HTTP_STATUS.RATE_LIMITED ? getQuotaCooldown(0) : COOLDOWN_MS.transientInitial;
|
|
}
|
|
|
|
function getScaledCooldown(
|
|
baseCooldownMs: number,
|
|
failureCount: number,
|
|
maxBackoffLevel = BACKOFF_CONFIG.maxLevel
|
|
) {
|
|
const safeBase = Number.isFinite(baseCooldownMs) && baseCooldownMs > 0 ? baseCooldownMs : 1000;
|
|
const exponent = Math.min(Math.max(0, failureCount - 1), Math.max(0, maxBackoffLevel));
|
|
return safeBase * Math.pow(2, exponent);
|
|
}
|
|
|
|
// Auto-cleanup expired lockouts every 15 seconds (lazy init for Cloudflare Workers compatibility)
|
|
let _cleanupTimer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
function ensureCleanupTimer() {
|
|
if (_cleanupTimer) return;
|
|
try {
|
|
_cleanupTimer = setInterval(() => {
|
|
const now = Date.now();
|
|
for (const key of modelLockouts.keys()) cleanupModelLockKey(key, now);
|
|
for (const key of modelFailureState.keys()) cleanupModelLockKey(key, now);
|
|
}, 15_000);
|
|
if (typeof _cleanupTimer === "object" && "unref" in _cleanupTimer) {
|
|
(_cleanupTimer as { unref?: () => void }).unref?.(); // Don't prevent process exit (Node.js only)
|
|
}
|
|
} catch {
|
|
// Cloudflare Workers may not support setInterval outside handlers — skip cleanup timer
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Lock a specific model on a specific account
|
|
* @param {string} provider
|
|
* @param {string} connectionId
|
|
* @param {string} model
|
|
* @param {string} reason - from RateLimitReason
|
|
* @param {number} cooldownMs
|
|
*/
|
|
export function lockModel(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined,
|
|
reason: string,
|
|
cooldownMs: number,
|
|
metadata: Partial<ModelLockoutEntry> = {}
|
|
): void {
|
|
if (!model) return; // No model → skip model-level locking
|
|
ensureCleanupTimer();
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const newUntil = Date.now() + cooldownMs;
|
|
// Preserve the longer cooldown if an existing lock has more time remaining.
|
|
// Safe without a mutex: no await between get/set, so this runs atomically
|
|
// within Node.js's single-threaded event loop.
|
|
const existing = modelLockouts.get(key);
|
|
if (existing && existing.until > newUntil) {
|
|
if (metadata.failureCount && metadata.failureCount > existing.failureCount) {
|
|
existing.failureCount = metadata.failureCount;
|
|
existing.lastFailureAt = metadata.lastFailureAt ?? existing.lastFailureAt;
|
|
existing.resetAfterMs = metadata.resetAfterMs ?? existing.resetAfterMs;
|
|
modelLockouts.set(key, existing);
|
|
}
|
|
return;
|
|
}
|
|
const now = Date.now();
|
|
modelLockouts.set(key, {
|
|
reason,
|
|
until: newUntil,
|
|
lockedAt: now,
|
|
failureCount: metadata.failureCount ?? existing?.failureCount ?? 1,
|
|
lastFailureAt: metadata.lastFailureAt ?? now,
|
|
resetAfterMs: metadata.resetAfterMs ?? existing?.resetAfterMs ?? 0,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Pick the `exactCooldownMs` to apply to a model lockout (#1308).
|
|
*
|
|
* When the upstream response carried an explicit reset longer than the base
|
|
* cooldown — e.g. Antigravity "Resets in 160h", a `Retry-After` header, or a
|
|
* parseable reset text already extracted by `checkFallbackError`/`parseRetryFromErrorText`
|
|
* into `parsedCooldownMs` — honor it exactly so an exhausted model is not retried
|
|
* again within minutes. Otherwise preserve the previous behavior: return `0` to let
|
|
* `recordModelLockoutFailure` apply its exponential backoff, or the base cooldown when
|
|
* backoff is disabled.
|
|
*/
|
|
export function selectLockoutCooldownMs(
|
|
parsedCooldownMs: number,
|
|
settings: { baseCooldownMs: number; useExponentialBackoff: boolean }
|
|
): number {
|
|
if (typeof parsedCooldownMs === "number" && parsedCooldownMs > settings.baseCooldownMs) {
|
|
return parsedCooldownMs;
|
|
}
|
|
return settings.useExponentialBackoff ? 0 : settings.baseCooldownMs;
|
|
}
|
|
|
|
export function recordModelLockoutFailure(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string,
|
|
reason: string,
|
|
status: number,
|
|
fallbackCooldownMs: number,
|
|
profile: ProviderProfile | null = null,
|
|
options: { exactCooldownMs?: number | null; maxCooldownMs?: number } = {}
|
|
) {
|
|
ensureCleanupTimer();
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
const now = Date.now();
|
|
cleanupModelLockKey(key, now);
|
|
|
|
// For daily quota exhaustion (quota_exhausted), set cooldown until tomorrow 00:00
|
|
// Use exactCooldownMs to bypass exponential backoff, ensuring precise lock until midnight
|
|
if (reason === "quota_exhausted" && typeof options.exactCooldownMs !== "number") {
|
|
options = { ...options, exactCooldownMs: getMsUntilTomorrow() };
|
|
}
|
|
|
|
const resetAfterMs = getFailureWindowMs(profile);
|
|
const previous = modelFailureState.get(key);
|
|
// Escalation window extends past the previously applied cooldown so a model
|
|
// that fails again right after its lockout expires keeps escalating.
|
|
const withinWindow =
|
|
previous &&
|
|
now - previous.lastFailureAt <= previous.resetAfterMs + (previous.lastCooldownMs ?? 0);
|
|
const failureCount = withinWindow ? previous.failureCount + 1 : 1;
|
|
|
|
const baseCooldownMs = getModelLockBaseCooldown(status, fallbackCooldownMs, profile);
|
|
// Cap exponential backoff so repeated failures cannot produce absurdly long
|
|
// lockouts; exact cooldowns (e.g. daily-quota until-midnight) are not capped.
|
|
const maxCooldownMs =
|
|
typeof options.maxCooldownMs === "number" && options.maxCooldownMs > 0
|
|
? options.maxCooldownMs
|
|
: BACKOFF_CONFIG.max;
|
|
const cooldownMs =
|
|
typeof options.exactCooldownMs === "number" && options.exactCooldownMs > 0
|
|
? options.exactCooldownMs
|
|
: Math.min(
|
|
getScaledCooldown(
|
|
baseCooldownMs,
|
|
failureCount,
|
|
profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel
|
|
),
|
|
maxCooldownMs
|
|
);
|
|
|
|
modelFailureState.set(key, {
|
|
failureCount,
|
|
lastFailureAt: now,
|
|
resetAfterMs,
|
|
lastCooldownMs: cooldownMs,
|
|
});
|
|
|
|
lockModel(provider, connectionId, model, reason, cooldownMs, {
|
|
failureCount,
|
|
lastFailureAt: now,
|
|
resetAfterMs,
|
|
});
|
|
|
|
return {
|
|
cooldownMs,
|
|
failureCount,
|
|
resetAfterMs,
|
|
};
|
|
}
|
|
|
|
export function clearModelLock(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
): boolean {
|
|
if (!model) return false;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
const hadLock = modelLockouts.delete(key);
|
|
const hadFailureState = modelFailureState.delete(key);
|
|
return hadLock || hadFailureState;
|
|
}
|
|
|
|
/**
|
|
* Whether a provider should use per-model lockouts instead of connection-wide cooldowns.
|
|
* Compatible and passthrough providers multiplex multiple upstream models behind one
|
|
* connection, so transient 404/429 responses should stay model-scoped instead of
|
|
* poisoning the whole connection.
|
|
*
|
|
* @param provider - Provider ID
|
|
* @param _model - Model ID (reserved for future use)
|
|
* @param connectionPassthroughModels - Optional per-connection override from providerSpecificData.
|
|
* When provided, takes precedence over registry/provider-level logic.
|
|
*/
|
|
export function hasPerModelQuota(
|
|
provider: string | null | undefined,
|
|
_model: string | null | undefined = null,
|
|
connectionPassthroughModels?: boolean
|
|
): boolean {
|
|
// Connection-level override takes precedence (e.g., user-configured ModelScope)
|
|
if (typeof connectionPassthroughModels === "boolean") {
|
|
return connectionPassthroughModels;
|
|
}
|
|
if (!provider) return false;
|
|
if (getCanonicalLockProvider(provider) === "codex") return true;
|
|
if (provider === "gemini" || provider === "github") return true;
|
|
if (getPassthroughProviders().has(provider)) return true;
|
|
if (isCompatibleProvider(provider)) return true;
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Lock a model (not connection) for a provider with per-model quotas.
|
|
* No-ops for providers that don't use per-model lockouts.
|
|
*/
|
|
export function lockModelIfPerModelQuota(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null,
|
|
reason: string,
|
|
cooldownMs: number,
|
|
connectionPassthroughModels?: boolean
|
|
): boolean {
|
|
if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false;
|
|
// Skip model-level lock if the entire provider is in circuit-breaker cooldown.
|
|
// The provider cooldown already prevents all requests, so a model lock is redundant.
|
|
if (isProviderInCooldown(provider)) return false;
|
|
lockModel(provider, connectionId, model, reason, cooldownMs);
|
|
return true;
|
|
}
|
|
|
|
export function shouldMarkAccountExhaustedFrom429(
|
|
provider: string | null | undefined,
|
|
model: string | null | undefined = null,
|
|
connectionPassthroughModels?: boolean,
|
|
failureKind?: FailureKind
|
|
): boolean {
|
|
// A plain 429 means transient rate limiting / high traffic for many OAuth providers.
|
|
// Only connection-poison the quota cache when the upstream body explicitly says
|
|
// the long-window quota is exhausted; otherwise fallback should try another account
|
|
// without making this one look quota-depleted for 5 minutes.
|
|
if (failureKind === "rate_limit" || failureKind === "transient") return false;
|
|
return (
|
|
shouldPreserveQuotaSignalsFor429(provider) &&
|
|
!hasPerModelQuota(provider, model, connectionPassthroughModels)
|
|
);
|
|
}
|
|
|
|
export function classifyLockoutReason(status: number): string {
|
|
if (status === 429) return "rate_limit";
|
|
if (status === 403) return "quota_exhausted";
|
|
return "unknown";
|
|
}
|
|
|
|
export type DecayResult = { cleared: boolean; newFailureCount: number };
|
|
|
|
export function decayModelFailureCount(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string
|
|
): DecayResult {
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
const failure = modelFailureState.get(key);
|
|
if (!failure) return { cleared: false, newFailureCount: 0 };
|
|
|
|
const newFailureCount = Math.floor(failure.failureCount / 2);
|
|
if (newFailureCount === 0) {
|
|
modelFailureState.delete(key);
|
|
return { cleared: true, newFailureCount: 0 };
|
|
} else {
|
|
modelFailureState.set(key, {
|
|
...failure,
|
|
failureCount: newFailureCount,
|
|
});
|
|
return { cleared: false, newFailureCount };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clear all in-memory model lockouts and failure state (for tests / full reset).
|
|
*/
|
|
export function clearAllModelLockouts(): void {
|
|
modelLockouts.clear();
|
|
modelFailureState.clear();
|
|
}
|
|
|
|
/**
|
|
* Check if a specific model on a specific account is locked
|
|
* @returns {boolean}
|
|
*/
|
|
export function isModelLocked(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
): boolean {
|
|
if (!model) return false;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const entry = modelLockouts.get(key);
|
|
return Boolean(entry);
|
|
}
|
|
|
|
/**
|
|
* Get model lockout info (for debugging/dashboard)
|
|
*/
|
|
export function getModelLockoutInfo(
|
|
provider: string,
|
|
connectionId: string,
|
|
model: string | null | undefined
|
|
) {
|
|
if (!model) return null;
|
|
const key = getModelLockKey(provider, connectionId, model);
|
|
cleanupModelLockKey(key);
|
|
const entry = modelLockouts.get(key);
|
|
if (!entry) return null;
|
|
return {
|
|
reason: entry.reason,
|
|
remainingMs: entry.until - Date.now(),
|
|
lockedAt: new Date(entry.lockedAt).toISOString(),
|
|
failureCount: entry.failureCount,
|
|
};
|
|
}
|
|
|
|
export type ModelLockoutInfo = {
|
|
provider: string;
|
|
connectionId: string;
|
|
model: string;
|
|
reason: string;
|
|
remainingMs: number;
|
|
failureCount: number;
|
|
lockedAt: string;
|
|
until: number;
|
|
};
|
|
|
|
/**
|
|
* Get all active model lockouts (for dashboard)
|
|
*/
|
|
export function getAllModelLockouts(): ModelLockoutInfo[] {
|
|
const now = Date.now();
|
|
const active: ModelLockoutInfo[] = [];
|
|
for (const key of modelLockouts.keys()) {
|
|
cleanupModelLockKey(key, now);
|
|
}
|
|
for (const [key, entry] of modelLockouts) {
|
|
const [provider, connectionId, ...modelParts] = key.split(":");
|
|
const model = modelParts.join(":");
|
|
active.push({
|
|
provider,
|
|
connectionId,
|
|
model,
|
|
reason: entry.reason,
|
|
remainingMs: entry.until - now,
|
|
failureCount: entry.failureCount,
|
|
lockedAt: new Date(entry.lockedAt).toISOString(),
|
|
until: entry.until,
|
|
});
|
|
}
|
|
return active;
|
|
}
|
|
|
|
// ─── Provider Breaker Compatibility Wrappers ────────────────────────────────
|
|
// Legacy helpers now delegate to the shared provider circuit breaker.
|
|
|
|
type ProviderBreakerProfile = {
|
|
failureThreshold?: number;
|
|
degradationThreshold?: number;
|
|
resetTimeoutMs?: number;
|
|
circuitBreakerThreshold?: number;
|
|
circuitBreakerReset?: number;
|
|
};
|
|
|
|
function getProviderBreaker(provider: string | null | undefined) {
|
|
return provider ? getCircuitBreaker(provider) : null;
|
|
}
|
|
|
|
function configureProviderBreaker(
|
|
provider: string | null | undefined,
|
|
profile?: ProviderBreakerProfile | null
|
|
) {
|
|
if (!provider) return null;
|
|
|
|
const resolvedProfile = { ...getProviderProfile(provider), ...profile };
|
|
// Issue #2100 follow-up: resolve useUpstream429BreakerHints from the
|
|
// provider profile (stored override) or fall back to per-provider default.
|
|
// Stored value type is `boolean | undefined` — never `null` after PATCH.
|
|
const userValue = resolvedProfile.useUpstream429BreakerHints;
|
|
const useHints = resolveUseUpstream429BreakerHints(provider, userValue);
|
|
return getCircuitBreaker(provider, {
|
|
failureThreshold: resolvedProfile.failureThreshold ?? resolvedProfile.circuitBreakerThreshold,
|
|
resetTimeout: resolvedProfile.resetTimeoutMs ?? resolvedProfile.circuitBreakerReset,
|
|
...(useHints
|
|
? {
|
|
cooldownByKind: {
|
|
rate_limit: 60_000,
|
|
quota_exhausted: 3_600_000,
|
|
} satisfies Partial<Record<FailureKind, number>>,
|
|
classifyError: classify429FromError,
|
|
}
|
|
: {}),
|
|
degradationThreshold: resolvedProfile.degradationThreshold,
|
|
maxBackoffMultiplier: resolvedProfile.maxBackoffMultiplier,
|
|
backoffEscalationCount: resolvedProfile.backoffEscalationCount,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Check if a provider is currently blocked by the shared circuit breaker.
|
|
*/
|
|
export function isProviderInCooldown(provider: string | null | undefined): boolean {
|
|
const breaker = getProviderBreaker(provider);
|
|
return breaker ? !breaker.canExecute() : false;
|
|
}
|
|
|
|
/**
|
|
* Get remaining retry-after time for a provider breaker.
|
|
*/
|
|
export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null {
|
|
const breaker = getProviderBreaker(provider);
|
|
if (!breaker || breaker.canExecute()) return null;
|
|
const remaining = breaker.getRetryAfterMs();
|
|
return remaining > 0 ? remaining : null;
|
|
}
|
|
|
|
export function getProviderBreakerState(provider: string | null | undefined) {
|
|
const breaker = getProviderBreaker(provider);
|
|
return breaker?.getStatus?.() ?? null;
|
|
}
|
|
|
|
/**
|
|
* Record a provider failure against the shared circuit breaker.
|
|
* Delegates to the existing CircuitBreaker utility which handles
|
|
* failure counting, threshold detection, and state transitions.
|
|
*
|
|
* IMPORTANT: If the breaker is already OPEN (in cooldown), we skip
|
|
* recording the failure to prevent resetting the cooldown timer.
|
|
* This matches the original behavior where failures during cooldown
|
|
* were ignored to avoid indefinite lockout.
|
|
*/
|
|
export function recordProviderFailure(
|
|
provider: string | null | undefined,
|
|
log?: { warn?: (...args: unknown[]) => void },
|
|
connectionId?: string | null,
|
|
profile?: ProviderBreakerProfile | null
|
|
): void {
|
|
if (!provider) return;
|
|
|
|
// Deduplicate rapid-fire failures from the same connection
|
|
if (connectionId) {
|
|
const dedupKey = `${provider}:${connectionId}`;
|
|
const now = Date.now();
|
|
const lastFailure = lastConnectionFailure.get(dedupKey);
|
|
if (lastFailure && now - lastFailure < CONNECTION_FAILURE_DEDUP_MS) {
|
|
return;
|
|
}
|
|
lastConnectionFailure.delete(dedupKey);
|
|
lastConnectionFailure.set(dedupKey, now);
|
|
pruneConnectionFailureDedupeEntries();
|
|
}
|
|
|
|
const breaker = configureProviderBreaker(provider, profile);
|
|
if (!breaker) return;
|
|
|
|
if (!breaker.canExecute()) return;
|
|
|
|
breaker._onFailure();
|
|
|
|
if (!breaker.canExecute()) {
|
|
log?.warn?.(`[ProviderFailure] ${provider}: circuit breaker opened after repeated failures`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reset the shared provider breaker.
|
|
*/
|
|
export function clearProviderFailure(provider: string | null | undefined): void {
|
|
const breaker = getProviderBreaker(provider);
|
|
breaker?.reset();
|
|
}
|
|
|
|
/**
|
|
* Get all providers currently blocked by the shared breaker.
|
|
*/
|
|
export function getProvidersInCooldown(): Array<{
|
|
provider: string;
|
|
failureCount: number;
|
|
cooldownRemainingMs: number | null;
|
|
lastFailureAt: number | null;
|
|
}> {
|
|
return getAllCircuitBreakerStatuses()
|
|
.filter((status) => {
|
|
const breaker = getProviderBreaker(status.name);
|
|
return Boolean(breaker && !breaker.canExecute());
|
|
})
|
|
.map((status) => ({
|
|
provider: status.name,
|
|
failureCount: status.failureCount,
|
|
cooldownRemainingMs: status.retryAfterMs || null,
|
|
lastFailureAt: status.lastFailureTime,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Check if a status code should be counted toward provider failure threshold
|
|
*/
|
|
export function isProviderFailureCode(status: number): boolean {
|
|
return PROVIDER_FAILURE_ERROR_CODES.has(status);
|
|
}
|
|
|
|
/**
|
|
* Returns true when a checkFallbackError result signals that the entire provider
|
|
* quota is exhausted for this request, so the combo router can skip remaining
|
|
* targets from the same provider (#1731).
|
|
*
|
|
* Covers:
|
|
* - reason === "quota_exhausted" (subscription, daily, credits)
|
|
* - creditsExhausted flag
|
|
* - dailyQuotaExhausted flag
|
|
*/
|
|
export function isProviderExhaustedReason(result: {
|
|
reason?: string;
|
|
creditsExhausted?: boolean;
|
|
dailyQuotaExhausted?: boolean;
|
|
}): boolean {
|
|
if (result.creditsExhausted || result.dailyQuotaExhausted) return true;
|
|
return result.reason === RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
|
|
// ─── Retry-After Parsing ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Parse retry-after information from JSON error response bodies.
|
|
* Providers embed retry info in different formats.
|
|
*
|
|
* @param {string|object} responseBody - Raw response body or parsed JSON
|
|
* @returns {{ retryAfterMs: number|null, reason: string }}
|
|
*/
|
|
export function parseRetryAfterFromBody(responseBody: unknown): {
|
|
retryAfterMs: number | null;
|
|
reason: RateLimitReasonValue;
|
|
} {
|
|
let body: JsonRecord;
|
|
try {
|
|
body = toJsonRecord(typeof responseBody === "string" ? JSON.parse(responseBody) : responseBody);
|
|
} catch {
|
|
return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
if (Object.keys(body).length === 0) {
|
|
return { retryAfterMs: null, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
// Gemini: { error: { details: [{ retryDelay: "33s" }] } }
|
|
const error = toJsonRecord(body.error);
|
|
const details = error.details || body.details || [];
|
|
for (const detail of Array.isArray(details) ? details : []) {
|
|
const detailRecord = toJsonRecord(detail);
|
|
if (detailRecord.retryDelay) {
|
|
return {
|
|
retryAfterMs: parseDelayString(detailRecord.retryDelay),
|
|
reason: RateLimitReason.RATE_LIMIT_EXCEEDED,
|
|
};
|
|
}
|
|
}
|
|
|
|
// OpenAI: "Please retry after 20s" in message
|
|
const msg = String(error.message || body.message || "");
|
|
const retryMatch = /retry\s+after\s+(\d+)\s*s/i.exec(msg);
|
|
if (retryMatch) {
|
|
return {
|
|
retryAfterMs: Number.parseInt(retryMatch[1], 10) * 1000,
|
|
reason: RateLimitReason.RATE_LIMIT_EXCEEDED,
|
|
};
|
|
}
|
|
|
|
// Anthropic: error type classification
|
|
const errorType = String(error.type || body.type || "");
|
|
if (errorType === "rate_limit_error") {
|
|
return { retryAfterMs: null, reason: RateLimitReason.RATE_LIMIT_EXCEEDED };
|
|
}
|
|
|
|
// Classify by error message keywords
|
|
const reason = classifyErrorText(msg || errorType);
|
|
return { retryAfterMs: null, reason };
|
|
}
|
|
|
|
/**
|
|
* Parse delay strings like "33s", "2m", "1h", "1500ms"
|
|
*/
|
|
function parseDelayString(value: unknown): number | null {
|
|
if (!value) return null;
|
|
const str = String(value).trim();
|
|
const msMatch = /^(\d+)\s*ms$/i.exec(str);
|
|
if (msMatch) return Number.parseInt(msMatch[1], 10);
|
|
const secMatch = /^(\d+)\s*s$/i.exec(str);
|
|
if (secMatch) return Number.parseInt(secMatch[1], 10) * 1000;
|
|
const minMatch = /^(\d+)\s*m$/i.exec(str);
|
|
if (minMatch) return Number.parseInt(minMatch[1], 10) * 60 * 1000;
|
|
const hrMatch = /^(\d+)\s*h$/i.exec(str);
|
|
if (hrMatch) return Number.parseInt(hrMatch[1], 10) * 3600 * 1000;
|
|
// Bare number → seconds
|
|
const num = Number.parseInt(str, 10);
|
|
return Number.isNaN(num) ? null : num * 1000;
|
|
}
|
|
|
|
// T07: parse retry time from error text body with combined "XhYmZs" format.
|
|
export function parseRetryFromErrorText(errorText: unknown): number | null {
|
|
if (!errorText || typeof errorText !== "string") return null;
|
|
const msg: string = String(errorText);
|
|
|
|
const bodyHintMs = parseRetryHintFromJsonBody(msg, MAX_PROVIDER_COOLDOWN_MS);
|
|
if (bodyHintMs !== null) return bodyHintMs;
|
|
|
|
// Issue #2321: parse embedded absolute ISO retry timestamps.
|
|
const isoMatch =
|
|
/\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i.exec(
|
|
msg
|
|
);
|
|
if (isoMatch) {
|
|
const parsedTs = Date.parse(isoMatch[1]);
|
|
if (Number.isFinite(parsedTs)) {
|
|
const waitMs = parsedTs - Date.now();
|
|
if (waitMs > 0) return waitMs;
|
|
}
|
|
}
|
|
|
|
const match = /reset after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg);
|
|
if (match?.[1] || match?.[2] || match?.[3]) return computeDurationMs(match);
|
|
|
|
// Variant without "reset after": "will reset after XhYmZs"
|
|
const altMatch = /will reset after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg);
|
|
if (altMatch?.[1] || altMatch?.[2] || altMatch?.[3]) return computeDurationMs(altMatch);
|
|
|
|
// Antigravity / Cloud Code phrasing: "Resets in 164h27m24s".
|
|
const resetsInMatch = /resets? in (\d+h)?(\d+m)?(\d+s)?/i.exec(msg);
|
|
if (resetsInMatch?.[1] || resetsInMatch?.[2] || resetsInMatch?.[3]) {
|
|
return computeDurationMs(resetsInMatch);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Compute total milliseconds from regex match groups (Xh)(Ym)(Zs)
|
|
* Capped at 30 days to prevent adversarial/buggy upstream from locking indefinitely.
|
|
*/
|
|
const MAX_PROVIDER_COOLDOWN_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
|
|
function computeDurationMs(match: RegExpMatchArray): number | null {
|
|
let totalMs = 0;
|
|
if (match[1]) totalMs += Number.parseInt(match[1], 10) * 3600 * 1000; // hours
|
|
if (match[2]) totalMs += Number.parseInt(match[2], 10) * 60 * 1000; // minutes
|
|
if (match[3]) totalMs += Number.parseInt(match[3], 10) * 1000; // seconds
|
|
return totalMs > 0 ? Math.min(totalMs, MAX_PROVIDER_COOLDOWN_MS) : null;
|
|
}
|
|
|
|
function isSubscriptionQuotaText(lower: string): boolean {
|
|
return (
|
|
lower.includes("usage limit reached") ||
|
|
lower.includes("usage limit has been") ||
|
|
lower.includes("claude pro usage limit") ||
|
|
lower.includes("you've reached your usage limit") ||
|
|
lower.includes("you have reached your usage limit")
|
|
);
|
|
}
|
|
|
|
// ─── Error Classification ───────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Classify error text into RateLimitReason
|
|
*/
|
|
export function classifyErrorText(errorText: unknown): RateLimitReasonValue {
|
|
if (!errorText) return RateLimitReason.UNKNOWN;
|
|
const lower = String(errorText).toLowerCase();
|
|
|
|
if (
|
|
lower.includes("quota exceeded") ||
|
|
lower.includes("quota depleted") ||
|
|
lower.includes("quota will reset") ||
|
|
lower.includes("your quota will reset") ||
|
|
lower.includes("quota has been exceeded") ||
|
|
lower.includes("hour quota") ||
|
|
lower.includes("billing") ||
|
|
looksLikeQuotaExhausted(lower) ||
|
|
// Issue #2321: Anthropic OAuth (Claude Code Pro/Team) 429 bodies surface
|
|
// the subscription quota with phrases that contain neither "quota" nor
|
|
// "billing". Without these patterns the error was classified as a
|
|
// transient RATE_LIMIT_EXCEEDED (~5s base cooldown), which cascades all
|
|
// Pro accounts into a tight retry loop until the 5h window resets.
|
|
isSubscriptionQuotaText(lower)
|
|
) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
// T10: credits_exhausted signals
|
|
if (isCreditsExhausted(lower)) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
// T06: account_deactivated signals
|
|
if (isAccountDeactivated(lower)) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
const configuredRule = matchErrorRuleByText(errorText);
|
|
if (configuredRule?.reason) return configuredRule.reason;
|
|
if (lower.includes("rate_limit")) return RateLimitReason.RATE_LIMIT_EXCEEDED;
|
|
if (lower.includes("resource exhausted") || lower.includes("high demand"))
|
|
return RateLimitReason.MODEL_CAPACITY;
|
|
if (
|
|
lower.includes("unauthorized") ||
|
|
lower.includes("invalid api key") ||
|
|
lower.includes("authentication")
|
|
) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
if (lower.includes("server error") || lower.includes("internal error")) {
|
|
return RateLimitReason.SERVER_ERROR;
|
|
}
|
|
return RateLimitReason.UNKNOWN;
|
|
}
|
|
|
|
/**
|
|
* Classify HTTP status + error text into RateLimitReason
|
|
*
|
|
* If context (provider, headers, body) is supplied, provider-specific rules
|
|
* are evaluated FIRST. A provider like Opencode can signal account-wide quota
|
|
* exhaustion via `x-ratelimit-remaining-requests: 0` even when the body says
|
|
* "rate limit" — without context, classifyError falls through to the global
|
|
* text rules and misclassifies as RATE_LIMIT_EXCEEDED. With context, the
|
|
* provider rule takes precedence.
|
|
*/
|
|
export function classifyError(
|
|
status: number,
|
|
errorText: unknown,
|
|
context?: { provider?: string | null; headers?: Record<string, string> | null; body?: unknown }
|
|
): RateLimitReasonValue {
|
|
// Provider-specific rules take priority — they have the most accurate signal
|
|
// (e.g. `x-ratelimit-remaining-requests: 0` is irrefutable account exhaustion).
|
|
if (context?.provider) {
|
|
const match = getProviderErrorRuleMatch(
|
|
context.provider,
|
|
status,
|
|
context.headers ?? null,
|
|
context.body
|
|
);
|
|
if (match) return match.reason;
|
|
}
|
|
|
|
// Text classification takes priority (more specific)
|
|
const textReason = classifyErrorText(errorText);
|
|
if (textReason !== RateLimitReason.UNKNOWN) return textReason;
|
|
|
|
// Fall back to status code
|
|
if (status === HTTP_STATUS.UNAUTHORIZED || status === HTTP_STATUS.FORBIDDEN) {
|
|
return RateLimitReason.AUTH_ERROR;
|
|
}
|
|
if (status === HTTP_STATUS.PAYMENT_REQUIRED) {
|
|
return RateLimitReason.QUOTA_EXHAUSTED;
|
|
}
|
|
if (status === HTTP_STATUS.RATE_LIMITED) {
|
|
return RateLimitReason.RATE_LIMIT_EXCEEDED;
|
|
}
|
|
if (status === HTTP_STATUS.SERVICE_UNAVAILABLE || status === 529) {
|
|
return RateLimitReason.MODEL_CAPACITY;
|
|
}
|
|
if (status >= 500) {
|
|
return RateLimitReason.SERVER_ERROR;
|
|
}
|
|
return RateLimitReason.UNKNOWN;
|
|
}
|
|
|
|
// ─── Daily Quota Helpers ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Calculate milliseconds from now until tomorrow at midnight (00:00:00).
|
|
* Used to lock a model until the next day when daily quota is exhausted.
|
|
* @returns {number} Milliseconds until tomorrow
|
|
*/
|
|
export function getMsUntilTomorrow(): number {
|
|
const nowMs = Date.now();
|
|
const tomorrow = new Date(nowMs);
|
|
tomorrow.setDate(tomorrow.getDate() + 1);
|
|
tomorrow.setHours(0, 0, 0, 0);
|
|
const ms = tomorrow.getTime() - nowMs;
|
|
// Guard against DST edge cases: if ms is negative (shouldn't happen) or
|
|
// unreasonably large (>25h due to spring-forward), cap at 24 hours.
|
|
return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000;
|
|
}
|
|
|
|
/**
|
|
* Check if error text indicates daily quota exhaustion (as opposed to rate limiting).
|
|
* Daily quota errors typically mention "today's quota" or "try again tomorrow".
|
|
* @param {string} errorText - Error message text
|
|
* @returns {boolean} True if daily quota is exhausted
|
|
*/
|
|
export function isDailyQuotaExhausted(errorText: string): boolean {
|
|
if (!errorText) return false;
|
|
const lower = errorText.toLowerCase();
|
|
return (
|
|
lower.includes("today's quota") ||
|
|
lower.includes("daily quota") ||
|
|
lower.includes("try again tomorrow")
|
|
);
|
|
}
|
|
|
|
// ─── Configurable Backoff ───────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Get backoff duration from configurable steps.
|
|
* @param {number} failureCount - Number of consecutive failures
|
|
* @returns {number} Duration in ms
|
|
*/
|
|
export function getBackoffDuration(failureCount: number): number {
|
|
const idx = Math.min(failureCount, BACKOFF_STEPS_MS.length - 1);
|
|
return BACKOFF_STEPS_MS[idx];
|
|
}
|
|
|
|
// ─── Original API (Backward Compatible) ────────────────────────────────────
|
|
|
|
/**
|
|
* Calculate exponential backoff cooldown for rate limits (429)
|
|
* Level 0: 1s, Level 1: 2s, Level 2: 4s... → max 2 min
|
|
* @param {number} backoffLevel - Current backoff level
|
|
* @returns {number} Cooldown in milliseconds
|
|
*/
|
|
export function getQuotaCooldown(backoffLevel = 0) {
|
|
return calculateBackoffCooldown(backoffLevel);
|
|
}
|
|
|
|
/**
|
|
* Check if error should trigger account fallback (switch to next account)
|
|
* @param {number} status - HTTP status code
|
|
* @param {string} errorText - Error message text
|
|
* @param {number} backoffLevel - Current backoff level for exponential backoff
|
|
* @param {string} [model] - Optional model name for model-level lockout
|
|
* @param {string} [provider] - Provider ID for profile-aware cooldowns
|
|
* @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number, reason?: string }}
|
|
*/
|
|
export function checkFallbackError(
|
|
status: number,
|
|
errorText: string | null,
|
|
backoffLevel: number = 0,
|
|
_model: string | null = null,
|
|
provider: string | null = null,
|
|
headers: Headers | Record<string, string> | null = null,
|
|
profileOverride: ProviderProfile | null = null,
|
|
structuredError?: { code?: string | null; type?: string | null } | null
|
|
): {
|
|
shouldFallback: boolean;
|
|
cooldownMs: number;
|
|
baseCooldownMs?: number;
|
|
newBackoffLevel?: number;
|
|
usedUpstreamRetryHint?: boolean;
|
|
reason?: string;
|
|
permanent?: boolean;
|
|
creditsExhausted?: boolean;
|
|
dailyQuotaExhausted?: boolean;
|
|
/** G-02: true when the error originates from an embedded service supervisor (not the upstream AI
|
|
* provider itself). Callers should apply connection cooldown only — do NOT record a provider
|
|
* circuit-breaker failure when this flag is set. */
|
|
skipProviderBreaker?: boolean;
|
|
quotaResetHintMs?: number;
|
|
/** #6061: the provider-configured cooldown (ms) before backoff scaling, surfaced so the
|
|
* caller can persist an explicit reset window instead of the engine's scaled cooldown. */
|
|
configuredCooldownMs?: number;
|
|
} {
|
|
// G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown).
|
|
// These are NOT upstream AI provider failures — they are local supervisor state changes.
|
|
// Apply a short 5s connection cooldown without tripping the provider circuit breaker.
|
|
if (status === 503 && headers) {
|
|
const hintValue =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("x-omni-fallback-hint")
|
|
: (headers as Record<string, string>)["x-omni-fallback-hint"] ||
|
|
(headers as Record<string, string>)["X-Omni-Fallback-Hint"];
|
|
if (typeof hintValue === "string" && hintValue.toLowerCase() === "connection_cooldown") {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 5_000,
|
|
baseCooldownMs: 5_000,
|
|
newBackoffLevel: 0,
|
|
reason: "service_not_running",
|
|
skipProviderBreaker: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
const errorStr = (errorText || "").toString();
|
|
const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null);
|
|
const maxBackoffSteps = profile?.maxBackoffSteps ?? BACKOFF_CONFIG.maxLevel;
|
|
const retryableStatuses = new Set([
|
|
HTTP_STATUS.REQUEST_TIMEOUT,
|
|
HTTP_STATUS.RATE_LIMITED,
|
|
HTTP_STATUS.SERVER_ERROR,
|
|
HTTP_STATUS.BAD_GATEWAY,
|
|
HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
HTTP_STATUS.GATEWAY_TIMEOUT,
|
|
]);
|
|
|
|
function parseResetFromHeaders(headers: Headers | Record<string, string> | null): number | null {
|
|
if (!headers) return null;
|
|
const recordHeaders = headers as Record<string, string>;
|
|
|
|
// Retry-After header
|
|
const retryAfter =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("retry-after")
|
|
: recordHeaders["retry-after"] || recordHeaders["Retry-After"];
|
|
|
|
if (retryAfter) {
|
|
const seconds = Number.parseInt(retryAfter, 10);
|
|
if (!Number.isNaN(seconds) && String(seconds) === String(retryAfter).trim()) {
|
|
return Date.now() + seconds * 1000;
|
|
}
|
|
const date = new Date(retryAfter);
|
|
if (!Number.isNaN(date.getTime())) return date.getTime();
|
|
}
|
|
|
|
// X-RateLimit-Reset
|
|
const rlReset =
|
|
typeof (headers as Headers).get === "function"
|
|
? (headers as Headers).get("x-ratelimit-reset")
|
|
: recordHeaders["x-ratelimit-reset"] || recordHeaders["X-RateLimit-Reset"];
|
|
|
|
if (rlReset) {
|
|
const ts = Number.parseInt(rlReset, 10);
|
|
if (!Number.isNaN(ts)) {
|
|
return ts > 10000000000 ? ts : ts * 1000;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function getUpstreamRetryHintMs() {
|
|
if (!profile?.useUpstreamRetryHints) return null;
|
|
const resetTime = parseResetFromHeaders(headers);
|
|
if (resetTime) {
|
|
const waitMs = Math.max(resetTime - Date.now(), 0);
|
|
if (waitMs > 0) return waitMs;
|
|
}
|
|
|
|
const retryFromErrorText = parseRetryFromErrorText(errorStr);
|
|
if (retryFromErrorText && retryFromErrorText > 0) {
|
|
return retryFromErrorText;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) {
|
|
void reason;
|
|
const baseCooldownMs =
|
|
typeof profile?.baseCooldownMs === "number" && profile.baseCooldownMs >= 0
|
|
? profile.baseCooldownMs
|
|
: COOLDOWN_MS.transientInitial;
|
|
return {
|
|
baseCooldownMs,
|
|
cooldownMs: getScaledCooldown(baseCooldownMs, level + 1, maxBackoffSteps),
|
|
newBackoffLevel: Math.min(level + 1, maxBackoffSteps),
|
|
};
|
|
}
|
|
|
|
function buildRetryableFallback(reason: RateLimitReasonValue) {
|
|
const upstreamRetryHintMs = getUpstreamRetryHintMs();
|
|
if (typeof upstreamRetryHintMs === "number" && upstreamRetryHintMs > 0) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: upstreamRetryHintMs,
|
|
baseCooldownMs: upstreamRetryHintMs,
|
|
newBackoffLevel: 0,
|
|
usedUpstreamRetryHint: true,
|
|
reason,
|
|
};
|
|
}
|
|
|
|
const scaled = getScaledBaseCooldown(reason, backoffLevel);
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: scaled.cooldownMs,
|
|
baseCooldownMs: scaled.baseCooldownMs,
|
|
newBackoffLevel: scaled.newBackoffLevel,
|
|
usedUpstreamRetryHint: false,
|
|
reason,
|
|
};
|
|
}
|
|
|
|
const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED;
|
|
const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);
|
|
const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429;
|
|
|
|
// Check error message FIRST - specific patterns take priority over status codes
|
|
if (errorText) {
|
|
// T06 (sub2api #1037): Permanent account deactivation — do NOT retry, mark as permanent failure
|
|
if (isAccountDeactivated(errorStr)) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 365 * 24 * 60 * 60 * 1000, // 1 year = effectively permanent
|
|
reason: RateLimitReason.AUTH_ERROR,
|
|
permanent: true,
|
|
};
|
|
}
|
|
|
|
// T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit
|
|
if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
creditsExhausted: true,
|
|
};
|
|
}
|
|
|
|
// Daily quota exhausted — lock model until tomorrow
|
|
if (shouldUseQuotaSignal && isDailyQuotaExhausted(errorStr)) {
|
|
const msUntilTomorrow = getMsUntilTomorrow();
|
|
// Cap at 24 hours to handle timezone edge cases
|
|
const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs,
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
dailyQuotaExhausted: true,
|
|
};
|
|
}
|
|
|
|
// Issue #2321: Anthropic OAuth (Claude Pro/Team) returns 429 with
|
|
// "Usage Limit Reached" for the 5-hour subscription quota. The
|
|
// pattern-based classifier now flags these as QUOTA_EXHAUSTED, but
|
|
// without a dedicated branch the request would still fall through to
|
|
// the generic 429 retry path (~5s base cooldown). Honor upstream
|
|
// Retry-After / reset hints only when the profile enables them;
|
|
// otherwise apply a local 1h cooldown so all Pro accounts on the same
|
|
// subscription tier stop cycling through tight retries without letting
|
|
// upstream-provided windows bypass the operator setting. (We
|
|
// deliberately do not use COOLDOWN_MS.paymentRequired here — that
|
|
// constant is 2 minutes, which is shorter than the recovery time of a
|
|
// subscription quota.)
|
|
if (
|
|
shouldUseQuotaSignal &&
|
|
!isCreditsExhausted(errorStr) &&
|
|
!isDailyQuotaExhausted(errorStr) &&
|
|
isSubscriptionQuotaText(errorStr.toLowerCase())
|
|
) {
|
|
// getUpstreamRetryHintMs() gates both headers and body reset text on
|
|
// profile.useUpstreamRetryHints.
|
|
const hintMs = getUpstreamRetryHintMs();
|
|
const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour
|
|
const bodyHint = parseRetryFromErrorText(errorStr);
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: hintMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS,
|
|
reason: RateLimitReason.QUOTA_EXHAUSTED,
|
|
usedUpstreamRetryHint: Boolean(hintMs),
|
|
quotaResetHintMs: bodyHint ?? undefined,
|
|
};
|
|
}
|
|
|
|
const quotaResetHintMs = parseRetryFromErrorText(errorStr);
|
|
if (
|
|
shouldUseQuotaSignal &&
|
|
quotaResetHintMs &&
|
|
classifyErrorText(errorStr) === RateLimitReason.QUOTA_EXHAUSTED
|
|
) {
|
|
const fallbackResult = buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED);
|
|
return {
|
|
...fallbackResult,
|
|
quotaResetHintMs,
|
|
};
|
|
}
|
|
|
|
// #2929: A route-restriction 403 (e.g. Fireworks Fire Pass keys returning
|
|
// "Fire Pass API keys are not authorized for this route." on the /models
|
|
// endpoint) means the key is valid but lacks access to THIS route — it still
|
|
// serves chat. It must NOT cool down the connection or be classified as an
|
|
// auth error, otherwise a single model-listing 403 marks the key unavailable.
|
|
if (
|
|
status === HTTP_STATUS.FORBIDDEN &&
|
|
errorStr.toLowerCase().includes("not authorized for this route")
|
|
) {
|
|
return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
if (
|
|
status === HTTP_STATUS.FORBIDDEN &&
|
|
provider &&
|
|
getProviderCategory(provider) === "apikey" &&
|
|
!errorStr.toLowerCase().includes("has not been used in project") &&
|
|
!errorStr.toLowerCase().includes("hour quota") &&
|
|
!errorStr.toLowerCase().includes("quota has been exceeded")
|
|
) {
|
|
return buildRetryableFallback(RateLimitReason.AUTH_ERROR);
|
|
}
|
|
}
|
|
|
|
// Gemini-specific: use known published RPM/RPD limits to distinguish 429 types.
|
|
// Gemini returns the same error body for both, so we use per-model request
|
|
// counters to decide: if daily count >= RPD → quota_exhausted (midnight lockout);
|
|
// if minute count >= RPM → rate_limit_exceeded (exponential backoff).
|
|
if (provider === "gemini" && status === HTTP_STATUS.RATE_LIMITED && _model) {
|
|
if (isRpdExhausted(_model)) {
|
|
return buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED);
|
|
}
|
|
if (isRpmExhausted(_model)) {
|
|
return buildRetryableFallback(RateLimitReason.RATE_LIMIT_EXCEEDED);
|
|
}
|
|
}
|
|
|
|
const configuredRule =
|
|
isRateLimitStatus && !preserveQuota429
|
|
? matchErrorRuleByStatus(status)
|
|
: findMatchingErrorRule(status, errorStr);
|
|
if (configuredRule) {
|
|
if (configuredRule.backoff) {
|
|
// Provider-specific rules in `providerRuleRegistry` are MORE SPECIFIC
|
|
// than the configured (global) rule, so we check them first. If a
|
|
// provider rule matches, it overrides the configured rule's reason
|
|
// (e.g. Opencode's `x-ratelimit-remaining-requests: 0` overrides
|
|
// 429 → RATE_LIMIT_EXCEEDED). We do NOT call the full `classifyError`
|
|
// here because its global status fallback would otherwise override
|
|
// specific configured reasons (e.g. 503 → SERVER_ERROR would be
|
|
// shadowed by 503 → MODEL_CAPACITY).
|
|
const providerMatch = provider
|
|
? getProviderErrorRuleMatch(provider, status, headers, structuredError ?? null)
|
|
: null;
|
|
const reason = providerMatch
|
|
? providerMatch.reason
|
|
: (configuredRule.reason ?? RateLimitReason.UNKNOWN);
|
|
// Fix C: thread `providerMatch.cooldownMs` through so a configured rule
|
|
// like the "Monthly usage limit reached. Resets in N days." matcher can
|
|
// declare an explicit cooldown (e.g. 13 days) and have it win over the
|
|
// scaled backoff default returned by `buildRetryableFallback`. Without
|
|
// this, the rule's reason is used but its cooldownMs is silently
|
|
// dropped — which is exactly the user-visible bug where a 13-day
|
|
// upstream quota reset was being treated as ~60s.
|
|
const providerCooldownMs =
|
|
providerMatch?.cooldownMs !== undefined && providerMatch.cooldownMs > 0
|
|
? providerMatch.cooldownMs
|
|
: undefined;
|
|
const fallback = buildRetryableFallback(reason);
|
|
if (providerCooldownMs !== undefined) {
|
|
return {
|
|
...fallback,
|
|
cooldownMs: providerCooldownMs,
|
|
baseCooldownMs: providerCooldownMs,
|
|
configuredCooldownMs: providerCooldownMs,
|
|
};
|
|
}
|
|
return fallback;
|
|
}
|
|
const cooldownMs = configuredRule.cooldownMs ?? 0;
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs,
|
|
baseCooldownMs: cooldownMs,
|
|
configuredCooldownMs: cooldownMs,
|
|
reason: configuredRule.reason ?? RateLimitReason.UNKNOWN,
|
|
};
|
|
}
|
|
|
|
if (status === HTTP_STATUS.NOT_ACCEPTABLE || retryableStatuses.has(status)) {
|
|
return buildRetryableFallback(RateLimitReason.SERVER_ERROR);
|
|
}
|
|
|
|
// 400 — context overflow / malformed request / model access denied
|
|
if (status === HTTP_STATUS.BAD_REQUEST) {
|
|
// Check structured error codes first (more reliable, no false positives)
|
|
// OpenAI: error.code === "model_not_found"
|
|
// Anthropic: error.type === "not_found_error" / "permission_error"
|
|
const structuredCode =
|
|
typeof structuredError?.code === "string" ? structuredError.code.toLowerCase() : "";
|
|
const structuredType =
|
|
typeof structuredError?.type === "string" ? structuredError.type.toLowerCase() : "";
|
|
// A clear bad-credential error must never be reclassified as model-access
|
|
// (which would silently exhaust every combo target). Structured detection
|
|
// below still catches genuine model_not_found / not_found_error codes.
|
|
const looksLikeAuthCredentialError = AUTH_CREDENTIAL_ERROR_PATTERNS.some((p) =>
|
|
p.test(errorStr)
|
|
);
|
|
const matchesModelAccessPattern =
|
|
!looksLikeAuthCredentialError && MODEL_ACCESS_DENIED_PATTERNS.some((p) => p.test(errorStr));
|
|
|
|
const isModelAccessDeniedStructured =
|
|
!!structuredError &&
|
|
(MODEL_ACCESS_DENIED_CODES.has(structuredCode) ||
|
|
MODEL_ACCESS_DENIED_TYPES.has(structuredType) ||
|
|
// Ambiguous types (e.g. Anthropic permission_error) only count as a model
|
|
// access denial when the message text confirms it is about the model.
|
|
(MODEL_ACCESS_AMBIGUOUS_TYPES.has(structuredType) && matchesModelAccessPattern));
|
|
|
|
const isOverflow = CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(errorStr));
|
|
const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr));
|
|
const isParamValidation = PARAM_VALIDATION_PATTERNS.some((p) => p.test(errorStr));
|
|
const isModelAccessDenied = isModelAccessDeniedStructured || matchesModelAccessPattern;
|
|
|
|
if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied) {
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: 0,
|
|
reason: RateLimitReason.MODEL_CAPACITY,
|
|
};
|
|
}
|
|
|
|
// Some providers (e.g. MiMoCode) signal throttling with a non-standard 400 whose
|
|
// body carries rate-limit semantics ("Detected high-frequency non-compliant
|
|
// requests from you.") instead of a 429. Detected here (AFTER malformed/overflow
|
|
// detection above, so a genuinely malformed 400 still wins and keeps its #2101
|
|
// zero-cooldown MODEL_CAPACITY classification), it is fallback-worthy at
|
|
// connection-cooldown scope so combo can fail over to another target (#4976).
|
|
if (RATE_LIMIT_TEXT_PATTERNS.some((p) => p.test(errorStr))) {
|
|
return buildRetryableFallback(RateLimitReason.RATE_LIMIT_EXCEEDED);
|
|
}
|
|
|
|
// Generic 400 is not account-fallback-worthy. Combo routing may still try a
|
|
// different provider/model because combo fallback is target-level orchestration.
|
|
return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN };
|
|
}
|
|
|
|
// All other errors - fallback with transient cooldown
|
|
return {
|
|
shouldFallback: true,
|
|
cooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient,
|
|
baseCooldownMs: profile?.baseCooldownMs ?? COOLDOWN_MS.transient,
|
|
reason: RateLimitReason.UNKNOWN,
|
|
};
|
|
}
|
|
|
|
// ─── Account State Management ───────────────────────────────────────────────
|
|
|
|
/**
|
|
* Normalize a stored cooldown timestamp to epoch milliseconds.
|
|
*
|
|
* `rate_limited_until` is a TEXT column, but some write paths persist a raw
|
|
* epoch NUMBER (e.g. `setConnectionRateLimitUntil` on the Antigravity full-quota
|
|
* path). SQLite TEXT affinity coerces it to a numeric string like
|
|
* "1781696905131.0", which `new Date(...)` cannot parse (→ NaN). Accept numeric
|
|
* epoch strings/numbers as well as ISO strings and Date objects (#3954).
|
|
*/
|
|
export function cooldownUntilMs(value: string | number | Date | null | undefined): number {
|
|
if (value === null || value === undefined || value === "") return NaN;
|
|
if (value instanceof Date) return value.getTime();
|
|
if (typeof value === "number") return value;
|
|
const raw = value.trim();
|
|
if (/^\d+(\.\d+)?$/.test(raw)) return Number(raw);
|
|
return new Date(raw).getTime();
|
|
}
|
|
|
|
/**
|
|
* Check if account is currently unavailable (cooldown not expired)
|
|
*/
|
|
export function isAccountUnavailable(unavailableUntil: string | Date | null | undefined): boolean {
|
|
if (!unavailableUntil) return false;
|
|
const ms = cooldownUntilMs(unavailableUntil);
|
|
return Number.isFinite(ms) && ms > Date.now();
|
|
}
|
|
|
|
/**
|
|
* Calculate unavailable until timestamp
|
|
*/
|
|
export function getUnavailableUntil(cooldownMs: number): string {
|
|
return new Date(Date.now() + cooldownMs).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Get the earliest rateLimitedUntil from a list of accounts
|
|
*/
|
|
export function getEarliestRateLimitedUntil(
|
|
accounts: Array<{ rateLimitedUntil?: string | null }>
|
|
): string | null {
|
|
let earliest: number | null = null;
|
|
const now = Date.now();
|
|
for (const acc of accounts) {
|
|
if (!acc.rateLimitedUntil) continue;
|
|
const until = cooldownUntilMs(acc.rateLimitedUntil);
|
|
if (!Number.isFinite(until) || until <= now) continue;
|
|
if (!earliest || until < earliest) earliest = until;
|
|
}
|
|
if (!earliest) return null;
|
|
return new Date(earliest).toISOString();
|
|
}
|
|
|
|
/**
|
|
* Format rateLimitedUntil to human-readable "reset after Xm Ys"
|
|
*/
|
|
export function formatRetryAfter(
|
|
rateLimitedUntil: string | number | Date | null | undefined
|
|
): string {
|
|
if (!rateLimitedUntil) return "";
|
|
const diffMs = new Date(rateLimitedUntil).getTime() - Date.now();
|
|
if (diffMs <= 0) return "reset after 0s";
|
|
const totalSec = Math.ceil(diffMs / 1000);
|
|
const h = Math.floor(totalSec / 3600);
|
|
const m = Math.floor((totalSec % 3600) / 60);
|
|
const s = totalSec % 60;
|
|
const parts: string[] = [];
|
|
if (h > 0) parts.push(`${h}h`);
|
|
if (m > 0) parts.push(`${m}m`);
|
|
if (s > 0 || parts.length === 0) parts.push(`${s}s`);
|
|
return `reset after ${parts.join(" ")}`;
|
|
}
|
|
|
|
/**
|
|
* Filter available accounts (not in cooldown)
|
|
*/
|
|
export function filterAvailableAccounts<T extends AccountState>(
|
|
accounts: T[],
|
|
excludeId: string | null = null
|
|
): T[] {
|
|
const now = Date.now();
|
|
return accounts.filter((acc) => {
|
|
if (excludeId && acc.id === excludeId) return false;
|
|
if (acc.rateLimitedUntil) {
|
|
const until = cooldownUntilMs(acc.rateLimitedUntil);
|
|
if (Number.isFinite(until) && until > now) return false;
|
|
}
|
|
return true;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reset account state when request succeeds
|
|
*/
|
|
export function resetAccountState<T extends AccountState | null | undefined>(
|
|
account: T
|
|
): T | AccountState {
|
|
if (!account) return account;
|
|
// Persist the cooldown clear so a successfully-retried connection is no longer
|
|
// marked as rate-limited in `provider_connections.rate_limited_until`. Mirrors
|
|
// Fix A: the in-memory AccountState and the DB row must agree so the cascade
|
|
// survives the request boundary and `clearStaleCrashCooldowns` doesn't have
|
|
// to rediscover what we already know is healthy.
|
|
// Best-effort: a DB write failure must not crash the request path.
|
|
const connId = (account as AccountState | null | undefined)?.id;
|
|
if (typeof connId === "string" && connId.length > 0) {
|
|
try {
|
|
setConnectionRateLimitUntil(connId, null);
|
|
} catch {
|
|
// ignore — best effort
|
|
}
|
|
}
|
|
return {
|
|
...account,
|
|
rateLimitedUntil: null,
|
|
backoffLevel: 0,
|
|
lastError: null,
|
|
status: "active",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Apply error state to account
|
|
*/
|
|
export function applyErrorState<T extends AccountState | null | undefined>(
|
|
account: T,
|
|
status: number,
|
|
errorText: string | null,
|
|
provider: string | null = null
|
|
): T | AccountState {
|
|
if (!account) return account;
|
|
|
|
const backoffLevel = account.backoffLevel || 0;
|
|
const fallbackDecision = checkFallbackError(status, errorText, backoffLevel, null, provider);
|
|
const { cooldownMs, reason } = fallbackDecision;
|
|
const newBackoffLevel =
|
|
"newBackoffLevel" in fallbackDecision ? fallbackDecision.newBackoffLevel : undefined;
|
|
|
|
// Cooldown may be overridden by a configured provider rule (see
|
|
// `accountFallback.ts:1511-1540` provider-match branch + the
|
|
// `providerMatch.cooldownMs` thread-through added by Fix C). When the
|
|
// configured rule sets an explicit cooldownMs, it wins over the scaled
|
|
// backoff default that `checkFallbackError` returned.
|
|
const configuredCooldownMs =
|
|
"configuredCooldownMs" in fallbackDecision
|
|
? (fallbackDecision as { configuredCooldownMs?: number }).configuredCooldownMs
|
|
: undefined;
|
|
const effectiveCooldownMs =
|
|
typeof configuredCooldownMs === "number" && configuredCooldownMs > 0
|
|
? configuredCooldownMs
|
|
: cooldownMs;
|
|
|
|
const nextState: T | AccountState = {
|
|
...account,
|
|
rateLimitedUntil: effectiveCooldownMs > 0 ? getUnavailableUntil(effectiveCooldownMs) : null,
|
|
backoffLevel: newBackoffLevel ?? backoffLevel,
|
|
lastError: { status, message: errorText, timestamp: new Date().toISOString(), reason },
|
|
status: "error",
|
|
};
|
|
|
|
// Persist the cooldown to `provider_connections.rate_limited_until` so the
|
|
// cascade survives the request boundary. Before Fix A the cooldown only
|
|
// lived in the in-memory AccountState object returned here, which was
|
|
// discarded the moment the request ended — the same exhausted key was
|
|
// re-picked on the very next request.
|
|
// Best-effort try/catch mirrors `open-sse/executors/antigravity.ts:343`
|
|
// (`markConnectionQuotaExhausted`) so a DB failure can never crash the
|
|
// chat path. See issue #1 (per-account 429 cascade not persisting).
|
|
const connId = (account as AccountState | null | undefined)?.id;
|
|
if (
|
|
typeof connId === "string" &&
|
|
connId.length > 0 &&
|
|
effectiveCooldownMs > 0 &&
|
|
nextState.rateLimitedUntil
|
|
) {
|
|
try {
|
|
const untilMs = cooldownUntilMs(nextState.rateLimitedUntil);
|
|
if (Number.isFinite(untilMs) && untilMs > Date.now()) {
|
|
setConnectionRateLimitUntil(connId, untilMs);
|
|
}
|
|
} catch {
|
|
// ignore — best effort
|
|
}
|
|
}
|
|
|
|
return nextState;
|
|
}
|
|
|
|
/**
|
|
* Get account health score (0-100) for P2C selection (Phase 9)
|
|
* @param {object} account
|
|
* @returns {number} score 0 = unhealthy, 100 = perfectly healthy
|
|
*/
|
|
export function getAccountHealth(
|
|
account: AccountState | null | undefined,
|
|
model?: unknown
|
|
): number {
|
|
if (!account) return 0;
|
|
let score = 100;
|
|
score -= (account.backoffLevel || 0) * 10;
|
|
if (account.lastError) score -= 20;
|
|
if (account.rateLimitedUntil && isAccountUnavailable(account.rateLimitedUntil)) score -= 30;
|
|
return Math.max(0, score);
|
|
}
|