import test from "node:test"; import assert from "node:assert/strict"; const accountFallback = await import("../../open-sse/services/accountFallback.ts"); const accountSelector = await import("../../open-sse/services/accountSelector.ts"); const { RateLimitReason, COOLDOWN_MS, PROVIDER_PROFILES } = await import("../../open-sse/config/constants.ts"); const { getCircuitBreaker } = await import("../../src/shared/utils/circuitBreaker.ts"); const { isOAuthInvalidToken, parseRetryFromErrorText, checkFallbackError, filterAvailableAccounts, getEarliestRateLimitedUntil, formatRetryAfter, applyErrorState, lockModelIfPerModelQuota, isModelLocked, getModelLockoutInfo, hasPerModelQuota, getProviderProfile, recordModelLockoutFailure, clearModelLock, shouldMarkAccountExhaustedFrom429, recordProviderFailure, isProviderInCooldown, getProviderCooldownRemainingMs, clearProviderFailure, isProviderFailureCode, getProvidersInCooldown, getProviderBreakerState, isCreditsExhausted, CREDITS_EXHAUSTED_SIGNALS, } = accountFallback; const { selectAccount } = accountSelector; /** Build a full ProviderProfile from partial overrides (test helper). */ function makeProfile(overrides: Record = {}): any { return { baseCooldownMs: 125, useUpstreamRetryHints: false, maxBackoffSteps: 3, failureThreshold: 60, resetTimeoutMs: 5000, transientCooldown: 125, rateLimitCooldown: 125, maxBackoffLevel: 3, circuitBreakerThreshold: 60, circuitBreakerReset: 5000, providerFailureThreshold: 5, providerFailureWindowMs: 300000, providerCooldownMs: 60000, ...overrides, }; } function withMockedNow(now, fn) { const originalNow = Date.now; Date.now = () => now; try { return fn(); } finally { Date.now = originalNow; } } test("isOAuthInvalidToken detects refreshable oauth failures", () => { assert.equal( isOAuthInvalidToken("Invalid authentication credentials for this OAuth 2 session"), true ); assert.equal(isOAuthInvalidToken("plain rate limit"), false); }); test("parseRetryFromErrorText parses both compact reset formats", () => { assert.equal(parseRetryFromErrorText("Your quota will reset after 2h30m14s"), 9_014_000); assert.equal(parseRetryFromErrorText("The pool will reset after 45m"), 2_700_000); assert.equal(parseRetryFromErrorText("This will reset after 30s"), 30_000); assert.equal(parseRetryFromErrorText("No reset metadata"), null); }); test("parseRetryFromErrorText parses Antigravity 'Resets in XhYmZs' phrasing", () => { assert.equal( parseRetryFromErrorText( "Individual quota reached. Contact your administrator to enable overages. " + "Resets in 164h27m24s." ), (164 * 3600 + 27 * 60 + 24) * 1000 ); assert.equal(parseRetryFromErrorText("Resets in 2h7m23s"), 7_643_000); assert.equal(parseRetryFromErrorText("Reset in 45m"), 2_700_000); }); test("parseRetryFromErrorText caps extreme reset windows at 30 days", () => { const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000; // 100 days → capped to 30 assert.equal(parseRetryFromErrorText("Resets in 2400h"), thirtyDaysMs); // Absurd value → capped assert.equal(parseRetryFromErrorText("Resets in 999999h"), thirtyDaysMs); }); test("checkFallbackError locks Antigravity quota-reached 429 for the full reset window", () => { const message = "Individual quota reached. Contact your administrator to enable overages. " + "Resets in 164h27m24s."; const result = checkFallbackError( 429, message, 0, "gemini-3-flash-agent", "antigravity", null, makeProfile({ useUpstreamRetryHints: true }) ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.usedUpstreamRetryHint, true); // Full parsed window (≈164.46h), under the 30-day cap — not the generic ~5s rate-limit backoff. assert.equal(result.cooldownMs, (164 * 3600 + 27 * 60 + 24) * 1000); }); test("recordModelLockoutFailure honors a multi-day exactCooldownMs (under 30-day cap)", () => { const provider = "antigravity"; const connectionId = "conn-quota-window"; const model = "gemini-3-flash-agent"; const exactCooldownMs = (164 * 3600 + 27 * 60 + 24) * 1000; clearModelLock(provider, connectionId, model); const lockout = recordModelLockoutFailure( provider, connectionId, model, "quota_exhausted", 429, 0, makeProfile(), { exactCooldownMs } ); assert.equal(lockout.cooldownMs, exactCooldownMs); assert.equal(isModelLocked(provider, connectionId, model), true); clearModelLock(provider, connectionId, model); }); test("checkFallbackError marks deactivated accounts as permanent auth failures", () => { const result = checkFallbackError(401, "This account has been deactivated"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.AUTH_ERROR); assert.equal(result.permanent, true); assert.ok(result.cooldownMs >= 300 * 24 * 60 * 60 * 1000); }); test("checkFallbackError classifies 'free tier of the model has been exhausted' as quota exhausted", () => { const result = checkFallbackError(429, "free tier of the model has been exhausted"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); }); test("checkFallbackError treats non-429 exhausted credits as long quota cooldowns", () => { const result = checkFallbackError(402, "credit_balance_too_low"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); test("checkFallbackError keeps API-key 429 exhausted-credit text on the resilience cooldown path", () => { const result = checkFallbackError( 429, "credit_balance_too_low", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.creditsExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => { const result = checkFallbackError( 429, "credit_balance_too_low", 0, null, "codex", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => { const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, makeProfile()); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError honors Retry-After header for rate limits", () => { withMockedNow(1_700_000_000_000, () => { const headers = new Headers({ "retry-after": "120" }); const result = checkFallbackError(429, "Rate limit hit", 3, null, "openai", headers); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.newBackoffLevel, 0); assert.equal(result.cooldownMs, 120_000); }); }); test("checkFallbackError honors x-ratelimit-reset for transient 5xx errors", () => { withMockedNow(1_700_000_000_000, () => { const resetSeconds = Math.floor((Date.now() + 90_000) / 1000); const headers = new Headers({ "x-ratelimit-reset": String(resetSeconds) }); const result = checkFallbackError(503, "upstream unavailable", 1, null, "openai", headers); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.SERVER_ERROR); assert.equal(result.newBackoffLevel, 0); assert.ok(result.cooldownMs >= 89_000); assert.ok(result.cooldownMs <= 90_000); }); }); test("checkFallbackError keeps generic 400 client errors terminal", () => { const result = checkFallbackError(400, "bad request payload"); assert.deepEqual(result, { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN, }); }); test("checkFallbackError treats a genuine 400 model-access error as combo fallback", () => { const result = checkFallbackError(400, "The model `foo` does not exist or is not available"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT treat a bad-credential 400 as model-access fallback", () => { // Phrased so it would otherwise match MODEL_ACCESS_DENIED_PATTERNS ("...api key // ... model"), but the bad-credential signal must keep it terminal so the real // auth error surfaces instead of silently exhausting every combo target. const result = checkFallbackError(400, "Invalid API key provided for model gpt-4o"); assert.deepEqual(result, { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN, }); }); test("checkFallbackError still honors structured model_not_found even with credential-like text", () => { // Structured codes are authoritative and unaffected by the credential guard. const result = checkFallbackError(400, "unauthorized-ish blob", 0, null, "openai", null, null, { code: "model_not_found", }); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("filterAvailableAccounts skips exclusion and active cooldowns but keeps recovered ones", () => { withMockedNow(1_700_000_000_000, () => { const accounts = [ { id: "exclude-me" }, { id: "cooling", rateLimitedUntil: new Date(Date.now() + 60_000).toISOString() }, { id: "recovered", rateLimitedUntil: new Date(Date.now() - 1_000).toISOString() }, { id: "healthy" }, ]; const available = filterAvailableAccounts(accounts, "exclude-me"); assert.deepEqual( available.map((account) => account.id), ["recovered", "healthy"] ); }); }); test("getEarliestRateLimitedUntil returns the shortest future cooldown and formatRetryAfter humanizes it", () => { withMockedNow(1_700_000_000_000, () => { const earliest = getEarliestRateLimitedUntil([ { rateLimitedUntil: new Date(Date.now() - 5_000).toISOString() }, { rateLimitedUntil: new Date(Date.now() + 90_000).toISOString() }, { rateLimitedUntil: new Date(Date.now() + 30_000).toISOString() }, ]); assert.equal(earliest, new Date(Date.now() + 30_000).toISOString()); assert.equal(formatRetryAfter(earliest), "reset after 30s"); }); }); test("applyErrorState and selectAccount advance to the next account after an auth failure", () => { withMockedNow(1_700_000_000_000, () => { const accounts = [ { id: "conn-a", backoffLevel: 0 }, { id: "conn-b", backoffLevel: 0 }, ]; const firstSelection = selectAccount(accounts, "fill-first"); assert.equal(firstSelection.account.id, "conn-a"); const failedFirst = applyErrorState(firstSelection.account, 401, "Unauthorized", "claude"); assert.equal(failedFirst.status, "error"); assert.equal(failedFirst.lastError.reason, RateLimitReason.AUTH_ERROR); const candidates = filterAvailableAccounts([failedFirst, accounts[1]], failedFirst.id); const nextSelection = selectAccount(candidates, "fill-first"); assert.equal(nextSelection.account.id, "conn-b"); }); }); test("lockModelIfPerModelQuota only locks supported providers and real models", () => { const geminiConnectionId = `gemini-${Date.now()}`; const openAiConnectionId = `openai-${Date.now()}`; const compatibleConnectionId = `compatible-${Date.now()}`; const compatibleProvider = "openai-compatible-custom-node"; const compatibleModel = "custom-model-a"; assert.equal(hasPerModelQuota("gemini"), true); assert.equal(hasPerModelQuota("openai"), false); assert.equal(hasPerModelQuota(compatibleProvider, compatibleModel), true); assert.equal( lockModelIfPerModelQuota( "gemini", geminiConnectionId, "gemini-2.5-pro", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked("gemini", geminiConnectionId, "gemini-2.5-pro"), true); assert.equal( lockModelIfPerModelQuota( "openai", openAiConnectionId, "gpt-5-mini", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), false ); assert.equal(isModelLocked("openai", openAiConnectionId, "gpt-5-mini"), false); assert.equal( lockModelIfPerModelQuota( compatibleProvider, compatibleConnectionId, compatibleModel, RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked(compatibleProvider, compatibleConnectionId, compatibleModel), true); }); test("getProviderProfile differentiates oauth and api-key providers", () => { const oauthProfile = getProviderProfile("claude"); assert.equal(oauthProfile.transientCooldown, PROVIDER_PROFILES.oauth.transientCooldown); assert.equal( oauthProfile.rateLimitCooldown, oauthProfile.useUpstreamRetryHints ? 0 : oauthProfile.baseCooldownMs ); assert.equal(oauthProfile.maxBackoffLevel, PROVIDER_PROFILES.oauth.maxBackoffLevel); assert.equal( oauthProfile.circuitBreakerThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold ); assert.equal(oauthProfile.circuitBreakerReset, PROVIDER_PROFILES.oauth.circuitBreakerReset); assert.equal(oauthProfile.baseCooldownMs, PROVIDER_PROFILES.oauth.transientCooldown); assert.equal(oauthProfile.failureThreshold, PROVIDER_PROFILES.oauth.circuitBreakerThreshold); assert.equal(oauthProfile.resetTimeoutMs, PROVIDER_PROFILES.oauth.circuitBreakerReset); const apiKeyProfile = getProviderProfile("openai"); assert.equal(apiKeyProfile.transientCooldown, PROVIDER_PROFILES.apikey.transientCooldown); assert.equal( apiKeyProfile.rateLimitCooldown, apiKeyProfile.useUpstreamRetryHints ? 0 : apiKeyProfile.baseCooldownMs ); assert.equal(apiKeyProfile.maxBackoffLevel, PROVIDER_PROFILES.apikey.maxBackoffLevel); assert.equal( apiKeyProfile.circuitBreakerThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold ); assert.equal(apiKeyProfile.circuitBreakerReset, PROVIDER_PROFILES.apikey.circuitBreakerReset); assert.equal(apiKeyProfile.baseCooldownMs, PROVIDER_PROFILES.apikey.transientCooldown); assert.equal(apiKeyProfile.failureThreshold, PROVIDER_PROFILES.apikey.circuitBreakerThreshold); assert.equal(apiKeyProfile.resetTimeoutMs, PROVIDER_PROFILES.apikey.circuitBreakerReset); }); test("shouldMarkAccountExhaustedFrom429 skips connection poisoning for compatible providers", () => { assert.equal(shouldMarkAccountExhaustedFrom429("gemini", "gemini-2.5-pro"), false); assert.equal( shouldMarkAccountExhaustedFrom429("openai-compatible-custom-node", "any-model"), false ); assert.equal(shouldMarkAccountExhaustedFrom429("openai", "gpt-4o-mini"), false); assert.equal(shouldMarkAccountExhaustedFrom429("claude", "claude-sonnet-4-6"), true); }); test("shouldMarkAccountExhaustedFrom429 does not poison quota cache for transient 429s", () => { assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "rate_limit"), false ); assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "transient"), false ); assert.equal( shouldMarkAccountExhaustedFrom429("kiro", "claude-opus-4.7", undefined, "quota_exhausted"), true ); }); test("hasPerModelQuota returns true for GitHub Copilot provider (#1624)", () => { assert.equal(hasPerModelQuota("github"), true); assert.equal(hasPerModelQuota("github", "gpt-5.1-codex-max"), true); assert.equal(hasPerModelQuota("github", "gpt-5-mini"), true); }); test("Codex Spark 429s are scoped away from normal Codex models", () => { const connectionId = `codex-${Date.now()}`; clearModelLock("codex", connectionId, "gpt-5.3-codex-spark"); clearModelLock("codex", connectionId, "gpt-5.3-codex"); assert.equal(hasPerModelQuota("codex", "gpt-5.3-codex-spark"), true); assert.equal(shouldMarkAccountExhaustedFrom429("codex", "gpt-5.3-codex-spark"), false); assert.equal( lockModelIfPerModelQuota( "codex", connectionId, "gpt-5.3-codex-spark", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked("codex", connectionId, "gpt-5.3-codex-spark"), true); assert.equal(isModelLocked("codex", connectionId, "codex-spark-mini"), true); assert.equal(isModelLocked("codex", connectionId, "gpt-5.3-codex"), false); clearModelLock("codex", connectionId, "gpt-5.3-codex-spark"); clearModelLock("codex", connectionId, "gpt-5.3-codex"); }); test("shouldMarkAccountExhaustedFrom429 skips connection-wide lockout for GitHub (#1624)", () => { assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5.1-codex-max"), false); assert.equal(shouldMarkAccountExhaustedFrom429("github", "gpt-5-mini"), false); assert.equal(shouldMarkAccountExhaustedFrom429("github", "claude-haiku-4.5"), false); }); test("lockModelIfPerModelQuota locks individual GitHub models without poisoning the connection (#1624)", () => { const connectionId = `github-${Date.now()}`; // A 429 on a high-PRU model should lock ONLY that model assert.equal( lockModelIfPerModelQuota( "github", connectionId, "gpt-5.1-codex-max", RateLimitReason.RATE_LIMIT_EXCEEDED, 30_000 ), true ); assert.equal(isModelLocked("github", connectionId, "gpt-5.1-codex-max"), true); // Other models on the same connection should remain unlocked assert.equal(isModelLocked("github", connectionId, "gpt-5-mini"), false); assert.equal(isModelLocked("github", connectionId, "claude-haiku-4.5"), false); }); test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and reset window", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const compatibleProvider = "openai-compatible-custom-node"; const compatibleModel = "custom-model-a"; const profile = makeProfile({ maxBackoffSteps: 2, maxBackoffLevel: 2, resetTimeoutMs: 500, }); const first = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); now += 50; const second = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); now += 50; const third = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); const info = getModelLockoutInfo(compatibleProvider, "conn-compatible", compatibleModel); assert.equal(first.failureCount, 1); assert.equal(first.cooldownMs, 125); assert.equal(second.failureCount, 2); assert.equal(second.cooldownMs, 250); assert.equal(third.failureCount, 3); assert.equal(third.cooldownMs, 500); assert.equal(info.failureCount, 3); clearModelLock(compatibleProvider, "conn-compatible", compatibleModel); now += 600; const afterReset = recordModelLockoutFailure( compatibleProvider, "conn-compatible", compatibleModel, "rate_limited", 429, 0, profile ); assert.equal(afterReset.failureCount, 1); assert.equal(afterReset.cooldownMs, 125); } finally { Date.now = originalNow; clearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a"); } }); // Provider-level failure circuit breaker tests test("isProviderFailureCode correctly identifies provider-wide transient error codes", () => { assert.equal(isProviderFailureCode(429), true); assert.equal(isProviderFailureCode(408), true); assert.equal(isProviderFailureCode(500), true); assert.equal(isProviderFailureCode(502), true); assert.equal(isProviderFailureCode(503), true); assert.equal(isProviderFailureCode(504), true); assert.equal(isProviderFailureCode(401), false); assert.equal(isProviderFailureCode(403), false); assert.equal(isProviderFailureCode(400), false); assert.equal(isProviderFailureCode(404), false); assert.equal(isProviderFailureCode(200), false); }); test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider"; // Clear any existing state clearProviderFailure(provider); assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold; // Record failures up to threshold - 1 for (let i = 0; i < threshold - 1; i++) { recordProviderFailure(provider); now += 1000; // 1 second between failures } assert.equal(isProviderInCooldown(provider), false); // Final failure to trigger threshold recordProviderFailure(provider); assert.equal(isProviderInCooldown(provider), true); const remaining = getProviderCooldownRemainingMs(provider); assert.ok(remaining !== null); assert.ok(remaining > 0); assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max // Check getProvidersInCooldown returns the provider const inCooldown = getProvidersInCooldown(); assert.ok(inCooldown.some((p) => p.provider === provider)); assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, threshold); // Simulate cooldown expiration now += 11 * 60 * 1000; // 11 minutes later assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); assert.equal( getProvidersInCooldown().some((p) => p.provider === provider), false ); } finally { Date.now = originalNow; clearProviderFailure("test-provider"); } }); test("recordProviderFailure honors runtime provider breaker profile", () => { const provider = "test-provider-runtime-profile"; clearProviderFailure(provider); try { const runtimeProfile = { failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold + 7, resetTimeoutMs: PROVIDER_PROFILES.apikey.circuitBreakerReset + 45_000, }; recordProviderFailure(provider, undefined, "conn-runtime-profile", runtimeProfile); const breaker = getCircuitBreaker(provider); assert.equal(breaker.failureThreshold, runtimeProfile.failureThreshold); assert.equal(breaker.resetTimeout, runtimeProfile.resetTimeoutMs); assert.equal(isProviderInCooldown(provider), false); const breakerAfterStatusCheck = getCircuitBreaker(provider); assert.equal(breakerAfterStatusCheck.failureThreshold, runtimeProfile.failureThreshold); assert.equal(breakerAfterStatusCheck.resetTimeout, runtimeProfile.resetTimeoutMs); } finally { clearProviderFailure(provider); } }); test("recordProviderFailure preserves provider breaker cooldown while open", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-open-cooldown-stability"; const profile = { failureThreshold: 1, resetTimeoutMs: 60_000 }; clearProviderFailure(provider); recordProviderFailure(provider, undefined, "conn-open-cooldown", profile); assert.equal(isProviderInCooldown(provider), true); const openedAt = getProviderBreakerState(provider)?.lastFailureTime; const initialRemaining = getProviderCooldownRemainingMs(provider); assert.equal(openedAt, now); assert.equal(initialRemaining, 60_000); now += 10_000; recordProviderFailure(provider, undefined, "conn-open-cooldown-later", profile); assert.equal(getProviderBreakerState(provider)?.lastFailureTime, openedAt); assert.equal(getProviderCooldownRemainingMs(provider), 50_000); } finally { Date.now = originalNow; clearProviderFailure("test-provider-open-cooldown-stability"); } }); test("recordProviderFailure keeps recent connection dedupe entries when pruning", () => { const originalNow = Date.now; const now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-dedupe-prune"; const profile = { failureThreshold: 20_000, resetTimeoutMs: 60_000 }; clearProviderFailure(provider); for (let i = 0; i <= 10_000; i++) { recordProviderFailure(provider, undefined, `conn-${i}`, profile); } const beforeDuplicate = getProviderBreakerState(provider)?.failureCount; recordProviderFailure(provider, undefined, "conn-10000", profile); const afterDuplicate = getProviderBreakerState(provider)?.failureCount; assert.equal(afterDuplicate, beforeDuplicate); assert.equal(isProviderInCooldown(provider), false); } finally { Date.now = originalNow; clearProviderFailure("test-provider-dedupe-prune"); } }); test("recordProviderFailure refreshes insertion order for existing dedupe keys", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-dedupe-lru"; const profile = { failureThreshold: 20_000, resetTimeoutMs: 60_000 }; clearProviderFailure(provider); for (let i = 0; i < 9_999; i++) { recordProviderFailure(provider, undefined, `conn-${i}`, profile); } now += 10_000; recordProviderFailure(provider, undefined, "conn-0", profile); for (let i = 10_000; i < 10_050; i++) { recordProviderFailure(provider, undefined, `conn-${i}`, profile); } const breakerState = getProviderBreakerState(provider); assert.equal(breakerState?.failureCount !== undefined, true); assert.equal(isProviderInCooldown(provider), false); } finally { Date.now = originalNow; clearProviderFailure("test-provider-dedupe-lru"); } }); test("checkFallbackError no longer mutates provider breaker state on per-connection failures", () => { const provider = "test-provider-check"; clearProviderFailure(provider); for (let i = 0; i < 5; i++) { checkFallbackError(429, "rate limited", 0, null, provider); } assert.equal(isProviderInCooldown(provider), false); clearProviderFailure(provider); }); test("checkFallbackError does not record provider failure for non-transient errors", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-no-record"; clearProviderFailure(provider); // Simulate 5 auth errors (401) - should NOT trigger provider cooldown for (let i = 0; i < 5; i++) { checkFallbackError(401, "unauthorized", 0, null, provider); now += 1000; } // Provider should NOT be in cooldown assert.equal(isProviderInCooldown(provider), false); } finally { Date.now = originalNow; clearProviderFailure("test-provider-no-record"); } }); test("clearProviderFailure removes provider from cooldown", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "test-provider-clear"; clearProviderFailure(provider); // Trigger cooldown const threshold = PROVIDER_PROFILES.apikey.circuitBreakerThreshold; for (let i = 0; i < threshold; i++) { recordProviderFailure(provider); now += 1000; } assert.equal(isProviderInCooldown(provider), true); // Clear the failure state clearProviderFailure(provider); assert.equal(isProviderInCooldown(provider), false); assert.equal(getProviderCooldownRemainingMs(provider), null); } finally { Date.now = originalNow; clearProviderFailure("test-provider-clear"); } }); // Daily quota exhausted detection tests test("isDailyQuotaExhausted detects today's quota errors", () => { const { isDailyQuotaExhausted } = accountFallback; assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true); assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true); assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true); assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false); assert.equal(isDailyQuotaExhausted(""), false); assert.equal(isDailyQuotaExhausted(null), false); }); test("getMsUntilTomorrow returns positive value less than 24 hours", () => { const { getMsUntilTomorrow } = accountFallback; const ms = getMsUntilTomorrow(); assert.ok(ms > 0, "should be positive"); assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours"); }); test("checkFallbackError locks model until tomorrow for non-429 daily quota exhaustion", () => { const result = checkFallbackError( 402, "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow" ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); assert.ok(result.cooldownMs > 0, "cooldown should be positive"); assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours"); }); test("checkFallbackError routes API-key 429 'try again tomorrow' through resilience cooldown", () => { const result = checkFallbackError( 429, "Please try again tomorrow", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.dailyQuotaExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => { const result = checkFallbackError( 429, "You have exceeded your daily quota", 0, null, "openai", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.dailyQuotaExhausted, undefined); assert.equal(result.cooldownMs, 125); }); test("checkFallbackError preserves OAuth 429 daily quota semantics", () => { const result = checkFallbackError( 429, "You have exceeded your daily quota", 0, null, "codex", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); assert.ok(result.cooldownMs > 0); }); // ModelScope daily quota lockout tests (commit 0456a1f5) test("recordModelLockoutFailure sets cooldown until tomorrow 0:00 for quota_exhausted reason", () => { const originalNow = Date.now; // Use a fixed local time (noon) to ensure predictable results const testDate = new Date(); testDate.setHours(12, 0, 0, 0); // Set to noon today const now = testDate.getTime(); Date.now = () => now; try { const provider = "modelscope"; const connectionId = "test-conn-modelscope-1"; const model = "qwen/Qwen2.5-Coder-32B-Instruct"; // Clear any existing state clearModelLock(provider, connectionId, model); const profile = makeProfile(); // Calculate milliseconds until tomorrow 00:00 local time const tomorrow = new Date(now); tomorrow.setDate(tomorrow.getDate() + 1); tomorrow.setHours(0, 0, 0, 0); const expectedMsUntilTomorrow = tomorrow.getTime() - now; // Account for timezone offset: function uses local time, test env may use UTC const timezoneOffset = new Date().getTimezoneOffset() * 60 * 1000; // Record failure with quota_exhausted reason const result = recordModelLockoutFailure( provider, connectionId, model, "quota_exhausted", 429, 0, // fallbackCooldownMs should be overridden to ms until tomorrow profile ); // Verify the cooldown is set to ms until tomorrow 0:00 (with tolerance) // The cooldown should be close to expectedMsUntilTomorrow const tolerance = 60 * 1000; // 1 minute tolerance // Calculate difference between actual and expected values const diff = Math.abs(result.cooldownMs - expectedMsUntilTomorrow); // Allow ±5 minutes tolerance (300,000 ms) assert.ok( diff <= 300_000, `cooldown should be ms until tomorrow 0:00 (expected ${expectedMsUntilTomorrow}ms, got ${result.cooldownMs}ms, diff ${diff}ms)` ); // Verify model is locked assert.equal(isModelLocked(provider, connectionId, model), true); const lockInfo = getModelLockoutInfo(provider, connectionId, model); assert.ok(lockInfo !== null, "lockInfo should not be null"); assert.ok(lockInfo.remainingMs > 0, "remaining time should be positive"); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("modelscope", "test-conn-modelscope-1", "qwen/Qwen2.5-Coder-32B-Instruct"); } }); test("recordModelLockoutFailure uses regular backoff for non-quota reasons", () => { const originalNow = Date.now; const now = 1_700_000_000_000; Date.now = () => now; try { const provider = "modelscope"; const connectionId = "test-conn-modelscope-2"; const model = "qwen/Qwen2.5-Coder-32B-Instruct"; clearModelLock(provider, connectionId, model); const profile = makeProfile({ baseCooldownMs: 5000, transientCooldown: 5000, rateLimitCooldown: 5000, }); // Record failure with rate_limited reason (not quota_exhausted) const result = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 0, profile ); // Verify the cooldown uses regular profile baseCooldownMs (5000ms) assert.ok( result.cooldownMs < 24 * 60 * 60 * 1000, "cooldown should be less than 24h for non-quota reasons" ); assert.equal(result.cooldownMs, 5000, "cooldown should use profile baseCooldownMs"); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("modelscope", "test-conn-modelscope-2", "qwen/Qwen2.5-Coder-32B-Instruct"); } }); // Test for hour quota related error messages test("checkFallbackError classifies hour quota errors correctly", () => { // For OAuth providers (e.g., codex), hour quota errors should be QUOTA_EXHAUSTED const result1 = checkFallbackError( 429, "Coding Plan hour quota has been exceeded", 0, null, "codex" ); assert.equal(result1.shouldFallback, true); assert.equal(result1.reason, RateLimitReason.QUOTA_EXHAUSTED); const result2 = checkFallbackError(429, "hour quota exceeded", 0, null, "codex"); assert.equal(result2.shouldFallback, true); assert.equal(result2.reason, RateLimitReason.QUOTA_EXHAUSTED); const result3 = checkFallbackError(429, "Your hour quota is exceeded", 0, null, "codex"); assert.equal(result3.shouldFallback, true); assert.equal(result3.reason, RateLimitReason.QUOTA_EXHAUSTED); const result4 = checkFallbackError(429, "hour quota depleted", 0, null, "codex"); assert.equal(result4.shouldFallback, true); assert.equal(result4.reason, RateLimitReason.QUOTA_EXHAUSTED); // For API-key providers with 402 status, hour quota errors should be QUOTA_EXHAUSTED const result5 = checkFallbackError(402, "hour quota has been exceeded", 0, null, "openai"); assert.equal(result5.shouldFallback, true); assert.equal(result5.reason, RateLimitReason.QUOTA_EXHAUSTED); const result6 = checkFallbackError( 403, "Coding Plan hour quota has been exceeded", 0, null, "openai" ); assert.equal(result6.shouldFallback, true); assert.equal(result6.reason, RateLimitReason.QUOTA_EXHAUSTED); }); // Test for classifyErrorText function with hour quota test("classifyErrorText handles hour quota messages", () => { const { classifyErrorText } = accountFallback; assert.equal( classifyErrorText("Coding Plan hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED ); assert.equal(classifyErrorText("hour quota exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("Your hour quota is exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("hour quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED); assert.equal(classifyErrorText("quota has been exceeded"), RateLimitReason.QUOTA_EXHAUSTED); }); // ─── Model Access Denied (structured error codes + regex fallback) ───── test("checkFallbackError detects model access denied via structured error code (OpenAI)", () => { const result = checkFallbackError( 400, "The model `gpt-5` does not exist", 0, null, "openai", null, null, { code: "model_not_found", type: null } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError detects model access denied via structured error type (Anthropic not_found_error)", () => { const result = checkFallbackError( 400, "model: claude-sonnet-4-7-20260515", 0, null, "anthropic", null, null, { code: null, type: "not_found_error" } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError detects model access denied via structured error type (Anthropic permission_error) when the message confirms the model", () => { const result = checkFallbackError( 400, "you do not have access to the requested model", 0, null, "anthropic", null, null, { code: null, type: "permission_error" } ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT fallback on a permission_error that is a key/feature scope issue (not model access)", () => { // permission_error is ambiguous on Anthropic — also raised for API-key scope, // org restrictions and feature gating. Without a model-related message it must // surface the real error instead of silently exhausting every combo target. const result = checkFallbackError( 400, "Your API key does not have permission to use the Message Batches API", 0, null, "anthropic", null, null, { code: null, type: "permission_error" } ); assert.equal(result.shouldFallback, false); }); test("checkFallbackError detects model access denied via regex fallback (invalid model)", () => { const result = checkFallbackError( 400, "Invalid model: gpt-5-turbo", 0, null, "some-provider", null, null ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 0); assert.equal(result.reason, RateLimitReason.MODEL_CAPACITY); }); test("checkFallbackError does NOT fallback on generic 400 without model access denied", () => { const result = checkFallbackError(400, "bad request payload", 0, null, "openai", null, null); assert.equal(result.shouldFallback, false); }); test("checkFallbackError ignores structured error with unrelated code on 400", () => { const result = checkFallbackError(400, "something went wrong", 0, null, "openai", null, null, { code: "invalid_api_key", type: null, }); // "invalid_api_key" is not in MODEL_ACCESS_DENIED_CODES, // no MODEL_ACCESS_DENIED_PATTERNS match either → shouldFallback: false assert.equal(result.shouldFallback, false); }); // ─── Gemini RPM 429 Classification (CREDITS_EXHAUSTED_SIGNALS fix) ───── test("isCreditsExhausted returns false for Gemini RPM 429 body text", () => { const geminiRpmText = "Resource has been exhausted (e.g. check quota)."; assert.equal(isCreditsExhausted(geminiRpmText), false); }); test("isCreditsExhausted returns true for actual credits-exhausted signals", () => { assert.equal(isCreditsExhausted("insufficient_quota"), true); assert.equal(isCreditsExhausted("credits exhausted"), true); assert.equal(isCreditsExhausted("payment required"), true); assert.equal(isCreditsExhausted("free tier of the model has been exhausted"), true); assert.equal(isCreditsExhausted("exceeded your current usage quota"), true); // #5239: "Insufficient account balance" out-of-credit bodies assert.equal(isCreditsExhausted("Insufficient account balance"), true); assert.equal(isCreditsExhausted("insufficient_balance"), true); }); test("CREDITS_EXHAUSTED_SIGNALS no longer contains generic gRPC resource-exhausted patterns", () => { // These patterns were removed because they falsely matched Gemini RPM 429 errors assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("resource has been exhausted"), false); assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("resource_exhausted"), false); assert.equal(CREDITS_EXHAUSTED_SIGNALS.includes("check quota"), false); }); test("checkFallbackError classifies Gemini RPM 429 as RATE_LIMIT_EXCEEDED (not QUOTA_EXHAUSTED)", () => { // provider=null → preserveQuota429=true → text quota checks run // isCreditsExhausted must NOT match Gemini's "Resource has been exhausted" const result = checkFallbackError( 429, "Resource has been exhausted (e.g. check quota).", 0, null, null, null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.creditsExhausted, undefined); assert.equal(result.dailyQuotaExhausted, undefined); assert.ok(result.cooldownMs > 0, "cooldownMs should be positive"); }); test("checkFallbackError classifies Gemini RPM 429 as RATE_LIMIT_EXCEEDED for API-key provider", () => { // provider="gemini" → preserveQuota429=false → status-based rule applies const result = checkFallbackError( 429, "Resource has been exhausted (e.g. check quota).", 0, null, "gemini", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); assert.equal(result.cooldownMs, 125); // makeProfile().baseCooldownMs }); test("checkFallbackError still classifies genuine OAuth quota-exhausted text as QUOTA_EXHAUSTED", () => { // Regression: OAuth providers must still get QUOTA_EXHAUSTED for actual quota messages const result = checkFallbackError( 429, "Coding Plan hour quota has been exceeded", 0, null, "codex", null, makeProfile() ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); }); test("checkFallbackError preserves daily-quota exhaustion for non-429 status codes", () => { // Non-429 status codes with daily quota text must still be QUOTA_EXHAUSTED const result = checkFallbackError( 402, "You have exceeded today's quota, please try again tomorrow" ); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); }); // ─── Gemini 429 → Model Lockout: rate_limited (not quota_exhausted) ──── test("Gemini RPM 429: recordModelLockoutFailure uses exponential backoff for rate_limited reason", () => { const originalNow = Date.now; const now = 1_700_000_000_000; Date.now = () => now; const provider = "gemini"; const connectionId = "test-conn-gemini-rpm"; const model = "gemini/gemma-4-31b-it"; try { clearModelLock(provider, connectionId, model); const profile = makeProfile({ baseCooldownMs: 5000, transientCooldown: 5000, rateLimitCooldown: 5000, }); // auth.ts flow: 429 + fallbackResult.reason=RATE_LIMIT_EXCEEDED // → reason="rate_limited" → recordModelLockoutFailure const first = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 0, profile ); assert.equal(first.failureCount, 1); assert.equal(first.cooldownMs, 5000, "first failure: 5s base cooldown"); const second = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 0, profile ); assert.equal(second.failureCount, 2); assert.equal(second.cooldownMs, 10000, "second failure: 10s exponential backoff"); assert.equal(isModelLocked(provider, connectionId, model), true); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("gemini", "test-conn-gemini-rpm", "gemini/gemma-4-31b-it"); } }); test("Gemini RPD (quota_exhausted) still triggers midnight lockout in recordModelLockoutFailure", () => { // Regression: real daily quota exhaustion must still produce midnight reset const originalNow = Date.now; const testDate = new Date(); testDate.setHours(12, 0, 0, 0); const now = testDate.getTime(); Date.now = () => now; const provider = "gemini"; const connectionId = "test-conn-gemini-rpd"; const model = "gemini/gemma-4-31b-it"; try { clearModelLock(provider, connectionId, model); const profile = makeProfile(); const result = recordModelLockoutFailure( provider, connectionId, model, "quota_exhausted", 429, 0, profile ); // Must lock until midnight, NOT exponential backoff const tomorrow = new Date(now); tomorrow.setDate(tomorrow.getDate() + 1); tomorrow.setHours(0, 0, 0, 0); const expected = tomorrow.getTime() - now; assert.ok( Math.abs(result.cooldownMs - expected) <= 300_000, `cooldown should be until tomorrow (expected ~${expected}, got ${result.cooldownMs})` ); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; clearModelLock("gemini", "test-conn-gemini-rpd", "gemini/gemma-4-31b-it"); } }); // ─── G-02: X-Omni-Fallback-Hint: connection_cooldown ───────────────────────── // When 9router executor signals a supervisor-not-running 503, checkFallbackError // must return 5s cooldown with skipProviderBreaker:true — not trip the circuit breaker. test("G-02: X-Omni-Fallback-Hint connection_cooldown on 503 returns 5s cooldown + skipProviderBreaker", () => { const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal(result.shouldFallback, true); assert.equal(result.cooldownMs, 5_000); assert.equal(result.skipProviderBreaker, true); assert.equal(result.newBackoffLevel, 0); assert.equal(result.reason, "service_not_running"); }); test("G-02: X-Omni-Fallback-Hint connection_cooldown header lookup is case-insensitive (lowercase header key)", () => { // Headers object normalises keys to lowercase — test the plain-object path const headers: Record = { "x-omni-fallback-hint": "connection_cooldown" }; const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal(result.skipProviderBreaker, true); assert.equal(result.cooldownMs, 5_000); }); test("G-02: hint header is ignored for non-503 status codes", () => { const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); // 502 should NOT trigger the hint path even if the header is present const result = checkFallbackError(502, "bad gateway", 0, null, "9router", headers); assert.equal(result.skipProviderBreaker, undefined); // normal path, no skip flag }); test("G-02: 503 without hint header follows normal circuit-breaker path", () => { // A plain 503 from a real upstream must still feed the circuit breaker const result = checkFallbackError(503, "service unavailable", 0, null, "openai", null); assert.equal(result.skipProviderBreaker, undefined); assert.ok(result.cooldownMs > 0); }); test("G-02: five consecutive 503 service_not_running do NOT trip provider circuit breaker (flag)", () => { // Verify that every call returns skipProviderBreaker:true so callers can skip recordProviderFailure const headers = new Headers({ "X-Omni-Fallback-Hint": "connection_cooldown" }); for (let i = 0; i < 5; i++) { const result = checkFallbackError( 503, "9router is not running (state: stopped)", 0, null, "9router", headers ); assert.equal( result.skipProviderBreaker, true, `call ${i + 1} should have skipProviderBreaker:true` ); } // Verify the circuit breaker for 9router is NOT open after those 5 calls const { isProviderInCooldown, clearProviderFailure } = accountFallback; assert.equal( isProviderInCooldown("9router"), false, "9router circuit breaker must remain closed" ); clearProviderFailure("9router"); // cleanup }); test("recordModelLockoutFailure caps cooldown at BACKOFF_CONFIG.max to prevent absurdly long lockouts", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "openai"; const connectionId = "conn-capped"; const model = "gpt-5-trillium"; clearModelLock(provider, connectionId, model); // Fire 9 consecutive failures so the backoff exceeds the 120s cap // baseCooldownMs=1000 (getQuotaCooldown(0)), failure 9: 1000*2^8=256000 > 120000 let lastResult; for (let i = 0; i < 9; i++) { lastResult = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 0, null ); now += 50; // each failure within the reset window } assert.ok( lastResult.cooldownMs <= 120_000, `cooldown ${lastResult.cooldownMs}ms should not exceed BACKOFF_CONFIG.max (120000ms)` ); assert.equal(lastResult.cooldownMs, 120_000); assert.equal(lastResult.failureCount, 9); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; } }); test("recordModelLockoutFailure groups provider aliases under canonical provider", () => { const providerAlias = "cx"; const providerCanonical = "codex"; const connectionId = "conn-alias-test"; const model = "gpt-5.5"; clearModelLock(providerCanonical, connectionId, model); clearModelLock(providerAlias, connectionId, model); const result1 = recordModelLockoutFailure( providerAlias, connectionId, model, "rate_limited", 429, 1000, null ); assert.equal(isModelLocked(providerAlias, connectionId, model), true); assert.equal(isModelLocked(providerCanonical, connectionId, model), true); clearModelLock(providerCanonical, connectionId, model); }); test("recordModelLockoutFailure escalates backoff correctly after cooldown expiration (long interval)", () => { const originalNow = Date.now; let now = 1_700_000_000_000; Date.now = () => now; try { const provider = "openai"; const connectionId = "conn-long-interval"; const model = "gpt-5-escalate"; clearModelLock(provider, connectionId, model); const profile = makeProfile({ baseCooldownMs: 120000, resetTimeoutMs: 30000, }); const first = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 120000, profile, { maxCooldownMs: 1800000 } ); assert.equal(first.failureCount, 1); assert.equal(first.cooldownMs, 120000); now += 130000; const second = recordModelLockoutFailure( provider, connectionId, model, "rate_limited", 429, 120000, profile, { maxCooldownMs: 1800000 } ); assert.equal(second.failureCount, 2); assert.equal(second.cooldownMs, 240000); clearModelLock(provider, connectionId, model); } finally { Date.now = originalNow; } }); // ── Custom banned signals (PR #3454) ────────────────────────────────────────── // Operators can extend ACCOUNT_DEACTIVATED_SIGNALS with provider-specific // permanent-ban phrasing via Settings → Security. These persist in the // key_value settings store and are applied at boot + on hot-reload through // setCustomBannedSignals(). Regression guard for the merge/detection behavior. const { setCustomBannedSignals, getMergedBannedSignals, isAccountDeactivated, ACCOUNT_DEACTIVATED_SIGNALS, } = accountFallback; test("getMergedBannedSignals returns built-in list unchanged when no custom signals", () => { setCustomBannedSignals([]); const merged = getMergedBannedSignals(); assert.deepEqual(merged, ACCOUNT_DEACTIVATED_SIGNALS); }); test("getMergedBannedSignals appends custom signals to the built-in list", () => { setCustomBannedSignals(["api key revoked", "tenant suspended"]); const merged = getMergedBannedSignals(); // Built-ins still present for (const sig of ACCOUNT_DEACTIVATED_SIGNALS) { assert.ok(merged.includes(sig), `built-in signal "${sig}" must survive merge`); } // Custom appended assert.ok(merged.includes("api key revoked")); assert.ok(merged.includes("tenant suspended")); setCustomBannedSignals([]); // cleanup }); test("isAccountDeactivated still matches built-in signals when custom list is empty", () => { setCustomBannedSignals([]); assert.equal(isAccountDeactivated("Your account has been suspended"), true); assert.equal(isAccountDeactivated("rate limit exceeded, retry later"), false); }); test("isAccountDeactivated matches a custom signal after setCustomBannedSignals", () => { setCustomBannedSignals([]); // Before registration the custom phrase is not a ban signal assert.equal( isAccountDeactivated("Error: API key revoked by administrator"), false, "custom phrase must not match before it is registered" ); setCustomBannedSignals(["api key revoked"]); // Case-insensitive substring match against the merged list assert.equal( isAccountDeactivated("Error: API key revoked by administrator"), true, "custom phrase must match once registered (case-insensitive substring)" ); // Built-ins remain matchable alongside custom signals assert.equal(isAccountDeactivated("account_deactivated"), true); setCustomBannedSignals([]); // cleanup — restore module state for other tests });