/** * Proxy Egress IP visibility. * * The proxy logs already capture the INBOUND client IP (x-forwarded-for), but * NOT the OUTBOUND/egress IP — the address the upstream actually sees. For * rotating providers (codex/openai) this is critical: when several accounts * egress through the SAME IP at high volume, the provider flags it as anomaly * and revokes the tokens ("Your authentication token has been invalidated"). * * This module resolves the real egress IP (via an echo-IP service through the * resolved proxy/dispatcher) and detects same-rotation-group accounts sharing * an egress IP, so the operator can confirm exactly which IP each account is * entering and leaving by. */ import { request as undiciRequest } from "undici"; import { createProxyDispatcher, proxyConfigToUrl } from "@omniroute/open-sse/utils/proxyDispatcher.ts"; import { rotationGroupFor } from "@omniroute/open-sse/services/refreshSerializer.ts"; const EGRESS_ECHO_URL = "https://api64.ipify.org?format=json"; const EGRESS_PROBE_TIMEOUT_MS = 6000; const EGRESS_CACHE_TTL_MS = 5 * 60 * 1000; export interface EgressProbeResult { ip: string | null; latencyMs: number; error?: string; } export type EgressProbe = (proxyUrl: string | null) => Promise; const egressCache = new Map(); async function defaultEgressProbe(proxyUrl: string | null): Promise { const start = Date.now(); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), EGRESS_PROBE_TIMEOUT_MS); try { const dispatcher = proxyUrl ? createProxyDispatcher(proxyUrl) : undefined; const res = await undiciRequest(EGRESS_ECHO_URL, { method: "GET", dispatcher, signal: controller.signal, headersTimeout: EGRESS_PROBE_TIMEOUT_MS, bodyTimeout: EGRESS_PROBE_TIMEOUT_MS, }); const text = await res.body.text(); let ip: string | null = null; try { ip = (JSON.parse(text) as { ip?: string }).ip ?? null; } catch { // non-JSON body — leave ip null } return { ip, latencyMs: Date.now() - start }; } catch (error) { return { ip: null, latencyMs: Date.now() - start, error: error instanceof Error ? error.message : String(error), }; } finally { clearTimeout(timeout); } } let probe: EgressProbe = defaultEgressProbe; /** Test seam: override the network probe. */ export function _setEgressProbeForTests(fn: EgressProbe | null): void { probe = fn ?? defaultEgressProbe; } export function clearEgressCache(): void { egressCache.clear(); } /** * Synchronous read of the cached egress IP for a proxy URL (null = direct). * Non-blocking — used by the request hot path to log the egress IP without an * echo-IP round-trip. Returns null if not yet probed. */ export function getCachedEgressIp(proxyUrl: string | null): string | null { const cached = egressCache.get(proxyUrl ?? "__direct__"); if (!cached) return null; if (Date.now() - cached.at >= EGRESS_CACHE_TTL_MS) return null; return cached.ip; } const warmingInFlight = new Set(); /** * Fire-and-forget: populate the egress cache for a proxy URL in the background * so subsequent proxy log lines carry the real egress IP. Deduped per URL. */ export function warmEgressIp(proxyUrl: string | null): void { const key = proxyUrl ?? "__direct__"; if (warmingInFlight.has(key) || getCachedEgressIp(proxyUrl) !== null) return; warmingInFlight.add(key); void resolveEgressIp(proxyUrl) .catch(() => undefined) .finally(() => warmingInFlight.delete(key)); } /** * Resolve the egress IP for a given proxy URL (null = direct/host IP). * Cached per proxyUrl to avoid an echo-IP round-trip on every call. */ export async function resolveEgressIp( proxyUrl: string | null, opts: { cacheTtlMs?: number; force?: boolean } = {} ): Promise { const key = proxyUrl ?? "__direct__"; const ttl = opts.cacheTtlMs ?? EGRESS_CACHE_TTL_MS; const cached = egressCache.get(key); if (!opts.force && cached && Date.now() - cached.at < ttl) { return { ip: cached.ip, latencyMs: 0, cached: true }; } const result = await probe(proxyUrl); egressCache.set(key, { ip: result.ip, at: Date.now() }); return { ...result, cached: false }; } export interface ConnectionEgress { connectionId: string; provider: string; account: string | null; proxyLevel: string; proxyHost: string | null; egressIp: string | null; error?: string; } export interface EgressSharingWarning { egressIp: string; rotationGroup: string; connections: string[]; // connectionId/account labels sharing this IP within one rotation group } export interface EgressDiagnostic { connections: ConnectionEgress[]; byEgressIp: Record; sharedWithinRotationGroup: EgressSharingWarning[]; } /** * PURE: group egress results by IP and flag IPs shared by ≥2 accounts of the * SAME rotation group (codex+openai share one Auth0 family — the exact * condition that triggers anomaly revocation). Direct/unknown IPs are reported * but only same-group sharing is a warning. */ export function analyzeEgressSharing(connections: ConnectionEgress[]): { byEgressIp: Record; sharedWithinRotationGroup: EgressSharingWarning[]; } { const byEgressIp: Record = {}; // ip -> rotationGroup -> labels const byIpGroup = new Map>(); for (const c of connections) { if (!c.egressIp) continue; const label = c.account || c.connectionId; (byEgressIp[c.egressIp] ??= []).push(label); const group = rotationGroupFor(c.provider) || `provider:${c.provider}`; let groups = byIpGroup.get(c.egressIp); if (!groups) { groups = new Map(); byIpGroup.set(c.egressIp, groups); } const list = groups.get(group) ?? []; list.push(label); groups.set(group, list); } const sharedWithinRotationGroup: EgressSharingWarning[] = []; for (const [egressIp, groups] of byIpGroup) { for (const [rotationGroup, labels] of groups) { if (labels.length >= 2) { sharedWithinRotationGroup.push({ egressIp, rotationGroup, connections: labels }); } } } return { byEgressIp, sharedWithinRotationGroup }; } /** * Diagnose egress IPs for every OAuth connection: resolve each connection's * proxy, probe the real egress IP, and flag same-rotation-group IP sharing. */ export async function diagnoseAllEgressIps(deps?: { getConnections?: () => Promise< Array<{ id: string; provider: string; name?: string; email?: string; authType?: string }> >; resolveProxy?: ( connectionId: string ) => Promise<{ proxy?: unknown; level?: string } | null>; }): Promise { const getConnections = deps?.getConnections ?? (async () => { const { getProviderConnections } = await import("./localDb"); return (await getProviderConnections({ authType: "oauth" })) as Array<{ id: string; provider: string; name?: string; email?: string; }>; }); const resolveProxy = deps?.resolveProxy ?? (async (connectionId: string) => { const { resolveProxyForConnection } = await import("./db/settings"); return resolveProxyForConnection(connectionId); }); const conns = await getConnections(); const results: ConnectionEgress[] = []; for (const c of conns) { const resolved = await resolveProxy(c.id); const proxyObj = (resolved?.proxy ?? null) as { type?: string; host?: string; port?: number | string; } | null; const proxyUrl = proxyObj ? proxyConfigToUrl(proxyObj) : null; const egress = await resolveEgressIp(proxyUrl); results.push({ connectionId: c.id, provider: c.provider, account: c.email || c.name || c.id.slice(0, 8), proxyLevel: resolved?.level || "direct", proxyHost: proxyObj?.host ?? null, egressIp: egress.ip, ...(egress.error ? { error: egress.error } : {}), }); } const { byEgressIp, sharedWithinRotationGroup } = analyzeEgressSharing(results); return { connections: results, byEgressIp, sharedWithinRotationGroup }; } export interface ProxyValidationResult { proxyId: string; host: string; port: number | string; alive: boolean; egressIp: string | null; latencyMs: number; previousStatus: string | null; newStatus: "active" | "error"; } /** * Validate every proxy in the registry by probing its real egress IP, and * persist the result to `proxy_registry.status` (active/error). Combined with * PROXY_ALIVE_PREDICATE in resolution, a dead proxy is automatically taken out * of rotation — fixing the "all proxies marked active but actually dead" state * that left codex accounts falling back to the shared host /64 IP. * * Deps are injectable for tests. */ export async function validateProxyPool(deps?: { listProxies?: () => Promise< Array<{ id: string; type: string; host: string; port: number | string; username?: string | null; password?: string | null; status?: string | null }> >; markStatus?: (id: string, status: string, meta: { latencyMs: number; egressIp: string | null }) => Promise; }): Promise { const listProxies = deps?.listProxies ?? (async () => { const { listProxies: real } = await import("./db/proxies"); return (await real({ includeSecrets: true })) as Array<{ id: string; type: string; host: string; port: number | string; username?: string | null; password?: string | null; status?: string | null; }>; }); const markStatus = deps?.markStatus ?? (async (id: string, status: string) => { const { updateProxy } = await import("./db/proxies"); await updateProxy(id, { status }); }); const proxies = await listProxies(); const report: ProxyValidationResult[] = []; for (const p of proxies) { const url = proxyConfigToUrl({ type: p.type, host: p.host, port: p.port, username: p.username ?? undefined, password: p.password ?? undefined, }); const probe = await resolveEgressIp(url, { force: true }); const alive = !!probe.ip && !probe.error; const newStatus: "active" | "error" = alive ? "active" : "error"; await markStatus(p.id, newStatus, { latencyMs: probe.latencyMs, egressIp: probe.ip }); report.push({ proxyId: p.id, host: p.host, port: p.port, alive, egressIp: probe.ip, latencyMs: probe.latencyMs, previousStatus: p.status ?? null, newStatus, }); } return report; } export interface DistributionPlan { assignments: Array<{ connectionId: string; account: string; proxyId: string }>; unassigned: Array<{ connectionId: string; account: string }>; sharingRisk: boolean; note: string; } /** * PURE: plan a 1-proxy-per-connection assignment so no two accounts of the same * rotation group share an egress IP (the codex anomaly trigger). Default is * strict 1:1 — extras are left UNASSIGNED (better unrouted than sharing an IP). * allowSharing=true round-robins instead, flagging sharingRisk. */ export function planProxyDistribution( connections: Array<{ id: string; account?: string }>, liveProxyIds: string[], opts: { allowSharing?: boolean } = {} ): DistributionPlan { const assignments: DistributionPlan["assignments"] = []; const unassigned: DistributionPlan["unassigned"] = []; let sharingRisk = false; connections.forEach((c, i) => { const account = c.account || c.id.slice(0, 8); if (liveProxyIds.length === 0) { unassigned.push({ connectionId: c.id, account }); return; } if (opts.allowSharing) { assignments.push({ connectionId: c.id, account, proxyId: liveProxyIds[i % liveProxyIds.length] }); } else if (i < liveProxyIds.length) { assignments.push({ connectionId: c.id, account, proxyId: liveProxyIds[i] }); } else { unassigned.push({ connectionId: c.id, account }); } }); if (opts.allowSharing && liveProxyIds.length < connections.length) sharingRisk = true; const note = liveProxyIds.length === 0 ? "No live proxies available — add working proxies before distributing." : liveProxyIds.length < connections.length && !opts.allowSharing ? `Only ${liveProxyIds.length} live proxies for ${connections.length} accounts — ${unassigned.length} left unassigned (avoid shared-IP anomaly).` : "1 distinct proxy per account."; return { assignments, unassigned, sharingRisk, note }; } /** * Apply a distribution plan: assign each proxy to its connection (account scope). */ export async function applyProxyDistribution( plan: DistributionPlan, deps?: { assign?: (connectionId: string, proxyId: string) => Promise } ): Promise<{ applied: number }> { const assign = deps?.assign ?? (async (connectionId: string, proxyId: string) => { const { assignProxyToScope } = await import("./db/proxies"); await assignProxyToScope("account", connectionId, proxyId); }); let applied = 0; for (const a of plan.assignments) { await assign(a.connectionId, a.proxyId); applied++; } return { applied }; }