// @ts-nocheck import "./setupPolyfill.ts"; import { AsyncLocalStorage } from "node:async_hooks"; import { fetch as undiciFetch } from "undici"; import { buildVercelRelayHeaders, createProxyDispatcher, getDefaultDispatcher, getRetryDispatcher, isRelayType, normalizeProxyUrl, proxyConfigToUrl, proxyUrlForLogs, } from "./proxyDispatcher.ts"; import tlsClient from "./tlsClient.ts"; import { isProxyReachable } from "@/lib/proxyHealth"; import { isControlPlaneProxyDirectFallbackEnabled, isFeatureFlagEnabled, } from "@/shared/utils/featureFlags"; import { findWorkingProxy } from "./proxyFallback.ts"; function isTlsFingerprintEnabled() { return process.env.ENABLE_TLS_FINGERPRINT === "true"; } /** Per-request tracking of whether TLS fingerprint was used */ type TlsFingerprintStore = { used: boolean }; const tlsFingerprintContext = new AsyncLocalStorage(); /** * #5217 (Gap-secondary): a mutable sink that records the proxy actually applied * by `runWithProxyContext` for the in-flight request. Executors that pin their * own per-account proxy *internally* (e.g. OpencodeExecutor wraps its dispatch * in `runWithProxyContext(account.proxy, …)`) never propagate that choice back * to the caller's `proxyInfo`, so the post-execution `[ProxyEgress]` line logged * `proxy=direct` even though `[ProxyFetch] Applied request proxy context: …` * fired. Wrapping the execution in `runWithAppliedProxyCapture(sink, fn)` lets * the egress logger read the innermost applied proxy (the last writer wins, which * is the executor's per-account proxy). */ export type AppliedProxySink = { proxy: unknown }; const appliedProxyContext = new AsyncLocalStorage(); /** * Run `fn` with an applied-proxy capture sink in context. Any * `runWithProxyContext` call inside `fn` that ends up applying a proxy records * that proxy config into `sink.proxy` (innermost wins). The sink is a plain * mutable object the caller retains, so it can read `sink.proxy` after `fn` * resolves. Pure plumbing — no behavioral change to the request itself. */ export function runWithAppliedProxyCapture(sink: AppliedProxySink, fn: () => T): T { return appliedProxyContext.run(sink, fn); } type FetchWithDispatcherOptions = RequestInit & { dispatcher?: unknown }; type FetchWithDispatcher = ( input: RequestInfo | URL, init?: FetchWithDispatcherOptions ) => Promise; /** * Flatten a fetch error's `cause` chain (and any Happy-Eyeballs `AggregateError` * sub-errors) into a single diagnostic line: code/syscall/errno/address:port + a * truncated message. undici/native both reject with a bare `TypeError: fetch failed` * whose real reason hides in `.cause`; surfacing it is what makes dispatcher-failure * bursts (#4252) diagnosable. Never includes a stack trace (Rule #12). Pure + testable. */ export function describeFetchCause(err: unknown): string { const parts: string[] = []; const seen = new Set(); let cur: unknown = err; for (let depth = 0; cur && depth < 5 && !seen.has(cur); depth++) { seen.add(cur); const e = cur as Record; const seg = [ typeof e.name === "string" && e.name !== "Error" ? e.name : null, typeof e.message === "string" ? e.message.slice(0, 160) : null, e.code != null ? `code=${String(e.code)}` : null, e.syscall != null ? `syscall=${String(e.syscall)}` : null, e.errno != null ? `errno=${String(e.errno)}` : null, e.address != null ? `address=${String(e.address)}${e.port != null ? `:${String(e.port)}` : ""}` : null, ] .filter(Boolean) .join(" "); if (seg) parts.push(seg); if (Array.isArray(e.errors)) { for (const sub of (e.errors as unknown[]).slice(0, 4)) { const s = (sub ?? {}) as Record; const subSeg = [ s.code != null ? `code=${String(s.code)}` : null, s.syscall != null ? `syscall=${String(s.syscall)}` : null, s.address != null ? `address=${String(s.address)}${s.port != null ? `:${String(s.port)}` : ""}` : null, ] .filter(Boolean) .join(" "); if (subSeg) parts.push(`↳ ${subSeg}`); else if (typeof s.message === "string") parts.push(`↳ ${s.message.slice(0, 80)}`); } } cur = e.cause; } return parts.join(" | ") || String(err); } function isStreamLikeBody(body: unknown): boolean { return ( body !== null && body !== undefined && typeof body === "object" && (typeof (body as Record).getReader === "function" || typeof (body as Record).stream === "function") ); } function requestHasNonReplayableBody( input: RequestInfo | URL, options: FetchWithDispatcherOptions ): boolean { if (isStreamLikeBody(options.body as unknown)) return true; if (typeof Request !== "undefined" && input instanceof Request) { if (input.bodyUsed) return true; if (input.body !== null) return true; } return false; } /** Injectable dependencies for testability (Approach B DI). */ export type ProxyFetchDeps = { undiciFetch?: FetchWithDispatcher; nativeFetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise; }; type PatchState = { originalFetch: typeof globalThis.fetch; proxyContext: AsyncLocalStorage; isPatched: boolean; }; const isCloud = typeof caches !== "undefined" && typeof caches === "object"; const PATCH_STATE_KEY = Symbol.for("omniroute.proxyFetch.state"); function getPatchState(): PatchState { const scopedGlobal = globalThis as typeof globalThis & { [PATCH_STATE_KEY]?: PatchState; }; if (!scopedGlobal[PATCH_STATE_KEY]) { scopedGlobal[PATCH_STATE_KEY] = { originalFetch: globalThis.fetch, proxyContext: new AsyncLocalStorage(), isPatched: false, }; } return scopedGlobal[PATCH_STATE_KEY]; } const patchState = getPatchState(); const originalFetch = patchState.originalFetch; const originalFetchWithDispatcher = originalFetch as FetchWithDispatcher; const proxyContext = patchState.proxyContext; function noProxyMatch(targetUrl) { const noProxy = process.env.NO_PROXY || process.env.no_proxy; if (!noProxy) return false; let target; try { target = new URL(targetUrl); } catch { return false; } const hostname = target.hostname.toLowerCase(); const port = target.port || (target.protocol === "https:" ? "443" : "80"); const patterns = noProxy .split(",") .map((p) => p.trim().toLowerCase()) .filter(Boolean); return patterns.some((pattern) => { if (pattern === "*") return true; const [patternHost, patternPort] = pattern.split(":"); if (patternPort && patternPort !== port) return false; if (!patternHost) return false; // Support wildcard matching (e.g. 192.168.* or *.local). // Uses a linear glob scan instead of dynamic RegExp to avoid ReDoS. if (patternHost.includes("*")) { const parts = patternHost.split("*"); let pos = 0; let ok = hostname.startsWith(parts[0]); if (ok) { pos = parts[0].length; for (let i = 1; i < parts.length && ok; i++) { const seg = parts[i]; if (i === parts.length - 1) { ok = seg === "" || (hostname.endsWith(seg) && hostname.length - seg.length >= pos); } else { const idx = seg ? hostname.indexOf(seg, pos) : pos; if (idx === -1) { ok = false; } else { pos = idx + seg.length; } } } } if (ok) return true; } if (patternHost.startsWith(".")) { return hostname.endsWith(patternHost) || hostname === patternHost.slice(1); } return hostname === patternHost || hostname.endsWith(`.${patternHost}`); }); } function isLocalAddress(hostname: string): boolean { const host = hostname .replace(/^\[/, "") .replace(/\]$/, "") .replace(/^::ffff:/i, ""); if (host === "localhost" || host === "0.0.0.0" || host === "127.0.0.1" || host === "::1") { return true; } if (host.endsWith(".local") || host.endsWith(".lan") || host.endsWith(".internal")) return true; // RFC1918 + loopback + link-local (169.254, incl. cloud metadata 169.254.169.254) // + CGNAT (100.64/10). 127/8 covers all loopback, not just 127.0.0.1. if (host.startsWith("192.168.")) return true; if (host.startsWith("10.")) return true; if (host.startsWith("127.")) return true; if (host.startsWith("169.254.")) return true; if (/^172\.(1[6-9]|2\d|3[0-1])\./.test(host)) return true; if (/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(host)) return true; // IPv6 ULA (fc00::/7 → fc/fd prefix) and link-local (fe80::/10) if (/^f[cd][0-9a-f]*:/i.test(host) || host.startsWith("fe80:")) return true; return false; } function resolveEnvProxyUrl(targetUrl) { if (noProxyMatch(targetUrl)) return null; let protocol; try { protocol = new URL(targetUrl).protocol; } catch { return null; } const proxyUrl = protocol === "https:" ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy; if (!proxyUrl) return null; return normalizeProxyUrl(proxyUrl, "environment proxy"); } export function resolveProxyForRequest(targetUrl) { let target; try { target = new URL(targetUrl); } catch { target = null; } // Always bypass proxy for local/LAN addresses if (target && isLocalAddress(target.hostname.toLowerCase())) { return { source: "direct", proxyUrl: null }; } const contextProxy = proxyContext.getStore(); if (contextProxy) { return { source: "context", proxyUrl: proxyConfigToUrl(contextProxy) }; } const envProxyUrl = resolveEnvProxyUrl(targetUrl); if (envProxyUrl) { return { source: "env", proxyUrl: envProxyUrl }; } return { source: "direct", proxyUrl: null }; } function getTargetUrl(input) { if (typeof input === "string") return input; if (input && typeof input.url === "string") return input.url; return String(input); } export async function runWithProxyContext( proxyConfig, fn, opts?: { directFallbackOnUnreachable?: boolean } ) { if (typeof fn !== "function") { throw new TypeError("runWithProxyContext requires a callback function"); } // Inherit existing context if no specific proxyConfig is provided const currentContext = proxyContext.getStore(); const effectiveProxyConfig = proxyConfig || currentContext || null; const resolvedProxyUrl = effectiveProxyConfig ? proxyConfigToUrl(effectiveProxyConfig) : null; // The caller must opt in, and the runtime feature flag must also be enabled. // This fallback changes egress IP, so upgrades must not silently turn it on. const directFallbackOnUnreachable = opts?.directFallbackOnUnreachable === true && isControlPlaneProxyDirectFallbackEnabled(); // Run fn with the proxy context cleared so the request egresses directly. const runDirect = () => proxyContext.run(null, fn); // T14: Proxy Fast-Fail // Perform a short TCP reachability check before issuing upstream requests. // Skip for edge-relay types (vercel / deno): proxyConfigToUrl returns // "https://" which is the relay endpoint itself, not an HTTP proxy — // the actual routing is handled via x-relay-* headers below. const isVercelRelay = isRelayType((effectiveProxyConfig as { type?: string })?.type); if (resolvedProxyUrl && !isVercelRelay) { const reachable = await isProxyReachable(resolvedProxyUrl); if (!reachable) { const proxyLabel = proxyUrlForLogs(resolvedProxyUrl); if (directFallbackOnUnreachable) { console.warn( `[ProxyFetch] Proxy unreachable (${proxyLabel}); using a direct connection for this request.` ); return runDirect(); } const err = new Error(`[Proxy Fast-Fail] Proxy unreachable: ${proxyLabel}`) as Error & { code?: string; statusCode?: number; }; err.code = "PROXY_UNREACHABLE"; err.statusCode = 503; throw err; } } // Fail-closed family check: when the proxy URL carries a ?family=ipv6|ipv4 marker // (set for HOSTNAME proxies by proxyConfigToUrl), verify the hostname actually has a // record in that family before egressing. Refuse early rather than silently fall back // to the other family. No-op for IP literals (their family is intrinsic). if (resolvedProxyUrl && !isVercelRelay) { try { const u = new URL(resolvedProxyUrl); const fam = u.searchParams.get("family"); if (fam === "ipv6" || fam === "ipv4") { const { assertHostnameSupportsFamily } = await import("./proxyFamilyResolve.ts"); await assertHostnameSupportsFamily(u.hostname, fam === "ipv6" ? 6 : 4); } } catch (familyErr) { if (directFallbackOnUnreachable) { console.warn( `[ProxyFetch] Proxy family pre-check failed (${proxyUrlForLogs(resolvedProxyUrl)}); using a direct connection for this request.` ); return runDirect(); } const e = familyErr as Error & { code?: string; statusCode?: number }; e.code = e.code || "PROXY_FAMILY_UNAVAILABLE"; e.statusCode = e.statusCode || 503; throw e; } } return proxyContext.run(effectiveProxyConfig, async () => { if (resolvedProxyUrl && effectiveProxyConfig !== currentContext) { console.log( `[ProxyFetch] Applied request proxy context: ${proxyUrlForLogs(resolvedProxyUrl)}` ); } // #5217: record the proxy actually applied so a post-execution egress logger // reflects the real egress (executors that pin a per-account proxy internally // otherwise leave proxyInfo reading "direct"). Innermost runWithProxyContext // wins, which is exactly the per-account proxy the executor selected. if (effectiveProxyConfig) { const sink = appliedProxyContext.getStore(); if (sink) sink.proxy = effectiveProxyConfig; } return fn(); }); } /** * Like {@link runWithProxyContext}, but if the assigned proxy is unreachable or fails * its pre-checks the request can degrade to a DIRECT connection instead of throwing. * * For control-plane flows — OAuth code/token exchange, connection tests, token refresh — * where a dead pinned proxy must not block reaching the upstream (it otherwise surfaces * as a generic "Internal server error"). Data-plane chat keeps strict pinning via * runWithProxyContext so per-account egress-IP isolation is preserved. * * This remains disabled unless OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK is enabled * from Feature Flags or the environment. */ export async function runWithProxyContextOrDirect(proxyConfig, fn) { return runWithProxyContext(proxyConfig, fn, { directFallbackOnUnreachable: true }); } async function patchedFetch( input: RequestInfo | URL, options: FetchWithDispatcherOptions = {}, deps: ProxyFetchDeps = {} ) { if (options?.dispatcher) { // When a dispatcher is present, we MUST use the undici library fetch // to ensure version compatibility. Node 22 built-in fetch (undici v6) // is incompatible with undici v8 dispatchers (missing onRequestStart, etc.) const _undiciDispatcher = deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise); return _undiciDispatcher(input, options); } const targetUrl = getTargetUrl(input); let resolved; try { resolved = resolveProxyForRequest(targetUrl); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(`[ProxyFetch] Proxy configuration error: ${message}`); throw error; } const { source, proxyUrl } = resolved; if (!proxyUrl) { // TLS fingerprint spoofing for direct connections (no proxy configured) if (isTlsFingerprintEnabled() && tlsClient.available) { try { const store = tlsFingerprintContext.getStore(); if (store) store.used = true; return await tlsClient.fetch(targetUrl, { ...options, headers: options.headers, signal: options.signal ?? undefined, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.warn( `[ProxyFetch] TLS fingerprint failed, falling back to native fetch: ${message}` ); const store = tlsFingerprintContext.getStore(); if (store) store.used = false; } } // Direct connection (no proxy) — use undici with custom dispatcher for timeout control. // Falls back to original native fetch if dispatcher initialization fails (#1054). // Retries once on transient dispatcher errors before falling back (fix: proxyfetch-undici-retry). // // Non-replayable body guard: if the body is stream-like (ReadableStream/Blob) // or the input is a Request that carries a body, the first dispatcher attempt // owns that body. Retrying or falling back to native fetch would replay a // consumed/locked body and can mask the original transport error with // "Response body object should not be disturbed or locked". const hasNonReplayableBody = requestHasNonReplayableBody(input, options); const maxAttempts = hasNonReplayableBody ? 1 : 2; const _undiciDirect = deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise); const _nativeFallback = (deps.nativeFetch as FetchWithDispatcher | undefined) ?? originalFetchWithDispatcher; let lastDispatcherError: unknown = null; for (let attempt = 0; attempt < maxAttempts; attempt++) { try { return await _undiciDirect(input, { ...options, // #4252: first attempt uses the pooled keep-alive dispatcher; a retry // (after a transient socket error) uses the no-keep-alive dispatcher so // it opens a FRESH socket instead of grabbing another stale pooled one // — the burst pattern was the retry re-hitting a dead pooled socket and // then falling through to native fetch (which also pools) → 502. dispatcher: attempt === 0 ? getDefaultDispatcher() : getRetryDispatcher(), }); } catch (dispatcherError) { const msg = dispatcherError instanceof Error ? dispatcherError.message : String(dispatcherError); // CAUTION: Do NOT fallback to native fetch if the error is a version mismatch (invalid onRequestStart) // because the native fetch will definitely fail with the undici v8 dispatcher. if (msg.includes("onRequestStart")) { console.error( `[ProxyFetch] Fatal version mismatch: Dispatcher (v8) vs Fetch (v6/native). Hardware upgrade or SOCKS5 config isolation required. Error: ${msg}` ); throw dispatcherError; } // Only retry/fallback for connection/dispatcher errors, not HTTP errors. // Prefer the .code property when available (more stable across undici // versions than message-string matching); fall back to substring match // for errors that lack a structured code. const errCode = (dispatcherError as { code?: unknown })?.code; if ( msg.includes("fetch failed") || errCode === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || (typeof errCode === "string" && errCode.startsWith("UND_ERR")) || msg.includes("UND_ERR") ) { if (attempt === 0 && maxAttempts > 1) { // First failure — retry once with a short jittered delay before giving up. lastDispatcherError = dispatcherError; await new Promise((r) => setTimeout(r, 25 + Math.random() * 50)); continue; } if (hasNonReplayableBody) { const detail = `dispatcher=[${describeFetchCause(dispatcherError)}] native=[skipped: non-replayable request body]`; console.warn( `[ProxyFetch] skipping native fetch fallback for non-replayable body: ${detail}` ); if (dispatcherError instanceof Error) { (dispatcherError as Error & { proxyFetchDetail?: string }).proxyFetchDetail = detail; } throw dispatcherError; } // All attempts exhausted — try proxy fallback before native fetch if (source === "direct" && isFeatureFlagEnabled("PROXY_AUTO_SELECT_ENABLED")) { let targetHostname = ""; try { targetHostname = new URL(targetUrl).hostname; } catch { // ignore } if (targetHostname) { const fallbackProxyUrl = await findWorkingProxy(targetHostname, targetUrl); if (fallbackProxyUrl) { try { const dispatcher = createProxyDispatcher(fallbackProxyUrl); return await _undiciDirect(input, { ...options, dispatcher }); } catch { // Proxy also failed — fall through to native fetch } } } } // Preserve original phrase intact for monitoring: "Undici dispatcher failed, falling back to native fetch" // #4252: append the flattened err.cause (code/syscall/errno/address) — the bare // "fetch failed" message hides what actually broke, making bursts undiagnosable. console.warn( `[ProxyFetch] Undici dispatcher failed, falling back to native fetch (after retry): ${describeFetchCause(dispatcherError)}` ); try { return await _nativeFallback(input, options); } catch (nativeError) { // #4252: both the undici dispatcher AND native fetch failed. Surface BOTH // causes (server log) and tag the propagated error so the combo executor sees // a diagnosable failure IMMEDIATELY instead of a bare "fetch failed" — the // latter left jobs sitting until the 30s semaphore queue timeout, which then // tripped the circuit breaker. const detail = `dispatcher=[${describeFetchCause(dispatcherError)}] native=[${describeFetchCause(nativeError)}]`; console.warn(`[ProxyFetch] native fetch fallback ALSO failed: ${detail}`); if (nativeError instanceof Error) { (nativeError as Error & { proxyFetchDetail?: string }).proxyFetchDetail = detail; } throw nativeError; } } throw dispatcherError; } } // Should not be reached, but satisfy TypeScript control-flow. throw lastDispatcherError; } // Edge relay (vercel / deno): instead of routing through an HTTP proxy // dispatcher, we send x-relay-* headers to the edge function which forwards // the request upstream. Both backends share the same envelope shape. const contextProxy = proxyContext.getStore(); if ( contextProxy && typeof contextProxy === "object" && isRelayType((contextProxy as { type?: string }).type) ) { const vc = contextProxy as { type?: string; host?: string; relayAuth?: string }; if (!vc.relayAuth) { // Generic message without internal labels — this throw can bubble up to // catch blocks that put error.message in response bodies (combo per-model // timeout, executor catch-all). Don't leak "[ProxyFetch]" diagnostics. const label = vc.type === "vercel" ? "Vercel relay" : `${vc.type || "Edge"} relay`; throw new Error(`${label} configuration error: missing relayAuth`); } const targetUrl = getTargetUrl(input); const relayHeaders = buildVercelRelayHeaders(targetUrl, vc.relayAuth); const mergedHeaders = new Headers(options?.headers); for (const [k, v] of Object.entries(relayHeaders)) mergedHeaders.set(k, v); // Pass host through proxyUrlForLogs so the same redaction policy applies // to relay routing logs (the rest of this module already follows that rule). const hostForLogs = proxyUrlForLogs(vc.host ? `https://${vc.host}` : ""); if (process.env.OMNIROUTE_PROXY_FETCH_DEBUG === "true") { console.debug(`[ProxyFetch] Routing via ${vc.type || "edge"} relay: ${hostForLogs}`); } return await originalFetch(`https://${vc.host}`, { ...options, headers: mergedHeaders, duplex: "half", }); } try { const dispatcher = createProxyDispatcher(proxyUrl); const _undiciProxy = deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise); return await _undiciProxy(input, { ...options, dispatcher, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(`[ProxyFetch] Proxy request failed (${source}, fail-closed): ${message}`); throw error; } } /** * Named export for proxyFetch — identical to the patched globalThis.fetch but * accepts an optional ProxyFetchDeps for unit test dependency injection. * Production code should use globalThis.fetch (or the default export) instead. */ export async function proxyFetch( input: RequestInfo | URL, options: RequestInit = {}, deps: ProxyFetchDeps = {} ): Promise { return patchedFetch(input, options as FetchWithDispatcherOptions, deps); } if (!isCloud && !patchState.isPatched) { globalThis.fetch = patchedFetch; patchState.isPatched = true; } /** * Run a function with TLS fingerprint tracking context. * After fn completes, returns { result, tlsFingerprintUsed }. */ export async function runWithTlsTracking(fn) { const store = { used: false }; const result = await tlsFingerprintContext.run(store, fn); return { result, tlsFingerprintUsed: store.used }; } /** Check if TLS fingerprint is enabled and available */ export function isTlsFingerprintActive() { return isTlsFingerprintEnabled() && tlsClient.available; } /** * Get the original unpatched global fetch function (Node.js native fetch * before the proxy/TLS fingerprint patch was applied). * Use this to bypass the patched fetch for specific requests when the * proxy dispatcher has compatibility issues with a particular endpoint. */ export function getOriginalFetch(): typeof globalThis.fetch { return originalFetch; } export default isCloud ? originalFetch : patchedFetch;