wehub-resource-sync
667 lines
26 KiB
TypeScript
667 lines
26 KiB
TypeScript
// @ts-nocheck
|
|
import "./setupPolyfill.ts";
|
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
import { fetch as undiciFetch } from "undici";
|
|
import {
|
|
buildVercelRelayHeaders,
|
|
createProxyDispatcher,
|
|
getDefaultDispatcher,
|
|
getRetryDispatcher,
|
|
isRelayType,
|
|
normalizeProxyUrl,
|
|
proxyConfigToUrl,
|
|
proxyUrlForLogs,
|
|
} from "./proxyDispatcher.ts";
|
|
import tlsClient from "./tlsClient.ts";
|
|
import { isProxyReachable } from "@/lib/proxyHealth";
|
|
import {
|
|
isControlPlaneProxyDirectFallbackEnabled,
|
|
isFeatureFlagEnabled,
|
|
} from "@/shared/utils/featureFlags";
|
|
import { findWorkingProxy } from "./proxyFallback.ts";
|
|
|
|
function isTlsFingerprintEnabled() {
|
|
return process.env.ENABLE_TLS_FINGERPRINT === "true";
|
|
}
|
|
|
|
/** Per-request tracking of whether TLS fingerprint was used */
|
|
type TlsFingerprintStore = { used: boolean };
|
|
const tlsFingerprintContext = new AsyncLocalStorage<TlsFingerprintStore>();
|
|
|
|
/**
|
|
* #5217 (Gap-secondary): a mutable sink that records the proxy actually applied
|
|
* by `runWithProxyContext` for the in-flight request. Executors that pin their
|
|
* own per-account proxy *internally* (e.g. OpencodeExecutor wraps its dispatch
|
|
* in `runWithProxyContext(account.proxy, …)`) never propagate that choice back
|
|
* to the caller's `proxyInfo`, so the post-execution `[ProxyEgress]` line logged
|
|
* `proxy=direct` even though `[ProxyFetch] Applied request proxy context: …`
|
|
* fired. Wrapping the execution in `runWithAppliedProxyCapture(sink, fn)` lets
|
|
* the egress logger read the innermost applied proxy (the last writer wins, which
|
|
* is the executor's per-account proxy).
|
|
*/
|
|
export type AppliedProxySink = { proxy: unknown };
|
|
const appliedProxyContext = new AsyncLocalStorage<AppliedProxySink>();
|
|
|
|
/**
|
|
* Run `fn` with an applied-proxy capture sink in context. Any
|
|
* `runWithProxyContext` call inside `fn` that ends up applying a proxy records
|
|
* that proxy config into `sink.proxy` (innermost wins). The sink is a plain
|
|
* mutable object the caller retains, so it can read `sink.proxy` after `fn`
|
|
* resolves. Pure plumbing — no behavioral change to the request itself.
|
|
*/
|
|
export function runWithAppliedProxyCapture<T>(sink: AppliedProxySink, fn: () => T): T {
|
|
return appliedProxyContext.run(sink, fn);
|
|
}
|
|
|
|
type FetchWithDispatcherOptions = RequestInit & { dispatcher?: unknown };
|
|
type FetchWithDispatcher = (
|
|
input: RequestInfo | URL,
|
|
init?: FetchWithDispatcherOptions
|
|
) => Promise<Response>;
|
|
|
|
/**
|
|
* Flatten a fetch error's `cause` chain (and any Happy-Eyeballs `AggregateError`
|
|
* sub-errors) into a single diagnostic line: code/syscall/errno/address:port + a
|
|
* truncated message. undici/native both reject with a bare `TypeError: fetch failed`
|
|
* whose real reason hides in `.cause`; surfacing it is what makes dispatcher-failure
|
|
* bursts (#4252) diagnosable. Never includes a stack trace (Rule #12). Pure + testable.
|
|
*/
|
|
export function describeFetchCause(err: unknown): string {
|
|
const parts: string[] = [];
|
|
const seen = new Set<unknown>();
|
|
let cur: unknown = err;
|
|
for (let depth = 0; cur && depth < 5 && !seen.has(cur); depth++) {
|
|
seen.add(cur);
|
|
const e = cur as Record<string, unknown>;
|
|
const seg = [
|
|
typeof e.name === "string" && e.name !== "Error" ? e.name : null,
|
|
typeof e.message === "string" ? e.message.slice(0, 160) : null,
|
|
e.code != null ? `code=${String(e.code)}` : null,
|
|
e.syscall != null ? `syscall=${String(e.syscall)}` : null,
|
|
e.errno != null ? `errno=${String(e.errno)}` : null,
|
|
e.address != null
|
|
? `address=${String(e.address)}${e.port != null ? `:${String(e.port)}` : ""}`
|
|
: null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
if (seg) parts.push(seg);
|
|
if (Array.isArray(e.errors)) {
|
|
for (const sub of (e.errors as unknown[]).slice(0, 4)) {
|
|
const s = (sub ?? {}) as Record<string, unknown>;
|
|
const subSeg = [
|
|
s.code != null ? `code=${String(s.code)}` : null,
|
|
s.syscall != null ? `syscall=${String(s.syscall)}` : null,
|
|
s.address != null
|
|
? `address=${String(s.address)}${s.port != null ? `:${String(s.port)}` : ""}`
|
|
: null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
if (subSeg) parts.push(`↳ ${subSeg}`);
|
|
else if (typeof s.message === "string") parts.push(`↳ ${s.message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
cur = e.cause;
|
|
}
|
|
return parts.join(" | ") || String(err);
|
|
}
|
|
|
|
|
|
function isStreamLikeBody(body: unknown): boolean {
|
|
return (
|
|
body !== null &&
|
|
body !== undefined &&
|
|
typeof body === "object" &&
|
|
(typeof (body as Record<string, unknown>).getReader === "function" ||
|
|
typeof (body as Record<string, unknown>).stream === "function")
|
|
);
|
|
}
|
|
|
|
function requestHasNonReplayableBody(
|
|
input: RequestInfo | URL,
|
|
options: FetchWithDispatcherOptions
|
|
): boolean {
|
|
if (isStreamLikeBody(options.body as unknown)) return true;
|
|
if (typeof Request !== "undefined" && input instanceof Request) {
|
|
if (input.bodyUsed) return true;
|
|
if (input.body !== null) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** Injectable dependencies for testability (Approach B DI). */
|
|
export type ProxyFetchDeps = {
|
|
undiciFetch?: FetchWithDispatcher;
|
|
nativeFetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
|
};
|
|
|
|
type PatchState = {
|
|
originalFetch: typeof globalThis.fetch;
|
|
proxyContext: AsyncLocalStorage<unknown>;
|
|
isPatched: boolean;
|
|
};
|
|
|
|
const isCloud = typeof caches !== "undefined" && typeof caches === "object";
|
|
const PATCH_STATE_KEY = Symbol.for("omniroute.proxyFetch.state");
|
|
|
|
function getPatchState(): PatchState {
|
|
const scopedGlobal = globalThis as typeof globalThis & {
|
|
[PATCH_STATE_KEY]?: PatchState;
|
|
};
|
|
|
|
if (!scopedGlobal[PATCH_STATE_KEY]) {
|
|
scopedGlobal[PATCH_STATE_KEY] = {
|
|
originalFetch: globalThis.fetch,
|
|
proxyContext: new AsyncLocalStorage(),
|
|
isPatched: false,
|
|
};
|
|
}
|
|
return scopedGlobal[PATCH_STATE_KEY];
|
|
}
|
|
|
|
const patchState = getPatchState();
|
|
const originalFetch = patchState.originalFetch;
|
|
const originalFetchWithDispatcher = originalFetch as FetchWithDispatcher;
|
|
const proxyContext = patchState.proxyContext;
|
|
|
|
function noProxyMatch(targetUrl) {
|
|
const noProxy = process.env.NO_PROXY || process.env.no_proxy;
|
|
if (!noProxy) return false;
|
|
|
|
let target;
|
|
try {
|
|
target = new URL(targetUrl);
|
|
} catch {
|
|
return false;
|
|
}
|
|
|
|
const hostname = target.hostname.toLowerCase();
|
|
const port = target.port || (target.protocol === "https:" ? "443" : "80");
|
|
const patterns = noProxy
|
|
.split(",")
|
|
.map((p) => p.trim().toLowerCase())
|
|
.filter(Boolean);
|
|
|
|
return patterns.some((pattern) => {
|
|
if (pattern === "*") return true;
|
|
|
|
const [patternHost, patternPort] = pattern.split(":");
|
|
if (patternPort && patternPort !== port) return false;
|
|
|
|
if (!patternHost) return false;
|
|
|
|
// Support wildcard matching (e.g. 192.168.* or *.local).
|
|
// Uses a linear glob scan instead of dynamic RegExp to avoid ReDoS.
|
|
if (patternHost.includes("*")) {
|
|
const parts = patternHost.split("*");
|
|
let pos = 0;
|
|
let ok = hostname.startsWith(parts[0]);
|
|
if (ok) {
|
|
pos = parts[0].length;
|
|
for (let i = 1; i < parts.length && ok; i++) {
|
|
const seg = parts[i];
|
|
if (i === parts.length - 1) {
|
|
ok = seg === "" || (hostname.endsWith(seg) && hostname.length - seg.length >= pos);
|
|
} else {
|
|
const idx = seg ? hostname.indexOf(seg, pos) : pos;
|
|
if (idx === -1) {
|
|
ok = false;
|
|
} else {
|
|
pos = idx + seg.length;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (ok) return true;
|
|
}
|
|
|
|
if (patternHost.startsWith(".")) {
|
|
return hostname.endsWith(patternHost) || hostname === patternHost.slice(1);
|
|
}
|
|
return hostname === patternHost || hostname.endsWith(`.${patternHost}`);
|
|
});
|
|
}
|
|
|
|
function isLocalAddress(hostname: string): boolean {
|
|
const host = hostname
|
|
.replace(/^\[/, "")
|
|
.replace(/\]$/, "")
|
|
.replace(/^::ffff:/i, "");
|
|
if (host === "localhost" || host === "0.0.0.0" || host === "127.0.0.1" || host === "::1") {
|
|
return true;
|
|
}
|
|
if (host.endsWith(".local") || host.endsWith(".lan") || host.endsWith(".internal")) return true;
|
|
// RFC1918 + loopback + link-local (169.254, incl. cloud metadata 169.254.169.254)
|
|
// + CGNAT (100.64/10). 127/8 covers all loopback, not just 127.0.0.1.
|
|
if (host.startsWith("192.168.")) return true;
|
|
if (host.startsWith("10.")) return true;
|
|
if (host.startsWith("127.")) return true;
|
|
if (host.startsWith("169.254.")) return true;
|
|
if (/^172\.(1[6-9]|2\d|3[0-1])\./.test(host)) return true;
|
|
if (/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(host)) return true;
|
|
// IPv6 ULA (fc00::/7 → fc/fd prefix) and link-local (fe80::/10)
|
|
if (/^f[cd][0-9a-f]*:/i.test(host) || host.startsWith("fe80:")) return true;
|
|
return false;
|
|
}
|
|
|
|
function resolveEnvProxyUrl(targetUrl) {
|
|
if (noProxyMatch(targetUrl)) return null;
|
|
|
|
let protocol;
|
|
try {
|
|
protocol = new URL(targetUrl).protocol;
|
|
} catch {
|
|
return null;
|
|
}
|
|
|
|
const proxyUrl =
|
|
protocol === "https:"
|
|
? process.env.HTTPS_PROXY ||
|
|
process.env.https_proxy ||
|
|
process.env.ALL_PROXY ||
|
|
process.env.all_proxy
|
|
: process.env.HTTP_PROXY ||
|
|
process.env.http_proxy ||
|
|
process.env.ALL_PROXY ||
|
|
process.env.all_proxy;
|
|
|
|
if (!proxyUrl) return null;
|
|
return normalizeProxyUrl(proxyUrl, "environment proxy");
|
|
}
|
|
|
|
export function resolveProxyForRequest(targetUrl) {
|
|
let target;
|
|
try {
|
|
target = new URL(targetUrl);
|
|
} catch {
|
|
target = null;
|
|
}
|
|
|
|
// Always bypass proxy for local/LAN addresses
|
|
if (target && isLocalAddress(target.hostname.toLowerCase())) {
|
|
return { source: "direct", proxyUrl: null };
|
|
}
|
|
|
|
const contextProxy = proxyContext.getStore();
|
|
if (contextProxy) {
|
|
return { source: "context", proxyUrl: proxyConfigToUrl(contextProxy) };
|
|
}
|
|
|
|
const envProxyUrl = resolveEnvProxyUrl(targetUrl);
|
|
if (envProxyUrl) {
|
|
return { source: "env", proxyUrl: envProxyUrl };
|
|
}
|
|
|
|
return { source: "direct", proxyUrl: null };
|
|
}
|
|
|
|
function getTargetUrl(input) {
|
|
if (typeof input === "string") return input;
|
|
if (input && typeof input.url === "string") return input.url;
|
|
return String(input);
|
|
}
|
|
|
|
export async function runWithProxyContext(
|
|
proxyConfig,
|
|
fn,
|
|
opts?: { directFallbackOnUnreachable?: boolean }
|
|
) {
|
|
if (typeof fn !== "function") {
|
|
throw new TypeError("runWithProxyContext requires a callback function");
|
|
}
|
|
|
|
// Inherit existing context if no specific proxyConfig is provided
|
|
const currentContext = proxyContext.getStore();
|
|
const effectiveProxyConfig = proxyConfig || currentContext || null;
|
|
|
|
const resolvedProxyUrl = effectiveProxyConfig ? proxyConfigToUrl(effectiveProxyConfig) : null;
|
|
|
|
// The caller must opt in, and the runtime feature flag must also be enabled.
|
|
// This fallback changes egress IP, so upgrades must not silently turn it on.
|
|
const directFallbackOnUnreachable =
|
|
opts?.directFallbackOnUnreachable === true && isControlPlaneProxyDirectFallbackEnabled();
|
|
// Run fn with the proxy context cleared so the request egresses directly.
|
|
const runDirect = () => proxyContext.run(null, fn);
|
|
|
|
// T14: Proxy Fast-Fail
|
|
// Perform a short TCP reachability check before issuing upstream requests.
|
|
// Skip for edge-relay types (vercel / deno): proxyConfigToUrl returns
|
|
// "https://<host>" which is the relay endpoint itself, not an HTTP proxy —
|
|
// the actual routing is handled via x-relay-* headers below.
|
|
const isVercelRelay = isRelayType((effectiveProxyConfig as { type?: string })?.type);
|
|
if (resolvedProxyUrl && !isVercelRelay) {
|
|
const reachable = await isProxyReachable(resolvedProxyUrl);
|
|
if (!reachable) {
|
|
const proxyLabel = proxyUrlForLogs(resolvedProxyUrl);
|
|
if (directFallbackOnUnreachable) {
|
|
console.warn(
|
|
`[ProxyFetch] Proxy unreachable (${proxyLabel}); using a direct connection for this request.`
|
|
);
|
|
return runDirect();
|
|
}
|
|
const err = new Error(`[Proxy Fast-Fail] Proxy unreachable: ${proxyLabel}`) as Error & {
|
|
code?: string;
|
|
statusCode?: number;
|
|
};
|
|
err.code = "PROXY_UNREACHABLE";
|
|
err.statusCode = 503;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// Fail-closed family check: when the proxy URL carries a ?family=ipv6|ipv4 marker
|
|
// (set for HOSTNAME proxies by proxyConfigToUrl), verify the hostname actually has a
|
|
// record in that family before egressing. Refuse early rather than silently fall back
|
|
// to the other family. No-op for IP literals (their family is intrinsic).
|
|
if (resolvedProxyUrl && !isVercelRelay) {
|
|
try {
|
|
const u = new URL(resolvedProxyUrl);
|
|
const fam = u.searchParams.get("family");
|
|
if (fam === "ipv6" || fam === "ipv4") {
|
|
const { assertHostnameSupportsFamily } = await import("./proxyFamilyResolve.ts");
|
|
await assertHostnameSupportsFamily(u.hostname, fam === "ipv6" ? 6 : 4);
|
|
}
|
|
} catch (familyErr) {
|
|
if (directFallbackOnUnreachable) {
|
|
console.warn(
|
|
`[ProxyFetch] Proxy family pre-check failed (${proxyUrlForLogs(resolvedProxyUrl)}); using a direct connection for this request.`
|
|
);
|
|
return runDirect();
|
|
}
|
|
const e = familyErr as Error & { code?: string; statusCode?: number };
|
|
e.code = e.code || "PROXY_FAMILY_UNAVAILABLE";
|
|
e.statusCode = e.statusCode || 503;
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
return proxyContext.run(effectiveProxyConfig, async () => {
|
|
if (resolvedProxyUrl && effectiveProxyConfig !== currentContext) {
|
|
console.log(
|
|
`[ProxyFetch] Applied request proxy context: ${proxyUrlForLogs(resolvedProxyUrl)}`
|
|
);
|
|
}
|
|
// #5217: record the proxy actually applied so a post-execution egress logger
|
|
// reflects the real egress (executors that pin a per-account proxy internally
|
|
// otherwise leave proxyInfo reading "direct"). Innermost runWithProxyContext
|
|
// wins, which is exactly the per-account proxy the executor selected.
|
|
if (effectiveProxyConfig) {
|
|
const sink = appliedProxyContext.getStore();
|
|
if (sink) sink.proxy = effectiveProxyConfig;
|
|
}
|
|
return fn();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Like {@link runWithProxyContext}, but if the assigned proxy is unreachable or fails
|
|
* its pre-checks the request can degrade to a DIRECT connection instead of throwing.
|
|
*
|
|
* For control-plane flows — OAuth code/token exchange, connection tests, token refresh —
|
|
* where a dead pinned proxy must not block reaching the upstream (it otherwise surfaces
|
|
* as a generic "Internal server error"). Data-plane chat keeps strict pinning via
|
|
* runWithProxyContext so per-account egress-IP isolation is preserved.
|
|
*
|
|
* This remains disabled unless OMNIROUTE_CONTROL_PLANE_PROXY_DIRECT_FALLBACK is enabled
|
|
* from Feature Flags or the environment.
|
|
*/
|
|
export async function runWithProxyContextOrDirect(proxyConfig, fn) {
|
|
return runWithProxyContext(proxyConfig, fn, { directFallbackOnUnreachable: true });
|
|
}
|
|
|
|
async function patchedFetch(
|
|
input: RequestInfo | URL,
|
|
options: FetchWithDispatcherOptions = {},
|
|
deps: ProxyFetchDeps = {}
|
|
) {
|
|
if (options?.dispatcher) {
|
|
// When a dispatcher is present, we MUST use the undici library fetch
|
|
// to ensure version compatibility. Node 22 built-in fetch (undici v6)
|
|
// is incompatible with undici v8 dispatchers (missing onRequestStart, etc.)
|
|
const _undiciDispatcher =
|
|
deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise<Response>);
|
|
return _undiciDispatcher(input, options);
|
|
}
|
|
|
|
const targetUrl = getTargetUrl(input);
|
|
let resolved;
|
|
try {
|
|
resolved = resolveProxyForRequest(targetUrl);
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
console.error(`[ProxyFetch] Proxy configuration error: ${message}`);
|
|
throw error;
|
|
}
|
|
const { source, proxyUrl } = resolved;
|
|
|
|
if (!proxyUrl) {
|
|
// TLS fingerprint spoofing for direct connections (no proxy configured)
|
|
if (isTlsFingerprintEnabled() && tlsClient.available) {
|
|
try {
|
|
const store = tlsFingerprintContext.getStore();
|
|
if (store) store.used = true;
|
|
return await tlsClient.fetch(targetUrl, {
|
|
...options,
|
|
headers: options.headers,
|
|
signal: options.signal ?? undefined,
|
|
});
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
console.warn(
|
|
`[ProxyFetch] TLS fingerprint failed, falling back to native fetch: ${message}`
|
|
);
|
|
const store = tlsFingerprintContext.getStore();
|
|
if (store) store.used = false;
|
|
}
|
|
}
|
|
// Direct connection (no proxy) — use undici with custom dispatcher for timeout control.
|
|
// Falls back to original native fetch if dispatcher initialization fails (#1054).
|
|
// Retries once on transient dispatcher errors before falling back (fix: proxyfetch-undici-retry).
|
|
//
|
|
// Non-replayable body guard: if the body is stream-like (ReadableStream/Blob)
|
|
// or the input is a Request that carries a body, the first dispatcher attempt
|
|
// owns that body. Retrying or falling back to native fetch would replay a
|
|
// consumed/locked body and can mask the original transport error with
|
|
// "Response body object should not be disturbed or locked".
|
|
const hasNonReplayableBody = requestHasNonReplayableBody(input, options);
|
|
const maxAttempts = hasNonReplayableBody ? 1 : 2;
|
|
const _undiciDirect =
|
|
deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise<Response>);
|
|
const _nativeFallback =
|
|
(deps.nativeFetch as FetchWithDispatcher | undefined) ?? originalFetchWithDispatcher;
|
|
let lastDispatcherError: unknown = null;
|
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
try {
|
|
return await _undiciDirect(input, {
|
|
...options,
|
|
// #4252: first attempt uses the pooled keep-alive dispatcher; a retry
|
|
// (after a transient socket error) uses the no-keep-alive dispatcher so
|
|
// it opens a FRESH socket instead of grabbing another stale pooled one
|
|
// — the burst pattern was the retry re-hitting a dead pooled socket and
|
|
// then falling through to native fetch (which also pools) → 502.
|
|
dispatcher: attempt === 0 ? getDefaultDispatcher() : getRetryDispatcher(),
|
|
});
|
|
} catch (dispatcherError) {
|
|
const msg =
|
|
dispatcherError instanceof Error ? dispatcherError.message : String(dispatcherError);
|
|
// CAUTION: Do NOT fallback to native fetch if the error is a version mismatch (invalid onRequestStart)
|
|
// because the native fetch will definitely fail with the undici v8 dispatcher.
|
|
if (msg.includes("onRequestStart")) {
|
|
console.error(
|
|
`[ProxyFetch] Fatal version mismatch: Dispatcher (v8) vs Fetch (v6/native). Hardware upgrade or SOCKS5 config isolation required. Error: ${msg}`
|
|
);
|
|
throw dispatcherError;
|
|
}
|
|
// Only retry/fallback for connection/dispatcher errors, not HTTP errors.
|
|
// Prefer the .code property when available (more stable across undici
|
|
// versions than message-string matching); fall back to substring match
|
|
// for errors that lack a structured code.
|
|
const errCode = (dispatcherError as { code?: unknown })?.code;
|
|
if (
|
|
msg.includes("fetch failed") ||
|
|
errCode === "ECONNREFUSED" ||
|
|
msg.includes("ECONNREFUSED") ||
|
|
(typeof errCode === "string" && errCode.startsWith("UND_ERR")) ||
|
|
msg.includes("UND_ERR")
|
|
) {
|
|
if (attempt === 0 && maxAttempts > 1) {
|
|
// First failure — retry once with a short jittered delay before giving up.
|
|
lastDispatcherError = dispatcherError;
|
|
await new Promise((r) => setTimeout(r, 25 + Math.random() * 50));
|
|
continue;
|
|
}
|
|
if (hasNonReplayableBody) {
|
|
const detail = `dispatcher=[${describeFetchCause(dispatcherError)}] native=[skipped: non-replayable request body]`;
|
|
console.warn(
|
|
`[ProxyFetch] skipping native fetch fallback for non-replayable body: ${detail}`
|
|
);
|
|
if (dispatcherError instanceof Error) {
|
|
(dispatcherError as Error & { proxyFetchDetail?: string }).proxyFetchDetail = detail;
|
|
}
|
|
throw dispatcherError;
|
|
}
|
|
|
|
// All attempts exhausted — try proxy fallback before native fetch
|
|
if (source === "direct" && isFeatureFlagEnabled("PROXY_AUTO_SELECT_ENABLED")) {
|
|
let targetHostname = "";
|
|
try {
|
|
targetHostname = new URL(targetUrl).hostname;
|
|
} catch {
|
|
// ignore
|
|
}
|
|
if (targetHostname) {
|
|
const fallbackProxyUrl = await findWorkingProxy(targetHostname, targetUrl);
|
|
if (fallbackProxyUrl) {
|
|
try {
|
|
const dispatcher = createProxyDispatcher(fallbackProxyUrl);
|
|
return await _undiciDirect(input, { ...options, dispatcher });
|
|
} catch {
|
|
// Proxy also failed — fall through to native fetch
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Preserve original phrase intact for monitoring: "Undici dispatcher failed, falling back to native fetch"
|
|
// #4252: append the flattened err.cause (code/syscall/errno/address) — the bare
|
|
// "fetch failed" message hides what actually broke, making bursts undiagnosable.
|
|
console.warn(
|
|
`[ProxyFetch] Undici dispatcher failed, falling back to native fetch (after retry): ${describeFetchCause(dispatcherError)}`
|
|
);
|
|
try {
|
|
return await _nativeFallback(input, options);
|
|
} catch (nativeError) {
|
|
// #4252: both the undici dispatcher AND native fetch failed. Surface BOTH
|
|
// causes (server log) and tag the propagated error so the combo executor sees
|
|
// a diagnosable failure IMMEDIATELY instead of a bare "fetch failed" — the
|
|
// latter left jobs sitting until the 30s semaphore queue timeout, which then
|
|
// tripped the circuit breaker.
|
|
const detail = `dispatcher=[${describeFetchCause(dispatcherError)}] native=[${describeFetchCause(nativeError)}]`;
|
|
console.warn(`[ProxyFetch] native fetch fallback ALSO failed: ${detail}`);
|
|
if (nativeError instanceof Error) {
|
|
(nativeError as Error & { proxyFetchDetail?: string }).proxyFetchDetail = detail;
|
|
}
|
|
throw nativeError;
|
|
}
|
|
}
|
|
throw dispatcherError;
|
|
}
|
|
}
|
|
// Should not be reached, but satisfy TypeScript control-flow.
|
|
throw lastDispatcherError;
|
|
}
|
|
|
|
// Edge relay (vercel / deno): instead of routing through an HTTP proxy
|
|
// dispatcher, we send x-relay-* headers to the edge function which forwards
|
|
// the request upstream. Both backends share the same envelope shape.
|
|
const contextProxy = proxyContext.getStore();
|
|
if (
|
|
contextProxy &&
|
|
typeof contextProxy === "object" &&
|
|
isRelayType((contextProxy as { type?: string }).type)
|
|
) {
|
|
const vc = contextProxy as { type?: string; host?: string; relayAuth?: string };
|
|
if (!vc.relayAuth) {
|
|
// Generic message without internal labels — this throw can bubble up to
|
|
// catch blocks that put error.message in response bodies (combo per-model
|
|
// timeout, executor catch-all). Don't leak "[ProxyFetch]" diagnostics.
|
|
const label = vc.type === "vercel" ? "Vercel relay" : `${vc.type || "Edge"} relay`;
|
|
throw new Error(`${label} configuration error: missing relayAuth`);
|
|
}
|
|
const targetUrl = getTargetUrl(input);
|
|
const relayHeaders = buildVercelRelayHeaders(targetUrl, vc.relayAuth);
|
|
const mergedHeaders = new Headers(options?.headers);
|
|
for (const [k, v] of Object.entries(relayHeaders)) mergedHeaders.set(k, v);
|
|
// Pass host through proxyUrlForLogs so the same redaction policy applies
|
|
// to relay routing logs (the rest of this module already follows that rule).
|
|
const hostForLogs = proxyUrlForLogs(vc.host ? `https://${vc.host}` : "");
|
|
if (process.env.OMNIROUTE_PROXY_FETCH_DEBUG === "true") {
|
|
console.debug(`[ProxyFetch] Routing via ${vc.type || "edge"} relay: ${hostForLogs}`);
|
|
}
|
|
return await originalFetch(`https://${vc.host}`, {
|
|
...options,
|
|
headers: mergedHeaders,
|
|
duplex: "half",
|
|
});
|
|
}
|
|
|
|
try {
|
|
const dispatcher = createProxyDispatcher(proxyUrl);
|
|
const _undiciProxy =
|
|
deps.undiciFetch ?? (undiciFetch as unknown as (...args: unknown[]) => Promise<Response>);
|
|
return await _undiciProxy(input, {
|
|
...options,
|
|
dispatcher,
|
|
});
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
console.error(`[ProxyFetch] Proxy request failed (${source}, fail-closed): ${message}`);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Named export for proxyFetch — identical to the patched globalThis.fetch but
|
|
* accepts an optional ProxyFetchDeps for unit test dependency injection.
|
|
* Production code should use globalThis.fetch (or the default export) instead.
|
|
*/
|
|
export async function proxyFetch(
|
|
input: RequestInfo | URL,
|
|
options: RequestInit = {},
|
|
deps: ProxyFetchDeps = {}
|
|
): Promise<Response> {
|
|
return patchedFetch(input, options as FetchWithDispatcherOptions, deps);
|
|
}
|
|
|
|
if (!isCloud && !patchState.isPatched) {
|
|
globalThis.fetch = patchedFetch;
|
|
patchState.isPatched = true;
|
|
}
|
|
|
|
/**
|
|
* Run a function with TLS fingerprint tracking context.
|
|
* After fn completes, returns { result, tlsFingerprintUsed }.
|
|
*/
|
|
export async function runWithTlsTracking(fn) {
|
|
const store = { used: false };
|
|
const result = await tlsFingerprintContext.run(store, fn);
|
|
return { result, tlsFingerprintUsed: store.used };
|
|
}
|
|
|
|
/** Check if TLS fingerprint is enabled and available */
|
|
export function isTlsFingerprintActive() {
|
|
return isTlsFingerprintEnabled() && tlsClient.available;
|
|
}
|
|
|
|
/**
|
|
* Get the original unpatched global fetch function (Node.js native fetch
|
|
* before the proxy/TLS fingerprint patch was applied).
|
|
* Use this to bypass the patched fetch for specific requests when the
|
|
* proxy dispatcher has compatibility issues with a particular endpoint.
|
|
*/
|
|
export function getOriginalFetch(): typeof globalThis.fetch {
|
|
return originalFetch;
|
|
}
|
|
|
|
export default isCloud ? originalFetch : patchedFetch;
|