chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:39:12 +08:00
commit d8dcd5f6d1
8604 changed files with 2479390 additions and 0 deletions
@@ -0,0 +1,126 @@
/**
* FingerprintRotator — Browser fingerprint profiles for session diversity
*
* Each profile mimics a real browser user-agent + Sec-CH-UA headers.
* Rotating through these prevents providers from associating requests
* to a single browser identity for rate-limiting purposes.
*/
import { type Fingerprint } from "./types.ts";
// ─── Profiles ──────────────────────────────────────────────────────────────
const PROFILES: Fingerprint[] = [
{
id: "chrome-mac",
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
acceptLanguage: "en-US,en;q=0.9",
secChUa: '"Not-A.Brand";v="99", "Chromium";v="149", "Google Chrome";v="149"',
secChUaPlatform: '"macOS"',
secChUaMobile: "?0",
},
{
id: "chrome-linux",
userAgent:
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
acceptLanguage: "en-US,en;q=0.9",
secChUa: '"Not-A.Brand";v="99", "Chromium";v="149", "Google Chrome";v="149"',
secChUaPlatform: '"Linux"',
secChUaMobile: "?0",
},
{
id: "chrome-win",
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36",
acceptLanguage: "en-US,en;q=0.9",
secChUa: '"Not-A.Brand";v="99", "Chromium";v="149", "Google Chrome";v="149"',
secChUaPlatform: '"Windows"',
secChUaMobile: "?0",
},
{
id: "firefox-mac",
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0",
acceptLanguage: "en-US,en;q=0.9",
},
{
id: "firefox-linux",
userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:152.0) Gecko/20100101 Firefox/152.0",
acceptLanguage: "en-US,en;q=0.9",
},
{
id: "safari-mac",
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.4 Safari/605.1.15",
acceptLanguage: "en-US,en;q=0.9",
},
{
id: "edge-mac",
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36 Edg/149.0.0.0",
acceptLanguage: "en-US,en;q=0.9",
secChUa: '"Not-A.Brand";v="99", "Chromium";v="149", "Microsoft Edge";v="149"',
secChUaPlatform: '"macOS"',
secChUaMobile: "?0",
},
{
id: "edge-win",
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36 Edg/149.0.0.0",
acceptLanguage: "en-US,en;q=0.9",
secChUa: '"Not-A.Brand";v="99", "Chromium";v="149", "Microsoft Edge";v="149"',
secChUaPlatform: '"Windows"',
secChUaMobile: "?0",
},
];
// ─── FingerprintRotator ────────────────────────────────────────────────────
export class FingerprintRotator {
private index = 0;
/** Get the next profile in round-robin order */
next(): Fingerprint {
const profile = PROFILES[this.index % PROFILES.length];
this.index++;
return profile;
}
/** Get a random profile */
random(): Fingerprint {
return PROFILES[Math.floor(Math.random() * PROFILES.length)];
}
/** Reset the round-robin counter */
reset(): void {
this.index = 0;
}
/** Number of available profiles */
get count(): number {
return PROFILES.length;
}
/** Build a Headers object from a fingerprint (with optional extras) */
buildHeaders(fingerprint: Fingerprint, extra?: Record<string, string>): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": "application/json",
Accept: "application/json, text/plain, */*",
"Accept-Language": fingerprint.acceptLanguage ?? "en-US,en;q=0.9",
"User-Agent": fingerprint.userAgent,
...extra,
};
if (fingerprint.secChUa) {
headers["Sec-CH-UA"] = fingerprint.secChUa;
headers["Sec-CH-UA-Mobile"] = fingerprint.secChUaMobile ?? "?0";
headers["Sec-CH-UA-Platform"] = fingerprint.secChUaPlatform ?? '"Windows"';
}
return headers;
}
/** List all profiles */
listAll(): Fingerprint[] {
return [...PROFILES];
}
}
+28
View File
@@ -0,0 +1,28 @@
/**
* Session Pool — Barrel exports
*
* Usage:
* import { SessionPool, Session, FingerprintRotator, SessionFactory, withSessionPool } from "./sessionPool/index.ts";
* import type { PoolConfig, PoolStats, PoolSessionDetail } from "./sessionPool/types.ts";
*/
export { Session } from "./session.ts";
export { SessionPool } from "./sessionPool.ts";
export { SessionFactory } from "./sessionFactory.ts";
export { FingerprintRotator } from "./fingerprintRotator.ts";
export { withSessionPool } from "./webExecutorWrapper.ts";
export { PoolRegistry } from "./poolRegistry.ts";
export type {
Fingerprint,
SessionState,
SessionResult,
PoolConfig,
PoolStats,
PoolSessionDetail,
WebExecutorFn,
WebExecutorRequest,
WebExecutorResponse,
} from "./types.ts";
export { DEFAULT_POOL_CONFIG } from "./types.ts";
@@ -0,0 +1,93 @@
/**
* Pool Registry — Global registry for active session pools.
*
* Provides a rendezvous point between executors (which create pools)
* and MCP tools / API handlers (which query pool state).
*
* Usage (executor side):
* PoolRegistry.register("pollinations", myPool);
*
* Usage (MCP tool / API side):
* const stats = PoolRegistry.getStats("pollinations");
* const all = PoolRegistry.getAllStats();
*/
import type { SessionPool } from "./sessionPool.ts";
import type { PoolStats, PoolSessionDetail } from "./types.ts";
type PoolEntry = {
pool: SessionPool;
createdAt: number;
};
class PoolRegistryImpl {
private pools = new Map<string, PoolEntry>();
/** Register a pool for a provider. Overwrites any previous pool. */
register(provider: string, pool: SessionPool): void {
this.pools.set(provider, { pool, createdAt: Date.now() });
}
/** Unregister a pool */
unregister(provider: string): boolean {
return this.pools.delete(provider);
}
/** Get a pool by provider name */
getPool(provider: string): SessionPool | undefined {
return this.pools.get(provider)?.pool;
}
/** List all registered provider names */
listProviders(): string[] {
return Array.from(this.pools.keys());
}
/** Get stats for a specific provider's pool */
getStats(provider: string): (PoolStats & { createdAt: number }) | null {
const entry = this.pools.get(provider);
if (!entry) return null;
return { ...entry.pool.getStats(), createdAt: entry.createdAt };
}
/** Get stats for all registered pools */
getAllStats(): Array<PoolStats & { createdAt: number }> {
const result: Array<PoolStats & { createdAt: number }> = [];
for (const [, entry] of this.pools) {
result.push({ ...entry.pool.getStats(), createdAt: entry.createdAt });
}
return result;
}
/** Get per-session details for a specific provider */
getSessionDetails(provider: string): PoolSessionDetail[] | null {
const entry = this.pools.get(provider);
if (!entry) return null;
return entry.pool.getSessionDetails();
}
/** Reset (shutdown + recreate) a pool */
resetPool(provider: string): boolean {
const entry = this.pools.get(provider);
if (!entry) return false;
entry.pool.shutdown();
this.pools.delete(provider);
return true;
}
/** Warm up a pool to a target session count */
async warmPool(provider: string, count: number): Promise<boolean> {
const entry = this.pools.get(provider);
if (!entry) return false;
await entry.pool.warmUp(count);
return true;
}
/** Count of registered pools */
get size(): number {
return this.pools.size;
}
}
/** Singleton global pool registry */
export const PoolRegistry = new PoolRegistryImpl();
+119
View File
@@ -0,0 +1,119 @@
/**
* Session — State machine for one pool session
*
* Lifecycle:
* active ──(429)──▶ cooldown ──(timeout)──▶ active
* active ──(5xx)──▶ dead ──(pruned)──▶ removed
* active ──(TTL)──▶ replaced with fresh session
*
* Each session tracks:
* - Fingerprint (UA + headers for one browser identity)
* - Request metrics (total, success, fail, inflight)
* - Cooldown state (exponential backoff on rate limits)
*/
import { type Fingerprint, type SessionStatus } from "./types.ts";
export class Session {
readonly id: string;
readonly fingerprint: Fingerprint;
readonly createdAt: number;
status: SessionStatus = "active";
inflight = 0;
totalRequests = 0;
successfulRequests = 0;
failedRequests = 0;
consecutiveFails = 0;
cooldownUntil = 0;
lastUsedAt = 0;
constructor(
fingerprint: Fingerprint,
private readonly cooldownBase: number,
private readonly cooldownMax: number,
private readonly cooldownJitter: number,
) {
this.id = `sess-${fingerprint.id}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
this.fingerprint = fingerprint;
this.createdAt = Date.now();
}
/** Whether this session can accept requests right now */
get isAvailable(): boolean {
if (this.status === "dead") return false;
if (this.status === "cooldown") {
if (Date.now() >= this.cooldownUntil) {
// Auto-recover from cooldown
this.status = "active";
this.consecutiveFails = 0;
return true;
}
return false;
}
return true;
}
/** Mark a successful request */
markSuccess(): void {
this.successfulRequests++;
this.consecutiveFails = 0;
}
/** Enter cooldown with exponential backoff */
markCooldown(): void {
this.consecutiveFails++;
const base = Math.min(
this.cooldownBase * Math.pow(2, this.consecutiveFails - 1),
this.cooldownMax,
);
const jitter = Math.random() * this.cooldownJitter;
this.cooldownUntil = Date.now() + base + jitter;
this.status = "cooldown";
}
/** Mark session as dead (non-recoverable error) */
markDead(): void {
this.status = "dead";
}
/** Increment inflight counter and mark as used */
acquire(): void {
this.inflight++;
this.totalRequests++;
this.lastUsedAt = Date.now();
}
/** Decrement inflight counter */
release(): void {
this.inflight = Math.max(0, this.inflight - 1);
}
/** Milliseconds remaining in cooldown */
get cooldownRemaining(): number {
if (this.status !== "cooldown") return 0;
return Math.max(0, this.cooldownUntil - Date.now());
}
/** Age in milliseconds */
get age(): number {
return Date.now() - this.createdAt;
}
/** Build headers for this session's fingerprint */
buildHeaders(extra?: Record<string, string>): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": "application/json",
Accept: "application/json, text/plain, */*",
"Accept-Language": this.fingerprint.acceptLanguage ?? "en-US,en;q=0.9",
"User-Agent": this.fingerprint.userAgent,
...extra,
};
if (this.fingerprint.secChUa) {
headers["Sec-CH-UA"] = this.fingerprint.secChUa;
headers["Sec-CH-UA-Mobile"] = this.fingerprint.secChUaMobile ?? "?0";
headers["Sec-CH-UA-Platform"] = this.fingerprint.secChUaPlatform ?? '"Windows"';
}
return headers;
}
}
@@ -0,0 +1,57 @@
/**
* SessionFactory — Creates initialized Session instances
*
* For zero-auth providers (Pollinations, Puter): just assigns a fingerprint.
* For cookie-based providers (ChatGPT Web, DeepSeek Web): would launch
* headless Playwright, solve Turnstile, and extract cookies.
*
* Currently only zero-auth is implemented. Cookie-based provider support
* is planned for Phase 3.
*/
import { FingerprintRotator } from "./fingerprintRotator.ts";
import { Session } from "./session.ts";
import type { PoolConfig } from "./types.ts";
export class SessionFactory {
private rotator = new FingerprintRotator();
constructor(private config: PoolConfig) {}
/**
* Create a new session with the next available fingerprint.
* For zero-auth providers, this is a lightweight operation
* (just picks a fingerprint). For cookie-based providers this
* would involve Playwright browser automation.
*/
createSession(): Session {
// Round-robin so each session in a warm pool gets a distinct fingerprint
// (a pool of look-alike sessions defeats the purpose). The rotator still
// exposes random() for callers that want unpredictability over time.
const fingerprint = this.rotator.next();
return new Session(
fingerprint,
this.config.cooldownBase,
this.config.cooldownMax,
this.config.cooldownJitter,
);
}
/** Reset the fingerprint rotator (e.g., after config change) */
resetRotator(): void {
this.rotator.reset();
}
/** Number of available fingerprint profiles */
get profileCount(): number {
return this.rotator.count;
}
/** Build headers from session fingerprint */
buildHeaders(
session: Session,
extra?: Record<string, string>,
): Record<string, string> {
return session.buildHeaders(extra);
}
}
@@ -0,0 +1,326 @@
/**
* SessionPool — Pool manager for anonymous web sessions
*
* Manages N sessions across N browser fingerprints, distributing requests
* round-robin to avoid rate-limiting any single "browser identity."
*
* Key behaviors:
* - acquire(): Returns the next available session (round-robin)
* - reportSuccess(session): Updates metrics, marks session healthy
* - reportFailure(session, status): Cooldown on 429, dead on 5xx
* - Scalability: Round-robin across sessions prevents one session from
* being overused while others sit idle.
* - Auto-heal: Sessions in cooldown auto-recover after the backoff window.
* - Self-repair: Dead sessions are replaced on next acquire if below maxSessions.
* - Thread-safe for concurrent request patterns (each acquire/release pair
* is atomic within one async flow).
*
* Usage:
* const pool = new SessionPool("pollinations", poolConfig, factory);
* await pool.ensureMinSessions();
* const session = pool.acquire();
* try {
* const res = await fetch(url, { headers: session.buildHeaders() });
* if (!res.ok && res.status === 429) pool.reportCooldown(session);
* else pool.reportSuccess(session);
* } catch { pool.reportDead(session); }
* finally { session.release(); }
*/
import { type EventEmitter } from "node:events";
import { createHash, randomUUID } from "node:crypto";
import { setTimeout as sleep } from "node:timers/promises";
import { Session } from "./session.ts";
import { SessionFactory } from "./sessionFactory.ts";
import {
type PoolConfig,
type PoolSessionDetail,
type PoolStats,
DEFAULT_POOL_CONFIG,
} from "./types.ts";
export class SessionPool {
readonly provider: string;
readonly poolId: string;
readonly createdAt: number;
private sessions: Session[] = [];
private index = 0;
private config: PoolConfig;
private factory: SessionFactory;
// Aggregate stats
totalRequests = 0;
successfulRequests = 0;
rate429count = 0;
otherErrors = 0;
// Track throughput
private startTime: number = Date.now();
private lastLog = 0;
constructor(
provider: string,
config?: Partial<PoolConfig>,
factory?: SessionFactory,
) {
this.provider = provider;
this.poolId = `pool-${provider}-${Date.now().toString(36)}`;
this.createdAt = Date.now();
this.config = { ...DEFAULT_POOL_CONFIG, ...config };
this.factory = factory ?? new SessionFactory(this.config);
}
// ─── Pool Lifecycle ──────────────────────────────────────────────────
/** Ensure the pool has at least minSessions ready */
async ensureMinSessions(): Promise<void> {
const needed = this.config.minSessions - this.sessions.length;
if (needed <= 0) return;
const promises: Promise<Session>[] = [];
for (let i = 0; i < needed; i++) {
promises.push(Promise.resolve(this.createSession()));
}
await Promise.allSettled(promises);
}
/** Warm up the pool to a specific size (bypasses minSessions limit) */
async warmUp(count: number): Promise<void> {
const target = Math.min(count, this.config.maxSessions);
const needed = target - this.sessions.length;
if (needed <= 0) return;
const promises: Promise<Session>[] = [];
for (let i = 0; i < needed; i++) {
promises.push(Promise.resolve(this.createSession()));
}
await Promise.allSettled(promises);
}
/** Graceful shutdown — mark all sessions dead */
async shutdown(): Promise<void> {
for (const s of this.sessions) {
s.markDead();
}
this.sessions = [];
}
// ─── Acquire / Release ───────────────────────────────────────────────
/**
* Acquire the next available session (round-robin with availability check).
* Returns null if no sessions are available (all on cooldown/dead).
*/
acquire(): Session | null {
// First pass: try round-robin from current index
if (this.sessions.length === 0) return null;
const startIdx = this.index % this.sessions.length;
for (let i = 0; i < this.sessions.length; i++) {
const idx = (startIdx + i) % this.sessions.length;
const session = this.sessions[idx];
if (session.isAvailable) {
// Skip sessions that hit max inflight limit (safety valve)
// For anonymous web providers we allow high concurrency per session
this.index = (idx + 1) % this.sessions.length;
session.acquire();
this.totalRequests++;
return session;
}
}
// No ready sessions — try to create a new one if under max
if (this.sessions.length < this.config.maxSessions) {
const session = this.createSession();
session.acquire();
this.totalRequests++;
return session;
}
// Last resort: wait for the nearest cooldown to expire (caller should retry)
return null;
}
/**
* Report a successful request. Updates metrics pool-wide and per-session.
*/
reportSuccess(session: Session): void {
session.markSuccess();
this.successfulRequests++;
}
/**
* Report a rate-limit (429). Puts the session into exponential-backoff cooldown.
*/
reportCooldown(session: Session): void {
session.markCooldown();
this.rate429count++;
this.maybeLog();
}
/**
* Report a non-recoverable error. Marks session as dead.
*/
reportDead(session: Session): void {
session.markDead();
this.otherErrors++;
}
// ─── Health / Stats ──────────────────────────────────────────────────
/** Count of available (active, not in cooldown) sessions */
get availableCount(): number {
return this.sessions.filter((s) => s.isAvailable).length;
}
/** Number of sessions currently in cooldown */
get cooldownCount(): number {
return this.sessions.filter((s) => s.status === "cooldown").length;
}
/** Number of dead sessions */
get deadCount(): number {
return this.sessions.filter((s) => s.status === "dead").length;
}
/** Total sessions managed */
get totalCount(): number {
return this.sessions.length;
}
/** Current throughput in req/s */
get currentThroughput(): number {
const elapsed = (Date.now() - this.startTime) / 1000;
return elapsed > 0 ? this.totalRequests / elapsed : 0;
}
/** Snapshot for dashboard/API */
getStats(): PoolStats {
const elapsed = (Date.now() - this.startTime) / 1000;
return {
provider: this.provider,
sessions: {
total: this.sessions.length,
active: this.availableCount,
cooldown: this.cooldownCount,
dead: this.deadCount,
},
requests: {
total: this.totalRequests,
success: this.successfulRequests,
rate429: this.rate429count,
otherErrors: this.otherErrors,
},
throughput: this.currentThroughput.toFixed(1),
successRate:
this.totalRequests > 0
? ((this.successfulRequests / this.totalRequests) * 100).toFixed(1)
: "100.0",
elapsed: elapsed.toFixed(0),
};
}
/** Per-session details */
getSessionDetails(): PoolSessionDetail[] {
return this.sessions.map((s) => ({
id: s.id,
fingerprint: s.fingerprint.id,
status: s.status,
totalRequests: s.totalRequests,
successfulRequests: s.successfulRequests,
successRate:
s.totalRequests > 0
? ((s.successfulRequests / s.totalRequests) * 100).toFixed(1)
: "100.0",
inflight: s.inflight,
cooldownRemaining: s.cooldownRemaining > 0
? `${(s.cooldownRemaining / 1000).toFixed(1)}s`
: "0s",
age: `${(s.age / 1000).toFixed(0)}s`,
}));
}
/** As acquire(), but blocks until a session is available */
async acquireBlocking(timeoutMs = 10_000): Promise<Session> {
const deadline = Date.now() + timeoutMs;
// Fast path
const fast = this.acquire();
if (fast) return fast;
// Wait-loop with backoff (50ms → 200ms)
let delay = 50;
while (Date.now() < deadline) {
await sleep(delay);
const session = this.acquire();
if (session) return session;
delay = Math.min(delay * 2, 200);
}
throw new Error(
`[SessionPool:${this.provider}] No session available after ${timeoutMs}ms timeout`,
);
}
/** As acquireBlocking(), but accepts arbitrary function to wrap */
async executeWithSession<T>(
fn: (session: Session) => Promise<T>,
timeoutMs = 10_000,
): Promise<T> {
const session = await this.acquireBlocking(timeoutMs);
try {
const result = await fn(session);
return result;
} finally {
session.release();
}
}
// ─── Internal ────────────────────────────────────────────────────────
/** Create and register a new session */
private createSession(): Session {
const session = this.factory.createSession();
this.sessions.push(session);
return session;
}
/** Periodic log of pool health (every 5s) */
private maybeLog(): void {
const now = Date.now();
if (now - this.lastLog < 5_000) return;
this.lastLog = now;
const stats = this.getStats();
if (stats.requests.total % 50 === 0) {
// Rate-limited to avoid log spam
}
}
/** Remove dead sessions and idle sessions older than maxIdleMs */
pruneDeadSessions(maxIdleMs = 300_000): void {
const now = Date.now();
const before = this.sessions.length;
this.sessions = this.sessions.filter((s) => {
if (s.status === "dead") return false;
// Prune idle sessions older than maxIdleMs (default 5min)
if (s.inflight === 0 && s.lastUsedAt > 0 && now - s.lastUsedAt > maxIdleMs) return false;
return true;
});
// If we pruned sessions, ensure minimum
if (this.sessions.length < before && this.sessions.length < this.config.minSessions) {
this.ensureMinSessions();
}
}
/** Start periodic pruning (every 60s) */
startAutoPrune(intervalMs = 60_000): ReturnType<typeof setInterval> {
const timer = setInterval(() => this.pruneDeadSessions(), intervalMs);
timer.unref();
return timer;
}
}
+100
View File
@@ -0,0 +1,100 @@
/**
* Session Pool — Type Definitions
*
* Core types for the anonymous session pool system:
* SessionPool → manages N sessions, each with a unique browser fingerprint
* Fingerprint → UA + headers for one browser-like identity
* Session → state machine tracking one session through its lifecycle
* PoolConfig → hot-reloadable pool parameters
* PoolStats → real-time pool health snapshot
*/
// ─── Fingerprint Types ─────────────────────────────────────────────────────
export interface Fingerprint {
id: string;
userAgent: string;
acceptLanguage: string;
secChUa?: string;
secChUaPlatform?: string;
secChUaMobile?: string;
}
export type FingerprintProfile = Fingerprint;
// ─── Session Types ─────────────────────────────────────────────────────────
export type SessionStatus = "active" | "cooldown" | "dead";
export interface SessionState {
id: string;
fingerprint: Fingerprint;
status: SessionStatus;
inflight: number;
totalRequests: number;
successfulRequests: number;
failedRequests: number;
consecutiveFails: number;
cooldownUntil: number;
createdAt: number;
lastUsedAt: number;
}
export interface SessionResult {
status: "ok" | "rate_limited" | "dead" | "error";
}
// ─── Pool Types ────────────────────────────────────────────────────────────
export interface PoolConfig {
minSessions: number;
maxSessions: number;
cooldownBase: number; // ms, default 1000
cooldownMax: number; // ms, default 30000
cooldownJitter: number; // ms, default 5000
requestTimeout: number; // ms, default 30000
requestJitter: number; // ms, default 50
}
export interface PoolStats {
provider: string;
sessions: {
total: number;
active: number;
cooldown: number;
dead: number;
};
requests: {
total: number;
success: number;
rate429: number;
otherErrors: number;
};
throughput: string; // req/s
successRate: string; // percentage
elapsed: string;
}
export interface PoolSessionDetail {
id: string;
fingerprint: string;
status: SessionStatus;
totalRequests: number;
successfulRequests: number;
successRate: string;
inflight: number;
cooldownRemaining: string;
age: string;
}
// ─── Defaults ──────────────────────────────────────────────────────────────
export const DEFAULT_POOL_CONFIG: PoolConfig = {
minSessions: 6,
maxSessions: 20,
cooldownBase: 1_000,
cooldownMax: 30_000,
cooldownJitter: 5_000,
requestTimeout: 30_000,
requestJitter: 50,
};
@@ -0,0 +1,132 @@
/**
* WebExecutorWrapper — Wraps any web executor with session pool support
*
* This is the integration bridge between the session pool and OmniRoute's
* executor system. It intercepts the fetch() call to add session-pool
* headers (fingerprint-based User-Agent, Sec-CH-UA, etc.) and handles
* 429/5xx responses with pool-level cooldown management.
*
* Future: For cookie-based providers (ChatGPT Web, DeepSeek Web, etc.)
* the wrapper will also inject cookies from the Playwright-authenticated
* session.
*/
import { Session } from "./session.ts";
import { SessionPool } from "./sessionPool.ts";
export interface WebExecutorRequest {
url: string;
method?: string;
headers?: Record<string, string>;
body?: string;
signal?: AbortSignal;
[key: string]: unknown;
}
export interface WebExecutorResponse {
status: number;
statusText?: string;
headers?: Record<string, string>;
body: string;
ok: boolean;
[key: string]: unknown;
}
export interface WebExecutorFn {
(req: WebExecutorRequest): Promise<WebExecutorResponse>;
}
/**
* Decorate a web executor function with session pool support.
*
* Before the underlying executor fires:
* 1. Acquires a session from the pool (reusable, fingerprint-isolated)
* 2. Merges session headers (UA + Sec-CH-UA) into the request
* 3. Handles 429 → pool cooldown, 5xx → session death
*
* For zero-auth providers like Pollinations, Puter, etc. this is all
* that's needed for "truly unlimited" — the fingerprint rotation alone
* defeats burst-based rate limiting.
*/
export function withSessionPool(
executor: WebExecutorFn,
pool: SessionPool,
options?: {
/** When true, wraps the response body for error handling */
wrapResponse?: boolean;
},
): WebExecutorFn {
const wrapResponse = options?.wrapResponse ?? true;
return async (req: WebExecutorRequest): Promise<WebExecutorResponse> => {
// Acquire session from pool (blocking with backoff)
let session: Session | null = null;
try {
session = await pool.acquireBlocking();
} catch (err) {
return {
status: 503,
statusText: "Service Unavailable",
body: JSON.stringify({
error: "session_pool_exhausted",
message: `[SessionPool:${pool.provider}] ${(err as Error).message}`,
}),
ok: false,
headers: {},
};
}
try {
// Build request with session fingerprint headers
const sessionHeaders = session.buildHeaders(req.headers);
const poolReq: WebExecutorRequest = {
...req,
headers: sessionHeaders,
};
// Execute the underlying web request
const res = await executor(poolReq);
// Handle response status
if (res.status === 429) {
pool.reportCooldown(session);
if (wrapResponse) {
return {
...res,
body: JSON.stringify({
error: "pool_rate_limited",
message: `[SessionPool:${pool.provider}] Rate limited, session ${session.id} in cooldown`,
}),
};
}
return res;
}
if (res.status >= 500) {
pool.reportDead(session);
return res;
}
// Success
pool.reportSuccess(session);
pool.totalRequests++;
return res;
} catch (err) {
// Network error — cooldown, not dead (may be transient)
pool.reportCooldown(session);
return {
status: 502,
statusText: "Bad Gateway",
body: JSON.stringify({
error: "pool_network_error",
message: (err as Error).message,
}),
ok: false,
headers: {},
};
} finally {
session.release();
}
};
}