/** * Node.js-only instrumentation logic. * * Separated from instrumentation.ts so that Turbopack's Edge bundler * does not trace into Node.js-only modules (fs, path, os, better-sqlite3, etc.) * and emit spurious "not supported in Edge Runtime" warnings. */ function getRandomBytes(byteLength: number): Uint8Array { const bytes = new Uint8Array(byteLength); globalThis.crypto.getRandomValues(bytes); return bytes; } function toBase64(bytes: Uint8Array): string { return btoa(String.fromCodePoint(...bytes)); } function toHex(bytes: Uint8Array): string { return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); } /** * Rename a Node process title so OmniRoute is identifiable in `ps`/`htop` * instead of the generic Next.js standalone server name. * * Only rewrites titles that start with "next-server", preserving any * trailing suffix (e.g. " (v16.2.9)"). Every other title — including one * that has already been renamed, or one that merely contains * "next-server" elsewhere — passes through unchanged. Empty/undefined-safe. */ export function renameProcessTitle(currentTitle: string): string { if (!currentTitle) return currentTitle; if (!currentTitle.startsWith("next-server")) return currentTitle; return `omniroute${currentTitle.slice("next-server".length)}`; } function isBackgroundServicesDisabled(): boolean { const raw = process.env.OMNIROUTE_DISABLE_BACKGROUND_SERVICES; if (!raw) return false; return new Set(["1", "true", "yes", "on"]).has(raw.trim().toLowerCase()); } async function ensureSecrets(): Promise { let getPersistedSecret = (_key: string): string | null => null; let persistSecret = (_key: string, _value: string): void => {}; try { ({ getPersistedSecret, persistSecret } = await import("@/lib/db/secrets")); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn( "[STARTUP] Secret persistence unavailable; falling back to process-local secrets:", msg ); } if (!process.env.JWT_SECRET || process.env.JWT_SECRET.trim() === "") { const persisted = getPersistedSecret("jwtSecret"); if (persisted) { process.env.JWT_SECRET = persisted; console.log("[STARTUP] JWT_SECRET restored from persistent store"); } else { const generated = toBase64(getRandomBytes(48)); process.env.JWT_SECRET = generated; persistSecret("jwtSecret", generated); console.log("[STARTUP] JWT_SECRET auto-generated and persisted (random 64-char secret)"); } } if (!process.env.API_KEY_SECRET || process.env.API_KEY_SECRET.trim() === "") { const persisted = getPersistedSecret("apiKeySecret"); if (persisted) { process.env.API_KEY_SECRET = persisted; } else { const generated = toHex(getRandomBytes(32)); process.env.API_KEY_SECRET = generated; persistSecret("apiKeySecret", generated); console.log( "[STARTUP] API_KEY_SECRET auto-generated and persisted (random 64-char hex secret)" ); } } } export async function registerNodejs(): Promise { // Rename the process title so OmniRoute is identifiable in ps/htop instead // of the generic "next-server" standalone server name. process.title = renameProcessTitle(process.title); // Initialize proxy fetch patch FIRST (before any HTTP requests) await import("@omniroute/open-sse/index.ts"); console.log("[STARTUP] Global fetch proxy patch initialized"); await ensureSecrets(); const { enforceWebRuntimeEnv } = await import("@/lib/env/runtimeEnv"); enforceWebRuntimeEnv(); // Trigger request-log layout migration during startup, before any request hits usageDb. await import("@/lib/usage/migrations"); const { initConsoleInterceptor } = await import("@/lib/consoleInterceptor"); initConsoleInterceptor(); // Clear stale transient connection cooldowns persisted from an unclean crash. // A crash mid-burst can leave far-future `rate_limited_until` values in the DB // that cause every connection to be skipped by getProviderCredentials(), making // all subsequent requests time out at Bottleneck's maxWaitMs (120 s default). // Terminal states (banned / expired / credits_exhausted) are intentionally kept. // See: https://github.com/diegosouzapw/OmniRoute/issues/3625 (Part A) try { const { clearStaleCrashCooldowns } = await import("@/lib/db/providers"); const { cleared } = clearStaleCrashCooldowns(); if (cleared > 0) { console.log( `[STARTUP] Cleared ${cleared} stale transient connection cooldown(s) from prior crash (#3625)` ); } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Could not clear stale crash cooldowns (non-fatal):", msg); } const [ { initGracefulShutdown }, { initApiBridgeServer }, { startBackgroundRefresh }, { ensureCloudSyncInitialized }, { startProviderLimitsSyncScheduler }, { getSettings }, { applyRuntimeSettings }, { startRuntimeConfigHotReload }, { startSpendBatchWriter }, { registerDefaultGuardrails }, { ensurePersistentManagementPasswordHash }, { skillExecutor }, { registerBuiltinSkills }, ] = await Promise.all([ import("@/lib/gracefulShutdown"), import("@/lib/apiBridgeServer"), import("@/domain/quotaCache"), import("@/lib/initCloudSync"), import("@/shared/services/providerLimitsSyncScheduler"), import("@/lib/db/settings"), import("@/lib/config/runtimeSettings"), import("@/lib/config/hotReload"), import("@/lib/spend/batchWriter"), import("@/lib/guardrails"), import("@/lib/auth/managementPassword"), import("@/lib/skills/executor"), import("@/lib/skills/builtins"), ]); // Proxy health scheduler (auto-removes dead proxies on interval) await import("@/lib/proxyHealth/scheduler"); initGracefulShutdown(); initApiBridgeServer(); startSpendBatchWriter(); registerDefaultGuardrails(); registerBuiltinSkills(skillExecutor); console.log("[STARTUP] Spend batch writer started"); console.log("[STARTUP] Guardrail registry initialized"); console.log("[STARTUP] Builtin skill handlers registered"); if (!isBackgroundServicesDisabled()) { startBackgroundRefresh(); console.log("[STARTUP] Quota cache background refresh started"); startProviderLimitsSyncScheduler(); console.log("[STARTUP] Provider limits sync scheduler started"); const cloudSyncInitialized = await ensureCloudSyncInitialized(); console.log( `[STARTUP] Cloud/model sync background bootstrap ${cloudSyncInitialized ? "initialized" : "skipped"}` ); const { initBatchProcessor } = await import("@omniroute/open-sse/services/batchProcessor"); initBatchProcessor(); console.log("[STARTUP] Batch processor started"); } try { const [ { migrateCodexConnectionDefaultsFromLegacySettings }, { startSessionAccountAffinityCleanup }, { seedDefaultModelAliases }, ] = await Promise.all([ import("@/lib/providers/codexConnectionDefaults"), import("@/lib/db/sessionAccountAffinity"), import("@/lib/modelAliasSeed"), ]); let settings = await getSettings(); const passwordState = await ensurePersistentManagementPasswordHash({ logger: console, settings, source: "startup", }); settings = passwordState.settings; const runtimeChanges = await applyRuntimeSettings(settings, { force: true, source: "startup" }); if (runtimeChanges.length > 0) { console.log( `[STARTUP] Runtime settings hydrated: ${runtimeChanges .map((entry) => entry.section) .join(", ")}` ); } // Restore Global System Prompt into in-memory config (#2468/#2470) if (settings.systemPrompt) { const { setSystemPromptConfig } = await import("@omniroute/open-sse/services/systemPrompt.ts"); setSystemPromptConfig(settings.systemPrompt); console.log("[STARTUP] Global System Prompt restored from settings"); } // Restore the proxy-level Thinking-Budget config (#5312 RC-A). It lives in // `settings.thinkingBudget` and is NOT covered by applyRuntimeSettings, so // without this the dashboard mode (auto/custom/adaptive) silently reverts to // the passthrough default on every restart. Previously this was only wired into // the unused `server-init.ts`, so it never ran in production. const { hydrateThinkingBudgetConfig } = await import( "@omniroute/open-sse/services/thinkingBudget.ts" ); if (hydrateThinkingBudgetConfig(settings)) { console.log("[STARTUP] Thinking-Budget config restored from settings"); } const seededModelAliases = await seedDefaultModelAliases(); console.log( `[STARTUP] Model alias seed: applied=${seededModelAliases.applied.length}, skipped=${seededModelAliases.skipped.length}, failed=${seededModelAliases.failed.length}` ); startSessionAccountAffinityCleanup(); const migration = await migrateCodexConnectionDefaultsFromLegacySettings(); if (migration.migrated) { console.log( `[STARTUP] Migrated Codex connection defaults for ${migration.updatedConnectionIds.length} connection(s)` ); if (settings.cloudEnabled === true) { const [{ syncToCloud }, { getConsistentMachineId }] = await Promise.all([ import("@/lib/cloudSync"), import("@/shared/utils/machineId"), ]); const machineId = await getConsistentMachineId(); await syncToCloud(machineId); console.log("[STARTUP] Synced migrated Codex connection defaults to cloud"); } } startRuntimeConfigHotReload(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Could not restore runtime settings:", msg); } try { const { initAuditLog, cleanupExpiredLogs } = await import("@/lib/compliance/index"); initAuditLog(); console.log("[COMPLIANCE] Audit log table initialized"); const cleanup = await cleanupExpiredLogs(); if ( cleanup.deletedUsage || cleanup.deletedCallLogs || cleanup.deletedProxyLogs || cleanup.deletedRequestDetailLogs || cleanup.deletedAuditLogs || cleanup.deletedMcpAuditLogs ) { console.log("[COMPLIANCE] Expired log cleanup:", cleanup); } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[COMPLIANCE] Could not initialize audit log:", msg); } await import("@/lib/db/core").then(({ ensureDbInitialized }) => ensureDbInitialized()); // Storage-configured scheduled VACUUM (#4437): registers the timer from // Settings > System & Storage and persists lastVacuumAt for the UI. try { const { initVacuumScheduler } = await import("@/lib/db/vacuumScheduler"); initVacuumScheduler(); console.log("[STARTUP] Scheduled VACUUM initialized (#4437)"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Could not initialize vacuum scheduler (non-fatal):", msg); } if (!isBackgroundServicesDisabled()) { try { const { bootstrapEmbeddedServices } = await import("@/lib/services/bootstrap"); await bootstrapEmbeddedServices(); console.log("[STARTUP] Embedded services bootstrap complete"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Embedded services bootstrap failed (non-fatal):", msg); } try { const { initEmbedWsProxy } = await import("@/lib/services/embedWsProxy"); initEmbedWsProxy(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Embed WS proxy failed to start (non-fatal):", msg); } try { const { autoRefreshDaemon } = await import("@omniroute/open-sse/services/autoRefreshDaemon"); autoRefreshDaemon.start(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Auto-refresh daemon failed to start (non-fatal):", msg); } // Proactive connection-cooldown recovery (#8): re-validate connections whose // transient `rate_limited_until` window has elapsed OUTSIDE the request hot // path, so the first request after a cooldown does not pay the probe latency. // Lazy/self-recovery still happens in getProviderCredentials; this front-runs it. try { const { initConnectionRecoveryScheduler } = await import("@/lib/quota/connectionRecovery"); initConnectionRecoveryScheduler(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Connection recovery scheduler failed to start (non-fatal):", msg); } try { // Arena ELO sync: model intelligence from the Arena AI leaderboard, powering the // Free Provider Rankings page. On by default; configurable from Dashboard Feature Flags. // Non-blocking — the initial sync is fire-and-forget and never fatal. const { initArenaEloSync } = await import("@/lib/arenaEloSync"); const started = await initArenaEloSync(); if (started) { console.log("[STARTUP] Arena ELO sync initialized"); } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Arena ELO sync failed to start (non-fatal):", msg); } // Pricing sync: opt-in external pricing data (self-gated by PRICING_SYNC_ENABLED inside // initPricingSync). Was only wired into the unused server-init.ts, so it never ran in the // standalone runtime even when enabled. Non-blocking, never fatal. try { const { initPricingSync } = await import("@/lib/pricingSync"); await initPricingSync(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Pricing sync failed to start (non-fatal):", msg); } // models.dev capability sync: opt-in via Settings > AI (self-gated by // settings.modelsDevSyncEnabled inside initModelsDevSync). Previously had no caller at all, // so the toggle was inert. Non-blocking, never fatal. try { const { initModelsDevSync } = await import("@/lib/modelsDevSync"); await initModelsDevSync(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] models.dev sync failed to start (non-fatal):", msg); } // Context-window self-correction (5004): periodically reconcile provider-declared // windows (from /models discovery) into auto:discovery overrides. Reuses already-synced // data (no new fetch); disable via CONTEXT_WINDOW_RECONCILE_INTERVAL=0. Never fatal. try { const { startContextWindowReconcile } = await import("@/lib/contextWindowResolver"); startContextWindowReconcile(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] context-window reconcile failed to start (non-fatal):", msg); } // TV6 typed memory decay: optional periodic sweep of decayed episodic memories. Doubly // opt-in (no-op unless MEMORY_TYPED_DECAY_ENABLED=true AND // MEMORY_TYPED_DECAY_SWEEP_INTERVAL>0). Never deletes by default. Never fatal. try { const { startMemoryDecaySweep } = await import("@/lib/memory/typedDecay"); startMemoryDecaySweep(); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] memory decay sweep failed to start (non-fatal):", msg); } // Real-time dashboard WebSocket daemon (port 20129): powers Combo Studio Live, // the Home live-pulse, and Live Compression. liveServer.ts auto-starts the // daemon on import (gated by OMNIROUTE_ENABLE_LIVE_WS, default ON) — but NOTHING // imported it in the packaged standalone/PM2 runtime. Only the unused // `server-init.ts` and a dev-only helper script (`scripts/start-ws-server.mjs`) // ever pulled it into a module graph, so in the published `omniroute` bin the // daemon never bound its port and every live dashboard reported "Live disabled — // WebSocket disconnected". Importing it here (the instrumentation hook that DOES // run in standalone) fires that flag-gated auto-start. Side-effect import + the // module's own `.catch` keep it non-fatal. try { await import("@/server/ws/liveServer"); console.log("[STARTUP] Live dashboard WebSocket daemon bootstrap invoked"); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.warn("[STARTUP] Live dashboard WebSocket daemon failed to start (non-fatal):", msg); } } }