import { spawn } from "node:child_process"; import { dirname } from "node:path"; import { writePidFile, cleanupPidFile, killAllSubprocesses } from "../utils/pid.mjs"; import { RESTART_RESET_MS, DEFAULT_MAX_RESTARTS, shouldExitInsteadOfRestart, computeRestartDelayMs, waitUntilPortFree, } from "./supervisorPolicy.mjs"; import { buildNodeHeapArgs } from "../../../scripts/build/runtime-env.mjs"; const CRASH_LOG_LINES = 50; export class ServerSupervisor { constructor({ serverPath, env, maxRestarts = DEFAULT_MAX_RESTARTS, memoryLimit = 512, onCrashCallback }) { this.serverPath = serverPath; this.env = env; this.maxRestarts = maxRestarts; this.memoryLimit = memoryLimit; this.onCrashCallback = onCrashCallback; this.restartCount = 0; this.startedAt = 0; this.crashLog = []; this.child = null; this.isShuttingDown = false; } start() { this.startedAt = Date.now(); this.crashLog = []; const showLog = process.env.OMNIROUTE_SHOW_LOG === "1"; // #5238: skip the explicit CLI --max-old-space-size when the user pinned the // heap via NODE_OPTIONS (a CLI arg would shadow/override their value). The // calibrated heap is already carried by env.NODE_OPTIONS either way. const heapArgs = buildNodeHeapArgs(process.env, this.memoryLimit); this.child = spawn("node", [...heapArgs, this.serverPath], { cwd: dirname(this.serverPath), env: this.env, stdio: showLog ? "inherit" : ["ignore", "ignore", "pipe"], }); writePidFile("server", this.child.pid); if (this.child.stderr) { this.child.stderr.on("data", (data) => { const lines = data.toString().split("\n").filter(Boolean); this.crashLog.push(...lines); if (this.crashLog.length > CRASH_LOG_LINES) { this.crashLog = this.crashLog.slice(-CRASH_LOG_LINES); } }); } this.child.on("error", (err) => this.handleExit(-1, err)); this.child.on("exit", (code) => this.handleExit(code)); return this.child; } handleExit(code) { // Node.js v24+ requires process.exit() to receive a number. Spawn-error events // deliver err.code (a string like 'ENOENT') via the 'error' listener; normalise here. const exitCode = typeof code === "number" ? code : null; cleanupPidFile("server"); // #4425: only exit on an intentional shutdown. A spontaneous code-0 exit (e.g. a // systemd MemoryMax cgroup kill, which reports the process exited cleanly) is anomalous // and must be restarted, not treated as a graceful stop that leaves the gateway dead. if (shouldExitInsteadOfRestart(this.isShuttingDown)) { process.exit(exitCode ?? 0); return; } const aliveMs = Date.now() - this.startedAt; if (aliveMs >= RESTART_RESET_MS) this.restartCount = 0; if (this.restartCount >= this.maxRestarts) { console.error(`\n⚠ Server crashed ${this.maxRestarts} times in <30s.`); if (this.onCrashCallback) { const action = this.onCrashCallback(this.crashLog); if (action === "disable-mitm-and-retry") { console.error("⚠ Disabling MITM and retrying...\n"); this.restartCount = 0; this.start(); return; } } this.dumpCrashLog(); process.exit(exitCode ?? 1); return; } this.restartCount++; const delay = computeRestartDelayMs(this.restartCount); console.error( `\n⚠ Server exited (code=${code ?? "?"}). Restarting in ${delay / 1000}s... (${this.restartCount}/${this.maxRestarts})` ); if (this.crashLog.length) this.dumpCrashLog(); // #4425: after a crash the OS may not have released the listen socket yet — restarting // immediately produced the EADDRINUSE cascade that exhausted the restart budget. Wait // (bounded) for the port to free up before respawning. setTimeout(async () => { await waitUntilPortFree(process.env.PORT || 20128); this.start(); }, delay); } dumpCrashLog() { console.error("\n--- Server crash log ---"); this.crashLog.forEach((l) => console.error(l)); console.error("--- End crash log ---\n"); } stop() { this.isShuttingDown = true; if (this.child?.pid) { try { process.kill(this.child.pid, "SIGTERM"); } catch {} setTimeout(() => { try { process.kill(this.child.pid, "SIGKILL"); } catch {} }, 5000); } killAllSubprocesses(); } } export function detectMitmCrash(crashLog) { const text = crashLog.join("\n").toLowerCase(); const signals = ["mitm", "tls socket", "certificate", "hosts", "eaccess"]; return signals.filter((s) => text.includes(s)).length >= 2; }