chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { dirname } from "node:path";
|
||||
import { writePidFile, cleanupPidFile, killAllSubprocesses } from "../utils/pid.mjs";
|
||||
import {
|
||||
RESTART_RESET_MS,
|
||||
DEFAULT_MAX_RESTARTS,
|
||||
shouldExitInsteadOfRestart,
|
||||
computeRestartDelayMs,
|
||||
waitUntilPortFree,
|
||||
} from "./supervisorPolicy.mjs";
|
||||
import { buildNodeHeapArgs } from "../../../scripts/build/runtime-env.mjs";
|
||||
|
||||
const CRASH_LOG_LINES = 50;
|
||||
|
||||
export class ServerSupervisor {
|
||||
constructor({ serverPath, env, maxRestarts = DEFAULT_MAX_RESTARTS, memoryLimit = 512, onCrashCallback }) {
|
||||
this.serverPath = serverPath;
|
||||
this.env = env;
|
||||
this.maxRestarts = maxRestarts;
|
||||
this.memoryLimit = memoryLimit;
|
||||
this.onCrashCallback = onCrashCallback;
|
||||
this.restartCount = 0;
|
||||
this.startedAt = 0;
|
||||
this.crashLog = [];
|
||||
this.child = null;
|
||||
this.isShuttingDown = false;
|
||||
}
|
||||
|
||||
start() {
|
||||
this.startedAt = Date.now();
|
||||
this.crashLog = [];
|
||||
|
||||
const showLog = process.env.OMNIROUTE_SHOW_LOG === "1";
|
||||
// #5238: skip the explicit CLI --max-old-space-size when the user pinned the
|
||||
// heap via NODE_OPTIONS (a CLI arg would shadow/override their value). The
|
||||
// calibrated heap is already carried by env.NODE_OPTIONS either way.
|
||||
const heapArgs = buildNodeHeapArgs(process.env, this.memoryLimit);
|
||||
this.child = spawn("node", [...heapArgs, this.serverPath], {
|
||||
cwd: dirname(this.serverPath),
|
||||
env: this.env,
|
||||
stdio: showLog ? "inherit" : ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
|
||||
writePidFile("server", this.child.pid);
|
||||
|
||||
if (this.child.stderr) {
|
||||
this.child.stderr.on("data", (data) => {
|
||||
const lines = data.toString().split("\n").filter(Boolean);
|
||||
this.crashLog.push(...lines);
|
||||
if (this.crashLog.length > CRASH_LOG_LINES) {
|
||||
this.crashLog = this.crashLog.slice(-CRASH_LOG_LINES);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
this.child.on("error", (err) => this.handleExit(-1, err));
|
||||
this.child.on("exit", (code) => this.handleExit(code));
|
||||
|
||||
return this.child;
|
||||
}
|
||||
|
||||
handleExit(code) {
|
||||
// Node.js v24+ requires process.exit() to receive a number. Spawn-error events
|
||||
// deliver err.code (a string like 'ENOENT') via the 'error' listener; normalise here.
|
||||
const exitCode = typeof code === "number" ? code : null;
|
||||
cleanupPidFile("server");
|
||||
|
||||
// #4425: only exit on an intentional shutdown. A spontaneous code-0 exit (e.g. a
|
||||
// systemd MemoryMax cgroup kill, which reports the process exited cleanly) is anomalous
|
||||
// and must be restarted, not treated as a graceful stop that leaves the gateway dead.
|
||||
if (shouldExitInsteadOfRestart(this.isShuttingDown)) {
|
||||
process.exit(exitCode ?? 0);
|
||||
return;
|
||||
}
|
||||
|
||||
const aliveMs = Date.now() - this.startedAt;
|
||||
if (aliveMs >= RESTART_RESET_MS) this.restartCount = 0;
|
||||
|
||||
if (this.restartCount >= this.maxRestarts) {
|
||||
console.error(`\n⚠ Server crashed ${this.maxRestarts} times in <30s.`);
|
||||
if (this.onCrashCallback) {
|
||||
const action = this.onCrashCallback(this.crashLog);
|
||||
if (action === "disable-mitm-and-retry") {
|
||||
console.error("⚠ Disabling MITM and retrying...\n");
|
||||
this.restartCount = 0;
|
||||
this.start();
|
||||
return;
|
||||
}
|
||||
}
|
||||
this.dumpCrashLog();
|
||||
process.exit(exitCode ?? 1);
|
||||
return;
|
||||
}
|
||||
|
||||
this.restartCount++;
|
||||
const delay = computeRestartDelayMs(this.restartCount);
|
||||
console.error(
|
||||
`\n⚠ Server exited (code=${code ?? "?"}). Restarting in ${delay / 1000}s... (${this.restartCount}/${this.maxRestarts})`
|
||||
);
|
||||
if (this.crashLog.length) this.dumpCrashLog();
|
||||
// #4425: after a crash the OS may not have released the listen socket yet — restarting
|
||||
// immediately produced the EADDRINUSE cascade that exhausted the restart budget. Wait
|
||||
// (bounded) for the port to free up before respawning.
|
||||
setTimeout(async () => {
|
||||
await waitUntilPortFree(process.env.PORT || 20128);
|
||||
this.start();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
dumpCrashLog() {
|
||||
console.error("\n--- Server crash log ---");
|
||||
this.crashLog.forEach((l) => console.error(l));
|
||||
console.error("--- End crash log ---\n");
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.isShuttingDown = true;
|
||||
if (this.child?.pid) {
|
||||
try {
|
||||
process.kill(this.child.pid, "SIGTERM");
|
||||
} catch {}
|
||||
setTimeout(() => {
|
||||
try {
|
||||
process.kill(this.child.pid, "SIGKILL");
|
||||
} catch {}
|
||||
}, 5000);
|
||||
}
|
||||
killAllSubprocesses();
|
||||
}
|
||||
}
|
||||
|
||||
export function detectMitmCrash(crashLog) {
|
||||
const text = crashLog.join("\n").toLowerCase();
|
||||
const signals = ["mitm", "tls socket", "certificate", "hosts", "eaccess"];
|
||||
return signals.filter((s) => text.includes(s)).length >= 2;
|
||||
}
|
||||
Reference in New Issue
Block a user