chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,349 @@
|
||||
import { Worker } from "node:worker_threads";
|
||||
import path from "node:path";
|
||||
import {
|
||||
getMeter,
|
||||
type Counter,
|
||||
type Histogram,
|
||||
type Meter,
|
||||
type ObservableGauge,
|
||||
} from "@internal/tracing";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { signalsEmitter } from "~/services/signals.server";
|
||||
import { singleton } from "~/utils/singleton";
|
||||
|
||||
export type TransformKind = "traces" | "logs" | "metrics";
|
||||
|
||||
type TaskMessage = {
|
||||
id: number;
|
||||
kind: TransformKind;
|
||||
payload: Uint8Array;
|
||||
spanAttributeValueLengthLimit: number;
|
||||
defaultEventStore: string;
|
||||
};
|
||||
|
||||
type Task = {
|
||||
message: TaskMessage;
|
||||
transfer: ArrayBuffer[];
|
||||
resolve: (r: any) => void;
|
||||
reject: (e: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
worker?: Worker;
|
||||
// Wall-clock stamp at enqueue; the task-duration histogram measures enqueue -> terminal state
|
||||
// (queue wait + worker compute), so the gap from the worker-reported compute time is queue wait.
|
||||
enqueuedAt: number;
|
||||
};
|
||||
|
||||
type ReapReason = "error" | "exit" | "timeout";
|
||||
|
||||
const TASK_TIMEOUT_MS = 30_000;
|
||||
const MAX_QUEUE_DEPTH = 2_000;
|
||||
const RESPAWN_BASE_MS = 500;
|
||||
const RESPAWN_MAX_MS = 30_000;
|
||||
const SHUTDOWN_DRAIN_MS = 5_000;
|
||||
|
||||
// Hand-rolled worker_threads pool: one in-flight task per worker so CPU-bound transforms run
|
||||
// fully in parallel. The main thread stays the only DB reader and broadcasts pricing to workers.
|
||||
export class OtlpWorkerPool {
|
||||
private readonly workers: Worker[] = [];
|
||||
private readonly idle: Worker[] = [];
|
||||
private readonly queue: number[] = [];
|
||||
private readonly tasks = new Map<number, Task>();
|
||||
private readonly busyByWorker = new Map<Worker, number>();
|
||||
private nextId = 1;
|
||||
private consecutiveFailures = 0;
|
||||
private isShuttingDown = false;
|
||||
private latestPricingModels: unknown[];
|
||||
|
||||
// Pre-allocated per-kind {kind} attribute objects so the per-task record path never allocates.
|
||||
private readonly _kindAttrs: Record<TransformKind, { kind: TransformKind }> = {
|
||||
traces: { kind: "traces" },
|
||||
logs: { kind: "logs" },
|
||||
metrics: { kind: "metrics" },
|
||||
};
|
||||
private _taskDurationHistogram?: Histogram;
|
||||
private _computeDurationHistogram?: Histogram;
|
||||
private _tasksCounter?: Counter;
|
||||
private _respawnsCounter?: Counter;
|
||||
|
||||
constructor(
|
||||
private readonly size: number,
|
||||
private readonly workerPath: string,
|
||||
pricingModels: unknown[],
|
||||
meter?: Meter
|
||||
) {
|
||||
this.latestPricingModels = pricingModels;
|
||||
this.#setupOtelMetrics(meter);
|
||||
for (let i = 0; i < size; i++) this.spawn();
|
||||
logger.info("OtlpWorkerPool started", { size, workerPath });
|
||||
}
|
||||
|
||||
#setupOtelMetrics(meterOverride: Meter | undefined): void {
|
||||
const meter = meterOverride ?? getMeter("ingest");
|
||||
|
||||
this._taskDurationHistogram = meter.createHistogram("ingest.worker_pool.task.duration", {
|
||||
description: "Enqueue-to-completion time for a transform task (queue wait + worker compute)",
|
||||
unit: "ms",
|
||||
});
|
||||
this._computeDurationHistogram = meter.createHistogram("ingest.worker_pool.compute.duration", {
|
||||
description: "Worker-reported compute time (decode + convert + enrich)",
|
||||
unit: "ms",
|
||||
});
|
||||
this._tasksCounter = meter.createCounter("ingest.worker_pool.tasks", {
|
||||
description: "Transform tasks by terminal outcome",
|
||||
unit: "tasks",
|
||||
});
|
||||
this._respawnsCounter = meter.createCounter("ingest.worker_pool.respawns", {
|
||||
description: "Worker respawns by reason",
|
||||
unit: "respawns",
|
||||
});
|
||||
|
||||
// Pull-based gauges: read at export time only, zero hot-path cost.
|
||||
const queueDepthGauge: ObservableGauge = meter.createObservableGauge(
|
||||
"ingest.worker_pool.queue_depth",
|
||||
{ description: "Tasks queued and awaiting a free worker", unit: "tasks" }
|
||||
);
|
||||
const workersGauge: ObservableGauge = meter.createObservableGauge(
|
||||
"ingest.worker_pool.workers",
|
||||
{
|
||||
description: "Pool workers by state (alive workers, idle workers)",
|
||||
unit: "workers",
|
||||
}
|
||||
);
|
||||
|
||||
meter.addBatchObservableCallback(
|
||||
(result) => {
|
||||
result.observe(queueDepthGauge, this.queue.length);
|
||||
result.observe(workersGauge, this.workers.length, { state: "alive" });
|
||||
result.observe(workersGauge, this.idle.length, { state: "idle" });
|
||||
},
|
||||
[queueDepthGauge, workersGauge]
|
||||
);
|
||||
}
|
||||
|
||||
#recordTaskEnd(task: Task, outcome: string, computeMs?: number): void {
|
||||
this._taskDurationHistogram?.record(
|
||||
Date.now() - task.enqueuedAt,
|
||||
this._kindAttrs[task.message.kind]
|
||||
);
|
||||
this._tasksCounter?.add(1, { kind: task.message.kind, outcome });
|
||||
if (computeMs !== undefined) {
|
||||
this._computeDurationHistogram?.record(computeMs, this._kindAttrs[task.message.kind]);
|
||||
}
|
||||
}
|
||||
|
||||
private spawn() {
|
||||
const worker = new Worker(this.workerPath, {
|
||||
workerData: { pricingModels: this.latestPricingModels },
|
||||
});
|
||||
|
||||
worker.on(
|
||||
"message",
|
||||
(msg: { id: number; ok: boolean; result?: any; error?: string; computeMs?: number }) => {
|
||||
if (this.workers.indexOf(worker) === -1) return; // late message from an already-reaped worker
|
||||
this.consecutiveFailures = 0;
|
||||
this.busyByWorker.delete(worker);
|
||||
const task = this.tasks.get(msg.id);
|
||||
if (task) {
|
||||
clearTimeout(task.timer);
|
||||
this.tasks.delete(msg.id);
|
||||
if (msg.ok) {
|
||||
this.#recordTaskEnd(task, "ok", msg.computeMs);
|
||||
task.resolve(msg.result);
|
||||
} else {
|
||||
this.#recordTaskEnd(task, "error", msg.computeMs);
|
||||
task.reject(new Error(msg.error ?? "otlp worker error"));
|
||||
}
|
||||
}
|
||||
this.release(worker);
|
||||
}
|
||||
);
|
||||
|
||||
worker.on("error", (error) => {
|
||||
logger.error("OtlpWorkerPool worker error", { error: error.message });
|
||||
this.reap(worker, error, "error");
|
||||
});
|
||||
|
||||
worker.on("exit", (code) => {
|
||||
// Any exit means this worker is gone, including a clean exit while it held a task; reap()
|
||||
// no-ops if the worker was already removed (e.g. error fired first).
|
||||
this.reap(worker, new Error(`otlp worker exited with code ${code}`), "exit");
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
this.idle.push(worker);
|
||||
}
|
||||
|
||||
// On crash/timeout: fail the worker's in-flight task (if still pending), drop the worker, and
|
||||
// respawn with exponential backoff so a persistently failing worker can't tight-loop.
|
||||
private reap(worker: Worker, error: Error, reason: ReapReason) {
|
||||
const wi = this.workers.indexOf(worker);
|
||||
if (wi === -1) return; // already reaped (error + exit can both fire for one crash)
|
||||
this.workers.splice(wi, 1);
|
||||
|
||||
const ii = this.idle.indexOf(worker);
|
||||
if (ii !== -1) this.idle.splice(ii, 1);
|
||||
|
||||
const inFlightId = this.busyByWorker.get(worker);
|
||||
this.busyByWorker.delete(worker);
|
||||
if (inFlightId !== undefined) {
|
||||
const task = this.tasks.get(inFlightId);
|
||||
if (task) {
|
||||
clearTimeout(task.timer);
|
||||
this.tasks.delete(inFlightId);
|
||||
// A timed-out task already recorded its own end + was removed from the map, so this only
|
||||
// fires for a crash that killed a task mid-flight.
|
||||
this.#recordTaskEnd(task, "crash");
|
||||
task.reject(error);
|
||||
}
|
||||
}
|
||||
|
||||
this._respawnsCounter?.add(1, { reason });
|
||||
void worker.terminate().catch(() => {});
|
||||
this.scheduleRespawn();
|
||||
}
|
||||
|
||||
private scheduleRespawn() {
|
||||
if (this.isShuttingDown) return;
|
||||
if (this.workers.length >= this.size) return;
|
||||
const delay = Math.min(RESPAWN_BASE_MS * 2 ** this.consecutiveFailures, RESPAWN_MAX_MS);
|
||||
this.consecutiveFailures++;
|
||||
setTimeout(() => {
|
||||
if (this.isShuttingDown) return;
|
||||
if (this.workers.length < this.size) this.spawn();
|
||||
this.drain();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private release(worker: Worker) {
|
||||
this.idle.push(worker);
|
||||
this.drain();
|
||||
}
|
||||
|
||||
private drain() {
|
||||
while (this.queue.length > 0 && this.idle.length > 0) {
|
||||
const worker = this.idle.pop()!;
|
||||
const id = this.queue.shift()!;
|
||||
const task = this.tasks.get(id);
|
||||
if (!task) continue;
|
||||
task.worker = worker;
|
||||
this.busyByWorker.set(worker, id);
|
||||
worker.postMessage(task.message, task.transfer);
|
||||
}
|
||||
}
|
||||
|
||||
private onTimeout(id: number) {
|
||||
const task = this.tasks.get(id);
|
||||
if (!task) return;
|
||||
this.tasks.delete(id);
|
||||
this.#recordTaskEnd(task, "timeout");
|
||||
const err = new Error(`otlp worker task timed out after ${TASK_TIMEOUT_MS}ms`);
|
||||
if (task.worker) {
|
||||
// Dispatched to a stuck worker: reap it. The task is already removed, so reap won't
|
||||
// double-reject.
|
||||
this.reap(task.worker, err, "timeout");
|
||||
} else {
|
||||
const qi = this.queue.indexOf(id);
|
||||
if (qi !== -1) this.queue.splice(qi, 1);
|
||||
}
|
||||
task.reject(err);
|
||||
}
|
||||
|
||||
runTransform(
|
||||
kind: TransformKind,
|
||||
payload: Uint8Array,
|
||||
config: { spanAttributeValueLengthLimit: number; defaultEventStore: string }
|
||||
): Promise<any> {
|
||||
if (this.isShuttingDown) {
|
||||
return Promise.reject(new Error("otlp worker pool is shutting down"));
|
||||
}
|
||||
if (this.queue.length >= MAX_QUEUE_DEPTH) {
|
||||
this._tasksCounter?.add(1, { kind, outcome: "rejected" });
|
||||
return Promise.reject(new Error("otlp worker pool queue is full"));
|
||||
}
|
||||
const id = this.nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => this.onTimeout(id), TASK_TIMEOUT_MS);
|
||||
this.tasks.set(id, {
|
||||
message: {
|
||||
id,
|
||||
kind,
|
||||
payload,
|
||||
spanAttributeValueLengthLimit: config.spanAttributeValueLengthLimit,
|
||||
defaultEventStore: config.defaultEventStore,
|
||||
},
|
||||
// Zero-copy the payload into the worker; the request owns a fresh ArrayBuffer.
|
||||
transfer: [payload.buffer as ArrayBuffer],
|
||||
resolve,
|
||||
reject,
|
||||
timer,
|
||||
enqueuedAt: Date.now(),
|
||||
});
|
||||
this.queue.push(id);
|
||||
this.drain();
|
||||
});
|
||||
}
|
||||
|
||||
broadcastPricing(models: unknown[]) {
|
||||
this.latestPricingModels = models;
|
||||
for (const worker of this.workers) {
|
||||
worker.postMessage({ type: "pricing", models });
|
||||
}
|
||||
logger.info("OtlpWorkerPool broadcast pricing", {
|
||||
models: models.length,
|
||||
workers: this.workers.length,
|
||||
});
|
||||
}
|
||||
|
||||
get queueDepth() {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
// Stop taking new work, let in-flight tasks finish (bounded), then terminate every worker.
|
||||
// Terminated workers fire "exit", but reap() no-ops on an already-removed worker, and the
|
||||
// isShuttingDown guard stops any pending respawn, so shutdown is quiet.
|
||||
async shutdown(): Promise<void> {
|
||||
if (this.isShuttingDown) return;
|
||||
this.isShuttingDown = true;
|
||||
|
||||
logger.info("OtlpWorkerPool shutting down", {
|
||||
workers: this.workers.length,
|
||||
inFlight: this.tasks.size,
|
||||
});
|
||||
|
||||
const deadline = Date.now() + SHUTDOWN_DRAIN_MS;
|
||||
while (this.tasks.size > 0 && Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
|
||||
const workers = this.workers.splice(0);
|
||||
this.idle.length = 0;
|
||||
this.queue.length = 0;
|
||||
this.busyByWorker.clear();
|
||||
// Reject anything that didn't drain within the deadline.
|
||||
for (const [, task] of this.tasks) {
|
||||
clearTimeout(task.timer);
|
||||
task.reject(new Error("otlp worker pool shutting down"));
|
||||
}
|
||||
this.tasks.clear();
|
||||
await Promise.all(workers.map((worker) => worker.terminate().catch(() => {})));
|
||||
}
|
||||
}
|
||||
|
||||
export function getOtlpWorkerPool(
|
||||
size: number,
|
||||
pricingModels: unknown[],
|
||||
workerPath?: string,
|
||||
meter?: Meter
|
||||
): OtlpWorkerPool {
|
||||
// singleton() stores on globalThis so the pool (and its worker threads) survive Remix HMR in dev
|
||||
// rather than leaking an orphaned pool + workers on every reload.
|
||||
return singleton("otlpWorkerPool", () => {
|
||||
const resolvedPath = workerPath ?? path.join(process.cwd(), "build", "otlpTransformWorker.cjs");
|
||||
const created = new OtlpWorkerPool(size, resolvedPath, pricingModels, meter);
|
||||
// Drain + terminate workers on shutdown so they aren't force-killed mid-task (which would
|
||||
// churn respawns). The main thread stays the only DB writer, so inserts are unaffected.
|
||||
signalsEmitter.on("SIGTERM", () => void created.shutdown());
|
||||
signalsEmitter.on("SIGINT", () => void created.shutdown());
|
||||
return created;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user