chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,106 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { ModelPricingRegistry } from "@internal/llm-model-catalog";
+import type { LlmModelWithPricing } from "@internal/llm-model-catalog";
+import {
+  ExportLogsServiceRequest,
+  ExportMetricsServiceRequest,
+  ExportTraceServiceRequest,
+} from "@trigger.dev/otlp-importer";
+import {
+  convertLogsToCreateableEvents,
+  convertMetricsToClickhouseRows,
+  convertSpansToCreateableEvents,
+  filterResourceLogs,
+  filterResourceMetrics,
+  filterResourceSpans,
+} from "./otlpTransform.server";
+import { enrichCreatableEvents, setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server";
+
+type TransformTask = {
+  id: number;
+  kind: "traces" | "logs" | "metrics";
+  payload: Uint8Array;
+  spanAttributeValueLengthLimit: number;
+  defaultEventStore: string;
+};
+
+type PricingUpdate = { type: "pricing"; models: LlmModelWithPricing[] };
+
+// The main thread is the only DB reader; it broadcasts the compiled model rows here.
+const registry = new ModelPricingRegistry();
+setLlmPricingRegistry(registry);
+
+function applyPricing(models: LlmModelWithPricing[]) {
+  registry.loadFromModels(models);
+}
+
+if (Array.isArray(workerData?.pricingModels)) {
+  applyPricing(workerData.pricingModels);
+}
+
+function runTask(task: TransformTask) {
+  const bytes = new Uint8Array(task.payload);
+
+  if (task.kind === "traces") {
+    const request = ExportTraceServiceRequest.decode(bytes);
+    const eventsWithStores = filterResourceSpans(request.resourceSpans).flatMap((resourceSpan) =>
+      convertSpansToCreateableEvents(
+        resourceSpan,
+        task.spanAttributeValueLengthLimit,
+        task.defaultEventStore
+      )
+    );
+    for (const group of eventsWithStores) {
+      group.events = enrichCreatableEvents(group.events);
+    }
+    return { eventsWithStores };
+  }
+
+  if (task.kind === "logs") {
+    const request = ExportLogsServiceRequest.decode(bytes);
+    const eventsWithStores = filterResourceLogs(request.resourceLogs).flatMap((resourceLog) =>
+      convertLogsToCreateableEvents(
+        resourceLog,
+        task.spanAttributeValueLengthLimit,
+        task.defaultEventStore
+      )
+    );
+    for (const group of eventsWithStores) {
+      group.events = enrichCreatableEvents(group.events);
+    }
+    return { eventsWithStores };
+  }
+
+  const request = ExportMetricsServiceRequest.decode(bytes);
+  const rows = filterResourceMetrics(request.resourceMetrics).flatMap((resourceMetrics) =>
+    convertMetricsToClickhouseRows(resourceMetrics, task.spanAttributeValueLengthLimit)
+  );
+  return { rows };
+}
+
+if (!parentPort) {
+  throw new Error("otlpTransformWorker must be run as a worker thread");
+}
+
+parentPort.on("message", (message: TransformTask | PricingUpdate) => {
+  if ("type" in message && message.type === "pricing") {
+    applyPricing(message.models);
+    return;
+  }
+
+  const task = message as TransformTask;
+  try {
+    // The worker has no MeterProvider, so it can't emit metrics itself. It measures its own
+    // compute time (decode + convert + enrich) and hands it back for the main thread to record.
+    const startedAt = performance.now();
+    const result = runTask(task);
+    const computeMs = performance.now() - startedAt;
+    parentPort!.postMessage({ id: task.id, ok: true, result, computeMs });
+  } catch (error) {
+    parentPort!.postMessage({
+      id: task.id,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+});