chore: import upstream snapshot with attribution

apps/supervisor/.env.example

@@ -0,0 +1,20 @@
+# This needs to match the token of the worker group you want to connect to
+TRIGGER_WORKER_TOKEN=
+
+# This needs to match the MANAGED_WORKER_SECRET env var on the webapp
+MANAGED_WORKER_SECRET=managed-secret
+
+# Point this at the webapp in prod
+TRIGGER_API_URL=http://localhost:3030
+
+# Point this at the webapp or an OTel collector in prod
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3030/otel
+# Use this on macOS
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:3030/otel
+
+# Optional settings
+DEBUG=1
+TRIGGER_DEQUEUE_INTERVAL_MS=1000
+
+# Dev-only: stream this process's logs over a local telnet/TCP socket (nc localhost 6769). Uncomment to enable.
+# SUPERVISOR_TELNET_LOGS_PORT=6769

apps/supervisor/.nvmrc

@@ -0,0 +1 @@
+v22.23.1

apps/supervisor/CLAUDE.md

@@ -0,0 +1,20 @@
+# Supervisor
+
+Node.js app that manages task execution containers. Receives work from the platform, starts Docker/Kubernetes containers, monitors execution, and reports results.
+
+## Key Directories
+
+- `src/services/` - Core service logic
+- `src/workloadManager/` - Container orchestration abstraction (Docker or Kubernetes)
+- `src/workloadServer/` - HTTP server for workload communication (heartbeats, snapshots)
+- `src/clients/` - Platform communication (webapp/coordinator)
+- `src/env.ts` - Environment configuration
+
+## Architecture
+
+- **WorkloadManager**: Abstracts Docker vs Kubernetes execution
+- **SupervisorSession**: Manages the dequeue loop with EWMA-based dynamic scaling
+- **ResourceMonitor**: Tracks CPU/memory during execution
+- **PodCleaner/FailedPodHandler**: Kubernetes-specific cleanup
+
+Communicates with the platform via Socket.io and HTTP. Receives task assignments through the dequeue protocol from the webapp.

apps/supervisor/Containerfile

@@ -0,0 +1,55 @@
+FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
+
+WORKDIR /app
+
+FROM node-22-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@2.10.0 prune --scope=supervisor --docker
+
+FROM node-22-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+RUN corepack enable && corepack prepare pnpm@10.33.2 --activate
+
+FROM base AS deps-fetcher
+RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile
+
+FROM deps-fetcher AS dev-deps
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --offline --ignore-scripts
+
+FROM base AS builder
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+COPY --chown=node:node .configs/tsconfig.base.json .configs/tsconfig.base.json
+COPY --chown=node:node scripts/updateVersion.ts scripts/updateVersion.ts
+COPY --chown=node:node scripts/retry-prisma-generate.mjs scripts/retry-prisma-generate.mjs
+
+RUN pnpm run generate && \
+  pnpm run --filter supervisor... build&& \
+  pnpm deploy --legacy --filter=supervisor --prod /prod/supervisor
+
+FROM base AS runner
+
+ENV NODE_ENV production
+
+COPY --from=builder /prod/supervisor /app/apps/supervisor
+
+EXPOSE 8000
+USER node
+
+# ensure pnpm is installed during build and not silently downloaded at runtime
+RUN pnpm -v
+
+CMD [ "/usr/bin/dumb-init", "--", "pnpm", "run", "--filter", "supervisor", "start"]

apps/supervisor/README.md

@@ -0,0 +1,105 @@
+# Supervisor
+
+## Dev setup
+
+1. Create a worker group
+
+```sh
+api_url=http://localhost:3030
+wg_name=my-worker
+
+# edit this
+admin_pat=tr_pat_...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"name\": \"$wg_name\"}"
+```
+
+If the worker group is newly created, the response will include a `token` field. If the group already exists, no token is returned.
+
+2. Create `.env` and set the worker token
+
+```sh
+cp .env.example .env
+
+# Then edit your .env and set this to the token.plaintext value
+TRIGGER_WORKER_TOKEN=tr_wgt_...
+```
+
+3. Start the supervisor
+
+```sh
+pnpm dev
+```
+
+4. Build CLI, then deploy a test project
+
+```sh
+pnpm exec trigger deploy --self-hosted
+
+# The additional network flag is required on linux
+pnpm exec trigger deploy --self-hosted --network host
+```
+
+## Worker group management
+
+### Shared variables
+
+```sh
+api_url=http://localhost:3030
+admin_pat=tr_pat_... # edit this
+```
+
+- These are used by all commands
+
+### Create a worker group
+
+```sh
+wg_name=my-worker
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"name\": \"$wg_name\"}"
+```
+
+- If the worker group already exists, no token will be returned
+
+### Set a worker group as default for a project
+
+```sh
+wg_name=my-worker
+project_id=clsw6q8wz...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"name\": \"$wg_name\", \"projectId\": \"$project_id\", \"makeDefaultForProject\": true}"
+```
+
+- If the worker group doesn't exist, yet it will be created
+- If the worker group already exists, it will be attached to the project as default. No token will be returned.
+
+### Remove the default worker group from a project
+
+```sh
+project_id=clsw6q8wz...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"projectId\": \"$project_id\", \"removeDefaultFromProject\": true}"
+```
+
+- The project will then use the global default again
+- When `removeDefaultFromProject: true` no other actions will be performed

apps/supervisor/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "supervisor",
+  "private": true,
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx --require dotenv/config --watch src/index.ts || (echo '!! Remember to run: nvm use'; exit 1)",
+    "start": "node dist/index.js",
+    "test:run": "vitest --no-file-parallelism --run",
+    "test:watch": "vitest --no-file-parallelism",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@aws-sdk/client-ecr": "^3.839.0",
+    "@internal/compute": "workspace:*",
+    "@kubernetes/client-node": "^1.0.0",
+    "@trigger.dev/core": "workspace:*",
+    "dockerode": "^4.0.6",
+    "ioredis": "~5.6.0",
+    "p-limit": "^6.2.0",
+    "prom-client": "^15.1.0",
+    "socket.io": "4.7.4",
+    "std-env": "^3.8.0",
+    "zod": "3.25.76"
+  },
+  "devDependencies": {
+    "@internal/testcontainers": "workspace:*",
+    "@types/dockerode": "^3.3.33"
+  }
+}

apps/supervisor/src/backpressure/backpressureMetrics.ts

@@ -0,0 +1,34 @@
+import { Counter, Gauge, type Registry } from "prom-client";
+
+/** Prometheus metrics for dequeue backpressure. */
+export class BackpressureMetrics {
+  /** 1 while backpressure is engaged (computed signal, set even in dry-run). */
+  readonly engaged: Gauge<string>;
+  /** 1 when running in dry-run (gates inert). */
+  readonly dryRun: Gauge<string>;
+  /** Dequeue attempts the gate skipped - or would have, in dry-run (labelled). */
+  readonly skipsTotal: Counter<string>;
+
+  constructor(opts: { register: Registry; prefix?: string }) {
+    const prefix = opts.prefix ?? "supervisor_backpressure";
+
+    this.engaged = new Gauge({
+      name: `${prefix}_engaged`,
+      help: "1 while dequeue backpressure is engaged (computed signal, regardless of dry-run)",
+      registers: [opts.register],
+    });
+
+    this.dryRun = new Gauge({
+      name: `${prefix}_dry_run`,
+      help: "1 when dequeue backpressure is in dry-run mode (gates inert)",
+      registers: [opts.register],
+    });
+
+    this.skipsTotal = new Counter({
+      name: `${prefix}_skipped_dequeues_total`,
+      help: "Dequeue attempts skipped by backpressure (or would be, in dry-run)",
+      labelNames: ["dry_run"],
+      registers: [opts.register],
+    });
+  }
+}

apps/supervisor/src/backpressure/backpressureMonitor.test.ts

@@ -0,0 +1,353 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { Registry } from "prom-client";
+import { BackpressureMonitor, type BackpressureSignalSource } from "./backpressureMonitor.js";
+import { BackpressureMetrics } from "./backpressureMetrics.js";
+
+function countingSource(verdict: { engaged: boolean } | null): {
+  source: BackpressureSignalSource;
+  reads: () => number;
+} {
+  let reads = 0;
+  return {
+    source: {
+      read: async () => {
+        reads++;
+        return verdict;
+      },
+    },
+    reads: () => reads,
+  };
+}
+
+describe("BackpressureMonitor", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("when disabled, never skips dequeue and never reads the signal source", () => {
+    // Even though the source would report "engaged", a disabled monitor must be
+    // a complete no-op: this is the backwards-compatibility guarantee.
+    const { source, reads } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({ enabled: false, source });
+
+    monitor.start();
+
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+    expect(reads()).toBe(0);
+
+    monitor.stop();
+  });
+
+  it("when enabled and the source reports engaged, skips dequeue after a refresh", async () => {
+    const { source } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0); // flush the initial async read
+
+    expect(monitor.shouldSkipDequeue()).toBe(true);
+
+    monitor.stop();
+  });
+
+  it("when enabled and the source reports clear, does not skip dequeue", async () => {
+    const { source } = countingSource({ engaged: false });
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("fails open (stops skipping) when the source throws", async () => {
+    let call = 0;
+    const source: BackpressureSignalSource = {
+      read: async () => {
+        call++;
+        if (call === 1) {
+          return { engaged: true };
+        }
+        throw new Error("signal source unreachable");
+      },
+    };
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(monitor.shouldSkipDequeue()).toBe(true); // engaged from the first read
+
+    await vi.advanceTimersByTimeAsync(1000); // next refresh throws
+    expect(monitor.shouldSkipDequeue()).toBe(false); // fail-open: a dead source must not pin the brake
+
+    monitor.stop();
+  });
+
+  it("fails open when the source reports unknown (null)", async () => {
+    const { source } = countingSource(null);
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("fails open when the cached verdict goes stale (older than max age)", async () => {
+    // Source stops updating (e.g. hangs) after the first read; the verdict ages out.
+    const source: BackpressureSignalSource = {
+      read: async () => ({ engaged: true, ts: Date.now() }),
+    };
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1_000_000, // effectively only the initial read fires
+      maxVerdictAgeMs: 15_000,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(monitor.shouldSkipDequeue()).toBe(true);
+
+    await vi.advanceTimersByTimeAsync(15_001); // verdict now older than max age
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("does not read the source on the hot path (reads are driven by the refresh tick)", async () => {
+    const { source, reads } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(reads()).toBe(1); // just the initial refresh
+
+    for (let i = 0; i < 1000; i++) {
+      monitor.shouldSkipDequeue();
+    }
+
+    expect(reads()).toBe(1); // hot-path calls performed zero I/O
+
+    monitor.stop();
+  });
+
+  it("does not start an overlapping refresh while one is in flight", async () => {
+    let reads = 0;
+    const source: BackpressureSignalSource = {
+      // Never resolves - simulates a hung read.
+      read: () => {
+        reads++;
+        return new Promise<{ engaged: boolean } | null>(() => {});
+      },
+    };
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(3000); // several intervals while the first read hangs
+
+    expect(reads).toBe(1); // in-flight guard prevents stacking
+
+    monitor.stop();
+  });
+
+  it("stops refreshing after stop()", async () => {
+    const { source, reads } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    const readsAtStop = reads();
+
+    monitor.stop();
+    await vi.advanceTimersByTimeAsync(5000);
+
+    expect(reads()).toBe(readsAtStop);
+  });
+
+  it("isEngaged reflects the hard engaged state (the signal for freezing scale-up)", async () => {
+    const { source } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(monitor.isEngaged()).toBe(true);
+
+    monitor.stop();
+  });
+
+  it("isEngaged is false when clear and when stale", async () => {
+    const source: BackpressureSignalSource = {
+      read: async () => ({ engaged: true, ts: Date.now() }),
+    };
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1_000_000,
+      maxVerdictAgeMs: 15_000,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(monitor.isEngaged()).toBe(true);
+
+    await vi.advanceTimersByTimeAsync(15_001); // stale → fail-open
+    expect(monitor.isEngaged()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("ramps the dequeue gate after release instead of resuming instantly", async () => {
+    let engaged = true;
+    let rnd = 0.5;
+    const source: BackpressureSignalSource = { read: async () => ({ engaged }) };
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1000,
+      rampMs: 10_000,
+      random: () => rnd,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(monitor.shouldSkipDequeue()).toBe(true); // hard engaged
+
+    // Release: the next refresh observes the clear verdict and starts the ramp.
+    engaged = false;
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(monitor.isEngaged()).toBe(false);
+
+    // Just after release (progress ~0): skip probability ~1, so skip regardless.
+    rnd = 0.99;
+    expect(monitor.shouldSkipDequeue()).toBe(true);
+
+    // Halfway through the ramp (progress 0.5): skip probability 0.5.
+    await vi.advanceTimersByTimeAsync(5000);
+    rnd = 0.4;
+    expect(monitor.shouldSkipDequeue()).toBe(true); // 0.4 < 0.5 → skip
+    rnd = 0.6;
+    expect(monitor.shouldSkipDequeue()).toBe(false); // 0.6 ≥ 0.5 → allow
+
+    // Past the ramp window: never skip.
+    await vi.advanceTimersByTimeAsync(5000);
+    rnd = 0.0;
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("fails open on an engaged verdict with no timestamp when staleness is enforced", async () => {
+    // A verdict claiming engaged but carrying no ts can't be checked for freshness;
+    // when maxVerdictAgeMs is set we must not trust it (else a dead producer could
+    // pin the brake forever).
+    const { source } = countingSource({ engaged: true }); // no ts
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1000,
+      maxVerdictAgeMs: 15_000,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(monitor.computeEngaged()).toBe(false);
+    expect(monitor.shouldSkipDequeue()).toBe(false);
+
+    monitor.stop();
+  });
+
+  it("in dry-run, the gates are inert but computeEngaged still reflects the real signal", async () => {
+    const { source } = countingSource({ engaged: true });
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1000,
+      dryRun: true,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(monitor.computeEngaged()).toBe(true); // real signal, for observability/metrics
+    expect(monitor.isEngaged()).toBe(false); // inert: no scale-up freeze
+    expect(monitor.shouldSkipDequeue()).toBe(false); // inert: no dequeue skip
+
+    monitor.stop();
+  });
+
+  it("logs on verdict transitions", async () => {
+    let engaged = true;
+    const source: BackpressureSignalSource = { read: async () => ({ engaged }) };
+    const logs: Array<{ message: string; meta?: Record<string, unknown> }> = [];
+    const logger = {
+      info: (message: string, meta?: Record<string, unknown>) => logs.push({ message, meta }),
+    };
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1000,
+      logger,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(logs.some((l) => l.meta?.engaged === true)).toBe(true);
+
+    engaged = false;
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(logs.some((l) => l.meta?.engaged === false)).toBe(true);
+
+    monitor.stop();
+  });
+
+  it("records prometheus metrics", async () => {
+    const { source } = countingSource({ engaged: true });
+    const register = new Registry();
+    const metrics = new BackpressureMetrics({ register });
+    const monitor = new BackpressureMonitor({
+      enabled: true,
+      source,
+      refreshIntervalMs: 1000,
+      metrics,
+    });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(await register.metrics()).toContain("supervisor_backpressure_engaged 1");
+
+    monitor.shouldSkipDequeue();
+    expect(await register.metrics()).toMatch(
+      /supervisor_backpressure_skipped_dequeues_total\{dry_run="false"\} [1-9]/
+    );
+
+    monitor.stop();
+  });
+
+  it("resumes instantly when no ramp is configured", async () => {
+    let engaged = true;
+    const source: BackpressureSignalSource = { read: async () => ({ engaged }) };
+    const monitor = new BackpressureMonitor({ enabled: true, source, refreshIntervalMs: 1000 });
+
+    monitor.start();
+    await vi.advanceTimersByTimeAsync(0);
+    expect(monitor.shouldSkipDequeue()).toBe(true);
+
+    engaged = false;
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(monitor.shouldSkipDequeue()).toBe(false); // no ramp → instant resume
+
+    monitor.stop();
+  });
+});

apps/supervisor/src/backpressure/backpressureMonitor.ts

@@ -0,0 +1,179 @@
+import type { BackpressureMetrics } from "./backpressureMetrics.js";
+
+export interface BackpressureLogger {
+  info(message: string, meta?: Record<string, unknown>): void;
+}
+
+export type BackpressureVerdict = {
+  engaged: boolean;
+  /** Epoch ms the verdict was produced. Used for consumer-side staleness fail-open. */
+  ts?: number;
+};
+
+/**
+ * Source of the current backpressure verdict. `read()` returns `null` when the
+ * verdict is unknown (missing/unreadable) - the monitor treats unknown as
+ * "not engaged" (fail-open).
+ */
+export interface BackpressureSignalSource {
+  read(): Promise<BackpressureVerdict | null>;
+}
+
+export type BackpressureMonitorOptions = {
+  enabled: boolean;
+  source: BackpressureSignalSource;
+  refreshIntervalMs?: number;
+  /**
+   * If set, a cached verdict older than this is treated as unknown (fail-open).
+   * Guards against the source silently going stale (e.g. hanging reads).
+   */
+  maxVerdictAgeMs?: number;
+  /**
+   * If set, after backpressure releases the dequeue gate stays partially engaged
+   * for this long, skipping a linearly-decaying fraction of attempts so the
+   * aggregate dequeue rate ramps from ~0 to full instead of snapping to full and
+   * re-flooding a freshly-recovered cluster. 0/unset = instant resume.
+   */
+  rampMs?: number;
+  /** Injectable RNG for the resume ramp; defaults to Math.random. */
+  random?: () => number;
+  /**
+   * When true, the gates are inert (never skip dequeues, never freeze scale-up).
+   * computeEngaged() still reflects the real signal so it can be observed.
+   */
+  dryRun?: boolean;
+  logger?: BackpressureLogger;
+  metrics?: BackpressureMetrics;
+};
+
+const DEFAULT_REFRESH_INTERVAL_MS = 1000;
+
+export class BackpressureMonitor {
+  private verdict: BackpressureVerdict | null = null;
+  private timer?: ReturnType<typeof setInterval>;
+  private refreshInFlight = false;
+  private wasEngaged = false;
+  private releasedAt?: number;
+
+  constructor(private readonly opts: BackpressureMonitorOptions) {
+    this.opts.metrics?.dryRun.set(this.opts.dryRun ? 1 : 0);
+  }
+
+  start(): void {
+    if (!this.opts.enabled) {
+      return;
+    }
+
+    void this.refreshTick();
+    this.timer = setInterval(
+      () => void this.refreshTick(),
+      this.opts.refreshIntervalMs ?? DEFAULT_REFRESH_INTERVAL_MS
+    );
+  }
+
+  /** Skip a tick if the previous refresh is still in flight, so slow/hung reads can't stack. */
+  private async refreshTick(): Promise<void> {
+    if (this.refreshInFlight) {
+      return;
+    }
+    this.refreshInFlight = true;
+    try {
+      await this.refresh();
+    } finally {
+      this.refreshInFlight = false;
+    }
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+  }
+
+  /**
+   * Raw hard backpressure state: true while the (fresh) verdict says engaged,
+   * ignoring dry-run. Used for observability/metrics so the real signal is
+   * visible even when the gates are inert.
+   */
+  computeEngaged(): boolean {
+    const verdict = this.verdict;
+    if (verdict?.engaged !== true) {
+      return false;
+    }
+
+    // When staleness enforcement is on, an engaged verdict must carry a fresh
+    // timestamp. A missing or stale ts can't be trusted (a dead producer could
+    // otherwise pin the brake forever), so fail open.
+    const maxAge = this.opts.maxVerdictAgeMs;
+    if (maxAge !== undefined) {
+      if (verdict.ts === undefined || Date.now() - verdict.ts > maxAge) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Effective hard state: the signal for freezing consumer-pool scale-up. Inert
+   * (false) in dry-run. Hot-path read, no I/O.
+   */
+  isEngaged(): boolean {
+    return this.opts.dryRun ? false : this.computeEngaged();
+  }
+
+  /** Hot-path read: synchronous, never performs I/O. Inert (false) in dry-run. */
+  shouldSkipDequeue(): boolean {
+    const wouldSkip = this.computeShouldSkip();
+    if (wouldSkip) {
+      this.opts.metrics?.skipsTotal.inc({ dry_run: this.opts.dryRun ? "true" : "false" });
+    }
+    return this.opts.dryRun ? false : wouldSkip;
+  }
+
+  private computeShouldSkip(): boolean {
+    if (this.computeEngaged()) {
+      return true;
+    }
+
+    // Post-release ramp: skip a linearly-decaying fraction of attempts so the
+    // aggregate dequeue rate climbs back to full over rampMs rather than snapping.
+    const rampMs = this.opts.rampMs;
+    if (rampMs && this.releasedAt !== undefined) {
+      const elapsed = Date.now() - this.releasedAt;
+      if (elapsed < rampMs) {
+        const skipProbability = 1 - elapsed / rampMs;
+        return (this.opts.random ?? Math.random)() < skipProbability;
+      }
+    }
+
+    return false;
+  }
+
+  private async refresh(): Promise<void> {
+    try {
+      this.verdict = await this.opts.source.read();
+    } catch {
+      // Fail-open: a dead/unreachable source must never pin the brake. Treat as
+      // unknown (no verdict) so dequeue resumes as if backpressure were off.
+      this.verdict = null;
+    }
+
+    // Track the engaged→released transition to anchor the resume ramp. Use the
+    // staleness-aware state so a stale verdict doesn't pin wasEngaged / the gauge.
+    const nowEngaged = this.computeEngaged();
+    this.opts.metrics?.engaged.set(nowEngaged ? 1 : 0);
+
+    if (nowEngaged !== this.wasEngaged) {
+      this.opts.logger?.info("backpressure verdict changed", {
+        engaged: nowEngaged,
+        dryRun: !!this.opts.dryRun,
+      });
+    }
+    if (this.wasEngaged && !nowEngaged) {
+      this.releasedAt = Date.now();
+    }
+    this.wasEngaged = nowEngaged;
+  }
+}

apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts

@@ -0,0 +1,95 @@
+import { describe, it, expect } from "vitest";
+import { parsePodCount, K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
+
+describe("parsePodCount", () => {
+  it("reads the pods object count", () => {
+    const text = [
+      "# HELP apiserver_storage_objects Number of stored objects",
+      "# TYPE apiserver_storage_objects gauge",
+      'apiserver_storage_objects{resource="pods"} 8421',
+      'apiserver_storage_objects{resource="configmaps"} 17',
+    ].join("\n");
+    expect(parsePodCount(text)).toBe(8421);
+  });
+
+  it("is tolerant of extra labels in any order", () => {
+    const text = 'apiserver_storage_objects{group="",resource="pods",extra="x"} 12';
+    expect(parsePodCount(text)).toBe(12);
+  });
+
+  it("parses scientific notation", () => {
+    const text = 'apiserver_storage_objects{resource="pods"} 1.2e+04';
+    expect(parsePodCount(text)).toBe(12000);
+  });
+
+  it("throws when the pods metric is absent", () => {
+    const text = 'apiserver_storage_objects{resource="configmaps"} 17';
+    expect(() => parsePodCount(text)).toThrow(/not found/);
+  });
+
+  it("throws on a non-finite value (e.g. 1e999)", () => {
+    const text = 'apiserver_storage_objects{resource="pods"} 1e999';
+    expect(() => parsePodCount(text)).toThrow();
+  });
+
+  it("throws on a negative value", () => {
+    const text = 'apiserver_storage_objects{resource="pods"} -5';
+    expect(() => parsePodCount(text)).toThrow();
+  });
+});
+
+function metrics(count: number): string {
+  return `apiserver_storage_objects{resource="pods"} ${count}`;
+}
+
+describe("K8sPodCountSignalSource", () => {
+  it("engages at the engage threshold and reports the count", async () => {
+    const counts: number[] = [];
+    const source = new K8sPodCountSignalSource({
+      fetchMetrics: async () => metrics(10000),
+      engageThreshold: 10000,
+      releaseThreshold: 5000,
+      reportPodCount: (c) => counts.push(c),
+    });
+    const verdict = await source.read();
+    expect(verdict.engaged).toBe(true);
+    expect(typeof verdict.ts).toBe("number");
+    expect(counts).toEqual([10000]);
+  });
+
+  it("does not engage below the engage threshold", async () => {
+    const source = new K8sPodCountSignalSource({
+      fetchMetrics: async () => metrics(9999),
+      engageThreshold: 10000,
+      releaseThreshold: 5000,
+    });
+    expect((await source.read()).engaged).toBe(false);
+  });
+
+  it("stays engaged in the hysteresis band, releases only below release threshold", async () => {
+    let count = 10000;
+    const source = new K8sPodCountSignalSource({
+      fetchMetrics: async () => metrics(count),
+      engageThreshold: 10000,
+      releaseThreshold: 5000,
+    });
+    expect((await source.read()).engaged).toBe(true); // engage
+    count = 7000;
+    expect((await source.read()).engaged).toBe(true); // band -> still engaged
+    count = 4999;
+    expect((await source.read()).engaged).toBe(false); // below release -> off
+    count = 7000;
+    expect((await source.read()).engaged).toBe(false); // band again -> stays off
+  });
+
+  it("propagates scrape failures (monitor fails open on throw)", async () => {
+    const source = new K8sPodCountSignalSource({
+      fetchMetrics: async () => {
+        throw new Error("connection refused");
+      },
+      engageThreshold: 10000,
+      releaseThreshold: 5000,
+    });
+    await expect(source.read()).rejects.toThrow("connection refused");
+  });
+});

apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts

@@ -0,0 +1,46 @@
+import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js";
+
+// Reads the apiserver's stored-pod-object count from a Prometheus /metrics scrape.
+const POD_COUNT_RE = /^apiserver_storage_objects\{[^}]*resource="pods"[^}]*\}\s+([0-9.eE+]+)/m;
+
+export function parsePodCount(metricsText: string): number {
+  const match = metricsText.match(POD_COUNT_RE);
+  if (!match) {
+    throw new Error('apiserver_storage_objects{resource="pods"} not found in metrics');
+  }
+  const value = Number(match[1]);
+  if (!Number.isFinite(value)) {
+    throw new Error(`unparseable pod count: ${match[1]}`);
+  }
+  return value;
+}
+
+export type K8sPodCountSignalSourceOptions = {
+  fetchMetrics: () => Promise<string>;
+  engageThreshold: number;
+  releaseThreshold: number;
+  reportPodCount?: (count: number) => void;
+};
+
+// Engage/release with hysteresis so a count hovering near the line doesn't flap.
+export class K8sPodCountSignalSource implements BackpressureSignalSource {
+  private engaged = false;
+
+  constructor(private readonly opts: K8sPodCountSignalSourceOptions) {}
+
+  async read(): Promise<BackpressureVerdict> {
+    const text = await this.opts.fetchMetrics();
+    const count = parsePodCount(text);
+    this.opts.reportPodCount?.(count);
+
+    if (this.engaged) {
+      if (count < this.opts.releaseThreshold) {
+        this.engaged = false;
+      }
+    } else if (count >= this.opts.engageThreshold) {
+      this.engaged = true;
+    }
+
+    return { engaged: this.engaged, ts: Date.now() };
+  }
+}

apps/supervisor/src/backpressure/redisBackpressureSignalSource.test.ts

@@ -0,0 +1,65 @@
+import { redisTest } from "@internal/testcontainers";
+import { Redis } from "ioredis";
+import { describe, expect } from "vitest";
+import { RedisBackpressureSignalSource } from "./redisBackpressureSignalSource.js";
+
+const KEY = "backpressure:test";
+
+describe("RedisBackpressureSignalSource", () => {
+  redisTest("returns null when the key is absent", async ({ redisOptions }) => {
+    const redis = new Redis(redisOptions);
+    try {
+      const source = new RedisBackpressureSignalSource(redis, KEY);
+      expect(await source.read()).toBeNull();
+    } finally {
+      await redis.quit();
+    }
+  });
+
+  redisTest("parses a valid engaged verdict", async ({ redisOptions }) => {
+    const redis = new Redis(redisOptions);
+    try {
+      await redis.set(KEY, JSON.stringify({ engaged: true, ts: 1_700_000_000_000 }));
+      const source = new RedisBackpressureSignalSource(redis, KEY);
+      expect(await source.read()).toEqual({ engaged: true, ts: 1_700_000_000_000 });
+    } finally {
+      await redis.quit();
+    }
+  });
+
+  redisTest("parses a clear verdict", async ({ redisOptions }) => {
+    const redis = new Redis(redisOptions);
+    try {
+      await redis.set(KEY, JSON.stringify({ engaged: false }));
+      const source = new RedisBackpressureSignalSource(redis, KEY);
+      expect(await source.read()).toEqual({ engaged: false });
+    } finally {
+      await redis.quit();
+    }
+  });
+
+  redisTest("returns null for malformed JSON (fail-open)", async ({ redisOptions }) => {
+    const redis = new Redis(redisOptions);
+    try {
+      await redis.set(KEY, "not json {");
+      const source = new RedisBackpressureSignalSource(redis, KEY);
+      expect(await source.read()).toBeNull();
+    } finally {
+      await redis.quit();
+    }
+  });
+
+  redisTest(
+    "returns null for valid JSON of the wrong shape (fail-open)",
+    async ({ redisOptions }) => {
+      const redis = new Redis(redisOptions);
+      try {
+        await redis.set(KEY, JSON.stringify({ foo: "bar" }));
+        const source = new RedisBackpressureSignalSource(redis, KEY);
+        expect(await source.read()).toBeNull();
+      } finally {
+        await redis.quit();
+      }
+    }
+  );
+});

apps/supervisor/src/backpressure/redisBackpressureSignalSource.ts

@@ -0,0 +1,35 @@
+import type { Redis } from "ioredis";
+import { z } from "zod";
+import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js";
+
+const VerdictSchema = z.object({
+  engaged: z.boolean(),
+  ts: z.number().optional(),
+});
+
+/** Reads the backpressure verdict from a Redis key written by the cluster-side aggregator. */
+export class RedisBackpressureSignalSource implements BackpressureSignalSource {
+  constructor(
+    private readonly redis: Redis,
+    private readonly key: string
+  ) {}
+
+  async read(): Promise<BackpressureVerdict | null> {
+    const raw = await this.redis.get(this.key);
+    if (raw === null) {
+      return null;
+    }
+
+    // A malformed or wrong-shaped value is treated as unknown (null) so the
+    // monitor fails open rather than acting on garbage.
+    let json: unknown;
+    try {
+      json = JSON.parse(raw);
+    } catch {
+      return null;
+    }
+
+    const parsed = VerdictSchema.safeParse(json);
+    return parsed.success ? parsed.data : null;
+  }
+}

apps/supervisor/src/clients/kubernetes.ts

@@ -0,0 +1,108 @@
+import * as k8s from "@kubernetes/client-node";
+import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node";
+import { assertExhaustive } from "@trigger.dev/core/utils";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import * as https from "node:https";
+
+export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+
+const logger = new SimpleStructuredLogger("kubernetes-client");
+
+export function createK8sApi() {
+  const kubeConfig = getKubeConfig();
+
+  function makeInformer<T extends KubernetesObject>(
+    path: string,
+    listPromiseFn: ListPromise<T>,
+    labelSelector?: string,
+    fieldSelector?: string
+  ): Informer<T> {
+    return k8s.makeInformer(kubeConfig, path, listPromiseFn, labelSelector, fieldSelector);
+  }
+
+  const api = {
+    core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+    batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+    apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+    makeInformer,
+  };
+
+  return api;
+}
+
+export type K8sApi = ReturnType<typeof createK8sApi>;
+
+function getKubeConfig() {
+  logger.debug("getKubeConfig()", { RUNTIME_ENV });
+
+  const kubeConfig = new k8s.KubeConfig();
+
+  switch (RUNTIME_ENV) {
+    case "local":
+      kubeConfig.loadFromDefault();
+      break;
+    case "kubernetes":
+      kubeConfig.loadFromCluster();
+      break;
+    default:
+      assertExhaustive(RUNTIME_ENV);
+  }
+
+  return kubeConfig;
+}
+
+export { k8s };
+
+/**
+ * Builds a function that scrapes the apiserver's Prometheus /metrics endpoint.
+ * One lightweight aggregate read - not a pod listing. Requires the service
+ * account to be granted GET on the /metrics non-resource URL.
+ */
+export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> {
+  const kubeConfig = getKubeConfig();
+
+  return async () => {
+    const cluster = kubeConfig.getCurrentCluster();
+    if (!cluster) {
+      throw new Error("no current cluster in kubeconfig");
+    }
+    const url = new URL(`${cluster.server}/metrics`);
+    const opts: https.RequestOptions = {
+      method: "GET",
+      protocol: url.protocol,
+      hostname: url.hostname,
+      port: url.port,
+      path: url.pathname,
+    };
+    // applyToHTTPSOptions sets the cluster CA, client cert/key, and auth headers
+    // (incl. exec plugins) on the request - so TLS verifies against the cluster
+    // CA, not the system store. The fetch-options path attaches the CA as an
+    // https.Agent, which global fetch (undici) ignores.
+    await kubeConfig.applyToHTTPSOptions(opts);
+
+    return new Promise<string>((resolve, reject) => {
+      const req = https.request(opts, (res) => {
+        const status = res.statusCode ?? 0;
+        let body = "";
+        res.setEncoding("utf8");
+        res.on("data", (chunk) => {
+          body += chunk;
+        });
+        res.on("end", () => {
+          if (status >= 200 && status < 300) {
+            resolve(body);
+          } else {
+            reject(new Error(`apiserver /metrics scrape failed: ${status}`));
+          }
+        });
+      });
+      // Without this a hung connect/TLS/read never settles, and the monitor's
+      // refreshInFlight guard would freeze the source (silent fail-open).
+      req.setTimeout(timeoutMs, () => {
+        req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`));
+      });
+      req.on("error", reject);
+      req.end();
+    });
+  };
+}

apps/supervisor/src/env.test.ts

@@ -0,0 +1,64 @@
+import { describe, it, expect, vi } from "vitest";
+
+// Mock std-env before importing env.ts so the module-level `Env.parse(stdEnv)`
+// doesn't fail in a test environment that lacks required vars.
+vi.mock("std-env", () => ({
+  env: {
+    TRIGGER_API_URL: "http://localhost:3030",
+    TRIGGER_WORKER_TOKEN: "test-token",
+    MANAGED_WORKER_SECRET: "test-secret",
+    OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318",
+  },
+}));
+
+const { Env } = await import("./env.js");
+
+// Minimal env that satisfies all required fields; everything else has defaults.
+const base = {
+  TRIGGER_API_URL: "http://localhost:3030",
+  TRIGGER_WORKER_TOKEN: "test-token",
+  MANAGED_WORKER_SECRET: "test-secret",
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318",
+};
+
+describe("Env superRefine - backpressure source awareness", () => {
+  it("pod-count source can be enabled without a Redis host", () => {
+    expect(() =>
+      Env.parse({
+        ...base,
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true",
+      })
+    ).not.toThrow();
+  });
+
+  it("redis source requires a Redis host", () => {
+    expect(() =>
+      Env.parse({
+        ...base,
+        TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: "true",
+      })
+    ).toThrow();
+  });
+
+  it("both sources can be enabled together (with a Redis host)", () => {
+    expect(() =>
+      Env.parse({
+        ...base,
+        TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: "true",
+        TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST: "localhost",
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true",
+      })
+    ).not.toThrow();
+  });
+
+  it("rejects pod-count release >= engage when the source is enabled", () => {
+    expect(() =>
+      Env.parse({
+        ...base,
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true",
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: "100",
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: "100",
+      })
+    ).toThrow();
+  });
+});

apps/supervisor/src/env.ts

@@ -0,0 +1,385 @@
+import { randomUUID } from "crypto";
+import { env as stdEnv } from "std-env";
+import { z } from "zod";
+import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
+
+export const Env = z
+  .object({
+    // This will come from `spec.nodeName` in k8s
+    TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+    TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
+
+    // Opt-in, dev-only: stream this process's logs over a local telnet/TCP socket on this port.
+    SUPERVISOR_TELNET_LOGS_PORT: z.coerce.number().optional(),
+
+    // Required settings
+    TRIGGER_API_URL: z.string().url(),
+    TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file
+    MANAGED_WORKER_SECRET: z.string(),
+    OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners
+
+    // Workload API settings (coordinator mode) - the workload API is what the run controller connects to
+    TRIGGER_WORKLOAD_API_ENABLED: BoolEnv.default(true),
+    TRIGGER_WORKLOAD_API_PROTOCOL: z
+      .string()
+      .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase()))
+      .default("http"),
+    TRIGGER_WORKLOAD_API_DOMAIN: z.string().optional(), // If unset, will use orchestrator-specific default
+    TRIGGER_WORKLOAD_API_HOST_INTERNAL: z.string().default("0.0.0.0"),
+    TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on
+    TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller
+
+    // Runner settings
+    RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(),
+    RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(),
+    RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv)
+    RUNNER_PRETTY_LOGS: BoolEnv.default(false),
+
+    // Dequeue settings (provider mode)
+    TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true),
+    // Which worker-queue class this supervisor fleet serves. "default" pulls the
+    // region queue (standard/agent runs); "scheduled" pulls the dedicated
+    // scheduled-lineage queue. Run a separate fleet per class for isolation.
+    TRIGGER_WORKER_QUEUE_CLASS: z.enum(["default", "scheduled"]).default("default"),
+    TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
+    TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
+    TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1),
+    TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1),
+    TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10),
+    TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"),
+    TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds
+    TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds
+    TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer)
+    TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current)
+    TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms)
+    TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate)
+
+    // Dequeue backpressure - off by default. When enabled, the supervisor reads a
+    // verdict from Redis (written by the cluster-side aggregator) and pauses dequeues
+    // while the worker cluster can't schedule pods. Disabled = total no-op: no Redis
+    // client is created, no reads happen, and the dequeue loop is unaffected.
+    TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED: BoolEnv.default(false),
+    // Safety default: even when enabled, backpressure only logs what it would do.
+    // Set to false to actually skip dequeues / freeze scale-up.
+    TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN: BoolEnv.default(true),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY: z.string().default("engine:dequeue:backpressure"),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS: z.coerce.number().int().positive().default(1000),
+    TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS: z.coerce.number().int().min(0).default(30_000), // Resume ramp window after release; 0 = instant resume
+
+    TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(15_000), // Stale verdict → fail-open (treat as not engaged)
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST: z.string().optional(),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT: z.coerce.number().int().optional(),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME: z.string().optional(),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD: z.string().optional(),
+    TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_TLS_DISABLED: BoolEnv.default(false),
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: BoolEnv.default(false),
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN: BoolEnv.default(true),
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(10_000),
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(5_000),
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(5_000),
+    // Hard timeout on the apiserver /metrics scrape. A hung request would otherwise
+    // never settle and freeze the monitor's refresh loop (fail-open silently).
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(10_000),
+
+    // Optional services
+    TRIGGER_WARM_START_URL: z.string().optional(),
+    TRIGGER_CHECKPOINT_URL: z.string().optional(),
+    TRIGGER_METADATA_URL: z.string().optional(),
+
+    // Warm-start delivery verification: after a warm-start hit, probe the
+    // platform and cold-start the run if no runner acted on the dispatch
+    TRIGGER_WARM_START_VERIFY_ENABLED: BoolEnv.default(false),
+    TRIGGER_WARM_START_VERIFY_DELAY_MS: z.coerce
+      .number()
+      .int()
+      .min(1_000)
+      .max(60_000)
+      .default(10_000),
+
+    // Used by the resource monitor
+    RESOURCE_MONITOR_ENABLED: BoolEnv.default(false),
+    RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
+    RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+
+    // Docker settings
+    DOCKER_API_VERSION: z.string().optional(),
+    DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64
+    DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true),
+    DOCKER_REGISTRY_USERNAME: z.string().optional(),
+    DOCKER_REGISTRY_PASSWORD: z.string().optional(),
+    DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1
+    DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true),
+    DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true),
+    /**
+     * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:<name|id>`.
+     * Any other value is taken as a custom network's name to which all runners should connect to.
+     *
+     * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation.
+     *
+     * **WARNING**: Specifying multiple networks will slightly increase startup times.
+     *
+     * @default "host"
+     */
+    DOCKER_RUNNER_NETWORKS: z.string().default("host"),
+
+    // Compute settings
+    COMPUTE_GATEWAY_URL: z.string().url().optional(),
+    COMPUTE_GATEWAY_AUTH_TOKEN: z.string().optional(),
+    COMPUTE_GATEWAY_TIMEOUT_MS: z.coerce.number().int().default(30_000),
+    COMPUTE_SNAPSHOTS_ENABLED: BoolEnv.default(false),
+    COMPUTE_TRACE_SPANS_ENABLED: BoolEnv.default(true),
+    COMPUTE_TRACE_OTLP_ENDPOINT: z.string().url().optional(), // Override for span export (derived from TRIGGER_API_URL if unset)
+    COMPUTE_SNAPSHOT_DELAY_MS: z.coerce.number().int().min(0).max(60_000).default(5_000),
+    COMPUTE_SNAPSHOT_DISPATCH_LIMIT: z.coerce.number().int().min(1).max(100).default(10),
+    // Instance create retries for transient placement failures (1 = no retries)
+    COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(10).default(3),
+    COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS: z.coerce
+      .number()
+      .int()
+      .min(0)
+      .max(10_000)
+      .default(250),
+
+    // Kubernetes settings
+    KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
+    KUBERNETES_NAMESPACE: z.string().default("default"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
+    KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
+    KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
+    KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false),
+    KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0),
+    KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit
+    KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
+    KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
+
+    // Per-preset overrides of the global KUBERNETES_CPU_REQUEST_RATIO
+    KUBERNETES_CPU_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(),
+
+    // Per-preset overrides of the global KUBERNETES_MEMORY_REQUEST_RATIO
+    KUBERNETES_MEMORY_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(),
+
+    KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
+    KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods
+
+    // Pod DNS config — override the cluster default ndots to `KUBERNETES_POD_DNS_NDOTS`.
+    // Default k8s ndots is 5: any name with fewer than 5 dots (e.g. `api.example.com`, 2 dots) is first walked
+    // through every entry in the cluster search list (`<ns>.svc.cluster.local`, `svc.cluster.local`, `cluster.local`)
+    // before being tried as-is, turning one resolution into 4+ CoreDNS queries (×2 with A+AAAA).
+    // Overriding the default can be useful to cut CoreDNS query amplification for external domains.
+    // Note: before enabling, make sure no code path relies on search-list expansion for names with dots ≥ the value
+    // set here — those names will now hit their as-is form first and could resolve externally before falling back.
+    KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED: BoolEnv.default(false),
+    KUBERNETES_POD_DNS_NDOTS: z.coerce.number().int().min(1).max(15).default(2),
+    // Large machine affinity settings - large-* presets prefer a dedicated pool
+    KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED: BoolEnv.default(false),
+    KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY: z
+      .string()
+      .trim()
+      .min(1)
+      .default("node.cluster.x-k8s.io/machinepool"),
+    KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_VALUE: z
+      .string()
+      .trim()
+      .min(1)
+      .default("large-machines"),
+    KUBERNETES_LARGE_MACHINE_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(100),
+
+    // Project affinity settings - pods from the same project prefer the same node
+    KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false),
+    KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50),
+    KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z
+      .string()
+      .trim()
+      .min(1)
+      .default("kubernetes.io/hostname"),
+
+    // Schedule affinity settings - runs from schedule trees prefer a dedicated pool
+    KUBERNETES_SCHEDULED_RUN_AFFINITY_ENABLED: BoolEnv.default(false),
+    KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_KEY: z
+      .string()
+      .trim()
+      .min(1)
+      .default("node.cluster.x-k8s.io/machinepool"),
+    KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_VALUE: z
+      .string()
+      .trim()
+      .min(1)
+      .default("scheduled-runs"),
+    KUBERNETES_SCHEDULED_RUN_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(80),
+    KUBERNETES_SCHEDULED_RUN_ANTI_AFFINITY_WEIGHT: z.coerce
+      .number()
+      .int()
+      .min(1)
+      .max(100)
+      .default(20),
+
+    // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool
+    // Comma-separated list of tolerations in the format: key=value:effect
+    // For Exists operator (no value): key:effect
+    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z
+      .string()
+      .transform((val, ctx) => {
+        const tolerations = val
+          .split(",")
+          .map((entry) => entry.trim())
+          .filter((entry) => entry.length > 0)
+          .map((entry) => {
+            const colonIdx = entry.lastIndexOf(":");
+            if (colonIdx === -1) {
+              ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `Invalid toleration format (missing effect): "${entry}"`,
+              });
+              return z.NEVER;
+            }
+
+            const effect = entry.slice(colonIdx + 1);
+            const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
+            if (!validEffects.includes(effect)) {
+              ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
+                  ", "
+                )}`,
+              });
+              return z.NEVER;
+            }
+
+            const keyValue = entry.slice(0, colonIdx);
+            const eqIdx = keyValue.indexOf("=");
+            const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx);
+
+            if (!key) {
+              ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `Invalid toleration format (empty key): "${entry}"`,
+              });
+              return z.NEVER;
+            }
+
+            if (eqIdx === -1) {
+              return { key, operator: "Exists" as const, effect };
+            }
+
+            return {
+              key,
+              operator: "Equal" as const,
+              value: keyValue.slice(eqIdx + 1),
+              effect,
+            };
+          });
+
+        return tolerations;
+      })
+      .optional(),
+
+    // Placement tags settings
+    PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
+    PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"),
+
+    // Metrics
+    METRICS_ENABLED: BoolEnv.default(true),
+    METRICS_COLLECT_DEFAULTS: BoolEnv.default(true),
+    METRICS_HOST: z.string().default("127.0.0.1"),
+    METRICS_PORT: z.coerce.number().int().default(9090),
+
+    // Pod cleaner
+    POD_CLEANER_ENABLED: BoolEnv.default(true),
+    POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000),
+    POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500),
+
+    // Failed pod handler
+    FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true),
+    FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000),
+
+    // Debug
+    DEBUG: BoolEnv.default(false),
+    SEND_RUN_DEBUG_LOGS: BoolEnv.default(false),
+
+    // Wide-event observability - off by default. Emits one flat-keyed JSON
+    // line per natural unit of work (dequeue iteration, HTTP request, socket
+    // lifecycle). High-QPS hotpath, so the kill switch must be honoured.
+    TRIGGER_WIDE_EVENTS_ENABLED: BoolEnv.default(false),
+    // When true, also emit wide events for high-frequency HTTP routes
+    // (heartbeat, snapshots-since, logs/debug). Off in prod to keep event
+    // volume manageable; on in test environments for full-fidelity debugging.
+    TRIGGER_WIDE_EVENTS_NOISY_ROUTES: BoolEnv.default(false),
+  })
+  .superRefine((data, ctx) => {
+    if (
+      data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED &&
+      data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE >=
+        data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE
+    ) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE must be less than TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE",
+        path: ["TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE"],
+      });
+    }
+    if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_METADATA_URL) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "TRIGGER_METADATA_URL is required when COMPUTE_SNAPSHOTS_ENABLED is true",
+        path: ["TRIGGER_METADATA_URL"],
+      });
+    }
+    if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_WORKLOAD_API_DOMAIN) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "TRIGGER_WORKLOAD_API_DOMAIN is required when COMPUTE_SNAPSHOTS_ENABLED is true",
+        path: ["TRIGGER_WORKLOAD_API_DOMAIN"],
+      });
+    }
+    if (
+      data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED &&
+      !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST
+    ) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST is required when TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED is true",
+        path: ["TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST"],
+      });
+    }
+  })
+  .transform((data) => ({
+    ...data,
+    COMPUTE_TRACE_OTLP_ENDPOINT: data.COMPUTE_TRACE_OTLP_ENDPOINT ?? `${data.TRIGGER_API_URL}/otel`,
+  }));
+
+export const env = Env.parse(stdEnv);

apps/supervisor/src/envUtil.test.ts

@@ -0,0 +1,80 @@
+import { describe, it, expect } from "vitest";
+import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
+
+describe("BoolEnv", () => {
+  it("should parse string 'true' as true", () => {
+    expect(BoolEnv.parse("true")).toBe(true);
+    expect(BoolEnv.parse("TRUE")).toBe(true);
+    expect(BoolEnv.parse("True")).toBe(true);
+  });
+
+  it("should parse string '1' as true", () => {
+    expect(BoolEnv.parse("1")).toBe(true);
+  });
+
+  it("should parse string 'false' as false", () => {
+    expect(BoolEnv.parse("false")).toBe(false);
+    expect(BoolEnv.parse("FALSE")).toBe(false);
+    expect(BoolEnv.parse("False")).toBe(false);
+  });
+
+  it("should handle whitespace", () => {
+    expect(BoolEnv.parse(" true ")).toBe(true);
+    expect(BoolEnv.parse(" 1 ")).toBe(true);
+  });
+
+  it("should pass through boolean values", () => {
+    expect(BoolEnv.parse(true)).toBe(true);
+    expect(BoolEnv.parse(false)).toBe(false);
+  });
+
+  it("should return false for invalid inputs", () => {
+    expect(BoolEnv.parse("invalid")).toBe(false);
+    expect(BoolEnv.parse("")).toBe(false);
+  });
+});
+
+describe("AdditionalEnvVars", () => {
+  it("should parse single key-value pair", () => {
+    expect(AdditionalEnvVars.parse("FOO=bar")).toEqual({ FOO: "bar" });
+  });
+
+  it("should parse multiple key-value pairs", () => {
+    expect(AdditionalEnvVars.parse("FOO=bar,BAZ=qux")).toEqual({
+      FOO: "bar",
+      BAZ: "qux",
+    });
+  });
+
+  it("should handle whitespace", () => {
+    expect(AdditionalEnvVars.parse(" FOO = bar , BAZ = qux ")).toEqual({
+      FOO: "bar",
+      BAZ: "qux",
+    });
+  });
+
+  it("should return undefined for empty string", () => {
+    expect(AdditionalEnvVars.parse("")).toBeUndefined();
+  });
+
+  it("should return undefined for invalid format", () => {
+    expect(AdditionalEnvVars.parse("invalid")).toBeUndefined();
+  });
+
+  it("should skip invalid pairs but include valid ones", () => {
+    expect(AdditionalEnvVars.parse("FOO=bar,INVALID,BAZ=qux")).toEqual({
+      FOO: "bar",
+      BAZ: "qux",
+    });
+  });
+
+  it("should pass through undefined", () => {
+    expect(AdditionalEnvVars.parse(undefined)).toBeUndefined();
+  });
+
+  it("should handle empty values", () => {
+    expect(AdditionalEnvVars.parse("FOO=,BAR=value")).toEqual({
+      BAR: "value",
+    });
+  });
+});

apps/supervisor/src/envUtil.ts

@@ -0,0 +1,47 @@
+import { z } from "zod";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+const logger = new SimpleStructuredLogger("env-util");
+
+const baseBoolEnv = z.preprocess((val) => {
+  if (typeof val !== "string") {
+    return val;
+  }
+
+  return ["true", "1"].includes(val.toLowerCase().trim());
+}, z.boolean());
+
+// Create a type-safe version that only accepts boolean defaults
+export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & {
+  default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>;
+};
+
+export const AdditionalEnvVars = z.preprocess((val) => {
+  if (typeof val !== "string") {
+    return val;
+  }
+
+  if (!val) {
+    return undefined;
+  }
+
+  try {
+    const result = val.split(",").reduce(
+      (acc, pair) => {
+        const [key, value] = pair.split("=");
+        if (!key || !value) {
+          return acc;
+        }
+        acc[key.trim()] = value.trim();
+        return acc;
+      },
+      {} as Record<string, string>
+    );
+
+    // Return undefined if no valid key-value pairs were found
+    return Object.keys(result).length === 0 ? undefined : result;
+  } catch (error) {
+    logger.warn("Failed to parse additional env vars", { error, val });
+    return undefined;
+  }
+}, z.record(z.string(), z.string()).optional());

apps/supervisor/src/index.ts

@@ -0,0 +1,763 @@
+import { SupervisorSession } from "@trigger.dev/core/v3/workers";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { formatLogLine, startTelnetLogServer } from "@trigger.dev/core/v3/telnetLogServer";
+import { env } from "./env.js";
+import { WorkloadServer } from "./workloadServer/index.js";
+import type { WorkloadManagerOptions, WorkloadManager } from "./workloadManager/types.js";
+import Docker from "dockerode";
+import { z } from "zod";
+import { type DequeuedMessage } from "@trigger.dev/core/v3";
+import {
+  DockerResourceMonitor,
+  KubernetesResourceMonitor,
+  NoopResourceMonitor,
+  type ResourceMonitor,
+} from "./resourceMonitor.js";
+import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js";
+import { DockerWorkloadManager } from "./workloadManager/docker.js";
+import { ComputeWorkloadManager } from "./workloadManager/compute.js";
+import {
+  HttpServer,
+  CheckpointClient,
+  isKubernetesEnvironment,
+} from "@trigger.dev/core/v3/serverOnly";
+import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
+import { collectDefaultMetrics, Gauge, Histogram } from "prom-client";
+import { register } from "./metrics.js";
+import { PodCleaner } from "./services/podCleaner.js";
+import { FailedPodHandler } from "./services/failedPodHandler.js";
+import { getWorkerToken } from "./workerToken.js";
+import { OtlpTraceService } from "./services/otlpTraceService.js";
+import {
+  WarmStartVerificationService,
+  type WarmStartTimings,
+} from "./services/warmStartVerificationService.js";
+import { extractTraceparent, getRestoreRunnerId } from "./util.js";
+import { Redis } from "ioredis";
+import { BackpressureMonitor } from "./backpressure/backpressureMonitor.js";
+import { RedisBackpressureSignalSource } from "./backpressure/redisBackpressureSignalSource.js";
+import { BackpressureMetrics } from "./backpressure/backpressureMetrics.js";
+import { K8sPodCountSignalSource } from "./backpressure/k8sPodCountSignalSource.js";
+import {
+  fromContext,
+  recordPhaseSince,
+  runWideEvent,
+  setExtra,
+  setMeta,
+  type WideEventOptions,
+} from "./wideEvents/index.js";
+
+if (env.METRICS_COLLECT_DEFAULTS) {
+  collectDefaultMetrics({ register });
+}
+
+const workloadCreateDuration = new Histogram({
+  name: "workload_create_duration_seconds",
+  help: "Duration of workload manager create calls. A create may include backend-internal retries, so one observation can span multiple attempts.",
+  labelNames: ["backend", "outcome"],
+  buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
+  registers: [register],
+});
+
+class ManagedSupervisor {
+  private readonly workerSession: SupervisorSession;
+  private readonly metricsServer?: HttpServer;
+  private readonly workloadServer: WorkloadServer;
+  private readonly workloadManager: WorkloadManager;
+  private readonly workloadManagerBackend: "compute" | "kubernetes" | "docker";
+  private readonly computeManager?: ComputeWorkloadManager;
+  private readonly logger = new SimpleStructuredLogger("managed-supervisor");
+  private readonly resourceMonitor: ResourceMonitor;
+  private readonly checkpointClient?: CheckpointClient;
+  private readonly warmStartVerifier?: WarmStartVerificationService;
+
+  private readonly podCleaner?: PodCleaner;
+  private readonly failedPodHandler?: FailedPodHandler;
+  private readonly tracing?: OtlpTraceService;
+  private readonly backpressureMonitors: BackpressureMonitor[] = [];
+  private readonly backpressureRedis?: Redis;
+
+  private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED);
+  private readonly warmStartUrl = env.TRIGGER_WARM_START_URL;
+
+  private readonly wideEventOpts: WideEventOptions = {
+    service: "supervisor",
+    env: { nodeId: env.TRIGGER_WORKER_INSTANCE_NAME },
+    enabled: env.TRIGGER_WIDE_EVENTS_ENABLED,
+  };
+  private readonly wideEventsNoisyRoutes = env.TRIGGER_WIDE_EVENTS_NOISY_ROUTES;
+
+  constructor() {
+    // Strip secret-like env vars before debug-logging the rest. Add any new
+    // secret env var here so it never lands in the DEBUG "Starting up" log.
+    const {
+      TRIGGER_WORKER_TOKEN,
+      MANAGED_WORKER_SECRET,
+      COMPUTE_GATEWAY_AUTH_TOKEN,
+      DOCKER_REGISTRY_PASSWORD,
+      TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD,
+      ...envWithoutSecrets
+    } = env;
+
+    if (env.DEBUG) {
+      this.logger.debug("Starting up", { envWithoutSecrets });
+    }
+
+    if (this.warmStartUrl) {
+      this.logger.log("🔥 Warm starts enabled", {
+        warmStartUrl: this.warmStartUrl,
+      });
+    }
+
+    const workloadManagerOptions = {
+      workloadApiProtocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
+      workloadApiDomain: env.TRIGGER_WORKLOAD_API_DOMAIN,
+      workloadApiPort: env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL,
+      warmStartUrl: this.warmStartUrl,
+      metadataUrl: env.TRIGGER_METADATA_URL,
+      imagePullSecrets: env.KUBERNETES_IMAGE_PULL_SECRETS?.split(","),
+      heartbeatIntervalSeconds: env.RUNNER_HEARTBEAT_INTERVAL_SECONDS,
+      snapshotPollIntervalSeconds: env.RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS,
+      additionalEnvVars: env.RUNNER_ADDITIONAL_ENV_VARS,
+      dockerAutoremove: env.DOCKER_AUTOREMOVE_EXITED_CONTAINERS,
+    } satisfies WorkloadManagerOptions;
+
+    this.resourceMonitor = env.RESOURCE_MONITOR_ENABLED
+      ? this.isKubernetes
+        ? new KubernetesResourceMonitor(createK8sApi(), env.TRIGGER_WORKER_INSTANCE_NAME)
+        : new DockerResourceMonitor(new Docker())
+      : new NoopResourceMonitor();
+
+    if (env.COMPUTE_GATEWAY_URL) {
+      if (!env.TRIGGER_WORKLOAD_API_DOMAIN) {
+        throw new Error("TRIGGER_WORKLOAD_API_DOMAIN is not set, cannot create compute manager");
+      }
+
+      const callbackUrl = `${env.TRIGGER_WORKLOAD_API_PROTOCOL}://${env.TRIGGER_WORKLOAD_API_DOMAIN}:${env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL}/api/v1/compute/snapshot-complete`;
+
+      if (env.COMPUTE_TRACE_SPANS_ENABLED) {
+        this.tracing = new OtlpTraceService({
+          endpointUrl: env.COMPUTE_TRACE_OTLP_ENDPOINT,
+        });
+      }
+
+      const computeManager = new ComputeWorkloadManager({
+        ...workloadManagerOptions,
+        gateway: {
+          url: env.COMPUTE_GATEWAY_URL,
+          authToken: env.COMPUTE_GATEWAY_AUTH_TOKEN,
+          timeoutMs: env.COMPUTE_GATEWAY_TIMEOUT_MS,
+        },
+        snapshots: {
+          enabled: env.COMPUTE_SNAPSHOTS_ENABLED,
+          delayMs: env.COMPUTE_SNAPSHOT_DELAY_MS,
+          dispatchLimit: env.COMPUTE_SNAPSHOT_DISPATCH_LIMIT,
+          callbackUrl,
+        },
+        tracing: this.tracing,
+        runner: {
+          instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
+          otelEndpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+          prettyLogs: env.RUNNER_PRETTY_LOGS,
+          sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
+        },
+        createRetry: {
+          maxAttempts: env.COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS,
+          baseDelayMs: env.COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS,
+        },
+      });
+      this.computeManager = computeManager;
+      this.workloadManager = computeManager;
+      this.workloadManagerBackend = "compute";
+    } else if (this.isKubernetes) {
+      this.workloadManager = new KubernetesWorkloadManager(workloadManagerOptions);
+      this.workloadManagerBackend = "kubernetes";
+    } else {
+      this.workloadManager = new DockerWorkloadManager(workloadManagerOptions);
+      this.workloadManagerBackend = "docker";
+    }
+
+    if (this.isKubernetes) {
+      if (env.POD_CLEANER_ENABLED) {
+        this.logger.log("🧹 Pod cleaner enabled", {
+          namespace: env.KUBERNETES_NAMESPACE,
+          batchSize: env.POD_CLEANER_BATCH_SIZE,
+          intervalMs: env.POD_CLEANER_INTERVAL_MS,
+        });
+        this.podCleaner = new PodCleaner({
+          register,
+          namespace: env.KUBERNETES_NAMESPACE,
+          batchSize: env.POD_CLEANER_BATCH_SIZE,
+          intervalMs: env.POD_CLEANER_INTERVAL_MS,
+        });
+      } else {
+        this.logger.warn("Pod cleaner disabled");
+      }
+
+      if (env.FAILED_POD_HANDLER_ENABLED) {
+        this.logger.log("🔁 Failed pod handler enabled", {
+          namespace: env.KUBERNETES_NAMESPACE,
+          reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
+        });
+        this.failedPodHandler = new FailedPodHandler({
+          register,
+          namespace: env.KUBERNETES_NAMESPACE,
+          reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
+        });
+      } else {
+        this.logger.warn("Failed pod handler disabled");
+      }
+    }
+
+    if (env.TRIGGER_DEQUEUE_INTERVAL_MS > env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS) {
+      this.logger.warn(
+        `⚠️  TRIGGER_DEQUEUE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_INTERVAL_MS}) is greater than TRIGGER_DEQUEUE_IDLE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS}) - did you mix them up?`
+      );
+    }
+
+    // Redis-verdict source (external aggregator). Keeps existing metric names.
+    if (env.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED) {
+      this.backpressureRedis = new Redis({
+        host: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST,
+        port: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT,
+        username: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME,
+        password: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD,
+        ...(env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_TLS_DISABLED ? {} : { tls: {} }),
+        maxRetriesPerRequest: null,
+      });
+      this.backpressureRedis.on("error", (error) =>
+        this.logger.error("Backpressure redis error", { error: error.message })
+      );
+      this.backpressureMonitors.push(
+        new BackpressureMonitor({
+          enabled: true,
+          source: new RedisBackpressureSignalSource(
+            this.backpressureRedis,
+            env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY
+          ),
+          refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS,
+          maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS,
+          rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS,
+          dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN,
+          logger: this.logger,
+          metrics: new BackpressureMetrics({ register }),
+        })
+      );
+      this.logger.log("🛑 Dequeue backpressure enabled (redis source)", {
+        key: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY,
+        refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS,
+        dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN,
+      });
+    }
+
+    // Pod-count source (in-process apiserver scrape). Namespaced metrics so the
+    // redis source's metric names are preserved.
+    if (env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED) {
+      // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
+      const podCountGauge = new Gauge({
+        name: "supervisor_cluster_pod_count",
+        help: "Total pod objects stored in the cluster, scraped for backpressure",
+        registers: [register],
+      });
+      this.backpressureMonitors.push(
+        new BackpressureMonitor({
+          enabled: true,
+          source: new K8sPodCountSignalSource({
+            fetchMetrics: createApiserverMetricsFetcher(
+              env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
+            ),
+            engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
+            releaseThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE,
+            reportPodCount: (count) => podCountGauge.set(count),
+          }),
+          refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS,
+          maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS,
+          rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS,
+          dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN,
+          logger: this.logger,
+          metrics: new BackpressureMetrics({
+            register,
+            prefix: "supervisor_backpressure_pod_count",
+          }),
+        })
+      );
+      this.logger.log("🛑 Dequeue backpressure enabled (pod-count source)", {
+        engage: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
+        release: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE,
+        refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS,
+        dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN,
+      });
+    }
+
+    this.workerSession = new SupervisorSession({
+      workerToken: getWorkerToken(),
+      apiUrl: env.TRIGGER_API_URL,
+      instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
+      managedWorkerSecret: env.MANAGED_WORKER_SECRET,
+      dequeueIntervalMs: env.TRIGGER_DEQUEUE_INTERVAL_MS,
+      dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS,
+      queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
+      maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT,
+      queueClass: env.TRIGGER_WORKER_QUEUE_CLASS,
+      metricsRegistry: register,
+      scaling: {
+        strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY,
+        minConsumerCount: env.TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT,
+        maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
+        scaleUpCooldownMs: env.TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS,
+        scaleDownCooldownMs: env.TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS,
+        targetRatio: env.TRIGGER_DEQUEUE_SCALING_TARGET_RATIO,
+        ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA,
+        batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS,
+        dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR,
+        // Freeze scale-up while backpressure is hard-engaged (not during the resume
+        // ramp). Undefined when backpressure is disabled → no effect on scaling.
+        shouldPauseScaling: () => this.backpressureMonitors.some((m) => m.isEngaged()),
+      },
+      runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
+      heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
+      sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
+      preDequeue: async () => {
+        // Synchronous, hot-path-safe cached read; false when no monitors are active.
+        const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue());
+
+        if (!env.RESOURCE_MONITOR_ENABLED || this.isKubernetes) {
+          // Resource monitor is not used in k8s; backpressure is the only gate there.
+          return { skipDequeue: skipForBackpressure };
+        }
+
+        const resources = await this.resourceMonitor.getNodeResources();
+
+        return {
+          maxResources: {
+            cpu: resources.cpuAvailable,
+            memory: resources.memoryAvailable,
+          },
+          skipDequeue:
+            skipForBackpressure ||
+            resources.cpuAvailable < 0.25 ||
+            resources.memoryAvailable < 0.25,
+        };
+      },
+      preSkip: async () => {
+        // When the node is full, it should still try to warm start runs
+        // await this.tryWarmStartAllThisNode();
+      },
+    });
+
+    if (env.TRIGGER_CHECKPOINT_URL) {
+      this.logger.log("🥶 Checkpoints enabled", {
+        checkpointUrl: env.TRIGGER_CHECKPOINT_URL,
+      });
+
+      this.checkpointClient = new CheckpointClient({
+        apiUrl: new URL(env.TRIGGER_CHECKPOINT_URL),
+        workerClient: this.workerSession.httpClient,
+        orchestrator: this.isKubernetes ? "KUBERNETES" : "DOCKER",
+      });
+    }
+
+    if (env.TRIGGER_WARM_START_VERIFY_ENABLED && this.warmStartUrl) {
+      this.logger.log("Warm-start delivery verification enabled", {
+        delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS,
+      });
+
+      this.warmStartVerifier = new WarmStartVerificationService({
+        workerClient: this.workerSession.httpClient,
+        delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS,
+        createWorkload: (message, timings) => this.createWorkload(message, timings),
+        wideEventOpts: this.wideEventOpts,
+      });
+    }
+
+    this.workerSession.on("runNotification", async ({ time, run }) => {
+      this.logger.verbose("runNotification", { time, run });
+
+      this.workloadServer.notifyRun({ run });
+    });
+
+    this.workerSession.on(
+      "runQueueMessage",
+      async ({ time, message, dequeueResponseMs, pollingIntervalMs }) => {
+        this.logger.verbose(`Received message with timestamp ${time.toLocaleString()}`, message);
+
+        const traceparent = extractTraceparent(message.run.traceContext);
+
+        await runWideEvent(
+          {
+            ...this.wideEventOpts,
+            op: "dequeue",
+            kind: "inbound",
+            traceparent,
+            setup: (state) => {
+              setMeta(state, "run_id", message.run.friendlyId);
+              setMeta(state, "env_id", message.environment.id);
+              setMeta(state, "org_id", message.organization.id);
+              setMeta(state, "project_id", message.project.id);
+              if (message.deployment.friendlyId) {
+                setMeta(state, "deployment_id", message.deployment.friendlyId);
+              }
+              setMeta(state, "machine_preset", message.run.machine.name);
+              state.extras.iteration = "dequeue";
+              state.extras.dequeue_response_ms = dequeueResponseMs;
+              state.extras.polling_interval_ms = pollingIntervalMs;
+              state.extras.completed_waitpoints = message.completedWaitpoints.length;
+            },
+          },
+          async () => {
+            if (message.completedWaitpoints.length > 0) {
+              this.logger.debug("Run has completed waitpoints", {
+                runId: message.run.id,
+                completedWaitpoints: message.completedWaitpoints.length,
+              });
+            }
+
+            if (!message.image) {
+              setExtra(fromContext(), "path_taken", "skipped_no_image");
+              this.logger.error("Run has no image", { runId: message.run.id });
+              return;
+            }
+
+            const { checkpoint, ...rest } = message;
+
+            // Register trace context early so snapshot spans work for all paths
+            // (cold create, restore, warm start). Re-registration on restore is safe
+            // since dequeue always provides fresh context.
+            if (this.computeManager?.traceSpansEnabled && traceparent) {
+              this.workloadServer.registerRunTraceContext(message.run.friendlyId, {
+                traceparent,
+                envId: message.environment.id,
+                orgId: message.organization.id,
+                projectId: message.project.id,
+              });
+            }
+
+            if (checkpoint) {
+              setExtra(fromContext(), "path_taken", "restore");
+              this.logger.debug("Restoring run", { runId: message.run.id });
+
+              if (this.computeManager) {
+                const restoreStart = performance.now();
+                try {
+                  const runnerId = getRestoreRunnerId(message.run.friendlyId, checkpoint.id);
+
+                  const didRestore = await this.computeManager.restore({
+                    snapshotId: checkpoint.location,
+                    runnerId,
+                    runFriendlyId: message.run.friendlyId,
+                    snapshotFriendlyId: message.snapshot.friendlyId,
+                    machine: message.run.machine,
+                    traceContext: message.run.traceContext,
+                    envId: message.environment.id,
+                    orgId: message.organization.id,
+                    projectId: message.project.id,
+                    hasPrivateLink: message.organization.hasPrivateLink,
+                    dequeuedAt: message.dequeuedAt,
+                  });
+                  recordPhaseSince("restore", restoreStart, undefined);
+                  setExtra(fromContext(), "did_restore", didRestore);
+
+                  if (didRestore) {
+                    this.logger.debug("Compute restore successful", {
+                      runId: message.run.id,
+                      runnerId,
+                    });
+                  } else {
+                    this.logger.error("Compute restore failed", {
+                      runId: message.run.id,
+                      runnerId,
+                    });
+                  }
+                } catch (error) {
+                  recordPhaseSince(
+                    "restore",
+                    restoreStart,
+                    error instanceof Error ? error : new Error(String(error))
+                  );
+                  this.logger.error("Failed to restore run (compute)", { error });
+                }
+
+                return;
+              }
+
+              if (!this.checkpointClient) {
+                this.logger.error("No checkpoint client", { runId: message.run.id });
+                return;
+              }
+
+              const restoreStart = performance.now();
+              try {
+                const didRestore = await this.checkpointClient.restoreRun({
+                  runFriendlyId: message.run.friendlyId,
+                  snapshotFriendlyId: message.snapshot.friendlyId,
+                  body: {
+                    ...rest,
+                    checkpoint,
+                  },
+                });
+                recordPhaseSince("restore", restoreStart, undefined);
+                setExtra(fromContext(), "did_restore", didRestore);
+
+                if (didRestore) {
+                  this.logger.debug("Restore successful", { runId: message.run.id });
+                } else {
+                  this.logger.error("Restore failed", { runId: message.run.id });
+                }
+              } catch (error) {
+                recordPhaseSince(
+                  "restore",
+                  restoreStart,
+                  error instanceof Error ? error : new Error(String(error))
+                );
+                this.logger.error("Failed to restore run", { error });
+              }
+
+              return;
+            }
+
+            this.logger.debug("Scheduling run", { runId: message.run.id });
+
+            const warmStartStart = performance.now();
+            const didWarmStart = await this.tryWarmStart(message, traceparent);
+            const warmStartCheckMs = Math.round(performance.now() - warmStartStart);
+            recordPhaseSince("warm_start", warmStartStart, undefined);
+            setExtra(fromContext(), "did_warm_start", didWarmStart);
+
+            if (didWarmStart) {
+              setExtra(fromContext(), "path_taken", "warm_start");
+              this.logger.debug("Warm start successful", { runId: message.run.id });
+              // A hit only means the response was written to the long-poll
+              // socket, not that the runner received it. Schedule a delivery
+              // verification that cold-starts the run if nobody acts on it.
+              this.warmStartVerifier?.schedule(message, {
+                dequeueResponseMs,
+                pollingIntervalMs,
+                warmStartCheckMs,
+              });
+              return;
+            }
+
+            setExtra(fromContext(), "path_taken", "cold_create");
+
+            await this.createWorkload(message, {
+              dequeueResponseMs,
+              pollingIntervalMs,
+              warmStartCheckMs,
+            });
+          }
+        );
+      }
+    );
+
+    if (env.METRICS_ENABLED) {
+      this.metricsServer = new HttpServer({
+        port: env.METRICS_PORT,
+        host: env.METRICS_HOST,
+        metrics: {
+          register,
+          expose: true,
+        },
+      });
+    }
+
+    // Responds to workload requests only
+    this.workloadServer = new WorkloadServer({
+      port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
+      host: env.TRIGGER_WORKLOAD_API_HOST_INTERNAL,
+      workerClient: this.workerSession.httpClient,
+      checkpointClient: this.checkpointClient,
+      computeManager: this.computeManager,
+      tracing: this.tracing,
+      wideEventOpts: this.wideEventOpts,
+      wideEventsNoisyRoutes: this.wideEventsNoisyRoutes,
+    });
+
+    this.workloadServer.on("runConnected", this.onRunConnected.bind(this));
+    this.workloadServer.on("runDisconnected", this.onRunDisconnected.bind(this));
+  }
+
+  async onRunConnected({ run }: { run: { friendlyId: string } }) {
+    this.logger.debug("Run connected", { run });
+    // The dispatched run reached a runner on this node - no fallback needed.
+    this.warmStartVerifier?.cancel(run.friendlyId);
+    this.workerSession.subscribeToRunNotifications([run.friendlyId]);
+  }
+
+  async onRunDisconnected({ run }: { run: { friendlyId: string } }) {
+    this.logger.debug("Run disconnected", { run });
+    this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]);
+  }
+
+  private async createWorkload(message: DequeuedMessage, timings: WarmStartTimings) {
+    const createStart = performance.now();
+    try {
+      if (!message.deployment.friendlyId) {
+        // mostly a type guard, deployments always exists for deployed environments
+        // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments.
+        throw new Error("Deployment is missing");
+      }
+
+      if (!message.image) {
+        // same type-guard situation as deployment above
+        throw new Error("Image is missing");
+      }
+
+      await this.workloadManager.create({
+        dequeuedAt: message.dequeuedAt,
+        dequeueResponseMs: timings.dequeueResponseMs,
+        pollingIntervalMs: timings.pollingIntervalMs,
+        warmStartCheckMs: timings.warmStartCheckMs,
+        envId: message.environment.id,
+        envType: message.environment.type,
+        image: message.image,
+        machine: message.run.machine,
+        orgId: message.organization.id,
+        projectId: message.project.id,
+        deploymentFriendlyId: message.deployment.friendlyId,
+        deploymentVersion: message.backgroundWorker.version,
+        runId: message.run.id,
+        runFriendlyId: message.run.friendlyId,
+        version: message.version,
+        nextAttemptNumber: message.run.attemptNumber,
+        snapshotId: message.snapshot.id,
+        snapshotFriendlyId: message.snapshot.friendlyId,
+        placementTags: message.placementTags,
+        traceContext: message.run.traceContext,
+        annotations: message.run.annotations,
+        hasPrivateLink: message.organization.hasPrivateLink,
+      });
+      recordPhaseSince("workload_create", createStart, undefined);
+      workloadCreateDuration.observe(
+        { backend: this.workloadManagerBackend, outcome: "success" },
+        (performance.now() - createStart) / 1000
+      );
+
+      // Disabled for now
+      // this.resourceMonitor.blockResources({
+      //   cpu: message.run.machine.cpu,
+      //   memory: message.run.machine.memory,
+      // });
+    } catch (error) {
+      recordPhaseSince(
+        "workload_create",
+        createStart,
+        error instanceof Error ? error : new Error(String(error))
+      );
+      workloadCreateDuration.observe(
+        { backend: this.workloadManagerBackend, outcome: "error" },
+        (performance.now() - createStart) / 1000
+      );
+      this.logger.error("Failed to create workload", {
+        runId: message.run.friendlyId,
+        error,
+      });
+    }
+  }
+
+  private async tryWarmStart(
+    dequeuedMessage: DequeuedMessage,
+    traceparent: string | undefined
+  ): Promise<boolean> {
+    if (!this.warmStartUrl) {
+      return false;
+    }
+
+    const warmStartUrlWithPath = new URL("/warm-start", this.warmStartUrl);
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+    };
+    // Propagate the inbound W3C traceparent so the upstream warm-start
+    // receiver continues the same trace instead of minting a new one. Gated
+    // by the same kill switch as the wide-event emission so the whole PR is
+    // a no-op on the wire when disabled.
+    if (this.wideEventOpts.enabled && traceparent) {
+      headers.traceparent = traceparent;
+    }
+
+    try {
+      const res = await fetch(warmStartUrlWithPath.href, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ dequeuedMessage }),
+      });
+
+      if (!res.ok) {
+        this.logger.error("Warm start failed", {
+          runId: dequeuedMessage.run.id,
+        });
+        return false;
+      }
+
+      const data = await res.json();
+      const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data);
+
+      if (!parsedData.success) {
+        this.logger.error("Warm start response invalid", {
+          runId: dequeuedMessage.run.id,
+          data,
+        });
+        return false;
+      }
+
+      return parsedData.data.didWarmStart;
+    } catch (error) {
+      this.logger.error("Warm start error", {
+        runId: dequeuedMessage.run.id,
+        error,
+      });
+      return false;
+    }
+  }
+
+  async start() {
+    this.logger.log("Starting up");
+
+    // Optional services
+    this.backpressureMonitors.forEach((m) => m.start());
+    await this.podCleaner?.start();
+    await this.failedPodHandler?.start();
+    await this.metricsServer?.start();
+
+    if (env.TRIGGER_WORKLOAD_API_ENABLED) {
+      this.logger.log("Workload API enabled", {
+        protocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
+        domain: env.TRIGGER_WORKLOAD_API_DOMAIN,
+        port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
+      });
+      await this.workloadServer.start();
+    } else {
+      this.logger.warn("Workload API disabled");
+    }
+
+    await this.workerSession.start();
+  }
+
+  async stop() {
+    this.logger.log("Shutting down");
+    // Stop the verifier first: its timer can otherwise fire mid-shutdown and
+    // cold-create a workload on a node that is going down.
+    this.warmStartVerifier?.stop();
+    await this.workloadServer.stop();
+    await this.workerSession.stop();
+
+    // Optional services
+    this.backpressureMonitors.forEach((m) => m.stop());
+    await this.backpressureRedis?.quit();
+    await this.podCleaner?.stop();
+    await this.failedPodHandler?.stop();
+    await this.metricsServer?.stop();
+  }
+}
+
+// Opt-in, dev-only: mirror this process's structured logs to a local telnet/TCP stream.
+if (env.SUPERVISOR_TELNET_LOGS_PORT && env.SUPERVISOR_TELNET_LOGS_PORT > 0) {
+  const telnetLogServer = startTelnetLogServer({
+    port: env.SUPERVISOR_TELNET_LOGS_PORT,
+    name: "supervisor",
+  });
+  SimpleStructuredLogger.onLog = (log) => telnetLogServer.broadcast(formatLogLine(log));
+}
+
+const worker = new ManagedSupervisor();
+worker.start();

apps/supervisor/src/metrics.ts

@@ -0,0 +1,3 @@
+import { Registry } from "prom-client";
+
+export const register = new Registry();

apps/supervisor/src/resourceMonitor.ts

@@ -0,0 +1,278 @@
+import type Docker from "dockerode";
+import type { MachineResources } from "@trigger.dev/core/v3";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { env } from "./env.js";
+import type { K8sApi } from "./clients/kubernetes.js";
+
+const logger = new SimpleStructuredLogger("resource-monitor");
+
+interface NodeResources {
+  cpuTotal: number; // in cores
+  cpuAvailable: number;
+  memoryTotal: number; // in bytes
+  memoryAvailable: number;
+}
+
+interface ResourceRequest {
+  cpu: number; // in cores
+  memory: number; // in bytes
+}
+
+export abstract class ResourceMonitor {
+  protected cacheTimeoutMs = 5_000;
+  protected lastUpdateMs = 0;
+
+  protected cachedResources: NodeResources = {
+    cpuTotal: 0,
+    cpuAvailable: 0,
+    memoryTotal: 0,
+    memoryAvailable: 0,
+  };
+
+  protected resourceParser: ResourceParser;
+
+  constructor(Parser: new () => ResourceParser) {
+    this.resourceParser = new Parser();
+  }
+
+  abstract getNodeResources(fromCache?: boolean): Promise<NodeResources>;
+
+  blockResources(resources: MachineResources): void {
+    const { cpu, memory } = this.toResourceRequest(resources);
+
+    logger.debug("[ResourceMonitor] Blocking resources", {
+      raw: resources,
+      converted: { cpu, memory },
+    });
+
+    this.cachedResources.cpuAvailable -= cpu;
+    this.cachedResources.memoryAvailable -= memory;
+  }
+
+  async wouldFit(request: ResourceRequest): Promise<boolean> {
+    const resources = await this.getNodeResources();
+    return resources.cpuAvailable >= request.cpu && resources.memoryAvailable >= request.memory;
+  }
+
+  private toResourceRequest(resources: MachineResources): ResourceRequest {
+    return {
+      cpu: resources.cpu ?? 0,
+      memory: this.gbToBytes(resources.memory ?? 0),
+    };
+  }
+
+  private gbToBytes(gb: number): number {
+    return gb * 1024 * 1024 * 1024;
+  }
+
+  protected isCacheValid(): boolean {
+    return this.cachedResources !== null && Date.now() - this.lastUpdateMs < this.cacheTimeoutMs;
+  }
+
+  protected applyOverrides(resources: NodeResources): NodeResources {
+    if (
+      !env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL &&
+      !env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB
+    ) {
+      return resources;
+    }
+
+    logger.debug("[ResourceMonitor] 🛡️ Applying resource overrides", {
+      cpuTotal: env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL,
+      memoryTotalGb: env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB,
+    });
+
+    const cpuTotal = env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL ?? resources.cpuTotal;
+    const memoryTotal = env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB
+      ? this.gbToBytes(env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB)
+      : resources.memoryTotal;
+
+    const cpuDiff = cpuTotal - resources.cpuTotal;
+    const memoryDiff = memoryTotal - resources.memoryTotal;
+
+    const cpuAvailable = Math.max(0, resources.cpuAvailable + cpuDiff);
+    const memoryAvailable = Math.max(0, resources.memoryAvailable + memoryDiff);
+
+    return {
+      cpuTotal,
+      cpuAvailable,
+      memoryTotal,
+      memoryAvailable,
+    };
+  }
+}
+
+type SystemInfo = {
+  NCPU: number | undefined;
+  MemTotal: number | undefined;
+};
+
+export class DockerResourceMonitor extends ResourceMonitor {
+  private docker: Docker;
+
+  constructor(docker: Docker) {
+    super(DockerResourceParser);
+    this.docker = docker;
+  }
+
+  async getNodeResources(fromCache?: boolean): Promise<NodeResources> {
+    if (this.isCacheValid() || fromCache) {
+      // logger.debug("[ResourceMonitor] Using cached resources");
+      return this.cachedResources;
+    }
+
+    const info: SystemInfo = await this.docker.info();
+    const stats = await this.docker.listContainers({ all: true });
+
+    // Get system-wide resources
+    const cpuTotal = info.NCPU ?? 0;
+    const memoryTotal = info.MemTotal ?? 0;
+
+    // Calculate used resources from running containers
+    let cpuUsed = 0;
+    let memoryUsed = 0;
+
+    for (const container of stats) {
+      if (container.State === "running") {
+        const c = this.docker.getContainer(container.Id);
+        const { HostConfig } = await c.inspect();
+
+        const cpu = this.resourceParser.cpu(HostConfig.NanoCpus ?? 0);
+        const memory = this.resourceParser.memory(HostConfig.Memory ?? 0);
+
+        cpuUsed += cpu;
+        memoryUsed += memory;
+      }
+    }
+
+    this.cachedResources = this.applyOverrides({
+      cpuTotal,
+      cpuAvailable: cpuTotal - cpuUsed,
+      memoryTotal,
+      memoryAvailable: memoryTotal - memoryUsed,
+    });
+
+    this.lastUpdateMs = Date.now();
+
+    return this.cachedResources;
+  }
+}
+
+export class KubernetesResourceMonitor extends ResourceMonitor {
+  private k8s: K8sApi;
+  private nodeName: string;
+
+  constructor(k8s: K8sApi, nodeName: string) {
+    super(KubernetesResourceParser);
+    this.k8s = k8s;
+    this.nodeName = nodeName;
+  }
+
+  async getNodeResources(fromCache?: boolean): Promise<NodeResources> {
+    if (this.isCacheValid() || fromCache) {
+      logger.debug("[ResourceMonitor] Using cached resources");
+      return this.cachedResources;
+    }
+
+    const node = await this.k8s.core.readNode({ name: this.nodeName });
+    const pods = await this.k8s.core.listPodForAllNamespaces({
+      // TODO: ensure this includes all pods that consume resources
+      fieldSelector: `spec.nodeName=${this.nodeName},status.phase=Running`,
+    });
+
+    const allocatable = node.status?.allocatable;
+    const cpuTotal = this.resourceParser.cpu(allocatable?.cpu ?? "0");
+    const memoryTotal = this.resourceParser.memory(allocatable?.memory ?? "0");
+
+    // Sum up resources requested by all pods on this node
+    let cpuRequested = 0;
+    let memoryRequested = 0;
+
+    for (const pod of pods.items) {
+      if (pod.status?.phase === "Running") {
+        if (!pod.spec) {
+          continue;
+        }
+
+        for (const container of pod.spec.containers) {
+          const resources = container.resources?.requests ?? {};
+          cpuRequested += this.resourceParser.cpu(resources.cpu ?? "0");
+          memoryRequested += this.resourceParser.memory(resources.memory ?? "0");
+        }
+      }
+    }
+
+    this.cachedResources = this.applyOverrides({
+      cpuTotal,
+      cpuAvailable: cpuTotal - cpuRequested,
+      memoryTotal,
+      memoryAvailable: memoryTotal - memoryRequested,
+    });
+
+    this.lastUpdateMs = Date.now();
+
+    return this.cachedResources;
+  }
+}
+
+export class NoopResourceMonitor extends ResourceMonitor {
+  constructor() {
+    super(NoopResourceParser);
+  }
+
+  async getNodeResources(): Promise<NodeResources> {
+    return {
+      cpuTotal: 0,
+      cpuAvailable: Infinity,
+      memoryTotal: 0,
+      memoryAvailable: Infinity,
+    };
+  }
+}
+
+abstract class ResourceParser {
+  abstract cpu(cpu: number | string): number;
+  abstract memory(memory: number | string): number;
+}
+
+class DockerResourceParser extends ResourceParser {
+  cpu(cpu: number): number {
+    return cpu / 1e9;
+  }
+
+  memory(memory: number): number {
+    return memory;
+  }
+}
+
+class KubernetesResourceParser extends ResourceParser {
+  cpu(cpu: string): number {
+    if (cpu.endsWith("m")) {
+      return parseInt(cpu.slice(0, -1)) / 1000;
+    }
+    return parseInt(cpu);
+  }
+
+  memory(memory: string): number {
+    if (memory.endsWith("Ki")) {
+      return parseInt(memory.slice(0, -2)) * 1024;
+    }
+    if (memory.endsWith("Mi")) {
+      return parseInt(memory.slice(0, -2)) * 1024 * 1024;
+    }
+    if (memory.endsWith("Gi")) {
+      return parseInt(memory.slice(0, -2)) * 1024 * 1024 * 1024;
+    }
+    return parseInt(memory);
+  }
+}
+
+class NoopResourceParser extends ResourceParser {
+  cpu(cpu: number): number {
+    return cpu;
+  }
+
+  memory(memory: number): number {
+    return memory;
+  }
+}

apps/supervisor/src/services/computeSnapshotService.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it, vi } from "vitest";
+import { setTimeout as sleep } from "node:timers/promises";
+import { ComputeSnapshotService } from "./computeSnapshotService.js";
+import type { ComputeWorkloadManager } from "../workloadManager/compute.js";
+import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers";
+
+// The TimerWheel ticks every 100ms, so a 200ms delay dispatches within ~300ms.
+const DELAY_MS = 200;
+// Long enough that a pending snapshot would certainly have dispatched.
+const SETTLE_MS = 600;
+
+function createService() {
+  const snapshot = vi.fn(
+    async (_opts: { runnerId: string; metadata: Record<string, string> }) => true
+  );
+
+  const computeManager = {
+    snapshotDelayMs: DELAY_MS,
+    snapshotDispatchLimit: 1,
+    snapshot,
+  } as unknown as ComputeWorkloadManager;
+
+  const service = new ComputeSnapshotService({
+    computeManager,
+    workerClient: {} as SupervisorHttpClient,
+    wideEventOpts: { service: "supervisor-test", env: {}, enabled: false },
+  });
+
+  return { service, snapshot };
+}
+
+function delayedSnapshot(runnerId = "runner-1") {
+  return {
+    runnerId,
+    runFriendlyId: "run_1",
+    snapshotFriendlyId: "snapshot_1",
+  };
+}
+
+describe("ComputeSnapshotService", () => {
+  it("dispatches a scheduled snapshot after the delay", async () => {
+    const { service, snapshot } = createService();
+    try {
+      service.schedule("run_1", delayedSnapshot());
+
+      await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
+      expect(snapshot).toHaveBeenCalledWith({
+        runnerId: "runner-1",
+        metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" },
+      });
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("cancel before the delay expires prevents the dispatch", async () => {
+    const { service, snapshot } = createService();
+    try {
+      service.schedule("run_1", delayedSnapshot());
+
+      expect(service.cancel("run_1")).toBe(true);
+
+      await sleep(SETTLE_MS);
+      expect(snapshot).not.toHaveBeenCalled();
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("cancel returns false when nothing is pending", () => {
+    const { service } = createService();
+    try {
+      expect(service.cancel("run_1")).toBe(false);
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("cancel with a matching runnerId cancels the pending snapshot", async () => {
+    const { service, snapshot } = createService();
+    try {
+      service.schedule("run_1", delayedSnapshot("runner-a"));
+
+      expect(service.cancel("run_1", "runner-a")).toBe(true);
+
+      await sleep(SETTLE_MS);
+      expect(snapshot).not.toHaveBeenCalled();
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("cancel with a different runnerId leaves the pending snapshot alone", async () => {
+    const { service, snapshot } = createService();
+    try {
+      service.schedule("run_1", delayedSnapshot("runner-a"));
+
+      // A stale runner for a reassigned run must not cancel the new runner's snapshot.
+      expect(service.cancel("run_1", "runner-b")).toBe(false);
+
+      await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
+      expect(snapshot).toHaveBeenCalledWith(expect.objectContaining({ runnerId: "runner-a" }));
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("re-scheduling the same run replaces the pending snapshot", async () => {
+    const { service, snapshot } = createService();
+    try {
+      service.schedule("run_1", delayedSnapshot());
+      service.schedule("run_1", {
+        runnerId: "runner-1",
+        runFriendlyId: "run_1",
+        snapshotFriendlyId: "snapshot_2",
+      });
+
+      await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 });
+      await sleep(SETTLE_MS);
+
+      expect(snapshot).toHaveBeenCalledTimes(1);
+      expect(snapshot).toHaveBeenCalledWith({
+        runnerId: "runner-1",
+        metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_2" },
+      });
+    } finally {
+      service.stop();
+    }
+  });
+});

apps/supervisor/src/services/computeSnapshotService.ts

@@ -0,0 +1,323 @@
+import pLimit from "p-limit";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
+import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers";
+import { type SnapshotCallbackPayload } from "@internal/compute";
+import type { ComputeWorkloadManager } from "../workloadManager/compute.js";
+import { TimerWheel } from "./timerWheel.js";
+import type { OtlpTraceService } from "./otlpTraceService.js";
+import {
+  emitOneShot,
+  fromContext,
+  recordPhaseSince,
+  runWideEvent,
+  setExtra,
+  setMeta,
+  type WideEventOptions,
+} from "../wideEvents/index.js";
+
+type DelayedSnapshot = {
+  runnerId: string;
+  runFriendlyId: string;
+  snapshotFriendlyId: string;
+};
+
+export type RunTraceContext = {
+  traceparent: string;
+  envId: string;
+  orgId: string;
+  projectId: string;
+};
+
+export type ComputeSnapshotServiceOptions = {
+  computeManager: ComputeWorkloadManager;
+  workerClient: SupervisorHttpClient;
+  tracing?: OtlpTraceService;
+  wideEventOpts: WideEventOptions;
+};
+
+export class ComputeSnapshotService {
+  private readonly logger = new SimpleStructuredLogger("compute-snapshot-service");
+
+  private static readonly MAX_TRACE_CONTEXTS = 10_000;
+  private readonly runTraceContexts = new Map<string, RunTraceContext>();
+  private readonly timerWheel: TimerWheel<DelayedSnapshot>;
+  private readonly dispatchLimit: ReturnType<typeof pLimit>;
+
+  private readonly computeManager: ComputeWorkloadManager;
+  private readonly workerClient: SupervisorHttpClient;
+  private readonly tracing?: OtlpTraceService;
+  private readonly wideEventOpts: WideEventOptions;
+
+  constructor(opts: ComputeSnapshotServiceOptions) {
+    this.computeManager = opts.computeManager;
+    this.workerClient = opts.workerClient;
+    this.tracing = opts.tracing;
+    this.wideEventOpts = opts.wideEventOpts;
+
+    this.dispatchLimit = pLimit(this.computeManager.snapshotDispatchLimit);
+    this.timerWheel = new TimerWheel<DelayedSnapshot>({
+      delayMs: this.computeManager.snapshotDelayMs,
+      onExpire: (item) => {
+        this.dispatchLimit(() => this.dispatch(item.data)).catch((error) => {
+          this.logger.error("Snapshot dispatch failed", {
+            runId: item.data.runFriendlyId,
+            runnerId: item.data.runnerId,
+            error,
+          });
+        });
+      },
+    });
+    this.timerWheel.start();
+  }
+
+  /** Schedule a delayed snapshot for a run. Replaces any pending snapshot for the same run. */
+  schedule(runFriendlyId: string, data: DelayedSnapshot) {
+    this.timerWheel.submit(runFriendlyId, data);
+    emitOneShot({
+      ...this.wideEventOpts,
+      op: "snapshot.schedule",
+      kind: "event",
+      populate: (state) => {
+        state.meta.run_id = runFriendlyId;
+        state.meta.snapshot_id = data.snapshotFriendlyId;
+        state.extras.runner_id = data.runnerId;
+        state.extras.delay_ms = this.computeManager.snapshotDelayMs;
+      },
+    });
+    this.logger.debug("Snapshot scheduled", {
+      runFriendlyId,
+      snapshotFriendlyId: data.snapshotFriendlyId,
+      delayMs: this.computeManager.snapshotDelayMs,
+    });
+  }
+
+  /**
+   * Cancel a pending delayed snapshot. Returns true if one was cancelled.
+   * When `runnerId` is given, only a snapshot scheduled for that same runner
+   * is cancelled - a stale runner for a run that has since been reassigned
+   * must not cancel the new runner's pending snapshot.
+   */
+  cancel(runFriendlyId: string, runnerId?: string): boolean {
+    if (runnerId) {
+      const pending = this.timerWheel.peek(runFriendlyId);
+      if (pending && pending.data.runnerId !== runnerId) {
+        return false;
+      }
+    }
+    const cancelled = this.timerWheel.cancel(runFriendlyId);
+    if (cancelled) {
+      emitOneShot({
+        ...this.wideEventOpts,
+        op: "snapshot.canceled",
+        kind: "event",
+        populate: (state) => {
+          state.meta.run_id = runFriendlyId;
+        },
+      });
+      this.logger.debug("Snapshot cancelled", { runFriendlyId });
+    }
+    return cancelled;
+  }
+
+  /** Handle the callback from the gateway after a snapshot completes or fails. */
+  async handleCallback(body: SnapshotCallbackPayload) {
+    const snapshotId = body.status === "completed" ? body.snapshot_id : undefined;
+    const runId = body.metadata?.runId;
+    const snapshotFriendlyId = body.metadata?.snapshotFriendlyId;
+
+    // Enrich the wrapping route's wide event with snapshot metadata. The
+    // `/api/v1/compute/snapshot-complete` route is registered with `wideRoute`,
+    // so `fromContext()` returns the State of that route and these calls
+    // become extras/meta on the same wide event - no nested emission.
+    const state = fromContext();
+    if (state) {
+      state.extras["snapshot.status"] = body.status;
+      if (body.instance_id) state.extras["snapshot.instance_id"] = body.instance_id;
+      if (body.duration_ms !== undefined) state.extras["snapshot.duration_ms"] = body.duration_ms;
+      if (snapshotId) state.extras["snapshot.id"] = snapshotId;
+      if (body.status === "failed" && body.error) state.extras["snapshot.error"] = body.error;
+    }
+    if (runId) setMeta(state, "run_id", runId);
+    if (snapshotFriendlyId) setMeta(state, "snapshot_id", snapshotFriendlyId);
+
+    this.logger.debug("Snapshot callback", {
+      snapshotId,
+      instanceId: body.instance_id,
+      status: body.status,
+      error: body.status === "failed" ? body.error : undefined,
+      metadata: body.metadata,
+      durationMs: body.duration_ms,
+    });
+
+    if (!runId || !snapshotFriendlyId) {
+      this.logger.error("Snapshot callback missing metadata", { body });
+      return { ok: false as const, status: 400 };
+    }
+
+    this.#emitSnapshotSpan(runId, body.duration_ms, snapshotId);
+
+    if (body.status === "completed") {
+      const submitStart = performance.now();
+      const result = await this.workerClient.submitSuspendCompletion({
+        runId,
+        snapshotId: snapshotFriendlyId,
+        body: {
+          success: true,
+          checkpoint: {
+            type: "COMPUTE",
+            location: body.snapshot_id,
+          },
+        },
+      });
+      recordPhaseSince(
+        "submit_completion",
+        submitStart,
+        result.success ? undefined : new Error(String(result.error))
+      );
+
+      if (result.success) {
+        this.logger.debug("Suspend completion submitted", {
+          runId,
+          instanceId: body.instance_id,
+          snapshotId: body.snapshot_id,
+        });
+      } else {
+        setExtra(state, "submit_completion.error", String(result.error));
+        this.logger.error("Failed to submit suspend completion", {
+          runId,
+          snapshotFriendlyId,
+          error: result.error,
+        });
+      }
+    } else {
+      const submitStart = performance.now();
+      const result = await this.workerClient.submitSuspendCompletion({
+        runId,
+        snapshotId: snapshotFriendlyId,
+        body: {
+          success: false,
+          error: body.error ?? "Snapshot failed",
+        },
+      });
+      recordPhaseSince(
+        "submit_completion",
+        submitStart,
+        result.success ? undefined : new Error(String(result.error))
+      );
+
+      if (!result.success) {
+        setExtra(state, "submit_completion.error", String(result.error));
+        this.logger.error("Failed to submit suspend failure", {
+          runId,
+          snapshotFriendlyId,
+          error: result.error,
+        });
+      }
+    }
+
+    return { ok: true as const, status: 200 };
+  }
+
+  registerTraceContext(runFriendlyId: string, ctx: RunTraceContext) {
+    // Evict oldest entries if we've hit the cap. This is best-effort: on a busy
+    // supervisor, entries for long-lived runs may be evicted before their snapshot
+    // callback arrives, causing those snapshot spans to be silently dropped.
+    // That's acceptable - trace spans are observability sugar, not correctness.
+    if (this.runTraceContexts.size >= ComputeSnapshotService.MAX_TRACE_CONTEXTS) {
+      const firstKey = this.runTraceContexts.keys().next().value;
+      if (firstKey) {
+        this.runTraceContexts.delete(firstKey);
+      }
+    }
+
+    this.runTraceContexts.set(runFriendlyId, ctx);
+  }
+
+  /** Stop the timer wheel, dropping pending snapshots. */
+  stop(): string[] {
+    // Intentionally drop pending snapshots rather than dispatching them. The supervisor
+    // is shutting down, so our callback URL will be dead by the time the gateway responds.
+    // Runners detect the supervisor is gone and reconnect to a new instance, which
+    // re-triggers the snapshot workflow. Snapshots are an optimization, not a correctness
+    // requirement - runs continue fine without them.
+    const remaining = this.timerWheel.stop();
+    const droppedRuns = remaining.map((item) => item.key);
+
+    if (droppedRuns.length > 0) {
+      this.logger.info("Stopped, dropped pending snapshots", { count: droppedRuns.length });
+      this.logger.debug("Dropped snapshot details", { runs: droppedRuns });
+    }
+
+    return droppedRuns;
+  }
+
+  /** Dispatch a snapshot request to the gateway. */
+  private async dispatch(snapshot: DelayedSnapshot): Promise<void> {
+    await runWideEvent(
+      {
+        ...this.wideEventOpts,
+        op: "snapshot.dispatch",
+        kind: "scheduled",
+        setup: (state) => {
+          state.meta.run_id = snapshot.runFriendlyId;
+          state.meta.snapshot_id = snapshot.snapshotFriendlyId;
+          state.extras.runner_id = snapshot.runnerId;
+        },
+      },
+      async () => {
+        const result = await this.computeManager.snapshot({
+          runnerId: snapshot.runnerId,
+          metadata: {
+            runId: snapshot.runFriendlyId,
+            snapshotFriendlyId: snapshot.snapshotFriendlyId,
+          },
+        });
+
+        if (!result) {
+          throw new Error("Snapshot dispatch returned no result");
+        }
+      }
+    );
+  }
+
+  #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) {
+    if (!this.tracing) return;
+
+    const ctx = this.runTraceContexts.get(runFriendlyId);
+    if (!ctx) return;
+
+    const parsed = parseTraceparent(ctx.traceparent);
+    if (!parsed) return;
+
+    const endEpochMs = Date.now();
+    const startEpochMs = durationMs ? endEpochMs - durationMs : endEpochMs;
+
+    const spanAttributes: Record<string, string | number | boolean> = {
+      "compute.type": "snapshot",
+    };
+
+    if (durationMs !== undefined) {
+      spanAttributes["compute.total_ms"] = durationMs;
+    }
+
+    if (snapshotId) {
+      spanAttributes["compute.snapshot_id"] = snapshotId;
+    }
+
+    this.tracing.emit({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.snapshot",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": ctx.envId,
+        "ctx.organization.id": ctx.orgId,
+        "ctx.project.id": ctx.projectId,
+        "ctx.run.id": runFriendlyId,
+      },
+      spanAttributes,
+    });
+  }
+}

apps/supervisor/src/services/failedPodHandler.test.ts

@@ -0,0 +1,581 @@
+import { describe, it, expect, beforeAll, afterEach } from "vitest";
+import { FailedPodHandler } from "./failedPodHandler.js";
+import { type K8sApi, createK8sApi } from "../clients/kubernetes.js";
+import { Registry } from "prom-client";
+import { setTimeout } from "timers/promises";
+
+// These tests require live K8s cluster credentials - skip by default
+describe.skipIf(!process.env.K8S_INTEGRATION_TESTS)("FailedPodHandler Integration Tests", () => {
+  const k8s = createK8sApi();
+  const namespace = "integration-test";
+  const register = new Registry();
+
+  beforeAll(async () => {
+    // Create the test namespace if it doesn't exist
+    try {
+      await k8s.core.readNamespace({ name: namespace });
+    } catch (_error) {
+      await k8s.core.createNamespace({
+        body: {
+          metadata: {
+            name: namespace,
+          },
+        },
+      });
+    }
+
+    // Clear any existing pods in the namespace
+    await deleteAllPodsInNamespace({ k8sApi: k8s, namespace });
+  });
+
+  afterEach(async () => {
+    // Clear metrics to avoid conflicts
+    register.clear();
+
+    // Delete any remaining pods in the namespace
+    await deleteAllPodsInNamespace({ k8sApi: k8s, namespace });
+  });
+
+  it("should process and delete failed pods with app=task-run label", async () => {
+    const handler = new FailedPodHandler({ namespace, k8s, register });
+
+    try {
+      // Create failed pods with the correct label
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 2,
+        shouldFail: true,
+      });
+
+      // Wait for pods to reach Failed state
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Failed",
+      });
+
+      // Start the handler
+      await handler.start();
+
+      // Wait for pods to be deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      // Verify metrics
+      const metrics = handler.getMetrics();
+
+      // Check informer events were recorded
+      const informerEvents = await metrics.informerEventsTotal.get();
+      expect(informerEvents.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "add",
+          }),
+          value: 2,
+        })
+      );
+      expect(informerEvents.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "connect",
+          }),
+          value: 1,
+        })
+      );
+      expect(informerEvents.values).not.toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "error",
+          }),
+        })
+      );
+
+      // Check pods were processed
+      const processedPods = await metrics.processedPodsTotal.get();
+      expect(processedPods.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            status: "Failed",
+          }),
+          value: 2,
+        })
+      );
+
+      // Check pods were deleted
+      const deletedPods = await metrics.deletedPodsTotal.get();
+      expect(deletedPods.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            status: "Failed",
+          }),
+          value: 2,
+        })
+      );
+
+      // Check no deletion errors were recorded
+      const deletionErrors = await metrics.deletionErrorsTotal.get();
+      expect(deletionErrors.values).toHaveLength(0);
+
+      // Check processing durations were recorded
+      const durations = await metrics.processingDurationSeconds.get();
+      const failedDurations = durations.values.filter(
+        (v) => v.labels.namespace === namespace && v.labels.status === "Failed"
+      );
+      expect(failedDurations.length).toBeGreaterThan(0);
+    } finally {
+      await handler.stop();
+    }
+  }, 30000);
+
+  it("should ignore pods without app=task-run label", async () => {
+    const handler = new FailedPodHandler({ namespace, k8s, register });
+
+    try {
+      // Create failed pods without the task-run label
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        shouldFail: true,
+        labels: { app: "not-task-run" },
+      });
+
+      // Wait for pod to reach Failed state
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Failed",
+      });
+
+      await handler.start();
+
+      // Wait a reasonable time to ensure pod isn't deleted
+      await setTimeout(5000);
+
+      // Verify pod still exists
+      const exists = await podExists({ k8sApi: k8s, namespace, podName: podNames[0]! });
+      expect(exists).toBe(true);
+
+      // Verify no metrics were recorded
+      const metrics = handler.getMetrics();
+      const processedPods = await metrics.processedPodsTotal.get();
+      expect(processedPods.values).toHaveLength(0);
+    } finally {
+      await handler.stop();
+    }
+  }, 30000);
+
+  it("should not process pods that are being deleted", async () => {
+    const handler = new FailedPodHandler({ namespace, k8s, register });
+
+    try {
+      // Create a failed pod that we'll mark for deletion
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        shouldFail: true,
+        command: ["/bin/sh", "-c", "sleep 30"],
+      });
+
+      // Wait for pod to reach Failed state
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Running",
+      });
+
+      // Delete the pod but don't wait for deletion
+      await k8s.core.deleteNamespacedPod({
+        namespace,
+        name: podNames[0]!,
+        gracePeriodSeconds: 5,
+      });
+
+      // Start the handler
+      await handler.start();
+
+      // Wait for pod to be fully deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      // Verify metrics show we skipped processing
+      const metrics = handler.getMetrics();
+      const processedPods = await metrics.processedPodsTotal.get();
+      expect(processedPods.values).toHaveLength(0);
+    } finally {
+      await handler.stop();
+    }
+  }, 30000);
+
+  it("should detect and process pods that fail after handler starts", async () => {
+    const handler = new FailedPodHandler({ namespace, k8s, register });
+
+    try {
+      // Start the handler
+      await handler.start();
+
+      // Create failed pods with the correct label
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 3,
+        shouldFail: true,
+      });
+
+      // Wait for pods to be deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      // Verify metrics
+      const metrics = handler.getMetrics();
+
+      // Check informer events were recorded
+      const informerEvents = await metrics.informerEventsTotal.get();
+      expect(informerEvents.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "add",
+          }),
+          value: 3,
+        })
+      );
+      expect(informerEvents.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "connect",
+          }),
+          value: 1,
+        })
+      );
+      expect(informerEvents.values).not.toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "error",
+          }),
+        })
+      );
+
+      // Check pods were processed
+      const processedPods = await metrics.processedPodsTotal.get();
+      expect(processedPods.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            status: "Failed",
+          }),
+          value: 3,
+        })
+      );
+
+      // Check pods were deleted
+      const deletedPods = await metrics.deletedPodsTotal.get();
+      expect(deletedPods.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            status: "Failed",
+          }),
+          value: 3,
+        })
+      );
+
+      // Check no deletion errors were recorded
+      const deletionErrors = await metrics.deletionErrorsTotal.get();
+      expect(deletionErrors.values).toHaveLength(0);
+
+      // Check processing durations were recorded
+      const durations = await metrics.processingDurationSeconds.get();
+      const failedDurations = durations.values.filter(
+        (v) => v.labels.namespace === namespace && v.labels.status === "Failed"
+      );
+      expect(failedDurations.length).toBeGreaterThan(0);
+    } finally {
+      await handler.stop();
+    }
+  }, 60000);
+
+  it("should handle graceful shutdown pods differently", async () => {
+    const handler = new FailedPodHandler({ namespace, k8s, register });
+
+    try {
+      // Create first batch of pods before starting handler
+      const firstBatchPodNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 2,
+        exitCode: FailedPodHandler.GRACEFUL_SHUTDOWN_EXIT_CODE,
+      });
+
+      // Wait for pods to reach Failed state
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames: firstBatchPodNames,
+        phase: "Failed",
+      });
+
+      // Start the handler
+      await handler.start();
+
+      // Wait for first batch to be deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames: firstBatchPodNames,
+      });
+
+      // Create second batch of pods after handler is running
+      const secondBatchPodNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 3,
+        exitCode: FailedPodHandler.GRACEFUL_SHUTDOWN_EXIT_CODE,
+      });
+
+      // Wait for second batch to be deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames: secondBatchPodNames,
+      });
+
+      // Verify metrics
+      const metrics = handler.getMetrics();
+
+      // Check informer events were recorded for both batches
+      const informerEvents = await metrics.informerEventsTotal.get();
+      expect(informerEvents.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            verb: "add",
+          }),
+          value: 5, // 2 from first batch + 3 from second batch
+        })
+      );
+
+      // Check pods were processed as graceful shutdowns
+      const processedPods = await metrics.processedPodsTotal.get();
+
+      // Should not be marked as Failed
+      const failedPods = processedPods.values.find(
+        (v) => v.labels.namespace === namespace && v.labels.status === "Failed"
+      );
+      expect(failedPods).toBeUndefined();
+
+      // Should be marked as GracefulShutdown
+      const gracefulShutdowns = processedPods.values.find(
+        (v) => v.labels.namespace === namespace && v.labels.status === "GracefulShutdown"
+      );
+      expect(gracefulShutdowns).toBeDefined();
+      expect(gracefulShutdowns?.value).toBe(5); // Total from both batches
+
+      // Check pods were still deleted
+      const deletedPods = await metrics.deletedPodsTotal.get();
+      expect(deletedPods.values).toContainEqual(
+        expect.objectContaining({
+          labels: expect.objectContaining({
+            namespace,
+            status: "Failed",
+          }),
+          value: 5, // Total from both batches
+        })
+      );
+
+      // Check no deletion errors were recorded
+      const deletionErrors = await metrics.deletionErrorsTotal.get();
+      expect(deletionErrors.values).toHaveLength(0);
+    } finally {
+      await handler.stop();
+    }
+  }, 30000);
+});
+
+async function createTestPods({
+  k8sApi,
+  namespace,
+  count,
+  labels = { app: "task-run" },
+  shouldFail = false,
+  namePrefix = "test-pod",
+  command = ["/bin/sh", "-c", shouldFail ? "exit 1" : "exit 0"],
+  randomizeName = true,
+  exitCode,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  count: number;
+  labels?: Record<string, string>;
+  shouldFail?: boolean;
+  namePrefix?: string;
+  command?: string[];
+  randomizeName?: boolean;
+  exitCode?: number;
+}) {
+  const createdPods: string[] = [];
+
+  // If exitCode is specified, override the command
+  if (exitCode !== undefined) {
+    command = ["/bin/sh", "-c", `exit ${exitCode}`];
+  }
+
+  for (let i = 0; i < count; i++) {
+    const podName = randomizeName
+      ? `${namePrefix}-${i}-${Math.random().toString(36).substring(2, 15)}`
+      : `${namePrefix}-${i}`;
+    await k8sApi.core.createNamespacedPod({
+      namespace,
+      body: {
+        metadata: {
+          name: podName,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          containers: [
+            {
+              name: "run-controller", // Changed to match the name we check in failedPodHandler
+              image: "busybox:1.37.0",
+              command,
+            },
+          ],
+        },
+      },
+    });
+    createdPods.push(podName);
+  }
+
+  return createdPods;
+}
+
+async function waitForPodsDeletion({
+  k8sApi,
+  namespace,
+  podNames,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podNames: string[];
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+  const pendingPods = new Set(podNames);
+
+  while (pendingPods.size > 0 && Date.now() - startTime < timeoutMs) {
+    const pods = await k8sApi.core.listNamespacedPod({ namespace });
+    const existingPods = new Set(pods.items.map((pod) => pod.metadata?.name ?? ""));
+
+    for (const podName of pendingPods) {
+      if (!existingPods.has(podName)) {
+        pendingPods.delete(podName);
+      }
+    }
+
+    if (pendingPods.size > 0) {
+      await setTimeout(waitMs);
+    }
+  }
+
+  if (pendingPods.size > 0) {
+    throw new Error(
+      `Pods [${Array.from(pendingPods).join(", ")}] were not deleted within ${timeoutMs}ms`
+    );
+  }
+}
+
+async function podExists({
+  k8sApi,
+  namespace,
+  podName,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podName: string;
+}) {
+  const pods = await k8sApi.core.listNamespacedPod({ namespace });
+  return pods.items.some((p) => p.metadata?.name === podName);
+}
+
+async function waitForPodsPhase({
+  k8sApi,
+  namespace,
+  podNames,
+  phase,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podNames: string[];
+  phase: "Pending" | "Running" | "Succeeded" | "Failed" | "Unknown";
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+  const pendingPods = new Set(podNames);
+
+  while (pendingPods.size > 0 && Date.now() - startTime < timeoutMs) {
+    const pods = await k8sApi.core.listNamespacedPod({ namespace });
+
+    for (const pod of pods.items) {
+      if (pendingPods.has(pod.metadata?.name ?? "") && pod.status?.phase === phase) {
+        pendingPods.delete(pod.metadata?.name ?? "");
+      }
+    }
+
+    if (pendingPods.size > 0) {
+      await setTimeout(waitMs);
+    }
+  }
+
+  if (pendingPods.size > 0) {
+    throw new Error(
+      `Pods [${Array.from(pendingPods).join(
+        ", "
+      )}] did not reach phase ${phase} within ${timeoutMs}ms`
+    );
+  }
+}
+
+async function deleteAllPodsInNamespace({
+  k8sApi,
+  namespace,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+}) {
+  // Get all pods
+  const pods = await k8sApi.core.listNamespacedPod({ namespace });
+  const podNames = pods.items.map((p) => p.metadata?.name ?? "");
+
+  // Delete all pods
+  await k8sApi.core.deleteCollectionNamespacedPod({ namespace, gracePeriodSeconds: 0 });
+
+  // Wait for all pods to be deleted
+  await waitForPodsDeletion({ k8sApi, namespace, podNames });
+}

apps/supervisor/src/services/failedPodHandler.ts

@@ -0,0 +1,327 @@
+import type { Informer, V1Pod } from "@kubernetes/client-node";
+import { LogLevel, SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import type { Registry } from "prom-client";
+import { Counter, Histogram } from "prom-client";
+import { setTimeout } from "timers/promises";
+import type { K8sApi } from "../clients/kubernetes.js";
+import { createK8sApi } from "../clients/kubernetes.js";
+import { register } from "../metrics.js";
+
+type PodStatus = "Pending" | "Running" | "Succeeded" | "Failed" | "Unknown" | "GracefulShutdown";
+
+export type FailedPodHandlerOptions = {
+  namespace: string;
+  reconnectIntervalMs?: number;
+  k8s?: K8sApi;
+  register?: Registry;
+};
+
+export class FailedPodHandler {
+  private readonly id: string;
+  private readonly logger: SimpleStructuredLogger;
+  private readonly k8s: K8sApi;
+  private readonly namespace: string;
+
+  private isRunning = false;
+
+  private readonly informer: Informer<V1Pod>;
+  private readonly reconnectIntervalMs: number;
+  private reconnecting = false;
+
+  // Metrics
+  private readonly register: Registry;
+  private readonly processedPodsTotal: Counter;
+  private readonly deletedPodsTotal: Counter;
+  private readonly deletionErrorsTotal: Counter;
+  private readonly processingDurationSeconds: Histogram<string>;
+  private readonly informerEventsTotal: Counter;
+
+  static readonly GRACEFUL_SHUTDOWN_EXIT_CODE = 200;
+
+  constructor(opts: FailedPodHandlerOptions) {
+    this.id = Math.random().toString(36).substring(2, 15);
+    this.logger = new SimpleStructuredLogger("failed-pod-handler", LogLevel.debug, {
+      id: this.id,
+    });
+
+    this.k8s = opts.k8s ?? createK8sApi();
+
+    this.namespace = opts.namespace;
+    this.reconnectIntervalMs = opts.reconnectIntervalMs ?? 1000;
+
+    this.informer = this.k8s.makeInformer(
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      () =>
+        this.k8s.core.listNamespacedPod({
+          namespace: this.namespace,
+          labelSelector: "app=task-run",
+          fieldSelector: "status.phase=Failed",
+        }),
+      "app=task-run",
+      "status.phase=Failed"
+    );
+
+    // Whenever a matching pod is added to the informer cache
+    this.informer.on("add", this.onPodCompleted.bind(this));
+
+    // Informer events
+    this.informer.on("connect", this.makeOnConnect("failed-pod-informer").bind(this));
+    this.informer.on("error", this.makeOnError("failed-pod-informer").bind(this));
+
+    // Initialize metrics
+    this.register = opts.register ?? register;
+
+    this.processedPodsTotal = new Counter({
+      name: "failed_pod_handler_processed_pods_total",
+      help: "Total number of failed pods processed",
+      labelNames: ["namespace", "status"],
+      registers: [this.register],
+    });
+
+    this.deletedPodsTotal = new Counter({
+      name: "failed_pod_handler_deleted_pods_total",
+      help: "Total number of pods deleted",
+      labelNames: ["namespace", "status"],
+      registers: [this.register],
+    });
+
+    this.deletionErrorsTotal = new Counter({
+      name: "failed_pod_handler_deletion_errors_total",
+      help: "Total number of errors encountered while deleting pods",
+      labelNames: ["namespace", "error_type"],
+      registers: [this.register],
+    });
+
+    this.processingDurationSeconds = new Histogram({
+      name: "failed_pod_handler_processing_duration_seconds",
+      help: "The duration of pod processing",
+      labelNames: ["namespace", "status"],
+      registers: [this.register],
+    });
+
+    this.informerEventsTotal = new Counter({
+      name: "failed_pod_handler_informer_events_total",
+      help: "Total number of informer events",
+      labelNames: ["namespace", "verb"],
+      registers: [this.register],
+    });
+  }
+
+  async start() {
+    if (this.isRunning) {
+      this.logger.warn("failed pod handler already running");
+      return;
+    }
+
+    this.isRunning = true;
+
+    this.logger.info("starting failed pod handler");
+    await this.informer.start();
+  }
+
+  async stop() {
+    if (!this.isRunning) {
+      this.logger.warn("failed pod handler not running");
+      return;
+    }
+
+    this.isRunning = false;
+
+    this.logger.info("stopping failed pod handler");
+    await this.informer.stop();
+  }
+
+  private async withHistogram<T>(
+    histogram: Histogram<string>,
+    promise: Promise<T>,
+    labels?: Record<string, string>
+  ): Promise<T> {
+    const end = histogram.startTimer({ namespace: this.namespace, ...labels });
+    try {
+      return await promise;
+    } finally {
+      end();
+    }
+  }
+
+  /**
+   * Returns the non-nullable status of a pod
+   */
+  private podStatus(pod: V1Pod): PodStatus {
+    return (pod.status?.phase ?? "Unknown") as PodStatus;
+  }
+
+  private async onPodCompleted(pod: V1Pod) {
+    this.logger.debug("pod-completed", this.podSummary(pod));
+    this.informerEventsTotal.inc({ namespace: this.namespace, verb: "add" });
+
+    if (!pod.metadata?.name) {
+      this.logger.error("pod-completed: no name", this.podSummary(pod));
+      return;
+    }
+
+    if (!pod.status) {
+      this.logger.error("pod-completed: no status", this.podSummary(pod));
+      return;
+    }
+
+    if (pod.metadata?.deletionTimestamp) {
+      this.logger.verbose("pod-completed: pod is being deleted", this.podSummary(pod));
+      return;
+    }
+
+    const podStatus = this.podStatus(pod);
+
+    switch (podStatus) {
+      case "Succeeded":
+        await this.withHistogram(this.processingDurationSeconds, this.onPodSucceeded(pod), {
+          status: podStatus,
+        });
+        break;
+      case "Failed":
+        await this.withHistogram(this.processingDurationSeconds, this.onPodFailed(pod), {
+          status: podStatus,
+        });
+        break;
+      default:
+        this.logger.error("pod-completed: unknown phase", this.podSummary(pod));
+    }
+  }
+
+  private async onPodSucceeded(pod: V1Pod) {
+    this.logger.debug("pod-succeeded", this.podSummary(pod));
+    this.processedPodsTotal.inc({
+      namespace: this.namespace,
+      status: this.podStatus(pod),
+    });
+  }
+
+  private async onPodFailed(pod: V1Pod) {
+    this.logger.debug("pod-failed", this.podSummary(pod));
+
+    try {
+      await this.processFailedPod(pod);
+    } catch (error) {
+      this.logger.error("pod-failed: error processing pod", this.podSummary(pod), { error });
+    } finally {
+      await this.deletePod(pod);
+    }
+  }
+
+  private async processFailedPod(pod: V1Pod) {
+    this.logger.verbose("pod-failed: processing pod", this.podSummary(pod));
+
+    const mainContainer = pod.status?.containerStatuses?.find((c) => c.name === "run-controller");
+
+    // If it's our special "graceful shutdown" exit code, don't process it further, just delete it
+    if (
+      mainContainer?.state?.terminated?.exitCode === FailedPodHandler.GRACEFUL_SHUTDOWN_EXIT_CODE
+    ) {
+      this.logger.debug("pod-failed: graceful shutdown detected", this.podSummary(pod));
+      this.processedPodsTotal.inc({
+        namespace: this.namespace,
+        status: "GracefulShutdown",
+      });
+      return;
+    }
+
+    this.processedPodsTotal.inc({
+      namespace: this.namespace,
+      status: this.podStatus(pod),
+    });
+  }
+
+  private async deletePod(pod: V1Pod) {
+    this.logger.verbose("pod-failed: deleting pod", this.podSummary(pod));
+    try {
+      await this.k8s.core.deleteNamespacedPod({
+        name: pod.metadata!.name!,
+        namespace: this.namespace,
+      });
+      this.deletedPodsTotal.inc({
+        namespace: this.namespace,
+        status: this.podStatus(pod),
+      });
+    } catch (error) {
+      this.logger.error("pod-failed: error deleting pod", this.podSummary(pod), { error });
+      this.deletionErrorsTotal.inc({
+        namespace: this.namespace,
+        error_type: error instanceof Error ? error.name : "unknown",
+      });
+    }
+  }
+
+  private makeOnError(informerName: string) {
+    return (err?: unknown) => this.onError(informerName, err);
+  }
+
+  private async onError(informerName: string, err?: unknown) {
+    if (!this.isRunning) {
+      this.logger.warn("onError: informer not running");
+      return;
+    }
+
+    // Guard against multiple simultaneous reconnections
+    if (this.reconnecting) {
+      this.logger.debug("onError: reconnection already in progress, skipping", {
+        informerName,
+      });
+      return;
+    }
+
+    this.reconnecting = true;
+
+    try {
+      const error = err instanceof Error ? err : undefined;
+      this.logger.error("error event fired", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+      });
+      this.informerEventsTotal.inc({ namespace: this.namespace, verb: "error" });
+
+      // Reconnect on errors
+      await setTimeout(this.reconnectIntervalMs);
+      await this.informer.start();
+    } catch (handlerError) {
+      const error = handlerError instanceof Error ? handlerError : undefined;
+      this.logger.error("onError: reconnection attempt failed", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+        errorStack: error?.stack,
+      });
+    } finally {
+      this.reconnecting = false;
+    }
+  }
+
+  private makeOnConnect(informerName: string) {
+    return () => this.onConnect(informerName);
+  }
+
+  private async onConnect(informerName: string) {
+    this.logger.info(`informer connected: ${informerName}`);
+    this.informerEventsTotal.inc({ namespace: this.namespace, verb: "connect" });
+  }
+
+  private podSummary(pod: V1Pod) {
+    return {
+      name: pod.metadata?.name,
+      namespace: pod.metadata?.namespace,
+      status: pod.status?.phase,
+      deletionTimestamp: pod.metadata?.deletionTimestamp,
+    };
+  }
+
+  // Method to expose metrics for testing
+  public getMetrics() {
+    return {
+      processedPodsTotal: this.processedPodsTotal,
+      deletedPodsTotal: this.deletedPodsTotal,
+      deletionErrorsTotal: this.deletionErrorsTotal,
+      informerEventsTotal: this.informerEventsTotal,
+      processingDurationSeconds: this.processingDurationSeconds,
+    };
+  }
+}

apps/supervisor/src/services/otlpTraceService.test.ts

@@ -0,0 +1,177 @@
+import { describe, it, expect } from "vitest";
+import { buildPayload } from "./otlpTraceService.js";
+
+describe("buildPayload", () => {
+  it("builds valid OTLP JSON with timing attributes", () => {
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      parentSpanId: "1234567890abcdef",
+      spanName: "compute.provision",
+      startTimeMs: 1000,
+      endTimeMs: 1250,
+      resourceAttributes: {
+        "ctx.environment.id": "env_123",
+        "ctx.organization.id": "org_456",
+        "ctx.project.id": "proj_789",
+        "ctx.run.id": "run_abc",
+      },
+      spanAttributes: {
+        "compute.total_ms": 250,
+        "compute.gateway.schedule_ms": 1,
+        "compute.cache.image_cached": true,
+      },
+    });
+
+    expect(payload.resourceSpans).toHaveLength(1);
+
+    const resourceSpan = payload.resourceSpans[0]!;
+
+    // $trigger=true so the webapp accepts it
+    const triggerAttr = resourceSpan.resource.attributes.find((a) => a.key === "$trigger");
+    expect(triggerAttr).toEqual({ key: "$trigger", value: { boolValue: true } });
+
+    // Resource attributes
+    const envAttr = resourceSpan.resource.attributes.find((a) => a.key === "ctx.environment.id");
+    expect(envAttr).toEqual({
+      key: "ctx.environment.id",
+      value: { stringValue: "env_123" },
+    });
+
+    // Span basics
+    const span = resourceSpan.scopeSpans[0]!.spans[0]!;
+    expect(span.name).toBe("compute.provision");
+    expect(span.traceId).toBe("abcd1234abcd1234abcd1234abcd1234");
+    expect(span.parentSpanId).toBe("1234567890abcdef");
+
+    // Integer attribute
+    const totalMs = span.attributes.find((a) => a.key === "compute.total_ms");
+    expect(totalMs).toEqual({ key: "compute.total_ms", value: { intValue: 250 } });
+
+    // Boolean attribute
+    const cached = span.attributes.find((a) => a.key === "compute.cache.image_cached");
+    expect(cached).toEqual({ key: "compute.cache.image_cached", value: { boolValue: true } });
+  });
+
+  it("generates a valid 16-char hex span ID", () => {
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.spanId).toMatch(/^[0-9a-f]{16}$/);
+  });
+
+  it("converts timestamps to nanoseconds", () => {
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1250,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.startTimeUnixNano).toBe("1000000000");
+    expect(span.endTimeUnixNano).toBe("1250000000");
+  });
+
+  it("converts real epoch timestamps without precision loss", () => {
+    // Date.now() values exceed Number.MAX_SAFE_INTEGER when multiplied by 1e6
+    const startMs = 1711929600000; // 2024-04-01T00:00:00Z
+    const endMs = 1711929600250;
+
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: startMs,
+      endTimeMs: endMs,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.startTimeUnixNano).toBe("1711929600000000000");
+    expect(span.endTimeUnixNano).toBe("1711929600250000000");
+  });
+
+  it("preserves sub-millisecond precision from performance.now() arithmetic", () => {
+    // provisionStartEpochMs = Date.now() - (performance.now() - startMs) produces fractional ms.
+    // Use small epoch + fraction to avoid IEEE 754 noise in the fractional part.
+    const startMs = 1000.322;
+    const endMs = 1045.789;
+
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: startMs,
+      endTimeMs: endMs,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.startTimeUnixNano).toBe("1000322000");
+    expect(span.endTimeUnixNano).toBe("1045789000");
+  });
+
+  it("sub-ms precision affects ordering for real epoch values", () => {
+    // Two spans within the same millisecond should have different nanosecond timestamps
+    const spanA = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "a",
+      startTimeMs: 1711929600000.3,
+      endTimeMs: 1711929600001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const spanB = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "b",
+      startTimeMs: 1711929600000.7,
+      endTimeMs: 1711929600001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const startA = BigInt(spanA.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.startTimeUnixNano);
+    const startB = BigInt(spanB.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.startTimeUnixNano);
+    // A should sort before B (both in the same ms but different sub-ms positions)
+    expect(startA).toBeLessThan(startB);
+  });
+
+  it("omits parentSpanId when not provided", () => {
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.parentSpanId).toBeUndefined();
+  });
+
+  it("handles double values for non-integer numbers", () => {
+    const payload = buildPayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: { "compute.cpu": 0.25 },
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    const cpu = span.attributes.find((a) => a.key === "compute.cpu");
+    expect(cpu).toEqual({ key: "compute.cpu", value: { doubleValue: 0.25 } });
+  });
+});

apps/supervisor/src/services/otlpTraceService.ts

@@ -0,0 +1,104 @@
+import { randomBytes } from "crypto";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+export type OtlpTraceServiceOptions = {
+  endpointUrl: string;
+  timeoutMs?: number;
+};
+
+export type OtlpTraceSpan = {
+  traceId: string;
+  parentSpanId?: string;
+  spanName: string;
+  startTimeMs: number;
+  endTimeMs: number;
+  resourceAttributes: Record<string, string | number | boolean>;
+  spanAttributes: Record<string, string | number | boolean>;
+};
+
+export class OtlpTraceService {
+  private readonly logger = new SimpleStructuredLogger("otlp-trace");
+
+  constructor(private opts: OtlpTraceServiceOptions) {}
+
+  /** Fire-and-forget: build payload and send to the configured OTLP endpoint */
+  emit(span: OtlpTraceSpan): void {
+    const payload = buildPayload(span);
+
+    fetch(`${this.opts.endpointUrl}/v1/traces`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(this.opts.timeoutMs ?? 5_000),
+    }).catch((err) => {
+      this.logger.warn("failed to send compute trace span", {
+        error: err instanceof Error ? err.message : String(err),
+      });
+    });
+  }
+}
+
+// ── Payload builder (internal) ───────────────────────────────────────────────
+
+/** @internal Exported for tests only */
+export function buildPayload(span: OtlpTraceSpan) {
+  const spanId = randomBytes(8).toString("hex");
+
+  return {
+    resourceSpans: [
+      {
+        resource: {
+          attributes: [
+            { key: "$trigger", value: { boolValue: true } },
+            ...toOtlpAttributes(span.resourceAttributes),
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: { name: "supervisor.compute" },
+            spans: [
+              {
+                traceId: span.traceId,
+                spanId,
+                parentSpanId: span.parentSpanId,
+                name: span.spanName,
+                kind: 3, // SPAN_KIND_CLIENT
+                startTimeUnixNano: msToNano(span.startTimeMs),
+                endTimeUnixNano: msToNano(span.endTimeMs),
+                attributes: toOtlpAttributes(span.spanAttributes),
+                status: { code: 1 }, // STATUS_CODE_OK
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  };
+}
+
+function toOtlpAttributes(
+  attrs: Record<string, string | number | boolean>
+): Array<{ key: string; value: Record<string, unknown> }> {
+  return Object.entries(attrs).map(([key, value]) => ({
+    key,
+    value: toOtlpValue(value),
+  }));
+}
+
+function toOtlpValue(value: string | number | boolean): Record<string, unknown> {
+  if (typeof value === "string") return { stringValue: value };
+  if (typeof value === "boolean") return { boolValue: value };
+  if (Number.isInteger(value)) return { intValue: value };
+  return { doubleValue: value };
+}
+
+/**
+ * Convert epoch milliseconds to nanosecond string, preserving sub-ms precision.
+ * Fractional ms from performance.now() arithmetic carry meaningful microsecond
+ * data that affects span sort ordering when events happen within the same ms.
+ */
+function msToNano(ms: number): string {
+  const wholeMs = Math.trunc(ms);
+  const fracNs = Math.round((ms - wholeMs) * 1_000_000);
+  return String(BigInt(wholeMs) * 1_000_000n + BigInt(fracNs));
+}

apps/supervisor/src/services/podCleaner.test.ts

@@ -0,0 +1,473 @@
+import { PodCleaner } from "./podCleaner.js";
+import { type K8sApi, createK8sApi } from "../clients/kubernetes.js";
+import { setTimeout } from "timers/promises";
+import { describe, it, expect, beforeAll, afterEach } from "vitest";
+import { Registry } from "prom-client";
+
+// These tests require live K8s cluster credentials - skip by default
+describe.skipIf(!process.env.K8S_INTEGRATION_TESTS)("PodCleaner Integration Tests", () => {
+  const k8s = createK8sApi();
+  const namespace = "integration-test";
+  const register = new Registry();
+
+  beforeAll(async () => {
+    // Create the test namespace, only if it doesn't exist
+    try {
+      await k8s.core.readNamespace({ name: namespace });
+    } catch (_error) {
+      await k8s.core.createNamespace({
+        body: {
+          metadata: {
+            name: namespace,
+          },
+        },
+      });
+    }
+  });
+
+  afterEach(async () => {
+    // Clear metrics to avoid conflicts
+    register.clear();
+
+    // Delete all pods in the namespace
+    await k8s.core.deleteCollectionNamespacedPod({ namespace, gracePeriodSeconds: 0 });
+  });
+
+  it("should clean up succeeded pods", async () => {
+    const podCleaner = new PodCleaner({ namespace, k8s, register });
+
+    try {
+      // Create a test pod that's in succeeded state
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        namePrefix: "test-succeeded-pod",
+      });
+
+      if (!podNames[0]) {
+        throw new Error("Failed to create test pod");
+      }
+      const podName = podNames[0];
+
+      // Wait for pod to complete
+      await waitForPodPhase({
+        k8sApi: k8s,
+        namespace,
+        podName,
+        phase: "Succeeded",
+      });
+
+      // Start the pod cleaner
+      await podCleaner.start();
+
+      // Wait for pod to be deleted
+      await waitForPodDeletion({
+        k8sApi: k8s,
+        namespace,
+        podName,
+      });
+
+      // Verify pod was deleted
+      expect(await podExists({ k8sApi: k8s, namespace, podName })).toBe(false);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+
+  it("should accurately track deletion metrics", async () => {
+    const podCleaner = new PodCleaner({ namespace, k8s, register });
+    try {
+      // Create a test pod that's in succeeded state
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        namePrefix: "test-succeeded-pod",
+      });
+
+      // Wait for pod to be in succeeded state
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Succeeded",
+      });
+
+      await podCleaner.start();
+
+      // Wait for pod to be deleted
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      const metrics = podCleaner.getMetrics();
+      const deletionCycles = await metrics.deletionCyclesTotal.get();
+      const deletionTimestamp = await metrics.lastDeletionTimestamp.get();
+
+      expect(deletionCycles?.values[0]?.value).toBeGreaterThan(0);
+      expect(deletionTimestamp?.values[0]?.value).toBeGreaterThan(0);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+
+  it("should handle different batch sizes - small", async () => {
+    const podCleaner = new PodCleaner({
+      namespace,
+      k8s,
+      register,
+      batchSize: 1,
+    });
+
+    try {
+      // Create some pods that will succeed
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 2,
+      });
+
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Succeeded",
+      });
+
+      await podCleaner.start();
+
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      const metrics = podCleaner.getMetrics();
+      const cycles = await metrics.deletionCyclesTotal.get();
+
+      expect(cycles?.values[0]?.value).toBe(2);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+
+  it("should handle different batch sizes - large", async () => {
+    const podCleaner = new PodCleaner({
+      namespace,
+      k8s,
+      register,
+      batchSize: 5000,
+    });
+
+    try {
+      // Create some pods that will succeed
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 10,
+      });
+
+      await waitForPodsPhase({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+        phase: "Succeeded",
+      });
+
+      await podCleaner.start();
+
+      await waitForPodsDeletion({
+        k8sApi: k8s,
+        namespace,
+        podNames,
+      });
+
+      const metrics = podCleaner.getMetrics();
+      const cycles = await metrics.deletionCyclesTotal.get();
+
+      expect(cycles?.values[0]?.value).toBe(1);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+
+  it("should not delete pods without app=task-run label", async () => {
+    const podCleaner = new PodCleaner({ namespace, k8s, register });
+
+    try {
+      // Create a test pod without the task-run label
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        labels: { app: "different-label" },
+        namePrefix: "non-task-run-pod",
+      });
+
+      if (!podNames[0]) {
+        throw new Error("Failed to create test pod");
+      }
+      const podName = podNames[0];
+
+      // Wait for pod to complete
+      await waitForPodPhase({
+        k8sApi: k8s,
+        namespace,
+        podName,
+        phase: "Succeeded",
+      });
+
+      await podCleaner.start();
+
+      // Wait a reasonable time to ensure pod isn't deleted
+      await setTimeout(5000);
+
+      // Verify pod still exists
+      expect(await podExists({ k8sApi: k8s, namespace, podName })).toBe(true);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+
+  it("should not delete pods that are still running", async () => {
+    const podCleaner = new PodCleaner({ namespace, k8s, register });
+
+    try {
+      // Create a test pod with a long-running command
+      const podNames = await createTestPods({
+        k8sApi: k8s,
+        namespace,
+        count: 1,
+        namePrefix: "running-pod",
+        command: ["sleep", "30"], // Will keep pod running
+      });
+
+      if (!podNames[0]) {
+        throw new Error("Failed to create test pod");
+      }
+      const podName = podNames[0];
+
+      // Wait for pod to be running
+      await waitForPodPhase({
+        k8sApi: k8s,
+        namespace,
+        podName,
+        phase: "Running",
+      });
+
+      await podCleaner.start();
+
+      // Wait a reasonable time to ensure pod isn't deleted
+      await setTimeout(5000);
+
+      // Verify pod still exists
+      expect(await podExists({ k8sApi: k8s, namespace, podName })).toBe(true);
+    } finally {
+      await podCleaner.stop();
+    }
+  }, 30000);
+});
+
+// Helper functions
+async function waitForPodPhase({
+  k8sApi,
+  namespace,
+  podName,
+  phase,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podName: string;
+  phase: string;
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+
+  while (Date.now() - startTime < timeoutMs) {
+    const pod = await k8sApi.core.readNamespacedPod({
+      namespace,
+      name: podName,
+    });
+    if (pod.status?.phase === phase) {
+      return;
+    }
+    await setTimeout(waitMs);
+  }
+
+  throw new Error(`Pod ${podName} did not reach phase ${phase} within ${timeoutMs}ms`);
+}
+
+async function waitForPodDeletion({
+  k8sApi,
+  namespace,
+  podName,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podName: string;
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+
+  while (Date.now() - startTime < timeoutMs) {
+    try {
+      await k8sApi.core.readNamespacedPod({
+        namespace,
+        name: podName,
+      });
+      await setTimeout(waitMs);
+    } catch (_error) {
+      // Pod was deleted
+      return;
+    }
+  }
+
+  throw new Error(`Pod ${podName} was not deleted within ${timeoutMs}ms`);
+}
+
+async function createTestPods({
+  k8sApi,
+  namespace,
+  count,
+  labels = { app: "task-run" },
+  shouldFail = false,
+  namePrefix = "test-pod",
+  command = ["/bin/sh", "-c", shouldFail ? "exit 1" : "exit 0"],
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  count: number;
+  labels?: Record<string, string>;
+  shouldFail?: boolean;
+  namePrefix?: string;
+  command?: string[];
+}) {
+  const createdPods: string[] = [];
+
+  for (let i = 0; i < count; i++) {
+    const podName = `${namePrefix}-${i}`;
+    await k8sApi.core.createNamespacedPod({
+      namespace,
+      body: {
+        metadata: {
+          name: podName,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          containers: [
+            {
+              name: "test",
+              image: "busybox:1.37.0",
+              command,
+            },
+          ],
+        },
+      },
+    });
+    createdPods.push(podName);
+  }
+
+  return createdPods;
+}
+
+async function waitForPodsPhase({
+  k8sApi,
+  namespace,
+  podNames,
+  phase,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podNames: string[];
+  phase: "Pending" | "Running" | "Succeeded" | "Failed" | "Unknown";
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+  const pendingPods = new Set(podNames);
+
+  while (pendingPods.size > 0 && Date.now() - startTime < timeoutMs) {
+    const pods = await k8sApi.core.listNamespacedPod({ namespace });
+
+    for (const pod of pods.items) {
+      if (pendingPods.has(pod.metadata?.name ?? "") && pod.status?.phase === phase) {
+        pendingPods.delete(pod.metadata?.name ?? "");
+      }
+    }
+
+    if (pendingPods.size > 0) {
+      await setTimeout(waitMs);
+    }
+  }
+
+  if (pendingPods.size > 0) {
+    throw new Error(
+      `Pods [${Array.from(pendingPods).join(
+        ", "
+      )}] did not reach phase ${phase} within ${timeoutMs}ms`
+    );
+  }
+}
+
+async function waitForPodsDeletion({
+  k8sApi,
+  namespace,
+  podNames,
+  timeoutMs = 10000,
+  waitMs = 1000,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podNames: string[];
+  timeoutMs?: number;
+  waitMs?: number;
+}) {
+  const startTime = Date.now();
+  const pendingPods = new Set(podNames);
+
+  while (pendingPods.size > 0 && Date.now() - startTime < timeoutMs) {
+    const pods = await k8sApi.core.listNamespacedPod({ namespace });
+    const existingPods = new Set(pods.items.map((pod) => pod.metadata?.name ?? ""));
+
+    for (const podName of pendingPods) {
+      if (!existingPods.has(podName)) {
+        pendingPods.delete(podName);
+      }
+    }
+
+    if (pendingPods.size > 0) {
+      await setTimeout(waitMs);
+    }
+  }
+
+  if (pendingPods.size > 0) {
+    throw new Error(
+      `Pods [${Array.from(pendingPods).join(", ")}] were not deleted within ${timeoutMs}ms`
+    );
+  }
+}
+
+async function podExists({
+  k8sApi,
+  namespace,
+  podName,
+}: {
+  k8sApi: K8sApi;
+  namespace: string;
+  podName: string;
+}) {
+  const pods = await k8sApi.core.listNamespacedPod({ namespace });
+  return pods.items.some((p) => p.metadata?.name === podName);
+}

apps/supervisor/src/services/podCleaner.ts

@@ -0,0 +1,119 @@
+import { IntervalService } from "@trigger.dev/core/v3";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import type { Registry } from "prom-client";
+import { Counter, Gauge } from "prom-client";
+import type { K8sApi } from "../clients/kubernetes.js";
+import { createK8sApi } from "../clients/kubernetes.js";
+import { register } from "../metrics.js";
+
+export type PodCleanerOptions = {
+  namespace: string;
+  k8s?: K8sApi;
+  register?: Registry;
+  batchSize?: number;
+  intervalMs?: number;
+};
+
+export class PodCleaner {
+  private readonly logger = new SimpleStructuredLogger("pod-cleaner");
+  private readonly k8s: K8sApi;
+  private readonly namespace: string;
+
+  private readonly batchSize: number;
+  private readonly deletionInterval: IntervalService;
+
+  // Metrics
+  private readonly register: Registry;
+  private readonly deletionCyclesTotal: Counter;
+  private readonly lastDeletionTimestamp: Gauge;
+
+  constructor(opts: PodCleanerOptions) {
+    this.k8s = opts.k8s ?? createK8sApi();
+
+    this.namespace = opts.namespace;
+    this.batchSize = opts.batchSize ?? 500;
+
+    this.deletionInterval = new IntervalService({
+      intervalMs: opts.intervalMs ?? 10000,
+      leadingEdge: true,
+      onInterval: this.deleteCompletedPods.bind(this),
+    });
+
+    // Initialize metrics
+    this.register = opts.register ?? register;
+
+    this.deletionCyclesTotal = new Counter({
+      name: "pod_cleaner_deletion_cycles_total",
+      help: "Total number of pod deletion cycles run",
+      labelNames: ["namespace", "status", "batch_size"],
+      registers: [this.register],
+    });
+
+    this.lastDeletionTimestamp = new Gauge({
+      name: "pod_cleaner_last_deletion_timestamp",
+      help: "Timestamp of the last deletion cycle",
+      labelNames: ["namespace"],
+      registers: [this.register],
+    });
+  }
+
+  async start() {
+    this.deletionInterval.start();
+  }
+
+  async stop() {
+    this.deletionInterval.stop();
+  }
+
+  private async deleteCompletedPods() {
+    let continuationToken: string | undefined;
+
+    do {
+      try {
+        const result = await this.k8s.core.deleteCollectionNamespacedPod({
+          namespace: this.namespace,
+          labelSelector: "app=task-run",
+          fieldSelector: "status.phase=Succeeded",
+          limit: this.batchSize,
+          _continue: continuationToken,
+          gracePeriodSeconds: 0,
+          propagationPolicy: "Background",
+          timeoutSeconds: 30,
+        });
+
+        // Update continuation token for next batch
+        continuationToken = result.metadata?._continue;
+
+        // Increment the deletion cycles counter
+        this.deletionCyclesTotal.inc({
+          namespace: this.namespace,
+          batch_size: this.batchSize,
+          status: "succeeded",
+        });
+
+        this.logger.debug("Deleted batch of pods", { continuationToken });
+      } catch (err) {
+        this.logger.error("Failed to delete batch of pods", {
+          err: err instanceof Error ? err.message : String(err),
+        });
+
+        this.deletionCyclesTotal.inc({
+          namespace: this.namespace,
+          batch_size: this.batchSize,
+          status: "failed",
+        });
+        break;
+      }
+    } while (continuationToken);
+
+    this.lastDeletionTimestamp.set({ namespace: this.namespace }, Date.now());
+  }
+
+  // Method to expose metrics for testing
+  public getMetrics() {
+    return {
+      deletionCyclesTotal: this.deletionCyclesTotal,
+      lastDeletionTimestamp: this.lastDeletionTimestamp,
+    };
+  }
+}

apps/supervisor/src/services/timerWheel.test.ts

@@ -0,0 +1,271 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { TimerWheel } from "./timerWheel.js";
+
+describe("TimerWheel", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("dispatches item after delay", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "snapshot-data");
+
+    // Not yet
+    vi.advanceTimersByTime(2900);
+    expect(dispatched).toEqual([]);
+
+    // After delay
+    vi.advanceTimersByTime(200);
+    expect(dispatched).toEqual(["run-1"]);
+
+    wheel.stop();
+  });
+
+  it("cancels item before it fires", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    vi.advanceTimersByTime(1000);
+    expect(wheel.cancel("run-1")).toBe(true);
+
+    vi.advanceTimersByTime(5000);
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("peek returns the pending item without removing it", () => {
+    const wheel = new TimerWheel<string>({ delayMs: 3000, onExpire: () => {} });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    expect(wheel.peek("run-1")).toEqual({ key: "run-1", data: "data" });
+    expect(wheel.size).toBe(1);
+    expect(wheel.peek("run-2")).toBeUndefined();
+
+    // Dispatched items are no longer peekable
+    vi.advanceTimersByTime(3100);
+    expect(wheel.peek("run-1")).toBeUndefined();
+
+    wheel.stop();
+  });
+
+  it("cancel returns false for unknown key", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+    expect(wheel.cancel("nonexistent")).toBe(false);
+  });
+
+  it("deduplicates: resubmitting same key replaces the entry", () => {
+    const dispatched: { key: string; data: string }[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push({ key: item.key, data: item.data }),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "old-data");
+
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-1", "new-data");
+
+    // Original would have fired at t=3000, but was replaced
+    // New one fires at t=1000+3000=4000
+    vi.advanceTimersByTime(2100);
+    expect(dispatched).toEqual([]);
+
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual([{ key: "run-1", data: "new-data" }]);
+
+    wheel.stop();
+  });
+
+  it("handles many concurrent items", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    for (let i = 0; i < 1000; i++) {
+      wheel.submit(`run-${i}`, `data-${i}`);
+    }
+    expect(wheel.size).toBe(1000);
+
+    vi.advanceTimersByTime(3100);
+    expect(dispatched.length).toBe(1000);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("handles items submitted at different times", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    wheel.submit("run-1", "data");
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-2", "data");
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-3", "data");
+
+    // t=2000: nothing yet
+    expect(dispatched).toEqual([]);
+
+    // t=3100: run-1 fires
+    vi.advanceTimersByTime(1100);
+    expect(dispatched).toEqual(["run-1"]);
+
+    // t=4100: run-2 fires
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual(["run-1", "run-2"]);
+
+    // t=5100: run-3 fires
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual(["run-1", "run-2", "run-3"]);
+
+    wheel.stop();
+  });
+
+  it("setDelay changes delay for new items only", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    wheel.submit("run-1", "data"); // 3s delay
+
+    vi.advanceTimersByTime(500);
+    wheel.setDelay(1000);
+    wheel.submit("run-2", "data"); // 1s delay
+
+    // t=1500: run-2 should have fired (submitted at t=500 with 1s delay)
+    vi.advanceTimersByTime(1100);
+    expect(dispatched).toEqual(["run-2"]);
+
+    // t=3100: run-1 fires at its original 3s delay
+    vi.advanceTimersByTime(1500);
+    expect(dispatched).toEqual(["run-2", "run-1"]);
+
+    wheel.stop();
+  });
+
+  it("stop returns unprocessed items", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data-1");
+    wheel.submit("run-2", "data-2");
+    wheel.submit("run-3", "data-3");
+
+    const remaining = wheel.stop();
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+    expect(remaining.length).toBe(3);
+    expect(remaining.map((r) => r.key).sort()).toEqual(["run-1", "run-2", "run-3"]);
+    expect(remaining.find((r) => r.key === "run-1")?.data).toBe("data-1");
+  });
+
+  it("after stop, new submissions are silently dropped", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.stop();
+
+    wheel.submit("run-late", "data");
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+  });
+
+  it("tracks size correctly through submit/cancel/dispatch", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+
+    wheel.start();
+
+    wheel.submit("a", "data");
+    wheel.submit("b", "data");
+    expect(wheel.size).toBe(2);
+
+    wheel.cancel("a");
+    expect(wheel.size).toBe(1);
+
+    vi.advanceTimersByTime(3100);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("clamps delay to valid range", () => {
+    const dispatched: string[] = [];
+
+    // Very small delay (should be at least 1 tick = 100ms)
+    const wheel = new TimerWheel<string>({
+      delayMs: 0,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    vi.advanceTimersByTime(200);
+    expect(dispatched).toEqual(["run-1"]);
+
+    wheel.stop();
+  });
+
+  it("multiple cancel calls are safe", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    expect(wheel.cancel("run-1")).toBe(true);
+    expect(wheel.cancel("run-1")).toBe(false);
+
+    wheel.stop();
+  });
+});

apps/supervisor/src/services/timerWheel.ts

@@ -0,0 +1,166 @@
+/**
+ * TimerWheel implements a hashed timer wheel for efficiently managing large numbers
+ * of delayed operations with O(1) submit, cancel, and per-item dispatch.
+ *
+ * Used by the supervisor to delay snapshot requests so that short-lived waitpoints
+ * (e.g. triggerAndWait that resolves in <3s) skip the snapshot entirely.
+ *
+ * The wheel is a ring buffer of slots. A single setInterval advances a cursor.
+ * When the cursor reaches a slot, all items in that slot are dispatched.
+ *
+ * Fixed capacity: 600 slots at 100ms tick = 60s max delay.
+ */
+
+const TICK_MS = 100;
+const NUM_SLOTS = 600; // 60s max delay at 100ms tick
+
+export type TimerWheelItem<T> = {
+  key: string;
+  data: T;
+};
+
+export type TimerWheelOptions<T> = {
+  /** Called when an item's delay expires. */
+  onExpire: (item: TimerWheelItem<T>) => void;
+  /** Delay in milliseconds before items fire. Clamped to [100, 60000]. */
+  delayMs: number;
+};
+
+type Entry<T> = {
+  key: string;
+  data: T;
+  slotIndex: number;
+};
+
+export class TimerWheel<T> {
+  private slots: Set<string>[];
+  private entries: Map<string, Entry<T>>;
+  private cursor: number;
+  private intervalId: ReturnType<typeof setInterval> | null;
+  private onExpire: (item: TimerWheelItem<T>) => void;
+  private delaySlots: number;
+
+  constructor(opts: TimerWheelOptions<T>) {
+    this.slots = Array.from({ length: NUM_SLOTS }, () => new Set());
+    this.entries = new Map();
+    this.cursor = 0;
+    this.intervalId = null;
+    this.onExpire = opts.onExpire;
+    this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.ceil(opts.delayMs / TICK_MS)));
+  }
+
+  /** Start the timer wheel. Must be called before submitting items. */
+  start(): void {
+    if (this.intervalId) return;
+    this.intervalId = setInterval(() => this.tick(), TICK_MS);
+    // Don't hold the process open just for the timer wheel
+    if (this.intervalId && typeof this.intervalId === "object" && "unref" in this.intervalId) {
+      this.intervalId.unref();
+    }
+  }
+
+  /**
+   * Stop the timer wheel and return all unprocessed items.
+   * The wheel keeps running normally during graceful shutdown - call stop()
+   * only when you're ready to tear down. Caller decides what to do with leftovers.
+   */
+  stop(): TimerWheelItem<T>[] {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+
+    const remaining: TimerWheelItem<T>[] = [];
+    for (const [key, entry] of this.entries) {
+      remaining.push({ key, data: entry.data });
+    }
+
+    for (const slot of this.slots) {
+      slot.clear();
+    }
+    this.entries.clear();
+
+    return remaining;
+  }
+
+  /**
+   * Update the delay for future submissions. Already-queued items keep their original timing.
+   * Clamped to [TICK_MS, 60000ms].
+   */
+  setDelay(delayMs: number): void {
+    this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.ceil(delayMs / TICK_MS)));
+  }
+
+  /**
+   * Submit an item to be dispatched after the configured delay.
+   * If an item with the same key already exists, it is replaced (dedup).
+   * No-op if the wheel is stopped.
+   */
+  submit(key: string, data: T): void {
+    if (!this.intervalId) return;
+
+    // Dedup: remove existing entry for this key
+    this.cancel(key);
+
+    const slotIndex = (this.cursor + this.delaySlots) % NUM_SLOTS;
+    const entry: Entry<T> = { key, data, slotIndex };
+
+    this.entries.set(key, entry);
+    this.slot(slotIndex).add(key);
+  }
+
+  /**
+   * Cancel a pending item. Returns true if the item was found and removed.
+   */
+  cancel(key: string): boolean {
+    const entry = this.entries.get(key);
+    if (!entry) return false;
+
+    this.slot(entry.slotIndex).delete(key);
+    this.entries.delete(key);
+    return true;
+  }
+
+  /** Look up a pending item without removing it. */
+  peek(key: string): TimerWheelItem<T> | undefined {
+    const entry = this.entries.get(key);
+    return entry ? { key, data: entry.data } : undefined;
+  }
+
+  /** Number of pending items in the wheel. */
+  get size(): number {
+    return this.entries.size;
+  }
+
+  /** Whether the wheel is running. */
+  get running(): boolean {
+    return this.intervalId !== null;
+  }
+
+  /** Get a slot by index. The array is fully initialized so this always returns a Set. */
+  private slot(index: number): Set<string> {
+    const s = this.slots[index];
+    if (!s) throw new Error(`TimerWheel: invalid slot index ${index}`);
+    return s;
+  }
+
+  /** Advance the cursor and dispatch all items in the current slot. */
+  private tick(): void {
+    this.cursor = (this.cursor + 1) % NUM_SLOTS;
+    const slot = this.slot(this.cursor);
+
+    if (slot.size === 0) return;
+
+    // Collect items to dispatch (copy keys since we mutate during iteration)
+    const keys = [...slot];
+    slot.clear();
+
+    for (const key of keys) {
+      const entry = this.entries.get(key);
+      if (!entry) continue;
+
+      this.entries.delete(key);
+      this.onExpire({ key, data: entry.data });
+    }
+  }
+}

apps/supervisor/src/services/warmStartVerificationService.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it, vi } from "vitest";
+import { setTimeout as sleep } from "node:timers/promises";
+import { WarmStartVerificationService } from "./warmStartVerificationService.js";
+import type { DequeuedMessage } from "@trigger.dev/core/v3";
+import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers";
+
+// The TimerWheel ticks every 100ms, so a 1000ms delay (the env minimum)
+// fires within ~1.1s.
+const DELAY_MS = 1_000;
+// Long enough that a pending verification would certainly have fired.
+const SETTLE_MS = 1_600;
+
+const DEQUEUED_SNAPSHOT_ID = "snapshot_dequeued";
+
+function makeMessage(runFriendlyId = "run_1"): DequeuedMessage {
+  return {
+    run: { friendlyId: runFriendlyId },
+    snapshot: { friendlyId: DEQUEUED_SNAPSHOT_ID },
+  } as unknown as DequeuedMessage;
+}
+
+function createService(opts: { latestSnapshotId?: string; probeError?: boolean }) {
+  const getLatestSnapshot = vi.fn(async (_runId: string) =>
+    opts.probeError
+      ? { success: false as const, error: "connection refused" }
+      : {
+          success: true as const,
+          data: { execution: { snapshot: { friendlyId: opts.latestSnapshotId } } },
+        }
+  );
+
+  const createWorkload = vi.fn(async () => {});
+
+  const service = new WarmStartVerificationService({
+    workerClient: { getLatestSnapshot } as unknown as SupervisorHttpClient,
+    delayMs: DELAY_MS,
+    createWorkload,
+    wideEventOpts: { service: "supervisor-test", env: {}, enabled: false },
+  });
+
+  return { service, getLatestSnapshot, createWorkload };
+}
+
+describe("WarmStartVerificationService", () => {
+  it("falls back to a cold create when the snapshot is unchanged", async () => {
+    const { service, createWorkload } = createService({
+      latestSnapshotId: DEQUEUED_SNAPSHOT_ID,
+    });
+    try {
+      const message = makeMessage();
+      const timings = { warmStartCheckMs: 12 };
+      service.schedule(message, timings);
+
+      await vi.waitFor(() => expect(createWorkload).toHaveBeenCalledTimes(1), {
+        timeout: 3_000,
+      });
+      expect(createWorkload).toHaveBeenCalledWith(message, timings);
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("does nothing when the snapshot has moved on (delivered)", async () => {
+    const { service, getLatestSnapshot, createWorkload } = createService({
+      latestSnapshotId: "snapshot_executing",
+    });
+    try {
+      service.schedule(makeMessage(), { warmStartCheckMs: 12 });
+
+      await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), {
+        timeout: 3_000,
+      });
+      await sleep(100);
+      expect(createWorkload).not.toHaveBeenCalled();
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("never falls back when the probe errors", async () => {
+    const { service, getLatestSnapshot, createWorkload } = createService({ probeError: true });
+    try {
+      service.schedule(makeMessage(), { warmStartCheckMs: 12 });
+
+      await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), {
+        timeout: 3_000,
+      });
+      await sleep(100);
+      expect(createWorkload).not.toHaveBeenCalled();
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("cancel before the delay prevents the probe entirely", async () => {
+    const { service, getLatestSnapshot, createWorkload } = createService({
+      latestSnapshotId: DEQUEUED_SNAPSHOT_ID,
+    });
+    try {
+      service.schedule(makeMessage(), { warmStartCheckMs: 12 });
+
+      expect(service.cancel("run_1")).toBe(true);
+
+      await sleep(SETTLE_MS);
+      expect(getLatestSnapshot).not.toHaveBeenCalled();
+      expect(createWorkload).not.toHaveBeenCalled();
+    } finally {
+      service.stop();
+    }
+  });
+
+  it("re-scheduling the same run replaces the pending verification", async () => {
+    const { service, getLatestSnapshot } = createService({
+      latestSnapshotId: "snapshot_executing",
+    });
+    try {
+      service.schedule(makeMessage(), { warmStartCheckMs: 1 });
+      service.schedule(makeMessage(), { warmStartCheckMs: 2 });
+
+      await vi.waitFor(() => expect(getLatestSnapshot).toHaveBeenCalledTimes(1), {
+        timeout: 3_000,
+      });
+      await sleep(SETTLE_MS);
+      expect(getLatestSnapshot).toHaveBeenCalledTimes(1);
+    } finally {
+      service.stop();
+    }
+  });
+});

apps/supervisor/src/services/warmStartVerificationService.ts

@@ -0,0 +1,173 @@
+import pLimit from "p-limit";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import type { DequeuedMessage } from "@trigger.dev/core/v3";
+import type { SupervisorHttpClient } from "@trigger.dev/core/v3/workers";
+import { tryCatch } from "@trigger.dev/core";
+import { TimerWheel } from "./timerWheel.js";
+import { emitOneShot, type WideEventOptions } from "../wideEvents/index.js";
+
+const PROBE_CONCURRENCY_LIMIT = 10;
+
+export type WarmStartTimings = {
+  dequeueResponseMs?: number;
+  pollingIntervalMs?: number;
+  warmStartCheckMs: number;
+};
+
+type PendingVerification = {
+  message: DequeuedMessage;
+  timings: WarmStartTimings;
+};
+
+export type WarmStartVerificationServiceOptions = {
+  workerClient: SupervisorHttpClient;
+  /** How long after a warm-start hit to verify the runner acted on it. */
+  delayMs: number;
+  /** Cold-creates the workload for a dispatched-but-lost run. */
+  createWorkload: (message: DequeuedMessage, timings: WarmStartTimings) => Promise<void>;
+  wideEventOpts: WideEventOptions;
+};
+
+/**
+ * Verifies that warm-start dispatches were actually acted on.
+ *
+ * Firestarter's `didWarmStart: true` means "response written to a socket",
+ * not "runner received it". A silently dead poller (no FIN - e.g. a VM torn
+ * down mid-poll) leaves the dispatched run stuck in PENDING_EXECUTING until
+ * the run engine's heartbeat redrive minutes later, burning a queue
+ * redelivery each time (TRI-10659).
+ *
+ * After a hit, the dequeued message is retained for `delayMs`, then the
+ * platform is asked for the run's latest snapshot. If it is still the exact
+ * snapshot we dequeued, no runner ever started the attempt - fall through to
+ * the regular cold-create path with the original message. Double-starts are
+ * impossible: `startRunAttempt` runs under a per-run lock and rejects stale
+ * snapshot ids, so if the original runner revives and races the fallback
+ * workload, exactly one wins and the loser exits before executing anything.
+ *
+ * On a probe ERROR we deliberately do nothing: the runner's attempt-start
+ * goes through nested retries, so during platform brownouts a healthy runner
+ * can legitimately act late - falling back on uncertainty would stampede
+ * duplicate workloads exactly when the platform is degraded. The heartbeat
+ * redrive remains the backstop for that case (and for supervisor restarts,
+ * which drop the in-memory timers).
+ */
+export class WarmStartVerificationService {
+  private readonly logger = new SimpleStructuredLogger("warm-start-verification");
+
+  private readonly timerWheel: TimerWheel<PendingVerification>;
+  private readonly probeLimit: ReturnType<typeof pLimit>;
+
+  private readonly workerClient: SupervisorHttpClient;
+  private readonly delayMs: number;
+  private readonly createWorkload: WarmStartVerificationServiceOptions["createWorkload"];
+  private readonly wideEventOpts: WideEventOptions;
+
+  constructor(opts: WarmStartVerificationServiceOptions) {
+    this.workerClient = opts.workerClient;
+    this.delayMs = opts.delayMs;
+    this.createWorkload = opts.createWorkload;
+    this.wideEventOpts = opts.wideEventOpts;
+
+    this.probeLimit = pLimit(PROBE_CONCURRENCY_LIMIT);
+    this.timerWheel = new TimerWheel<PendingVerification>({
+      delayMs: opts.delayMs,
+      onExpire: (item) => {
+        this.probeLimit(() => this.verify(item.data)).catch((error) => {
+          this.logger.error("Verification failed", {
+            runId: item.data.message.run.friendlyId,
+            error,
+          });
+        });
+      },
+    });
+    this.timerWheel.start();
+  }
+
+  /** Schedule delivery verification for a warm-start hit. */
+  schedule(message: DequeuedMessage, timings: WarmStartTimings) {
+    this.timerWheel.submit(message.run.friendlyId, { message, timings });
+    this.logger.debug("Verification scheduled", {
+      runId: message.run.friendlyId,
+      snapshotId: message.snapshot.friendlyId,
+      delayMs: this.delayMs,
+    });
+  }
+
+  /**
+   * Cancel a pending verification, e.g. when the runner connects to this
+   * supervisor. Purely an optimization: the matched runner often lives on a
+   * different node and connects to that node's supervisor, so most healthy
+   * deliveries are confirmed by the probe, not by this.
+   */
+  cancel(runFriendlyId: string): boolean {
+    return this.timerWheel.cancel(runFriendlyId);
+  }
+
+  /** Stop the timer wheel, dropping pending verifications. The run engine's
+   * heartbeat redrive covers anything dropped here. */
+  stop() {
+    const remaining = this.timerWheel.stop();
+    if (remaining.length > 0) {
+      this.logger.info("Stopped, dropped pending verifications", { count: remaining.length });
+    }
+  }
+
+  private async verify({ message, timings }: PendingVerification) {
+    const runFriendlyId = message.run.friendlyId;
+
+    const result = await this.workerClient.getLatestSnapshot(runFriendlyId);
+
+    if (!result.success) {
+      // Never fall back on uncertainty - see class docs.
+      this.emitOutcome(message, "probe_error", String(result.error));
+      this.logger.warn("Verification probe failed, skipping", {
+        runId: runFriendlyId,
+        error: result.error,
+      });
+      return;
+    }
+
+    const latestSnapshotId = result.data.execution.snapshot.friendlyId;
+
+    if (latestSnapshotId !== message.snapshot.friendlyId) {
+      // Something acted on the run (attempt started, or it was cancelled or
+      // requeued) - the dispatch is no longer ours to worry about.
+      this.emitOutcome(message, "delivered");
+      return;
+    }
+
+    this.emitOutcome(message, "fallback");
+    this.logger.warn("Warm start dispatch was never acted on, cold starting", {
+      runId: runFriendlyId,
+      snapshotId: message.snapshot.friendlyId,
+    });
+
+    const [error] = await tryCatch(this.createWorkload(message, timings));
+    if (error) {
+      this.logger.error("Fallback workload create failed", {
+        runId: runFriendlyId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  private emitOutcome(
+    message: DequeuedMessage,
+    outcome: "delivered" | "fallback" | "probe_error",
+    error?: string
+  ) {
+    emitOneShot({
+      ...this.wideEventOpts,
+      op: "warmstart.verify",
+      kind: "event",
+      populate: (state) => {
+        state.meta.run_id = message.run.friendlyId;
+        state.meta.snapshot_id = message.snapshot.friendlyId;
+        state.extras.outcome = outcome;
+        state.extras.delay_ms = this.delayMs;
+        if (error) state.extras.error = error;
+      },
+    });
+  }
+}

apps/supervisor/src/util.ts

@@ -0,0 +1,44 @@
+import { isMacOS, isWindows } from "std-env";
+
+export function normalizeDockerHostUrl(url: string) {
+  const $url = new URL(url);
+
+  if ($url.hostname === "localhost") {
+    $url.hostname = getDockerHostDomain();
+  }
+
+  return $url.toString();
+}
+
+export function getDockerHostDomain() {
+  return isMacOS || isWindows ? "host.docker.internal" : "localhost";
+}
+
+/** Extract the W3C traceparent string from an untyped trace context record */
+export function extractTraceparent(traceContext?: Record<string, unknown>): string | undefined {
+  if (
+    traceContext &&
+    "traceparent" in traceContext &&
+    typeof traceContext.traceparent === "string"
+  ) {
+    return traceContext.traceparent;
+  }
+  return undefined;
+}
+
+export function getRunnerId(runId: string, attemptNumber?: number) {
+  const parts = ["runner", runId.replace("run_", "")];
+
+  if (attemptNumber && attemptNumber > 1) {
+    parts.push(`attempt-${attemptNumber}`);
+  }
+
+  return parts.join("-");
+}
+
+/** Derive a unique runnerId for a restore cycle using the checkpoint suffix */
+export function getRestoreRunnerId(runFriendlyId: string, checkpointId: string) {
+  const runIdShort = runFriendlyId.replace("run_", "");
+  const checkpointSuffix = checkpointId.slice(-8);
+  return `runner-${runIdShort}-${checkpointSuffix}`;
+}

apps/supervisor/src/wideEvents/baggage.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { encodeBaggage } from "./baggage.js";
+
+describe("encodeBaggage", () => {
+  it("returns empty string for an empty map", () => {
+    expect(encodeBaggage({})).toBe("");
+  });
+
+  it("encodes a single entry as k=v", () => {
+    expect(encodeBaggage({ run_id: "run-1" })).toBe("run_id=run-1");
+  });
+
+  it("sorts keys for stable output across hops", () => {
+    expect(encodeBaggage({ b: "2", a: "1", c: "3" })).toBe("a=1,b=2,c=3");
+  });
+
+  it("skips empty keys and empty values", () => {
+    expect(encodeBaggage({ "": "v", k: "", real: "x" })).toBe("real=x");
+  });
+
+  it("truncates values longer than the cap", () => {
+    const long = "x".repeat(1024);
+    const got = encodeBaggage({ k: long });
+    const value = got.slice("k=".length);
+    expect(value.length).toBe(256);
+  });
+
+  it("caps multibyte values by UTF-8 bytes, not code units", () => {
+    const long = "あ".repeat(512); // 3 UTF-8 bytes each
+    const got = encodeBaggage({ k: long });
+    const value = got.slice("k=".length);
+    expect(Buffer.byteLength(value, "utf8")).toBeLessThanOrEqual(256);
+  });
+
+  it("caps the number of entries", () => {
+    const meta: Record<string, string> = {};
+    for (let i = 0; i < 50; i++) {
+      // Sortable two-digit keys so we know which 32 survive.
+      meta[`k${String(i).padStart(2, "0")}`] = "v";
+    }
+    const got = encodeBaggage(meta);
+    expect(got.split(",").length).toBe(32);
+  });
+});

apps/supervisor/src/wideEvents/baggage.ts

@@ -0,0 +1,45 @@
+/**
+ * W3C Baggage (https://www.w3.org/TR/baggage/) encoding for outbound peer
+ * calls. Serialises a State's `meta` map into a `Baggage` header value so
+ * the downstream service auto-stamps the same labels onto its own wide
+ * events - even on early-error paths that bail before parsing the request
+ * body.
+ *
+ * Outbound discipline: only call this on peer-to-peer hops within the trust
+ * boundary. External-endpoint calls (image registries, cloud-provider
+ * APIs, third-party webhooks) must not include the Baggage header.
+ */
+
+import { truncateUtf8 } from "./truncate.js";
+
+/**
+ * Cap the number of entries serialised onto the header. A misbehaving
+ * caller's `meta` map shouldn't blow up downstream event width.
+ */
+const MAX_BAGGAGE_ENTRIES = 32;
+
+/**
+ * Cap each value's length. Defense against an upstream that stuffs
+ * unbounded payloads into a meta value.
+ */
+const MAX_BAGGAGE_VALUE_BYTES = 256;
+
+/**
+ * Encode a `meta` map as a Baggage header value (`k1=v1,k2=v2`). Keys are
+ * sorted for stable output across hops; an empty input yields the empty
+ * string so the caller can skip emitting the header entirely.
+ */
+export function encodeBaggage(meta: Record<string, string>): string {
+  const entries = Object.entries(meta).filter(([k, v]) => k && v);
+  if (entries.length === 0) return "";
+
+  entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+
+  const out: string[] = [];
+  for (const [k, raw] of entries) {
+    if (out.length >= MAX_BAGGAGE_ENTRIES) break;
+    const v = truncateUtf8(raw, MAX_BAGGAGE_VALUE_BYTES);
+    out.push(`${k}=${v}`);
+  }
+  return out.join(",");
+}

apps/supervisor/src/wideEvents/context.ts

@@ -0,0 +1,14 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { State } from "./state.js";
+
+/**
+ * AsyncLocalStorage threading per-operation `State` through the call stack.
+ * Wrappers enter a state via `wideEventStorage.run(state, () => fn())` and
+ * any code in the async call tree retrieves it via `fromContext()`.
+ */
+export const wideEventStorage = new AsyncLocalStorage<State>();
+
+/** Returns the State attached to the current async context, or null. */
+export function fromContext(): State | null {
+  return wideEventStorage.getStore() ?? null;
+}

apps/supervisor/src/wideEvents/emit.test.ts

@@ -0,0 +1,134 @@
+import { describe, it, expect } from "vitest";
+import { emit, EmitMessage } from "./emit.js";
+import { newState } from "./new.js";
+
+function captureEmit(state: Parameters<typeof emit>[0]): Record<string, unknown> {
+  const captured: string[] = [];
+  const origWrite = process.stdout.write;
+  process.stdout.write = ((chunk: unknown) => {
+    captured.push(String(chunk));
+    return true;
+  }) as typeof process.stdout.write;
+  try {
+    emit(state);
+  } finally {
+    process.stdout.write = origWrite;
+  }
+  expect(captured).toHaveLength(1);
+  const line = captured[0];
+  if (!line) throw new Error("no captured line");
+  return JSON.parse(line) as Record<string, unknown>;
+}
+
+describe("emit", () => {
+  it("emits a single line with the stable message + request_id", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    s.durationMs = 5;
+    const out = captureEmit(s);
+    expect(out.msg).toBe(EmitMessage);
+    expect(out.request_id).toBe(s.requestId);
+    expect(out.service).toBe("supervisor");
+    expect(out.ok).toBe(true);
+    expect(out.status).toBe(200);
+    expect(out.duration_ms).toBe(5);
+  });
+
+  it("emits start_time as an ISO timestamp set by newState", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    const out = captureEmit(s);
+    expect(typeof out.start_time).toBe("string");
+    // Microsecond-precision RFC3339 (6 fractional digits), parseable as a date.
+    expect(out.start_time).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/);
+    expect(Number.isNaN(new Date(out.start_time as string).getTime())).toBe(false);
+  });
+
+  it("omits start_time when unset", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    delete s.startTime;
+    s.statusCode = 200;
+    s.ok = true;
+    const out = captureEmit(s);
+    expect(out).not.toHaveProperty("start_time");
+  });
+
+  it("omits empty optional fields", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    const out = captureEmit(s);
+    expect(out).not.toHaveProperty("trace_id");
+    expect(out).not.toHaveProperty("version");
+    expect(out).not.toHaveProperty("commit_sha");
+    expect(out).not.toHaveProperty("error.code");
+  });
+
+  it("flattens meta keys as meta.<key>", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    s.meta.run_id = "run_abc";
+    s.meta.deployment_id = "dep_xyz";
+    const out = captureEmit(s);
+    expect(out["meta.run_id"]).toBe("run_abc");
+    expect(out["meta.deployment_id"]).toBe("dep_xyz");
+    expect(out).not.toHaveProperty("meta");
+  });
+
+  it("flattens phases as phase.<name>.<field>", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    s.phases.push({ name: "warm_start", durationMs: 12, ok: true, attempts: 1 });
+    s.phases.push({
+      name: "workload_create",
+      durationMs: 3,
+      ok: false,
+      attempts: 2,
+      errorCode: "Error",
+      errorMsg: "boom",
+      sub: { create_ms: 1 },
+    });
+    const out = captureEmit(s);
+    expect(out["phase.warm_start.duration_ms"]).toBe(12);
+    expect(out["phase.warm_start.ok"]).toBe(true);
+    expect(out["phase.warm_start.attempts"]).toBe(1);
+    expect(out["phase.workload_create.duration_ms"]).toBe(3);
+    expect(out["phase.workload_create.ok"]).toBe(false);
+    expect(out["phase.workload_create.attempts"]).toBe(2);
+    expect(out["phase.workload_create.error_code"]).toBe("Error");
+    expect(out["phase.workload_create.error_message"]).toBe("boom");
+    expect(out["phase.workload_create.create_ms"]).toBe(1);
+  });
+
+  it("includes error.code/message/kind when state.error is set", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 500;
+    s.error = { code: "InternalError", message: "kaboom", kind: "internal" };
+    const out = captureEmit(s);
+    expect(out["error.code"]).toBe("InternalError");
+    expect(out["error.message"]).toBe("kaboom");
+    expect(out["error.kind"]).toBe("internal");
+  });
+
+  it("truncates very long error messages", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.error = { code: "Big", message: "x".repeat(2000), kind: "internal" };
+    const out = captureEmit(s);
+    expect((out["error.message"] as string).length).toBe(512);
+  });
+
+  it("flattens extras at the top level", () => {
+    const s = newState({ service: "supervisor", env: {} });
+    s.statusCode = 200;
+    s.ok = true;
+    s.extras.route = "/health";
+    s.extras["dispatch.result"] = "hit";
+    const out = captureEmit(s);
+    expect(out.route).toBe("/health");
+    expect(out["dispatch.result"]).toBe("hit");
+  });
+});

apps/supervisor/src/wideEvents/emit.ts

@@ -0,0 +1,84 @@
+import type { State } from "./state.js";
+import { truncateUtf8 } from "./truncate.js";
+
+/**
+ * Stable slog message string for every wide event. Downstream filters (jq,
+ * Axiom queries, Vector pipelines) pin to this constant. The `service` field
+ * disambiguates which service emitted it.
+ */
+export const EmitMessage = "wide_event";
+
+const MAX_ERROR_MSG_BYTES = 512;
+
+/**
+ * Serializes a State as a single flat-keyed JSON line on stdout. Keys are
+ * flat (no nested objects) to keep jq filtering and Axiom indexing cheap.
+ * Empty optional fields are omitted.
+ */
+export function emit(state: State): void {
+  // Best-effort: an observability failure (serialization, stdout write) must
+  // never break or mask the caller's operation. Every call site relies on this.
+  try {
+    const out: Record<string, unknown> = {
+      msg: EmitMessage,
+      request_id: state.requestId,
+    };
+
+    if (state.traceId) out.trace_id = state.traceId;
+    appendIfSet(out, "start_time", state.startTime);
+    appendIfSet(out, "service", state.service);
+    appendIfSet(out, "version", state.version);
+    appendIfSet(out, "commit_sha", state.commitSha);
+    appendIfSet(out, "region", state.region);
+    appendIfSet(out, "node_id", state.nodeId);
+
+    appendIfSet(out, "op", state.op);
+    appendIfSet(out, "kind", state.kind);
+
+    out.ok = state.ok;
+    if (state.statusCode !== 0) out.status = state.statusCode;
+    out.duration_ms = state.durationMs;
+
+    if (state.error) {
+      appendIfSet(out, "error.code", state.error.code);
+      appendIfSet(out, "error.message", truncateUtf8(state.error.message, MAX_ERROR_MSG_BYTES));
+      appendIfSet(out, "error.kind", state.error.kind);
+    }
+
+    for (const [k, v] of Object.entries(state.meta)) {
+      out["meta." + k] = v;
+    }
+
+    for (const p of state.phases) {
+      const prefix = "phase." + p.name + ".";
+      out[prefix + "duration_ms"] = p.durationMs;
+      out[prefix + "ok"] = p.ok;
+      out[prefix + "attempts"] = p.attempts;
+      if (p.errorCode) out[prefix + "error_code"] = p.errorCode;
+      if (p.errorMsg) out[prefix + "error_message"] = p.errorMsg;
+      if (p.sub) {
+        for (const [sk, sv] of Object.entries(p.sub)) {
+          out[prefix + sk] = sv;
+        }
+      }
+    }
+
+    for (const [k, v] of Object.entries(state.extras)) {
+      out[k] = v;
+    }
+
+    process.stdout.write(JSON.stringify(out) + "\n");
+  } catch (err) {
+    try {
+      process.stderr.write(
+        `wide_event_emit_failed: ${err instanceof Error ? err.message : String(err)}\n`
+      );
+    } catch {
+      // last resort - drop the event rather than throw
+    }
+  }
+}
+
+function appendIfSet(out: Record<string, unknown>, key: string, value: string | undefined): void {
+  if (value) out[key] = value;
+}

apps/supervisor/src/wideEvents/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Wide-event observability surface for the supervisor. One flat-keyed JSON
+ * line per natural unit of work (HTTP request, dequeue iteration, socket
+ * lifecycle event). Events join across services via `trace_id` (parsed from
+ * the inbound W3C `traceparent`) and `meta.run_id`.
+ *
+ * Off by default behind a kill switch - the dispatch hotpath runs at high
+ * QPS, so logging pressure must be cleanly removable.
+ */
+export { type Env, isValidRequestId, newState, type NewStateOptions } from "./new.js";
+export { emit, EmitMessage } from "./emit.js";
+export { parseTraceId } from "./traceparent.js";
+export { fromContext, wideEventStorage } from "./context.js";
+export { type PhaseOpt, recordPhase, recordPhaseSince, timePhase } from "./record.js";
+export {
+  emitOneShot,
+  runWideEvent,
+  setExtra,
+  setMeta,
+  type WideEventLifecycleOptions,
+  type WideEventOptions,
+} from "./middleware.js";
+export type { ErrorInfo, PhaseRecord, State } from "./state.js";
+export { encodeBaggage } from "./baggage.js";

apps/supervisor/src/wideEvents/middleware.test.ts

@@ -0,0 +1,206 @@
+import { describe, it, expect } from "vitest";
+import { fromContext } from "./context.js";
+import { emitOneShot, runWideEvent, setMeta } from "./middleware.js";
+
+function captureStdout(fn: () => Promise<unknown> | unknown): Promise<string[]> {
+  const captured: string[] = [];
+  const orig = process.stdout.write;
+  process.stdout.write = ((chunk: unknown) => {
+    captured.push(String(chunk));
+    return true;
+  }) as typeof process.stdout.write;
+  return Promise.resolve(fn())
+    .finally(() => {
+      process.stdout.write = orig;
+    })
+    .then(() => captured);
+}
+
+describe("runWideEvent", () => {
+  it("emits one event with ok=true when no statusCode is set", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test", route: "/x", method: "POST" },
+        async () => undefined
+      );
+    });
+    expect(lines).toHaveLength(1);
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.ok).toBe(true);
+    expect(ev.service).toBe("supervisor");
+    expect(ev.route).toBe("/x");
+    expect(ev.method).toBe("POST");
+    expect(typeof ev.duration_ms).toBe("number");
+    expect(typeof ev.request_id).toBe("string");
+  });
+
+  it("derives ok from statusCode set via finalize", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test" },
+        async () => undefined,
+        (state) => {
+          state.statusCode = 200;
+        }
+      );
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.ok).toBe(true);
+    expect(ev.status).toBe(200);
+  });
+
+  it("treats 4xx as ok=false", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test" },
+        async () => undefined,
+        (state) => {
+          state.statusCode = 400;
+        }
+      );
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.ok).toBe(false);
+    expect(ev.status).toBe(400);
+  });
+
+  it("emits ok=false with error.kind=internal on throw", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test" },
+        async () => {
+          throw new Error("boom");
+        }
+      ).catch(() => undefined);
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.ok).toBe(false);
+    expect(ev.status).toBe(500);
+    expect(ev["error.kind"]).toBe("internal");
+    expect(ev["error.message"]).toBe("boom");
+  });
+
+  it("threads state through AsyncLocalStorage", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test" },
+        async () => {
+          setMeta(fromContext(), "run_id", "run_abc");
+        }
+      );
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev["meta.run_id"]).toBe("run_abc");
+    expect(ev.ok).toBe(true);
+  });
+
+  it("picks up inbound traceparent for trace_id", async () => {
+    const tp = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: true, op: "test", traceparent: tp },
+        async () => undefined
+      );
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.trace_id).toBe("4bf92f3577b34da6a3ce929d0e0e4736");
+  });
+
+  it("honours setup() to attach meta and extras before fn runs", async () => {
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        {
+          service: "supervisor",
+          env: {},
+          enabled: true,
+          op: "test",
+          setup: (state) => {
+            state.meta.run_id = "run_abc";
+            state.extras.iteration = "dequeue";
+          },
+        },
+        async () => undefined
+      );
+    });
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev["meta.run_id"]).toBe("run_abc");
+    expect(ev.iteration).toBe("dequeue");
+  });
+
+  it("short-circuits to pass-through when enabled=false", async () => {
+    let seenState: ReturnType<typeof fromContext> = null;
+    const lines = await captureStdout(async () => {
+      await runWideEvent(
+        { service: "supervisor", env: {}, enabled: false, op: "test" },
+        async () => {
+          seenState = fromContext();
+        }
+      );
+    });
+    expect(lines).toHaveLength(0);
+    expect(seenState).toBe(null);
+  });
+
+  it("isolates state across concurrent invocations", async () => {
+    const lines = await captureStdout(async () => {
+      await Promise.all(
+        ["a", "b", "c"].map((tag) =>
+          runWideEvent({ service: "supervisor", env: {}, enabled: true, op: "test" }, async () => {
+            const s = fromContext();
+            if (!s) throw new Error("no state");
+            s.meta.tag = tag;
+            await new Promise((r) => setTimeout(r, 5));
+            expect(s.meta.tag).toBe(tag);
+          })
+        )
+      );
+    });
+    const tags = lines.map((l) => (JSON.parse(l) as Record<string, unknown>)["meta.tag"]);
+    expect(tags.sort()).toEqual(["a", "b", "c"]);
+  });
+});
+
+describe("emitOneShot", () => {
+  it("emits a single event with populated meta when enabled", async () => {
+    const lines = await captureStdout(() => {
+      emitOneShot({
+        service: "supervisor",
+        env: {},
+        enabled: true,
+        op: "test",
+        populate: (s) => {
+          s.meta.run_id = "run_abc";
+          s.extras.event = "run:start";
+        },
+      });
+    });
+    expect(lines).toHaveLength(1);
+    const line = lines[0];
+    if (!line) throw new Error("no line");
+    const ev = JSON.parse(line) as Record<string, unknown>;
+    expect(ev.ok).toBe(true);
+    expect(ev["meta.run_id"]).toBe("run_abc");
+    expect(ev.event).toBe("run:start");
+  });
+
+  it("emits nothing when disabled", async () => {
+    const lines = await captureStdout(() => {
+      emitOneShot({ service: "supervisor", env: {}, enabled: false, op: "test" });
+    });
+    expect(lines).toHaveLength(0);
+  });
+});

apps/supervisor/src/wideEvents/middleware.ts

@@ -0,0 +1,132 @@
+import { emit } from "./emit.js";
+import { newState, type Env } from "./new.js";
+import { wideEventStorage } from "./context.js";
+import type { State } from "./state.js";
+
+/** Options common to every wide-event lifecycle. */
+export type WideEventOptions = {
+  service: string;
+  env: Env;
+  /**
+   * Kill switch. When false, lifecycles degenerate into transparent
+   * pass-through - no State allocation, no AsyncLocalStorage run, no emit.
+   * Important for the dispatch hotpath where logging pressure must be
+   * cleanly removable.
+   */
+  enabled: boolean;
+};
+
+/** Per-invocation options layered on top of `WideEventOptions`. */
+export type WideEventLifecycleOptions = WideEventOptions & {
+  /** Operation discriminator (`instance.create`, `dequeue`, ...). Required. */
+  op: string;
+  /** Event shape: `inbound` | `outbound` | `event` | `scheduled`. Optional. */
+  kind?: string;
+  /** Route template (HTTP only) captured into `extras.route`. */
+  route?: string;
+  /** HTTP method captured into `extras.method`. */
+  method?: string;
+  /** Inbound W3C traceparent (HTTP header, queue message field). */
+  traceparent?: string;
+  /** Inbound request id (e.g. `x-request-id` header). */
+  inboundRequestId?: string;
+  /** Runs after the state is built, before the wrapped fn. Use to attach meta. */
+  setup?: (state: State) => void;
+};
+
+/**
+ * Runs `fn` inside an AsyncLocalStorage state and emits one wide event on
+ * completion or error. `finalize` runs after `fn` returns but before emit -
+ * use it to read out-of-band outcome info (e.g. `res.statusCode` for an HTTP
+ * route) and assign to `state.statusCode`. The wrapper computes `ok` from
+ * `statusCode` if it's set; otherwise it defaults to true on success.
+ *
+ * Returns the original `fn` result. When `enabled=false`, `fn` runs unchanged
+ * with no event emitted.
+ */
+export async function runWideEvent<T>(
+  opts: WideEventLifecycleOptions,
+  fn: () => Promise<T> | T,
+  finalize?: (state: State) => void
+): Promise<T> {
+  if (!opts.enabled) {
+    return fn();
+  }
+
+  const state = newState({
+    service: opts.service,
+    env: opts.env,
+    inboundRequestId: opts.inboundRequestId,
+    traceparent: opts.traceparent,
+    op: opts.op,
+    kind: opts.kind,
+  });
+  if (opts.route) state.extras.route = opts.route;
+  if (opts.method) state.extras.method = opts.method;
+
+  const start = performance.now();
+  try {
+    if (opts.setup) opts.setup(state);
+    const result = await wideEventStorage.run(state, () => Promise.resolve(fn()));
+    state.durationMs = Math.round(performance.now() - start);
+    if (finalize) finalize(state);
+    if (state.statusCode !== 0) {
+      state.ok = state.statusCode >= 200 && state.statusCode < 300;
+    } else {
+      state.ok = true;
+    }
+    emit(state);
+    return result;
+  } catch (err) {
+    state.durationMs = Math.round(performance.now() - start);
+    const e = err instanceof Error ? err : new Error(String(err));
+    if (state.statusCode === 0) state.statusCode = 500;
+    state.ok = false;
+    state.error = {
+      code: e.name || "Error",
+      message: e.message,
+      kind: "internal",
+    };
+    emit(state);
+    throw err;
+  }
+}
+
+/**
+ * One-shot wide event with no wrapped operation. Use for socket lifecycle
+ * events (`run:start`, `run:stop`) where there is no surrounding async unit
+ * of work to time. `populate` runs synchronously to attach meta/extras
+ * before emit.
+ */
+export function emitOneShot(
+  opts: WideEventOptions & {
+    op: string;
+    kind?: string;
+    traceparent?: string;
+    populate?: (state: State) => void;
+  }
+): void {
+  if (!opts.enabled) return;
+  const state = newState({
+    service: opts.service,
+    env: opts.env,
+    traceparent: opts.traceparent,
+    op: opts.op,
+    kind: opts.kind,
+  });
+  if (opts.populate) opts.populate(state);
+  state.ok = true;
+  emit(state);
+}
+
+/** Convenience accessor for in-handler meta mutation. */
+export function setMeta(state: State | null, key: string, value: string): void {
+  if (!state) return;
+  state.meta[key] = value;
+}
+
+/** Convenience for free-form fields (did_warm_start, dispatch.result, ...). */
+export function setExtra(state: State | null, key: string, value: unknown): void {
+  if (!state) return;
+  state.extras[key] = value;
+}

apps/supervisor/src/wideEvents/new.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect } from "vitest";
+import { isValidRequestId, newState } from "./new.js";
+
+describe("isValidRequestId", () => {
+  it("accepts visible ASCII", () => {
+    expect(isValidRequestId("req-abc-123_456.7")).toBe(true);
+    expect(isValidRequestId("a")).toBe(true);
+  });
+
+  it("rejects empty string", () => {
+    expect(isValidRequestId("")).toBe(false);
+  });
+
+  it("rejects overlong strings (>128 bytes)", () => {
+    expect(isValidRequestId("a".repeat(128))).toBe(true);
+    expect(isValidRequestId("a".repeat(129))).toBe(false);
+  });
+
+  it("rejects whitespace, newlines, control chars", () => {
+    expect(isValidRequestId("has space")).toBe(false);
+    expect(isValidRequestId("has\ttab")).toBe(false);
+    expect(isValidRequestId("has\nnewline")).toBe(false);
+    expect(isValidRequestId("\x00null")).toBe(false);
+  });
+
+  it("rejects high-bit / non-ASCII", () => {
+    expect(isValidRequestId("café")).toBe(false);
+    expect(isValidRequestId("a\x7f")).toBe(false);
+  });
+});
+
+describe("newState", () => {
+  const env = { version: "1.0.0", commitSha: "abc123", region: "us-east-1", nodeId: "node-1" };
+
+  it("populates service identity from env", () => {
+    const s = newState({ service: "supervisor", env });
+    expect(s.service).toBe("supervisor");
+    expect(s.version).toBe("1.0.0");
+    expect(s.commitSha).toBe("abc123");
+    expect(s.region).toBe("us-east-1");
+    expect(s.nodeId).toBe("node-1");
+  });
+
+  it("mints a fresh request id when none provided", () => {
+    const s = newState({ service: "test", env: {} });
+    expect(s.requestId).toMatch(/^req-[0-9a-f]{32}$/);
+  });
+
+  it("honours a valid inbound request id", () => {
+    const s = newState({ service: "test", env: {}, inboundRequestId: "trace-abc-123" });
+    expect(s.requestId).toBe("trace-abc-123");
+  });
+
+  it("rejects unsafe inbound request id and mints a fresh one", () => {
+    const s = newState({ service: "test", env: {}, inboundRequestId: "has space" });
+    expect(s.requestId).toMatch(/^req-[0-9a-f]{32}$/);
+  });
+
+  it("parses traceparent into traceId and preserves the raw header", () => {
+    const tp = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+    const s = newState({ service: "test", env: {}, traceparent: tp });
+    expect(s.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736");
+    expect(s.traceparent).toBe(tp);
+  });
+
+  it("leaves traceId empty when no traceparent provided", () => {
+    const s = newState({ service: "test", env: {} });
+    expect(s.traceId).toBe("");
+    expect(s.traceparent).toBe("");
+  });
+
+  it("initialises empty meta/extras/phases", () => {
+    const s = newState({ service: "test", env: {} });
+    expect(s.meta).toEqual({});
+    expect(s.extras).toEqual({});
+    expect(s.phases).toEqual([]);
+    expect(s.ok).toBe(false);
+    expect(s.statusCode).toBe(0);
+    expect(s.durationMs).toBe(0);
+  });
+});

apps/supervisor/src/wideEvents/new.ts

@@ -0,0 +1,96 @@
+import { randomBytes } from "node:crypto";
+import { parseTraceId } from "./traceparent.js";
+import type { State } from "./state.js";
+
+const MAX_REQUEST_ID_LEN = 128;
+
+/**
+ * Validates an inbound request id. Non-empty, no longer than 128 bytes,
+ * composed entirely of visible ASCII (0x21..0x7E). Rejects newlines, control
+ * characters, whitespace, DEL, high-bit bytes - any of which could poison the
+ * log pipeline if echoed back verbatim.
+ */
+export function isValidRequestId(s: string): boolean {
+  if (s.length === 0 || s.length > MAX_REQUEST_ID_LEN) return false;
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i);
+    if (c < 0x21 || c > 0x7e) return false;
+  }
+  return true;
+}
+
+/**
+ * Service-level identity that's constant for the lifetime of the process.
+ * Populated once at startup, copied into every State.
+ */
+export type Env = {
+  version?: string;
+  commitSha?: string;
+  region?: string;
+  nodeId?: string;
+};
+
+export type NewStateOptions = {
+  service: string;
+  env: Env;
+  /** Optional inbound request id (e.g. from `x-request-id`). If unsafe or absent, a fresh `req-<hex>` is minted. */
+  inboundRequestId?: string;
+  /** Optional inbound W3C traceparent (HTTP header, queue message field). */
+  traceparent?: string;
+  /** Operation discriminator. Dotted `noun.verb`. Defaults to empty (set later). */
+  op?: string;
+  /** Event shape: `inbound` | `outbound` | `event` | `scheduled`. Defaults to empty. */
+  kind?: string;
+};
+
+/**
+ * Builds a State for a wide-event lifecycle.
+ *
+ *   - requestId: honours `inboundRequestId` if present and safe; otherwise
+ *     mints a fresh `req-<hex>` id.
+ *   - traceId: parsed from the provided traceparent (graceful empty if
+ *     absent or malformed).
+ *   - traceparent: preserved verbatim for downstream propagation.
+ */
+export function newState(opts: NewStateOptions): State {
+  const traceparent = opts.traceparent ?? "";
+  const inbound = opts.inboundRequestId ?? "";
+  const requestId = isValidRequestId(inbound) ? inbound : newRequestId();
+
+  return {
+    startTime: nowRfc3339(),
+    requestId,
+    traceId: parseTraceId(traceparent),
+    traceparent,
+    service: opts.service,
+    version: opts.env.version,
+    commitSha: opts.env.commitSha,
+    region: opts.env.region,
+    nodeId: opts.env.nodeId,
+    op: opts.op ?? "",
+    kind: opts.kind ?? "",
+    meta: {},
+    phases: [],
+    ok: false,
+    statusCode: 0,
+    durationMs: 0,
+    extras: {},
+  };
+}
+
+function newRequestId(): string {
+  return "req-" + randomBytes(16).toString("hex");
+}
+
+/**
+ * Current wall-clock time as an RFC3339 string with microsecond precision.
+ * `Date.toISOString()` only has millisecond resolution, which is too coarse to
+ * order multiple wide events emitted within the same millisecond.
+ * `performance.timeOrigin + performance.now()` gives a sub-millisecond wall-clock
+ * reading; we append the microsecond digits to the millisecond ISO string.
+ */
+function nowRfc3339(): string {
+  const ms = performance.timeOrigin + performance.now();
+  const micros = Math.floor((ms % 1) * 1000); // microseconds within the millisecond (0..999)
+  return new Date(ms).toISOString().slice(0, -1) + String(micros).padStart(3, "0") + "Z";
+}

apps/supervisor/src/wideEvents/record.test.ts

@@ -0,0 +1,112 @@
+import { describe, it, expect } from "vitest";
+import { fromContext, wideEventStorage } from "./context.js";
+import { recordPhase, recordPhaseSince, timePhase } from "./record.js";
+import { newState } from "./new.js";
+import type { State } from "./state.js";
+
+function makeState(): State {
+  return newState({ service: "test", env: {} });
+}
+
+describe("recordPhase", () => {
+  it("appends a successful phase", () => {
+    const s = makeState();
+    recordPhase(s, "lookup", performance.now() - 50, undefined);
+    expect(s.phases).toHaveLength(1);
+    const phase = s.phases[0];
+    if (!phase) throw new Error("missing phase");
+    expect(phase.name).toBe("lookup");
+    expect(phase.ok).toBe(true);
+    expect(phase.attempts).toBe(1);
+    expect(phase.durationMs).toBeGreaterThanOrEqual(45);
+  });
+
+  it("appends a failed phase with error code/message", () => {
+    const s = makeState();
+    recordPhase(s, "dispatch", performance.now(), new Error("nope"));
+    const phase = s.phases[0];
+    if (!phase) throw new Error("missing phase");
+    expect(phase.ok).toBe(false);
+    expect(phase.errorCode).toBe("Error");
+    expect(phase.errorMsg).toBe("nope");
+  });
+
+  it("truncates very long error messages", () => {
+    const s = makeState();
+    recordPhase(s, "x", performance.now(), new Error("y".repeat(2000)));
+    const phase = s.phases[0];
+    if (!phase) throw new Error("missing phase");
+    expect(phase.errorMsg?.length).toBe(512);
+  });
+
+  it("honours opts.attempts", () => {
+    const s = makeState();
+    recordPhase(s, "retry", performance.now(), undefined, { attempts: 3 });
+    expect(s.phases[0]?.attempts).toBe(3);
+  });
+
+  it("attaches sub-timings", () => {
+    const s = makeState();
+    recordPhase(s, "complex", performance.now(), undefined, { sub: { setup_ms: 10, work_ms: 5 } });
+    expect(s.phases[0]?.sub).toEqual({ setup_ms: 10, work_ms: 5 });
+  });
+
+  it("is a no-op when state is null", () => {
+    expect(() => recordPhase(null, "x", performance.now(), undefined)).not.toThrow();
+  });
+});
+
+describe("timePhase + AsyncLocalStorage threading", () => {
+  it("records via fromContext on success", async () => {
+    const s = makeState();
+    const value = await wideEventStorage.run(s, () => timePhase("work", async () => 42));
+    expect(value).toBe(42);
+    expect(s.phases).toHaveLength(1);
+    expect(s.phases[0]?.ok).toBe(true);
+  });
+
+  it("records via fromContext on error and rethrows", async () => {
+    const s = makeState();
+    await expect(
+      wideEventStorage.run(s, () =>
+        timePhase("work", async () => {
+          throw new Error("boom");
+        })
+      )
+    ).rejects.toThrow("boom");
+    expect(s.phases).toHaveLength(1);
+    expect(s.phases[0]?.ok).toBe(false);
+    expect(s.phases[0]?.errorMsg).toBe("boom");
+  });
+
+  it("runs fn unchanged when no state on context", async () => {
+    const value = await timePhase("work", async () => "ok");
+    expect(value).toBe("ok");
+  });
+});
+
+describe("recordPhaseSince", () => {
+  it("records using a caller-captured start time", async () => {
+    const s = makeState();
+    await wideEventStorage.run(s, async () => {
+      const start = performance.now();
+      await new Promise((r) => setTimeout(r, 10));
+      recordPhaseSince("spanning", start, undefined);
+    });
+    expect(s.phases).toHaveLength(1);
+    expect(s.phases[0]?.durationMs).toBeGreaterThanOrEqual(8);
+  });
+});
+
+describe("fromContext", () => {
+  it("returns null when no state attached", () => {
+    expect(fromContext()).toBe(null);
+  });
+
+  it("returns the state when inside wideEventStorage.run", () => {
+    const s = makeState();
+    wideEventStorage.run(s, () => {
+      expect(fromContext()).toBe(s);
+    });
+  });
+});

apps/supervisor/src/wideEvents/record.ts

@@ -0,0 +1,82 @@
+import { fromContext } from "./context.js";
+import type { PhaseRecord, State } from "./state.js";
+import { truncateUtf8 } from "./truncate.js";
+
+const MAX_ERROR_MSG_BYTES = 512;
+
+/** Optional knobs for a phase record. */
+export type PhaseOpt = {
+  /** Attempt count for the phase (default 1). */
+  attempts?: number;
+  /** Sub-timings to fold into `phase.<name>.<key>`. */
+  sub?: Record<string, number>;
+};
+
+/**
+ * Appends a phase outcome to `state.phases`. Safe to call on success
+ * (`err === undefined`) and error paths. `errorMsg` is truncated to 512 bytes
+ * to keep the wide event compact. No-op if state is null.
+ */
+export function recordPhase(
+  state: State | null,
+  name: string,
+  startMs: number,
+  err: Error | undefined,
+  opts: PhaseOpt = {}
+): void {
+  if (!state) return;
+  const p: PhaseRecord = {
+    name,
+    durationMs: Math.round(performance.now() - startMs),
+    ok: err === undefined,
+    attempts: opts.attempts ?? 1,
+  };
+  if (err) {
+    p.errorCode = err.name || "Error";
+    p.errorMsg = truncateUtf8(err.message, MAX_ERROR_MSG_BYTES);
+  }
+  if (opts.sub) p.sub = opts.sub;
+  state.phases.push(p);
+}
+
+/**
+ * Runs `fn` and appends a phase outcome to the State attached to the current
+ * async context. If no state is on context (test paths, background work),
+ * `fn` runs unchanged. The phase is recorded on both success and error paths
+ * so failed phases still appear in the wide event with duration_ms +
+ * error_code.
+ */
+export async function timePhase<T>(
+  name: string,
+  fn: () => Promise<T> | T,
+  opts: PhaseOpt = {}
+): Promise<T> {
+  const start = performance.now();
+  try {
+    const result = await fn();
+    recordPhase(fromContext(), name, start, undefined, opts);
+    return result;
+  } catch (err) {
+    recordPhase(fromContext(), name, start, asError(err), opts);
+    throw err;
+  }
+}
+
+/**
+ * Appends a phase outcome to the State attached to the current async context
+ * using a `startMs` captured by the caller. Use when the phase boundary spans
+ * multiple calls with intermediate error handling that can't fit inside a
+ * single `timePhase` closure. Nil-state safe.
+ */
+export function recordPhaseSince(
+  name: string,
+  startMs: number,
+  err: Error | undefined,
+  opts: PhaseOpt = {}
+): void {
+  recordPhase(fromContext(), name, startMs, err, opts);
+}
+
+function asError(e: unknown): Error {
+  return e instanceof Error ? e : new Error(String(e));
+}

apps/supervisor/src/wideEvents/state.ts

@@ -0,0 +1,84 @@
+/**
+ * Per-event accumulator backing a single wide event. The supervisor emits one
+ * flat-keyed JSON line per natural unit of work (dequeue iteration, HTTP
+ * request, socket lifecycle event). Optional fields are omitted on emit so
+ * events stay compact.
+ */
+export type State = {
+  /**
+   * Wall-clock time the event began, as an ISO-8601 string. Emitted as
+   * `start_time` so log collection orders events by when work started rather
+   * than by the collector's ingestion time.
+   */
+  startTime?: string;
+
+  // Cross-stack correlation.
+  requestId: string;
+  traceId: string;
+  /**
+   * Raw inbound W3C `traceparent`, preserved verbatim so outbound calls can
+   * propagate the same trace context without losing the parent span-id.
+   * Empty when no inbound traceparent was set.
+   */
+  traceparent: string;
+
+  // Service identity (set by `newState` from Env).
+  service: string;
+  version?: string;
+  commitSha?: string;
+  region?: string;
+  nodeId?: string;
+
+  /**
+   * Operation discriminator. Dotted `noun.verb` (e.g. `instance.create`,
+   * `snapshot.dispatch`). Low cardinality - bounded set per service, not
+   * unbounded. Empty allowed during construction but expected to be set
+   * before emit.
+   */
+  op: string;
+
+  /**
+   * Event shape. `inbound` for received requests, `outbound` for outgoing
+   * calls, `event` for ambient occurrences with no meaningful duration,
+   * `scheduled` for timer-driven work. Empty allowed; omitted from emit
+   * when empty.
+   */
+  kind: string;
+
+  // Caller-attached opaque metadata, flattened to `meta.<key>` on emit.
+  meta: Record<string, string>;
+
+  // Per-phase outcomes, in completion order.
+  phases: PhaseRecord[];
+
+  // Top-level outcome (set after the wrapped operation returns).
+  ok: boolean;
+  statusCode: number;
+  durationMs: number;
+  error?: ErrorInfo;
+
+  // Free-form ad-hoc additions (route, method, did_warm_start, ...).
+  extras: Record<string, unknown>;
+};
+
+/**
+ * Single named phase outcome. Retries collapse into `attempts > 1` with the
+ * last error reflected in errorCode/errorMsg.
+ */
+export type PhaseRecord = {
+  name: string;
+  durationMs: number;
+  ok: boolean;
+  attempts: number;
+  errorCode?: string;
+  errorMsg?: string;
+  sub?: Record<string, number>;
+};
+
+/** Top-level error summary for a failed operation. */
+export type ErrorInfo = {
+  code: string;
+  message: string;
+  /** Coarse classification - "client" | "upstream" | "internal" | "timeout". */
+  kind: string;
+};

apps/supervisor/src/wideEvents/traceparent.test.ts

@@ -0,0 +1,43 @@
+import { describe, it, expect } from "vitest";
+import { parseTraceId } from "./traceparent.js";
+
+describe("parseTraceId", () => {
+  const validTraceId = "4bf92f3577b34da6a3ce929d0e0e4736";
+  const validHeader = `00-${validTraceId}-00f067aa0ba902b7-01`;
+
+  it("extracts the trace-id from a valid W3C traceparent", () => {
+    expect(parseTraceId(validHeader)).toBe(validTraceId);
+  });
+
+  it("returns empty string for empty/null/undefined input", () => {
+    expect(parseTraceId("")).toBe("");
+    expect(parseTraceId(null)).toBe("");
+    expect(parseTraceId(undefined)).toBe("");
+  });
+
+  it("returns empty for wrong segment count", () => {
+    expect(parseTraceId("00-abc-def")).toBe("");
+    expect(parseTraceId("00-abc-def-01-extra")).toBe("");
+  });
+
+  it("returns empty for non-zero version byte", () => {
+    expect(parseTraceId(`01-${validTraceId}-00f067aa0ba902b7-01`)).toBe("");
+  });
+
+  it("returns empty for wrong-length trace-id", () => {
+    expect(parseTraceId("00-abc-00f067aa0ba902b7-01")).toBe("");
+  });
+
+  it("returns empty for non-hex trace-id", () => {
+    expect(parseTraceId("00-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz-00f067aa0ba902b7-01")).toBe("");
+  });
+
+  it("returns empty for all-zero trace-id", () => {
+    expect(parseTraceId("00-00000000000000000000000000000000-00f067aa0ba902b7-01")).toBe("");
+  });
+
+  it("accepts uppercase hex", () => {
+    const tid = "4BF92F3577B34DA6A3CE929D0E0E4736";
+    expect(parseTraceId(`00-${tid}-00f067aa0ba902b7-01`)).toBe(tid);
+  });
+});

apps/supervisor/src/wideEvents/traceparent.ts

@@ -0,0 +1,39 @@
+/**
+ * Extracts the trace-id from a W3C `traceparent` header. Returns "" when the
+ * header is absent, malformed, or carries an all-zero trace-id.
+ *
+ * Format: `<version>-<trace-id>-<span-id>-<flags>`
+ *   version : 2 hex chars, must be "00"
+ *   trace-id: 32 hex chars, non-zero
+ *   span-id : 16 hex chars (not validated - we only need trace-id)
+ *   flags   : 2 hex chars (not validated)
+ */
+export function parseTraceId(header: string | null | undefined): string {
+  if (!header) return "";
+  const parts = header.split("-");
+  if (parts.length !== 4) return "";
+  if (parts[0] !== "00") return "";
+  const tid = parts[1];
+  if (!tid || tid.length !== 32) return "";
+  if (!isHex(tid)) return "";
+  if (isAllZero(tid)) return "";
+  return tid;
+}
+
+function isHex(s: string): boolean {
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i);
+    const isDigit = c >= 0x30 && c <= 0x39;
+    const isLower = c >= 0x61 && c <= 0x66;
+    const isUpper = c >= 0x41 && c <= 0x46;
+    if (!isDigit && !isLower && !isUpper) return false;
+  }
+  return true;
+}
+
+function isAllZero(s: string): boolean {
+  for (let i = 0; i < s.length; i++) {
+    if (s.charCodeAt(i) !== 0x30) return false;
+  }
+  return true;
+}

apps/supervisor/src/wideEvents/truncate.test.ts

@@ -0,0 +1,30 @@
+import { describe, it, expect } from "vitest";
+import { truncateUtf8 } from "./truncate.js";
+
+describe("truncateUtf8", () => {
+  it("returns short ASCII unchanged", () => {
+    expect(truncateUtf8("hello", 512)).toBe("hello");
+  });
+
+  it("truncates ASCII to the byte cap", () => {
+    expect(truncateUtf8("x".repeat(1024), 256)).toBe("x".repeat(256));
+  });
+
+  it("never exceeds the byte cap for multibyte input", () => {
+    // "あ" is 3 UTF-8 bytes; 200 of them = 600 bytes.
+    const got = truncateUtf8("あ".repeat(200), 256);
+    expect(Buffer.byteLength(got, "utf8")).toBeLessThanOrEqual(256);
+  });
+
+  it("does not split a multibyte sequence", () => {
+    // 256 / 3 bytes = 85 whole chars (255 bytes), the 86th would overflow.
+    expect(truncateUtf8("あ".repeat(200), 256)).toBe("あ".repeat(85));
+  });
+
+  it("does not split a surrogate pair", () => {
+    // "😀" is 2 UTF-16 units / 4 UTF-8 bytes; only one fits under a 5-byte cap.
+    const got = truncateUtf8("😀😀", 5);
+    expect(got).toBe("😀");
+    expect(Buffer.byteLength(got, "utf8")).toBe(4);
+  });
+});

apps/supervisor/src/wideEvents/truncate.ts

@@ -0,0 +1,20 @@
+/**
+ * Truncate `value` to at most `maxBytes` UTF-8 bytes without splitting a
+ * multi-byte sequence or surrogate pair. Plain `.slice()` counts UTF-16 code
+ * units, so multibyte text can blow past a byte cap and cutting mid-pair
+ * leaves a lone surrogate that downstream JSON / Postgres consumers reject.
+ */
+export function truncateUtf8(value: string, maxBytes: number): string {
+  if (Buffer.byteLength(value, "utf8") <= maxBytes) return value;
+
+  let bytes = 0;
+  let end = 0;
+  // `for..of` yields whole code points, so a surrogate pair is never split.
+  for (const ch of value) {
+    const size = Buffer.byteLength(ch, "utf8");
+    if (bytes + size > maxBytes) break;
+    bytes += size;
+    end += ch.length;
+  }
+  return value.slice(0, end);
+}

apps/supervisor/src/workerToken.ts

@@ -0,0 +1,29 @@
+import { readFileSync } from "fs";
+import { env } from "./env.js";
+
+export function getWorkerToken() {
+  if (!env.TRIGGER_WORKER_TOKEN.startsWith("file://")) {
+    return env.TRIGGER_WORKER_TOKEN;
+  }
+
+  const tokenPath = env.TRIGGER_WORKER_TOKEN.replace("file://", "");
+
+  console.debug(
+    JSON.stringify({
+      message: "🔑 Reading worker token from file",
+      tokenPath,
+    })
+  );
+
+  try {
+    const token = readFileSync(tokenPath, "utf8").trim();
+    return token;
+  } catch (error) {
+    console.error(`Failed to read worker token from file: ${tokenPath}`, error);
+    throw new Error(
+      `Unable to read worker token from file: ${
+        error instanceof Error ? error.message : "Unknown error"
+      }`
+    );
+  }
+}

apps/supervisor/src/workloadManager/compute.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { ComputeClientError } from "@internal/compute";
+import { isRetryableCreateError, runnerNameForAttempt } from "./compute.js";
+
+describe("runnerNameForAttempt", () => {
+  it("keeps the unsuffixed name for the first attempt", () => {
+    expect(runnerNameForAttempt("runner-abc123", 1)).toBe("runner-abc123");
+  });
+
+  it("suffixes retry attempts deterministically", () => {
+    expect(runnerNameForAttempt("runner-abc123", 2)).toBe("runner-abc123-r2");
+    expect(runnerNameForAttempt("runner-abc123", 3)).toBe("runner-abc123-r3");
+  });
+});
+
+describe("isRetryableCreateError", () => {
+  it("retries statuses where the create definitely did not commit", () => {
+    expect(isRetryableCreateError(new ComputeClientError(500, "tap busy", "http://gw"))).toBe(true);
+    expect(isRetryableCreateError(new ComputeClientError(503, "no placement", "http://gw"))).toBe(
+      true
+    );
+  });
+
+  it("does not retry lost-response statuses (create may have committed)", () => {
+    expect(isRetryableCreateError(new ComputeClientError(502, "bad gateway", "http://gw"))).toBe(
+      false
+    );
+    expect(
+      isRetryableCreateError(new ComputeClientError(504, "gateway timeout", "http://gw"))
+    ).toBe(false);
+  });
+
+  it("does not retry 4xx responses", () => {
+    expect(isRetryableCreateError(new ComputeClientError(400, "bad request", "http://gw"))).toBe(
+      false
+    );
+    expect(isRetryableCreateError(new ComputeClientError(409, "conflict", "http://gw"))).toBe(
+      false
+    );
+  });
+
+  it("does not retry timeouts (instance may still be provisioning)", () => {
+    expect(isRetryableCreateError(new DOMException("timed out", "TimeoutError"))).toBe(false);
+  });
+
+  it("retries network-level fetch failures", () => {
+    expect(isRetryableCreateError(new TypeError("fetch failed"))).toBe(true);
+  });
+
+  it("does not retry unknown errors", () => {
+    expect(isRetryableCreateError(new Error("something else"))).toBe(false);
+    expect(isRetryableCreateError("string error")).toBe(false);
+  });
+});

apps/supervisor/src/workloadManager/compute.ts

@@ -0,0 +1,511 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
+import { flattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes";
+import {
+  type WorkloadManager,
+  type WorkloadManagerCreateOptions,
+  type WorkloadManagerOptions,
+} from "./types.js";
+import { ComputeClient, ComputeClientError, stripImageDigest } from "@internal/compute";
+import { setTimeout as sleep } from "node:timers/promises";
+import { extractTraceparent, getRunnerId } from "../util.js";
+import type { OtlpTraceService } from "../services/otlpTraceService.js";
+import { tryCatch } from "@trigger.dev/core";
+import { encodeBaggage, fromContext } from "../wideEvents/index.js";
+
+const DEFAULT_CREATE_MAX_ATTEMPTS = 3;
+const DEFAULT_CREATE_RETRY_BASE_DELAY_MS = 250;
+
+/**
+ * TEMPORARY (TRI-10293): a failed create can leave its instance name
+ * registered gateway/fcrun-side until async cleanup runs, so a same-name
+ * retry can 409 against our own residue. Until the gateway cleans up
+ * failed-create registrations properly, retry attempts get a deterministic
+ * suffix. Attempt 1 keeps the unsuffixed name so the non-retry path is
+ * unchanged; the suffixed name flows into both the instance name and
+ * TRIGGER_RUNNER_ID, which downstream flows treat as one opaque
+ * self-reported token. Only attempts following a ComputeClientError are
+ * suffixed - network-failure retries keep the same name on purpose, because
+ * the gateway's name-collision 409 is their safety net against
+ * double-creating an instance whose create response was lost.
+ */
+export function runnerNameForAttempt(runnerId: string, attempt: number): string {
+  return attempt === 1 ? runnerId : `${runnerId}-r${attempt}`;
+}
+
+/**
+ * Whether a failed instance create is worth retrying. Only statuses where
+ * the create definitely did NOT commit are retried: 500 means the agent or
+ * fcrun returned a create error (e.g. a netns slot holding the tap busy, a
+ * full node disk - placement may differ on retry), 503 means the gateway
+ * had nowhere to place it. 502/504 are excluded: the gateway emits those
+ * when it fails to reach the node or read its response, which can happen
+ * AFTER the agent committed the create - and the gateway only records the
+ * instance name on a clean 201, so a same-name retry would miss the
+ * collision check and could double-create the VM on another node. 4xx won't
+ * heal on retry, and timeouts may still be provisioning. Network-level
+ * fetch failures are safe: if the gateway processed the create, its name
+ * index is populated and the retry 409s harmlessly.
+ */
+export function isRetryableCreateError(error: unknown): boolean {
+  if (error instanceof ComputeClientError) {
+    return error.status === 500 || error.status === 503;
+  }
+  if (error instanceof DOMException && error.name === "TimeoutError") {
+    return false;
+  }
+  // Network-level fetch failures (gateway briefly unreachable)
+  return error instanceof TypeError;
+}
+
+type ComputeWorkloadManagerOptions = WorkloadManagerOptions & {
+  gateway: {
+    url: string;
+    authToken?: string;
+    timeoutMs: number;
+  };
+  snapshots: {
+    enabled: boolean;
+    delayMs: number;
+    dispatchLimit: number;
+    callbackUrl: string;
+  };
+  tracing?: OtlpTraceService;
+  runner: {
+    instanceName: string;
+    otelEndpoint: string;
+    prettyLogs: boolean;
+    sendRunDebugLogs: boolean;
+  };
+  createRetry?: {
+    maxAttempts: number;
+    baseDelayMs: number;
+  };
+};
+
+export class ComputeWorkloadManager implements WorkloadManager {
+  private readonly logger = new SimpleStructuredLogger("compute-workload-manager");
+  private readonly compute: ComputeClient;
+  private readonly createMaxAttempts: number;
+  private readonly createRetryBaseDelayMs: number;
+
+  constructor(private opts: ComputeWorkloadManagerOptions) {
+    this.createMaxAttempts = opts.createRetry?.maxAttempts ?? DEFAULT_CREATE_MAX_ATTEMPTS;
+    this.createRetryBaseDelayMs =
+      opts.createRetry?.baseDelayMs ?? DEFAULT_CREATE_RETRY_BASE_DELAY_MS;
+
+    if (opts.workloadApiDomain) {
+      this.logger.warn("⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
+
+    this.compute = new ComputeClient({
+      gatewayUrl: opts.gateway.url,
+      authToken: opts.gateway.authToken,
+      timeoutMs: opts.gateway.timeoutMs,
+      // Forward the current wide-event scope's traceparent + request_id so the
+      // downstream service continues the same trace and joins its own wide
+      // events to ours. Additionally serialize caller-supplied meta labels
+      // into the W3C Baggage header so the downstream service auto-stamps
+      // them even on early-error paths that bail before parsing the body.
+      // When called outside a wide-event scope (or when wide events are
+      // disabled), `fromContext` returns undefined and propagation is skipped.
+      getPropagationHeaders: () => {
+        const state = fromContext();
+        if (!state) return {};
+        const headers: Record<string, string> = { "x-request-id": state.requestId };
+        if (state.traceparent) {
+          headers.traceparent = state.traceparent;
+        }
+        const baggage = encodeBaggage(state.meta);
+        if (baggage) {
+          headers.baggage = baggage;
+        }
+        return headers;
+      },
+    });
+  }
+
+  get snapshotsEnabled(): boolean {
+    return this.opts.snapshots.enabled;
+  }
+
+  get snapshotDelayMs(): number {
+    return this.opts.snapshots.delayMs;
+  }
+
+  get snapshotDispatchLimit(): number {
+    return this.opts.snapshots.dispatchLimit;
+  }
+
+  get traceSpansEnabled(): boolean {
+    return !!this.opts.tracing;
+  }
+
+  async create(opts: WorkloadManagerCreateOptions) {
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
+
+    const envVars: Record<string, string> = {
+      OTEL_EXPORTER_OTLP_ENDPOINT: this.opts.runner.otelEndpoint,
+      TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()),
+      TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()),
+      TRIGGER_ENV_ID: opts.envId,
+      TRIGGER_DEPLOYMENT_ID: opts.deploymentFriendlyId,
+      TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion,
+      TRIGGER_RUN_ID: opts.runFriendlyId,
+      TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId,
+      TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol,
+      TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort),
+      TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "",
+      TRIGGER_WORKER_INSTANCE_NAME: this.opts.runner.instanceName,
+      TRIGGER_RUNNER_ID: runnerId,
+      TRIGGER_MACHINE_CPU: String(opts.machine.cpu),
+      TRIGGER_MACHINE_MEMORY: String(opts.machine.memory),
+      PRETTY_LOGS: String(this.opts.runner.prettyLogs),
+      TRIGGER_SEND_RUN_DEBUG_LOGS: String(this.opts.runner.sendRunDebugLogs),
+    };
+
+    if (this.opts.warmStartUrl) {
+      envVars.TRIGGER_WARM_START_URL = this.opts.warmStartUrl;
+    }
+
+    if (this.snapshotsEnabled && this.opts.metadataUrl) {
+      envVars.TRIGGER_METADATA_URL = this.opts.metadataUrl;
+    }
+
+    if (this.opts.heartbeatIntervalSeconds) {
+      envVars.TRIGGER_HEARTBEAT_INTERVAL_SECONDS = String(this.opts.heartbeatIntervalSeconds);
+    }
+
+    if (this.opts.snapshotPollIntervalSeconds) {
+      envVars.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS = String(
+        this.opts.snapshotPollIntervalSeconds
+      );
+    }
+
+    if (this.opts.additionalEnvVars) {
+      Object.assign(envVars, this.opts.additionalEnvVars);
+    }
+
+    // Strip image digest - resolve by tag, not digest
+    const imageRef = stripImageDigest(opts.image);
+
+    // Labels forwarded to the compute provider for network-policy selection.
+    // `org` is always set so every run carries its org identity.
+    const labels: Record<string, string> = {
+      org: opts.orgId,
+    };
+
+    // Wide event: single canonical log line emitted in finally
+    const event: Record<string, unknown> = {
+      // High-cardinality identifiers
+      runId: opts.runFriendlyId,
+      runnerId,
+      envId: opts.envId,
+      envType: opts.envType,
+      orgId: opts.orgId,
+      projectId: opts.projectId,
+      deploymentVersion: opts.deploymentVersion,
+      machine: opts.machine.name,
+      // Environment
+      instanceName: this.opts.runner.instanceName,
+      // Supervisor timing
+      dequeueResponseMs: opts.dequeueResponseMs,
+      pollingIntervalMs: opts.pollingIntervalMs,
+      warmStartCheckMs: opts.warmStartCheckMs,
+      // Request
+      image: imageRef,
+    };
+
+    const startMs = performance.now();
+
+    try {
+      const createRequest = {
+        name: runnerId,
+        image: imageRef,
+        env: envVars,
+        cpu: opts.machine.cpu,
+        memory_gb: opts.machine.memory,
+        metadata: {
+          runId: opts.runFriendlyId,
+          envId: opts.envId,
+          envType: opts.envType,
+          orgId: opts.orgId,
+          projectId: opts.projectId,
+          deploymentVersion: opts.deploymentVersion,
+          machine: opts.machine.name,
+        },
+        ...(Object.keys(labels).length > 0 ? { labels } : {}),
+      };
+
+      // Retry transient placement failures instead of abandoning the run: a
+      // swallowed create error leaves the run waiting for the run engine's
+      // PENDING_EXECUTING timeout (minutes) before it is redriven, while a
+      // retried create typically succeeds in under a second (TRI-10293).
+      let error: unknown;
+      let data: Awaited<ReturnType<typeof this.compute.instances.create>> | null | undefined;
+      let attempt = 1;
+      // Set after a ComputeClientError: the failed create may have left its
+      // name registered, so subsequent attempts use a suffixed name.
+      let suffixAttempts = false;
+      for (; attempt <= this.createMaxAttempts; attempt++) {
+        const attemptRunnerId = suffixAttempts ? runnerNameForAttempt(runnerId, attempt) : runnerId;
+        [error, data] = await tryCatch(
+          this.compute.instances.create(
+            attemptRunnerId === runnerId
+              ? createRequest
+              : {
+                  ...createRequest,
+                  name: attemptRunnerId,
+                  env: { ...envVars, TRIGGER_RUNNER_ID: attemptRunnerId },
+                }
+          )
+        );
+
+        if (!error) {
+          event.runnerId = attemptRunnerId;
+          break;
+        }
+
+        if (error instanceof ComputeClientError) {
+          suffixAttempts = true;
+        }
+
+        this.logger.warn("create instance attempt failed", {
+          runnerId: attemptRunnerId,
+          attempt,
+          error: error instanceof Error ? error.message : String(error),
+        });
+
+        if (!isRetryableCreateError(error) || attempt === this.createMaxAttempts) break;
+        await sleep(this.createRetryBaseDelayMs * attempt);
+      }
+      event.createAttempts = attempt;
+
+      if (error || !data) {
+        event.error = error instanceof Error ? error.message : String(error);
+        event.errorType =
+          error instanceof DOMException && error.name === "TimeoutError" ? "timeout" : "fetch";
+        // Intentional: errors are captured in the wide event, not thrown. This matches
+        // the Docker/K8s managers. The run will eventually time out if scheduling fails.
+        return;
+      }
+
+      event.instanceId = data.id;
+      event.ok = true;
+
+      // Parse timing data from compute response (optional - requires gateway timing flag)
+      if (data._timing) {
+        event.timing = data._timing;
+      }
+
+      this.#emitProvisionSpan(opts, startMs, data._timing);
+    } finally {
+      event.durationMs = Math.round(performance.now() - startMs);
+      event.ok ??= false;
+      this.logger.debug("create instance", event);
+    }
+  }
+
+  async snapshot(opts: { runnerId: string; metadata: Record<string, string> }): Promise<boolean> {
+    const [error] = await tryCatch(
+      this.compute.instances.snapshot(opts.runnerId, {
+        callback: {
+          url: this.opts.snapshots.callbackUrl,
+          metadata: opts.metadata,
+        },
+      })
+    );
+
+    if (error) {
+      this.logger.error("snapshot request failed", {
+        runnerId: opts.runnerId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return false;
+    }
+
+    this.logger.debug("snapshot request accepted", { runnerId: opts.runnerId });
+    return true;
+  }
+
+  async deleteInstance(runnerId: string): Promise<boolean> {
+    const [error] = await tryCatch(this.compute.instances.delete(runnerId));
+
+    if (error) {
+      this.logger.error("delete instance failed", {
+        runnerId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return false;
+    }
+
+    this.logger.debug("delete instance success", { runnerId });
+    return true;
+  }
+
+  #emitProvisionSpan(opts: WorkloadManagerCreateOptions, startMs: number, timing?: unknown) {
+    if (!this.traceSpansEnabled) return;
+
+    const parsed = parseTraceparent(extractTraceparent(opts.traceContext));
+    if (!parsed) return;
+
+    const endMs = performance.now();
+    const now = Date.now();
+    const provisionStartEpochMs = now - (endMs - startMs);
+    const endEpochMs = now;
+
+    // Span starts at dequeue time so events (dequeue) render in the thin-line section
+    // before "Started". The actual provision call time is in provisionStartEpochMs.
+    // Subtract 1ms so compute span always sorts before the attempt span (same dequeue time)
+    const startEpochMs = opts.dequeuedAt.getTime() - 1;
+
+    const spanAttributes: Record<string, string | number | boolean> = {
+      "compute.type": "create",
+      "compute.provision_start_ms": provisionStartEpochMs,
+      ...(timing
+        ? (flattenAttributes(timing, "compute") as Record<string, string | number | boolean>)
+        : {}),
+    };
+
+    if (opts.dequeueResponseMs !== undefined) {
+      spanAttributes["supervisor.dequeue_response_ms"] = opts.dequeueResponseMs;
+    }
+    if (opts.warmStartCheckMs !== undefined) {
+      spanAttributes["supervisor.warm_start_check_ms"] = opts.warmStartCheckMs;
+    }
+
+    // Use the platform API URL, not the runner OTLP endpoint (which may be a VM gateway IP)
+    this.opts.tracing?.emit({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.provision",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": opts.envId,
+        "ctx.organization.id": opts.orgId,
+        "ctx.project.id": opts.projectId,
+        "ctx.run.id": opts.runFriendlyId,
+      },
+      spanAttributes,
+    });
+  }
+
+  async restore(opts: {
+    snapshotId: string;
+    runnerId: string;
+    runFriendlyId: string;
+    snapshotFriendlyId: string;
+    machine: { cpu: number; memory: number };
+    // Trace context for OTel span emission
+    traceContext?: Record<string, unknown>;
+    envId?: string;
+    orgId?: string;
+    projectId?: string;
+    hasPrivateLink?: boolean;
+    dequeuedAt?: Date;
+  }): Promise<boolean> {
+    const metadata: Record<string, string> = {
+      TRIGGER_RUNNER_ID: opts.runnerId,
+      TRIGGER_RUN_ID: opts.runFriendlyId,
+      TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId,
+      TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol,
+      TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort),
+      TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "",
+      TRIGGER_WORKER_INSTANCE_NAME: this.opts.runner.instanceName,
+    };
+
+    // Resupply labels on restore (the provider doesn't persist them across a
+    // snapshot). orgId is optional on the restore opts type, so guard it.
+    const labels: Record<string, string> = {};
+    if (opts.orgId) {
+      labels.org = opts.orgId;
+    }
+
+    this.logger.verbose("restore request body", {
+      snapshotId: opts.snapshotId,
+      runnerId: opts.runnerId,
+    });
+
+    const startMs = performance.now();
+
+    const [error] = await tryCatch(
+      this.compute.snapshots.restore(opts.snapshotId, {
+        name: opts.runnerId,
+        metadata,
+        cpu: opts.machine.cpu,
+        memory_gb: opts.machine.memory,
+        ...(Object.keys(labels).length > 0 ? { labels } : {}),
+      })
+    );
+
+    const durationMs = Math.round(performance.now() - startMs);
+
+    if (error) {
+      this.logger.error("restore request failed", {
+        snapshotId: opts.snapshotId,
+        runnerId: opts.runnerId,
+        error: error instanceof Error ? error.message : String(error),
+        durationMs,
+      });
+      return false;
+    }
+
+    this.logger.debug("restore request success", {
+      snapshotId: opts.snapshotId,
+      runnerId: opts.runnerId,
+      durationMs,
+    });
+
+    this.#emitRestoreSpan(opts, startMs);
+
+    return true;
+  }
+
+  #emitRestoreSpan(
+    opts: {
+      snapshotId: string;
+      runnerId: string;
+      runFriendlyId: string;
+      traceContext?: Record<string, unknown>;
+      envId?: string;
+      orgId?: string;
+      projectId?: string;
+      dequeuedAt?: Date;
+    },
+    startMs: number
+  ) {
+    if (!this.traceSpansEnabled) return;
+
+    const parsed = parseTraceparent(extractTraceparent(opts.traceContext));
+    if (!parsed || !opts.envId || !opts.orgId || !opts.projectId) return;
+
+    const endMs = performance.now();
+    const now = Date.now();
+    const restoreStartEpochMs = now - (endMs - startMs);
+    const endEpochMs = now;
+
+    // Subtract 1ms so restore span always sorts before the attempt span
+    const startEpochMs = (opts.dequeuedAt?.getTime() ?? restoreStartEpochMs) - 1;
+
+    this.opts.tracing?.emit({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.restore",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": opts.envId,
+        "ctx.organization.id": opts.orgId,
+        "ctx.project.id": opts.projectId,
+        "ctx.run.id": opts.runFriendlyId,
+      },
+      spanAttributes: {
+        "compute.type": "restore",
+        "compute.snapshot_id": opts.snapshotId,
+      },
+    });
+  }
+}

apps/supervisor/src/workloadManager/docker.ts

@@ -0,0 +1,305 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import {
+  type WorkloadManager,
+  type WorkloadManagerCreateOptions,
+  type WorkloadManagerOptions,
+} from "./types.js";
+import { env } from "../env.js";
+import { getDockerHostDomain, getRunnerId, normalizeDockerHostUrl } from "../util.js";
+import Docker from "dockerode";
+import { tryCatch } from "@trigger.dev/core";
+import { ECRAuthService } from "./ecrAuth.js";
+
+export class DockerWorkloadManager implements WorkloadManager {
+  private readonly logger = new SimpleStructuredLogger("docker-workload-manager");
+  private readonly docker: Docker;
+
+  private readonly runnerNetworks: string[];
+  private readonly staticAuth?: Docker.AuthConfig;
+  private readonly platformOverride?: string;
+  private readonly ecrAuthService?: ECRAuthService;
+
+  constructor(private opts: WorkloadManagerOptions) {
+    this.docker = new Docker({
+      version: env.DOCKER_API_VERSION,
+    });
+
+    if (opts.workloadApiDomain) {
+      this.logger.warn("⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
+
+    this.runnerNetworks = env.DOCKER_RUNNER_NETWORKS.split(",");
+
+    this.platformOverride = env.DOCKER_PLATFORM;
+    if (this.platformOverride) {
+      this.logger.info("🖥️  Platform override", {
+        targetPlatform: this.platformOverride,
+        hostPlatform: process.arch,
+      });
+    }
+
+    if (env.DOCKER_REGISTRY_USERNAME && env.DOCKER_REGISTRY_PASSWORD && env.DOCKER_REGISTRY_URL) {
+      this.logger.info("🐋 Using Docker registry credentials", {
+        username: env.DOCKER_REGISTRY_USERNAME,
+        url: env.DOCKER_REGISTRY_URL,
+      });
+
+      this.staticAuth = {
+        username: env.DOCKER_REGISTRY_USERNAME,
+        password: env.DOCKER_REGISTRY_PASSWORD,
+        serveraddress: env.DOCKER_REGISTRY_URL,
+      };
+    } else if (ECRAuthService.hasAWSCredentials()) {
+      this.logger.info("🐋 AWS credentials found, initializing ECR auth service");
+      this.ecrAuthService = new ECRAuthService();
+    } else {
+      this.logger.warn(
+        "🐋 No Docker registry credentials or AWS credentials provided, skipping auth"
+      );
+    }
+  }
+
+  async create(opts: WorkloadManagerCreateOptions) {
+    this.logger.verbose("create()", { opts });
+
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
+
+    // Build environment variables
+    const envVars: string[] = [
+      `OTEL_EXPORTER_OTLP_ENDPOINT=${env.OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`,
+      `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
+      `TRIGGER_ENV_ID=${opts.envId}`,
+      `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`,
+      `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`,
+      `TRIGGER_RUN_ID=${opts.runFriendlyId}`,
+      `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
+      `TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
+      `TRIGGER_SUPERVISOR_API_PORT=${this.opts.workloadApiPort}`,
+      `TRIGGER_SUPERVISOR_API_DOMAIN=${this.opts.workloadApiDomain ?? getDockerHostDomain()}`,
+      `TRIGGER_WORKER_INSTANCE_NAME=${env.TRIGGER_WORKER_INSTANCE_NAME}`,
+      `TRIGGER_RUNNER_ID=${runnerId}`,
+      `TRIGGER_MACHINE_CPU=${opts.machine.cpu}`,
+      `TRIGGER_MACHINE_MEMORY=${opts.machine.memory}`,
+      `PRETTY_LOGS=${env.RUNNER_PRETTY_LOGS}`,
+      `TRIGGER_SEND_RUN_DEBUG_LOGS=${env.SEND_RUN_DEBUG_LOGS}`,
+    ];
+
+    if (this.opts.warmStartUrl) {
+      envVars.push(`TRIGGER_WARM_START_URL=${normalizeDockerHostUrl(this.opts.warmStartUrl)}`);
+    }
+
+    if (this.opts.metadataUrl) {
+      envVars.push(`TRIGGER_METADATA_URL=${this.opts.metadataUrl}`);
+    }
+
+    if (this.opts.heartbeatIntervalSeconds) {
+      envVars.push(`TRIGGER_HEARTBEAT_INTERVAL_SECONDS=${this.opts.heartbeatIntervalSeconds}`);
+    }
+
+    if (this.opts.snapshotPollIntervalSeconds) {
+      envVars.push(
+        `TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS=${this.opts.snapshotPollIntervalSeconds}`
+      );
+    }
+
+    if (this.opts.additionalEnvVars) {
+      Object.entries(this.opts.additionalEnvVars).forEach(([key, value]) => {
+        envVars.push(`${key}=${value}`);
+      });
+    }
+
+    const hostConfig: Docker.HostConfig = {
+      AutoRemove: !!this.opts.dockerAutoremove,
+    };
+
+    const [firstNetwork, ...remainingNetworks] = this.runnerNetworks;
+
+    // Always attach the first network at container creation time. This has the following benefits:
+    // - If there is only a single network to attach, this will prevent having to make a separate request.
+    // - If there are multiple networks to attach, this will ensure the runner won't also be connected to the bridge network
+    hostConfig.NetworkMode = firstNetwork;
+
+    if (env.DOCKER_ENFORCE_MACHINE_PRESETS) {
+      hostConfig.NanoCpus = opts.machine.cpu * 1e9;
+      hostConfig.Memory = opts.machine.memory * 1024 * 1024 * 1024;
+    }
+
+    let imageRef = opts.image;
+
+    if (env.DOCKER_STRIP_IMAGE_DIGEST) {
+      imageRef = opts.image.split("@")[0]!;
+    }
+
+    const containerCreateOpts: Docker.ContainerCreateOptions = {
+      name: runnerId,
+      Hostname: runnerId,
+      HostConfig: hostConfig,
+      Image: imageRef,
+      AttachStdout: false,
+      AttachStderr: false,
+      AttachStdin: false,
+    };
+
+    if (this.platformOverride) {
+      containerCreateOpts.platform = this.platformOverride;
+    }
+
+    const logger = this.logger.child({ opts, containerCreateOpts });
+
+    const [inspectError, inspectResult] = await tryCatch(this.docker.getImage(imageRef).inspect());
+
+    let shouldPull = !!inspectError;
+    if (this.platformOverride) {
+      const imageArchitecture = inspectResult?.Architecture;
+
+      // When the image architecture doesn't match the platform, we need to pull the image
+      if (imageArchitecture && !this.platformOverride.includes(imageArchitecture)) {
+        shouldPull = true;
+      }
+    }
+
+    // If the image is not present, try to pull it
+    if (shouldPull) {
+      logger.info("Pulling image", {
+        error: inspectError,
+        image: opts.image,
+        targetPlatform: this.platformOverride,
+        imageArchitecture: inspectResult?.Architecture,
+      });
+
+      // Get auth config (static or ECR)
+      const authConfig = await this.getAuthConfig();
+
+      // Ensure the image is present
+      const [createImageError, imageResponseReader] = await tryCatch(
+        this.docker.createImage(authConfig, {
+          fromImage: imageRef,
+          ...(this.platformOverride ? { platform: this.platformOverride } : {}),
+        })
+      );
+      if (createImageError) {
+        logger.error("Failed to pull image", { error: createImageError });
+        return;
+      }
+
+      const [imageReadError, imageResponse] = await tryCatch(readAllChunks(imageResponseReader));
+      if (imageReadError) {
+        logger.error("failed to read image response", { error: imageReadError });
+        return;
+      }
+
+      logger.debug("pulled image", { image: opts.image, imageResponse });
+    } else {
+      // Image is present, so we can use it to create the container
+    }
+
+    // Create container
+    const [createContainerError, container] = await tryCatch(
+      this.docker.createContainer({
+        ...containerCreateOpts,
+        // Add env vars here so they're not logged
+        Env: envVars,
+      })
+    );
+
+    if (createContainerError) {
+      logger.error("Failed to create container", { error: createContainerError });
+      return;
+    }
+
+    // If there are multiple networks to attach to we need to attach the remaining ones after creation
+    if (remainingNetworks.length > 0) {
+      await this.attachContainerToNetworks({
+        containerId: container.id,
+        networkNames: remainingNetworks,
+      });
+    }
+
+    // Start container
+    const [startError, startResult] = await tryCatch(container.start());
+
+    if (startError) {
+      logger.error("Failed to start container", { error: startError, containerId: container.id });
+      return;
+    }
+
+    logger.debug("create succeeded", { startResult, containerId: container.id });
+  }
+
+  /**
+   * Get authentication config for Docker operations
+   * Uses static credentials if available, otherwise attempts ECR auth
+   */
+  private async getAuthConfig(): Promise<Docker.AuthConfig | undefined> {
+    // Use static credentials if available
+    if (this.staticAuth) {
+      return this.staticAuth;
+    }
+
+    // Use ECR auth if service is available
+    if (this.ecrAuthService) {
+      const ecrAuth = await this.ecrAuthService.getAuthConfig();
+      return ecrAuth || undefined;
+    }
+
+    // No auth available
+    return undefined;
+  }
+
+  private async attachContainerToNetworks({
+    containerId,
+    networkNames,
+  }: {
+    containerId: string;
+    networkNames: string[];
+  }) {
+    this.logger.debug("Attaching container to networks", { containerId, networkNames });
+
+    const [error, networkResults] = await tryCatch(
+      this.docker.listNetworks({
+        filters: {
+          // Full name matches only to prevent unexpected results
+          name: networkNames.map((name) => `^${name}$`),
+        },
+      })
+    );
+
+    if (error) {
+      this.logger.error("Failed to list networks", { networkNames });
+      return;
+    }
+
+    const results = await Promise.allSettled(
+      networkResults.map((networkInfo) => {
+        const network = this.docker.getNetwork(networkInfo.Id);
+        return network.connect({ Container: containerId });
+      })
+    );
+
+    if (results.some((r) => r.status === "rejected")) {
+      this.logger.error("Failed to attach container to some networks", {
+        containerId,
+        networkNames,
+        results,
+      });
+      return;
+    }
+
+    this.logger.debug("Attached container to networks", {
+      containerId,
+      networkNames,
+      results,
+    });
+  }
+}
+
+async function readAllChunks(reader: NodeJS.ReadableStream) {
+  const chunks = [];
+  for await (const chunk of reader) {
+    chunks.push(chunk.toString());
+  }
+  return chunks;
+}

apps/supervisor/src/workloadManager/ecrAuth.ts

@@ -0,0 +1,144 @@
+import { ECRClient, GetAuthorizationTokenCommand } from "@aws-sdk/client-ecr";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { tryCatch } from "@trigger.dev/core";
+import type Docker from "dockerode";
+
+interface ECRTokenCache {
+  token: string;
+  username: string;
+  serverAddress: string;
+  expiresAt: Date;
+}
+
+export class ECRAuthService {
+  private readonly logger = new SimpleStructuredLogger("ecr-auth-service");
+  private readonly ecrClient: ECRClient;
+  private tokenCache: ECRTokenCache | null = null;
+
+  constructor() {
+    this.ecrClient = new ECRClient();
+
+    this.logger.info("🔐 ECR Auth Service initialized", {
+      region: this.ecrClient.config.region,
+    });
+  }
+
+  /**
+   * Check if we have AWS credentials configured
+   */
+  static hasAWSCredentials(): boolean {
+    if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
+      return true;
+    }
+
+    if (
+      process.env.AWS_PROFILE ||
+      process.env.AWS_ROLE_ARN ||
+      process.env.AWS_WEB_IDENTITY_TOKEN_FILE
+    ) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Check if the current token is still valid with a 10-minute buffer
+   */
+  private isTokenValid(): boolean {
+    if (!this.tokenCache) {
+      return false;
+    }
+
+    const now = new Date();
+    const bufferMs = 10 * 60 * 1000; // 10 minute buffer before expiration
+    return now < new Date(this.tokenCache.expiresAt.getTime() - bufferMs);
+  }
+
+  /**
+   * Get a fresh ECR authorization token from AWS
+   */
+  private async fetchNewToken(): Promise<ECRTokenCache | null> {
+    const [error, response] = await tryCatch(
+      this.ecrClient.send(new GetAuthorizationTokenCommand({}))
+    );
+
+    if (error) {
+      this.logger.error("Failed to get ECR authorization token", { error });
+      return null;
+    }
+
+    const authData = response.authorizationData?.[0];
+    if (!authData?.authorizationToken || !authData.proxyEndpoint) {
+      this.logger.error("Invalid ECR authorization response", { authData });
+      return null;
+    }
+
+    // Decode the base64 token to get username:password
+    const decoded = Buffer.from(authData.authorizationToken, "base64").toString("utf-8");
+    const [username, password] = decoded.split(":", 2);
+
+    if (!username || !password) {
+      this.logger.error("Failed to parse ECR authorization token");
+      return null;
+    }
+
+    const expiresAt = authData.expiresAt || new Date(Date.now() + 12 * 60 * 60 * 1000); // Default 12 hours
+
+    const tokenCache: ECRTokenCache = {
+      token: password,
+      username,
+      serverAddress: authData.proxyEndpoint,
+      expiresAt,
+    };
+
+    this.logger.info("🔐 Successfully fetched ECR token", {
+      username,
+      serverAddress: authData.proxyEndpoint,
+      expiresAt: expiresAt.toISOString(),
+    });
+
+    return tokenCache;
+  }
+
+  /**
+   * Get ECR auth config for Docker operations
+   * Returns cached token if valid, otherwise fetches a new one
+   */
+  async getAuthConfig(): Promise<Docker.AuthConfig | null> {
+    // Check if cached token is still valid
+    if (this.isTokenValid()) {
+      this.logger.debug("Using cached ECR token");
+      return {
+        username: this.tokenCache!.username,
+        password: this.tokenCache!.token,
+        serveraddress: this.tokenCache!.serverAddress,
+      };
+    }
+
+    // Fetch new token
+    this.logger.info("Fetching new ECR authorization token");
+    const newToken = await this.fetchNewToken();
+
+    if (!newToken) {
+      return null;
+    }
+
+    // Cache the new token
+    this.tokenCache = newToken;
+
+    return {
+      username: newToken.username,
+      password: newToken.token,
+      serveraddress: newToken.serverAddress,
+    };
+  }
+
+  /**
+   * Clear the cached token (useful for testing or forcing refresh)
+   */
+  clearCache(): void {
+    this.tokenCache = null;
+    this.logger.debug("ECR token cache cleared");
+  }
+}

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -0,0 +1,582 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import {
+  type WorkloadManager,
+  type WorkloadManagerCreateOptions,
+  type WorkloadManagerOptions,
+} from "./types.js";
+import type {
+  EnvironmentType,
+  MachinePreset,
+  MachinePresetName,
+  PlacementTag,
+} from "@trigger.dev/core/v3";
+import { PlacementTagProcessor } from "@trigger.dev/core/v3/serverOnly";
+import { env } from "../env.js";
+import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js";
+import { getRunnerId } from "../util.js";
+
+type ResourceQuantities = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
+const cpuRequestRatioByMachinePreset: Record<MachinePresetName, number | undefined> = {
+  micro: env.KUBERNETES_CPU_REQUEST_RATIO_MICRO,
+  "small-1x": env.KUBERNETES_CPU_REQUEST_RATIO_SMALL_1X,
+  "small-2x": env.KUBERNETES_CPU_REQUEST_RATIO_SMALL_2X,
+  "medium-1x": env.KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_1X,
+  "medium-2x": env.KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_2X,
+  "large-1x": env.KUBERNETES_CPU_REQUEST_RATIO_LARGE_1X,
+  "large-2x": env.KUBERNETES_CPU_REQUEST_RATIO_LARGE_2X,
+};
+
+const memoryRequestRatioByMachinePreset: Record<MachinePresetName, number | undefined> = {
+  micro: env.KUBERNETES_MEMORY_REQUEST_RATIO_MICRO,
+  "small-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_1X,
+  "small-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_2X,
+  "medium-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_1X,
+  "medium-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_2X,
+  "large-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_1X,
+  "large-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_2X,
+};
+
+export class KubernetesWorkloadManager implements WorkloadManager {
+  private readonly logger = new SimpleStructuredLogger("kubernetes-workload-provider");
+  private k8s: K8sApi;
+  private namespace = env.KUBERNETES_NAMESPACE;
+  private placementTagProcessor: PlacementTagProcessor;
+
+  // Resource settings
+  private readonly cpuRequestMinCores = env.KUBERNETES_CPU_REQUEST_MIN_CORES;
+  private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO;
+  private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB;
+  private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO;
+  private readonly memoryOverheadGb = env.KUBERNETES_MEMORY_OVERHEAD_GB;
+
+  constructor(private opts: WorkloadManagerOptions) {
+    this.k8s = createK8sApi();
+    this.placementTagProcessor = new PlacementTagProcessor({
+      enabled: env.PLACEMENT_TAGS_ENABLED,
+      prefix: env.PLACEMENT_TAGS_PREFIX,
+    });
+
+    if (opts.workloadApiDomain) {
+      this.logger.warn("[KubernetesWorkloadManager] ⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
+  }
+
+  private addPlacementTags(
+    podSpec: Omit<k8s.V1PodSpec, "containers">,
+    placementTags?: PlacementTag[]
+  ): Omit<k8s.V1PodSpec, "containers"> {
+    const nodeSelector = this.placementTagProcessor.convertToNodeSelector(
+      placementTags,
+      podSpec.nodeSelector
+    );
+
+    return {
+      ...podSpec,
+      nodeSelector,
+    };
+  }
+
+  private stripImageDigest(imageRef: string): string {
+    if (!env.KUBERNETES_STRIP_IMAGE_DIGEST) {
+      return imageRef;
+    }
+
+    const atIndex = imageRef.lastIndexOf("@");
+
+    if (atIndex === -1) {
+      return imageRef;
+    }
+
+    return imageRef.substring(0, atIndex);
+  }
+
+  private clamp(value: number, min: number, max: number): number {
+    return Math.min(Math.max(value, min), max);
+  }
+
+  async create(opts: WorkloadManagerCreateOptions) {
+    this.logger.verbose("[KubernetesWorkloadManager] Creating container", { opts });
+
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
+
+    try {
+      await this.k8s.core.createNamespacedPod({
+        namespace: this.namespace,
+        body: {
+          metadata: {
+            name: runnerId,
+            namespace: this.namespace,
+            labels: {
+              ...this.#getSharedLabels(opts),
+              app: "task-run",
+              "app.kubernetes.io/part-of": "trigger-worker",
+              "app.kubernetes.io/component": "create",
+            },
+          },
+          spec: {
+            ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags),
+            affinity: this.#getAffinity(opts),
+            tolerations: this.#getScheduleTolerations(this.#isScheduledRun(opts)),
+            terminationGracePeriodSeconds: 60 * 60,
+            containers: [
+              {
+                name: "run-controller",
+                image: this.stripImageDigest(opts.image),
+                ports: [
+                  {
+                    containerPort: 8000,
+                  },
+                ],
+                resources: this.#getResourcesForMachine(opts.machine),
+                env: [
+                  {
+                    name: "TRIGGER_DEQUEUED_AT_MS",
+                    value: opts.dequeuedAt.getTime().toString(),
+                  },
+                  {
+                    name: "TRIGGER_POD_SCHEDULED_AT_MS",
+                    value: Date.now().toString(),
+                  },
+                  {
+                    name: "TRIGGER_RUN_ID",
+                    value: opts.runFriendlyId,
+                  },
+                  {
+                    name: "TRIGGER_ENV_ID",
+                    value: opts.envId,
+                  },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_ID",
+                    value: opts.deploymentFriendlyId,
+                  },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_VERSION",
+                    value: opts.deploymentVersion,
+                  },
+                  {
+                    name: "TRIGGER_SNAPSHOT_ID",
+                    value: opts.snapshotFriendlyId,
+                  },
+                  {
+                    name: "TRIGGER_SUPERVISOR_API_PROTOCOL",
+                    value: this.opts.workloadApiProtocol,
+                  },
+                  {
+                    name: "TRIGGER_SUPERVISOR_API_PORT",
+                    value: `${this.opts.workloadApiPort}`,
+                  },
+                  {
+                    name: "TRIGGER_SUPERVISOR_API_DOMAIN",
+                    ...(this.opts.workloadApiDomain
+                      ? {
+                          value: this.opts.workloadApiDomain,
+                        }
+                      : {
+                          valueFrom: {
+                            fieldRef: {
+                              fieldPath: "status.hostIP",
+                            },
+                          },
+                        }),
+                  },
+                  {
+                    name: "TRIGGER_WORKER_INSTANCE_NAME",
+                    valueFrom: {
+                      fieldRef: {
+                        fieldPath: "spec.nodeName",
+                      },
+                    },
+                  },
+                  {
+                    name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+                    value: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+                  },
+                  {
+                    name: "TRIGGER_RUNNER_ID",
+                    value: runnerId,
+                  },
+                  {
+                    name: "TRIGGER_MACHINE_CPU",
+                    value: `${opts.machine.cpu}`,
+                  },
+                  {
+                    name: "TRIGGER_MACHINE_MEMORY",
+                    value: `${opts.machine.memory}`,
+                  },
+                  {
+                    name: "TRIGGER_SEND_RUN_DEBUG_LOGS",
+                    value: `${env.SEND_RUN_DEBUG_LOGS}`,
+                  },
+                  {
+                    name: "LIMITS_CPU",
+                    valueFrom: {
+                      resourceFieldRef: {
+                        resource: "limits.cpu",
+                      },
+                    },
+                  },
+                  {
+                    name: "LIMITS_MEMORY",
+                    valueFrom: {
+                      resourceFieldRef: {
+                        resource: "limits.memory",
+                      },
+                    },
+                  },
+                  ...(this.opts.warmStartUrl
+                    ? [{ name: "TRIGGER_WARM_START_URL", value: this.opts.warmStartUrl }]
+                    : []),
+                  ...(this.opts.metadataUrl
+                    ? [{ name: "TRIGGER_METADATA_URL", value: this.opts.metadataUrl }]
+                    : []),
+                  ...(this.opts.heartbeatIntervalSeconds
+                    ? [
+                        {
+                          name: "TRIGGER_HEARTBEAT_INTERVAL_SECONDS",
+                          value: `${this.opts.heartbeatIntervalSeconds}`,
+                        },
+                      ]
+                    : []),
+                  ...(this.opts.snapshotPollIntervalSeconds
+                    ? [
+                        {
+                          name: "TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS",
+                          value: `${this.opts.snapshotPollIntervalSeconds}`,
+                        },
+                      ]
+                    : []),
+                  ...(this.opts.additionalEnvVars
+                    ? Object.entries(this.opts.additionalEnvVars).map(([key, value]) => ({
+                        name: key,
+                        value: value,
+                      }))
+                    : []),
+                ],
+              },
+            ],
+          },
+        },
+      });
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      throw candidate;
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    this.#throwUnlessRecord(err);
+
+    if ("body" in err && err.body) {
+      this.logger.error("[KubernetesWorkloadManager] Create failed", { rawError: err.body });
+      this.#throwUnlessRecord(err.body);
+
+      if (typeof err.body.message === "string") {
+        throw new Error(err.body?.message);
+      } else {
+        throw err.body;
+      }
+    } else {
+      this.logger.error("[KubernetesWorkloadManager] Create failed", { rawError: err });
+      throw err;
+    }
+  }
+
+  #envTypeToLabelValue(type: EnvironmentType) {
+    switch (type) {
+      case "PRODUCTION":
+        return "prod";
+      case "STAGING":
+        return "stg";
+      case "DEVELOPMENT":
+        return "dev";
+      case "PREVIEW":
+        return "preview";
+    }
+  }
+
+  private getImagePullSecrets(): k8s.V1LocalObjectReference[] | undefined {
+    return this.opts.imagePullSecrets?.map((name) => ({ name }));
+  }
+
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: this.getImagePullSecrets(),
+      ...(env.KUBERNETES_SCHEDULER_NAME
+        ? {
+            schedulerName: env.KUBERNETES_SCHEDULER_NAME,
+          }
+        : {}),
+      ...(env.KUBERNETES_WORKER_NODETYPE_LABEL
+        ? {
+            nodeSelector: {
+              nodetype: env.KUBERNETES_WORKER_NODETYPE_LABEL,
+            },
+          }
+        : {}),
+      ...(env.KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED
+        ? {
+            dnsConfig: {
+              options: [{ name: "ndots", value: `${env.KUBERNETES_POD_DNS_NDOTS}` }],
+            },
+          }
+        : {}),
+    };
+  }
+
+  get #defaultResourceRequests(): ResourceQuantities {
+    return {
+      "ephemeral-storage": env.KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST,
+    };
+  }
+
+  get #defaultResourceLimits(): ResourceQuantities {
+    return {
+      "ephemeral-storage": env.KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT,
+    };
+  }
+
+  #isScheduledRun(opts: WorkloadManagerCreateOptions): boolean {
+    return opts.annotations?.rootTriggerSource === "schedule";
+  }
+
+  #getSharedLabels(opts: WorkloadManagerCreateOptions): Record<string, string> {
+    const labels: Record<string, string> = {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+      machine: opts.machine.name,
+      // We intentionally use a boolean label rather than exposing the full trigger source
+      // (e.g. sdk, api, cli, mcp, schedule) to keep label cardinality low in metrics.
+      // The schedule vs non-schedule distinction is all we need for the current metrics
+      // and pool-level scheduling decisions; finer-grained source breakdowns live in run annotations.
+      scheduled: String(this.#isScheduledRun(opts)),
+    };
+
+    // Add privatelink label for CiliumNetworkPolicy matching
+    if (opts.hasPrivateLink) {
+      labels.privatelink = opts.orgId;
+    }
+
+    return labels;
+  }
+
+  #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    const cpuRatio = cpuRequestRatioByMachinePreset[preset.name] ?? this.cpuRequestRatio;
+    const memoryRatio = memoryRequestRatioByMachinePreset[preset.name] ?? this.memoryRequestRatio;
+
+    const cpuRequest = preset.cpu * cpuRatio;
+    const memoryRequest = preset.memory * memoryRatio;
+
+    // Clamp between min and max
+    const clampedCpu = this.clamp(cpuRequest, this.cpuRequestMinCores, preset.cpu);
+    const clampedMemory = this.clamp(memoryRequest, this.memoryRequestMinGb, preset.memory);
+
+    return {
+      cpu: `${clampedCpu}`,
+      memory: `${clampedMemory}G`,
+    };
+  }
+
+  #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    const memoryLimit = this.memoryOverheadGb
+      ? preset.memory + this.memoryOverheadGb
+      : preset.memory;
+
+    return {
+      cpu: `${preset.cpu}`,
+      memory: `${memoryLimit}G`,
+    };
+  }
+
+  #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
+    return {
+      requests: {
+        ...this.#defaultResourceRequests,
+        ...this.#getResourceRequestsForMachine(preset),
+      },
+      limits: {
+        ...this.#defaultResourceLimits,
+        ...this.#getResourceLimitsForMachine(preset),
+      },
+    };
+  }
+
+  #isLargeMachine(preset: MachinePreset): boolean {
+    return preset.name.startsWith("large-");
+  }
+
+  #getAffinity(opts: WorkloadManagerCreateOptions): k8s.V1Affinity | undefined {
+    const largeNodeAffinity = this.#getNodeAffinityRules(opts.machine);
+    const scheduleNodeAffinity = this.#getScheduleNodeAffinityRules(this.#isScheduledRun(opts));
+    const podAffinity = this.#getProjectPodAffinity(opts.projectId);
+
+    // Merge node affinity rules from multiple sources
+    const preferred = [
+      ...(largeNodeAffinity?.preferredDuringSchedulingIgnoredDuringExecution ?? []),
+      ...(scheduleNodeAffinity?.preferredDuringSchedulingIgnoredDuringExecution ?? []),
+    ];
+    // Only large machine affinity produces hard requirements (non-large runs must stay off the large pool).
+    // Schedule affinity is soft both ways.
+    const required = [
+      ...(largeNodeAffinity?.requiredDuringSchedulingIgnoredDuringExecution?.nodeSelectorTerms ??
+        []),
+    ];
+
+    const hasNodeAffinity = preferred.length > 0 || required.length > 0;
+
+    if (!hasNodeAffinity && !podAffinity) {
+      return undefined;
+    }
+
+    return {
+      ...(hasNodeAffinity && {
+        nodeAffinity: {
+          ...(preferred.length > 0 && {
+            preferredDuringSchedulingIgnoredDuringExecution: preferred,
+          }),
+          ...(required.length > 0 && {
+            requiredDuringSchedulingIgnoredDuringExecution: { nodeSelectorTerms: required },
+          }),
+        },
+      }),
+      ...(podAffinity && { podAffinity }),
+    };
+  }
+
+  #getNodeAffinityRules(preset: MachinePreset): k8s.V1NodeAffinity | undefined {
+    if (!env.KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED) {
+      return undefined;
+    }
+
+    if (this.#isLargeMachine(preset)) {
+      // soft preference for the large-machine pool, falls back to standard if unavailable
+      return {
+        preferredDuringSchedulingIgnoredDuringExecution: [
+          {
+            weight: env.KUBERNETES_LARGE_MACHINE_AFFINITY_WEIGHT,
+            preference: {
+              matchExpressions: [
+                {
+                  key: env.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY,
+                  operator: "In",
+                  values: [env.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_VALUE],
+                },
+              ],
+            },
+          },
+        ],
+      };
+    }
+
+    // not schedulable in the large-machine pool
+    return {
+      requiredDuringSchedulingIgnoredDuringExecution: {
+        nodeSelectorTerms: [
+          {
+            matchExpressions: [
+              {
+                key: env.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY,
+                operator: "NotIn",
+                values: [env.KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_VALUE],
+              },
+            ],
+          },
+        ],
+      },
+    };
+  }
+
+  #getScheduleNodeAffinityRules(isScheduledRun: boolean): k8s.V1NodeAffinity | undefined {
+    if (
+      !env.KUBERNETES_SCHEDULED_RUN_AFFINITY_ENABLED ||
+      !env.KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_VALUE
+    ) {
+      return undefined;
+    }
+
+    if (isScheduledRun) {
+      // soft preference for the schedule pool
+      return {
+        preferredDuringSchedulingIgnoredDuringExecution: [
+          {
+            weight: env.KUBERNETES_SCHEDULED_RUN_AFFINITY_WEIGHT,
+            preference: {
+              matchExpressions: [
+                {
+                  key: env.KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_KEY,
+                  operator: "In",
+                  values: [env.KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_VALUE],
+                },
+              ],
+            },
+          },
+        ],
+      };
+    }
+
+    // soft anti-affinity: non-schedule runs prefer to avoid the schedule pool
+    return {
+      preferredDuringSchedulingIgnoredDuringExecution: [
+        {
+          weight: env.KUBERNETES_SCHEDULED_RUN_ANTI_AFFINITY_WEIGHT,
+          preference: {
+            matchExpressions: [
+              {
+                key: env.KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_KEY,
+                operator: "NotIn",
+                values: [env.KUBERNETES_SCHEDULED_RUN_AFFINITY_POOL_LABEL_VALUE],
+              },
+            ],
+          },
+        },
+      ],
+    };
+  }
+
+  #getScheduleTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined {
+    if (!isScheduledRun || !env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS?.length) {
+      return undefined;
+    }
+
+    return env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS;
+  }
+
+  #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined {
+    if (!env.KUBERNETES_PROJECT_AFFINITY_ENABLED) {
+      return undefined;
+    }
+
+    return {
+      preferredDuringSchedulingIgnoredDuringExecution: [
+        {
+          weight: env.KUBERNETES_PROJECT_AFFINITY_WEIGHT,
+          podAffinityTerm: {
+            labelSelector: {
+              matchExpressions: [
+                {
+                  key: "project",
+                  operator: "In",
+                  values: [projectId],
+                },
+              ],
+            },
+            topologyKey: env.KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY,
+          },
+        },
+      ],
+    };
+  }
+}

apps/supervisor/src/workloadManager/types.ts

@@ -0,0 +1,52 @@
+import type {
+  EnvironmentType,
+  MachinePreset,
+  PlacementTag,
+  RunAnnotations,
+} from "@trigger.dev/core/v3";
+
+export interface WorkloadManagerOptions {
+  workloadApiProtocol: "http" | "https";
+  workloadApiDomain?: string; // If unset, will use orchestrator-specific default
+  workloadApiPort: number;
+  warmStartUrl?: string;
+  metadataUrl?: string;
+  imagePullSecrets?: string[];
+  heartbeatIntervalSeconds?: number;
+  snapshotPollIntervalSeconds?: number;
+  additionalEnvVars?: Record<string, string>;
+  dockerAutoremove?: boolean;
+}
+
+export interface WorkloadManager {
+  create: (opts: WorkloadManagerCreateOptions) => Promise<unknown>;
+}
+
+export interface WorkloadManagerCreateOptions {
+  image: string;
+  machine: MachinePreset;
+  version: string;
+  nextAttemptNumber?: number;
+  dequeuedAt: Date;
+  placementTags?: PlacementTag[];
+  // Timing context (populated by supervisor handler, included in wide event)
+  dequeueResponseMs?: number;
+  pollingIntervalMs?: number;
+  warmStartCheckMs?: number;
+  // identifiers
+  envId: string;
+  envType: EnvironmentType;
+  orgId: string;
+  projectId: string;
+  deploymentFriendlyId: string;
+  deploymentVersion: string;
+  runId: string;
+  runFriendlyId: string;
+  snapshotId: string;
+  snapshotFriendlyId: string;
+  // Trace context for OTel span emission (W3C format: { traceparent: "00-...", tracestate?: "..." })
+  traceContext?: Record<string, unknown>;
+  annotations?: RunAnnotations;
+  // private networking
+  hasPrivateLink?: boolean;
+}

apps/supervisor/src/workloadServer/index.ts

@@ -0,0 +1,883 @@
+import { SnapshotCallbackPayloadSchema } from "@internal/compute";
+import { type CheckpointClient, HttpServer } from "@trigger.dev/core/v3/serverOnly";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import {
+  type WorkloadRunSnapshotsSinceResponseBody,
+  type SupervisorHttpClient,
+  WORKLOAD_HEADERS,
+  type WorkloadClientSocketData,
+  type WorkloadClientToServerEvents,
+  type WorkloadContinueRunExecutionResponseBody,
+  WorkloadDebugLogRequestBody,
+  type WorkloadDequeueFromVersionResponseBody,
+  WorkloadHeartbeatRequestBody,
+  type WorkloadHeartbeatResponseBody,
+  WorkloadRunAttemptCompleteRequestBody,
+  type WorkloadRunAttemptCompleteResponseBody,
+  WorkloadRunAttemptStartRequestBody,
+  type WorkloadRunAttemptStartResponseBody,
+  type WorkloadServerToClientEvents,
+  type WorkloadSuspendRunResponseBody,
+} from "@trigger.dev/core/v3/workers";
+import EventEmitter from "node:events";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { type Namespace, Server, type Socket } from "socket.io";
+import { z } from "zod";
+import { env } from "../env.js";
+import { register } from "../metrics.js";
+import {
+  ComputeSnapshotService,
+  type RunTraceContext,
+} from "../services/computeSnapshotService.js";
+import type { OtlpTraceService } from "../services/otlpTraceService.js";
+import {
+  emitOneShot,
+  runWideEvent,
+  setMeta,
+  type State,
+  type WideEventOptions,
+} from "../wideEvents/index.js";
+import type { ComputeWorkloadManager } from "../workloadManager/compute.js";
+
+// Use the official export when upgrading to socket.io@4.8.0
+interface DefaultEventsMap {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [event: string]: (...args: any[]) => void;
+}
+
+const WorkloadActionParams = z.object({
+  runFriendlyId: z.string(),
+  snapshotFriendlyId: z.string(),
+});
+
+// Workloads bundled into customer task images before CLI v4.4.4 use a strict
+// zod enum for checkpoint type that only allows DOCKER and KUBERNETES. The
+// workload never reads this field - it only validates the response shape - so
+// rewriting it to a known value keeps older runners working without affecting
+// the value stored in the database or seen by internal services.
+function legacifyCheckpointType<T extends { checkpoint?: { type: string } | null }>(item: T): T {
+  if (item.checkpoint?.type === "COMPUTE") {
+    return { ...item, checkpoint: { ...item.checkpoint, type: "KUBERNETES" } } as T;
+  }
+  return item;
+}
+
+type WorkloadServerEvents = {
+  runConnected: [
+    {
+      run: {
+        friendlyId: string;
+      };
+    },
+  ];
+  runDisconnected: [
+    {
+      run: {
+        friendlyId: string;
+      };
+    },
+  ];
+};
+
+type WorkloadServerOptions = {
+  port: number;
+  host?: string;
+  workerClient: SupervisorHttpClient;
+  checkpointClient?: CheckpointClient;
+  computeManager?: ComputeWorkloadManager;
+  tracing?: OtlpTraceService;
+  wideEventOpts: WideEventOptions;
+  /** When true, high-frequency HTTP routes also emit wide events. */
+  wideEventsNoisyRoutes: boolean;
+};
+
+export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
+  private checkpointClient?: CheckpointClient;
+  private readonly snapshotService?: ComputeSnapshotService;
+
+  private readonly logger = new SimpleStructuredLogger("workload-server");
+  private readonly wideEventOpts: WideEventOptions;
+  private readonly wideEventsNoisyRoutes: boolean;
+
+  private readonly httpServer: HttpServer;
+  private readonly websocketServer: Namespace<
+    WorkloadClientToServerEvents,
+    WorkloadServerToClientEvents,
+    DefaultEventsMap,
+    WorkloadClientSocketData
+  >;
+
+  private readonly runSockets = new Map<
+    string,
+    Socket<
+      WorkloadClientToServerEvents,
+      WorkloadServerToClientEvents,
+      DefaultEventsMap,
+      WorkloadClientSocketData
+    >
+  >();
+
+  private readonly workerClient: SupervisorHttpClient;
+
+  constructor(opts: WorkloadServerOptions) {
+    super();
+
+    const host = opts.host ?? "0.0.0.0";
+    const port = opts.port;
+
+    this.workerClient = opts.workerClient;
+    this.checkpointClient = opts.checkpointClient;
+    this.wideEventOpts = opts.wideEventOpts;
+    this.wideEventsNoisyRoutes = opts.wideEventsNoisyRoutes;
+
+    if (opts.computeManager?.snapshotsEnabled) {
+      this.snapshotService = new ComputeSnapshotService({
+        computeManager: opts.computeManager,
+        workerClient: opts.workerClient,
+        tracing: opts.tracing,
+        wideEventOpts: this.wideEventOpts,
+      });
+    }
+
+    this.httpServer = this.createHttpServer({ host, port });
+    this.websocketServer = this.createWebsocketServer();
+  }
+
+  private headerValueFromRequest(req: IncomingMessage, headerName: string): string | undefined {
+    const value = req.headers[headerName];
+
+    if (Array.isArray(value)) {
+      return value[0];
+    }
+
+    return value;
+  }
+
+  private runnerIdFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.RUNNER_ID);
+  }
+
+  private deploymentIdFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.DEPLOYMENT_ID);
+  }
+
+  private deploymentVersionFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.DEPLOYMENT_VERSION);
+  }
+
+  private projectRefFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.PROJECT_REF);
+  }
+
+  /**
+   * Sets common route meta on the wide-event state from URL params.
+   */
+  private attachRouteMeta(state: State, params: unknown): void {
+    if (!params || typeof params !== "object") return;
+    const p = params as Record<string, unknown>;
+    if (typeof p.runFriendlyId === "string") setMeta(state, "run_id", p.runFriendlyId);
+    if (typeof p.snapshotFriendlyId === "string") {
+      setMeta(state, "snapshot_id", p.snapshotFriendlyId);
+    }
+    if (typeof p.deploymentId === "string") setMeta(state, "deployment_id", p.deploymentId);
+  }
+
+  /**
+   * Wraps an HTTP route handler body with the wide-event lifecycle. Reads
+   * `traceparent` and `x-request-id` from `req.headers`, attaches `run_id` /
+   * `snapshot_id` / `deployment_id` meta from `params` when present, and
+   * captures the response status from `res.statusCode` after `fn` returns.
+   *
+   * Pass `highFrequency: true` for noisy routes (heartbeat, polling). Those
+   * still go through the wrapper but only emit when
+   * `TRIGGER_WIDE_EVENTS_NOISY_ROUTES` is on, so prod can keep them dark
+   * while test envs capture full-fidelity traffic for debugging.
+   */
+  private wideRoute<T>(
+    ctx: { req: IncomingMessage; res: ServerResponse; params?: unknown },
+    op: string,
+    route: string,
+    method: string,
+    fn: () => Promise<T> | T,
+    routeOpts: { highFrequency?: boolean } = {}
+  ): Promise<T> {
+    const enabled =
+      this.wideEventOpts.enabled && (!routeOpts.highFrequency || this.wideEventsNoisyRoutes);
+    return runWideEvent(
+      {
+        ...this.wideEventOpts,
+        enabled,
+        op,
+        kind: "inbound",
+        route,
+        method,
+        traceparent: this.headerValueFromRequest(ctx.req, "traceparent"),
+        inboundRequestId: this.headerValueFromRequest(ctx.req, "x-request-id"),
+        setup: (state) => this.attachRouteMeta(state, ctx.params),
+      },
+      fn,
+      (state) => {
+        state.statusCode = ctx.res.statusCode;
+      }
+    );
+  }
+
+  private createHttpServer({ host, port }: { host: string; port: number }) {
+    const httpServer = new HttpServer({
+      port,
+      host,
+      metrics: {
+        register,
+        expose: false,
+      },
+    })
+      .route("/health", "GET", {
+        handler: async ({ reply }) => {
+          reply.text("OK");
+        },
+      })
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/attempts/start",
+        "POST",
+        {
+          paramsSchema: WorkloadActionParams,
+          bodySchema: WorkloadRunAttemptStartRequestBody,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "attempt.start",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/attempts/start",
+              "POST",
+              async () => {
+                const { req, reply, params, body } = ctx;
+                const startResponse = await this.workerClient.startRunAttempt(
+                  params.runFriendlyId,
+                  params.snapshotFriendlyId,
+                  body,
+                  this.runnerIdFromRequest(req)
+                );
+
+                if (!startResponse.success) {
+                  this.logger.error("Failed to start run", {
+                    params,
+                    error: startResponse.error,
+                  });
+                  reply.empty(500);
+                  return;
+                }
+
+                reply.json(startResponse.data satisfies WorkloadRunAttemptStartResponseBody);
+                return;
+              }
+            ),
+        }
+      )
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/attempts/complete",
+        "POST",
+        {
+          paramsSchema: WorkloadActionParams,
+          bodySchema: WorkloadRunAttemptCompleteRequestBody,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "attempt.complete",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/attempts/complete",
+              "POST",
+              async () => {
+                const { req, reply, params, body } = ctx;
+                const runnerId = this.runnerIdFromRequest(req);
+
+                // A completion attempt invalidates any pending delayed snapshot
+                // regardless of outcome: the runner has finished executing, so the
+                // suspended state the snapshot was scheduled to capture no longer
+                // exists. Cancel BEFORE the async completion call - the timer
+                // wheel can tick during the await, so cancelling after it leaves
+                // a real window for a due snapshot to dispatch and pause a VM
+                // that has moved on. The runnerId guard keeps a stale duplicate
+                // runner's completion from cancelling a fresh runner's snapshot,
+                // and the runner can't schedule a new suspend until it receives
+                // this route's reply, so nothing legitimate can be cancelled here.
+                this.snapshotService?.cancel(params.runFriendlyId, runnerId);
+
+                const completeResponse = await this.workerClient.completeRunAttempt(
+                  params.runFriendlyId,
+                  params.snapshotFriendlyId,
+                  body,
+                  runnerId
+                );
+
+                if (!completeResponse.success) {
+                  this.logger.error("Failed to complete run", {
+                    params,
+                    error: completeResponse.error,
+                  });
+                  reply.empty(500);
+                  return;
+                }
+
+                reply.json(completeResponse.data satisfies WorkloadRunAttemptCompleteResponseBody);
+                return;
+              }
+            ),
+        }
+      )
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/heartbeat",
+        "POST",
+        {
+          paramsSchema: WorkloadActionParams,
+          bodySchema: WorkloadHeartbeatRequestBody,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "heartbeat",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/heartbeat",
+              "POST",
+              async () => {
+                const { req, reply, params, body } = ctx;
+                const heartbeatResponse = await this.workerClient.heartbeatRun(
+                  params.runFriendlyId,
+                  params.snapshotFriendlyId,
+                  body,
+                  this.runnerIdFromRequest(req)
+                );
+
+                if (!heartbeatResponse.success) {
+                  this.logger.error("Failed to heartbeat run", {
+                    params,
+                    error: heartbeatResponse.error,
+                  });
+                  reply.empty(500);
+                  return;
+                }
+
+                reply.json({
+                  ok: true,
+                } satisfies WorkloadHeartbeatResponseBody);
+              },
+              { highFrequency: true }
+            ),
+        }
+      )
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/suspend",
+        "GET",
+        {
+          paramsSchema: WorkloadActionParams,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "suspend",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/suspend",
+              "GET",
+              async () => {
+                const { reply, params, req } = ctx;
+                const runnerId = this.runnerIdFromRequest(req);
+                const deploymentVersion = this.deploymentVersionFromRequest(req);
+                const projectRef = this.projectRefFromRequest(req);
+
+                this.logger.debug("Suspend request", {
+                  params,
+                  runnerId,
+                  deploymentVersion,
+                  projectRef,
+                });
+
+                if (!runnerId || !deploymentVersion || !projectRef) {
+                  this.logger.error("Invalid headers for suspend request", {
+                    ...params,
+                    runnerId,
+                    deploymentVersion,
+                    projectRef,
+                  });
+                  reply.json(
+                    {
+                      ok: false,
+                      error: "Invalid headers",
+                    } satisfies WorkloadSuspendRunResponseBody,
+                    false,
+                    400
+                  );
+                  return;
+                }
+
+                if (this.snapshotService) {
+                  // Compute mode: delay snapshot to avoid wasted work on short-lived waitpoints.
+                  // If the run continues before the delay expires, the snapshot is cancelled.
+                  reply.json({ ok: true } satisfies WorkloadSuspendRunResponseBody, false, 202);
+
+                  this.snapshotService.schedule(params.runFriendlyId, {
+                    runnerId,
+                    runFriendlyId: params.runFriendlyId,
+                    snapshotFriendlyId: params.snapshotFriendlyId,
+                  });
+
+                  return;
+                }
+
+                if (!this.checkpointClient) {
+                  reply.json(
+                    {
+                      ok: false,
+                      error: "Checkpoints disabled",
+                    } satisfies WorkloadSuspendRunResponseBody,
+                    false,
+                    400
+                  );
+                  return;
+                }
+
+                reply.json(
+                  {
+                    ok: true,
+                  } satisfies WorkloadSuspendRunResponseBody,
+                  false,
+                  202
+                );
+
+                const suspendResult = await this.checkpointClient.suspendRun({
+                  runFriendlyId: params.runFriendlyId,
+                  snapshotFriendlyId: params.snapshotFriendlyId,
+                  body: {
+                    runnerId,
+                    runId: params.runFriendlyId,
+                    snapshotId: params.snapshotFriendlyId,
+                    projectRef,
+                    deploymentVersion,
+                  },
+                });
+
+                if (!suspendResult) {
+                  this.logger.error("Failed to suspend run", { params });
+                  return;
+                }
+              }
+            ),
+        }
+      )
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/continue",
+        "GET",
+        {
+          paramsSchema: WorkloadActionParams,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "continue",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/continue",
+              "GET",
+              async () => {
+                const { req, reply, params } = ctx;
+                this.logger.debug("Run continuation request", { params });
+
+                // Cancel any pending delayed snapshot for this run
+                this.snapshotService?.cancel(params.runFriendlyId);
+
+                const continuationResult = await this.workerClient.continueRunExecution(
+                  params.runFriendlyId,
+                  params.snapshotFriendlyId,
+                  this.runnerIdFromRequest(req)
+                );
+
+                if (!continuationResult.success) {
+                  this.logger.error("Failed to continue run execution", { params });
+                  reply.json(
+                    {
+                      ok: false,
+                      error: "Failed to continue run execution",
+                    },
+                    false,
+                    400
+                  );
+                  return;
+                }
+
+                reply.json(continuationResult.data as WorkloadContinueRunExecutionResponseBody);
+              }
+            ),
+        }
+      )
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/since/:snapshotFriendlyId",
+        "GET",
+        {
+          paramsSchema: WorkloadActionParams,
+          handler: async (ctx) =>
+            this.wideRoute(
+              ctx,
+              "snapshots.since",
+              "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/since/:snapshotFriendlyId",
+              "GET",
+              async () => {
+                const { req, reply, params } = ctx;
+                const sinceSnapshotResponse = await this.workerClient.getSnapshotsSince(
+                  params.runFriendlyId,
+                  params.snapshotFriendlyId,
+                  this.runnerIdFromRequest(req)
+                );
+
+                if (!sinceSnapshotResponse.success) {
+                  this.logger.error("Failed to get snapshots since", {
+                    runId: params.runFriendlyId,
+                    error: sinceSnapshotResponse.error,
+                  });
+                  reply.empty(500);
+                  return;
+                }
+
+                reply.json({
+                  snapshots: sinceSnapshotResponse.data.snapshots.map(legacifyCheckpointType),
+                } satisfies WorkloadRunSnapshotsSinceResponseBody);
+              },
+              { highFrequency: true }
+            ),
+        }
+      )
+      .route("/api/v1/workload-actions/deployments/:deploymentId/dequeue", "GET", {
+        paramsSchema: z.object({
+          deploymentId: z.string(),
+        }),
+
+        handler: async (ctx) =>
+          this.wideRoute(
+            ctx,
+            "deployment.dequeue",
+            "/api/v1/workload-actions/deployments/:deploymentId/dequeue",
+            "GET",
+            async () => {
+              const { req, reply, params } = ctx;
+              const dequeueResponse = await this.workerClient.dequeueFromVersion(
+                params.deploymentId,
+                1,
+                this.runnerIdFromRequest(req)
+              );
+
+              if (!dequeueResponse.success) {
+                this.logger.error("Failed to get latest snapshot", {
+                  deploymentId: params.deploymentId,
+                  error: dequeueResponse.error,
+                });
+                reply.empty(500);
+                return;
+              }
+
+              reply.json(
+                dequeueResponse.data.map(
+                  legacifyCheckpointType
+                ) satisfies WorkloadDequeueFromVersionResponseBody
+              );
+            }
+          ),
+      });
+
+    if (env.SEND_RUN_DEBUG_LOGS) {
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
+        bodySchema: WorkloadDebugLogRequestBody,
+        handler: async (ctx) =>
+          this.wideRoute(
+            ctx,
+            "logs.debug",
+            "/api/v1/workload-actions/runs/:runFriendlyId/logs/debug",
+            "POST",
+            async () => {
+              const { req, reply, params, body } = ctx;
+              reply.empty(204);
+
+              await this.workerClient.sendDebugLog(
+                params.runFriendlyId,
+                body,
+                this.runnerIdFromRequest(req)
+              );
+            },
+            { highFrequency: true }
+          ),
+      });
+    } else {
+      // Disabled: drop immediately without reading/parsing the body and without
+      // any log we can't switch off. Older runners still POST per log line; the
+      // route stays registered (an unregistered route would log "No route match"
+      // per request) but sheds the request at minimal cost. Request metrics still
+      // count it. 204 is non-retryable on the runner client, so no retry storm.
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        skipBodyParsing: true,
+        handler: async (ctx) => {
+          ctx.reply.empty(204);
+        },
+      });
+    }
+
+    // Snapshot callback endpoint (inbound from compute path)
+    httpServer.route("/api/v1/compute/snapshot-complete", "POST", {
+      bodySchema: SnapshotCallbackPayloadSchema,
+      handler: async (ctx) =>
+        this.wideRoute(
+          ctx,
+          "snapshot.callback",
+          "/api/v1/compute/snapshot-complete",
+          "POST",
+          async () => {
+            const { reply, body } = ctx;
+            if (!this.snapshotService) {
+              reply.empty(404);
+              return;
+            }
+
+            const result = await this.snapshotService.handleCallback(body);
+            reply.empty(result.status);
+          }
+        ),
+    });
+
+    return httpServer;
+  }
+
+  private createWebsocketServer() {
+    const io = new Server(this.httpServer.server);
+
+    const websocketServer: Namespace<
+      WorkloadClientToServerEvents,
+      WorkloadServerToClientEvents,
+      DefaultEventsMap,
+      WorkloadClientSocketData
+    > = io.of("/workload");
+
+    websocketServer.on("disconnect", (socket) => {
+      this.logger.verbose("[WS] disconnect", socket.id);
+    });
+    websocketServer.use(async (socket, next) => {
+      const setSocketDataFromHeader = (
+        dataKey: keyof typeof socket.data,
+        headerName: string,
+        required: boolean = true
+      ) => {
+        const value = socket.handshake.headers[headerName];
+
+        if (value) {
+          if (Array.isArray(value)) {
+            if (value[0]) {
+              socket.data[dataKey] = value[0];
+              return;
+            }
+          } else {
+            socket.data[dataKey] = value;
+            return;
+          }
+        }
+
+        if (required) {
+          this.logger.error("[WS] missing required header", { headerName });
+          throw new Error("missing header");
+        }
+      };
+
+      try {
+        setSocketDataFromHeader("deploymentId", WORKLOAD_HEADERS.DEPLOYMENT_ID);
+        setSocketDataFromHeader("runnerId", WORKLOAD_HEADERS.RUNNER_ID);
+      } catch (error) {
+        this.logger.error("[WS] setSocketDataFromHeader error", { error });
+        socket.disconnect(true);
+        return;
+      }
+
+      this.logger.debug("[WS] auth success", socket.data);
+
+      next();
+    });
+    websocketServer.on("connection", (socket) => {
+      const socketLogger = this.logger.child({
+        socketId: socket.id,
+        socketData: socket.data,
+      });
+
+      const getSocketMetadata = () => {
+        return {
+          deploymentId: socket.data.deploymentId,
+          runId: socket.data.runFriendlyId,
+          snapshotId: socket.data.snapshotId,
+          runnerId: socket.data.runnerId,
+        };
+      };
+
+      const emitSocketLifecycle = (
+        event: "run_connected" | "run_disconnected",
+        friendlyId: string,
+        disconnectReason?: string
+      ) => {
+        emitOneShot({
+          ...this.wideEventOpts,
+          op: event === "run_connected" ? "socket.run.connected" : "socket.run.disconnected",
+          kind: "event",
+          populate: (state) => {
+            state.extras.event = event;
+            setMeta(state, "run_id", friendlyId);
+            if (socket.data.deploymentId) {
+              setMeta(state, "deployment_id", socket.data.deploymentId);
+            }
+            if (socket.data.runnerId) setMeta(state, "runner_id", socket.data.runnerId);
+            state.extras.socket_id = socket.id;
+            if (disconnectReason) state.extras.disconnect_reason = disconnectReason;
+          },
+        });
+      };
+
+      const runConnected = (friendlyId: string) => {
+        socketLogger.debug("runConnected", { ...getSocketMetadata() });
+
+        // If there's already a run ID set, we should "disconnect" it from this socket
+        if (socket.data.runFriendlyId && socket.data.runFriendlyId !== friendlyId) {
+          socketLogger.debug("runConnected: disconnecting existing run", {
+            ...getSocketMetadata(),
+            newRunId: friendlyId,
+            oldRunId: socket.data.runFriendlyId,
+          });
+          runDisconnected(socket.data.runFriendlyId, "socket_run_replaced");
+        }
+
+        this.runSockets.set(friendlyId, socket);
+        this.emit("runConnected", { run: { friendlyId } });
+        socket.data.runFriendlyId = friendlyId;
+        emitSocketLifecycle("run_connected", friendlyId);
+      };
+
+      const runDisconnected = (friendlyId: string, reason: string) => {
+        socketLogger.debug("runDisconnected", { ...getSocketMetadata() });
+
+        // The run is gone from this runner (crash, exit, or replaced by a new
+        // run), so a pending delayed snapshot for it is stale. Genuine
+        // waitpoint suspensions keep the socket connected, so this doesn't
+        // cancel a snapshot that's still wanted; the runnerId match guards
+        // against a stale duplicate runner cancelling a fresh runner's
+        // snapshot after the run was reassigned. Caveat: socket.data.runnerId
+        // is frozen at the websocket handshake, so after a same-supervisor
+        // restore (new runner id, socket not recreated) this guard refuses
+        // the cancel - a missed cancel, never a wrong one. The
+        // attempt.complete cancel uses the runner's current HTTP header id
+        // and is unaffected.
+        this.snapshotService?.cancel(friendlyId, socket.data.runnerId);
+
+        this.runSockets.delete(friendlyId);
+        this.emit("runDisconnected", { run: { friendlyId } });
+        socket.data.runFriendlyId = undefined;
+        emitSocketLifecycle("run_disconnected", friendlyId, reason);
+      };
+
+      socketLogger.debug("wsServer socket connected", { ...getSocketMetadata() });
+
+      // FIXME: where does this get set?
+      if (socket.data.runFriendlyId) {
+        runConnected(socket.data.runFriendlyId);
+      }
+
+      socket.on("disconnecting", (reason, description) => {
+        socketLogger.verbose("Socket disconnecting", {
+          ...getSocketMetadata(),
+          reason,
+          description,
+        });
+
+        if (socket.data.runFriendlyId) {
+          runDisconnected(socket.data.runFriendlyId, `socket_disconnecting:${reason}`);
+        }
+      });
+
+      socket.on("disconnect", (reason, description) => {
+        socketLogger.debug("Socket disconnected", { ...getSocketMetadata(), reason, description });
+      });
+
+      socket.on("error", (error) => {
+        socketLogger.error("Socket error", {
+          ...getSocketMetadata(),
+          error: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+          },
+        });
+      });
+
+      socket.on("run:start", async (message) => {
+        const log = socketLogger.child({
+          eventName: "run:start",
+          ...getSocketMetadata(),
+          ...message,
+        });
+
+        log.debug("Handling run:start");
+
+        try {
+          runConnected(message.run.friendlyId);
+        } catch (error) {
+          log.error("run:start error", { error });
+        }
+      });
+
+      socket.on("run:stop", async (message) => {
+        const log = socketLogger.child({
+          eventName: "run:stop",
+          ...getSocketMetadata(),
+          ...message,
+        });
+
+        log.debug("Handling run:stop");
+
+        try {
+          runDisconnected(message.run.friendlyId, "run_stop_message");
+          // Don't delete trace context here - run:stop fires after each snapshot/shutdown
+          // but the run may be restored on a new VM and snapshot again. Trace context is
+          // re-populated on dequeue, and entries are small (4 strings per run).
+        } catch (error) {
+          log.error("run:stop error", { error });
+        }
+      });
+    });
+
+    return websocketServer;
+  }
+
+  notifyRun({ run }: { run: { friendlyId: string } }) {
+    try {
+      const runSocket = this.runSockets.get(run.friendlyId);
+
+      if (!runSocket) {
+        this.logger.debug("notifyRun: Run socket not found", { run });
+
+        this.workerClient.sendDebugLog(run.friendlyId, {
+          time: new Date(),
+          message: "run:notify socket not found on supervisor",
+        });
+
+        return;
+      }
+
+      runSocket.emit("run:notify", { version: "1", run });
+      this.logger.debug("run:notify sent", { run });
+
+      this.workerClient.sendDebugLog(run.friendlyId, {
+        time: new Date(),
+        message: "run:notify supervisor -> runner",
+      });
+    } catch (error) {
+      this.logger.error("Error in notifyRun", { run, error });
+
+      this.workerClient.sendDebugLog(run.friendlyId, {
+        time: new Date(),
+        message: "run:notify error on supervisor",
+      });
+    }
+  }
+
+  registerRunTraceContext(runFriendlyId: string, ctx: RunTraceContext) {
+    this.snapshotService?.registerTraceContext(runFriendlyId, ctx);
+  }
+
+  async start() {
+    await this.httpServer.start();
+  }
+
+  async stop() {
+    this.snapshotService?.stop();
+    await this.httpServer.stop();
+  }
+}

apps/supervisor/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../.configs/tsconfig.base.json",
+  "include": ["src/**/*.ts"],
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist"
+  }
+}

apps/webapp/.env

@@ -0,0 +1 @@
+../../.env

apps/webapp/.gitignore

@@ -0,0 +1,23 @@
+node_modules
+
+/.cache
+/build
+/public/build
+
+/cypress/screenshots
+/cypress/videos
+
+/app/styles/tailwind.css
+
+# Ensure the .env symlink is not removed by accident
+!.env
+
+# Storybook build outputs
+build-storybook.log
+.out
+.storybook-out
+storybook-static
+
+/prisma/seed.js
+/prisma/populate.js
+.memory-snapshots

apps/webapp/.server-changes/task-filter-icons.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Use the "all tasks" icon (`TasksIcon`) for the Tasks/Task type filter buttons on the Runs, Logs, Errors, and metrics dashboard pages (and the task-derived queue indicators), and show the clock icon for Scheduled runs in the run inspector header instead of the standard task icon.

apps/webapp/CLAUDE.md

@@ -0,0 +1,131 @@
+# Webapp
+
+Remix 2.17.4 app serving as the main API, dashboard, and orchestration engine. Uses an Express server (`server.ts`).
+
+## Verifying Changes
+
+**Never run `pnpm run build --filter webapp` to verify changes.** Building proves almost nothing about correctness. The webapp is an app, not a public package — use typecheck from the repo root:
+
+```bash
+pnpm run typecheck --filter webapp   # ~1-2 minutes
+```
+
+Only run typecheck after major changes (new files, significant refactors, schema changes). For small edits, trust the types and let CI catch issues.
+
+Note: Public packages (`packages/*`) use `build` instead. See the root CLAUDE.md for details.
+
+## Testing Dashboard Changes with Chrome DevTools MCP
+
+Use the `chrome-devtools` MCP server to visually verify local dashboard changes. The webapp must be running (`pnpm run dev --filter webapp` from repo root).
+
+### Login
+
+```
+1. mcp__chrome-devtools__new_page(url: "http://localhost:3030")
+   → Redirects to /login
+2. mcp__chrome-devtools__click the "Continue with Email" link
+3. mcp__chrome-devtools__fill the email field with "local@trigger.dev"
+4. mcp__chrome-devtools__click "Send a magic link"
+   → Auto-logs in and redirects to the dashboard (no email verification needed locally)
+```
+
+### Navigating and Verifying
+
+- **take_snapshot**: Get an a11y tree of the page (text content, element UIDs for interaction). Prefer this over screenshots for understanding page structure.
+- **take_screenshot**: Capture what the page looks like visually. Use to verify styling, layout, and visual changes.
+- **navigate_page**: Go to specific URLs, e.g. `http://localhost:3030/orgs/references-bc08/projects/hello-world-SiWs/env/dev/runs`
+- **click / fill**: Interact with elements using UIDs from `take_snapshot`.
+- **evaluate_script**: Run JS in the browser console for debugging.
+- **list_console_messages**: Check for console errors after navigating.
+
+### Tips
+
+- Snapshots can be very large on complex pages (200K+ chars). Use `take_screenshot` first to orient, then `take_snapshot` only when you need element UIDs to interact.
+- The local seeded user email is `local@trigger.dev`.
+- Dashboard URL pattern: `http://localhost:3030/orgs/{orgSlug}/projects/{projectSlug}/env/{envSlug}/{section}`
+
+## Key File Locations
+
+- **Trigger API**: `app/routes/api.v1.tasks.$taskId.trigger.ts`
+- **Batch trigger**: `app/routes/api.v1.tasks.batch.ts`
+- **OTEL endpoints**: `app/routes/otel.v1.logs.ts`, `app/routes/otel.v1.traces.ts`
+- **Prisma setup**: `app/db.server.ts`
+- **Run engine config**: `app/v3/runEngine.server.ts`
+- **Services**: `app/v3/services/**/*.server.ts`
+- **Presenters**: `app/v3/presenters/**/*.server.ts`
+
+## Route Convention
+
+Routes use Remix flat-file convention with dot-separated segments:
+`api.v1.tasks.$taskId.trigger.ts` -> `/api/v1/tasks/:taskId/trigger`
+
+## Abort Signals
+
+**Never use `request.signal`** for detecting client disconnects. It is broken due to a Node.js bug ([nodejs/node#55428](https://github.com/nodejs/node/issues/55428)) where the AbortSignal chain is severed when Remix internally clones the Request object. Instead, use `getRequestAbortSignal()` from `app/services/httpAsyncStorage.server.ts`, which is wired directly to Express `res.on("close")` and fires reliably.
+
+```typescript
+import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server";
+
+// In route handlers, SSE streams, or any server-side code:
+const signal = getRequestAbortSignal();
+```
+
+## Environment Variables
+
+Access via `env` export from `app/env.server.ts`. **Never use `process.env` directly.**
+
+For testable code, **never import env.server.ts** in test files. Pass configuration as options instead:
+- `realtime/nativeRealtimeClient.server.ts` (testable service, takes config as constructor arg)
+- `realtime/nativeRealtimeClientInstance.server.ts` (creates singleton with env config)
+
+## Run Engine 2.0
+
+The webapp integrates `@internal/run-engine` via `app/v3/runEngine.server.ts`. This is the singleton engine instance. Services in `app/v3/services/` call engine methods for all run lifecycle operations (triggering, completing, cancelling, etc.).
+
+The `engineVersion.server.ts` file determines V1 vs V2 for a given environment. New code should always target V2.
+
+## Background Workers
+
+Background job workers use `@trigger.dev/redis-worker`:
+- `app/v3/commonWorker.server.ts`
+- `app/v3/alertsWorker.server.ts`
+- `app/v3/batchTriggerWorker.server.ts`
+
+Do NOT add new jobs using zodworker/graphile-worker (legacy).
+
+## Real-time
+
+- Socket.io: `app/v3/handleSocketIo.server.ts`, `app/v3/handleWebsockets.server.ts`
+- Electric SQL: Powers real-time data sync for the dashboard
+
+## Legacy V1 Code
+
+The `app/v3/` directory name is misleading - most code is actively used by V2. Only these specific files are V1-only legacy:
+- `app/v3/marqs/` (old MarQS queue system)
+- `app/v3/legacyRunEngineWorker.server.ts`
+- `app/v3/services/triggerTaskV1.server.ts`
+- `app/v3/services/cancelTaskRunV1.server.ts`
+- `app/v3/authenticatedSocketConnection.server.ts`
+- `app/v3/sharedSocketConnection.ts`
+
+Some services (e.g., `cancelTaskRun.server.ts`, `batchTriggerV3.server.ts`) branch on `RunEngineVersion` to support both V1 and V2. When editing these, only modify V2 code paths.
+
+## Performance: Trigger Hot Path
+
+The `triggerTask.server.ts` service is the **highest-throughput code path** in the system. Every API trigger call goes through it. Keep it fast:
+
+- **Do NOT add database queries** to `triggerTask.server.ts` or `batchTriggerV3.server.ts`. Task defaults (TTL, etc.) are resolved via `backgroundWorkerTask.findFirst()` in the queue concern (`queues.server.ts`) - one query per request, in mutually exclusive branches depending on locked/non-locked path. Piggyback on the existing query instead of adding new ones.
+- **Two-stage resolution pattern**: Task metadata is resolved in two stages by design:
+  1. **Trigger time** (`triggerTask.server.ts`): Only TTL is resolved from task defaults. Everything else uses whatever the caller provides.
+  2. **Dequeue time** (`dequeueSystem.ts`): Full `BackgroundWorkerTask` is loaded and retry config, machine config, maxDuration, etc. are resolved against task defaults.
+- If you need to add a new task-level default, **add it to the existing `select` clause** in the `backgroundWorkerTask.findFirst()` query — do NOT add a second query. If the default doesn't need to be known at trigger time, resolve it at dequeue time instead.
+- Batch triggers (`batchTriggerV3.server.ts`) follow the same pattern — keep batch paths equally fast.
+
+## Prisma Query Patterns
+
+- **Always use `findFirst` instead of `findUnique`.** Prisma's `findUnique` has an implicit DataLoader that batches concurrent calls into a single `IN` query. This batching cannot be disabled and has active bugs even in Prisma 6.x: uppercase UUIDs returning null (#25484, confirmed 6.4.1), composite key SQL correctness issues (#22202), and 5-10x worse performance than manual DataLoader (#6573, open since 2021). `findFirst` is never batched and avoids this entire class of issues.
+
+## React Patterns
+
+- Only use `useCallback`/`useMemo` for context provider values, expensive derived data that is a dependency elsewhere, or stable refs required by a dependency array. Don't wrap ordinary event handlers or trivial computations.
+- Use named constants for sentinel/placeholder values (e.g. `const UNSET_VALUE = "__unset__"`) instead of raw string literals scattered across comparisons.

apps/webapp/README.md

@@ -0,0 +1,10 @@
+## Trigger webapp - powered by Remix
+
+To start, run with `pnpm run dev --filter webapp`
+
+### Build the docker image locally:
+
+```sh
+pnpm run docker:build:webapp
+docker run -it triggerdotdev-webapp sh
+```

apps/webapp/app/api/versions.ts

@@ -0,0 +1,57 @@
+import {
+  API_VERSION_HEADER_NAME,
+  API_VERSION as CORE_API_VERSION,
+} from "@trigger.dev/core/v3/serverOnly";
+import { z } from "zod";
+
+export const CURRENT_API_VERSION = CORE_API_VERSION;
+
+export const NON_SPECIFIC_API_VERSION = "none";
+
+export type API_VERSIONS = typeof CURRENT_API_VERSION | typeof NON_SPECIFIC_API_VERSION;
+
+export function getApiVersion(request: Request): API_VERSIONS {
+  const apiVersion = request.headers.get(API_VERSION_HEADER_NAME);
+
+  if (apiVersion === CURRENT_API_VERSION) {
+    return apiVersion;
+  }
+
+  return NON_SPECIFIC_API_VERSION;
+}
+
+// This has been copied from the core package to allow us to use these types in the webapp
+export const RunStatusUnspecifiedApiVersion = z.enum([
+  /// Task is waiting for a version update because it cannot execute without additional information (task, queue, etc.). Replaces WAITING_FOR_DEPLOY
+  "PENDING_VERSION",
+  /// Task hasn't been deployed yet but is waiting to be executed
+  "WAITING_FOR_DEPLOY",
+  /// Task is waiting to be executed by a worker
+  "QUEUED",
+  /// Task is currently being executed by a worker
+  "EXECUTING",
+  /// Task has failed and is waiting to be retried
+  "REATTEMPTING",
+  /// Task has been paused by the system, and will be resumed by the system
+  "FROZEN",
+  /// Task has been completed successfully
+  "COMPLETED",
+  /// Task has been canceled by the user
+  "CANCELED",
+  /// Task has been completed with errors
+  "FAILED",
+  /// Task has crashed and won't be retried, most likely the worker ran out of resources, e.g. memory or storage
+  "CRASHED",
+  /// Task was interrupted during execution, mostly this happens in development environments
+  "INTERRUPTED",
+  /// Task has failed to complete, due to an error in the system
+  "SYSTEM_FAILURE",
+  /// Task has been scheduled to run at a specific time
+  "DELAYED",
+  /// Task has expired and won't be executed
+  "EXPIRED",
+  /// Task has reached it's maxDuration and has been stopped
+  "TIMED_OUT",
+]);
+
+export type RunStatusUnspecifiedApiVersion = z.infer<typeof RunStatusUnspecifiedApiVersion>;

apps/webapp/app/assets/icons/AIChatIcon.tsx

@@ -0,0 +1,27 @@
+export function AIChatIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M5.42975 3.52374C7.56942 3.17905 9.76406 3 12 3C14.2359 3 16.4306 3.17905 18.5702 3.52374C20.0066 3.75513 21 5.01325 21 6.42589V13.5741C21 14.9868 20.0066 16.2449 18.5702 16.4763C16.8747 16.7494 15.1447 16.9185 13.3869 16.977C13.1832 16.9837 12.9952 17.0654 12.8594 17.2013L9.28032 20.7803C9.06582 20.9948 8.74323 21.059 8.46298 20.9429C8.18272 20.8268 7.99999 20.5534 7.99999 20.25V16.8073C7.13516 16.7236 6.27812 16.6129 5.42975 16.4763C3.99338 16.2449 2.99998 14.9868 2.99998 13.5741V6.42589C2.99998 5.01325 3.99337 3.75513 5.42975 3.52374Z"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinejoin="round"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12 6C12.3443 6 12.6501 6.22034 12.7589 6.54702L13.1795 7.8086C13.3387 8.28637 13.7136 8.66127 14.1914 8.82053L15.453 9.24106C15.7797 9.34995 16 9.65566 16 10C16 10.3443 15.7797 10.6501 15.453 10.7589L14.1914 11.1795C13.7136 11.3387 13.3387 11.7136 13.1795 12.1914L12.7589 13.453C12.6501 13.7797 12.3443 14 12 14C11.6557 14 11.3499 13.7797 11.2411 13.453L10.8205 12.1914C10.6613 11.7136 10.2864 11.3387 9.8086 11.1795L8.54702 10.7589C8.22034 10.6501 8 10.3443 8 10C8 9.65566 8.22034 9.34995 8.54702 9.24106L9.8086 8.82053C10.2864 8.66127 10.6613 8.28637 10.8205 7.8086L11.2411 6.54702C11.3499 6.22034 11.6557 6 12 6Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AIMetricsIcon.tsx

@@ -0,0 +1,34 @@
+export function AIMetricsIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M5.5 2C5.8013 2 6.0688 2.1928 6.16408 2.47864L6.53204 3.58252C6.67139 4.00057 6.99943 4.32861 7.41748 4.46796L8.52136 4.83592C8.8072 4.9312 9 5.1987 9 5.5C9 5.8013 8.8072 6.0688 8.52136 6.16408L7.41748 6.53204C6.99943 6.67139 6.67139 6.99943 6.53204 7.41748L6.16408 8.52136C6.0688 8.8072 5.8013 9 5.5 9C5.1987 9 4.9312 8.8072 4.83592 8.52136L4.46796 7.41748C4.32861 6.99943 4.00057 6.67139 3.58252 6.53204L2.47864 6.16408C2.1928 6.0688 2 5.8013 2 5.5C2 5.1987 2.1928 4.9312 2.47864 4.83592L3.58252 4.46796C4.00057 4.32861 4.32861 4.00057 4.46796 3.58252L4.83592 2.47864C4.9312 2.1928 5.1987 2 5.5 2Z"
+        fill="currentColor"
+      />
+      <path
+        d="M9 12C9 10.8954 9.89543 10 11 10H13C14.1046 10 15 10.8954 15 12V21H9V12Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M15 7C15 5.89543 15.8954 5 17 5H19C20.1046 5 21 5.89543 21 7V20C21 20.5523 20.5523 21 20 21H15V7Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M3 17C3 15.8954 3.89543 15 5 15H7C8.10457 15 9 15.8954 9 17V21H4C3.44772 21 3 20.5523 3 20V17Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AIPenIcon.tsx

@@ -0,0 +1,25 @@
+export function AIPenIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M6 2C6.34434 2 6.65005 2.22034 6.75895 2.54702L7.17947 3.8086C7.33873 4.28637 7.71363 4.66127 8.1914 4.82053L9.45298 5.24105C9.77966 5.34995 10 5.65566 10 6C10 6.34434 9.77966 6.65005 9.45298 6.75895L8.1914 7.17947C7.71363 7.33873 7.33873 7.71363 7.17947 8.1914L6.75895 9.45298C6.65005 9.77966 6.34434 10 6 10C5.65566 10 5.34995 9.77966 5.24105 9.45298L4.82053 8.1914C4.66127 7.71363 4.28637 7.33873 3.8086 7.17947L2.54702 6.75895C2.22034 6.65005 2 6.34434 2 6C2 5.65566 2.22034 5.34995 2.54702 5.24105L3.8086 4.82053C4.28637 4.66127 4.66127 4.28637 4.82053 3.8086L5.24105 2.54702C5.34995 2.22034 5.65566 2 6 2Z"
+        fill="currentColor"
+      />
+      <path
+        d="M18.7573 3.6275L20.3732 5.24335C21.1542 6.0244 21.1542 7.29073 20.3732 8.07178L9.7203 18.7246C9.57776 18.8671 9.39569 18.9631 9.19757 19.0002L4.03376 19.9669L5.00051 14.8031C5.03763 14.6051 5.13358 14.4229 5.27603 14.2804L15.9289 3.6275C16.7099 2.84645 17.9763 2.84645 18.7573 3.6275Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <line x1="17.6464" y1="10.3536" x2="13.6464" y2="6.35355" stroke="currentColor" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -0,0 +1,31 @@
+export function AISparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M9.14286 4.85718C9.46177 4.85718 9.74205 5.06859 9.82966 5.37523L10.6041 8.08589C10.9431 9.27235 11.8705 10.1998 13.057 10.5388L15.7677 11.3132C16.0743 11.4008 16.2857 11.6811 16.2857 12C16.2857 12.319 16.0743 12.5992 15.7677 12.6868L13.057 13.4613C11.8705 13.8003 10.9431 14.7277 10.6041 15.9142L9.82966 18.6248C9.74205 18.9315 9.46177 19.1429 9.14286 19.1429C8.82394 19.1429 8.54367 18.9315 8.45605 18.6248L7.68158 15.9142C7.34259 14.7277 6.41517 13.8003 5.22871 13.4613L2.51806 12.6868C2.21141 12.5992 2 12.319 2 12C2 11.6811 2.21141 11.4008 2.51806 11.3132L5.22871 10.5388C6.41517 10.1998 7.34259 9.27235 7.68158 8.08589L8.45605 5.37523C8.54367 5.06859 8.82394 4.85718 9.14286 4.85718Z"
+        fill="#7655FD"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M17.7143 2C18.0421 2 18.3278 2.22307 18.4072 2.54105L18.6538 3.5272C18.8777 4.42291 19.5771 5.12229 20.4728 5.34622L21.459 5.59276C21.7769 5.67225 22 5.95795 22 6.28571C22 6.61348 21.7769 6.89918 21.459 6.97867L20.4728 7.22521C19.5771 7.44914 18.8777 8.14851 18.6538 9.04423L18.4072 10.0304C18.3278 10.3484 18.0421 10.5714 17.7143 10.5714C17.3865 10.5714 17.1008 10.3484 17.0213 10.0304L16.7748 9.04423C16.5509 8.14852 15.8515 7.44914 14.9558 7.22521L13.9696 6.97867C13.6516 6.89918 13.4286 6.61348 13.4286 6.28571C13.4286 5.95795 13.6516 5.67225 13.9696 5.59276L14.9558 5.34622C15.8515 5.12229 16.5509 4.42291 16.7748 3.5272L17.0213 2.54105C17.1008 2.22307 17.3865 2 17.7143 2Z"
+        fill="#D946EF"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M16.2857 14.8572C16.5932 14.8572 16.8661 15.0539 16.9633 15.3456L17.3388 16.472C17.481 16.8986 17.8157 17.2333 18.2423 17.3755L19.3687 17.751C19.6604 17.8482 19.8571 18.1212 19.8571 18.4286C19.8571 18.7361 19.6604 19.009 19.3687 19.1062L18.2423 19.4817C17.8157 19.6239 17.481 19.9586 17.3388 20.3852L16.9633 21.5116C16.8661 21.8033 16.5932 22 16.2857 22C15.9783 22 15.7053 21.8033 15.6081 21.5116L15.2326 20.3852C15.0904 19.9586 14.7557 19.6239 14.3291 19.4817L13.2027 19.1062C12.911 19.009 12.7143 18.7361 12.7143 18.4286C12.7143 18.1212 12.911 17.8482 13.2027 17.751L14.3291 17.3755C14.7557 17.2333 15.0904 16.8986 15.2326 16.472L15.6081 15.3456C15.7053 15.0539 15.9783 14.8572 16.2857 14.8572Z"
+        fill="#4F46E5"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AbacusIcon.tsx

@@ -0,0 +1,71 @@
+export function AbacusIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_16909_120578)">
+        <path
+          d="M4 3V21"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M20 21V3"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M8 5L8 6"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M14 5L14 6"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M15 10L15 11"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M9 10L9 11"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M12 15L12 16"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M8 15L8 16"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3 21H21"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+      </g>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AiProviderIcons.tsx

@@ -0,0 +1,181 @@
+type IconProps = { className?: string };
+
+export function OpenAIIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M9.648 9.27004V7.36778C9.648 7.20757 9.70764 7.08738 9.84657 7.00738L13.6396 4.80477C14.1559 4.50443 14.7715 4.36434 15.4069 4.36434C17.7899 4.36434 19.2992 6.22658 19.2992 8.20884C19.2992 8.34898 19.2992 8.50919 19.2793 8.6694L15.3473 6.3466C15.1091 6.20651 14.8707 6.20651 14.6324 6.3466L9.648 9.27004ZM18.5048 16.6789V12.1334C18.5048 11.853 18.3855 11.6528 18.1473 11.5126L13.1629 8.58919L14.7913 7.64801C14.9303 7.56801 15.0495 7.56801 15.1884 7.64801L18.9814 9.85062C20.0737 10.4915 20.8084 11.853 20.8084 13.1745C20.8084 14.6962 19.9148 16.098 18.5048 16.6787V16.6789ZM8.47638 12.6742L6.848 11.7131C6.70907 11.6331 6.64943 11.5128 6.64943 11.3526V6.94746C6.64943 4.80498 8.2778 3.18295 10.4821 3.18295C11.3163 3.18295 12.0906 3.46334 12.7461 3.96392L8.834 6.24669C8.59578 6.38679 8.47658 6.58702 8.47658 6.86746V12.6743L8.47638 12.6742ZM11.9814 14.7165L9.648 13.395V10.5918L11.9814 9.27025L14.3146 10.5918V13.395L11.9814 14.7165ZM13.4807 20.8038C12.6466 20.8038 11.8723 20.5234 11.2168 20.0229L15.1288 17.7401C15.3671 17.6 15.4863 17.3997 15.4863 17.1193V11.3124L17.1346 12.2735C17.2735 12.3535 17.3331 12.4737 17.3331 12.634V17.0391C17.3331 19.1816 15.6848 20.8036 13.4807 20.8036V20.8038ZM8.77424 16.3385L4.9812 14.136C3.88892 13.4951 3.15426 12.1336 3.15426 10.8121C3.15426 9.27025 4.06775 7.88863 5.4776 7.30789V11.8733C5.4776 12.1537 5.59683 12.3539 5.83506 12.494L10.7997 15.3974L9.17134 16.3385C9.03241 16.4185 8.91317 16.4185 8.77424 16.3385ZM8.55592 19.6224C6.31192 19.6224 4.66364 17.9204 4.66364 15.8179C4.66364 15.6577 4.68355 15.4975 4.70329 15.3373L8.61535 17.62C8.85358 17.7602 9.09201 17.7602 9.33023 17.62L14.3146 14.7167V16.619C14.3146 16.7792 14.255 16.8994 14.1161 16.9794L10.3231 19.182C9.80672 19.4823 9.19109 19.6224 8.55571 19.6224H8.55592ZM13.4807 22.0052C15.8836 22.0052 17.8891 20.2832 18.3461 18.0004C20.5701 17.4197 21.9999 15.3172 21.9999 13.1747C21.9999 11.773 21.4043 10.4115 20.3319 9.43025C20.4312 9.00972 20.4908 8.58919 20.4908 8.16886C20.4908 5.30551 18.1872 3.16283 15.5261 3.16283C14.9901 3.16283 14.4737 3.24283 13.9574 3.42316C13.0636 2.54207 11.8324 1.98145 10.4821 1.98145C8.07927 1.98145 6.07369 3.70339 5.61678 5.98616C3.39269 6.5669 1.96289 8.6694 1.96289 10.8119C1.96289 12.2136 2.55857 13.5751 3.63095 14.5563C3.53166 14.9768 3.47207 15.3974 3.47207 15.8177C3.47207 18.6811 5.77567 20.8237 8.43668 20.8237C8.97277 20.8237 9.48911 20.7437 10.0055 20.5634C10.899 21.4445 12.1302 22.0052 13.4807 22.0052Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function AnthropicIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M17.3041 3.541h-3.6718l6.696 16.918H24Zm-10.6082 0L0 20.459h3.7442l1.3693-3.5527h7.0052l1.3693 3.5528h3.7442L10.5363 3.5409Zm-.3712 10.2232 2.2914-5.9456 2.2914 5.9456Z" />
+    </svg>
+  );
+}
+
+export function GeminiIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M12 0C12 6.627 6.627 12 0 12c6.627 0 12 5.373 12 12 0-6.627 5.373-12 12-12-6.627 0-12-5.373-12-12Z" />
+    </svg>
+  );
+}
+
+export function LlamaIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 12 12"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M3.4485 2C4.406 2 5.2065 2.466 6.1635 3.688L6.3045 3.5015C6.3995 3.3785 6.496 3.2595 6.5945 3.1465L6.751 2.9715C7.294 2.394 7.896 2 8.6125 2C9.249 2 9.847 2.2785 10.358 2.758L10.467 2.8645C11.332 3.747 11.9255 5.2195 11.9935 6.8775L11.999 7.0735L12 7.1985C12 7.949 11.86 8.578 11.591 9.0485L11.521 9.1635L11.467 9.24C11.3165 9.45 11.135 9.619 10.924 9.7445L10.7915 9.8155L10.748 9.8355C10.6986 9.85749 10.6482 9.87718 10.597 9.8945C10.3825 9.96542 10.1579 10.0006 9.932 9.9985C9.67 9.9985 9.434 9.965 9.213 9.891C8.906 9.789 8.6315 9.611 8.35 9.333L8.2365 9.2155C7.86 8.8095 7.4695 8.2275 6.99 7.4225L6.275 6.2175L6.003 5.77L5.12 7.335L4.9485 7.631C3.7985 9.578 3.1135 10 2.178 10C1.573 10 1.0755 9.79 0.71 9.409L0.626 9.317C0.384 9.0305 0.2075 8.6615 0.1045 8.2225L0.071 8.0625C0.0323456 7.84982 0.00961585 7.63456 0.003 7.4185L0 7.234C0.001 6.8615 0.03 6.489 0.087 6.119L0.137 5.8325C0.286 5.0675 0.551 4.3535 0.905 3.754L1.0095 3.584C1.598 2.669 2.404 2.0575 3.317 2.004L3.4485 2ZM3.432 3.3075L3.3315 3.3125C2.9165 3.354 2.5285 3.649 2.2055 4.101L2.1365 4.2005L2.1315 4.2095C1.7965 4.718 1.539 5.3985 1.4035 6.132L1.4015 6.143C1.33323 6.5148 1.29859 6.89199 1.298 7.27L1.299 7.364C1.301 7.454 1.3075 7.544 1.319 7.634L1.3405 7.7795C1.3865 8.031 1.469 8.2335 1.5835 8.3835L1.642 8.452C1.7935 8.6135 1.991 8.698 2.227 8.698C2.777 8.698 3.125 8.36 4.075 6.8775L5.1625 5.1775L5.3895 4.827L5.32 4.728C4.555 3.65 4.042 3.3075 3.432 3.3075ZM8.53 3.0315L8.442 3.035C8.1245 3.059 7.8305 3.2145 7.532 3.5015L7.434 3.6005C7.2145 3.8315 6.9905 4.1325 6.7505 4.504L6.8835 4.703C6.9735 4.84 7.0645 4.983 7.1585 5.132L7.305 5.3695L8.003 6.537L8.3505 7.094C8.642 7.557 8.8655 7.894 9.0545 8.135L9.161 8.266C9.302 8.429 9.4255 8.536 9.5495 8.6025L9.6005 8.6275C9.714 8.6775 9.829 8.6965 9.9595 8.6965C10.0475 8.6975 10.1345 8.685 10.2185 8.66C10.3875 8.608 10.5235 8.5 10.625 8.3415L10.6725 8.26L10.711 8.179C10.808 7.9495 10.856 7.649 10.856 7.2865L10.853 7.062C10.813 5.6265 10.384 4.376 9.753 3.663L9.665 3.5685C9.33 3.227 8.943 3.0315 8.53 3.0315Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function DeepseekIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_20374_57805)">
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M23.7479 4.48176C23.4939 4.35776 23.384 4.59476 23.236 4.71576C23.185 4.75476 23.142 4.80576 23.099 4.85176C22.727 5.24876 22.293 5.50876 21.726 5.47776C20.897 5.43176 20.189 5.69176 19.563 6.32576C19.43 5.54376 18.988 5.07776 18.316 4.77776C17.964 4.62176 17.608 4.46676 17.361 4.12776C17.189 3.88676 17.142 3.61776 17.056 3.35376C17.001 3.19376 16.946 3.03076 16.763 3.00376C16.563 2.97276 16.4849 3.13976 16.4069 3.27976C16.094 3.85176 15.9729 4.48176 15.9849 5.11976C16.0119 6.55576 16.618 7.69976 17.823 8.51276C17.96 8.60576 17.995 8.69976 17.952 8.83576C17.87 9.11576 17.772 9.38776 17.686 9.66876C17.631 9.84776 17.549 9.88576 17.357 9.80876C16.7082 9.52995 16.1189 9.12939 15.6209 8.62876C14.7639 7.80076 13.9899 6.88676 13.0239 6.17076C12.8001 6.00537 12.5703 5.84827 12.3349 5.69976C11.3499 4.74276 12.4649 3.95676 12.7229 3.86376C12.9929 3.76576 12.8159 3.43176 11.9439 3.43576C11.0719 3.43976 10.2739 3.73076 9.25695 4.11976C9.10582 4.17767 8.95033 4.22348 8.79195 4.25676C7.84158 4.07769 6.8696 4.0433 5.90895 4.15476C4.02395 4.36476 2.51895 5.25676 1.41195 6.77776C0.0819496 8.60576 -0.23105 10.6838 0.15195 12.8498C0.55495 15.1338 1.72095 17.0248 3.51195 18.5028C5.36995 20.0358 7.50895 20.7868 9.94995 20.6428C11.4319 20.5578 13.0829 20.3588 14.9439 18.7828C15.4139 19.0168 15.906 19.1098 16.724 19.1798C17.354 19.2388 17.96 19.1498 18.429 19.0518C19.164 18.8958 19.1129 18.2148 18.8479 18.0908C16.693 17.0868 17.166 17.4958 16.735 17.1648C17.831 15.8688 19.481 14.5228 20.127 10.1618C20.177 9.81476 20.134 9.59676 20.127 9.31676C20.123 9.14676 20.162 9.07976 20.357 9.06076C20.898 9.00463 21.4228 8.84327 21.902 8.58576C23.298 7.82276 23.862 6.57076 23.995 5.06876C24.015 4.83876 23.9909 4.60276 23.7479 4.48176ZM11.5809 17.9998C9.49195 16.3578 8.47895 15.8168 8.06095 15.8398C7.66895 15.8638 7.73995 16.3108 7.82595 16.6028C7.91595 16.8908 8.03295 17.0888 8.19695 17.3418C8.31095 17.5088 8.38895 17.7578 8.08395 17.9448C7.41095 18.3608 6.24195 17.8048 6.18695 17.7778C4.82595 16.9758 3.68695 15.9178 2.88595 14.4708C2.11195 13.0778 1.66195 11.5838 1.58795 9.98876C1.56795 9.60276 1.68095 9.46676 2.06495 9.39676C2.56906 9.30029 3.08558 9.28711 3.59395 9.35776C5.72595 9.66976 7.53995 10.6228 9.06195 12.1318C9.92995 12.9918 10.5869 14.0188 11.2639 15.0228C11.9839 16.0888 12.7579 17.1048 13.7439 17.9368C14.0919 18.2288 14.3689 18.4508 14.6349 18.6138C13.8329 18.7038 12.4949 18.7238 11.5809 17.9998ZM12.5809 11.5598C12.5808 11.5101 12.5927 11.4611 12.6157 11.4171C12.6387 11.373 12.672 11.3353 12.7129 11.307C12.7538 11.2787 12.8009 11.2609 12.8502 11.2549C12.8995 11.2489 12.9495 11.2551 12.9959 11.2728C13.0551 11.294 13.1062 11.3331 13.142 11.3848C13.1779 11.4364 13.1967 11.4979 13.1959 11.5608C13.1961 11.6014 13.1881 11.6416 13.1726 11.6791C13.157 11.7166 13.1341 11.7506 13.1053 11.7792C13.0764 11.8078 13.0422 11.8303 13.0045 11.8455C12.9669 11.8607 12.9266 11.8683 12.8859 11.8678C12.8457 11.8679 12.8057 11.86 12.7685 11.8445C12.7313 11.829 12.6976 11.8063 12.6693 11.7776C12.641 11.7489 12.6186 11.7149 12.6037 11.6775C12.5887 11.6401 12.5803 11.6 12.5809 11.5598ZM15.6909 13.1558C15.4909 13.2368 15.2919 13.3068 15.1009 13.3158C14.8136 13.3258 14.5316 13.236 14.3029 13.0618C14.0289 12.8318 13.8329 12.7038 13.7509 12.3038C13.7227 12.1083 13.7281 11.9094 13.7669 11.7158C13.8369 11.3888 13.7589 11.1788 13.5279 10.9888C13.3409 10.8328 13.1019 10.7898 12.8399 10.7898C12.7502 10.7845 12.6631 10.7578 12.5859 10.7118C12.4759 10.6578 12.3859 10.5218 12.4719 10.3538C12.4999 10.2998 12.6319 10.1678 12.6639 10.1438C13.0199 9.94176 13.4309 10.0078 13.8099 10.1598C14.1619 10.3038 14.4279 10.5678 14.8109 10.9418C15.2019 11.3928 15.2729 11.5178 15.4959 11.8558C15.6719 12.1208 15.8319 12.3928 15.9409 12.7038C16.0079 12.8988 15.9219 13.0578 15.6909 13.1558Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_20374_57805">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}
+
+export function XAIIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M17.7329 8.44672L18.0784 22.0139H20.8452L21.1911 3.50781L17.7329 8.44672Z"
+        fill="currentColor"
+      />
+      <path d="M21.1911 2H16.9692L10.3442 11.4621L12.4552 14.4768L21.1911 2Z" fill="currentColor" />
+      <path
+        d="M2.95508 22.0136H7.17691L9.28824 18.9989L7.17691 15.9839L2.95508 22.0136Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2.95508 8.44629L12.4546 22.0134H16.6764L7.17691 8.44629H2.95508Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function PerplexityIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M18.4875 2V8.06H20.75V16.6833H18.3042V22L12.44 16.8383V21.9592H11.5308V16.8325L5.66 22V16.6125H3.25V7.99H5.65333V2L11.5308 7.41167V2.15833H12.4392V7.56667L18.4875 2ZM12.44 9.53667V15.6358L17.395 19.9975V14.0333L12.44 9.53667ZM11.5242 9.47L6.56917 13.9683V19.9975L11.5242 15.6358V9.47083V9.47ZM18.3042 15.7867H19.8408V8.9575H13.2167L18.3042 13.5742V15.7867ZM10.8192 8.88667H4.15833V15.7158H5.65833V13.5692L10.8183 8.88583L10.8192 8.88667ZM6.5625 4.06333V7.98833H10.825L6.5625 4.06333ZM17.5783 4.06333L13.3158 7.98833H17.5783V4.06333Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function CerebrasIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M11.6535 20.6834C10.4382 20.6834 9.28717 20.4401 8.23625 20.0036C6.66345 19.3453 5.31944 18.2433 4.36862 16.8551C3.4178 15.4669 2.86732 13.7997 2.86732 11.9964C2.86732 10.7943 3.11039 9.65655 3.56078 8.61182C4.22564 7.0519 5.3409 5.72809 6.7421 4.7907C8.1433 3.85331 9.83047 3.30948 11.6535 3.30948V2C10.2594 2 8.92972 2.27907 7.71437 2.78712C5.8985 3.54562 4.35432 4.81217 3.26767 6.40788C2.17386 8.00356 1.5376 9.92844 1.5376 11.9964C1.5376 13.3774 1.82356 14.6941 2.33114 15.8891C3.09609 17.6851 4.38291 19.2093 5.99144 20.2898C7.60713 21.3703 9.55167 22 11.6463 22V20.6834H11.6535Z"
+        fill="currentColor"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M7.14949 17.3272C6.32735 16.6403 5.71253 15.8102 5.2979 14.9014C4.88323 13.9927 4.66877 13.0124 4.66877 12.0249C4.66877 11.2378 4.8046 10.4506 5.06911 9.69931C5.34079 8.94794 5.74113 8.23241 6.29159 7.58122C6.9779 6.7655 7.81433 6.15012 8.72225 5.73508C9.63021 5.32005 10.6239 5.11254 11.6105 5.11254C12.3969 5.11254 13.1904 5.24849 13.9411 5.51325C14.6989 5.78516 15.4138 6.18588 16.0643 6.7297L16.9151 5.72077C16.143 5.07676 15.2851 4.59018 14.3843 4.27533C13.4835 3.95332 12.547 3.7959 11.6105 3.7959C10.4309 3.7959 9.25846 4.04634 8.17178 4.54009C7.08514 5.03382 6.09141 5.77087 5.27643 6.73687C4.62587 7.50967 4.14689 8.36119 3.82518 9.25563C3.50347 10.1501 3.34619 11.0875 3.34619 12.0249C3.34619 13.1984 3.59641 14.3719 4.08969 15.4524C4.58298 16.5329 5.32649 17.5276 6.29876 18.3362L7.14949 17.3272Z"
+        fill="currentColor"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M9.18714 16.4758C8.32211 16.0179 7.64298 15.3524 7.17829 14.5724C6.7136 13.7925 6.47052 12.8909 6.47052 11.9821C6.47052 11.1807 6.65641 10.3721 7.06388 9.62074C7.52144 8.75492 8.19345 8.08228 8.97983 7.6243C9.76624 7.1592 10.667 6.9159 11.5821 6.9159C12.3828 6.9159 13.1978 7.10194 13.9556 7.50269L14.5704 6.33631C13.6196 5.83541 12.5901 5.59212 11.5749 5.59927C10.424 5.59927 9.28725 5.90697 8.30069 6.48655C7.31412 7.06618 6.46339 7.92487 5.89146 9.00535C5.39101 9.95706 5.14795 10.9803 5.14795 11.9821C5.14795 13.127 5.45536 14.2576 6.04159 15.2379C6.62782 16.2254 7.48568 17.0626 8.57236 17.635L9.18714 16.4758Z"
+        fill="currentColor"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M11.6608 15.2165C11.2104 15.2165 10.7815 15.1235 10.3955 14.9589C9.80924 14.7156 9.31596 14.3005 8.96564 13.7782C8.61536 13.2558 8.40804 12.6333 8.40804 11.9606C8.40804 11.5098 8.50095 11.0805 8.66537 10.6941C8.90845 10.1145 9.32309 9.61359 9.84496 9.26297C10.3669 8.91235 10.9888 8.70484 11.6608 8.70484V7.38818C11.0317 7.38818 10.4312 7.517 9.88072 7.74597C9.05858 8.09659 8.36511 8.66905 7.87183 9.39892C7.37142 10.136 7.08545 11.0233 7.08545 11.9678C7.08545 12.5975 7.21412 13.1986 7.4429 13.7496C7.79322 14.5725 8.37228 15.2666 9.10147 15.7603C9.83067 16.2469 10.71 16.5331 11.6608 16.5331V15.2165Z"
+        fill="currentColor"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12.7332 10.9234C12.5831 10.766 12.4187 10.6372 12.2542 10.5442C12.0898 10.4511 11.9183 10.401 11.7395 10.401C11.4965 10.401 11.2891 10.444 11.0961 10.5299C10.9102 10.6157 10.7458 10.7302 10.61 10.8805C10.4741 11.0236 10.374 11.1953 10.3026 11.3814C10.2311 11.5674 10.2025 11.7678 10.2025 11.9681C10.2025 12.1685 10.2382 12.3689 10.3026 12.5549C10.374 12.7409 10.4741 12.9127 10.61 13.0558C10.7458 13.1989 10.9031 13.3206 11.0961 13.4064C11.282 13.4923 11.4965 13.5352 11.7395 13.5352C11.9397 13.5352 12.1327 13.4923 12.3043 13.4136C12.4759 13.3277 12.626 13.2061 12.7475 13.0486L13.6197 13.986C13.491 14.1148 13.3409 14.2293 13.1693 14.3223C12.9978 14.4154 12.8262 14.4941 12.6546 14.5513C12.483 14.6086 12.3114 14.6515 12.1542 14.673C11.9969 14.7016 11.8539 14.7087 11.7395 14.7087C11.3463 14.7087 10.9746 14.6443 10.6314 14.5155C10.2811 14.3868 9.98084 14.2007 9.73064 13.9574C9.47326 13.7213 9.27311 13.4279 9.12298 13.0916C8.97285 12.7553 8.90137 12.376 8.90137 11.9681C8.90137 11.5531 8.97285 11.181 9.12298 10.8447C9.27311 10.5084 9.47326 10.2222 9.73064 9.97887C9.98801 9.74274 10.2883 9.55669 10.6314 9.42075C10.9817 9.29193 11.3535 9.22754 11.7395 9.22754C12.0755 9.22754 12.4115 9.29193 12.7475 9.42075C13.0835 9.54953 13.3838 9.7499 13.634 10.0218L12.7332 10.9234Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function MistralIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M7.28516 3.74658H4.1416V6.8809H7.28516V3.74658Z" />
+      <path d="M19.8579 3.74658H16.7144V6.8809H19.8579V3.74658Z" />
+      <path d="M10.4277 6.88086H4.1416V10.0152H10.4277V6.88086Z" />
+      <path d="M19.8588 6.88086H13.5728V10.0152H19.8588V6.88086Z" />
+      <path d="M19.8564 10.0137H4.1416V13.148H19.8564V10.0137Z" />
+      <path d="M7.28516 13.1484H4.1416V16.2828H7.28516V13.1484Z" />
+      <path d="M13.5723 13.1484H10.4287V16.2828H13.5723V13.1484Z" />
+      <path d="M19.8579 13.1484H16.7144V16.2828H19.8579V13.1484Z" />
+      <path d="M10.4286 16.2812H1V19.4157H10.4286V16.2812Z" />
+      <path d="M23.0024 16.2812H13.5728V19.4157H23.0024V16.2812Z" />
+    </svg>
+  );
+}
+
+export function AzureIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M13.05 4.24L6.56 18.05l2.77-.46 1.89-4.55 4.2 5.19.03-.01 2.99.53L13.05 4.24zm-4.1 7.51L2.49 20.21h5.23l1.23-3.04v-5.42z" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AnimatedHourglassIcon.tsx

@@ -0,0 +1,27 @@
+import { useAnimate } from "framer-motion";
+import { HourglassIcon } from "lucide-react";
+import { useEffect } from "react";
+
+export function AnimatedHourglassIcon({
+  className,
+  delay,
+}: {
+  className?: string;
+  delay?: number;
+}) {
+  const [scope, animate] = useAnimate();
+
+  useEffect(() => {
+    animate(
+      [
+        [scope.current, { rotate: 0 }, { duration: 0.7 }],
+        [scope.current, { rotate: 180 }, { duration: 0.3 }],
+        [scope.current, { rotate: 180 }, { duration: 0.7 }],
+        [scope.current, { rotate: 360 }, { duration: 0.3 }],
+      ],
+      { repeat: Infinity, delay }
+    );
+  }, []);
+
+  return <HourglassIcon ref={scope} className={className} />;
+}

apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx

@@ -0,0 +1,12 @@
+export function AnthropicLogoIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M17.3041 3.541h-3.6718l6.696 16.918H24Zm-10.6082 0L0 20.459h3.7442l1.3693-3.5527h7.0052l1.3693 3.5528h3.7442L10.5363 3.5409Zm-.3712 10.2232 2.2914-5.9456 2.2914 5.9456Z" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ArchiveIcon.tsx

@@ -0,0 +1,44 @@
+export function ArchiveIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_17398_782)">
+        <path
+          d="M19.2293 9C20.1562 9.00001 20.8612 9.83278 20.7088 10.7471L19.2781 19.3291C19.1173 20.2933 18.283 21 17.3055 21H6.69416C5.71662 21 4.88235 20.2933 4.7215 19.3291L3.29084 10.7471C3.13848 9.83289 3.8436 9.00021 4.77033 9H19.2293ZM9.95002 12.5C9.12179 12.5002 8.45002 13.1717 8.45002 14C8.45002 14.8283 9.12179 15.4998 9.95002 15.5H13.95L14.1033 15.4922C14.8597 15.4154 15.45 14.7767 15.45 14C15.45 13.2233 14.8597 12.5846 14.1033 12.5078L13.95 12.5H9.95002Z"
+          fill="currentColor"
+        />
+        <rect x="2" y="3" width="20" height="4" rx="1" fill="currentColor" />
+      </g>
+      <defs>
+        <clipPath id="clip0_17398_782">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}
+
+export function UnarchiveIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_17398_66731)">
+        <path
+          d="M19.2027 10C20.1385 10 20.8456 10.8478 20.6782 11.7686L19.2984 19.3574C19.1254 20.3084 18.2972 21 17.3306 21H6.66945C5.70287 21 4.87458 20.3084 4.70167 19.3574L3.32179 11.7686C3.15438 10.8478 3.86152 10 4.79738 10H10.9995V16C10.9995 16.5521 11.4475 16.9997 11.9995 17C12.5518 17 12.9995 16.5523 12.9995 16V10H19.2027Z"
+          fill="currentColor"
+        />
+        <rect x="11" y="4" width="2" height="6" fill="currentColor" />
+        <path
+          d="M15.5 6.5L12 3L8.5 6.5"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_17398_66731">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ArrowLeftRightIcon.tsx

@@ -0,0 +1,41 @@
+export function ArrowLeftRightIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M5 7.5L18 7.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M15.4 11L18.9 7.5L15.4 4"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M5 16.5L18 16.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M8.5 13L5 16.5L8.5 20"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ArrowRightSquareIcon.tsx

@@ -0,0 +1,33 @@
+export function ArrowRightSquareIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M10 12L19 12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M16.4 15.5L19.9 12L16.4 8.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 8V6C13 4.89543 12.1046 4 11 4H7C5.89543 4 5 4.89543 5 6V18C5 19.1046 5.89543 20 7 20H11C12.1046 20 13 19.1046 13 18V16"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ArrowTopRightBottomLeftIcon.tsx

@@ -0,0 +1,22 @@
+export function ArrowTopRightBottomLeftIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.8258 10.5L20.125 5.20083V8.5625C20.125 9.08027 20.5447 9.5 21.0625 9.5C21.5803 9.5 22 9.08027 22 8.5625V2.9375C22 2.41973 21.5803 2 21.0625 2H15.4375C14.9197 2 14.5 2.41973 14.5 2.9375C14.5 3.45527 14.9197 3.875 15.4375 3.875H18.7992L13.5 9.17417C13.1339 9.54029 13.1339 10.1339 13.5 10.5C13.8661 10.8661 14.4597 10.8661 14.8258 10.5Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2 21.0625V15.4375C2 14.9197 2.41973 14.5 2.9375 14.5C3.45527 14.5 3.875 14.9197 3.875 15.4375V18.7992L9.17417 13.5C9.54029 13.1339 10.1339 13.1339 10.5 13.5C10.8661 13.8661 10.8661 14.4597 10.5 14.8258L5.20083 20.125H8.5625C9.08027 20.125 9.5 20.5447 9.5 21.0625C9.5 21.5803 9.08027 22 8.5625 22H2.9375C2.69757 22 2.45765 21.9085 2.27459 21.7254C2.1847 21.6355 2.11689 21.5319 2.07114 21.4214C2.0253 21.3108 2 21.1896 2 21.0625Z"
+        fill="currentColor"
+      />
+      <path
+        d="M14.8258 10.5L20.125 5.20083V10C20.125 10.5178 20.5447 10.9375 21.0625 10.9375C21.5803 10.9375 22 10.5178 22 10V2.9375C22 2.41973 21.5803 2 21.0625 2H14C13.4822 2 13.0625 2.41973 13.0625 2.9375C13.0625 3.45527 13.4822 3.875 14 3.875H18.7992L13.5 9.17417C13.1339 9.54029 13.1339 10.1339 13.5 10.5C13.8661 10.8661 14.4597 10.8661 14.8258 10.5Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2 21.0625V13.9375C2 13.4197 2.41973 13 2.9375 13C3.45527 13 3.875 13.4197 3.875 13.9375V18.7992L9.17417 13.5C9.54029 13.1339 10.1339 13.1339 10.5 13.5C10.8661 13.8661 10.8661 14.4597 10.5 14.8258L5.20083 20.125H10.0625C10.5803 20.125 11 20.5447 11 21.0625C11 21.5803 10.5803 22 10.0625 22H2.9375C2.69757 22 2.45765 21.9085 2.27459 21.7254C2.1847 21.6355 2.11689 21.5319 2.07114 21.4214C2.0253 21.3108 2 21.1896 2 21.0625Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AttemptIcon.tsx

@@ -0,0 +1,30 @@
+export function AttemptIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect x="3" y="3" width="18" height="18" rx="2" stroke="currentColor" strokeWidth="2" />
+      <rect width="6" height="2" rx="1" transform="matrix(1 0 0 -1 9 15)" fill="currentColor" />
+      <rect
+        width="2.48444"
+        height="2"
+        rx="1"
+        transform="matrix(1 0 0 -1 10.8042 9)"
+        fill="currentColor"
+      />
+      <path
+        d="M12.125 7.06332C11.6119 6.88324 11.0493 7.14809 10.8613 7.65823L7.70722 16.2127C7.52479 16.7075 7.77801 17.2565 8.27281 17.4389C8.7645 17.6202 9.31041 17.3713 9.49607 16.8813L12.7262 8.35528C12.9243 7.83221 12.6528 7.24858 12.125 7.06332Z"
+        fill="currentColor"
+      />
+      <path
+        d="M11.9626 7.06332C12.4757 6.88324 13.0383 7.14809 13.2264 7.65823L16.3804 16.2127C16.5628 16.7075 16.3096 17.2565 15.8148 17.4389C15.3231 17.6202 14.7772 17.3713 14.5915 16.8813L11.3614 8.35528C11.1633 7.83221 11.4349 7.24858 11.9626 7.06332Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AvatarCircleIcon.tsx

@@ -0,0 +1,20 @@
+export function AvatarCircleIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <circle cx="12" cy="12" r="9" stroke="currentColor" strokeWidth="2" />
+      <circle cx="12" cy="9.5" r="2.5" stroke="currentColor" strokeWidth="2" />
+      <path
+        d="M6 19C7.00156 16.6478 9.32233 15 12.0254 15C14.6837 15 16.9724 16.5938 18 18.884"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BatchesIcon.tsx

@@ -0,0 +1,17 @@
+export function BatchesIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect x="3" y="3" width="7" height="7" rx="2" stroke="currentColor" strokeWidth="2" />
+      <rect x="3" y="14" width="7" height="7" rx="2" stroke="currentColor" strokeWidth="2" />
+      <rect x="14" y="3" width="7" height="7" rx="2" stroke="currentColor" strokeWidth="2" />
+      <rect x="14" y="14" width="7" height="7" rx="2" stroke="currentColor" strokeWidth="2" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BeakerIcon.tsx

@@ -0,0 +1,30 @@
+export function BeakerIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M9 10V4H15V10C16.3896 11.737 19.5 15.0137 19.5 17.4324C19.5 19.4768 17.5444 21 15.5 21H8.5C6.45569 21 4.5 19.5444 4.5 17.5C4.5 15.0813 7.6104 11.737 9 10Z"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <line
+        x1="8"
+        y1="4"
+        x2="16"
+        y2="4"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+      />
+      <line x1="6" y1="14" x2="18" y2="14" stroke="currentColor" strokeWidth="2" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BellIcon.tsx

@@ -0,0 +1,23 @@
+export function BellIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M15 18C15 19.6569 13.6569 21 12 21C10.3431 21 9 19.6569 9 18"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M12 3C15.4217 3 18.2705 5.63978 18.5762 9.06641C18.7369 10.8684 18.9014 12.5444 19.0156 13.1777C19.1051 13.6742 19.2358 14.3446 19.373 15.0332L19.7715 16.9971C19.7714 16.9973 19.7713 16.9981 19.7705 16.999L19.7695 17H4.23047L4.22949 16.999L4.22852 16.9971C4.46222 15.8599 4.80534 14.1705 4.98438 13.1777C5.09858 12.5444 5.26306 10.8684 5.42383 9.06641C5.72954 5.63978 8.57828 3 12 3Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BookIcon.tsx

@@ -0,0 +1,23 @@
+export function BookIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M3 5.11726C3 4.52125 3.51807 4.05756 4.11043 4.12338L11.1104 4.90116C11.6169 4.95743 12 5.38549 12 5.89504V18.4414C12 18.7394 11.741 18.9712 11.4448 18.9383L3.88957 18.0988C3.38313 18.0426 3 17.6145 3 17.105V5.11726Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M12 5.89504C12 5.38549 12.3831 4.95743 12.8896 4.90116L19.8896 4.12338C20.4819 4.05756 21 4.52125 21 5.11727V17.105C21 17.6145 20.6169 18.0426 20.1104 18.0988L12.5552 18.9383C12.259 18.9712 12 18.7394 12 18.4414V5.89504Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/Box3DIcon.tsx

@@ -0,0 +1,22 @@
+export function Box3DIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M11.5352 3.37305C11.8262 3.22036 12.174 3.22033 12.4649 3.37305L19.9913 7.32324C20.3204 7.49599 20.5264 7.83734 20.5265 8.20898V15.791C20.5264 16.1627 20.3204 16.504 19.9913 16.6768L12.4649 20.627C12.174 20.7797 11.8262 20.7796 11.5352 20.627L4.00888 16.6768C3.67978 16.504 3.47374 16.1627 3.47372 15.791V8.20898C3.47374 7.83731 3.67978 7.49598 4.00888 7.32324L11.5352 3.37305Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M4.44721 7.10557C3.95324 6.85858 3.35256 7.05881 3.10557 7.55279C2.85858 8.04676 3.05881 8.64744 3.55279 8.89443L4 8L4.44721 7.10557ZM12 20H13V12H12H11V20H12ZM12 12L12.4472 12.8944L20.4472 8.89443L20 8L19.5528 7.10557L11.5528 11.1056L12 12ZM4 8L3.55279 8.89443L11.5528 12.8944L12 12L12.4472 11.1056L4.44721 7.10557L4 8Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BugIcon.tsx

@@ -0,0 +1,73 @@
+export function BugIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M18 17L21 19"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M6 17L3 19"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M18 13H21"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M18 9L21 7"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M8 8H16C17.1046 8 18 8.89543 18 10V15C18 18.3137 15.3137 21 12 21C8.68629 21 6 18.3137 6 15V10C6 8.89543 6.89543 8 8 8Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M3 13H6"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M12 21V14"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M16 7C16 4.79086 14.2091 3 12 3C9.79086 3 8 4.79086 8 7"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+      />
+      <path
+        d="M6 9L3 7"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BulbIcon.tsx

@@ -0,0 +1,19 @@
+export function BulbIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M15 19V17V16.9584C15 16.5724 15.2256 16.2261 15.5579 16.0296C17.6182 14.8113 19 12.567 19 10C19 6.13401 15.866 3 12 3C8.13401 3 5 6.13401 5 10C5 12.567 6.38176 14.8113 8.44208 16.0296C8.77437 16.2261 9 16.5724 9 16.9584V17V19C9 20.1046 9.89543 21 11 21H13C14.1046 21 15 20.1046 15 19Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <line x1="9" y1="17" x2="15" y2="17" stroke="currentColor" strokeWidth="2" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/BunLogoIcon.tsx

@@ -0,0 +1,94 @@
+export function BunLogoIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 80 70" xmlns="http://www.w3.org/2000/svg">
+      <path
+        id="Shadow"
+        d="M71.09,20.74c-.16-.17-.33-.34-.5-.5s-.33-.34-.5-.5-.33-.34-.5-.5-.33-.34-.5-.5-.33-.34-.5-.5-.33-.34-.5-.5-.33-.34-.5-.5A26.46,26.46,0,0,1,75.5,35.7c0,16.57-16.82,30.05-37.5,30.05-11.58,0-21.94-4.23-28.83-10.86l.5.5.5.5.5.5.5.5.5.5.5.5.5.5C19.55,65.3,30.14,69.75,42,69.75c20.68,0,37.5-13.48,37.5-30C79.5,32.69,76.46,26,71.09,20.74Z"
+      />
+      <g id="Body">
+        <path
+          id="Background"
+          d="M73,35.7c0,15.21-15.67,27.54-35,27.54S3,50.91,3,35.7C3,26.27,9,17.94,18.22,13S33.18,3,38,3s8.94,4.13,19.78,10C67,17.94,73,26.27,73,35.7Z"
+          style={{ fill: "#fbf0df" }}
+        />
+        <path
+          id="Bottom_Shadow"
+          data-name="Bottom Shadow"
+          d="M73,35.7a21.67,21.67,0,0,0-.8-5.78c-2.73,33.3-43.35,34.9-59.32,24.94A40,40,0,0,0,38,63.24C57.3,63.24,73,50.89,73,35.7Z"
+          style={{ fill: "#f6dece" }}
+        />
+        <path
+          id="Light_Shine"
+          data-name="Light Shine"
+          d="M24.53,11.17C29,8.49,34.94,3.46,40.78,3.45A9.29,9.29,0,0,0,38,3c-2.42,0-5,1.25-8.25,3.13-1.13.66-2.3,1.39-3.54,2.15-2.33,1.44-5,3.07-8,4.7C8.69,18.13,3,26.62,3,35.7c0,.4,0,.8,0,1.19C9.06,15.48,20.07,13.85,24.53,11.17Z"
+          style={{ fill: "#fffefc" }}
+        />
+        <path
+          id="Top"
+          d="M35.12,5.53A16.41,16.41,0,0,1,29.49,18c-.28.25-.06.73.3.59,3.37-1.31,7.92-5.23,6-13.14C35.71,5,35.12,5.12,35.12,5.53Zm2.27,0A16.24,16.24,0,0,1,39,19c-.12.35.31.65.55.36C41.74,16.56,43.65,11,37.93,5,37.64,4.74,37.19,5.14,37.39,5.49Zm2.76-.17A16.42,16.42,0,0,1,47,17.12a.33.33,0,0,0,.65.11c.92-3.49.4-9.44-7.17-12.53C40.08,4.54,39.82,5.08,40.15,5.32ZM21.69,15.76a16.94,16.94,0,0,0,10.47-9c.18-.36.75-.22.66.18-1.73,8-7.52,9.67-11.12,9.45C21.32,16.4,21.33,15.87,21.69,15.76Z"
+          style={{ fill: "#ccbea7", fillRule: "evenodd" }}
+        />
+        <path
+          id="Outline"
+          d="M38,65.75C17.32,65.75.5,52.27.5,35.7c0-10,6.18-19.33,16.53-24.92,3-1.6,5.57-3.21,7.86-4.62,1.26-.78,2.45-1.51,3.6-2.19C32,1.89,35,.5,38,.5s5.62,1.2,8.9,3.14c1,.57,2,1.19,3.07,1.87,2.49,1.54,5.3,3.28,9,5.27C69.32,16.37,75.5,25.69,75.5,35.7,75.5,52.27,58.68,65.75,38,65.75ZM38,3c-2.42,0-5,1.25-8.25,3.13-1.13.66-2.3,1.39-3.54,2.15-2.33,1.44-5,3.07-8,4.7C8.69,18.13,3,26.62,3,35.7,3,50.89,18.7,63.25,38,63.25S73,50.89,73,35.7C73,26.62,67.31,18.13,57.78,13,54,11,51.05,9.12,48.66,7.64c-1.09-.67-2.09-1.29-3-1.84C42.63,4,40.42,3,38,3Z"
+        />
+      </g>
+      <g id="Mouth">
+        <g id="Background-2" data-name="Background">
+          <path
+            d="M45.05,43a8.93,8.93,0,0,1-2.92,4.71,6.81,6.81,0,0,1-4,1.88A6.84,6.84,0,0,1,34,47.71,8.93,8.93,0,0,1,31.12,43a.72.72,0,0,1,.8-.81H44.26A.72.72,0,0,1,45.05,43Z"
+            style={{ fill: "#b71422" }}
+          />
+        </g>
+        <g id="Tongue">
+          <path
+            id="Background-3"
+            data-name="Background"
+            d="M34,47.79a6.91,6.91,0,0,0,4.12,1.9,6.91,6.91,0,0,0,4.11-1.9,10.63,10.63,0,0,0,1-1.07,6.83,6.83,0,0,0-4.9-2.31,6.15,6.15,0,0,0-5,2.78C33.56,47.4,33.76,47.6,34,47.79Z"
+            style={{ fill: "#ff6164" }}
+          />
+          <path
+            id="Outline-2"
+            data-name="Outline"
+            d="M34.16,47a5.36,5.36,0,0,1,4.19-2.08,6,6,0,0,1,4,1.69c.23-.25.45-.51.66-.77a7,7,0,0,0-4.71-1.93,6.36,6.36,0,0,0-4.89,2.36A9.53,9.53,0,0,0,34.16,47Z"
+          />
+        </g>
+        <path
+          id="Outline-3"
+          data-name="Outline"
+          d="M38.09,50.19a7.42,7.42,0,0,1-4.45-2,9.52,9.52,0,0,1-3.11-5.05,1.2,1.2,0,0,1,.26-1,1.41,1.41,0,0,1,1.13-.51H44.26a1.44,1.44,0,0,1,1.13.51,1.19,1.19,0,0,1,.25,1h0a9.52,9.52,0,0,1-3.11,5.05A7.42,7.42,0,0,1,38.09,50.19Zm-6.17-7.4c-.16,0-.2.07-.21.09a8.29,8.29,0,0,0,2.73,4.37A6.23,6.23,0,0,0,38.09,49a6.28,6.28,0,0,0,3.65-1.73,8.3,8.3,0,0,0,2.72-4.37.21.21,0,0,0-.2-.09Z"
+        />
+      </g>
+      <g id="Face">
+        <ellipse
+          id="Right_Blush"
+          data-name="Right Blush"
+          cx="53.22"
+          cy="40.18"
+          rx="5.85"
+          ry="3.44"
+          style={{ fill: "#febbd0" }}
+        />
+        <ellipse
+          id="Left_Bluch"
+          data-name="Left Bluch"
+          cx="22.95"
+          cy="40.18"
+          rx="5.85"
+          ry="3.44"
+          style={{ fill: "#febbd0" }}
+        />
+        <path
+          id="Eyes"
+          d="M25.7,38.8a5.51,5.51,0,1,0-5.5-5.51A5.51,5.51,0,0,0,25.7,38.8Zm24.77,0A5.51,5.51,0,1,0,45,33.29,5.5,5.5,0,0,0,50.47,38.8Z"
+          style={{ fillRule: "evenodd" }}
+        />
+        <path
+          id="Iris"
+          d="M24,33.64a2.07,2.07,0,1,0-2.06-2.07A2.07,2.07,0,0,0,24,33.64Zm24.77,0a2.07,2.07,0,1,0-2.06-2.07A2.07,2.07,0,0,0,48.75,33.64Z"
+          style={{ fill: "#fff", fillRule: "evenodd" }}
+        />
+      </g>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChartArrowIcon.tsx

@@ -0,0 +1,42 @@
+export function ChartArrowIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M9 12C9 10.8954 9.89543 10 11 10H13C14.1046 10 15 10.8954 15 12V21H9V12Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M15 7C15 5.89543 15.8954 5 17 5H19C20.1046 5 21 5.89543 21 7V20C21 20.5523 20.5523 21 20 21H15V7Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M3 17C3 15.8954 3.89543 15 5 15H7C8.10457 15 9 15.8954 9 17V21H4C3.44772 21 3 20.5523 3 20V17Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M3 5.5L8 5.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M7.4 8L9.9 5.50005L7.39995 3"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChartBarIcon.tsx

@@ -0,0 +1,28 @@
+export function ChartBarIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M9 5C9 3.89543 9.89543 3 11 3H13C14.1046 3 15 3.89543 15 5V21H9V5Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M15 13C15 11.8954 15.8954 11 17 11H19C20.1046 11 21 11.8954 21 13V20C21 20.5523 20.5523 21 20 21H15V13Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path
+        d="M3 10C3 8.89543 3.89543 8 5 8H7C8.10457 8 9 8.89543 9 10V21H4C3.44772 21 3 20.5523 3 20V10Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChevronExtraSmallDown.tsx

@@ -0,0 +1,13 @@
+export function ChevronExtraSmallDown({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M15 6L9.75926 12.1142C9.36016 12.5798 8.63984 12.5798 8.24074 12.1142L3 6"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeMiterlimit="1.00244"
+        strokeLinecap="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChevronExtraSmallUp.tsx

@@ -0,0 +1,13 @@
+export function ChevronExtraSmallUp({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 12L8.24074 5.8858C8.63984 5.42019 9.36016 5.42019 9.75926 5.8858L15 12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeMiterlimit="1.00244"
+        strokeLinecap="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ClockIcon.tsx

@@ -0,0 +1,24 @@
+export function ClockIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <circle cx="12" cy="12" r="9" stroke="currentColor" strokeWidth="2" />
+      <line
+        x1="12"
+        y1="7"
+        x2="12"
+        y2="12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+      />
+      <path d="M15.5 15.5L12 12" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ClockRotateLeftIcon.tsx

@@ -0,0 +1,15 @@
+export function ClockRotateLeftIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M4.01784 10.9999C4.27072 9.07068 5.21806 7.29972 6.68252 6.01856C8.14697 4.73741 10.0282 4.03389 11.9739 4.03971C13.9197 4.04553 15.7966 4.76028 17.2534 6.05017C18.7101 7.34006 19.6469 9.11666 19.8882 11.0474C20.1296 12.9781 19.659 14.9306 18.5645 16.5394C17.4701 18.1482 15.8268 19.303 13.9424 19.7876C12.0579 20.2722 10.0615 20.0534 8.32671 19.1721C6.59196 18.2909 5.23784 16.8076 4.51784 14.9999M4.01784 19.9999L4.01784 14.9999L9.01784 14.9999"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path d="M12 12L12 8" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
+      <path d="M12 12L14 14" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/CloudProviderIcon.tsx

@@ -0,0 +1,76 @@
+export function CloudProviderIcon({
+  provider,
+  className,
+}: {
+  provider: "aws" | "digitalocean" | (string & {});
+  className?: string;
+}) {
+  switch (provider) {
+    case "aws":
+      return <AWS className={className} />;
+    case "digitalocean":
+      return <DigitalOcean className={className} />;
+    default:
+      return null;
+  }
+}
+
+export function AWS({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 240 240" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M67.6343 100.132C67.5545 102.527 67.8738 104.921 68.5123 107.236C69.2307 109.232 70.1088 111.147 71.0666 112.983C71.3859 113.542 71.6254 114.181 71.6254 114.819C71.5455 115.777 70.9868 116.735 70.1088 117.214L65.08 120.566C64.4414 120.965 63.723 121.285 63.0046 121.285C62.1266 121.205 61.2486 120.806 60.61 120.167C59.5723 119.05 58.5346 117.773 57.7364 116.416C56.9382 115.059 56.14 113.542 55.2619 111.706C49.0359 119.05 41.2134 122.722 31.7944 122.722C25.0894 122.722 19.7414 120.806 15.8301 116.974C11.9189 113.143 9.92334 108.034 9.92334 101.649C9.92334 94.8638 12.318 89.3561 17.1871 85.2054C22.0562 81.0547 28.5217 78.9793 36.7434 78.9793C39.5371 78.9793 42.4107 79.2188 45.2044 79.6179C48.1578 80.017 51.1911 80.6556 54.3839 81.374V75.547C54.3839 69.4806 53.1068 65.25 50.6323 62.7756C48.1578 60.3011 43.7677 59.1038 37.6214 59.1038C34.7478 59.1038 31.7944 59.4231 29.0007 60.1415C26.0473 60.8598 23.1737 61.7379 20.38 62.8554C19.5019 63.2545 18.5441 63.5738 17.5862 63.8931C17.1871 64.0527 16.7082 64.1325 16.3091 64.1325C15.1916 64.1325 14.6328 63.3343 14.6328 61.6581V57.7468C14.553 56.7889 14.7925 55.8311 15.1916 54.9531C15.8301 54.2347 16.5485 53.6759 17.4266 53.2768C20.6194 51.6804 23.9719 50.483 27.4841 49.6848C31.555 48.6472 35.7855 48.1682 39.9362 48.1682C49.435 48.1682 56.3795 50.3234 60.8495 54.6338C65.3195 58.9441 67.4746 65.4895 67.4746 74.2699V100.132H67.6343ZM35.2268 112.265C38.0205 112.265 40.8143 111.786 43.4484 110.828C46.4018 109.79 48.9561 108.034 51.0314 105.72C52.3086 104.283 53.2664 102.527 53.7453 100.611C54.3041 98.376 54.5436 95.9813 54.5436 93.6665V90.314C52.1489 89.7552 49.6744 89.2763 47.2 88.957C44.7255 88.6377 42.1712 88.4781 39.6968 88.4781C34.3487 88.4781 30.4375 89.5158 27.8034 91.6709C25.1692 93.8261 23.8921 96.8593 23.8921 100.85C23.8921 104.602 24.85 107.396 26.8455 109.312C28.841 111.227 31.555 112.265 35.2268 112.265ZM99.3234 120.886C98.2857 120.965 97.1682 120.726 96.2902 120.087C95.492 119.209 94.8534 118.172 94.614 116.974L75.8559 55.2723C75.4568 54.2347 75.2173 53.197 75.1375 52.0795C75.0577 51.1216 75.6963 50.2436 76.6541 50.0839H84.9556C86.4722 50.0839 87.5098 50.3234 88.0686 50.8822C88.6273 51.4409 89.1861 52.4786 89.665 53.9952L102.995 106.837L115.447 53.9952C115.846 52.3988 116.325 51.3611 116.964 50.8822C117.922 50.3234 119.039 50.0041 120.157 50.0839H126.543C128.059 50.0839 129.097 50.3234 129.735 50.8822C130.534 51.7602 131.092 52.7979 131.252 53.9952L143.864 107.476L157.673 53.9952C157.912 52.7979 158.471 51.7602 159.269 50.8822C160.227 50.3234 161.265 50.0041 162.382 50.0839H169.806C170.764 49.9243 171.642 50.6427 171.801 51.6005V52.0795C171.801 52.4786 171.722 52.9575 171.642 53.3566C171.482 54.075 171.322 54.7136 171.083 55.3522L151.846 117.054C151.367 118.651 150.808 119.688 150.17 120.167C149.292 120.726 148.174 121.045 147.137 120.965H140.272C138.755 120.965 137.718 120.726 137.079 120.167C136.281 119.289 135.722 118.172 135.562 116.974L123.19 65.4895L110.898 116.895C110.738 118.092 110.179 119.209 109.381 120.087C108.423 120.726 107.306 120.965 106.188 120.886H99.3234ZM201.894 123.041C197.743 123.041 193.593 122.562 189.602 121.604C185.61 120.646 182.497 119.608 180.422 118.411C179.384 117.932 178.506 117.134 177.948 116.176C177.628 115.458 177.469 114.739 177.469 113.941V109.87C177.469 108.194 178.107 107.396 179.305 107.396C179.784 107.396 180.262 107.476 180.741 107.635C181.22 107.795 181.939 108.114 182.737 108.433C185.531 109.711 188.484 110.589 191.517 111.227C194.63 111.866 197.823 112.185 201.016 112.185C206.045 112.185 209.956 111.307 212.67 109.551C215.304 107.955 216.901 105.081 216.821 101.968C216.901 99.8926 216.102 97.897 214.665 96.3804C213.229 94.8638 210.515 93.5068 206.604 92.2297L195.029 88.6377C189.202 86.8018 184.892 84.0879 182.258 80.4959C179.704 77.1434 178.267 73.1524 178.267 68.9218C178.187 65.8088 178.905 62.7756 180.422 60.0616C181.859 57.5073 183.775 55.2723 186.169 53.5163C188.644 51.6005 191.437 50.2436 194.471 49.3655C197.743 48.4077 201.096 47.9288 204.528 48.0086C206.284 48.0086 208.12 48.0884 209.876 48.3279C211.712 48.5673 213.388 48.8866 215.065 49.2059C216.741 49.5252 218.178 50.0041 219.614 50.483C220.812 50.8822 221.929 51.3611 222.967 51.9198C223.925 52.3988 224.723 53.0373 225.362 53.9154C225.92 54.7136 226.16 55.5916 226.08 56.5495V60.3011C226.08 61.9773 225.441 62.8554 224.244 62.8554C223.206 62.6957 222.169 62.3765 221.211 61.8975C216.422 59.7423 211.153 58.7047 205.885 58.7845C201.335 58.7845 197.743 59.5029 195.269 61.0195C192.794 62.5361 191.517 64.8509 191.517 68.1236C191.437 70.2788 192.315 72.3541 193.912 73.7909C195.508 75.3075 198.462 76.8241 202.692 78.1811L214.027 81.7731C219.774 83.609 223.925 86.1633 226.399 89.4359C228.874 92.6288 230.151 96.6199 230.071 100.611C230.151 103.804 229.432 106.997 227.996 109.87C226.559 112.584 224.563 114.979 222.169 116.895C219.535 118.97 216.501 120.487 213.309 121.365C209.637 122.482 205.805 123.041 201.894 123.041Z"
+        fill="white"
+      />
+      <path
+        d="M216.98 161.834C190.719 181.231 152.564 191.528 119.758 191.528C73.7806 191.528 32.3533 174.526 1.06324 146.269C-1.41123 144.034 0.823772 141.001 3.77717 142.757C37.6215 162.393 79.3681 174.286 122.552 174.286C153.682 174.126 184.493 167.821 213.149 155.768C217.539 153.772 221.291 158.641 216.98 161.834Z"
+        fill="#FF9900"
+      />
+      <path
+        d="M227.916 149.382C224.563 145.072 205.726 147.307 197.185 148.344C194.63 148.664 194.231 146.428 196.546 144.752C211.553 134.216 236.217 137.249 239.091 140.761C241.965 144.273 238.293 169.018 224.244 180.832C222.089 182.667 220.014 181.71 220.971 179.315C224.164 171.413 231.268 153.612 227.916 149.382Z"
+        fill="#FF9900"
+      />
+    </svg>
+  );
+}
+
+export function DigitalOcean({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 240 240" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_18907_83670)">
+        <path
+          d="M120.001 218.82V180.552C160.62 180.552 192.015 140.347 176.514 97.6503C170.768 81.8384 158.157 69.2205 142.339 63.4808C99.6439 47.9862 59.4601 79.3824 59.4532 119.988L59.4463 120.002H21.1802C21.1802 55.292 83.6945 4.91045 151.493 26.0827C181.122 35.327 204.667 58.8724 213.918 88.5025C235.089 156.31 184.703 218.82 120.001 218.82Z"
+          fill="#0069FF"
+        />
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M120.089 180.647H81.9333V142.497L81.9402 142.49H120.082L120.089 142.497V180.647Z"
+          fill="#0069FF"
+        />
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M81.925 209.964H52.6132L52.6063 209.957V180.644H81.9319V209.957L81.925 209.964Z"
+          fill="#0069FF"
+        />
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M52.6298 180.647H28.0573L28.0435 180.64V156.081L28.0573 156.06H52.616L52.6298 156.067V180.647Z"
+          fill="#0069FF"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_18907_83670">
+          <rect
+            width="197.64"
+            height="197.64"
+            fill="white"
+            transform="translate(21.1802 21.1802)"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/CodeSquareIcon.tsx

@@ -0,0 +1,28 @@
+export function CodeSquareIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect x="3" y="4" width="18" height="16" rx="2" stroke="currentColor" strokeWidth="2" />
+      <path
+        d="M10 9L7 12L10 15"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M14 15L17 12L14 9"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ConcurrencyIcon.tsx

@@ -0,0 +1,13 @@
+export function ConcurrencyIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <circle cx="3.75" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="14.25" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="3.75" cy="9" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="9" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="14.25" r="1.75" stroke="currentColor" />
+      <circle cx="14.25" cy="9" r="2.25" fill="currentColor" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ConnectionIcons.tsx

@@ -0,0 +1,73 @@
+export function ConnectedIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <rect
+        x="0.5"
+        y="-0.5"
+        width="19"
+        height="1"
+        rx="0.5"
+        transform="matrix(1 0 0 -1 1.99998 20)"
+        stroke="#878C99"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M14.4187 5L6.00002 5C4.89545 5 4.00002 5.89543 4.00002 7L4.00001 15C4.00002 16.1046 4.89545 17 6.00002 17L18 17C19.1046 17 20 16.1046 20 15L20 7.9816L18.5 9.4816L18.5 15C18.5 15.2761 18.2762 15.5 18 15.5L6.00001 15.5C5.72387 15.5 5.50001 15.2761 5.50001 15L5.50002 7C5.50002 6.72386 5.72387 6.5 6.00002 6.5L12.9187 6.5L14.4187 5Z"
+        fill="#878C99"
+      />
+      <path
+        d="M8.50002 9.75L11.5 12.75L18 6"
+        stroke="#28BF5C"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}
+
+export function DisconnectedIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M14.673 19L2.99998 19C2.44769 19 1.99998 19.4477 1.99998 20C1.99998 20.5523 2.44769 21 2.99998 21L16.673 21L14.673 19Z"
+        fill="#878C99"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M3.99999 8.32703L3.99999 15C3.99999 16.1046 4.89542 17 5.99999 17L12.673 17L11.173 15.5L5.99999 15.5C5.72385 15.5 5.49999 15.2761 5.49999 15L5.49999 9.82703L3.99999 8.32703ZM18.5 14.2641L18.5 7C18.5 6.72386 18.2761 6.5 18 6.5L10.7358 6.5L9.23585 5L18 5C19.1046 5 20 5.89543 20 7L20 15C20 15.2292 19.9614 15.4495 19.8904 15.6546L18.5 14.2641Z"
+        fill="#878C99"
+      />
+      <path d="M3.00001 3L21 21" stroke="#E11D48" strokeWidth="1.5" strokeLinecap="round" />
+    </svg>
+  );
+}
+
+export function CheckingConnectionIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <rect
+        x="0.5"
+        y="-0.5"
+        width="19"
+        height="1"
+        rx="0.5"
+        transform="matrix(1 0 0 -1 1.99998 20)"
+        stroke="#878C99"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M18 6.5L6.00001 6.5C5.72387 6.5 5.50001 6.72386 5.50001 7L5.50001 15C5.50001 15.2761 5.72387 15.5 6.00001 15.5L18 15.5C18.2762 15.5 18.5 15.2761 18.5 15V7C18.5 6.72386 18.2762 6.5 18 6.5ZM6.00001 5C4.89545 5 4.00001 5.89543 4.00001 7L4.00001 15C4.00001 16.1046 4.89545 17 6.00001 17L18 17C19.1046 17 20 16.1046 20 15V7C20 5.89543 19.1046 5 18 5L6.00001 5Z"
+        fill="#878C99"
+      />
+      <circle cx="9" cy="11" r="1" fill="#878C99" />
+      <circle cx="12" cy="11" r="1" fill="#878C99" />
+      <circle cx="15" cy="11" r="1" fill="#878C99" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/CreditCardIcon.tsx

@@ -0,0 +1,16 @@
+export function CreditCardIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect x="3" y="5" width="18" height="14" rx="2" stroke="currentColor" strokeWidth="2" />
+      <rect x="4" y="9" width="16" height="2" fill="currentColor" />
+      <rect x="6" y="13" width="7" height="2" rx="1" fill="currentColor" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/CubeSparkleIcon.tsx

@@ -0,0 +1,24 @@
+export function CubeSparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M12 7C12.4304 7 12.8126 7.27543 12.9487 7.68378L13.4743 9.26076C13.6734 9.85797 14.142 10.3266 14.7393 10.5257L16.3162 11.0513C16.7246 11.1875 17 11.5696 17 12C17 12.4305 16.7246 12.8126 16.3162 12.9487L14.7393 13.4744C14.142 13.6734 13.6734 14.1421 13.4743 14.7393L12.9487 16.3163C12.8126 16.7246 12.4304 17 12 17C11.5696 17 11.1874 16.7246 11.0513 16.3163L10.5256 14.7393C10.3266 14.1421 9.85794 13.6734 9.26073 13.4744L7.68374 12.9487C7.2754 12.8126 6.99997 12.4305 6.99997 12C6.99997 11.5696 7.2754 11.1875 7.68374 11.0513L9.26073 10.5257C9.85794 10.3266 10.3266 9.85797 10.5256 9.26076L11.0513 7.68378C11.1874 7.27543 11.5696 7 12 7Z"
+        fill="currentColor"
+      />
+      <path
+        d="M11 2.18502C11.6188 1.82775 12.3812 1.82775 13 2.18502L20 6.22647C20.6188 6.58373 21 7.24399 21 7.95852V16.0414C21 16.756 20.6188 17.4162 20 17.7735L13 21.8149C12.3812 22.1722 11.6188 22.1722 11 21.8149L4 17.7735C3.3812 17.4162 3 16.756 3 16.0414V7.95852C3 7.24399 3.3812 6.58373 4 6.22647L11 2.18502Z"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+    </svg>
+  );
+}