chore: import upstream snapshot with attribution

apps/webapp/app/v3/GCRARateLimiter.server.ts

@@ -0,0 +1,173 @@
+import type { Cluster } from "ioredis";
+import type Redis from "ioredis";
+
+/**
+ * Options for configuring the RateLimiter.
+ */
+export interface GCRARateLimiterOptions {
+  /** An instance of ioredis. */
+  redis: Redis | Cluster;
+  /**
+   * A string prefix to namespace keys in Redis.
+   * Defaults to "ratelimit:".
+   */
+  keyPrefix?: string;
+  /**
+   * The minimum interval between requests (the emission interval) in milliseconds.
+   * For example, 1000 ms for one request per second.
+   */
+  emissionInterval: number;
+  /**
+   * The burst tolerance in milliseconds. This represents how much “credit” can be
+   * accumulated to allow short bursts beyond the average rate.
+   * For example, if you want to allow 3 requests in a burst with an emission interval of 1000 ms,
+   * you might set this to 3000.
+   */
+  burstTolerance: number;
+  /**
+   * Expiration for the Redis key in milliseconds.
+   * Defaults to the larger of 60 seconds or (emissionInterval + burstTolerance).
+   */
+  keyExpiration?: number;
+}
+
+/**
+ * The result of a rate limit check.
+ */
+export interface RateLimitResult {
+  /** Whether the request is allowed. */
+  allowed: boolean;
+  /**
+   * If not allowed, this is the number of milliseconds the caller should wait
+   * before retrying.
+   */
+  retryAfter?: number;
+}
+
+/**
+ * A rate limiter using Redis and the Generic Cell Rate Algorithm (GCRA).
+ *
+ * The GCRA is implemented using a Lua script that runs atomically in Redis.
+ *
+ * When a request comes in, the algorithm:
+ *  - Retrieves the current "Theoretical Arrival Time" (TAT) from Redis (or initializes it if missing).
+ *  - If the current time is greater than or equal to the TAT, the request is allowed and the TAT is updated to now + emissionInterval.
+ *  - Otherwise, if the current time plus the burst tolerance is at least the TAT, the request is allowed and the TAT is incremented.
+ *  - If neither condition is met, the request is rejected and a Retry-After value is returned.
+ */
+export class GCRARateLimiter {
+  private redis: Redis | Cluster;
+  private keyPrefix: string;
+  private emissionInterval: number;
+  private burstTolerance: number;
+  private keyExpiration: number;
+
+  constructor(options: GCRARateLimiterOptions) {
+    this.redis = options.redis;
+    this.keyPrefix = options.keyPrefix || "gcra:ratelimit:";
+    this.emissionInterval = options.emissionInterval;
+    this.burstTolerance = options.burstTolerance;
+    // Default expiration: at least 60 seconds or the sum of emissionInterval and burstTolerance
+    this.keyExpiration =
+      options.keyExpiration || Math.max(60_000, this.emissionInterval + this.burstTolerance);
+
+    // Define a custom Redis command 'gcra' that implements the GCRA algorithm.
+    // Using defineCommand ensures the Lua script is loaded once and run atomically.
+    this.redis.defineCommand("gcra", {
+      numberOfKeys: 1,
+      lua: `
+--[[
+  GCRA Lua script
+  KEYS[1]         - The rate limit key (e.g. "ratelimit:<identifier>")
+  ARGV[1]         - Current time in ms (number)
+  ARGV[2]         - Emission interval in ms (number)
+  ARGV[3]         - Burst tolerance in ms (number)
+  ARGV[4]         - Key expiration in ms (number)
+  
+  Returns: { allowedFlag, value }
+    allowedFlag: 1 if allowed, 0 if rate-limited.
+    value: 0 when allowed; if not allowed, the number of ms to wait.
+]]--
+
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local emission_interval = tonumber(ARGV[2])
+local burst_tolerance = tonumber(ARGV[3])
+local expire = tonumber(ARGV[4])
+
+-- Get the stored Theoretical Arrival Time (TAT) or default to 0.
+local tat = tonumber(redis.call("GET", key) or 0)
+if tat == 0 then
+  tat = now
+end
+
+local allowed, new_tat, retry_after
+
+if now >= tat then
+  -- No delay: request is on schedule.
+  new_tat = now + emission_interval
+  allowed = true
+elseif (now + burst_tolerance) >= tat then
+  -- Within burst capacity: allow request.
+  new_tat = tat + emission_interval
+  allowed = true
+else
+  -- Request exceeds the allowed burst; calculate wait time.
+  allowed = false
+  retry_after = tat - (now + burst_tolerance)
+end
+
+if allowed then
+  redis.call("SET", key, new_tat, "PX", expire)
+  return {1, 0}
+else
+  return {0, retry_after}
+end
+`,
+    });
+  }
+
+  /**
+   * Checks whether a request associated with the given identifier is allowed.
+   *
+   * @param identifier A unique string identifying the subject of rate limiting (e.g. user ID, IP address, or domain).
+   * @returns A promise that resolves to a RateLimitResult.
+   *
+   * @example
+   * const result = await rateLimiter.check('user:12345');
+   * if (!result.allowed) {
+   *   // Tell the client to retry after result.retryAfter milliseconds.
+   * }
+   */
+  async check(identifier: string): Promise<RateLimitResult> {
+    const key = `${this.keyPrefix}${identifier}`;
+    const now = Date.now();
+
+    try {
+      // Call the custom 'gcra' command.
+      // The script returns an array: [allowedFlag, value]
+      //   - allowedFlag: 1 if allowed; 0 if rejected.
+      //   - value: 0 when allowed; if rejected, the number of ms to wait before retrying.
+      // @ts-expect-error: The custom command is defined via defineCommand.
+      const result: [number, number] = await this.redis.gcra(
+        key,
+        now,
+        this.emissionInterval,
+        this.burstTolerance,
+        this.keyExpiration
+      );
+      const allowed = result[0] === 1;
+      if (allowed) {
+        return { allowed: true };
+      } else {
+        return { allowed: false, retryAfter: result[1] };
+      }
+      // eslint-disable-next-line no-useless-catch -- catch documents the fail-open/fail-closed policy
+    } catch (error) {
+      // In a production system you might log the error and either
+      // allow the request (fail open) or deny it (fail closed).
+      // Here we choose to propagate the error.
+      throw error;
+    }
+  }
+}

apps/webapp/app/v3/accountsWebhookWorker.server.ts

@@ -0,0 +1,94 @@
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { ssoController } from "~/services/sso.server";
+import { applyDirectorySyncEffects } from "~/services/directorySyncEffects.server";
+
+// Dedicated worker for inbound account-management webhooks. The webhook
+// proxy route verifies the signature via the plugin and enqueues the
+// parsed event here; this worker calls back into the plugin to apply the
+// DB writes. The plugin owns the vendor-specific logic; the webapp owns
+// the queue runtime (this file), mirroring `commonWorker.server.ts`.
+//
+// Vendor-neutral by design: the catalog/job names and payload shape carry
+// no provider identity.
+const PayloadSchema = z
+  .object({
+    id: z.string(),
+    event: z.string(),
+    data: z.unknown(),
+  })
+  // Zod v3 treats a `z.unknown()` field as optional, so a payload missing
+  // `data` entirely would otherwise validate. Require the key's presence
+  // at runtime via refine — `.nonoptional()` is a v4-only API and isn't
+  // available on the classic `z` import this repo pins.
+  .refine((payload) => "data" in payload, {
+    message: "data is required",
+    path: ["data"],
+  });
+
+function initializeWorker() {
+  const redisOptions = {
+    keyPrefix: "accounts-webhook:worker:",
+    host: env.COMMON_WORKER_REDIS_HOST,
+    port: env.COMMON_WORKER_REDIS_PORT,
+    username: env.COMMON_WORKER_REDIS_USERNAME,
+    password: env.COMMON_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.COMMON_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  const worker = new RedisWorker({
+    name: "accounts-webhook-worker",
+    redisOptions,
+    catalog: {
+      "account.webhook.event": {
+        schema: PayloadSchema,
+        visibilityTimeoutMs: 30_000,
+        retry: { maxAttempts: 5 },
+      },
+    },
+    concurrency: {
+      workers: 2,
+      tasksPerWorker: 4,
+      limit: 8,
+    },
+    pollIntervalMs: 1_000,
+    immediatePollIntervalMs: 50,
+    shutdownTimeoutMs: 30_000,
+    jobs: {
+      "account.webhook.event": async ({ payload }) => {
+        // The plugin returns a Result; throw on error so the worker
+        // retries (a resolved err would otherwise be silently dropped).
+        // `z.unknown()` infers `data?: unknown`, but the contract's
+        // SsoWebhookEvent requires `data: unknown` — restate the fields
+        // explicitly so the optional-vs-required shapes line up.
+        const result = await ssoController.processWebhookEvent({
+          id: payload.id,
+          event: payload.event,
+          data: payload.data,
+        });
+        if (result.isErr()) {
+          throw new Error(`account webhook processing failed: ${result.error}`);
+        }
+        // Directory-sync events return membership effects to apply against
+        // public.* tables (the plugin never writes those). A throw here
+        // bubbles to the worker for retry; effects are idempotent.
+        await applyDirectorySyncEffects(result.value.effects);
+      },
+    },
+  });
+
+  // Only poll on worker-role instances (same gate as commonWorker) and
+  // only when the feature is enabled (no plugin loaded otherwise).
+  if (env.COMMON_WORKER_ENABLED === "true" && env.SSO_ENABLED) {
+    logger.debug(`👨‍🏭 Starting accounts webhook worker at host ${env.COMMON_WORKER_REDIS_HOST}`);
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const accountsWebhookWorker = singleton("accountsWebhookWorker", initializeWorker);

apps/webapp/app/v3/alertsRateLimiter.server.ts

@@ -0,0 +1,30 @@
+import { env } from "~/env.server";
+import { createRedisClient } from "~/redis.server";
+import { GCRARateLimiter } from "./GCRARateLimiter.server";
+import { singleton } from "~/utils/singleton";
+import { logger } from "~/services/logger.server";
+
+export const alertsRateLimiter = singleton("alertsRateLimiter", initializeAlertsRateLimiter);
+
+function initializeAlertsRateLimiter() {
+  const redis = createRedisClient("alerts:ratelimiter", {
+    keyPrefix: "alerts:ratelimiter:",
+    host: env.ALERT_RATE_LIMITER_REDIS_HOST,
+    port: env.ALERT_RATE_LIMITER_REDIS_PORT,
+    username: env.ALERT_RATE_LIMITER_REDIS_USERNAME,
+    password: env.ALERT_RATE_LIMITER_REDIS_PASSWORD,
+    tlsDisabled: env.ALERT_RATE_LIMITER_REDIS_TLS_DISABLED === "true",
+    clusterMode: env.ALERT_RATE_LIMITER_REDIS_CLUSTER_MODE_ENABLED === "1",
+  });
+
+  logger.debug(`🚦 Initializing alerts rate limiter at host ${env.ALERT_RATE_LIMITER_REDIS_HOST}`, {
+    emissionInterval: env.ALERT_RATE_LIMITER_EMISSION_INTERVAL,
+    burstTolerance: env.ALERT_RATE_LIMITER_BURST_TOLERANCE,
+  });
+
+  return new GCRARateLimiter({
+    redis,
+    emissionInterval: env.ALERT_RATE_LIMITER_EMISSION_INTERVAL,
+    burstTolerance: env.ALERT_RATE_LIMITER_BURST_TOLERANCE,
+  });
+}

apps/webapp/app/v3/alertsWorker.server.ts

@@ -0,0 +1,143 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { DeliverAlertService } from "./services/alerts/deliverAlert.server";
+import { DeliverErrorGroupAlertService } from "./services/alerts/deliverErrorGroupAlert.server";
+import { ErrorAlertEvaluator } from "./services/alerts/errorAlertEvaluator.server";
+import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server";
+import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
+
+function initializeWorker() {
+  const redisOptions = {
+    keyPrefix: "alerts:worker:",
+    host: env.ALERTS_WORKER_REDIS_HOST,
+    port: env.ALERTS_WORKER_REDIS_PORT,
+    username: env.ALERTS_WORKER_REDIS_USERNAME,
+    password: env.ALERTS_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.ALERTS_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(`👨‍🏭 Initializing alerts worker at host ${env.ALERTS_WORKER_REDIS_HOST}`);
+
+  const worker = new RedisWorker({
+    name: "alerts-worker",
+    redisOptions,
+    catalog: {
+      "v3.performTaskRunAlerts": {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: false,
+      },
+      "v3.performDeploymentAlerts": {
+        schema: z.object({
+          deploymentId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: false,
+      },
+      "v3.deliverAlert": {
+        schema: z.object({
+          alertId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: false,
+      },
+      "v3.evaluateErrorAlerts": {
+        schema: z.object({
+          projectId: z.string(),
+          scheduledAt: z.number(),
+        }),
+        visibilityTimeoutMs: 60_000 * 5,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+      "v3.deliverErrorGroupAlert": {
+        schema: z.object({
+          channelId: z.string(),
+          projectId: z.string(),
+          classification: z.enum(["new_issue", "regression", "unignored"]),
+          error: z.object({
+            fingerprint: z.string(),
+            environmentId: z.string(),
+            environmentSlug: z.string(),
+            environmentName: z.string(),
+            taskIdentifier: z.string(),
+            errorType: z.string(),
+            errorMessage: z.string(),
+            sampleStackTrace: z.string(),
+            firstSeen: z.string(),
+            lastSeen: z.string(),
+            occurrenceCount: z.number(),
+          }),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+        logErrors: true,
+      },
+    },
+    concurrency: {
+      workers: env.ALERTS_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.ALERTS_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.ALERTS_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.ALERTS_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.ALERTS_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.ALERTS_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("AlertsWorker", env.ALERTS_WORKER_LOG_LEVEL),
+    jobs: {
+      "v3.deliverAlert": async ({ payload }) => {
+        const service = new DeliverAlertService();
+
+        await service.call(payload.alertId);
+      },
+      "v3.performDeploymentAlerts": async ({ payload }) => {
+        const service = new PerformDeploymentAlertsService();
+
+        await service.call(payload.deploymentId);
+      },
+      "v3.performTaskRunAlerts": async ({ payload }) => {
+        const service = new PerformTaskRunAlertsService();
+        await service.call(payload.runId);
+      },
+      "v3.evaluateErrorAlerts": async ({ payload }) => {
+        const evaluator = new ErrorAlertEvaluator();
+        await evaluator.evaluate(payload.projectId, payload.scheduledAt);
+      },
+      "v3.deliverErrorGroupAlert": async ({ payload }) => {
+        const service = new DeliverErrorGroupAlertService();
+        await service.call(payload);
+      },
+    },
+  });
+
+  if (env.ALERTS_WORKER_ENABLED === "true") {
+    logger.debug(
+      `👨‍🏭 Starting alerts worker at host ${env.ALERTS_WORKER_REDIS_HOST}, pollInterval = ${env.ALERTS_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.ALERTS_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.ALERTS_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.ALERTS_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.ALERTS_WORKER_CONCURRENCY_LIMIT}`
+    );
+
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const alertsWorker = singleton("alertsWorker", initializeWorker);

apps/webapp/app/v3/batchTriggerWorker.server.ts

@@ -0,0 +1,105 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { RunEngineBatchTriggerService } from "~/runEngine/services/batchTrigger.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { BatchTriggerV3Service } from "./services/batchTriggerV3.server";
+// Import engine to ensure it's initialized (which initializes BatchQueue for v2 batches)
+import { engine } from "./runEngine.server";
+
+/**
+ * Legacy batch trigger worker for processing v3 and run engine v1 batches.
+ *
+ * NOTE: Run Engine v2 batches (batchVersion: "runengine:v2") use the new BatchQueue
+ * system with Deficit Round Robin scheduling, which is encapsulated within the RunEngine.
+ * See runEngine.server.ts for the configuration.
+ *
+ * This worker is kept for backwards compatibility with:
+ * - v3 batches (batchVersion: "v3") - handled by BatchTriggerV3Service
+ * - Run Engine v1 batches (batchVersion: "runengine:v1") - handled by RunEngineBatchTriggerService
+ */
+function initializeWorker() {
+  // Ensure the engine (and its BatchQueue) is initialized
+  void engine;
+  const redisOptions = {
+    keyPrefix: "batch-trigger:worker:",
+    host: env.BATCH_TRIGGER_WORKER_REDIS_HOST,
+    port: env.BATCH_TRIGGER_WORKER_REDIS_PORT,
+    username: env.BATCH_TRIGGER_WORKER_REDIS_USERNAME,
+    password: env.BATCH_TRIGGER_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(
+    `👨‍🏭 Initializing batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}`
+  );
+
+  const worker = new RedisWorker({
+    name: "batch-trigger-worker",
+    redisOptions,
+    catalog: {
+      "v3.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+        }),
+        visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "runengine.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+          parentRunId: z.string().optional(),
+          resumeParentOnCompletion: z.boolean().optional(),
+        }),
+        visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+    },
+    concurrency: {
+      workers: env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.BATCH_TRIGGER_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.BATCH_TRIGGER_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("BatchTriggerWorker", env.BATCH_TRIGGER_WORKER_LOG_LEVEL),
+    jobs: {
+      "v3.processBatchTaskRun": async ({ payload }) => {
+        const service = new BatchTriggerV3Service(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+      "runengine.processBatchTaskRun": async ({ payload }) => {
+        const service = new RunEngineBatchTriggerService(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+    },
+  });
+
+  if (env.BATCH_TRIGGER_WORKER_ENABLED === "true") {
+    logger.debug(
+      `👨‍🏭 Starting batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}, pollInterval = ${env.BATCH_TRIGGER_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT}`
+    );
+
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const batchTriggerWorker = singleton("batchTriggerWorker", initializeWorker);

apps/webapp/app/v3/billingLimitWorker.server.ts

@@ -0,0 +1,189 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { BillingLimitConvergeEnvironmentsService } from "./services/billingLimit/billingLimitConvergeEnvironmentsService.server";
+import type { BillingLimitConvergeTargetState } from "./services/billingLimit/billingLimitConstants";
+import {
+  buildBillingLimitInProgressCancelJobId,
+  buildBillingLimitResolveJobId,
+} from "./services/billingLimit/billingLimitConstants";
+import { runBillingLimitCancelInProgressRuns } from "./services/billingLimit/billingLimitCancelInProgressRuns.server";
+import { runPendingBillingLimitResolves } from "./services/billingLimit/billingLimitPendingResolveCoordinator.server";
+import type { PendingBillingLimitResolve } from "./services/billingLimit/billingLimitPendingResolve.types";
+
+function initializeWorker() {
+  const redisOptions = {
+    keyPrefix: "billing-limit:worker:",
+    host: env.BILLING_LIMIT_WORKER_REDIS_HOST,
+    port: env.BILLING_LIMIT_WORKER_REDIS_PORT,
+    username: env.BILLING_LIMIT_WORKER_REDIS_USERNAME,
+    password: env.BILLING_LIMIT_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(
+    `👨‍🏭 Initializing billing limit worker at host ${env.BILLING_LIMIT_WORKER_REDIS_HOST}`
+  );
+
+  const worker = new RedisWorker({
+    name: "billing-limit-worker",
+    redisOptions,
+    catalog: {
+      "billingLimit.convergeEnvironments": {
+        schema: z.object({
+          organizationId: z.string(),
+          targetState: z.enum(["grace", "rejected", "ok"]),
+        }),
+        visibilityTimeoutMs: 60_000 * 10,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "billingLimit.reconcileTick": {
+        schema: z.object({}),
+        visibilityTimeoutMs: 60_000 * 5,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "billingLimit.cancelInProgressRuns": {
+        schema: z.object({
+          organizationId: z.string(),
+          hitAt: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000 * 10,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "billingLimit.resolve": {
+        schema: z.object({
+          organizationId: z.string(),
+          resumeMode: z.enum(["queue", "new_only"]),
+          resolvedAt: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000 * 10,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+    },
+    concurrency: {
+      workers: env.BILLING_LIMIT_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.BILLING_LIMIT_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.BILLING_LIMIT_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.BILLING_LIMIT_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.BILLING_LIMIT_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.BILLING_LIMIT_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("BillingLimitWorker", env.BILLING_LIMIT_WORKER_LOG_LEVEL),
+    jobs: {
+      "billingLimit.convergeEnvironments": async ({ payload }) => {
+        await BillingLimitConvergeEnvironmentsService.runConverge(payload);
+      },
+      "billingLimit.reconcileTick": async () => {
+        await BillingLimitConvergeEnvironmentsService.runReconcileTick();
+        await scheduleBillingLimitReconcileTick(worker);
+      },
+      "billingLimit.cancelInProgressRuns": async ({ payload }) => {
+        await runBillingLimitCancelInProgressRuns(payload.organizationId, payload.hitAt);
+      },
+      "billingLimit.resolve": async ({ payload }) => {
+        await runPendingBillingLimitResolves([payload]);
+      },
+    },
+  });
+
+  return worker;
+}
+
+declare global {
+  // eslint-disable-next-line no-var
+  var __billingLimitWorkerStarted__: boolean | undefined;
+}
+
+/**
+ * Bootstraps the billing-limit redis worker on webapp startup.
+ *
+ * Constructed via the module singleton (for enqueue from webhooks); started
+ * here so `sideEffects: false` builds keep an explicit entry-point side
+ * effect — do not rely on a bare `import "~/v3/billingLimitWorker.server"`.
+ */
+export function initBillingLimitWorker(
+  opts: {
+    isEnabled?: () => boolean;
+  } = {}
+): void {
+  const isEnabled = opts.isEnabled ?? (() => env.BILLING_LIMIT_WORKER_ENABLED === "true");
+
+  if (!isEnabled()) {
+    return;
+  }
+
+  if (global.__billingLimitWorkerStarted__) {
+    return;
+  }
+
+  const worker = billingLimitWorker;
+
+  logger.debug(
+    `👨‍🏭 Starting billing limit worker at host ${env.BILLING_LIMIT_WORKER_REDIS_HOST}, reconcileIntervalMs = ${env.BILLING_LIMIT_RECONCILE_INTERVAL_MS}`
+  );
+  try {
+    worker.start();
+    global.__billingLimitWorkerStarted__ = true;
+    void scheduleBillingLimitReconcileTick(worker).catch((error) => {
+      logger.error("Failed to schedule initial billing-limit reconcile tick", {
+        error,
+      });
+    });
+  } catch (error) {
+    global.__billingLimitWorkerStarted__ = false;
+    throw error;
+  }
+}
+
+async function scheduleBillingLimitReconcileTick(worker: ReturnType<typeof initializeWorker>) {
+  await worker.enqueue({
+    id: "billingLimit.reconcileTick",
+    job: "billingLimit.reconcileTick",
+    payload: {},
+    availableAt: new Date(Date.now() + env.BILLING_LIMIT_RECONCILE_INTERVAL_MS),
+  });
+}
+
+export const billingLimitWorker = singleton("billingLimitWorker", initializeWorker);
+
+export async function enqueueBillingLimitConverge(
+  organizationId: string,
+  targetState: BillingLimitConvergeTargetState
+) {
+  return billingLimitWorker.enqueue({
+    id: `billingLimit.converge:${organizationId}:${targetState}`,
+    job: "billingLimit.convergeEnvironments",
+    payload: { organizationId, targetState },
+  });
+}
+
+export async function enqueueBillingLimitCancelInProgressRuns(
+  organizationId: string,
+  hitAt: string
+) {
+  return billingLimitWorker.enqueue({
+    id: buildBillingLimitInProgressCancelJobId(organizationId, hitAt),
+    job: "billingLimit.cancelInProgressRuns",
+    payload: { organizationId, hitAt },
+  });
+}
+
+export async function enqueueBillingLimitResolve(pending: PendingBillingLimitResolve) {
+  return billingLimitWorker.enqueue({
+    id: buildBillingLimitResolveJobId(pending.organizationId, pending.resolvedAt),
+    job: "billingLimit.resolve",
+    payload: pending,
+  });
+}

apps/webapp/app/v3/buildSettings.ts

@@ -0,0 +1,10 @@
+import { z } from "zod";
+
+export const BuildSettingsSchema = z.object({
+  triggerConfigFilePath: z.string().optional(),
+  installCommand: z.string().optional(),
+  preBuildCommand: z.string().optional(),
+  useNativeBuildServer: z.boolean().optional(),
+});
+
+export type BuildSettings = z.infer<typeof BuildSettingsSchema>;

apps/webapp/app/v3/canAccessAi.server.ts

@@ -0,0 +1,47 @@
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { makeFlag } from "~/v3/featureFlags.server";
+
+export async function canAccessAi(options: {
+  userId: string;
+  isAdmin: boolean;
+  isImpersonating: boolean;
+  organizationSlug: string;
+}): Promise<boolean> {
+  const { userId, isAdmin, isImpersonating, organizationSlug } = options;
+
+  // 1. If env var is set then globally enabled
+  if (env.AI_FEATURES_ENABLED === "1") {
+    return true;
+  }
+
+  // 2. Admins always have access
+  if (isAdmin || isImpersonating) {
+    return true;
+  }
+
+  // 3. Check if org/global feature flag is on
+  const org = await prisma.organization.findFirst({
+    where: {
+      slug: organizationSlug,
+      members: { some: { userId } },
+    },
+    select: {
+      featureFlags: true,
+    },
+  });
+
+  const flag = makeFlag();
+  const flagResult = await flag({
+    key: FEATURE_FLAG.hasAiAccess,
+    defaultValue: false,
+    overrides: (org?.featureFlags as Record<string, unknown>) ?? {},
+  });
+  if (flagResult) {
+    return true;
+  }
+
+  // 4. Not enabled anywhere
+  return false;
+}

apps/webapp/app/v3/canAccessDashboardAgent.server.ts

@@ -0,0 +1,51 @@
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { makeFlag } from "~/v3/featureFlags.server";
+
+/**
+ * Whether the in-dashboard AI agent is available to this user in this org.
+ * Gated by the global / per-org `hasDashboardAgentAccess` flag, with
+ * `DASHBOARD_AGENT_ENABLED` as the global default (a per-org override wins).
+ * Admins/impersonators bypass it only when `DASHBOARD_AGENT_ADMIN_PREVIEW` is on
+ * (default off). Enforced server-side so a non-flagged user can't start sessions.
+ */
+export async function canAccessDashboardAgent(options: {
+  userId: string;
+  isAdmin: boolean;
+  isImpersonating: boolean;
+  organizationSlug: string;
+  // When the caller already has the org's `featureFlags` loaded (e.g. a layout
+  // loader that queried the org with a membership check), pass them to skip the
+  // extra org lookup. Omit it and we query the org ourselves.
+  orgFeatureFlags?: Record<string, unknown> | null;
+}): Promise<boolean> {
+  const { userId, isAdmin, isImpersonating, organizationSlug, orgFeatureFlags } = options;
+
+  if ((isAdmin || isImpersonating) && env.DASHBOARD_AGENT_ADMIN_PREVIEW === "1") {
+    return true;
+  }
+
+  let overrides = orgFeatureFlags;
+  if (overrides === undefined) {
+    const org = await prisma.organization.findFirst({
+      where: {
+        slug: organizationSlug,
+        members: { some: { userId } },
+      },
+      select: {
+        featureFlags: true,
+      },
+    });
+    overrides = (org?.featureFlags as Record<string, unknown>) ?? {};
+  }
+
+  const flag = makeFlag();
+  const flagResult = await flag({
+    key: FEATURE_FLAG.hasDashboardAgentAccess,
+    defaultValue: env.DASHBOARD_AGENT_ENABLED === "1",
+    overrides: overrides ?? {},
+  });
+
+  return Boolean(flagResult);
+}

apps/webapp/app/v3/canAccessPrivateConnections.server.ts

@@ -0,0 +1,28 @@
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { makeFlag } from "~/v3/featureFlags.server";
+
+export async function canAccessPrivateConnections(options: {
+  organizationSlug: string;
+  userId: string;
+}): Promise<boolean> {
+  const { organizationSlug, userId } = options;
+
+  const org = await prisma.organization.findFirst({
+    where: {
+      slug: organizationSlug,
+      members: { some: { userId } },
+    },
+    select: {
+      featureFlags: true,
+    },
+  });
+
+  const flag = makeFlag();
+  return flag({
+    key: FEATURE_FLAG.hasPrivateConnections,
+    defaultValue: env.PRIVATE_CONNECTIONS_ENABLED === "1",
+    overrides: (org?.featureFlags as Record<string, unknown>) ?? {},
+  });
+}

apps/webapp/app/v3/canAccessQuery.server.ts

@@ -0,0 +1,48 @@
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { makeFlag } from "~/v3/featureFlags.server";
+
+export async function canAccessQuery(options: {
+  userId: string;
+  isAdmin: boolean;
+  isImpersonating: boolean;
+  organizationSlug: string;
+}): Promise<boolean> {
+  const { userId, isAdmin, isImpersonating, organizationSlug } = options;
+
+  // 1. If it's on then we have access
+  const globallyEnabled = env.QUERY_FEATURE_ENABLED === "1";
+  if (globallyEnabled) {
+    return true;
+  }
+
+  // 2. Admins always have access
+  if (isAdmin || isImpersonating) {
+    return true;
+  }
+
+  // 3. Check if org/global feature flag is on
+  const org = await prisma.organization.findFirst({
+    where: {
+      slug: organizationSlug,
+      members: { some: { userId } },
+    },
+    select: {
+      featureFlags: true,
+    },
+  });
+
+  const flag = makeFlag();
+  const flagResult = await flag({
+    key: FEATURE_FLAG.hasQueryAccess,
+    defaultValue: false,
+    overrides: (org?.featureFlags as Record<string, unknown>) ?? {},
+  });
+  if (flagResult) {
+    return true;
+  }
+
+  // 4. Not enabled anywhere
+  return false;
+}

apps/webapp/app/v3/commonWorker.server.ts

@@ -0,0 +1,325 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { DeliverEmailSchema } from "emails";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { RunEngineBatchTriggerService } from "~/runEngine/services/batchTrigger.server";
+import { sendEmail } from "~/services/email.server";
+import {
+  AttioUserSyncSchema,
+  AttioWorkspaceSyncSchema,
+  runAttioUserSync,
+  runAttioWorkspaceSync,
+} from "~/services/attio.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { DeliverAlertService } from "./services/alerts/deliverAlert.server";
+import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server";
+import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
+import { BatchTriggerV3Service } from "./services/batchTriggerV3.server";
+import { CancelDevSessionRunsService } from "./services/cancelDevSessionRuns.server";
+import { CancelTaskAttemptDependenciesService } from "./services/cancelTaskAttemptDependencies.server";
+import { EnqueueDelayedRunService } from "./services/enqueueDelayedRun.server";
+import { ExecuteTasksWaitingForDeployService } from "./services/executeTasksWaitingForDeploy";
+import { ExpireEnqueuedRunService } from "./services/expireEnqueuedRun.server";
+import { ResumeBatchRunService } from "./services/resumeBatchRun.server";
+import { ResumeTaskDependencyService } from "./services/resumeTaskDependency.server";
+import { RetryAttemptService } from "./services/retryAttempt.server";
+import { TimeoutDeploymentService } from "./services/timeoutDeployment.server";
+import { BulkActionService } from "./services/bulk/BulkActionV2.server";
+
+function initializeWorker() {
+  const redisOptions = {
+    keyPrefix: "common:worker:",
+    host: env.COMMON_WORKER_REDIS_HOST,
+    port: env.COMMON_WORKER_REDIS_PORT,
+    username: env.COMMON_WORKER_REDIS_USERNAME,
+    password: env.COMMON_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.COMMON_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(`👨‍🏭 Initializing common worker at host ${env.COMMON_WORKER_REDIS_HOST}`);
+
+  const worker = new RedisWorker({
+    name: "common-worker",
+    redisOptions,
+    catalog: {
+      scheduleEmail: {
+        schema: DeliverEmailSchema,
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "attio.syncWorkspace": {
+        schema: AttioWorkspaceSyncSchema,
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "attio.syncUser": {
+        schema: AttioUserSyncSchema,
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "v3.resumeBatchRun": {
+        schema: z.object({
+          batchRunId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "v3.resumeTaskDependency": {
+        schema: z.object({
+          dependencyId: z.string(),
+          sourceTaskAttemptId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "v3.timeoutDeployment": {
+        schema: z.object({
+          deploymentId: z.string(),
+          fromStatus: z.string(),
+          errorMessage: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "v3.executeTasksWaitingForDeploy": {
+        schema: z.object({
+          backgroundWorkerId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "v3.retryAttempt": {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "v3.cancelTaskAttemptDependencies": {
+        schema: z.object({
+          attemptId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 8,
+        },
+      },
+      "v3.cancelDevSessionRuns": {
+        schema: z.object({
+          runIds: z.array(z.string()),
+          cancelledAt: z.coerce.date(),
+          reason: z.string(),
+          cancelledSessionId: z.string().optional(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      // @deprecated, moved to batchTriggerWorker.server.ts
+      "v3.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      // @deprecated, moved to batchTriggerWorker.server.ts
+      "runengine.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+          parentRunId: z.string().optional(),
+          resumeParentOnCompletion: z.boolean().optional(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "v3.performTaskRunAlerts": {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "v3.performDeploymentAlerts": {
+        schema: z.object({
+          deploymentId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "v3.deliverAlert": {
+        schema: z.object({
+          alertId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      "v3.expireRun": {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 6,
+        },
+      },
+      "v3.enqueueDelayedRun": {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 6,
+        },
+      },
+      processBulkAction: {
+        schema: z.object({
+          bulkActionId: z.string(),
+        }),
+        visibilityTimeoutMs: 180_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+    },
+    concurrency: {
+      workers: env.COMMON_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.COMMON_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.COMMON_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.COMMON_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.COMMON_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("CommonWorker", env.COMMON_WORKER_LOG_LEVEL),
+    jobs: {
+      scheduleEmail: async ({ payload }) => {
+        await sendEmail(payload);
+      },
+      "attio.syncWorkspace": async ({ payload }) => {
+        await runAttioWorkspaceSync(payload);
+      },
+      "attio.syncUser": async ({ payload }) => {
+        await runAttioUserSync(payload);
+      },
+      "v3.resumeBatchRun": async ({ payload }) => {
+        const service = new ResumeBatchRunService();
+        await service.call(payload.batchRunId);
+      },
+      "v3.resumeTaskDependency": async ({ payload }) => {
+        const service = new ResumeTaskDependencyService();
+        await service.call(payload.dependencyId, payload.sourceTaskAttemptId);
+      },
+      "v3.timeoutDeployment": async ({ payload }) => {
+        const service = new TimeoutDeploymentService();
+        await service.call(payload.deploymentId, payload.fromStatus, payload.errorMessage);
+      },
+      "v3.executeTasksWaitingForDeploy": async ({ payload }) => {
+        const service = new ExecuteTasksWaitingForDeployService();
+        await service.call(payload.backgroundWorkerId);
+      },
+      "v3.retryAttempt": async ({ payload }) => {
+        const service = new RetryAttemptService();
+        await service.call(payload.runId);
+      },
+      "v3.cancelTaskAttemptDependencies": async ({ payload }) => {
+        const service = new CancelTaskAttemptDependenciesService();
+        await service.call(payload.attemptId);
+      },
+      "v3.cancelDevSessionRuns": async ({ payload }) => {
+        const service = new CancelDevSessionRunsService();
+        await service.call(payload);
+      },
+      // @deprecated, moved to batchTriggerWorker.server.ts
+      "v3.processBatchTaskRun": async ({ payload }) => {
+        const service = new BatchTriggerV3Service(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+      // @deprecated, moved to batchTriggerWorker.server.ts
+      "runengine.processBatchTaskRun": async ({ payload }) => {
+        const service = new RunEngineBatchTriggerService(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+      // @deprecated, moved to alertsWorker.server.ts
+      "v3.deliverAlert": async ({ payload }) => {
+        const service = new DeliverAlertService();
+
+        await service.call(payload.alertId);
+      },
+      // @deprecated, moved to alertsWorker.server.ts
+      "v3.performDeploymentAlerts": async ({ payload }) => {
+        const service = new PerformDeploymentAlertsService();
+
+        await service.call(payload.deploymentId);
+      },
+      // @deprecated, moved to alertsWorker.server.ts
+      "v3.performTaskRunAlerts": async ({ payload }) => {
+        const service = new PerformTaskRunAlertsService();
+        await service.call(payload.runId);
+      },
+      "v3.expireRun": async ({ payload }) => {
+        const service = new ExpireEnqueuedRunService();
+
+        await service.call(payload.runId);
+      },
+      "v3.enqueueDelayedRun": async ({ payload }) => {
+        const service = new EnqueueDelayedRunService();
+
+        await service.call(payload.runId);
+      },
+      processBulkAction: async ({ payload }) => {
+        const service = new BulkActionService();
+        await service.process(payload.bulkActionId);
+      },
+    },
+  });
+
+  if (env.COMMON_WORKER_ENABLED === "true") {
+    logger.debug(
+      `👨‍🏭 Starting common worker at host ${env.COMMON_WORKER_REDIS_HOST}, pollInterval = ${env.COMMON_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.COMMON_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.COMMON_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.COMMON_WORKER_CONCURRENCY_LIMIT}`
+    );
+
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const commonWorker = singleton("commonWorker", initializeWorker);

apps/webapp/app/v3/deduplicateVariableArray.server.ts

@@ -0,0 +1,14 @@
+import { type EnvironmentVariable } from "./environmentVariables/repository";
+
+/** Later variables override earlier ones */
+export function deduplicateVariableArray(variables: EnvironmentVariable[]) {
+  const result: EnvironmentVariable[] = [];
+  // Process array in reverse order so later variables override earlier ones
+  for (const variable of [...variables].reverse()) {
+    if (!result.some((v) => v.key === variable.key)) {
+      result.push(variable);
+    }
+  }
+  // Reverse back to maintain original order but with later variables taking precedence
+  return result.reverse();
+}

apps/webapp/app/v3/detectQueryTables.ts

@@ -0,0 +1,113 @@
+import {
+  parseTSQLSelect,
+  SyntaxError as TSQLSyntaxError,
+  type Field,
+  type JoinExpr,
+  type SelectQuery,
+  type SelectSetQuery,
+} from "@internal/tsql";
+
+/**
+ * Extract every known table a TRQL query reads — the FROM table, every JOIN in
+ * the chain, and any subqueries — for per-table JWT-scope authorization.
+ *
+ * `allowedTableNames` is the set of recognised table names (matched
+ * case-insensitively); anything not in it is ignored. Injected so this stays
+ * dependency-free (the caller derives it from the query schemas).
+ *
+ * Returns `null` when the query can't be parsed; callers MUST treat `null` as
+ * deny-by-default.
+ */
+export function detectQueryTables(query: string, allowedTableNames: Set<string>): string[] | null {
+  let ast: SelectQuery | SelectSetQuery;
+  try {
+    ast = parseTSQLSelect(query);
+  } catch (err) {
+    if (err instanceof TSQLSyntaxError) return null;
+    throw err;
+  }
+
+  const allowed = new Map(Array.from(allowedTableNames, (n) => [n.toLowerCase(), n]));
+  const seen = new Set<string>();
+  const scanned = new WeakSet<object>();
+
+  function visitSelect(q: SelectQuery): void {
+    // CTE bodies: `WITH r AS (SELECT ... FROM <table>) ...` — the table is
+    // read by the CTE even when the outer query only references the CTE alias.
+    if (q.ctes) {
+      for (const cte of Object.values(q.ctes)) {
+        scanForSubqueries(cte.expr);
+      }
+    }
+    // FROM / JOIN chain (tables + FROM-position subqueries).
+    if (q.select_from) visitJoin(q.select_from);
+    // Subqueries anywhere else (WHERE, SELECT list, GROUP BY, ORDER BY, etc.)
+    // can each embed a SELECT that reads a real table, e.g.
+    // `WHERE id IN (SELECT … FROM runs)`.
+    scanForSubqueries(q.select);
+    scanForSubqueries(q.where);
+    scanForSubqueries(q.prewhere);
+    scanForSubqueries(q.having);
+    scanForSubqueries(q.group_by);
+    scanForSubqueries(q.array_join_list);
+    scanForSubqueries(q.order_by);
+    scanForSubqueries(q.limit);
+    scanForSubqueries(q.offset);
+    scanForSubqueries(q.limit_by);
+    scanForSubqueries(q.window_exprs);
+  }
+  // Shape-agnostic walk of an expression subtree: descends every nested
+  // object/array and hands any embedded SELECT to the query visitors, so a new
+  // node shape can't silently reintroduce a detection gap. The WeakSet guards
+  // against back-reference cycles the AST might carry.
+  function scanForSubqueries(node: unknown): void {
+    if (node === null || typeof node !== "object") return;
+    if (scanned.has(node)) return;
+    scanned.add(node);
+    if (Array.isArray(node)) {
+      for (const item of node) scanForSubqueries(item);
+      return;
+    }
+    const expressionType = (node as { expression_type?: string }).expression_type;
+    if (expressionType === "select_query") {
+      visitSelect(node as SelectQuery);
+      return;
+    }
+    if (expressionType === "select_set_query") {
+      visitSelectSet(node as SelectSetQuery);
+      return;
+    }
+    for (const value of Object.values(node)) scanForSubqueries(value);
+  }
+  function visitSelectSet(q: SelectSetQuery): void {
+    visitAny(q.initial_select_query);
+    for (const node of q.subsequent_select_queries ?? []) {
+      visitAny(node.select_query);
+    }
+  }
+  function visitAny(q: SelectQuery | SelectSetQuery): void {
+    if (q.expression_type === "select_query") visitSelect(q);
+    else visitSelectSet(q);
+  }
+  function visitJoin(node: JoinExpr): void {
+    const tableExpr = node.table;
+    if (tableExpr) {
+      if ((tableExpr as Field).expression_type === "field") {
+        const name = (tableExpr as Field).chain[0];
+        const canonicalName =
+          typeof name === "string" ? allowed.get(name.toLowerCase()) : undefined;
+        if (canonicalName) seen.add(canonicalName);
+      } else if ((tableExpr as SelectQuery).expression_type === "select_query") {
+        visitSelect(tableExpr as SelectQuery);
+      } else if ((tableExpr as SelectSetQuery).expression_type === "select_set_query") {
+        visitSelectSet(tableExpr as SelectSetQuery);
+      }
+    }
+    if (node.next_join) visitJoin(node.next_join);
+  }
+
+  if (ast.expression_type === "select_set_query") visitSelectSet(ast);
+  else visitSelect(ast);
+
+  return Array.from(seen);
+}

apps/webapp/app/v3/dynamicFlushScheduler.server.ts

@@ -0,0 +1,499 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { tryCatch } from "@trigger.dev/core/utils";
+import { getMeter, type Counter, type Histogram, type Meter } from "@internal/tracing";
+import { nanoid } from "nanoid";
+import pLimit from "p-limit";
+import { signalsEmitter } from "~/services/signals.server";
+
+export type DynamicFlushSchedulerConfig<T> = {
+  batchSize: number;
+  flushInterval: number;
+  callback: (flushId: string, batch: T[]) => Promise<void>;
+  // New configuration options
+  minConcurrency?: number;
+  maxConcurrency?: number;
+  maxBatchSize?: number;
+  memoryPressureThreshold?: number; // Number of items that triggers increased concurrency
+  loadSheddingThreshold?: number; // Number of items that triggers load shedding
+  loadSheddingEnabled?: boolean;
+  isDroppableEvent?: (item: T) => boolean; // Function to determine if an event can be dropped
+  // Self-observability. `name` is the low-cardinality `scheduler` label that separates the
+  // task_events / llm_metrics / otlp_metrics instances in the same process. `meter` defaults to
+  // the global provider; inject one in tests. Instruments are no-op unless metrics are enabled.
+  meter?: Meter;
+  name?: string;
+};
+
+export class DynamicFlushScheduler<T> {
+  private batchQueue: T[][];
+  private currentBatch: T[];
+  private readonly BATCH_SIZE: number;
+  private readonly FLUSH_INTERVAL: number;
+  private flushTimer: NodeJS.Timeout | null;
+  private metricsReporterTimer: NodeJS.Timeout | undefined;
+  private readonly callback: (flushId: string, batch: T[]) => Promise<void>;
+
+  // New properties for dynamic scaling
+  private readonly minConcurrency: number;
+  private readonly maxConcurrency: number;
+  private readonly maxBatchSize: number;
+  private readonly memoryPressureThreshold: number;
+  private limiter: ReturnType<typeof pLimit>;
+  private currentBatchSize: number;
+  private totalQueuedItems: number = 0;
+  private consecutiveFlushFailures: number = 0;
+  private lastFlushTime: number = Date.now();
+  private metrics = {
+    flushedBatches: 0,
+    failedBatches: 0,
+    totalItemsFlushed: 0,
+    droppedEvents: 0,
+    droppedEventsByKind: new Map<string, number>(),
+  };
+  private isShuttingDown: boolean = false;
+
+  // New properties for load shedding
+  private readonly loadSheddingThreshold: number;
+  private readonly loadSheddingEnabled: boolean;
+  private readonly isDroppableEvent?: (item: T) => boolean;
+  private isLoadShedding: boolean = false;
+
+  private readonly logger: Logger = new Logger("EventRepo.DynamicFlushScheduler", "info");
+
+  // Pre-allocated attribute objects (closed label sets) so the hot flush path never allocates.
+  private readonly _metricAttrs: { scheduler: string };
+  private readonly _batchOkAttrs: { scheduler: string; outcome: string };
+  private readonly _batchFailedAttrs: { scheduler: string; outcome: string };
+  private _batchesCounter?: Counter;
+  private _itemsCounter?: Counter;
+  private _flushDurationHistogram?: Histogram;
+  private _batchSizeHistogram?: Histogram;
+  private _droppedEventsCounter?: Counter;
+
+  constructor(config: DynamicFlushSchedulerConfig<T>) {
+    const schedulerName = config.name ?? "unknown";
+    this._metricAttrs = { scheduler: schedulerName };
+    this._batchOkAttrs = { scheduler: schedulerName, outcome: "ok" };
+    this._batchFailedAttrs = { scheduler: schedulerName, outcome: "failed" };
+    this.batchQueue = [];
+    this.currentBatch = [];
+    this.BATCH_SIZE = config.batchSize;
+    this.currentBatchSize = config.batchSize;
+    this.FLUSH_INTERVAL = config.flushInterval;
+    this.callback = config.callback;
+    this.flushTimer = null;
+
+    // Initialize dynamic scaling parameters
+    this.minConcurrency = config.minConcurrency ?? 1;
+    this.maxConcurrency = config.maxConcurrency ?? 10;
+    this.maxBatchSize = config.maxBatchSize ?? config.batchSize * 5;
+    this.memoryPressureThreshold = config.memoryPressureThreshold ?? config.batchSize * 20;
+
+    // Initialize load shedding parameters
+    this.loadSheddingThreshold = config.loadSheddingThreshold ?? config.batchSize * 50;
+    this.loadSheddingEnabled = config.loadSheddingEnabled ?? true;
+    this.isDroppableEvent = config.isDroppableEvent;
+
+    // Start with minimum concurrency
+    this.limiter = pLimit(this.minConcurrency);
+
+    this.startFlushTimer();
+    this.startMetricsReporter();
+    this.setupShutdownHandlers();
+    this.#setupOtelMetrics(config.meter, schedulerName);
+  }
+
+  #setupOtelMetrics(meterOverride: Meter | undefined, name: string): void {
+    const meter = meterOverride ?? getMeter("ingest-flush");
+
+    this._batchesCounter = meter.createCounter("ingest.flush.batches", {
+      description: "Batches flushed to the sink, by outcome",
+      unit: "batches",
+    });
+    this._itemsCounter = meter.createCounter("ingest.flush.items", {
+      description: "Items successfully flushed to the sink",
+      unit: "items",
+    });
+    this._flushDurationHistogram = meter.createHistogram("ingest.flush.duration", {
+      description: "Wall-clock duration of a single batch flush",
+      unit: "ms",
+    });
+    this._batchSizeHistogram = meter.createHistogram("ingest.flush.batch_size", {
+      description: "Number of items in a flushed batch",
+      unit: "items",
+    });
+    this._droppedEventsCounter = meter.createCounter("ingest.flush.dropped_events", {
+      description: "Events dropped by load shedding before they reached the sink",
+      unit: "events",
+    });
+
+    // Pull-based gauges: read at export time only, so they add zero hot-path cost.
+    const queueDepthGauge = meter.createObservableGauge("ingest.flush.queue_depth", {
+      description: "Items queued and awaiting flush",
+      unit: "items",
+    });
+    const concurrencyGauge = meter.createObservableGauge("ingest.flush.concurrency", {
+      description: "Current concurrent-flush limit",
+      unit: "flushes",
+    });
+    const loadSheddingGauge = meter.createObservableGauge("ingest.flush.load_shedding", {
+      description: "1 while actively shedding load, otherwise 0",
+    });
+
+    meter.addBatchObservableCallback(
+      (result) => {
+        result.observe(queueDepthGauge, this.totalQueuedItems, this._metricAttrs);
+        result.observe(concurrencyGauge, this.limiter.concurrency, this._metricAttrs);
+        result.observe(loadSheddingGauge, this.isLoadShedding ? 1 : 0, this._metricAttrs);
+      },
+      [queueDepthGauge, concurrencyGauge, loadSheddingGauge]
+    );
+  }
+
+  addToBatch(items: T[]): void {
+    let itemsToAdd = items;
+
+    // Apply load shedding if enabled and we're over the threshold
+    if (this.loadSheddingEnabled && this.totalQueuedItems >= this.loadSheddingThreshold) {
+      const { kept, dropped } = this.applyLoadShedding(items);
+      itemsToAdd = kept;
+
+      if (dropped.length > 0) {
+        this.metrics.droppedEvents += dropped.length;
+        this._droppedEventsCounter?.add(dropped.length, this._metricAttrs);
+
+        // Track dropped events by kind if possible
+        dropped.forEach((item) => {
+          const kind = this.getEventKind(item);
+          if (kind) {
+            const currentCount = this.metrics.droppedEventsByKind.get(kind) || 0;
+            this.metrics.droppedEventsByKind.set(kind, currentCount + 1);
+          }
+        });
+
+        if (!this.isLoadShedding) {
+          this.isLoadShedding = true;
+        }
+
+        this.logger.warn("Load shedding", {
+          totalQueuedItems: this.totalQueuedItems,
+          threshold: this.loadSheddingThreshold,
+          droppedCount: dropped.length,
+        });
+      }
+    } else if (this.isLoadShedding && this.totalQueuedItems < this.loadSheddingThreshold * 0.8) {
+      this.isLoadShedding = false;
+      this.logger.info("Load shedding deactivated", {
+        totalQueuedItems: this.totalQueuedItems,
+        threshold: this.loadSheddingThreshold,
+        totalDropped: this.metrics.droppedEvents,
+      });
+    }
+
+    this.currentBatch.push(...itemsToAdd);
+    this.totalQueuedItems += itemsToAdd.length;
+
+    // Check if we need to create a batch (if we are shutting down, create a batch immediately because the flush timer is stopped)
+    if (this.currentBatch.length >= this.currentBatchSize || this.isShuttingDown) {
+      this.createBatch();
+    }
+
+    // Adjust concurrency based on queue pressure
+    this.adjustConcurrency();
+  }
+
+  private createBatch(): void {
+    if (this.currentBatch.length === 0) return;
+
+    this.batchQueue.push(this.currentBatch);
+    this.currentBatch = [];
+    this.flushBatches();
+    this.resetFlushTimer();
+  }
+
+  private setupShutdownHandlers(): void {
+    signalsEmitter.on("SIGTERM", () =>
+      this.shutdown().catch((error) => {
+        this.logger.error("Error shutting down dynamic flush scheduler", {
+          error,
+        });
+      })
+    );
+    signalsEmitter.on("SIGINT", () =>
+      this.shutdown().catch((error) => {
+        this.logger.error("Error shutting down dynamic flush scheduler", {
+          error,
+        });
+      })
+    );
+  }
+
+  private startFlushTimer(): void {
+    this.flushTimer = setInterval(() => this.checkAndFlush(), this.FLUSH_INTERVAL);
+  }
+
+  private resetFlushTimer(): void {
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+    }
+
+    if (this.isShuttingDown) return;
+
+    this.startFlushTimer();
+  }
+
+  private checkAndFlush(): void {
+    if (this.currentBatch.length > 0) {
+      this.createBatch();
+    }
+    this.flushBatches();
+  }
+
+  private async flushBatches(): Promise<void> {
+    const batchesToFlush: T[][] = [];
+
+    // Dequeue all available batches up to current concurrency limit
+    while (this.batchQueue.length > 0 && batchesToFlush.length < this.limiter.concurrency) {
+      const batch = this.batchQueue.shift();
+      if (batch) {
+        batchesToFlush.push(batch);
+      }
+    }
+
+    if (batchesToFlush.length === 0) return;
+
+    // Schedule all batches for concurrent processing
+    const flushPromises = batchesToFlush.map((batch) =>
+      this.limiter(async () => {
+        const itemCount = batch.length;
+
+        // eslint-disable-next-line no-this-alias
+        const self = this;
+
+        async function tryFlush(flushId: string, batchToFlush: T[], attempt: number = 1) {
+          try {
+            const startTime = Date.now();
+            await self.callback(flushId, batchToFlush);
+
+            const duration = Date.now() - startTime;
+            self.totalQueuedItems -= itemCount;
+            self.consecutiveFlushFailures = 0;
+            self.lastFlushTime = Date.now();
+            self.metrics.flushedBatches++;
+            self.metrics.totalItemsFlushed += itemCount;
+
+            self._flushDurationHistogram?.record(duration, self._metricAttrs);
+            self._batchSizeHistogram?.record(itemCount, self._metricAttrs);
+            self._itemsCounter?.add(itemCount, self._metricAttrs);
+            self._batchesCounter?.add(1, self._batchOkAttrs);
+
+            self.logger.debug("Batch flushed successfully", {
+              flushId,
+              itemCount,
+              duration,
+              remainingQueueDepth: self.totalQueuedItems,
+              activeConcurrency: self.limiter.activeCount,
+              pendingConcurrency: self.limiter.pendingCount,
+            });
+          } catch (error) {
+            self.consecutiveFlushFailures++;
+            self.metrics.failedBatches++;
+
+            self.logger.error("Error attempting to flush batch", {
+              flushId,
+              itemCount,
+              error,
+              consecutiveFailures: self.consecutiveFlushFailures,
+              attempt,
+            });
+
+            // Back off on failures
+            if (self.consecutiveFlushFailures > 5) {
+              self.adjustConcurrency(true);
+            }
+
+            if (attempt <= 3) {
+              await new Promise((resolve) => setTimeout(resolve, 500));
+              return await tryFlush(flushId, batchToFlush, attempt + 1);
+            } else {
+              throw error;
+            }
+          }
+        }
+
+        const [flushError] = await tryCatch(tryFlush(nanoid(), batch));
+
+        if (flushError) {
+          this.logger.error("Error flushing batch", {
+            error: flushError,
+          });
+          this._batchesCounter?.add(1, this._batchFailedAttrs);
+        }
+      })
+    );
+
+    // Don't await here - let them run concurrently
+    Promise.allSettled(flushPromises).then(() => {
+      const shouldContinueFlushing =
+        this.batchQueue.length > 0 && (this.consecutiveFlushFailures < 3 || this.isShuttingDown);
+      // After flush completes, check if we need to flush more
+      if (shouldContinueFlushing) {
+        this.flushBatches();
+      }
+    });
+  }
+
+  private lastConcurrencyAdjustment: number = Date.now();
+
+  private adjustConcurrency(backOff: boolean = false): void {
+    const currentConcurrency = this.limiter.concurrency;
+    let newConcurrency = currentConcurrency;
+
+    // Calculate pressure metrics - moved outside the if/else block
+    const queuePressure = this.totalQueuedItems / this.memoryPressureThreshold;
+    const timeSinceLastFlush = Date.now() - this.lastFlushTime;
+    const timeSinceLastAdjustment = Date.now() - this.lastConcurrencyAdjustment;
+
+    // Don't adjust too frequently (except for backoff)
+    if (!backOff && timeSinceLastAdjustment < 1000) {
+      return;
+    }
+
+    if (backOff) {
+      // Reduce concurrency on failures
+      newConcurrency = Math.max(this.minConcurrency, Math.floor(currentConcurrency * 0.75));
+    } else {
+      if (queuePressure > 0.8 || timeSinceLastFlush > this.FLUSH_INTERVAL * 2) {
+        // High pressure - increase concurrency
+        newConcurrency = Math.min(this.maxConcurrency, currentConcurrency + 2);
+      } else if (queuePressure < 0.2 && currentConcurrency > this.minConcurrency) {
+        // Low pressure - decrease concurrency
+        newConcurrency = Math.max(this.minConcurrency, currentConcurrency - 1);
+      }
+    }
+
+    // Adjust batch size based on pressure
+    if (this.totalQueuedItems > this.memoryPressureThreshold) {
+      this.currentBatchSize = Math.min(
+        this.maxBatchSize,
+        Math.floor(this.BATCH_SIZE * (1 + queuePressure))
+      );
+    } else {
+      this.currentBatchSize = this.BATCH_SIZE;
+    }
+
+    // Update concurrency if changed
+    if (newConcurrency !== currentConcurrency) {
+      this.limiter = pLimit(newConcurrency);
+
+      this.logger.debug("Adjusted flush concurrency", {
+        previousConcurrency: currentConcurrency,
+        newConcurrency,
+        queuePressure,
+        totalQueuedItems: this.totalQueuedItems,
+        currentBatchSize: this.currentBatchSize,
+        memoryPressureThreshold: this.memoryPressureThreshold,
+      });
+    }
+  }
+
+  private startMetricsReporter(): void {
+    // Report metrics every 30 seconds
+    this.metricsReporterTimer = setInterval(() => {
+      const droppedByKind: Record<string, number> = {};
+      this.metrics.droppedEventsByKind.forEach((count, kind) => {
+        droppedByKind[kind] = count;
+      });
+
+      this.logger.debug("DynamicFlushScheduler metrics", {
+        totalQueuedItems: this.totalQueuedItems,
+        batchQueueLength: this.batchQueue.length,
+        currentBatchLength: this.currentBatch.length,
+        currentConcurrency: this.limiter.concurrency,
+        activeConcurrent: this.limiter.activeCount,
+        pendingConcurrent: this.limiter.pendingCount,
+        currentBatchSize: this.currentBatchSize,
+        isLoadShedding: this.isLoadShedding,
+        metrics: {
+          ...this.metrics,
+          droppedByKind,
+        },
+      });
+    }, 30000);
+  }
+
+  private applyLoadShedding(items: T[]): { kept: T[]; dropped: T[] } {
+    if (!this.isDroppableEvent) {
+      // If no function provided to determine droppable events, keep all
+      return { kept: items, dropped: [] };
+    }
+
+    const kept: T[] = [];
+    const dropped: T[] = [];
+
+    for (const item of items) {
+      if (this.isDroppableEvent(item)) {
+        dropped.push(item);
+      } else {
+        kept.push(item);
+      }
+    }
+
+    return { kept, dropped };
+  }
+
+  private getEventKind(item: T): string | undefined {
+    // Try to extract the kind from the event if it has one
+    if (item && typeof item === "object" && "kind" in item) {
+      return String(item.kind);
+    }
+    return undefined;
+  }
+
+  // Method to get current status
+  getStatus() {
+    const droppedByKind: Record<string, number> = {};
+    this.metrics.droppedEventsByKind.forEach((count, kind) => {
+      droppedByKind[kind] = count;
+    });
+
+    return {
+      queuedItems: this.totalQueuedItems,
+      batchQueueLength: this.batchQueue.length,
+      currentBatchSize: this.currentBatch.length,
+      concurrency: this.limiter.concurrency,
+      activeFlushes: this.limiter.activeCount,
+      pendingFlushes: this.limiter.pendingCount,
+      isLoadShedding: this.isLoadShedding,
+      metrics: {
+        ...this.metrics,
+        droppedEventsByKind: droppedByKind,
+      },
+    };
+  }
+
+  // Graceful shutdown
+  async shutdown(): Promise<void> {
+    if (this.isShuttingDown) return;
+
+    this.isShuttingDown = true;
+
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+    }
+
+    if (this.metricsReporterTimer) {
+      clearInterval(this.metricsReporterTimer);
+    }
+
+    // Flush any remaining items
+    if (this.currentBatch.length > 0) {
+      this.createBatch();
+    }
+
+    // Wait for all pending flushes to complete
+    while (this.batchQueue.length > 0 || this.limiter.activeCount > 0) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
+}

apps/webapp/app/v3/engineDeprecation.server.ts

@@ -0,0 +1,34 @@
+import { env } from "~/env.server";
+
+/**
+ * Graceful sunset of the v3 engine (RunEngineVersion.V1).
+ *
+ * v3 maps to engine V1 (MarQS + Graphile); v4 is engine V2 (run-engine). A
+ * single master flag (DEPRECATE_V3_ENABLED, default off) gates every shutdown
+ * behaviour so the cloud can flip the switch while self-hosted instances still
+ * on V1 keep working until they migrate. This mirrors
+ * DEPRECATE_V3_CLI_DEPLOYS_ENABLED, which already gates deploys.
+ *
+ * The flag controls three surfaces:
+ *   1. Triggers that resolve to V1 are rejected with a graceful error.
+ *   2. The legacy `trigger dev` websocket (v3 CLIs only) is closed.
+ *   3. V1 run-lifecycle background jobs become no-ops to shed database load.
+ *
+ * Every call site also checks the run/project is actually V1, so v4 (V2) is
+ * never affected.
+ */
+
+export const V3_MIGRATION_URL = "https://trigger.dev/docs/migrating-from-v3";
+
+export const V3_TRIGGER_DEPRECATION_MESSAGE = `Trigger.dev v3 is no longer supported. Please upgrade your project to v4 to keep triggering tasks: ${V3_MIGRATION_URL}`;
+
+// Sent as a websocket close reason, which is capped at 123 bytes, so keep it short.
+export const V3_DEV_DEPRECATION_MESSAGE = `Trigger.dev v3 is no longer supported. Upgrade to v4: ${V3_MIGRATION_URL}`;
+
+/**
+ * Whether the v3 (engine V1) shutdown is being enforced. Guard every V1-only
+ * code path with `isV3Disabled() && <run/project is V1>` so v4 is untouched.
+ */
+export function isV3Disabled(): boolean {
+  return env.DEPRECATE_V3_ENABLED === "1";
+}

apps/webapp/app/v3/engineVersion.server.ts

@@ -0,0 +1,81 @@
+import { RunEngineVersion, type RuntimeEnvironmentType } from "@trigger.dev/database";
+import { $replica } from "~/db.server";
+import {
+  findCurrentWorkerFromEnvironment,
+  getCurrentWorkerDeploymentEngineVersion,
+} from "./models/workerDeployment.server";
+
+// Co-locate the per-env run-ops residency/mint decision next to the
+// engine-version decision. determineEngineVersion is intentionally left untouched so its
+// read-only callers (presenters, admin routes, pauseQueue) never pay the mint flag read.
+export { resolveRunIdMintKind, type RunIdMintKind } from "./runOpsMigration/runOpsMintKind.server";
+
+type Environment = {
+  id: string;
+  type: RuntimeEnvironmentType;
+  project: {
+    id: string;
+    engine: RunEngineVersion;
+  };
+};
+
+export async function determineEngineVersion({
+  environment,
+  workerVersion,
+  engineVersion: version,
+}: {
+  environment: Environment;
+  workerVersion?: string;
+  engineVersion?: RunEngineVersion;
+}): Promise<RunEngineVersion> {
+  if (version) {
+    return version;
+  }
+
+  // If the project is V1, then none of the background workers are running V2
+  if (environment.project.engine === RunEngineVersion.V1) {
+    return "V1";
+  }
+
+  /**
+   * The project has V2 enabled so it *could* be V2.
+   */
+
+  // A specific worker version is requested
+  if (workerVersion) {
+    const worker = await $replica.backgroundWorker.findUnique({
+      select: {
+        engine: true,
+      },
+      where: {
+        projectId_runtimeEnvironmentId_version: {
+          projectId: environment.project.id,
+          runtimeEnvironmentId: environment.id,
+          version: workerVersion,
+        },
+      },
+    });
+
+    if (!worker) {
+      throw new Error(`Worker not found: environment: ${environment.id} version: ${workerVersion}`);
+    }
+
+    return worker.engine;
+  }
+
+  // Dev: use the latest BackgroundWorker
+  if (environment.type === "DEVELOPMENT") {
+    const backgroundWorker = await findCurrentWorkerFromEnvironment(environment);
+    return backgroundWorker?.engine ?? "V1";
+  }
+
+  // Deployed: use the latest deployed BackgroundWorker
+  const currentDeploymentEngineVersion = await getCurrentWorkerDeploymentEngineVersion(
+    environment.id
+  );
+  if (currentDeploymentEngineVersion) {
+    return currentDeploymentEngineVersion;
+  }
+
+  return environment.project.engine;
+}

apps/webapp/app/v3/environmentVariableRules.server.ts

@@ -0,0 +1,46 @@
+import { type EnvironmentVariable } from "./environmentVariables/repository";
+
+type VariableRule =
+  | { type: "exact"; key: string }
+  | { type: "prefix"; prefix: string }
+  | { type: "whitelist"; key: string };
+
+const blacklistedVariables: VariableRule[] = [
+  { type: "exact", key: "TRIGGER_SECRET_KEY" },
+  { type: "exact", key: "TRIGGER_API_URL" },
+];
+
+const additionalExternalSyncReservedKeys = ["TRIGGER_VERSION", "TRIGGER_PREVIEW_BRANCH"];
+
+export function isBlacklistedVariable(key: string): boolean {
+  const whitelisted = blacklistedVariables.find((bv) => bv.type === "whitelist" && bv.key === key);
+  if (whitelisted) {
+    return false;
+  }
+
+  const exact = blacklistedVariables.find((bv) => bv.type === "exact" && bv.key === key);
+  if (exact) {
+    return true;
+  }
+
+  const prefix = blacklistedVariables.find(
+    (bv) => bv.type === "prefix" && key.startsWith(bv.prefix)
+  );
+  if (prefix) {
+    return true;
+  }
+
+  return false;
+}
+
+// Keys that must never be synced from an external integration (e.g. Vercel). Superset of
+// the repository blacklist so submitting a reserved key doesn't get the whole batch rejected.
+export function isReservedForExternalSync(key: string): boolean {
+  return isBlacklistedVariable(key) || additionalExternalSyncReservedKeys.includes(key);
+}
+
+export function removeBlacklistedVariables(
+  variables: EnvironmentVariable[]
+): EnvironmentVariable[] {
+  return variables.filter((v) => !isBlacklistedVariable(v.key));
+}

apps/webapp/app/v3/environmentVariables/repository.ts

@@ -0,0 +1,134 @@
+import type { RuntimeEnvironmentType } from "@trigger.dev/database";
+import { z } from "zod";
+
+export const EnvironmentVariableKey = z
+  .string()
+  .nonempty("Key is required")
+  .regex(/^\w+$/, "Keys can only use alphanumeric characters and underscores");
+
+export const EnvironmentVariableUpdaterSchema = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal("user"),
+    userId: z.string(),
+  }),
+  z.object({
+    type: z.literal("integration"),
+    integration: z.string(),
+  }),
+]);
+export type EnvironmentVariableUpdater = z.infer<typeof EnvironmentVariableUpdaterSchema>;
+
+export const CreateEnvironmentVariables = z.object({
+  override: z.boolean(),
+  environmentIds: z.array(z.string()),
+  isSecret: z.boolean().optional(),
+  parentEnvironmentId: z.string().optional(),
+  variables: z.array(z.object({ key: EnvironmentVariableKey, value: z.string() })),
+  lastUpdatedBy: EnvironmentVariableUpdaterSchema.optional(),
+});
+
+export type CreateEnvironmentVariables = z.infer<typeof CreateEnvironmentVariables>;
+
+export type CreateResult =
+  | {
+      success: true;
+    }
+  | {
+      success: false;
+      error: string;
+      variableErrors?: { key: string; error: string }[];
+    };
+
+export const EditEnvironmentVariable = z.object({
+  id: z.string(),
+  values: z.array(
+    z.object({
+      environmentId: z.string(),
+      value: z.string(),
+    })
+  ),
+  keepEmptyValues: z.boolean().optional(),
+  lastUpdatedBy: EnvironmentVariableUpdaterSchema.optional(),
+});
+export type EditEnvironmentVariable = z.infer<typeof EditEnvironmentVariable>;
+
+export const DeleteEnvironmentVariable = z.object({
+  id: z.string(),
+  environmentId: z.string().optional(),
+});
+export type DeleteEnvironmentVariable = z.infer<typeof DeleteEnvironmentVariable>;
+
+export const DeleteEnvironmentVariableValue = z.object({
+  id: z.string(),
+  environmentId: z.string(),
+});
+export type DeleteEnvironmentVariableValue = z.infer<typeof DeleteEnvironmentVariableValue>;
+
+export const EditEnvironmentVariableValue = z.object({
+  id: z.string(),
+  environmentId: z.string(),
+  value: z.string(),
+  lastUpdatedBy: EnvironmentVariableUpdaterSchema.optional(),
+});
+export type EditEnvironmentVariableValue = z.infer<typeof EditEnvironmentVariableValue>;
+
+export type Result =
+  | {
+      success: true;
+    }
+  | {
+      success: false;
+      error: string;
+    };
+
+export type ProjectEnvironmentVariable = {
+  key: string;
+  values: {
+    value: string;
+    environment: {
+      id: string;
+      type: RuntimeEnvironmentType;
+    };
+  }[];
+};
+
+export type EnvironmentVariable = {
+  key: string;
+  value: string;
+};
+
+export type EnvironmentVariableWithSecret = EnvironmentVariable & {
+  isSecret: boolean;
+};
+
+export interface Repository {
+  create(projectId: string, options: CreateEnvironmentVariables): Promise<CreateResult>;
+  edit(projectId: string, options: EditEnvironmentVariable): Promise<Result>;
+  editValue(projectId: string, options: EditEnvironmentVariableValue): Promise<Result>;
+  getProject(projectId: string): Promise<ProjectEnvironmentVariable[]>;
+  /**
+   * Fetch and decrypt only the given env var values (for dashboard display of non-secret rows).
+   * Map keys are `${environmentId}:${variableKey}`.
+   */
+  getVariableValuesForKeys(
+    projectId: string,
+    items: Array<{ environmentId: string; key: string }>
+  ): Promise<Map<string, string>>;
+  /**
+   * Get the environment variables for a given environment, it does NOT return values for secret variables
+   */
+  getEnvironmentWithRedactedSecrets(
+    projectId: string,
+    environmentId: string
+  ): Promise<EnvironmentVariableWithSecret[]>;
+  /**
+   * Get the environment variables for a given environment
+   */
+  getEnvironment(projectId: string, environmentId: string): Promise<EnvironmentVariable[]>;
+  /**
+   * Return all env vars, including secret variables with values. Should only be used for executing tasks.
+   */
+  getEnvironmentVariables(projectId: string, environmentId: string): Promise<EnvironmentVariable[]>;
+  delete(projectId: string, options: DeleteEnvironmentVariable): Promise<Result>;
+  deleteValue(projectId: string, options: DeleteEnvironmentVariableValue): Promise<Result>;
+}

apps/webapp/app/v3/eventRepository/common.server.ts

@@ -0,0 +1,181 @@
+import type { Attributes } from "@opentelemetry/api";
+import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
+import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
+import type {
+  ExceptionEventProperties,
+  SpanEvents,
+  TaskRunError,
+} from "@trigger.dev/core/v3/schemas";
+import { unflattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes";
+import { createHash } from "node:crypto";
+
+export function extractContextFromCarrier(carrier: Record<string, unknown>) {
+  const traceparent = carrier["traceparent"];
+  const tracestate = carrier["tracestate"];
+
+  if (typeof traceparent !== "string") {
+    return undefined;
+  }
+
+  return {
+    ...carrier,
+    traceparent: parseTraceparent(traceparent),
+    tracestate,
+  };
+}
+
+export function getNowInNanoseconds(): bigint {
+  return BigInt(new Date().getTime() * 1_000_000);
+}
+
+export function getDateFromNanoseconds(nanoseconds: bigint): Date {
+  return new Date(Number(nanoseconds) / 1_000_000);
+}
+
+export function calculateDurationFromStart(
+  startTime: bigint,
+  endTime: Date = new Date(),
+  minimumDuration?: number
+) {
+  const $endtime = typeof endTime === "string" ? new Date(endTime) : endTime;
+
+  const duration = Number(BigInt($endtime.getTime() * 1_000_000) - startTime);
+
+  if (minimumDuration && duration < minimumDuration) {
+    return minimumDuration;
+  }
+
+  return duration;
+}
+
+export function calculateDurationFromStartJsDate(startTime: Date, endTime: Date = new Date()) {
+  const $endtime = typeof endTime === "string" ? new Date(endTime) : endTime;
+
+  return ($endtime.getTime() - startTime.getTime()) * 1_000_000;
+}
+
+export function convertDateToNanoseconds(date: Date): bigint {
+  return BigInt(date.getTime()) * BigInt(1_000_000);
+}
+
+/**
+ * Returns a deterministically random 8-byte span ID formatted/encoded as a 16 lowercase hex
+ * characters corresponding to 64 bits, based on the trace ID and seed.
+ */
+export function generateDeterministicSpanId(traceId: string, seed: string) {
+  const hash = createHash("sha1");
+  hash.update(traceId);
+  hash.update(seed);
+  const buffer = hash.digest();
+  let hexString = "";
+  for (let i = 0; i < 8; i++) {
+    const val = buffer.readUInt8(i);
+    const str = val.toString(16).padStart(2, "0");
+    hexString += str;
+  }
+  return hexString;
+}
+
+const randomIdGenerator = new RandomIdGenerator();
+
+export function generateTraceId() {
+  return randomIdGenerator.generateTraceId();
+}
+
+export function generateSpanId() {
+  return randomIdGenerator.generateSpanId();
+}
+
+export function stripAttributePrefix(attributes: Attributes, prefix: string) {
+  const result: Attributes = {};
+
+  for (const [key, value] of Object.entries(attributes)) {
+    if (key.startsWith(prefix)) {
+      result[key.slice(prefix.length + 1)] = value;
+    } else {
+      result[key] = value;
+    }
+  }
+  return result;
+}
+
+export function parseEventsField(events: unknown): SpanEvents {
+  if (!events) return [];
+  if (!Array.isArray(events)) return [];
+
+  const unsafe = events
+    ? (events as any[]).map((e) => ({
+        ...e,
+        properties: unflattenAttributes(e.properties as Attributes),
+      }))
+    : undefined;
+
+  return unsafe as SpanEvents;
+}
+
+export function createExceptionPropertiesFromError(error: TaskRunError): ExceptionEventProperties {
+  switch (error.type) {
+    case "BUILT_IN_ERROR": {
+      return {
+        type: error.name,
+        message: error.message,
+        stacktrace: error.stackTrace,
+      };
+    }
+    case "CUSTOM_ERROR": {
+      return {
+        type: "Error",
+        message: error.raw,
+      };
+    }
+    case "INTERNAL_ERROR": {
+      return {
+        type: "Internal error",
+        message: [error.code, error.message].filter(Boolean).join(": "),
+        stacktrace: error.stackTrace,
+      };
+    }
+    case "STRING_ERROR": {
+      return {
+        type: "Error",
+        message: error.raw,
+      };
+    }
+  }
+}
+
+// Removes internal/private attribute keys from span properties.
+// Filters: "$" prefixed keys (private metadata) and "ctx." prefixed keys (Trigger.dev run context)
+export function removePrivateProperties(
+  attributes: Attributes | undefined | null
+): Attributes | undefined {
+  if (!attributes) {
+    return undefined;
+  }
+
+  const result: Attributes = {};
+
+  for (const [key, value] of Object.entries(attributes)) {
+    if (key.startsWith("$") || key.startsWith("ctx.")) {
+      continue;
+    }
+
+    result[key] = value;
+  }
+
+  if (Object.keys(result).length === 0) {
+    return undefined;
+  }
+
+  return result;
+}
+
+export function isEmptyObject(obj: object) {
+  for (var prop in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, prop)) {
+      return false;
+    }
+  }
+
+  return true;
+}

apps/webapp/app/v3/eventRepository/emergencySpanCap.server.ts

@@ -0,0 +1,8 @@
+import { env } from "~/env.server";
+
+// Emergency circuit breaker for trace views: when TRACE_VIEW_EMERGENCY_SPAN_CAP
+// is set, clamp a trace summary span limit to it. Unset = no clamping.
+export function clampToEmergencySpanCap(limit: number): number {
+  const cap = env.TRACE_VIEW_EMERGENCY_SPAN_CAP;
+  return cap === undefined ? limit : Math.min(limit, cap);
+}

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -0,0 +1,500 @@
+import type { Attributes, Tracer } from "@opentelemetry/api";
+import type {
+  ExceptionEventProperties,
+  SpanEvents,
+  TaskEventEnvironment,
+  TaskEventStyle,
+  TaskRunError,
+} from "@trigger.dev/core/v3";
+import type {
+  Prisma,
+  TaskEvent,
+  TaskEventKind,
+  TaskEventLevel,
+  TaskEventStatus,
+  TaskRun,
+} from "@trigger.dev/database";
+import type { MetricsV1Input } from "@internal/clickhouse";
+import type { DetailedTraceEvent, TaskEventStoreTable } from "../taskEventStore.server";
+export type { ExceptionEventProperties };
+
+// ============================================================================
+// Event Creation Types
+// ============================================================================
+
+export type LlmMetricsData = {
+  genAiSystem: string;
+  requestModel: string;
+  responseModel: string;
+  baseResponseModel: string;
+  matchedModelId: string;
+  operationId: string;
+  finishReason: string;
+  costSource: string;
+  pricingTierId: string;
+  pricingTierName: string;
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  usageDetails: Record<string, number>;
+  inputCost: number;
+  outputCost: number;
+  totalCost: number;
+  costDetails: Record<string, number>;
+  providerCost: number;
+  msToFirstChunk: number;
+  tokensPerSecond: number;
+  metadata: Record<string, string>;
+  promptSlug: string;
+  promptVersion: number;
+};
+
+export type CreateEventInput = Omit<
+  Prisma.TaskEventCreateInput,
+  | "id"
+  | "createdAt"
+  | "properties"
+  | "metadata"
+  | "style"
+  | "output"
+  | "payload"
+  | "serviceName"
+  | "serviceNamespace"
+  | "tracestate"
+  | "projectRef"
+  | "runIsTest"
+  | "workerId"
+  | "queueId"
+  | "queueName"
+  | "batchId"
+  | "taskPath"
+  | "taskExportName"
+  | "workerVersion"
+  | "idempotencyKey"
+  | "attemptId"
+  | "usageDurationMs"
+  | "usageCostInCents"
+  | "machinePreset"
+  | "machinePresetCpu"
+  | "machinePresetMemory"
+  | "machinePresetCentsPerMs"
+  | "links"
+> & {
+  properties: Attributes;
+  resourceProperties?: Attributes;
+  metadata: Attributes | undefined;
+  style: Attributes | undefined;
+  machineId?: string;
+  runTags?: string[];
+  /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */
+  _llmMetrics?: LlmMetricsData;
+};
+
+export type CreatableEventKind = TaskEventKind;
+export type CreatableEventStatus = TaskEventStatus;
+
+// ============================================================================
+// Task Run Types
+// ============================================================================
+
+export type CompleteableTaskRun = Pick<
+  TaskRun,
+  | "friendlyId"
+  | "traceId"
+  | "spanId"
+  | "parentSpanId"
+  | "createdAt"
+  | "completedAt"
+  | "taskIdentifier"
+  | "projectId"
+  | "runtimeEnvironmentId"
+  | "organizationId"
+  | "isTest"
+>;
+
+// ============================================================================
+// Trace and Event Types
+// ============================================================================
+
+export type TraceAttributes = Partial<
+  Pick<
+    CreateEventInput,
+    "isError" | "isCancelled" | "isDebug" | "runId" | "metadata" | "properties" | "style"
+  >
+>;
+
+export type SetAttribute<T extends TraceAttributes> = (key: keyof T, value: T[keyof T]) => void;
+
+export type TraceEventOptions = {
+  kind?: CreatableEventKind;
+  context?: Record<string, unknown>;
+  spanParentAsLink?: boolean;
+  spanIdSeed?: string;
+  attributes: TraceAttributes;
+  environment: TaskEventEnvironment;
+  taskSlug: string;
+  startTime?: bigint;
+  endTime?: Date;
+  immediate?: boolean;
+};
+
+export type EventBuilder = {
+  traceId: string;
+  spanId: string;
+  setAttribute: SetAttribute<TraceAttributes>;
+  stop: () => void;
+  failWithError: (error: TaskRunError) => void;
+};
+
+export type UpdateEventOptions = {
+  attributes: TraceAttributes;
+  endTime?: Date;
+  immediate?: boolean;
+  events?: SpanEvents;
+};
+
+// ============================================================================
+// Configuration Types
+// ============================================================================
+
+export type EventRepoConfig = {
+  batchSize: number;
+  batchInterval: number;
+  retentionInDays: number;
+  partitioningEnabled: boolean;
+  tracer?: Tracer;
+  minConcurrency?: number;
+  maxConcurrency?: number;
+  maxBatchSize?: number;
+  memoryPressureThreshold?: number;
+  loadSheddingThreshold?: number;
+  loadSheddingEnabled?: boolean;
+};
+
+// ============================================================================
+// Query Types
+// ============================================================================
+
+export type QueryOptions = Prisma.TaskEventWhereInput;
+
+export type TaskEventRecord = TaskEvent;
+
+export type QueriedEvent = Prisma.TaskEventGetPayload<{
+  select: {
+    spanId: true;
+    parentId: true;
+    runId: true;
+    message: true;
+    style: true;
+    startTime: true;
+    duration: true;
+    isError: true;
+    isPartial: true;
+    isCancelled: true;
+    level: true;
+    events: true;
+    kind: true;
+    attemptNumber: true;
+  };
+}>;
+
+export type PreparedEvent = Omit<QueriedEvent, "events" | "style" | "duration"> & {
+  duration: number;
+  events: SpanEvents;
+  style: TaskEventStyle;
+};
+
+export type PreparedDetailedEvent = Omit<DetailedTraceEvent, "events" | "style" | "duration"> & {
+  duration: number;
+  events: SpanEvents;
+  style: TaskEventStyle;
+};
+
+export type RunPreparedEvent = PreparedEvent & {
+  taskSlug?: string;
+};
+
+export type SpanDetail = {
+  // ============================================================================
+  // Core Identity & Structure
+  // ============================================================================
+  spanId: string; // Tree structure, span identification
+  parentId: string | null; // Tree hierarchy
+  message: string; // Displayed as span title
+
+  // ============================================================================
+  // Status & State
+  // ============================================================================
+  isError: boolean; // Error status display, filtering, status icons
+  isPartial: boolean; // In-progress status display, timeline calculations
+  isCancelled: boolean; // Cancelled status display, status determination
+  level: TaskEventLevel; // Text styling, timeline rendering decisions
+
+  // ============================================================================
+  // Timing
+  // ============================================================================
+  startTime: Date; // Timeline calculations, display
+  duration: number; // Timeline width, duration display, calculations
+
+  // ============================================================================
+  // Content & Display
+  // ============================================================================
+  events: SpanEvents; // Timeline events, SpanEvents component
+  style: TaskEventStyle; // Icons, variants, accessories (RunIcon, SpanTitle)
+  properties: Record<string, unknown> | string | number | boolean | null | undefined; // Displayed as JSON in span properties (CodeBlock)
+  resourceProperties?: Record<string, unknown> | string | number | boolean | null | undefined; // Displayed as JSON in span resource properties (CodeBlock)
+
+  // ============================================================================
+  // Entity & Relationships
+  // ============================================================================
+  entity: {
+    // Used for entity type switching in SpanEntity
+    type: string | undefined;
+    id: string | undefined;
+    metadata: string | undefined;
+  };
+
+  metadata: any; // Used by SpanPresenter for entity processing
+};
+
+// ============================================================================
+// Span and Link Types
+// ============================================================================
+
+export type SpanSummaryCommon = {
+  id: string;
+  parentId: string | undefined;
+  runId: string;
+  data: {
+    message: string;
+    events: SpanEvents;
+    startTime: Date;
+    duration: number;
+    isError: boolean;
+    isPartial: boolean;
+    isCancelled: boolean;
+    level: NonNullable<CreateEventInput["level"]>;
+    attemptNumber?: number;
+  };
+};
+
+export type SpanSummary = {
+  id: string;
+  parentId: string | undefined;
+  runId: string;
+  data: {
+    message: string;
+    style: TaskEventStyle;
+    events: SpanEvents;
+    startTime: Date;
+    duration: number;
+    isError: boolean;
+    isPartial: boolean;
+    isCancelled: boolean;
+    isDebug: boolean;
+    level: NonNullable<CreateEventInput["level"]>;
+    attemptNumber?: number;
+  };
+};
+
+export type SpanOverride = {
+  isCancelled?: boolean;
+  isError?: boolean;
+  duration?: number;
+  events?: SpanEvents;
+};
+
+export type TraceSummary = {
+  rootSpan: SpanSummary;
+  spans: Array<SpanSummary>;
+  overridesBySpanId?: Record<string, SpanOverride>;
+  /** Set when a subtree fetch hit the row cap before collecting all descendants. */
+  isTruncated?: boolean;
+};
+
+export type SpanDetailedSummary = {
+  id: string;
+  parentId: string | undefined;
+  runId: string;
+  data: {
+    message: string;
+    taskSlug?: string;
+    events: SpanEvents;
+    startTime: Date;
+    duration: number;
+    isError: boolean;
+    isPartial: boolean;
+    isCancelled: boolean;
+    level: NonNullable<CreateEventInput["level"]>;
+    attemptNumber?: number;
+    properties?: Attributes;
+  };
+  children: Array<SpanDetailedSummary>;
+};
+
+// A single trace event for the streaming export path (the "Download trace"
+// feature). Deliberately flat and self-contained: it carries its own parent ref
+// so hierarchy is reconstructable downstream without ever building a tree. Used
+// by `streamTraceEvents`, which yields these one at a time so an arbitrarily
+// large trace is never fully resident in memory.
+export type StreamedTraceEvent = {
+  spanId: string;
+  parentSpanId: string;
+  startTime: Date;
+  durationNs: number;
+  level: string;
+  message: string;
+  isError: boolean;
+  // Span attributes/properties as a raw JSON string, emitted verbatim (the
+  // ClickHouse store already materialises it as text — no per-row parse).
+  propertiesText: string;
+};
+
+export type TraceDetailedSummary = {
+  traceId: string;
+  rootSpan: SpanDetailedSummary;
+  /** Set when a fetch hit the row cap before collecting all spans. */
+  isTruncated?: boolean;
+};
+
+// ============================================================================
+// Event Repository Interface
+// ============================================================================
+
+/**
+ * Interface for the EventRepository class.
+ * Defines the public API for managing task events, traces, and spans.
+ */
+export interface IEventRepository {
+  maximumLiveReloadingSetting: number;
+  // Event insertion methods
+  insertMany(events: CreateEventInput[]): void;
+  insertManyImmediate(events: CreateEventInput[]): Promise<void>;
+  insertManyMetrics(rows: MetricsV1Input[]): void;
+
+  // Run event completion methods
+  completeSuccessfulRunEvent(params: { run: CompleteableTaskRun; endTime?: Date }): Promise<void>;
+
+  completeCachedRunEvent(params: {
+    run: CompleteableTaskRun;
+    blockedRun: CompleteableTaskRun;
+    spanId: string;
+    parentSpanId: string;
+    spanCreatedAt: Date;
+    isError: boolean;
+    endTime?: Date;
+  }): Promise<void>;
+
+  completeFailedRunEvent(params: {
+    run: CompleteableTaskRun;
+    endTime?: Date;
+    exception: { message?: string; type?: string; stacktrace?: string };
+  }): Promise<void>;
+
+  completeExpiredRunEvent(params: {
+    run: CompleteableTaskRun;
+    endTime?: Date;
+    ttl: string;
+  }): Promise<void>;
+
+  createAttemptFailedRunEvent(params: {
+    run: CompleteableTaskRun;
+    endTime?: Date;
+    attemptNumber: number;
+    exception: { message?: string; type?: string; stacktrace?: string };
+  }): Promise<void>;
+
+  cancelRunEvent(params: {
+    reason: string;
+    run: CompleteableTaskRun;
+    cancelledAt: Date;
+  }): Promise<void>;
+
+  // Query methods
+  getTraceSummary(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): Promise<TraceSummary | undefined>;
+
+  /** Fetch the anchor span, its ancestors (for override propagation), and all descendants. */
+  getTraceSubtreeSummary(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    anchorSpanId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): Promise<TraceSummary | undefined>;
+
+  getTraceDetailedSummary(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): Promise<TraceDetailedSummary | undefined>;
+
+  /** Fetch the anchor span subtree as a detailed hierarchical trace rooted at anchorSpanId. */
+  getTraceDetailedSubtreeSummary(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    anchorSpanId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): Promise<TraceDetailedSummary | undefined>;
+
+  // Streams a trace's events in start_time order, one at a time, without ever
+  // materialising the full result set or a tree. Powers the streaming trace
+  // export so arbitrarily large traces download with bounded memory.
+  streamTraceEvents(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): AsyncIterable<StreamedTraceEvent>;
+
+  getRunEvents(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    traceId: string,
+    runId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date
+  ): Promise<RunPreparedEvent[]>;
+
+  getSpan(
+    storeTable: TaskEventStoreTable,
+    environmentId: string,
+    spanId: string,
+    traceId: string,
+    startCreatedAt: Date,
+    endCreatedAt?: Date,
+    options?: { includeDebugLogs?: boolean }
+  ): Promise<SpanDetail | undefined>;
+
+  // Event recording methods
+  recordEvent(
+    message: string,
+    options: TraceEventOptions & { duration?: number; parentId?: string }
+  ): Promise<void>;
+
+  traceEvent<TResult>(
+    message: string,
+    options: TraceEventOptions & { incomplete?: boolean; isError?: boolean },
+    callback: (
+      e: EventBuilder,
+      traceContext: Record<string, string | undefined>,
+      traceparent?: { traceId: string; spanId: string }
+    ) => Promise<TResult>
+  ): Promise<TResult>;
+}

apps/webapp/app/v3/eventRepository/index.server.ts

@@ -0,0 +1,300 @@
+import { env } from "~/env.server";
+import { eventRepository } from "./eventRepository.server";
+import { type IEventRepository, type TraceEventOptions } from "./eventRepository.types";
+import { prisma } from "~/db.server";
+import { runStore } from "../runStore.server";
+import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { logger } from "~/services/logger.server";
+import { FEATURE_FLAG } from "../featureFlags";
+import { flag } from "../featureFlags.server";
+import { getTaskEventStore } from "../taskEventStore.server";
+import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
+
+export const EVENT_STORE_TYPES = {
+  POSTGRES: "postgres",
+  CLICKHOUSE: "clickhouse",
+  CLICKHOUSE_V2: "clickhouse_v2",
+} as const;
+
+export type EventStoreType = (typeof EVENT_STORE_TYPES)[keyof typeof EVENT_STORE_TYPES];
+/**
+ * Async variant of {@link resolveEventRepositoryForStore}. Awaits the factory's
+ * registry readiness before returning the ClickHouse event repository; for
+ * non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
+ * runs) it returns the Prisma event repository without ever touching the
+ * factory — so the factory never needs to know about Postgres.
+ */
+export async function getEventRepositoryForStore(
+  store: string,
+  organizationId: string
+): Promise<IEventRepository> {
+  if (store !== EVENT_STORE_TYPES.CLICKHOUSE && store !== EVENT_STORE_TYPES.CLICKHOUSE_V2) {
+    return eventRepository;
+  }
+  const { repository } = await clickhouseFactory.getEventRepositoryForOrganization(
+    store,
+    organizationId
+  );
+  return repository;
+}
+
+export async function getConfiguredEventRepository(
+  organizationId: string
+): Promise<{ repository: IEventRepository; store: EventStoreType }> {
+  const organization = await prisma.organization.findFirst({
+    select: {
+      id: true,
+      featureFlags: true,
+    },
+    where: {
+      id: organizationId,
+    },
+  });
+
+  if (!organization) {
+    throw new Error("Organization not found when configuring event repository");
+  }
+
+  // resolveTaskEventRepositoryFlag checks:
+  // 1. organization.featureFlags (highest priority)
+  // 2. global feature flags (via flags() function)
+  // 3. env.EVENT_REPOSITORY_DEFAULT_STORE (fallback)
+  const taskEventStore = await resolveTaskEventRepositoryFlag(
+    (organization.featureFlags as Record<string, unknown> | null) ?? undefined
+  );
+
+  if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) {
+    const { repository: resolvedRepository } =
+      await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
+    return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
+  }
+
+  if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE) {
+    const { repository: resolvedRepository } =
+      await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
+    return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
+  }
+
+  return { repository: eventRepository, store: EVENT_STORE_TYPES.POSTGRES };
+}
+
+export async function getEventRepository(
+  organizationId: string,
+  featureFlags: Record<string, unknown> | undefined,
+  parentStore: string | undefined
+): Promise<{ repository: IEventRepository; store: string }> {
+  const taskEventStore = parentStore ?? (await resolveTaskEventRepositoryFlag(featureFlags));
+
+  // Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed
+  // runs, or the legacy "postgres" value) resolve to the Prisma event repo.
+  if (
+    taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
+    taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
+  ) {
+    return { repository: eventRepository, store: getTaskEventStore() };
+  }
+
+  const { repository: resolvedRepository } =
+    await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId);
+
+  switch (taskEventStore) {
+    case EVENT_STORE_TYPES.CLICKHOUSE_V2: {
+      return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 };
+    }
+    case EVENT_STORE_TYPES.CLICKHOUSE: {
+      return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE };
+    }
+    default: {
+      return { repository: eventRepository, store: getTaskEventStore() };
+    }
+  }
+}
+
+export async function getV3EventRepository(
+  organizationId: string,
+  parentStore: string | undefined
+): Promise<{ repository: IEventRepository; store: string }> {
+  if (typeof parentStore === "string") {
+    // Support legacy Postgres store for self-hosters and runs persisted with a
+    // non-ClickHouse store — fall back to the Prisma-based event repository.
+    if (
+      parentStore !== EVENT_STORE_TYPES.CLICKHOUSE &&
+      parentStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2
+    ) {
+      return { repository: eventRepository, store: parentStore };
+    }
+
+    const { repository: resolvedRepository } =
+      await clickhouseFactory.getEventRepositoryForOrganization(parentStore, organizationId);
+    return { repository: resolvedRepository, store: parentStore };
+  }
+
+  if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse_v2") {
+    const { repository: resolvedRepository } =
+      await clickhouseFactory.getEventRepositoryForOrganization("clickhouse_v2", organizationId);
+    return { repository: resolvedRepository, store: "clickhouse_v2" };
+  } else if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse") {
+    const { repository: resolvedRepository } =
+      await clickhouseFactory.getEventRepositoryForOrganization("clickhouse", organizationId);
+    return { repository: resolvedRepository, store: "clickhouse" };
+  } else {
+    return { repository: eventRepository, store: getTaskEventStore() };
+  }
+}
+
+async function resolveTaskEventRepositoryFlag(
+  featureFlags: Record<string, unknown> | undefined
+): Promise<"clickhouse" | "clickhouse_v2" | "postgres"> {
+  const flagResult = await flag({
+    key: FEATURE_FLAG.taskEventRepository,
+    defaultValue: env.EVENT_REPOSITORY_DEFAULT_STORE,
+    overrides: featureFlags,
+  });
+
+  if (flagResult === "clickhouse_v2") {
+    return "clickhouse_v2";
+  }
+
+  if (flagResult === "clickhouse") {
+    return "clickhouse";
+  }
+
+  return flagResult;
+}
+
+export async function recordRunDebugLog(
+  runId: string,
+  message: string,
+  options: Omit<TraceEventOptions, "environment" | "taskSlug" | "startTime"> & {
+    duration?: number;
+    parentId?: string;
+    startTime?: Date;
+  }
+): Promise<
+  | {
+      success: true;
+    }
+  | {
+      success: false;
+      code: "RUN_NOT_FOUND" | "FAILED_TO_RECORD_EVENT";
+      error?: unknown;
+    }
+> {
+  if (env.EVENT_REPOSITORY_DEBUG_LOGS_DISABLED) {
+    // drop debug events silently
+    return {
+      success: true,
+    };
+  }
+
+  return recordRunEvent(runId, message, {
+    ...options,
+    attributes: {
+      ...options?.attributes,
+      isDebug: true,
+    },
+  });
+}
+
+async function recordRunEvent(
+  runId: string,
+  message: string,
+  options: Omit<TraceEventOptions, "environment" | "taskSlug" | "startTime"> & {
+    duration?: number;
+    parentId?: string;
+    startTime?: Date;
+  }
+): Promise<
+  | {
+      success: true;
+    }
+  | {
+      success: false;
+      code: "RUN_NOT_FOUND" | "FAILED_TO_RECORD_EVENT";
+      error?: unknown;
+    }
+> {
+  try {
+    const foundRun = await findRunForEventCreation(runId);
+
+    if (!foundRun) {
+      logger.error("Failed to find run for event creation", { runId });
+      return {
+        success: false,
+        code: "RUN_NOT_FOUND",
+      };
+    }
+
+    const $eventRepository = await getEventRepositoryForStore(
+      foundRun.taskEventStore,
+      foundRun.runtimeEnvironment.organizationId
+    );
+
+    const { attributes, startTime, ...optionsRest } = options;
+
+    await $eventRepository.recordEvent(message, {
+      environment: foundRun.runtimeEnvironment,
+      taskSlug: foundRun.taskIdentifier,
+      context: foundRun.traceContext as Record<string, string | undefined>,
+      attributes: {
+        runId: foundRun.friendlyId,
+        ...attributes,
+      },
+      startTime: BigInt((startTime?.getTime() ?? Date.now()) * 1_000_000),
+      ...optionsRest,
+    });
+
+    return {
+      success: true,
+    };
+  } catch (error) {
+    logger.error("Failed to record event for run", {
+      error: error instanceof Error ? error.message : error,
+      runId,
+    });
+
+    return {
+      success: false,
+      code: "FAILED_TO_RECORD_EVENT",
+      error,
+    };
+  }
+}
+
+async function findRunForEventCreation(runId: string) {
+  const foundRun = await runStore.findRun(
+    {
+      id: runId,
+    },
+    {
+      select: {
+        friendlyId: true,
+        taskIdentifier: true,
+        traceContext: true,
+        taskEventStore: true,
+        runtimeEnvironmentId: true,
+      },
+    },
+    prisma
+  );
+
+  if (!foundRun) {
+    return null;
+  }
+
+  const environment = await controlPlaneResolver.resolveAuthenticatedEnv(
+    foundRun.runtimeEnvironmentId
+  );
+
+  if (!environment) {
+    // Run exists but its environment could not be resolved (e.g. a lagging replica
+    // under split); distinguish this from a genuinely missing run.
+    logger.warn("Run found but environment unresolved for event creation", {
+      runId,
+      runtimeEnvironmentId: foundRun.runtimeEnvironmentId,
+    });
+    return null;
+  }
+
+  return { ...foundRun, runtimeEnvironment: environment };
+}

apps/webapp/app/v3/eventRepository/sanitizeRowsOnParseError.server.ts

@@ -0,0 +1,164 @@
+import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings";
+
+/**
+ * Replacement string we substitute for any attribute value that contains
+ * a lone UTF-16 surrogate. JSON-safe, distinctly recognisable in logs and
+ * the dashboard so operators can spot affected rows.
+ */
+export const INVALID_UTF16_SENTINEL = "[invalid-utf16]";
+
+/**
+ * ClickHouse's `JSON(max_dynamic_paths)` column fits each bare-integer
+ * JSON token into Int64 (signed) or UInt64 (unsigned). Bare integers
+ * outside `[-2^63, 2^64 - 1]` are rejected with `INCORRECT_DATA` (no
+ * silent fallback to Float64). `JSON.stringify` emits any integer-valued
+ * Number with `|value| < 1e21` as a bare integer (no exponent), so any
+ * JS Number above ~9.2e18 that *happens* to be integer-valued lands on
+ * the wire as a token CH cannot accept.
+ *
+ * The fix: replace such Numbers with their string form. CH's dynamic
+ * JSON column accepts a `String` subtype on the same path, so the row
+ * inserts cleanly on retry. The numeric value was already
+ * precision-lossy upstream (JS Number can't represent integers above
+ * 2^53 faithfully), so type-flipping to string is information-preserving
+ * relative to what arrived.
+ *
+ * Float-valued numbers (including very large ones like `1e25`) serialise
+ * with an exponent and are accepted by CH at any magnitude, so they're
+ * left alone.
+ */
+const UINT64_MAX = 18446744073709551615n;
+const INT64_MIN = -9223372036854775808n;
+
+function isUnsafeJsonInteger(value: number): boolean {
+  if (!Number.isFinite(value)) return false;
+  if (!Number.isInteger(value)) return false;
+  // JSON.stringify emits integer-valued Numbers as bare integer tokens
+  // (no exponent) only while `|value| < 1e21`; at or above that
+  // threshold `Number.prototype.toString` switches to exponential form,
+  // which CH accepts as Float64 at any magnitude. So the dangerous band
+  // is strictly between the Int64/UInt64 boundary and 1e21.
+  if (Math.abs(value) >= 1e21) return false;
+  // Compare via BigInt for exactness. The Number literal 18446744073709551615
+  // is rounded to 2**64 in float64 (the float spacing near 2^64 is 2048), so a
+  // direct `value > 18446744073709551615` would miss a Number whose float64
+  // value is exactly 2**64 — `JSON.stringify` of that emits
+  // "18446744073709552000", which exceeds UInt64.MAX and ClickHouse rejects.
+  // `BigInt(value)` is safe here because we already gated on Number.isInteger.
+  const asBigInt = BigInt(value);
+  return asBigInt > UINT64_MAX || asBigInt < INT64_MIN;
+}
+
+export type SanitizeResult = {
+  /** How many rows had at least one string field replaced. */
+  rowsTouched: number;
+  /** Total count of string fields replaced across all sanitized rows. */
+  fieldsSanitized: number;
+};
+
+/**
+ * Recognises ClickHouse's "Cannot parse JSON object" rejection — the
+ * deterministic-failure class our sanitizer is designed for. Bubbles up
+ * from `@clickhouse/client` as an `InsertError` whose `.message` retains
+ * the original ClickHouse error text.
+ */
+export function isClickHouseJsonParseError(err: unknown): boolean {
+  if (!err) return false;
+  const message =
+    typeof err === "object" && err !== null && "message" in err
+      ? String((err as { message?: unknown }).message ?? "")
+      : String(err);
+  return message.includes("Cannot parse JSON object");
+}
+
+/**
+ * Extracts the row index ClickHouse reported as the first to fail
+ * (`(at row N)`). Returns `null` if the message doesn't include one —
+ * caller should treat that as "sanitize from row 0".
+ */
+export function parseRowNumberFromError(errorMessage: string): number | null {
+  const match = errorMessage.match(/at row (\d+)/);
+  return match ? Number.parseInt(match[1], 10) : null;
+}
+
+/**
+ * Walks `value` recursively and replaces any string leaf that contains a
+ * lone UTF-16 surrogate with `INVALID_UTF16_SENTINEL`. Mutates objects
+ * and arrays in place; primitives are returned unchanged.
+ *
+ * Caller passes anything: a row object, a single field, an unknown JSON
+ * payload. The walker doesn't depend on the row's schema — it sanitizes
+ * every string in the structure, which is exactly what ClickHouse cares
+ * about when parsing the row's JSON form.
+ */
+export function sanitizeUnknownInPlace(value: unknown): { value: unknown; fixed: number } {
+  if (typeof value === "string") {
+    // `detectBadJsonStrings` works on JSON-escaped text — feed it the
+    // serialized form so any lone UTF-16 surrogate in the JS string is
+    // emitted as a `\uXXXX` escape it can spot. Valid surrogate pairs
+    // (e.g. emoji) are emitted as raw characters by JSON.stringify and
+    // exit at the function's fast path.
+    if (detectBadJsonStrings(JSON.stringify(value))) {
+      return { value: INVALID_UTF16_SENTINEL, fixed: 1 };
+    }
+    return { value, fixed: 0 };
+  }
+
+  if (typeof value === "number" && isUnsafeJsonInteger(value)) {
+    return { value: String(value), fixed: 1 };
+  }
+
+  if (Array.isArray(value)) {
+    let fixed = 0;
+    for (let i = 0; i < value.length; i++) {
+      const result = sanitizeUnknownInPlace(value[i]);
+      value[i] = result.value;
+      fixed += result.fixed;
+    }
+    return { value, fixed };
+  }
+
+  if (value !== null && typeof value === "object") {
+    let fixed = 0;
+    const obj = value as Record<string, unknown>;
+    for (const k of Object.keys(obj)) {
+      const result = sanitizeUnknownInPlace(obj[k]);
+      obj[k] = result.value;
+      fixed += result.fixed;
+    }
+    return { value, fixed };
+  }
+
+  return { value, fixed: 0 };
+}
+
+/**
+ * Sanitizes every row in `rows`, mutating each in place so callers can
+ * hand the same array to the retry insert.
+ *
+ * Rationale for scanning the whole batch (instead of starting from the
+ * row index ClickHouse reports): `at row N` semantics under
+ * `input_format_parallel_parsing` aren't well-defined — N can be
+ * chunk-relative rather than batch-global, and 0-vs-1 indexing differs
+ * between formats. Whole-batch scanning is robust to those quirks and
+ * also catches multiple bad rows in one pass (so a single retry covers
+ * the entire failure even if more than one row is poisoned).
+ *
+ * The cost is bounded: this only runs on the rare ClickHouse-rejection
+ * path, and `detectBadJsonStrings` exits in O(1) for clean strings
+ * (the fast `indexOf("\\u")` check), so healthy attributes are effectively
+ * free even when included in the walk.
+ */
+export function sanitizeRows<T extends object>(rows: T[]): SanitizeResult {
+  const result: SanitizeResult = { rowsTouched: 0, fieldsSanitized: 0 };
+
+  for (let i = 0; i < rows.length; i++) {
+    const { fixed } = sanitizeUnknownInPlace(rows[i]);
+    if (fixed > 0) {
+      result.rowsTouched++;
+      result.fieldsSanitized += fixed;
+    }
+  }
+
+  return result;
+}

apps/webapp/app/v3/eventRepository/traceExport.server.ts

@@ -0,0 +1,206 @@
+import { formatDurationNanoseconds } from "@trigger.dev/core/v3/utils/durations";
+import type { StreamedTraceEvent } from "./eventRepository.types";
+
+// Lines are batched into ~64KB chunks before being yielded to the gzip stream:
+// per-line chunks make `Readable.from` ~10x slower, while chunks this size keep
+// the pipe fed yet stay small enough to release the event loop frequently.
+const DEFAULT_FLUSH_BYTES = 64 * 1024;
+
+export type TraceExportContext = {
+  runFriendlyId: string;
+  traceId: string;
+  taskIdentifier?: string;
+  /** Absolute dashboard URL for the run (used by formats that link out). */
+  runUrl?: string;
+};
+
+/**
+ * A trace export format. `formatEvent` renders one {@link StreamedTraceEvent} to
+ * a string; `header`/`footer` bookend the stream. Formats are intentionally
+ * stateless across events so the export stays O(1) memory — see
+ * {@link streamTraceExport}.
+ */
+export type TraceExportFormat = {
+  name: TraceExportFormatName;
+  extension: string;
+  header?: (ctx: TraceExportContext) => string;
+  formatEvent: (event: StreamedTraceEvent, ctx: TraceExportContext) => string;
+  footer?: (ctx: TraceExportContext) => string;
+};
+
+export type TraceExportFormatName = "log" | "jsonl" | "markdown";
+
+/**
+ * Streams a trace export by piping events through a {@link TraceExportFormat}.
+ * Batches output into ~`flushBytes` chunks and releases the event loop between
+ * flushes; holds nothing across events but the buffer, so an arbitrarily large
+ * trace exports in bounded memory regardless of format.
+ */
+export async function* streamTraceExport(
+  events: AsyncIterable<StreamedTraceEvent>,
+  format: TraceExportFormat,
+  ctx: TraceExportContext,
+  options: { flushBytes?: number } = {}
+): AsyncGenerator<string> {
+  const flushBytes = options.flushBytes ?? DEFAULT_FLUSH_BYTES;
+
+  let buffer = format.header ? format.header(ctx) : "";
+
+  for await (const event of events) {
+    buffer += format.formatEvent(event, ctx);
+    if (buffer.length >= flushBytes) {
+      yield buffer;
+      buffer = "";
+      await new Promise<void>((resolve) => setImmediate(resolve));
+    }
+  }
+
+  if (format.footer) {
+    buffer += format.footer(ctx);
+  }
+
+  if (buffer.length > 0) {
+    yield buffer;
+  }
+}
+
+function hasProperties(propertiesText: string): boolean {
+  const trimmed = propertiesText.trim();
+  return trimmed.length > 0 && trimmed !== "{}";
+}
+
+// For error events, pull the error message out of the properties so formats can
+// surface it inline instead of burying it in the JSON blob. Only parses when the
+// event is actually an error (rare), so the common path stays parse-free. Handles
+// both trigger.dev's `error.*` shape and OTel's `exception.*` shape; the full
+// object (incl. stacktrace) still rides along in the properties.
+function errorMessage(event: StreamedTraceEvent): string | undefined {
+  if (!event.isError || !hasProperties(event.propertiesText)) return undefined;
+  try {
+    const props = JSON.parse(event.propertiesText) as {
+      error?: { message?: unknown };
+      exception?: { message?: unknown };
+    };
+    const message = props.error?.message ?? props.exception?.message;
+    return typeof message === "string" && message.length > 0
+      ? message.replace(/\s+/g, " ").trim()
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function lineage(event: StreamedTraceEvent): string {
+  return event.parentSpanId ? `${event.spanId} ← ${event.parentSpanId}` : event.spanId;
+}
+
+// ---------------------------------------------------------------------------
+// log — flat, chronological, grep-friendly. One line per event (+ a props line).
+// ---------------------------------------------------------------------------
+const logFormat: TraceExportFormat = {
+  name: "log",
+  extension: "txt",
+  formatEvent(event) {
+    const time = event.startTime.toISOString();
+    const level = event.level.padEnd(5);
+    const errMsg = errorMessage(event);
+    const status = event.isError ? (errMsg ? ` [ERROR: ${errMsg}]` : " [ERROR]") : "";
+    const duration =
+      event.durationNs > 0 ? ` (${formatDurationNanoseconds(event.durationNs)})` : "";
+
+    let out = `${time} ${level} [${lineage(event)}] ${event.message}${status}${duration}\n`;
+    if (hasProperties(event.propertiesText)) {
+      out += `    props: ${event.propertiesText.trim()}\n`;
+    }
+    return out;
+  },
+};
+
+// ---------------------------------------------------------------------------
+// jsonl — one JSON object per line, properties inlined as a nested object.
+// ---------------------------------------------------------------------------
+const jsonlFormat: TraceExportFormat = {
+  name: "jsonl",
+  extension: "jsonl",
+  formatEvent(event) {
+    let properties: unknown = undefined;
+    if (hasProperties(event.propertiesText)) {
+      try {
+        properties = JSON.parse(event.propertiesText);
+      } catch {
+        properties = event.propertiesText;
+      }
+    }
+
+    return (
+      JSON.stringify({
+        time: event.startTime.toISOString(),
+        level: event.level,
+        spanId: event.spanId,
+        parentSpanId: event.parentSpanId || undefined,
+        message: event.message,
+        durationNs: event.durationNs,
+        isError: event.isError || undefined,
+        errorMessage: errorMessage(event),
+        properties,
+      }) + "\n"
+    );
+  },
+};
+
+// ---------------------------------------------------------------------------
+// markdown — AI-friendly: YAML frontmatter (ids, task, dashboard URL) + a
+// scannable table, one row per event. Properties stay (inline code) so the
+// export isn't lossy; a column-friendly cell escaper keeps the table intact.
+// ---------------------------------------------------------------------------
+function mdCell(value: string): string {
+  // Pipes and newlines would break the table row; escape/flatten them. (GFM
+  // treats `\|` inside a table cell — including code spans — as a literal pipe.)
+  return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+
+const markdownFormat: TraceExportFormat = {
+  name: "markdown",
+  extension: "md",
+  header(ctx) {
+    const lines = ["---", `run: ${ctx.runFriendlyId}`, `trace: ${ctx.traceId}`];
+    if (ctx.taskIdentifier) lines.push(`task: ${ctx.taskIdentifier}`);
+    if (ctx.runUrl) lines.push(`url: ${ctx.runUrl}`);
+    lines.push("---", "", `# Trace for ${ctx.runFriendlyId}`, "");
+    if (ctx.runUrl) {
+      lines.push(`[View in dashboard](${ctx.runUrl})`, "");
+    }
+    lines.push(
+      "| time | level | event | duration | span ← parent | properties |",
+      "| --- | --- | --- | --- | --- | --- |"
+    );
+    return lines.join("\n") + "\n";
+  },
+  formatEvent(event) {
+    const time = event.startTime.toISOString();
+    const level = event.isError ? `${event.level} ❌` : event.level;
+    const duration = event.durationNs > 0 ? formatDurationNanoseconds(event.durationNs) : "—";
+    const lineage = event.parentSpanId ? `${event.spanId} ← ${event.parentSpanId}` : event.spanId;
+    const errMsg = errorMessage(event);
+    const eventCell = errMsg ? `${event.message} — ${errMsg}` : event.message;
+    const properties = hasProperties(event.propertiesText)
+      ? "`" + mdCell(event.propertiesText.trim()) + "`"
+      : "—";
+
+    return `| ${time} | ${level} | ${mdCell(eventCell)} | ${duration} | \`${lineage}\` | ${properties} |\n`;
+  },
+};
+
+const FORMATS: Record<TraceExportFormatName, TraceExportFormat> = {
+  log: logFormat,
+  jsonl: jsonlFormat,
+  markdown: markdownFormat,
+};
+
+/** Resolve a `?format=` value to a format, defaulting to `log`. */
+export function getTraceExportFormat(name: string | null | undefined): TraceExportFormat {
+  if (name && Object.prototype.hasOwnProperty.call(FORMATS, name)) {
+    return FORMATS[name as TraceExportFormatName];
+  }
+  return logFormat;
+}

apps/webapp/app/v3/failedTaskRun.server.ts

@@ -0,0 +1,304 @@
+import type {
+  TaskRunExecution,
+  TaskRunExecutionRetry,
+  TaskRunFailedExecutionResult,
+  V3TaskRunExecution,
+} from "@trigger.dev/core/v3";
+import { calculateNextRetryDelay, RetryOptions } from "@trigger.dev/core/v3";
+import type { Prisma, TaskRun } from "@trigger.dev/database";
+import * as semver from "semver";
+import { logger } from "~/services/logger.server";
+import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server";
+import { BaseService } from "./services/baseService.server";
+import { CompleteAttemptService } from "./services/completeAttempt.server";
+import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { isFailableRunStatus, isFinalAttemptStatus } from "./taskStatus";
+
+const FailedTaskRunRetryGetPayload = {
+  select: {
+    id: true,
+    attempts: {
+      orderBy: {
+        createdAt: "desc",
+      },
+      take: 1,
+    },
+    lockedById: true, // task
+    lockedToVersionId: true, // worker
+  },
+} as const;
+
+type TaskRunWithAttempts = Prisma.TaskRunGetPayload<typeof FailedTaskRunRetryGetPayload>;
+
+export class FailedTaskRunService extends BaseService {
+  public async call(anyRunId: string, completion: TaskRunFailedExecutionResult) {
+    logger.debug("[FailedTaskRunService] Handling failed task run", { anyRunId, completion });
+
+    const isFriendlyId = anyRunId.startsWith("run_");
+
+    const taskRun = await this.runStore.findRun(
+      {
+        friendlyId: isFriendlyId ? anyRunId : undefined,
+        id: !isFriendlyId ? anyRunId : undefined,
+      },
+      this._prisma
+    );
+
+    if (!taskRun) {
+      logger.error("[FailedTaskRunService] Task run not found", {
+        anyRunId,
+        completion,
+      });
+
+      return;
+    }
+
+    if (!isFailableRunStatus(taskRun.status)) {
+      logger.error("[FailedTaskRunService] Task run is not in a failable state", {
+        taskRun,
+        completion,
+      });
+
+      return;
+    }
+
+    const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
+    const retryResult = await retryHelper.call({
+      runId: taskRun.id,
+      completion,
+    });
+
+    logger.debug("[FailedTaskRunService] Completion result", {
+      runId: taskRun.id,
+      result: retryResult,
+    });
+  }
+}
+
+interface TaskRunWithWorker extends TaskRun {
+  lockedBy: { retryConfig: Prisma.JsonValue } | null;
+  lockedToVersion: { sdkVersion: string } | null;
+}
+
+export class FailedTaskRunRetryHelper extends BaseService {
+  async call({
+    runId,
+    completion,
+    isCrash,
+  }: {
+    runId: string;
+    completion: TaskRunFailedExecutionResult;
+    isCrash?: boolean;
+  }) {
+    const taskRun = await this.runStore.findRun(
+      {
+        id: runId,
+      },
+      FailedTaskRunRetryGetPayload,
+      this._prisma
+    );
+
+    if (!taskRun) {
+      logger.error("[FailedTaskRunRetryHelper] Task run not found", {
+        runId,
+        completion,
+      });
+
+      return "NO_TASK_RUN";
+    }
+
+    const retriableExecution = await this.#getRetriableAttemptExecution(taskRun, completion);
+
+    if (!retriableExecution) {
+      return "NO_EXECUTION";
+    }
+
+    logger.debug("[FailedTaskRunRetryHelper] Completing attempt", { taskRun, completion });
+
+    const completeAttempt = new CompleteAttemptService({
+      prisma: this._prisma,
+      isSystemFailure: !isCrash,
+      isCrash,
+    });
+    const completeResult = await completeAttempt.call({
+      completion,
+      execution: retriableExecution,
+    });
+
+    return completeResult;
+  }
+
+  async #getRetriableAttemptExecution(
+    run: TaskRunWithAttempts,
+    completion: TaskRunFailedExecutionResult
+  ): Promise<V3TaskRunExecution | undefined> {
+    let attempt = run.attempts[0];
+
+    // We need to create an attempt if:
+    // - None exists yet
+    // - The last attempt has a final status, e.g. we failed between attempts
+    if (!attempt || isFinalAttemptStatus(attempt.status)) {
+      logger.debug("[FailedTaskRunRetryHelper] No attempts found", {
+        run,
+        completion,
+      });
+
+      const createAttempt = new CreateTaskRunAttemptService(this._prisma);
+
+      try {
+        const { execution } = await createAttempt.call({
+          runId: run.id,
+          // This ensures we correctly respect `maxAttempts = 1` when failing before the first attempt was created
+          startAtZero: true,
+        });
+        return execution;
+      } catch (error) {
+        logger.error("[FailedTaskRunRetryHelper] Failed to create attempt", {
+          run,
+          completion,
+          error,
+        });
+
+        return;
+      }
+    }
+
+    // We already have an attempt with non-final status, let's use it
+    try {
+      const executionPayload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+        id: attempt.id,
+        skipStatusChecks: true,
+      });
+
+      return executionPayload?.execution;
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution payload", {
+        run,
+        completion,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static getExecutionRetry({
+    run,
+    execution,
+  }: {
+    run: TaskRunWithWorker;
+    execution: TaskRunExecution;
+  }): TaskRunExecutionRetry | undefined {
+    try {
+      const retryConfig = FailedTaskRunRetryHelper.getRetryConfig({ run, execution });
+      if (!retryConfig) {
+        return;
+      }
+
+      const delay = calculateNextRetryDelay(retryConfig, execution.attempt.number);
+
+      if (!delay) {
+        logger.debug("[FailedTaskRunRetryHelper] No more retries", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      return {
+        timestamp: Date.now() + delay,
+        delay,
+      };
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution retry", {
+        run,
+        execution,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static getRetryConfig({
+    run,
+    execution,
+  }: {
+    run: TaskRunWithWorker;
+    execution: TaskRunExecution;
+  }): RetryOptions | undefined {
+    try {
+      const retryConfig = run.lockedBy?.retryConfig;
+
+      if (!retryConfig) {
+        if (!run.lockedToVersion) {
+          logger.error("[FailedTaskRunRetryHelper] Run not locked to version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        const sdkVersion = run.lockedToVersion.sdkVersion ?? "0.0.0";
+        const isValid = semver.valid(sdkVersion);
+
+        if (!isValid) {
+          logger.error("[FailedTaskRunRetryHelper] Invalid SDK version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        // With older SDK versions, tasks only have a retry config stored in the DB if it's explicitly defined on the task itself
+        // It won't get populated with retry.default in trigger.config.ts
+        if (semver.lt(sdkVersion, FailedTaskRunRetryHelper.DEFAULT_RETRY_CONFIG_SINCE_VERSION)) {
+          logger.warn(
+            "[FailedTaskRunRetryHelper] SDK version not recent enough to determine retry config",
+            {
+              run,
+              execution,
+            }
+          );
+
+          return;
+        }
+      }
+
+      const parsedRetryConfig = RetryOptions.nullable().safeParse(retryConfig);
+
+      if (!parsedRetryConfig.success) {
+        logger.error("[FailedTaskRunRetryHelper] Invalid retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      if (!parsedRetryConfig.data) {
+        logger.debug("[FailedTaskRunRetryHelper] No retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      return parsedRetryConfig.data;
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution retry", {
+        run,
+        execution,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static DEFAULT_RETRY_CONFIG_SINCE_VERSION = "3.1.0";
+}

apps/webapp/app/v3/featureFlags.server.ts

@@ -0,0 +1,148 @@
+import { type z } from "zod";
+import { prisma, type PrismaClientOrTransaction } from "~/db.server";
+import {
+  type FeatureFlagCatalogSchema,
+  type FeatureFlagKey,
+  FeatureFlagCatalog,
+} from "~/v3/featureFlags";
+
+export type FlagsOptions<T extends FeatureFlagKey> = {
+  key: T;
+  defaultValue?: z.infer<(typeof FeatureFlagCatalog)[T]>;
+  overrides?: Record<string, unknown>;
+};
+
+export function makeFlag(_prisma: PrismaClientOrTransaction = prisma) {
+  function flag<T extends FeatureFlagKey>(
+    opts: FlagsOptions<T> & { defaultValue: z.infer<(typeof FeatureFlagCatalog)[T]> }
+  ): Promise<z.infer<(typeof FeatureFlagCatalog)[T]>>;
+  function flag<T extends FeatureFlagKey>(
+    opts: FlagsOptions<T>
+  ): Promise<z.infer<(typeof FeatureFlagCatalog)[T]> | undefined>;
+  async function flag<T extends FeatureFlagKey>(
+    opts: FlagsOptions<T>
+  ): Promise<z.infer<(typeof FeatureFlagCatalog)[T]> | undefined> {
+    const value = await _prisma.featureFlag.findFirst({
+      where: {
+        key: opts.key,
+      },
+    });
+
+    const flagSchema = FeatureFlagCatalog[opts.key];
+
+    if (opts.overrides?.[opts.key] !== undefined) {
+      const parsed = flagSchema.safeParse(opts.overrides[opts.key]);
+
+      if (parsed.success) {
+        return parsed.data;
+      }
+    }
+
+    if (value !== null) {
+      const parsed = flagSchema.safeParse(value.value);
+
+      if (parsed.success) {
+        return parsed.data;
+      }
+    }
+
+    return opts.defaultValue;
+  }
+
+  return flag;
+}
+
+export function makeSetFlag(_prisma: PrismaClientOrTransaction = prisma) {
+  return async function setFlag<T extends FeatureFlagKey>(
+    opts: FlagsOptions<T> & { value: z.infer<(typeof FeatureFlagCatalog)[T]> }
+  ): Promise<void> {
+    await _prisma.featureFlag.upsert({
+      where: {
+        key: opts.key,
+      },
+      create: {
+        key: opts.key,
+        value: opts.value,
+      },
+      update: {
+        value: opts.value,
+      },
+    });
+  };
+}
+
+export type AllFlagsOptions = {
+  defaultValues?: Partial<FeatureFlagCatalog>;
+  overrides?: Record<string, unknown>;
+};
+
+export function makeFlags(_prisma: PrismaClientOrTransaction = prisma) {
+  return async function flags(options?: AllFlagsOptions): Promise<Partial<FeatureFlagCatalog>> {
+    const rows = await _prisma.featureFlag.findMany();
+
+    // Build a map of key -> value from database
+    const dbValues = new Map<string, unknown>();
+    for (const row of rows) {
+      dbValues.set(row.key, row.value);
+    }
+
+    const result: Partial<FeatureFlagCatalog> = {};
+
+    // Process each flag in the catalog
+    for (const key of Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]) {
+      const schema = FeatureFlagCatalog[key];
+
+      // Priority: overrides > database > defaultValues
+      if (options?.overrides?.[key] !== undefined) {
+        const parsed = schema.safeParse(options.overrides[key]);
+        if (parsed.success) {
+          (result as any)[key] = parsed.data;
+          continue;
+        }
+      }
+
+      if (dbValues.has(key)) {
+        const parsed = schema.safeParse(dbValues.get(key));
+        if (parsed.success) {
+          (result as any)[key] = parsed.data;
+          continue;
+        }
+      }
+
+      if (options?.defaultValues?.[key] !== undefined) {
+        const parsed = schema.safeParse(options.defaultValues[key]);
+        if (parsed.success) {
+          (result as any)[key] = parsed.data;
+        }
+      }
+    }
+
+    return result;
+  };
+}
+
+export const flag = makeFlag();
+export const flags = makeFlags();
+export const setFlag = makeSetFlag();
+
+// Utility function to set multiple feature flags at once
+export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma) {
+  return async function setMultipleFlags(
+    flags: Partial<z.infer<typeof FeatureFlagCatalogSchema>>
+  ): Promise<{ key: string; value: any }[]> {
+    const setFlag = makeSetFlag(_prisma);
+    const updatedFlags: { key: string; value: any }[] = [];
+
+    for (const [key, value] of Object.entries(flags)) {
+      if (value !== undefined) {
+        await setFlag({
+          key: key as any,
+          value: value as any,
+        });
+        updatedFlags.push({ key, value });
+      }
+    }
+
+    return updatedFlags;
+  };
+}

apps/webapp/app/v3/featureFlags.ts

@@ -0,0 +1,133 @@
+import { z } from "zod";
+
+export const FEATURE_FLAG = {
+  defaultWorkerInstanceGroupId: "defaultWorkerInstanceGroupId",
+  taskEventRepository: "taskEventRepository",
+  hasQueryAccess: "hasQueryAccess",
+  hasLogsPageAccess: "hasLogsPageAccess",
+  hasAiAccess: "hasAiAccess",
+  hasDashboardAgentAccess: "hasDashboardAgentAccess",
+  hasComputeAccess: "hasComputeAccess",
+  hasPrivateConnections: "hasPrivateConnections",
+  hasSso: "hasSso",
+  mollifierEnabled: "mollifierEnabled",
+  workerQueueScheduledSplitEnabled: "workerQueueScheduledSplitEnabled",
+  realtimeBackend: "realtimeBackend",
+  computeMigrationEnabled: "computeMigrationEnabled",
+  computeMigrationFreePercentage: "computeMigrationFreePercentage",
+  computeMigrationPaidPercentage: "computeMigrationPaidPercentage",
+  computeMigrationRequireTemplate: "computeMigrationRequireTemplate",
+  devBranchesEnabled: "devBranchesEnabled",
+  runOpsMintKind: "runOpsMintKind",
+} as const;
+
+export const FeatureFlagCatalog = {
+  [FEATURE_FLAG.defaultWorkerInstanceGroupId]: z.string(),
+  [FEATURE_FLAG.taskEventRepository]: z.enum(["clickhouse", "clickhouse_v2", "postgres"]),
+  [FEATURE_FLAG.hasQueryAccess]: z.coerce.boolean(),
+  [FEATURE_FLAG.hasLogsPageAccess]: z.coerce.boolean(),
+  [FEATURE_FLAG.hasAiAccess]: z.coerce.boolean(),
+  // Gates the in-dashboard AI agent panel. Controllable globally and per-org
+  // (org wins). Defaults off via DASHBOARD_AGENT_ENABLED.
+  [FEATURE_FLAG.hasDashboardAgentAccess]: z.coerce.boolean(),
+  [FEATURE_FLAG.hasComputeAccess]: z.coerce.boolean(),
+  [FEATURE_FLAG.hasPrivateConnections]: z.coerce.boolean(),
+  [FEATURE_FLAG.hasSso]: z.coerce.boolean(),
+  [FEATURE_FLAG.mollifierEnabled]: z.coerce.boolean(),
+  [FEATURE_FLAG.workerQueueScheduledSplitEnabled]: z.coerce.boolean(),
+  // Which backend serves the realtime run feed. Controllable
+  // globally and per-org (org wins). Defaults to "electric" when unset.
+  // "shadow" serves Electric but diffs the native path in the background.
+  [FEATURE_FLAG.realtimeBackend]: z.enum(["electric", "native", "shadow"]),
+  // Strict z.boolean() (not z.coerce.boolean()): coercion turns the string "false"
+  // into true, which would silently flip this kill switch / per-org exclude the wrong
+  // way if written as a string via the admin PAT route. The admin toggle sends a real
+  // boolean, so this only rejects the dangerous stringified case.
+  [FEATURE_FLAG.computeMigrationEnabled]: z.boolean(),
+  [FEATURE_FLAG.computeMigrationFreePercentage]: z.coerce.number().int().min(0).max(100),
+  [FEATURE_FLAG.computeMigrationPaidPercentage]: z.coerce.number().int().min(0).max(100),
+  // When on, migrated orgs build their compute template in required mode at deploy
+  // (fails the deploy on error) instead of shadow. Strict boolean (see above).
+  [FEATURE_FLAG.computeMigrationRequireTemplate]: z.boolean(),
+  // Per-org access to development branches. Off unless enabled for the org.
+  [FEATURE_FLAG.devBranchesEnabled]: z.coerce.boolean(),
+  // Per-org run-ops-id mint cutover. Defaults to "cuid"; only honored when
+  // RUN_OPS_MINT_ENABLED is on AND isSplitEnabled() is true.
+  [FEATURE_FLAG.runOpsMintKind]: z.enum(["cuid", "runOpsId"]),
+};
+
+export type FeatureFlagKey = keyof typeof FeatureFlagCatalog;
+
+// Infrastructure flags that are read-only on the global flags page.
+// Shown with current/resolved value but no controls.
+export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [
+  FEATURE_FLAG.defaultWorkerInstanceGroupId,
+  FEATURE_FLAG.taskEventRepository,
+];
+
+// Flags that are read-only on the org-level dialog.
+// Shown with global value but no controls (org can't override these).
+export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [
+  FEATURE_FLAG.defaultWorkerInstanceGroupId,
+  FEATURE_FLAG.taskEventRepository,
+];
+
+// Create a Zod schema from the existing catalog
+export const FeatureFlagCatalogSchema = z.object(FeatureFlagCatalog);
+export type FeatureFlagCatalog = z.infer<typeof FeatureFlagCatalogSchema>;
+
+// Utility function to validate a feature flag value
+export function validateFeatureFlagValue<T extends FeatureFlagKey>(
+  key: T,
+  value: unknown
+): z.SafeParseReturnType<unknown, z.infer<(typeof FeatureFlagCatalog)[T]>> {
+  return FeatureFlagCatalog[key].safeParse(value);
+}
+
+// Utility function to validate all feature flags at once
+export function validateAllFeatureFlags(values: Record<string, unknown>) {
+  return FeatureFlagCatalogSchema.safeParse(values);
+}
+
+// Utility function to validate partial feature flags (all keys optional)
+export function validatePartialFeatureFlags(values: Record<string, unknown>) {
+  return FeatureFlagCatalogSchema.partial().safeParse(values);
+}
+
+// Utility types for catalog-driven UI rendering
+export type FlagControlType =
+  | { type: "boolean" }
+  | { type: "enum"; options: string[] }
+  | { type: "number"; min?: number; max?: number }
+  | { type: "string" };
+
+export function getFlagControlType(schema: z.ZodTypeAny): FlagControlType {
+  const typeName = schema._def.typeName;
+
+  if (typeName === "ZodBoolean") {
+    return { type: "boolean" };
+  }
+
+  if (typeName === "ZodEnum") {
+    return { type: "enum", options: schema._def.values as string[] };
+  }
+
+  // z.coerce.number() reports as ZodNumber; pull min/max out of its checks
+  // so the UI can render a constrained number input instead of free text.
+  if (typeName === "ZodNumber") {
+    const checks = (schema._def.checks ?? []) as Array<{ kind: string; value?: number }>;
+    const min = checks.find((c) => c.kind === "min")?.value;
+    const max = checks.find((c) => c.kind === "max")?.value;
+    return { type: "number", min, max };
+  }
+
+  return { type: "string" };
+}
+
+export function getAllFlagControlTypes(): Record<string, FlagControlType> {
+  const result: Record<string, FlagControlType> = {};
+  for (const [key, schema] of Object.entries(FeatureFlagCatalog)) {
+    result[key] = getFlagControlType(schema);
+  }
+  return result;
+}

apps/webapp/app/v3/friendlyIdentifiers.ts

@@ -0,0 +1 @@
+export { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic";

apps/webapp/app/v3/getDeploymentImageRef.server.ts

@@ -0,0 +1,565 @@
+import {
+  ECRClient,
+  CreateRepositoryCommand,
+  DescribeRepositoriesCommand,
+  type Repository,
+  type Tag,
+  RepositoryNotFoundException,
+  GetAuthorizationTokenCommand,
+  PutLifecyclePolicyCommand,
+  PutImageTagMutabilityCommand,
+  SetRepositoryPolicyCommand,
+} from "@aws-sdk/client-ecr";
+import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts";
+import { tryCatch } from "@trigger.dev/core";
+import { logger } from "~/services/logger.server";
+import { type RegistryConfig } from "./registryConfig.server";
+import type { EnvironmentType } from "@trigger.dev/core/v3";
+
+// Optional configuration for cross-account access
+export type AssumeRoleConfig = {
+  roleArn?: string;
+  externalId?: string;
+};
+
+async function getAssumedRoleCredentials({
+  region,
+  assumeRole,
+}: {
+  region: string;
+  assumeRole?: AssumeRoleConfig;
+}): Promise<{
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken: string;
+}> {
+  const sts = new STSClient({ region });
+
+  // Generate a unique session name using timestamp and random string
+  // This helps with debugging but doesn't affect concurrent sessions
+  const timestamp = Date.now();
+  const randomSuffix = Math.random().toString(36).substring(2, 8);
+  const sessionName = `TriggerWebappECRAccess_${timestamp}_${randomSuffix}`;
+
+  const [error, response] = await tryCatch(
+    sts.send(
+      new AssumeRoleCommand({
+        RoleArn: assumeRole?.roleArn,
+        RoleSessionName: sessionName,
+        // Sessions automatically expire after 1 hour
+        // AWS allows 5000 concurrent sessions by default
+        DurationSeconds: 3600,
+        ExternalId: assumeRole?.externalId,
+      })
+    )
+  );
+
+  if (error) {
+    logger.error("Failed to assume role", {
+      assumeRole,
+      sessionName,
+      error: error.message,
+    });
+    throw error;
+  }
+
+  if (!response.Credentials) {
+    throw new Error("STS: No credentials returned from assumed role");
+  }
+
+  if (
+    !response.Credentials.AccessKeyId ||
+    !response.Credentials.SecretAccessKey ||
+    !response.Credentials.SessionToken
+  ) {
+    throw new Error("STS: Invalid credentials returned from assumed role");
+  }
+
+  return {
+    accessKeyId: response.Credentials.AccessKeyId,
+    secretAccessKey: response.Credentials.SecretAccessKey,
+    sessionToken: response.Credentials.SessionToken,
+  };
+}
+
+export async function createEcrClient({
+  region,
+  assumeRole,
+}: {
+  region: string;
+  assumeRole?: AssumeRoleConfig;
+}) {
+  if (!assumeRole) {
+    return new ECRClient({ region });
+  }
+
+  // Get credentials for cross-account access
+  const credentials = await getAssumedRoleCredentials({ region, assumeRole });
+  return new ECRClient({
+    region,
+    credentials,
+  });
+}
+
+export async function getDeploymentImageRef({
+  registry,
+  projectRef,
+  nextVersion,
+  environmentType,
+  deploymentShortCode,
+}: {
+  registry: RegistryConfig;
+  projectRef: string;
+  nextVersion: string;
+  environmentType: EnvironmentType;
+  deploymentShortCode: string;
+}): Promise<{
+  imageRef: string;
+  isEcr: boolean;
+  repoCreated: boolean;
+}> {
+  const repositoryName = `${registry.namespace}/${projectRef}`;
+  const envType = environmentType.toLowerCase();
+  const imageRef = `${registry.host}/${repositoryName}:${nextVersion}.${envType}.${deploymentShortCode}`;
+
+  if (!isEcrRegistry(registry.host)) {
+    return {
+      imageRef,
+      isEcr: false,
+      repoCreated: false,
+    };
+  }
+
+  const [ecrRepoError, ecrData] = await tryCatch(
+    ensureEcrRepositoryExists({
+      repositoryName,
+      registryHost: registry.host,
+      registryTags: registry.ecrTags,
+      assumeRole: {
+        roleArn: registry.ecrAssumeRoleArn,
+        externalId: registry.ecrAssumeRoleExternalId,
+      },
+      defaultRepositoryPolicy: registry.ecrDefaultRepositoryPolicy,
+    })
+  );
+
+  if (ecrRepoError) {
+    logger.error("Failed to ensure ECR repository exists", {
+      repositoryName,
+      host: registry.host,
+      ecrRepoError: ecrRepoError.message,
+    });
+    throw ecrRepoError;
+  }
+
+  return {
+    imageRef,
+    isEcr: true,
+    repoCreated: ecrData.repoCreated,
+  };
+}
+
+export function isEcrRegistry(registryHost: string) {
+  try {
+    parseEcrRegistryDomain(registryHost);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function parseRegistryTags(tags: string): Tag[] {
+  if (!tags) {
+    return [];
+  }
+
+  return tags
+    .split(",")
+    .map((t) => {
+      const tag = t.trim();
+      if (tag.length === 0) {
+        return null;
+      }
+
+      // If there's no '=' in the tag, treat the whole tag as the key with an empty value
+      const equalIndex = tag.indexOf("=");
+      const key = equalIndex === -1 ? tag : tag.slice(0, equalIndex);
+      const value = equalIndex === -1 ? "" : tag.slice(equalIndex + 1);
+
+      if (key.trim().length === 0) {
+        logger.warn("Invalid ECR tag format (empty key), skipping tag", { tag: t });
+        return null;
+      }
+
+      return {
+        Key: key.trim(),
+        Value: value.trim(),
+      } as Tag;
+    })
+    .filter((tag): tag is Tag => tag !== null);
+}
+
+const untaggedImageExpirationPolicy = JSON.stringify({
+  rules: [
+    {
+      rulePriority: 1,
+      description: "Expire untagged images older than 3 days",
+      selection: {
+        tagStatus: "untagged",
+        countType: "sinceImagePushed",
+        countUnit: "days",
+        countNumber: 3,
+      },
+      action: { type: "expire" },
+    },
+  ],
+});
+
+async function createEcrRepository({
+  repositoryName,
+  region,
+  accountId,
+  registryTags,
+  assumeRole,
+  defaultRepositoryPolicy,
+}: {
+  repositoryName: string;
+  region: string;
+  accountId?: string;
+  registryTags?: string;
+  assumeRole?: AssumeRoleConfig;
+  defaultRepositoryPolicy?: string;
+}): Promise<Repository> {
+  const ecr = await createEcrClient({ region, assumeRole });
+
+  const result = await ecr.send(
+    new CreateRepositoryCommand({
+      repositoryName,
+      imageTagMutability: "IMMUTABLE_WITH_EXCLUSION",
+      imageTagMutabilityExclusionFilters: [
+        {
+          // only the `cache` tag will be mutable, all other tags will be immutable
+          filter: "cache",
+          filterType: "WILDCARD",
+        },
+      ],
+      encryptionConfiguration: {
+        encryptionType: "AES256",
+      },
+      registryId: accountId,
+      tags: registryTags ? parseRegistryTags(registryTags) : undefined,
+    })
+  );
+
+  if (!result.repository) {
+    logger.error("Failed to create ECR repository", { repositoryName, result });
+    throw new Error(`Failed to create ECR repository: ${repositoryName}`);
+  }
+
+  // When the `cache` tag is mutated, the old cache images are untagged.
+  // This policy matches those images and expires them to avoid bloating the repository.
+  await ecr.send(
+    new PutLifecyclePolicyCommand({
+      repositoryName: result.repository.repositoryName,
+      registryId: result.repository.registryId,
+      lifecyclePolicyText: untaggedImageExpirationPolicy,
+    })
+  );
+
+  // Apply an operator-provided IAM policy to the new repository. Useful for
+  // self-hosters whose ECR account is separate from the account running the
+  // EKS workers — without this the workers get 403 Forbidden when pulling the
+  // task image (default ECR policy only grants access to the registry owner).
+  // The existing-repo branch of `ensureEcrRepositoryExists` reconciles this
+  // same policy on every call, so a partial-create that fails here is
+  // self-healing on the next deploy.
+  if (defaultRepositoryPolicy) {
+    await applyEcrRepositoryPolicy({
+      repositoryName: result.repository.repositoryName!,
+      region,
+      accountId: result.repository.registryId ?? accountId,
+      assumeRole,
+      defaultRepositoryPolicy,
+    });
+  }
+
+  return result.repository;
+}
+
+async function applyEcrRepositoryPolicy({
+  repositoryName,
+  region,
+  accountId,
+  assumeRole,
+  defaultRepositoryPolicy,
+}: {
+  repositoryName: string;
+  region: string;
+  accountId?: string;
+  assumeRole?: AssumeRoleConfig;
+  defaultRepositoryPolicy: string;
+}): Promise<void> {
+  const ecr = await createEcrClient({ region, assumeRole });
+
+  await ecr.send(
+    new SetRepositoryPolicyCommand({
+      repositoryName,
+      registryId: accountId,
+      policyText: defaultRepositoryPolicy,
+    })
+  );
+}
+
+async function updateEcrRepositoryCacheSettings({
+  repositoryName,
+  region,
+  accountId,
+  assumeRole,
+}: {
+  repositoryName: string;
+  region: string;
+  accountId?: string;
+  assumeRole?: AssumeRoleConfig;
+}): Promise<void> {
+  logger.debug("Updating ECR repository tag mutability to IMMUTABLE_WITH_EXCLUSION", {
+    repositoryName,
+    region,
+  });
+
+  const ecr = await createEcrClient({ region, assumeRole });
+
+  await ecr.send(
+    new PutImageTagMutabilityCommand({
+      repositoryName,
+      registryId: accountId,
+      imageTagMutability: "IMMUTABLE_WITH_EXCLUSION",
+      imageTagMutabilityExclusionFilters: [
+        {
+          // only the `cache` tag will be mutable, all other tags will be immutable
+          filter: "cache",
+          filterType: "WILDCARD",
+        },
+      ],
+    })
+  );
+
+  // When the `cache` tag is mutated, the old cache images are untagged.
+  // This policy matches those images and expires them to avoid bloating the repository.
+  await ecr.send(
+    new PutLifecyclePolicyCommand({
+      repositoryName,
+      registryId: accountId,
+      lifecyclePolicyText: untaggedImageExpirationPolicy,
+    })
+  );
+
+  logger.debug("Successfully updated ECR repository to IMMUTABLE_WITH_EXCLUSION", {
+    repositoryName,
+    region,
+  });
+}
+
+async function getEcrRepository({
+  repositoryName,
+  region,
+  accountId,
+  assumeRole,
+}: {
+  repositoryName: string;
+  region: string;
+  accountId?: string;
+  assumeRole?: AssumeRoleConfig;
+}): Promise<Repository | undefined> {
+  const ecr = await createEcrClient({ region, assumeRole });
+
+  try {
+    const result = await ecr.send(
+      new DescribeRepositoriesCommand({
+        repositoryNames: [repositoryName],
+        registryId: accountId,
+      })
+    );
+
+    if (!result.repositories || result.repositories.length === 0) {
+      logger.debug("ECR repository not found", { repositoryName, region, result });
+      return undefined;
+    }
+
+    return result.repositories[0];
+  } catch (error) {
+    if (
+      error instanceof RepositoryNotFoundException ||
+      (error instanceof Error && error.message?.includes("does not exist"))
+    ) {
+      logger.debug("ECR repository not found: RepositoryNotFoundException", {
+        repositoryName,
+        region,
+      });
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+export type EcrRegistryComponents = {
+  accountId: string;
+  region: string;
+};
+
+export function parseEcrRegistryDomain(registryHost: string): EcrRegistryComponents {
+  const parts = registryHost.split(".");
+
+  const isValid =
+    parts.length === 6 &&
+    parts[1] === "dkr" &&
+    parts[2] === "ecr" &&
+    parts[4] === "amazonaws" &&
+    parts[5] === "com";
+
+  if (!isValid) {
+    throw new Error(`Invalid ECR registry host: ${registryHost}`);
+  }
+
+  return {
+    accountId: parts[0],
+    region: parts[3],
+  };
+}
+
+async function ensureEcrRepositoryExists({
+  repositoryName,
+  registryHost,
+  registryTags,
+  assumeRole,
+  defaultRepositoryPolicy,
+}: {
+  repositoryName: string;
+  registryHost: string;
+  registryTags?: string;
+  assumeRole?: AssumeRoleConfig;
+  defaultRepositoryPolicy?: string;
+}): Promise<{ repo: Repository; repoCreated: boolean }> {
+  const { region, accountId } = parseEcrRegistryDomain(registryHost);
+
+  const [getRepoError, existingRepo] = await tryCatch(
+    getEcrRepository({ repositoryName, region, accountId, assumeRole })
+  );
+
+  if (getRepoError) {
+    logger.error("Failed to get ECR repository", { repositoryName, region, getRepoError });
+    throw getRepoError;
+  }
+
+  if (existingRepo) {
+    logger.debug("ECR repository already exists", { repositoryName, region, existingRepo });
+
+    // check if the repository is missing the cache settings
+    if (existingRepo.imageTagMutability === "IMMUTABLE") {
+      const [updateError] = await tryCatch(
+        updateEcrRepositoryCacheSettings({ repositoryName, region, accountId, assumeRole })
+      );
+
+      if (updateError) {
+        logger.error("Failed to update ECR repository cache settings", {
+          repositoryName,
+          region,
+          updateError,
+        });
+      }
+    }
+
+    // Reconcile the default repository policy on every call. Idempotent, and
+    // covers two recovery cases: (1) a previous create succeeded but the
+    // SetRepositoryPolicy call failed mid-flight, leaving the repo without a
+    // policy; (2) the operator updated DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY
+    // and existing repos need to pick up the new value.
+    if (defaultRepositoryPolicy) {
+      const [policyError] = await tryCatch(
+        applyEcrRepositoryPolicy({
+          repositoryName,
+          region,
+          accountId,
+          assumeRole,
+          defaultRepositoryPolicy,
+        })
+      );
+
+      if (policyError) {
+        logger.error("Failed to reconcile ECR repository policy on existing repo", {
+          repositoryName,
+          region,
+          policyError,
+        });
+      }
+    }
+
+    return {
+      repo: existingRepo,
+      repoCreated: false,
+    };
+  }
+
+  const [createRepoError, newRepo] = await tryCatch(
+    createEcrRepository({
+      repositoryName,
+      region,
+      accountId,
+      registryTags,
+      assumeRole,
+      defaultRepositoryPolicy,
+    })
+  );
+
+  if (createRepoError) {
+    logger.error("Failed to create ECR repository", { repositoryName, region, createRepoError });
+    throw createRepoError;
+  }
+
+  if (newRepo.repositoryName !== repositoryName) {
+    logger.error("ECR repository name mismatch", { repositoryName, region, newRepo });
+    throw new Error(
+      `ECR repository name mismatch: ${repositoryName} !== ${newRepo.repositoryName}`
+    );
+  }
+
+  return {
+    repo: newRepo,
+    repoCreated: true,
+  };
+}
+
+export async function getEcrAuthToken({
+  registryHost,
+  assumeRole,
+}: {
+  registryHost: string;
+  assumeRole?: AssumeRoleConfig;
+}): Promise<{ username: string; password: string }> {
+  const { region, accountId } = parseEcrRegistryDomain(registryHost);
+  if (!region) {
+    logger.error("Invalid ECR registry host", { registryHost });
+    throw new Error("Invalid ECR registry host");
+  }
+
+  const ecr = await createEcrClient({ region, assumeRole });
+  const response = await ecr.send(
+    new GetAuthorizationTokenCommand({
+      registryIds: accountId ? [accountId] : undefined,
+    })
+  );
+
+  if (!response.authorizationData) {
+    throw new Error("Failed to get ECR authorization token");
+  }
+
+  const authData = response.authorizationData[0];
+
+  if (!authData.authorizationToken) {
+    throw new Error("No authorization token returned from ECR");
+  }
+
+  const authToken = Buffer.from(authData.authorizationToken, "base64").toString();
+  const [username, password] = authToken.split(":");
+
+  return { username, password };
+}

apps/webapp/app/v3/github.ts

@@ -0,0 +1,35 @@
+import { z } from "zod";
+
+export const BranchTrackingConfigSchema = z.object({
+  prod: z.object({
+    branch: z.string().optional(),
+  }),
+  staging: z.object({
+    branch: z.string().optional(),
+  }),
+});
+
+export type BranchTrackingConfig = z.infer<typeof BranchTrackingConfigSchema>;
+
+export function getTrackedBranchForEnvironment(
+  branchTracking: BranchTrackingConfig | undefined,
+  previewDeploymentsEnabled: boolean,
+  environment: {
+    type: "PRODUCTION" | "STAGING" | "DEVELOPMENT" | "PREVIEW";
+    branchName?: string;
+  }
+): string | undefined {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return branchTracking?.prod?.branch;
+    case "STAGING":
+      return branchTracking?.staging?.branch;
+    case "PREVIEW":
+      return previewDeploymentsEnabled ? environment.branchName : undefined;
+    case "DEVELOPMENT":
+      return undefined;
+    default:
+      environment.type satisfies never;
+      return undefined;
+  }
+}

apps/webapp/app/v3/globalFlagsRegistry.server.ts

@@ -0,0 +1,21 @@
+import { singleton } from "~/utils/singleton";
+import { env } from "~/env.server";
+import { flags } from "~/v3/featureFlags.server";
+import type { FeatureFlagCatalog } from "~/v3/featureFlags";
+import { createReloadingRegistry } from "~/utils/reloadingRegistry.server";
+
+/**
+ * In-memory snapshot of the global feature flags, refreshed every
+ * GLOBAL_FLAGS_RELOAD_INTERVAL_MS. `flags()` reads the DB-backed global values
+ * (no per-org overrides). Read synchronously on the trigger hot path; a cold
+ * read (before the first load) returns undefined and the resolver falls back to
+ * not-migrated.
+ */
+export const globalFlagsRegistry = singleton("globalFlagsRegistry", () =>
+  createReloadingRegistry<Partial<FeatureFlagCatalog>>({
+    name: "global-flags",
+    intervalMs: env.GLOBAL_FLAGS_RELOAD_INTERVAL_MS,
+    autoStart: process.env.NODE_ENV !== "test", // only auto-poll outside tests
+    load: () => flags(),
+  })
+);

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -0,0 +1,689 @@
+import type { EventBusEventArgs } from "@internal/run-engine";
+import { createAdapter } from "@socket.io/redis-adapter";
+import {
+  ClientToSharedQueueMessages,
+  CoordinatorSocketData,
+  CoordinatorToPlatformMessages,
+  PlatformToCoordinatorMessages,
+  PlatformToProviderMessages,
+  ProviderToPlatformMessages,
+  SharedQueueToClientMessages,
+} from "@trigger.dev/core/v3";
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import type {
+  WorkerClientToServerEvents,
+  WorkerServerToClientEvents,
+} from "@trigger.dev/core/v3/workers";
+import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace";
+import { defaultReconnectOnError } from "@internal/redis";
+import { Redis } from "ioredis";
+import type { Namespace, Socket } from "socket.io";
+import { Server } from "socket.io";
+import { env } from "~/env.server";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+import { authenticateApiRequestWithFailure } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { recordRunDebugLog } from "./eventRepository/index.server";
+import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server";
+import { engine } from "./runEngine.server";
+import { CompleteAttemptService } from "./services/completeAttempt.server";
+import { CrashTaskRunService } from "./services/crashTaskRun.server";
+import { CreateCheckpointService } from "./services/createCheckpoint.server";
+import { CreateDeploymentBackgroundWorkerServiceV3 } from "./services/createDeploymentBackgroundWorkerV3.server";
+import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { DeploymentIndexFailed } from "./services/deploymentIndexFailed.server";
+import { ResumeAttemptService } from "./services/resumeAttempt.server";
+import { UpdateFatalRunErrorService } from "./services/updateFatalRunError.server";
+import { WorkerGroupTokenService } from "./services/worker/workerGroupTokenService.server";
+import { SharedSocketConnection } from "./sharedSocketConnection";
+import { isV3Disabled } from "./engineDeprecation.server";
+
+export const socketIo = singleton("socketIo", initalizeIoServer);
+
+function initalizeIoServer() {
+  const io = initializeSocketIOServerInstance();
+
+  io.on("connection", (socket) => {
+    logger.log(`[socket.io][${socket.id}] connection at url: ${socket.request.url}`);
+  });
+
+  const coordinatorNamespace = createCoordinatorNamespace(io);
+  const providerNamespace = createProviderNamespace(io);
+  const sharedQueueConsumerNamespace = createSharedQueueConsumerNamespace(io);
+  const workerNamespace = createWorkerNamespace({
+    io,
+    namespace: "/worker",
+    authenticate: async (request) => {
+      const tokenService = new WorkerGroupTokenService();
+      const authenticatedInstance = await tokenService.authenticate(request);
+      if (!authenticatedInstance) {
+        return false;
+      }
+      return true;
+    },
+  });
+  const devWorkerNamespace = createWorkerNamespace({
+    io,
+    namespace: "/dev-worker",
+    authenticate: async (request) => {
+      const authentication = await authenticateApiRequestWithFailure(request);
+      if (!authentication.ok) {
+        return false;
+      }
+      if (authentication.environment.type !== "DEVELOPMENT") {
+        return false;
+      }
+      return true;
+    },
+  });
+
+  return {
+    io,
+    coordinatorNamespace,
+    providerNamespace,
+    sharedQueueConsumerNamespace,
+    workerNamespace,
+    devWorkerNamespace,
+  };
+}
+
+function initializeSocketIOServerInstance() {
+  if (env.REDIS_HOST && env.REDIS_PORT) {
+    const pubClient = new Redis({
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      reconnectOnError: defaultReconnectOnError,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    });
+    const subClient = pubClient.duplicate();
+
+    const io = new Server({
+      adapter: createAdapter(pubClient, subClient, {
+        key: "tr:socket.io:",
+        publishOnSpecificResponseChannel: true,
+      }),
+    });
+
+    return io;
+  }
+
+  return new Server();
+}
+
+function createCoordinatorNamespace(io: Server) {
+  const coordinator = new ZodNamespace({
+    // @ts-ignore - for some reason the built ZodNamespace Server type is not compatible with the Server type here, but only when doing typechecking
+    io,
+    name: "coordinator",
+    authToken: env.COORDINATOR_SECRET,
+    clientMessages: CoordinatorToPlatformMessages,
+    serverMessages: PlatformToCoordinatorMessages,
+    socketData: CoordinatorSocketData,
+    handlers: {
+      READY_FOR_EXECUTION: async (message) => {
+        const payload = await sharedQueueTasks.getLatestExecutionPayloadFromRun(
+          message.runId,
+          true,
+          !!message.totalCompletions
+        );
+
+        if (!payload) {
+          logger.error("Failed to retrieve execution payload", message);
+          return { success: false };
+        } else {
+          return { success: true, payload };
+        }
+      },
+      READY_FOR_LAZY_ATTEMPT: async (message) => {
+        try {
+          const payload = await sharedQueueTasks.getLazyAttemptPayload(
+            message.envId,
+            message.runId
+          );
+
+          if (!payload) {
+            logger.error(
+              "READY_FOR_LAZY_ATTEMPT: Failed to retrieve lazy attempt payload",
+              message
+            );
+            return { success: false, reason: "READY_FOR_LAZY_ATTEMPT: Failed to retrieve payload" };
+          }
+
+          return { success: true, lazyPayload: payload };
+        } catch (error) {
+          logger.error("READY_FOR_LAZY_ATTEMPT: Error while creating lazy attempt", {
+            runId: message.runId,
+            envId: message.envId,
+            totalCompletions: message.totalCompletions,
+            error,
+          });
+          return { success: false };
+        }
+      },
+      READY_FOR_RESUME: async (message) => {
+        const resumeAttempt = new ResumeAttemptService();
+        await resumeAttempt.call(message);
+      },
+      TASK_RUN_COMPLETED: async (message) => {
+        const completeAttempt = new CompleteAttemptService({
+          supportsRetryCheckpoints: message.version === "v1",
+        });
+        await completeAttempt.call({
+          completion: message.completion,
+          execution: message.execution,
+          checkpoint: message.checkpoint,
+        });
+      },
+      TASK_RUN_COMPLETED_WITH_ACK: async (message) => {
+        try {
+          const completeAttempt = new CompleteAttemptService({
+            supportsRetryCheckpoints: message.version === "v1",
+          });
+          await completeAttempt.call({
+            completion: message.completion,
+            execution: message.execution,
+            checkpoint: message.checkpoint,
+          });
+
+          return {
+            success: true,
+          };
+        } catch (error) {
+          const friendlyError =
+            error instanceof Error
+              ? {
+                  name: error.name,
+                  message: error.message,
+                  stack: error.stack,
+                }
+              : {
+                  name: "UnknownError",
+                  message: String(error),
+                };
+
+          logger.error("Error while completing attempt with ack", {
+            error: friendlyError,
+            message,
+          });
+
+          return {
+            success: false,
+            error: friendlyError,
+          };
+        }
+      },
+      TASK_RUN_FAILED_TO_RUN: async (message) => {
+        await sharedQueueTasks.taskRunFailed(message.completion);
+      },
+      TASK_HEARTBEAT: async (message) => {
+        await sharedQueueTasks.taskHeartbeat(message.attemptFriendlyId);
+      },
+      TASK_RUN_HEARTBEAT: async (message) => {
+        await sharedQueueTasks.taskRunHeartbeat(message.runId);
+      },
+      CHECKPOINT_CREATED: async (message) => {
+        try {
+          const createCheckpoint = new CreateCheckpointService();
+          const result = await createCheckpoint.call(message);
+
+          return { keepRunAlive: result?.keepRunAlive ?? false };
+        } catch (error) {
+          logger.error("Error while creating checkpoint", {
+            rawMessage: message,
+            error: error instanceof Error ? error.message : error,
+          });
+
+          return { keepRunAlive: false };
+        }
+      },
+      CREATE_WORKER: async (message) => {
+        try {
+          const environment = await findEnvironmentById(message.envId);
+
+          if (!environment) {
+            logger.error("Environment not found", { id: message.envId });
+            return { success: false };
+          }
+
+          const service = new CreateDeploymentBackgroundWorkerServiceV3();
+          const worker = await service.call(message.projectRef, environment, message.deploymentId, {
+            localOnly: false,
+            metadata: message.metadata,
+            supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
+          });
+
+          return { success: !!worker };
+        } catch (error) {
+          logger.error("Error while creating worker", {
+            error,
+            envId: message.envId,
+            projectRef: message.projectRef,
+            deploymentId: message.deploymentId,
+            version: message.version,
+          });
+          return { success: false };
+        }
+      },
+      CREATE_TASK_RUN_ATTEMPT: async (message) => {
+        try {
+          const environment = await findEnvironmentById(message.envId);
+
+          if (!environment) {
+            logger.error("CREATE_TASK_RUN_ATTEMPT: Environment not found", message);
+            return { success: false, reason: "Environment not found" };
+          }
+
+          const service = new CreateTaskRunAttemptService();
+          const { attempt } = await service.call({
+            runId: message.runId,
+            authenticatedEnv: environment,
+            setToExecuting: false,
+          });
+
+          const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+            id: attempt.id,
+            setToExecuting: true,
+            skipStatusChecks: true,
+          });
+
+          if (!payload) {
+            logger.error(
+              "CREATE_TASK_RUN_ATTEMPT: Failed to retrieve payload after attempt creation",
+              message
+            );
+            return {
+              success: false,
+              reason: "CREATE_TASK_RUN_ATTEMPT: Failed to retrieve payload",
+            };
+          }
+
+          return { success: true, executionPayload: payload };
+        } catch (error) {
+          logger.error("CREATE_TASK_RUN_ATTEMPT: Error while creating attempt", {
+            ...message,
+            error,
+          });
+          return { success: false };
+        }
+      },
+      INDEXING_FAILED: async (message) => {
+        try {
+          const service = new DeploymentIndexFailed();
+
+          await service.call(message.deploymentId, message.error);
+        } catch (error) {
+          logger.error("Error while processing index failure", {
+            deploymentId: message.deploymentId,
+            error,
+          });
+        }
+      },
+      RUN_CRASHED: async (message) => {
+        try {
+          const service = new CrashTaskRunService();
+
+          await service.call(message.runId, {
+            reason: `${message.error.name}: ${message.error.message}`,
+            logs: message.error.stack,
+          });
+        } catch (error) {
+          logger.error("Error while processing run failure", {
+            runId: message.runId,
+            error,
+          });
+        }
+      },
+    },
+    onConnection: async (socket, handler, sender, logger) => {
+      if (socket.data.supportsDynamicConfig) {
+        socket.emit("DYNAMIC_CONFIG", {
+          version: "v1",
+          checkpointThresholdInMs: env.CHECKPOINT_THRESHOLD_IN_MS,
+        });
+      }
+    },
+    postAuth: async (socket, next, logger) => {
+      function setSocketDataFromHeader(
+        dataKey: keyof typeof socket.data,
+        headerName: string,
+        required: boolean = true
+      ) {
+        const value = socket.handshake.headers[headerName];
+
+        if (value) {
+          socket.data[dataKey] = Array.isArray(value) ? value[0] : value;
+          return;
+        }
+
+        if (required) {
+          logger.error("missing required header", { headerName });
+          throw new Error("missing header");
+        }
+      }
+
+      try {
+        setSocketDataFromHeader("supportsDynamicConfig", "x-supports-dynamic-config", false);
+      } catch (error) {
+        logger.error("setSocketDataFromHeader error", { error });
+        socket.disconnect(true);
+        return;
+      }
+
+      logger.debug("success", socket.data);
+
+      next();
+    },
+  });
+
+  return coordinator.namespace;
+}
+
+function createProviderNamespace(io: Server) {
+  const provider = new ZodNamespace({
+    // @ts-ignore - for some reason the built ZodNamespace Server type is not compatible with the Server type here, but only when doing typechecking
+    io,
+    name: "provider",
+    authToken: env.PROVIDER_SECRET,
+    clientMessages: ProviderToPlatformMessages,
+    serverMessages: PlatformToProviderMessages,
+    handlers: {
+      WORKER_CRASHED: async (message) => {
+        try {
+          if (message.overrideCompletion) {
+            const updateErrorService = new UpdateFatalRunErrorService();
+            await updateErrorService.call(message.runId, { ...message });
+          } else {
+            const crashRunService = new CrashTaskRunService();
+            await crashRunService.call(message.runId, { ...message });
+          }
+        } catch (error) {
+          logger.error("Error while handling crashed worker", { error });
+        }
+      },
+      INDEXING_FAILED: async (message) => {
+        try {
+          const service = new DeploymentIndexFailed();
+
+          await service.call(message.deploymentId, message.error, message.overrideCompletion);
+        } catch (e) {
+          logger.error("Error while indexing", { error: e });
+        }
+      },
+    },
+  });
+
+  return provider.namespace;
+}
+
+function createSharedQueueConsumerNamespace(io: Server) {
+  const sharedQueue = new ZodNamespace({
+    // @ts-ignore - for some reason the built ZodNamespace Server type is not compatible with the Server type here, but only when doing typechecking
+    io,
+    name: "shared-queue",
+    authToken: env.PROVIDER_SECRET,
+    clientMessages: ClientToSharedQueueMessages,
+    serverMessages: SharedQueueToClientMessages,
+    onConnection: async (socket, handler, sender, logger) => {
+      // v3 (engine V1) shutdown: don't start the MarQS shared-queue consumer, so no
+      // deployed V1 runs are dequeued. This namespace is V1-only; v4 dequeues through
+      // the run-engine worker path. This is the code-level equivalent of taking the
+      // v3 coordinator offline.
+      if (isV3Disabled()) {
+        logger.warn("Refusing /shared-queue connection: v3 engine is shut down");
+        socket.disconnect(true);
+        return;
+      }
+
+      const sharedSocketConnection = new SharedSocketConnection({
+        // @ts-ignore - for some reason the built ZodNamespace Server type is not compatible with the Server type here, but only when doing typechecking
+        namespace: sharedQueue.namespace,
+        // @ts-ignore - for some reason the built ZodNamespace Server type is not compatible with the Server type here, but only when doing typechecking
+        socket,
+        logger,
+        poolSize: env.SHARED_QUEUE_CONSUMER_POOL_SIZE,
+      });
+
+      sharedSocketConnection.onClose.attach((closeEvent) => {
+        logger.info("Socket closed", { closeEvent });
+      });
+
+      await sharedSocketConnection.initialize();
+    },
+  });
+
+  return sharedQueue.namespace;
+}
+
+function headersFromHandshake(handshake: Socket["handshake"]) {
+  const headers = new Headers();
+
+  for (const [key, value] of Object.entries(handshake.headers)) {
+    if (typeof value !== "string") continue;
+    headers.append(key, value);
+  }
+
+  return headers;
+}
+
+function createWorkerNamespace({
+  io,
+  namespace,
+  authenticate,
+}: {
+  io: Server;
+  namespace: string;
+  authenticate: (request: Request) => Promise<boolean>;
+}) {
+  const worker: Namespace<WorkerClientToServerEvents, WorkerServerToClientEvents> =
+    io.of(namespace);
+
+  worker.use(async (socket, next) => {
+    try {
+      const headers = headersFromHandshake(socket.handshake);
+
+      logger.debug("Worker authentication", {
+        namespace,
+        socketId: socket.id,
+        headers: Object.fromEntries(headers),
+      });
+
+      const request = new Request("https://example.com", {
+        headers,
+      });
+
+      const success = await authenticate(request);
+
+      if (!success) {
+        throw new Error("unauthorized");
+      }
+
+      next();
+    } catch (error) {
+      // System handles auth failure by disconnecting the socket — not an
+      // error. Most volume is V1 /dev-worker reconnect churn from outdated
+      // CLIs anyway.
+      logger.warn("Worker authentication failed", {
+        namespace,
+        error: error instanceof Error ? error.message : error,
+      });
+
+      socket.disconnect(true);
+    }
+  });
+
+  worker.on("connection", async (socket) => {
+    logger.debug("worker connected", { namespace, socketId: socket.id });
+
+    const rooms = new Set<string>();
+
+    async function onNotification({
+      time,
+      run,
+      snapshot,
+    }: EventBusEventArgs<"workerNotification">[0]) {
+      if (!env.RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS) {
+        return;
+      }
+
+      logger.debug("[handleSocketIo] Received worker notification", {
+        namespace,
+        time,
+        runId: run.id,
+        snapshot,
+      });
+
+      // Record notification event
+      await recordRunDebugLog(run.id, `run:notify workerNotification event`, {
+        attributes: {
+          properties: {
+            snapshotId: snapshot.id,
+            snapshotStatus: snapshot.executionStatus,
+            rooms: Array.from(rooms),
+          },
+        },
+        startTime: time,
+      });
+    }
+
+    engine.eventBus.on("workerNotification", onNotification);
+
+    const interval = setInterval(() => {
+      logger.debug("Rooms for socket", {
+        namespace,
+        socketId: socket.id,
+        rooms: Array.from(rooms),
+      });
+    }, 5000);
+
+    socket.on("disconnect", (reason, description) => {
+      logger.debug("worker disconnected", {
+        namespace,
+        socketId: socket.id,
+        reason,
+        description,
+      });
+      clearInterval(interval);
+
+      engine.eventBus.off("workerNotification", onNotification);
+    });
+
+    socket.on("disconnecting", (reason, description) => {
+      logger.debug("worker disconnecting", {
+        namespace,
+        socketId: socket.id,
+        reason,
+        description,
+      });
+      clearInterval(interval);
+    });
+
+    socket.on("error", (error) => {
+      logger.error("worker error", {
+        namespace,
+        socketId: socket.id,
+        error: JSON.parse(JSON.stringify(error)),
+      });
+      clearInterval(interval);
+    });
+
+    socket.on("run:subscribe", async ({ version, runFriendlyIds }) => {
+      logger.debug("run:subscribe", { namespace, version, runFriendlyIds });
+
+      const settledResult = await Promise.allSettled(
+        runFriendlyIds.map(async (friendlyId) => {
+          const room = roomFromFriendlyRunId(friendlyId);
+
+          logger.debug("Joining room", { namespace, room });
+
+          socket.join(room);
+          rooms.add(room);
+
+          await recordRunDebugLog(
+            RunId.fromFriendlyId(friendlyId),
+            "run:subscribe received by platform",
+            {
+              attributes: {
+                properties: {
+                  friendlyId,
+                  runFriendlyIds,
+                  room,
+                },
+              },
+            }
+          );
+        })
+      );
+
+      for (const result of settledResult) {
+        if (result.status === "rejected") {
+          logger.error("Error joining room", {
+            namespace,
+            runFriendlyIds,
+            error: result.reason instanceof Error ? result.reason.message : result.reason,
+          });
+        }
+      }
+
+      logger.debug("Rooms for socket after subscribe", {
+        namespace,
+        socketId: socket.id,
+        rooms: Array.from(rooms),
+      });
+    });
+
+    socket.on("run:unsubscribe", async ({ version, runFriendlyIds }) => {
+      logger.debug("run:unsubscribe", { namespace, version, runFriendlyIds });
+
+      const settledResult = await Promise.allSettled(
+        runFriendlyIds.map(async (friendlyId) => {
+          const room = roomFromFriendlyRunId(friendlyId);
+
+          logger.debug("Leaving room", { namespace, room });
+
+          socket.leave(room);
+          rooms.delete(room);
+
+          await recordRunDebugLog(
+            RunId.fromFriendlyId(friendlyId),
+            "run:unsubscribe received by platform",
+            {
+              attributes: {
+                properties: {
+                  friendlyId,
+                  runFriendlyIds,
+                  room,
+                },
+              },
+            }
+          );
+        })
+      );
+
+      for (const result of settledResult) {
+        if (result.status === "rejected") {
+          logger.error("Error leaving room", {
+            namespace,
+            runFriendlyIds,
+            error: result.reason instanceof Error ? result.reason.message : result.reason,
+          });
+        }
+      }
+
+      logger.debug("Rooms for socket after unsubscribe", {
+        namespace,
+        socketId: socket.id,
+        rooms: Array.from(rooms),
+      });
+    });
+  });
+
+  return worker;
+}
+
+export function roomFromFriendlyRunId(id: string) {
+  return `room:${id}`;
+}

apps/webapp/app/v3/handleWebsockets.server.ts

@@ -0,0 +1,56 @@
+import type { IncomingMessage } from "node:http";
+import { WebSocketServer, type WebSocket } from "ws";
+import { authenticateApiKey } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "../utils/singleton";
+import { V3_DEV_DEPRECATION_MESSAGE } from "./engineDeprecation.server";
+
+export const wss = singleton("wss", initalizeWebSocketServer);
+
+function initalizeWebSocketServer() {
+  const server = new WebSocketServer({ noServer: true });
+
+  server.on("connection", handleWebSocketConnection);
+
+  return server;
+}
+
+async function handleWebSocketConnection(ws: WebSocket, req: IncomingMessage) {
+  logger.debug("Handle websocket connection", {
+    ipAddress: req.headers["x-forwarded-for"] || req.socket.remoteAddress,
+  });
+
+  const authHeader = req.headers.authorization;
+
+  if (!authHeader || typeof authHeader !== "string") {
+    ws.close(1008, "Missing Authorization header");
+    return;
+  }
+
+  const [authType, apiKey] = authHeader.split(" ");
+
+  if (authType !== "Bearer" || !apiKey) {
+    ws.close(1008, "Invalid Authorization header");
+    return;
+  }
+
+  const authenticationResult = await authenticateApiKey(apiKey);
+
+  if (!authenticationResult || !authenticationResult.ok) {
+    ws.close(1008, "Invalid API key");
+    return;
+  }
+
+  const authenticatedEnv = authenticationResult.environment;
+
+  // This websocket is only used by the legacy v3 `trigger dev` CLI (v4 uses a
+  // different dev transport). The v3 engine is end-of-lifed, so there is no
+  // longer any work to run here — close with the graceful upgrade message so
+  // an old CLI is told what to do instead of sitting connected.
+  logger.warn("Rejected deprecated v3 dev CLI websocket connection", {
+    environmentId: authenticatedEnv.id,
+    projectId: authenticatedEnv.projectId,
+    organizationId: authenticatedEnv.organizationId,
+  });
+  ws.close(1008, V3_DEV_DEPRECATION_MESSAGE);
+}

apps/webapp/app/v3/humanToCron.server.ts

@@ -0,0 +1,90 @@
+import OpenAI from "openai";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { safeJsonParse } from "~/utils/json";
+
+export const HumanToCronResult = z.object({
+  isValid: z.boolean(),
+  cron: z.string().optional(),
+  error: z.string().optional(),
+});
+
+export type HumanToCronResult = z.infer<typeof HumanToCronResult>;
+
+export const humanToCronSupported = typeof env.OPENAI_API_KEY === "string";
+
+export async function humanToCron(message: string, userId: string): Promise<HumanToCronResult> {
+  if (!humanToCronSupported) {
+    return {
+      isValid: false,
+      error: "OpenAI API key is not set",
+    };
+  }
+
+  const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY });
+
+  const completion = await openai.chat.completions.create({
+    model: "gpt-3.5-turbo-1106",
+    user: userId,
+    messages: [
+      {
+        role: "system",
+        content: `You are a helpful assistant who will turn nautral language into a valid CRON expresion. 
+          
+          The version of CRON that we use is an extension of the minimal.
+
+*    *    *    *    *
+┬    ┬    ┬    ┬    ┬
+│    │    │    │    |
+│    │    │    │    └ day of week (0 - 7, 1L - 7L) (0 or 7 is Sun)
+│    │    │    └───── month (1 - 12)
+│    │    └────────── day of month (1 - 31, L)
+│    └─────────────── hour (0 - 23)
+└──────────────────── minute (0 - 59)
+
+Supports mixed use of ranges and range increments (W character not supported currently). See tests for examples.
+
+          Return JSON in one of these formats, putting in the correct data where you see <THE CRON EXPRESSION> and <ERROR MESSAGE DESCRIBING WHY IT'S NOT VALID>:
+        1. If it's valid: { "isValid": true, "cron": "<THE CRON EXPRESSION>" }
+        2. If it's not possible to make a valid CRON expression: { "isValid": false, "error": "<ERROR MESSAGE DESCRIBING WHY IT'S NOT VALID>"}`,
+      },
+      {
+        role: "user",
+        content: `What is a valid CRON expression for this: ${message}`,
+      },
+    ],
+    response_format: { type: "json_object" },
+  });
+
+  if (!completion.choices[0]?.message.content) {
+    return {
+      isValid: false,
+      error: "No response from OpenAI",
+    };
+  }
+
+  logger.debug("OpenAI response", {
+    completion,
+  });
+
+  const jsonResponse = safeJsonParse(completion.choices[0].message.content);
+
+  if (!jsonResponse) {
+    return {
+      isValid: false,
+      error: "Invalid response from OpenAI",
+    };
+  }
+
+  const parsedResponse = HumanToCronResult.safeParse(jsonResponse);
+
+  if (!parsedResponse.success) {
+    return {
+      isValid: false,
+      error: `Invalid response from OpenAI: ${parsedResponse.error.message}`,
+    };
+  }
+
+  return parsedResponse.data;
+}

apps/webapp/app/v3/legacyRunEngineWorker.server.ts

@@ -0,0 +1,118 @@
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { Logger } from "@trigger.dev/core/logger";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { TaskRunHeartbeatFailedService } from "./taskRunHeartbeatFailed.server";
+import { completeBatchTaskRunItemV3, tryCompleteBatchV3 } from "./services/batchTriggerV3.server";
+import { prisma } from "~/db.server";
+import { marqs } from "./marqs/index.server";
+
+function initializeWorker() {
+  const redisOptions = {
+    keyPrefix: "legacy-run-engine:worker:",
+    host: env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST,
+    port: env.LEGACY_RUN_ENGINE_WORKER_REDIS_PORT,
+    username: env.LEGACY_RUN_ENGINE_WORKER_REDIS_USERNAME,
+    password: env.LEGACY_RUN_ENGINE_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.LEGACY_RUN_ENGINE_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(
+    `👨‍🏭 Initializing legacy run engine worker at host ${env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST}`
+  );
+
+  const worker = new RedisWorker({
+    name: "legacy-run-engine-worker",
+    redisOptions,
+    catalog: {
+      runHeartbeat: {
+        schema: z.object({
+          runId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 3,
+        },
+      },
+      completeBatchTaskRunItem: {
+        schema: z.object({
+          itemId: z.string(),
+          batchTaskRunId: z.string(),
+          scheduleResumeOnComplete: z.boolean(),
+          taskRunAttemptId: z.string().optional(),
+          attempt: z.number().optional(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 10,
+        },
+      },
+      tryCompleteBatchV3: {
+        schema: z.object({
+          batchId: z.string(),
+          scheduleResumeOnComplete: z.boolean(),
+        }),
+        visibilityTimeoutMs: 30_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      scheduleRequeueMessage: {
+        schema: z.object({
+          messageId: z.string(),
+        }),
+        visibilityTimeoutMs: 60_000,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+    },
+    concurrency: {
+      workers: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.LEGACY_RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("LegacyRunEngineWorker", env.LEGACY_RUN_ENGINE_WORKER_LOG_LEVEL),
+    jobs: {
+      runHeartbeat: async ({ payload }) => {
+        const service = new TaskRunHeartbeatFailedService();
+
+        await service.call(payload.runId);
+      },
+      completeBatchTaskRunItem: async ({ payload, attempt }) => {
+        await completeBatchTaskRunItemV3(
+          payload.itemId,
+          payload.batchTaskRunId,
+          prisma,
+          payload.scheduleResumeOnComplete,
+          payload.taskRunAttemptId,
+          attempt
+        );
+      },
+      tryCompleteBatchV3: async ({ payload }) => {
+        await tryCompleteBatchV3(payload.batchId, prisma, payload.scheduleResumeOnComplete);
+      },
+      scheduleRequeueMessage: async ({ payload }) => {
+        await marqs.requeueMessageById(payload.messageId);
+      },
+    },
+  });
+
+  if (env.LEGACY_RUN_ENGINE_WORKER_ENABLED === "true") {
+    logger.debug(
+      `👨‍🏭 Starting legacy run engine worker at host ${env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST}, pollInterval = ${env.LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT}`
+    );
+
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const legacyRunEngineWorker = singleton("legacyRunEngineWorker", initializeWorker);

apps/webapp/app/v3/llmPricingRegistry.server.ts

@@ -0,0 +1,167 @@
+import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-model-catalog";
+import type { LlmModelWithPricing } from "@internal/llm-model-catalog";
+import { prisma, $replica } from "~/db.server";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { signalsEmitter } from "~/services/signals.server";
+import { createRedisClient } from "~/redis.server";
+import { singleton } from "~/utils/singleton";
+import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server";
+
+type PricingReloadListener = (models: LlmModelWithPricing[]) => void;
+const pricingReloadListeners = new Set<PricingReloadListener>();
+
+// Notify subscribers (e.g. the OTLP worker pool) after each load/reload so they can rebuild
+// their own in-memory copy. No-op until something subscribes.
+function emitPricingReload() {
+  if (!llmPricingRegistry || !llmPricingRegistry.isLoaded || pricingReloadListeners.size === 0) {
+    return;
+  }
+  const models = llmPricingRegistry.toSerializable();
+  for (const listener of pricingReloadListeners) {
+    try {
+      listener(models);
+    } catch (err) {
+      logger.warn("LLM pricing reload listener failed", {
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+}
+
+export function subscribeToPricingReload(listener: PricingReloadListener): () => void {
+  pricingReloadListeners.add(listener);
+  if (llmPricingRegistry?.isLoaded) {
+    try {
+      listener(llmPricingRegistry.toSerializable());
+    } catch (err) {
+      logger.warn("LLM pricing reload listener failed", {
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+  return () => {
+    pricingReloadListeners.delete(listener);
+  };
+}
+
+async function initRegistry(registry: ModelPricingRegistry) {
+  if (env.LLM_PRICING_SEED_ON_STARTUP) {
+    await seedLlmPricing(prisma);
+  }
+
+  await registry.loadFromDatabase();
+}
+
+export const llmPricingRegistry = singleton("llmPricingRegistry", () => {
+  if (!env.LLM_COST_TRACKING_ENABLED) {
+    return null;
+  }
+
+  const registry = new ModelPricingRegistry($replica);
+
+  // Wire up the registry so enrichCreatableEvents can use it
+  setLlmPricingRegistry(registry);
+
+  initRegistry(registry)
+    .then(() => emitPricingReload())
+    .catch((err) => {
+      console.error("Failed to initialize LLM pricing registry", err);
+    });
+
+  // Periodic reload (backstop for the pub/sub path below)
+  const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS;
+  const interval = setInterval(() => {
+    registry
+      .reload()
+      .then(() => emitPricingReload())
+      .catch((err) => {
+        console.error("Failed to reload LLM pricing registry", err);
+      });
+  }, reloadInterval);
+
+  // Pub/sub reload is opt-in per process (default off). Without it, the
+  // registry stays accurate via the existing 5-minute interval. Enable on
+  // the OTel-ingesting services where pricing freshness directly affects
+  // span cost enrichment; dashboard and worker services don't need it and
+  // shouldn't pile onto each publish with a full-table reload.
+  if (env.LLM_PRICING_RELOAD_PUBSUB_ENABLED) {
+    const subscriber = createRedisClient("llm-pricing:subscriber", {
+      keyPrefix: "llm-pricing:subscriber:",
+      host: env.COMMON_WORKER_REDIS_HOST,
+      port: env.COMMON_WORKER_REDIS_PORT,
+      username: env.COMMON_WORKER_REDIS_USERNAME,
+      password: env.COMMON_WORKER_REDIS_PASSWORD,
+      tlsDisabled: env.COMMON_WORKER_REDIS_TLS_DISABLED === "true",
+      clusterMode: env.COMMON_WORKER_REDIS_CLUSTER_MODE_ENABLED === "1",
+    });
+
+    subscriber.subscribe(env.LLM_PRICING_RELOAD_CHANNEL).catch((err) => {
+      logger.warn("Failed to subscribe to LLM pricing reload channel", {
+        channel: env.LLM_PRICING_RELOAD_CHANNEL,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    });
+
+    // Coalesce reload calls so a burst of publishes only triggers one
+    // reload. The first publish schedules a reload at
+    // T+LLM_PRICING_RELOAD_DEBOUNCE_MS; subsequent publishes during that
+    // window are no-ops because the trailing reload picks up everything
+    // when it queries the DB. Bounds reload rate to at most 1 per debounce
+    // window regardless of publisher chattiness.
+    const debounceMs = env.LLM_PRICING_RELOAD_DEBOUNCE_MS;
+    let pendingReloadTimer: NodeJS.Timeout | null = null;
+
+    function scheduleReload() {
+      if (pendingReloadTimer) return;
+      pendingReloadTimer = setTimeout(() => {
+        pendingReloadTimer = null;
+        registry
+          .reload()
+          .then(() => emitPricingReload())
+          .catch((err) => {
+            logger.warn("Failed to reload LLM pricing registry from pub/sub", {
+              error: err instanceof Error ? err.message : String(err),
+            });
+          });
+      }, debounceMs);
+    }
+
+    subscriber.on("message", (channel) => {
+      if (channel !== env.LLM_PRICING_RELOAD_CHANNEL) return;
+      scheduleReload();
+    });
+
+    signalsEmitter.on("SIGTERM", () => {
+      clearInterval(interval);
+      if (pendingReloadTimer) clearTimeout(pendingReloadTimer);
+      void subscriber.quit().catch(() => {});
+    });
+    signalsEmitter.on("SIGINT", () => {
+      clearInterval(interval);
+      if (pendingReloadTimer) clearTimeout(pendingReloadTimer);
+      void subscriber.quit().catch(() => {});
+    });
+  } else {
+    signalsEmitter.on("SIGTERM", () => clearInterval(interval));
+    signalsEmitter.on("SIGINT", () => clearInterval(interval));
+  }
+
+  return registry;
+});
+
+/**
+ * Wait for the LLM pricing registry to finish its initial load, with a timeout.
+ * After the first call resolves (or times out), subsequent calls are no-ops.
+ */
+export async function waitForLlmPricingReady(): Promise<void> {
+  if (!llmPricingRegistry || llmPricingRegistry.isLoaded) return;
+
+  const timeoutMs = env.LLM_PRICING_READY_TIMEOUT_MS;
+  if (timeoutMs <= 0) return;
+
+  await Promise.race([
+    llmPricingRegistry.isReady,
+    new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
+  ]);
+}

apps/webapp/app/v3/machinePresets.server.ts

@@ -0,0 +1,66 @@
+import type { MachinePreset } from "@trigger.dev/core/v3";
+import { MachineConfig, MachinePresetName } from "@trigger.dev/core/v3";
+import { defaultMachine, machines } from "~/services/platform.v3.server";
+import { logger } from "~/services/logger.server";
+
+export function machinePresetFromConfig(config: unknown): MachinePreset {
+  const parsedConfig = MachineConfig.safeParse(config);
+
+  if (!parsedConfig.success) {
+    logger.info("Failed to parse machine config", { config });
+
+    return machinePresetFromName("small-1x");
+  }
+
+  if (parsedConfig.data.preset) {
+    return machinePresetFromName(parsedConfig.data.preset);
+  }
+
+  if (parsedConfig.data.cpu && parsedConfig.data.memory) {
+    const name = derivePresetNameFromValues(parsedConfig.data.cpu, parsedConfig.data.memory);
+
+    return machinePresetFromName(name);
+  }
+
+  return machinePresetFromName("small-1x");
+}
+
+export function machinePresetFromName(name: MachinePresetName): MachinePreset {
+  return {
+    name,
+    ...machines[name],
+  };
+}
+
+export function machinePresetFromRun(run: { machinePreset: string | null }): MachinePreset | null {
+  const presetName = MachinePresetName.safeParse(run.machinePreset).data;
+
+  if (!presetName) {
+    return null;
+  }
+
+  return machinePresetFromName(presetName);
+}
+
+// Finds the smallest machine preset name that satisfies the given CPU and memory requirements
+function derivePresetNameFromValues(cpu: number, memory: number): MachinePresetName {
+  for (const [name, preset] of Object.entries(machines)) {
+    if (preset.cpu >= cpu && preset.memory >= memory) {
+      return name as MachinePresetName;
+    }
+  }
+
+  return defaultMachine;
+}
+
+export function allMachines(): Record<string, MachinePreset> {
+  return Object.fromEntries(
+    Object.entries(machines).map(([name, preset]) => [
+      name,
+      {
+        name: name as MachinePresetName,
+        ...preset,
+      },
+    ])
+  );
+}

apps/webapp/app/v3/marqs/asyncWorker.server.ts

@@ -0,0 +1,37 @@
+export class AsyncWorker {
+  private running = false;
+  private timeout?: NodeJS.Timeout;
+
+  constructor(
+    private readonly fn: () => Promise<void>,
+    private readonly interval: number
+  ) {}
+
+  start() {
+    if (this.running) {
+      return;
+    }
+
+    this.running = true;
+
+    this.#run();
+  }
+
+  stop() {
+    this.running = false;
+  }
+
+  async #run() {
+    if (!this.running) {
+      return;
+    }
+
+    try {
+      await this.fn();
+    } catch (e) {
+      console.error(e);
+    }
+
+    this.timeout = setTimeout(this.#run.bind(this), this.interval);
+  }
+}

apps/webapp/app/v3/marqs/concurrencyMonitor.server.ts

@@ -0,0 +1,204 @@
+import type { Logger } from "@trigger.dev/core/logger";
+import type { Redis } from "ioredis";
+import { prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import type { MarQS } from "./index.server";
+import { marqs as marqsv3 } from "./index.server";
+import { env } from "~/env.server";
+
+export type MarqsConcurrencyMonitorOptions = {
+  dryRun?: boolean;
+  abortSignal?: AbortSignal;
+};
+
+export interface MarqsConcurrencyResolveCompletedRunsCallback {
+  (candidateRunIds: string[]): Promise<Array<{ id: string }>>;
+}
+
+export class MarqsConcurrencyMonitor {
+  private _logger: Logger;
+
+  constructor(
+    private marqs: MarQS,
+    private callback: MarqsConcurrencyResolveCompletedRunsCallback,
+    private options: MarqsConcurrencyMonitorOptions = {}
+  ) {
+    this._logger = logger.child({
+      component: "marqs",
+      operation: "concurrencyMonitor",
+      dryRun: this.dryRun,
+      marqs: marqs.name,
+    });
+  }
+
+  get dryRun() {
+    return typeof this.options.dryRun === "boolean" ? this.options.dryRun : false;
+  }
+
+  get keys() {
+    return this.marqs.keys;
+  }
+
+  get signal() {
+    return this.options.abortSignal;
+  }
+
+  public async call() {
+    this._logger.debug("[MarqsConcurrencyMonitor] Initiating monitoring");
+
+    const stats = {
+      streamCallbacks: 0,
+      processedKeys: 0,
+    };
+
+    const { stream, redis } = this.marqs.queueConcurrencyScanStream(
+      10,
+      () => {
+        this._logger.debug("[MarqsConcurrencyMonitor] stream closed", {
+          stats,
+        });
+      },
+      (error) => {
+        this._logger.debug("[MarqsConcurrencyMonitor] stream error", {
+          stats,
+          error: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+          },
+        });
+      }
+    );
+
+    stream.on("data", async (keys) => {
+      stream.pause();
+
+      if (this.signal?.aborted) {
+        stream.destroy();
+        return;
+      }
+
+      stats.streamCallbacks++;
+
+      const uniqueKeys = Array.from(new Set<string>(keys));
+
+      if (uniqueKeys.length === 0) {
+        stream.resume();
+        return;
+      }
+
+      this._logger.debug("[MarqsConcurrencyMonitor] correcting queues concurrency", {
+        keys: uniqueKeys,
+      });
+
+      stats.processedKeys += uniqueKeys.length;
+
+      await Promise.allSettled(uniqueKeys.map((key) => this.#processKey(key, redis))).finally(
+        () => {
+          stream.resume();
+        }
+      );
+    });
+  }
+
+  async #processKey(key: string, redis: Redis) {
+    key = this.keys.stripKeyPrefix(key);
+    const envKey = this.keys.envCurrentConcurrencyKeyFromQueue(key);
+
+    let runIds: string[] = [];
+
+    try {
+      // Next, we need to get all the items from the key, and any parent keys (org, env, queue) using sunion.
+      runIds = await redis.sunion(envKey, key);
+    } catch (e) {
+      this._logger.error("[MarqsConcurrencyMonitor] error during sunion", {
+        key,
+        envKey,
+        runIds,
+        error: e,
+      });
+    }
+
+    if (runIds.length === 0) {
+      return;
+    }
+
+    const perfNow = performance.now();
+
+    const completeRuns = await this.callback(runIds);
+
+    const durationMs = performance.now() - perfNow;
+
+    const completedRunIds = completeRuns.map((run) => run.id);
+
+    if (completedRunIds.length === 0) {
+      this._logger.debug("[MarqsConcurrencyMonitor] no completed runs found", {
+        key,
+        envKey,
+        runIds,
+        durationMs,
+      });
+
+      return;
+    }
+
+    this._logger.debug("[MarqsConcurrencyMonitor] removing completed runs from queue", {
+      key,
+      envKey,
+      completedRunIds,
+      durationMs,
+    });
+
+    if (this.dryRun) {
+      return;
+    }
+
+    const pipeline = redis.pipeline();
+
+    pipeline.srem(key, ...completedRunIds);
+    pipeline.srem(envKey, ...completedRunIds);
+
+    try {
+      await pipeline.exec();
+    } catch (e) {
+      this._logger.error("[MarqsConcurrencyMonitor] error removing completed runs from queue", {
+        key,
+        envKey,
+        completedRunIds,
+        error: e,
+      });
+    }
+  }
+
+  static async initiateV3Monitoring(abortSignal?: AbortSignal) {
+    if (!marqsv3) {
+      return;
+    }
+
+    const instance = new MarqsConcurrencyMonitor(
+      marqsv3,
+      (runIds) =>
+        prisma.taskRun.findMany({
+          select: { id: true },
+          where: {
+            id: {
+              in: runIds,
+            },
+            status: {
+              in: [
+                "CANCELED",
+                "COMPLETED_SUCCESSFULLY",
+                "COMPLETED_WITH_ERRORS",
+                "CRASHED",
+                "SYSTEM_FAILURE",
+                "INTERRUPTED",
+              ],
+            },
+          },
+        }),
+      { dryRun: env.V3_MARQS_CONCURRENCY_MONITOR_ENABLED === "0", abortSignal }
+    );
+
+    await instance.call();
+  }
+}

apps/webapp/app/v3/marqs/constants.server.ts

@@ -0,0 +1,4 @@
+export const MARQS_RESUME_PRIORITY_TIMESTAMP_OFFSET = 31_556_952 * 1000; // 1 year
+export const MARQS_RETRY_PRIORITY_TIMESTAMP_OFFSET = 15_778_476 * 1000; // 6 months
+export const MARQS_DELAYED_REQUEUE_THRESHOLD_IN_MS = 500;
+export const MARQS_SCHEDULED_REQUEUE_AVAILABLE_AT_THRESHOLD_IN_MS = 500;

apps/webapp/app/v3/marqs/devPubSub.server.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+import { singleton } from "~/utils/singleton";
+import type { ZodSubscriber } from "../utils/zodPubSub.server";
+import { ZodPubSub } from "../utils/zodPubSub.server";
+import { env } from "~/env.server";
+import { Gauge } from "prom-client";
+import { metricsRegister } from "~/metrics.server";
+
+const messageCatalog = {
+  CANCEL_ATTEMPT: z.object({
+    version: z.literal("v1").default("v1"),
+    backgroundWorkerId: z.string(),
+    attemptId: z.string(),
+    taskRunId: z.string(),
+  }),
+};
+
+export type DevSubscriber = ZodSubscriber<typeof messageCatalog>;
+
+export const devPubSub = singleton("devPubSub", initializeDevPubSub);
+
+function initializeDevPubSub() {
+  const pubSub = new ZodPubSub({
+    redis: {
+      port: env.PUBSUB_REDIS_PORT,
+      host: env.PUBSUB_REDIS_HOST,
+      username: env.PUBSUB_REDIS_USERNAME,
+      password: env.PUBSUB_REDIS_PASSWORD,
+      tlsDisabled: env.PUBSUB_REDIS_TLS_DISABLED === "true",
+      clusterMode: env.PUBSUB_REDIS_CLUSTER_MODE_ENABLED === "1",
+    },
+    schema: messageCatalog,
+  });
+
+  new Gauge({
+    name: "dev_pub_sub_subscribers",
+    help: "Number of dev pub sub subscribers",
+    collect() {
+      this.set(pubSub.subscriberCount);
+    },
+    registers: [metricsRegister],
+  });
+
+  return pubSub;
+}

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -0,0 +1,623 @@
+import type { Context, Span } from "@opentelemetry/api";
+import { ROOT_CONTEXT, SpanKind, context, trace } from "@opentelemetry/api";
+import type {
+  V3TaskRunExecution,
+  TaskRunExecutionLazyAttemptPayload,
+  TaskRunExecutionResult,
+  TaskRunFailedExecutionResult,
+  serverWebsocketMessages,
+} from "@trigger.dev/core/v3";
+import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
+import type { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
+import type { BackgroundWorker, BackgroundWorkerTask } from "@trigger.dev/database";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { createNewSession, disconnectSession } from "~/models/runtimeEnvironment.server";
+import { findQueueInEnvironment, sanitizeQueueName } from "~/models/taskQueue.server";
+import type { RedisClient } from "~/redis.server";
+import { createRedisClient } from "~/redis.server";
+import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { resolveVariablesForEnvironment } from "../environmentVariables/environmentVariablesRepository.server";
+import { FailedTaskRunService } from "../failedTaskRun.server";
+import { CancelDevSessionRunsService } from "../services/cancelDevSessionRuns.server";
+import { CompleteAttemptService } from "../services/completeAttempt.server";
+import { attributesFromAuthenticatedEnv, tracer } from "../tracer.server";
+import type { DevSubscriber } from "./devPubSub.server";
+import { devPubSub } from "./devPubSub.server";
+
+const MessageBody = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal("EXECUTE"),
+    taskIdentifier: z.string(),
+  }),
+]);
+
+type BackgroundWorkerWithTasks = BackgroundWorker & { tasks: BackgroundWorkerTask[] };
+
+export type DevQueueConsumerOptions = {
+  maximumItemsPerTrace?: number;
+  traceTimeoutSeconds?: number;
+  ipAddress?: string;
+};
+
+export class DevQueueConsumer {
+  private _backgroundWorkers: Map<string, BackgroundWorkerWithTasks> = new Map();
+  private _backgroundWorkerSubscriber: Map<string, DevSubscriber> = new Map();
+  private _deprecatedWorkers: Map<string, BackgroundWorkerWithTasks> = new Map();
+  private _enabled = false;
+  private _maximumItemsPerTrace: number;
+  private _traceTimeoutSeconds: number;
+  private _perTraceCountdown: number | undefined;
+  private _lastNewTrace: Date | undefined;
+  private _currentSpanContext: Context | undefined;
+  private _taskFailures: number = 0;
+  private _taskSuccesses: number = 0;
+  private _currentSpan: Span | undefined;
+  private _endSpanInNextIteration = false;
+  private _inProgressRuns: Map<string, string> = new Map(); // Keys are task run friendly IDs, values are TaskRun internal ids/queue message ids
+  private _connectionLostAt?: Date;
+  private _redisClient: RedisClient;
+
+  constructor(
+    public id: string,
+    public env: AuthenticatedEnvironment,
+    private _sender: ZodMessageSender<typeof serverWebsocketMessages>,
+    private _options: DevQueueConsumerOptions = {}
+  ) {
+    this._traceTimeoutSeconds = _options.traceTimeoutSeconds ?? 60;
+    this._maximumItemsPerTrace = _options.maximumItemsPerTrace ?? 1_000;
+    this._redisClient = createRedisClient("tr:devQueueConsumer", {
+      keyPrefix: "tr:devQueueConsumer:",
+      ...devPubSub.redisOptions,
+    });
+  }
+
+  // This method is called when a background worker is deprecated and will no longer be used unless a run is locked to it
+  public async deprecateBackgroundWorker(id: string) {
+    const backgroundWorker = this._backgroundWorkers.get(id);
+
+    if (!backgroundWorker) {
+      return;
+    }
+
+    logger.debug("[DevQueueConsumer] Deprecating background worker", {
+      backgroundWorker: backgroundWorker.id,
+      env: this.env.id,
+    });
+
+    this._deprecatedWorkers.set(id, backgroundWorker);
+    this._backgroundWorkers.delete(id);
+  }
+
+  public async registerBackgroundWorker(id: string, inProgressRuns: string[] = []) {
+    const backgroundWorker = await prisma.backgroundWorker.findFirst({
+      where: { friendlyId: id, runtimeEnvironmentId: this.env.id },
+      include: {
+        tasks: true,
+      },
+    });
+
+    if (!backgroundWorker) {
+      return;
+    }
+
+    if (this._backgroundWorkers.has(backgroundWorker.id)) {
+      return;
+    }
+
+    this._backgroundWorkers.set(backgroundWorker.id, backgroundWorker);
+
+    logger.debug("[DevQueueConsumer] Registered background worker", {
+      backgroundWorker: backgroundWorker.id,
+      inProgressRuns,
+      env: this.env.id,
+    });
+
+    const subscriber = await devPubSub.subscribe(`backgroundWorker:${backgroundWorker.id}:*`);
+
+    subscriber.on("CANCEL_ATTEMPT", async (message) => {
+      await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
+        backgroundWorkerId: backgroundWorker.friendlyId,
+        data: {
+          type: "CANCEL_ATTEMPT",
+          taskAttemptId: message.attemptId,
+          taskRunId: message.taskRunId,
+        },
+      });
+    });
+
+    this._backgroundWorkerSubscriber.set(backgroundWorker.id, subscriber);
+
+    for (const runId of inProgressRuns) {
+      this._inProgressRuns.set(runId, runId);
+    }
+
+    // Start reading from the queue if we haven't already
+    await this.#enable();
+  }
+
+  public async taskAttemptCompleted(
+    workerId: string,
+    completion: TaskRunExecutionResult,
+    execution: V3TaskRunExecution
+  ) {
+    if (completion.ok) {
+      this._taskSuccesses++;
+    } else {
+      this._taskFailures++;
+    }
+
+    logger.debug("[DevQueueConsumer] taskAttemptCompleted()", {
+      taskRunCompletion: completion,
+      execution,
+      env: this.env.id,
+    });
+
+    const service = new CompleteAttemptService();
+    const result = await service.call({ completion, execution, env: this.env });
+
+    if (result === "COMPLETED") {
+      this._inProgressRuns.delete(execution.run.id);
+    }
+  }
+
+  public async taskRunFailed(workerId: string, completion: TaskRunFailedExecutionResult) {
+    this._taskFailures++;
+
+    logger.debug("[DevQueueConsumer] taskRunFailed()", { completion, env: this.env.id });
+
+    this._inProgressRuns.delete(completion.id);
+
+    const service = new FailedTaskRunService();
+
+    await service.call(completion.id, completion);
+  }
+
+  /**
+   * @deprecated Use `taskRunHeartbeat` instead
+   */
+  public async taskHeartbeat(workerId: string, id: string) {
+    logger.debug("[DevQueueConsumer] taskHeartbeat()", { id });
+
+    const taskRunAttempt = await prisma.taskRunAttempt.findFirst({
+      where: { friendlyId: id },
+    });
+
+    if (!taskRunAttempt) {
+      return;
+    }
+
+    await marqs?.heartbeatMessage(taskRunAttempt.taskRunId);
+  }
+
+  public async taskRunHeartbeat(workerId: string, id: string) {
+    logger.debug("[DevQueueConsumer] taskRunHeartbeat()", { id });
+
+    await marqs?.heartbeatMessage(id);
+  }
+
+  public async stop(reason: string = "CLI disconnected") {
+    if (!this._enabled) {
+      return;
+    }
+
+    logger.debug("[DevQueueConsumer] Stopping dev queue consumer", { env: this.env });
+
+    this._enabled = false;
+
+    // Create the session
+    const session = await disconnectSession(this.env.id);
+
+    const runIds = Array.from(this._inProgressRuns.values());
+    this._inProgressRuns.clear();
+
+    if (runIds.length > 0) {
+      await CancelDevSessionRunsService.enqueue(
+        {
+          runIds,
+          cancelledAt: new Date(),
+          reason,
+          cancelledSessionId: session?.id,
+        },
+        new Date(Date.now() + 1000 * 10) // 10 seconds from now
+      );
+    }
+
+    // We need to unsubscribe from the background worker channels
+    for (const [id, subscriber] of this._backgroundWorkerSubscriber) {
+      logger.debug("Unsubscribing from background worker channel", { id });
+
+      await subscriber.stopListening();
+      this._backgroundWorkerSubscriber.delete(id);
+
+      logger.debug("Unsubscribed from background worker channel", { id });
+    }
+
+    // We need to end the current span
+    if (this._currentSpan) {
+      this._currentSpan.end();
+    }
+  }
+
+  async #enable() {
+    if (this._enabled) {
+      return;
+    }
+
+    await this._redisClient.set(`connection:${this.env.id}`, this.id, "EX", 60 * 60 * 24); // 24 hours
+
+    this._enabled = true;
+    // Create the session
+    await createNewSession(this.env, this._options.ipAddress ?? "unknown");
+
+    this._perTraceCountdown = this._options.maximumItemsPerTrace;
+    this._lastNewTrace = new Date();
+    this._taskFailures = 0;
+    this._taskSuccesses = 0;
+
+    this.#doWork().finally(() => {});
+  }
+
+  async #doWork() {
+    if (!this._enabled) {
+      return;
+    }
+
+    const canSendMessage = await this._sender.validateCanSendMessage();
+
+    if (!canSendMessage) {
+      this._connectionLostAt ??= new Date();
+
+      if (Date.now() - this._connectionLostAt.getTime() > 60 * 1000) {
+        logger.debug("Connection lost for more than 60 seconds, stopping the consumer", {
+          env: this.env,
+        });
+
+        await this.stop("Connection lost for more than 60 seconds");
+        return;
+      }
+
+      setTimeout(() => this.#doWork(), 1000);
+      return;
+    }
+
+    this._connectionLostAt = undefined;
+
+    const currentConnection = await this._redisClient.get(`connection:${this.env.id}`);
+
+    if (currentConnection && currentConnection !== this.id) {
+      logger.debug("Another connection is active, stopping the consumer", {
+        currentConnection,
+        env: this.env,
+      });
+
+      await this.stop("Another connection is active");
+      return;
+    }
+
+    // Check if the trace has expired
+    if (
+      this._perTraceCountdown === 0 ||
+      Date.now() - this._lastNewTrace!.getTime() > this._traceTimeoutSeconds * 1000 ||
+      this._currentSpanContext === undefined ||
+      this._endSpanInNextIteration
+    ) {
+      if (this._currentSpan) {
+        this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
+        this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
+
+        logger.debug("Ending DevQueueConsumer.doWork() trace", {
+          isRecording: this._currentSpan.isRecording(),
+        });
+
+        this._currentSpan.end();
+      }
+
+      // Create a new trace
+      this._currentSpan = tracer.startSpan(
+        "DevQueueConsumer.doWork()",
+        {
+          kind: SpanKind.CONSUMER,
+          attributes: {
+            ...attributesFromAuthenticatedEnv(this.env),
+          },
+        },
+        ROOT_CONTEXT
+      );
+
+      // Get the span trace context
+      this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);
+
+      this._perTraceCountdown = this._options.maximumItemsPerTrace;
+      this._lastNewTrace = new Date();
+      this._taskFailures = 0;
+      this._taskSuccesses = 0;
+      this._endSpanInNextIteration = false;
+    }
+
+    return context.with(this._currentSpanContext ?? ROOT_CONTEXT, async () => {
+      await this.#doWorkInternal();
+      this._perTraceCountdown = this._perTraceCountdown! - 1;
+    });
+  }
+
+  async #doWorkInternal() {
+    // Attempt to dequeue a message from the environment's queue
+    // If no message is available, reschedule the worker to run again in 1 second
+    // If a message is available, find the BackgroundWorkerTask that matches the message's taskIdentifier
+    // If no matching task is found, nack the message and reschedule the worker to run again in 1 second
+    // If the matching task is found, create the task attempt and lock the task run, then send the task run to the client
+    // Store the message as a processing message
+    // If the websocket connection disconnects before the task run is completed, nack the message
+    // When the task run completes, ack the message
+    // Using a heartbeat mechanism, if the client keeps responding with a heartbeat, we'll keep the message processing and increase the visibility timeout.
+
+    const message = await marqs?.dequeueMessageInEnv(this.env);
+
+    if (!message) {
+      setTimeout(() => this.#doWork(), 1000);
+      return;
+    }
+
+    const dequeuedStart = Date.now();
+
+    const messageBody = MessageBody.safeParse(message.data);
+
+    if (!messageBody.success) {
+      logger.error("Failed to parse message", {
+        queueMessage: message.data,
+        error: messageBody.error,
+        env: this.env,
+      });
+
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "Failed to parse message.data with MessageBody schema in DevQueueConsumer"
+      );
+
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const existingTaskRun = await prisma.taskRun.findFirst({
+      where: {
+        id: message.messageId,
+      },
+    });
+
+    if (!existingTaskRun) {
+      logger.debug("Failed to find existing task run, acking", {
+        messageId: message.messageId,
+      });
+
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "Failed to find task run in DevQueueConsumer"
+      );
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const backgroundWorker = existingTaskRun.lockedToVersionId
+      ? (this._deprecatedWorkers.get(existingTaskRun.lockedToVersionId) ??
+        this._backgroundWorkers.get(existingTaskRun.lockedToVersionId))
+      : this.#getLatestBackgroundWorker();
+
+    if (!backgroundWorker) {
+      logger.debug("Failed to find background worker, acking", {
+        messageId: message.messageId,
+        lockedToVersionId: existingTaskRun.lockedToVersionId,
+        deprecatedWorkers: Array.from(this._deprecatedWorkers.keys()),
+        backgroundWorkers: Array.from(this._backgroundWorkers.keys()),
+        latestWorker: this.#getLatestBackgroundWorker(),
+      });
+
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "Failed to find background worker in DevQueueConsumer"
+      );
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const backgroundTask = backgroundWorker.tasks.find(
+      (task) => task.slug === existingTaskRun.taskIdentifier
+    );
+
+    if (!backgroundTask) {
+      logger.warn("No matching background task found for task run", {
+        taskRun: existingTaskRun.id,
+        taskIdentifier: existingTaskRun.taskIdentifier,
+        backgroundWorker: backgroundWorker.id,
+        taskSlugs: backgroundWorker.tasks.map((task) => task.slug),
+      });
+
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "No matching background task found in DevQueueConsumer"
+      );
+
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const lockedAt = new Date();
+    const startedAt = existingTaskRun.startedAt ?? new Date();
+
+    const lockedTaskRun = await prisma.taskRun.update({
+      where: {
+        id: message.messageId,
+      },
+      data: {
+        lockedAt,
+        lockedById: backgroundTask.id,
+        status: "EXECUTING",
+        lockedToVersionId: backgroundWorker.id,
+        taskVersion: backgroundWorker.version,
+        sdkVersion: backgroundWorker.sdkVersion,
+        cliVersion: backgroundWorker.cliVersion,
+        startedAt,
+        maxDurationInSeconds: getMaxDuration(
+          existingTaskRun.maxDurationInSeconds,
+          backgroundTask.maxDurationInSeconds
+        ),
+      },
+    });
+
+    if (!lockedTaskRun) {
+      logger.warn("Failed to lock task run", {
+        taskRun: existingTaskRun.id,
+        taskIdentifier: existingTaskRun.taskIdentifier,
+        backgroundWorker: backgroundWorker.id,
+        messageId: message.messageId,
+      });
+
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "Failed to lock task run in DevQueueConsumer"
+      );
+
+      setTimeout(() => this.#doWork(), 100);
+      return;
+    }
+
+    const queue = await findQueueInEnvironment(
+      lockedTaskRun.queue,
+      this.env.id,
+      backgroundTask.id,
+      backgroundTask
+    );
+
+    if (!queue) {
+      logger.debug("[DevQueueConsumer] Failed to find queue", {
+        queueName: lockedTaskRun.queue,
+        sanitizedName: sanitizeQueueName(lockedTaskRun.queue),
+        taskRun: lockedTaskRun.id,
+        messageId: message.messageId,
+      });
+
+      await marqs?.nackMessage(message.messageId);
+      setTimeout(() => this.#doWork(), 1000);
+      return;
+    }
+
+    if (!this._enabled) {
+      logger.debug("Dev queue consumer is disabled", { env: this.env, queueMessage: message });
+
+      await marqs?.nackMessage(message.messageId);
+      return;
+    }
+
+    const variables = await resolveVariablesForEnvironment(this.env);
+
+    if (backgroundWorker.supportsLazyAttempts) {
+      const payload: TaskRunExecutionLazyAttemptPayload = {
+        traceContext: lockedTaskRun.traceContext as Record<string, unknown>,
+        environment: variables.reduce((acc: Record<string, string>, curr) => {
+          acc[curr.key] = curr.value;
+          return acc;
+        }, {}),
+        runId: lockedTaskRun.friendlyId,
+        messageId: lockedTaskRun.id,
+        isTest: lockedTaskRun.isTest,
+        isReplay: !!lockedTaskRun.replayedFromTaskRunFriendlyId,
+        metrics: [
+          {
+            name: "start",
+            event: "dequeue",
+            timestamp: dequeuedStart,
+            duration: Date.now() - dequeuedStart,
+          },
+        ],
+      };
+
+      try {
+        await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
+          backgroundWorkerId: backgroundWorker.friendlyId,
+          data: {
+            type: "EXECUTE_RUN_LAZY_ATTEMPT",
+            payload,
+          },
+        });
+
+        logger.debug("Executing the run", {
+          messageId: message.messageId,
+        });
+
+        this._inProgressRuns.set(lockedTaskRun.friendlyId, message.messageId);
+      } catch (e) {
+        if (e instanceof Error) {
+          this._currentSpan?.recordException(e);
+        } else {
+          this._currentSpan?.recordException(new Error(String(e)));
+        }
+
+        this._endSpanInNextIteration = true;
+
+        // We now need to unlock the task run and delete the task run attempt
+        await prisma.$transaction([
+          prisma.taskRun.update({
+            where: {
+              id: lockedTaskRun.id,
+            },
+            data: {
+              lockedAt: null,
+              lockedById: null,
+              status: "PENDING",
+              startedAt: existingTaskRun.startedAt,
+            },
+          }),
+        ]);
+
+        this._inProgressRuns.delete(lockedTaskRun.friendlyId);
+
+        // Finally we need to nack the message so it can be retried
+        await marqs?.nackMessage(message.messageId);
+      } finally {
+        setTimeout(() => this.#doWork(), 100);
+      }
+    } else {
+      logger.debug("We no longer support non-lazy attempts, aborting this run", {
+        messageId: message.messageId,
+        backgroundWorker,
+      });
+      await marqs?.acknowledgeMessage(
+        message.messageId,
+        "Non-lazy attempts are no longer supported in DevQueueConsumer"
+      );
+
+      setTimeout(() => this.#doWork(), 100);
+    }
+  }
+
+  // Get the latest background worker based on the version.
+  // Versions are in the format of 20240101.1 and 20240101.2, or even 20240101.10, 20240101.11, etc.
+  #getLatestBackgroundWorker() {
+    const workers = Array.from(this._backgroundWorkers.values());
+
+    if (workers.length === 0) {
+      return;
+    }
+
+    return workers.reduce((acc, curr) => {
+      const accParts = acc.version.split(".").map(Number);
+      const currParts = curr.version.split(".").map(Number);
+
+      // Compare the major part
+      if (accParts[0] < currParts[0]) {
+        return curr;
+      } else if (accParts[0] > currParts[0]) {
+        return acc;
+      }
+
+      // Compare the minor part (assuming all versions have two parts)
+      if (accParts[1] < currParts[1]) {
+        return curr;
+      } else {
+        return acc;
+      }
+    });
+  }
+}

apps/webapp/app/v3/marqs/fairDequeuingStrategy.server.ts

@@ -0,0 +1,598 @@
+import type { Cache as UnkeyCache } from "@unkey/cache";
+import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache";
+import { createLRUMemoryStore } from "@internal/cache";
+import { randomUUID } from "crypto";
+import type { Redis } from "ioredis";
+import type { EnvQueues, MarQSFairDequeueStrategy, MarQSKeyProducer } from "./types";
+import seedrandom from "seedrandom";
+import type { Tracer } from "@opentelemetry/api";
+import { startSpan } from "../tracing.server";
+
+export type FairDequeuingStrategyBiases = {
+  /**
+   * How much to bias towards environments with higher concurrency limits
+   * 0 = no bias, 1 = full bias based on limit differences
+   */
+  concurrencyLimitBias: number;
+
+  /**
+   * How much to bias towards environments with more available capacity
+   * 0 = no bias, 1 = full bias based on available capacity
+   */
+  availableCapacityBias: number;
+
+  /**
+   * Controls randomization of queue ordering within environments
+   * 0 = strict age-based ordering (oldest first)
+   * 1 = completely random ordering
+   * Values between 0-1 blend between age-based and random ordering
+   */
+  queueAgeRandomization: number;
+};
+
+export type FairDequeuingStrategyOptions = {
+  redis: Redis;
+  keys: MarQSKeyProducer;
+  defaultEnvConcurrency: number;
+  parentQueueLimit: number;
+  tracer: Tracer;
+  seed?: string;
+  /**
+   * Configure biasing for environment shuffling
+   * If not provided, no biasing will be applied (completely random shuffling)
+   */
+  biases?: FairDequeuingStrategyBiases;
+  reuseSnapshotCount?: number;
+  maximumEnvCount?: number;
+  /**
+   * Maximum number of queues to process per environment
+   * If not provided, all queues in an environment will be processed
+   */
+  maximumQueuePerEnvCount?: number;
+};
+
+type FairQueueConcurrency = {
+  current: number;
+  limit: number;
+  reserve: number;
+};
+
+type FairQueue = { id: string; age: number; org: string; env: string };
+
+type FairQueueSnapshot = {
+  id: string;
+  envs: Record<string, { concurrency: FairQueueConcurrency }>;
+  queues: Array<FairQueue>;
+};
+
+type WeightedEnv = {
+  envId: string;
+  weight: number;
+};
+
+type WeightedQueue = {
+  queue: FairQueue;
+  weight: number;
+};
+
+const emptyFairQueueSnapshot: FairQueueSnapshot = {
+  id: "empty",
+  envs: {},
+  queues: [],
+};
+
+const defaultBiases: FairDequeuingStrategyBiases = {
+  concurrencyLimitBias: 0,
+  availableCapacityBias: 0,
+  queueAgeRandomization: 0, // Default to completely age-based ordering
+};
+
+export class FairDequeuingStrategy implements MarQSFairDequeueStrategy {
+  private _cache: UnkeyCache<{
+    concurrencyLimit: number;
+  }>;
+
+  private _rng: seedrandom.PRNG;
+  private _reusedSnapshotForConsumer: Map<
+    string,
+    { snapshot: FairQueueSnapshot; reuseCount: number }
+  > = new Map();
+
+  constructor(private options: FairDequeuingStrategyOptions) {
+    const ctx = new DefaultStatefulContext();
+    const memory = createLRUMemoryStore(500);
+
+    this._cache = createCache({
+      concurrencyLimit: new Namespace<number>(ctx, {
+        stores: [memory],
+        fresh: 60_000, // The time in milliseconds that a value is considered fresh. Cache hits within this time will return the cached value.
+        stale: 180_000, // The time in milliseconds that a value is considered stale. Cache hits within this time will return the cached value and trigger a background refresh.
+      }),
+    });
+
+    this._rng = seedrandom(options.seed);
+  }
+
+  async distributeFairQueuesFromParentQueue(
+    parentQueue: string,
+    consumerId: string
+  ): Promise<Array<EnvQueues>> {
+    return await startSpan(
+      this.options.tracer,
+      "distributeFairQueuesFromParentQueue",
+      async (span) => {
+        span.setAttribute("consumer_id", consumerId);
+        span.setAttribute("parent_queue", parentQueue);
+
+        const snapshot = await this.#createQueueSnapshot(parentQueue, consumerId);
+
+        span.setAttributes({
+          snapshot_env_count: Object.keys(snapshot.envs).length,
+          snapshot_queue_count: snapshot.queues.length,
+        });
+
+        const queues = snapshot.queues;
+
+        if (queues.length === 0) {
+          return [];
+        }
+
+        const envQueues = this.#shuffleQueuesByEnv(snapshot);
+
+        span.setAttribute(
+          "shuffled_queue_count",
+          envQueues.reduce((sum, env) => sum + env.queues.length, 0)
+        );
+
+        if (envQueues[0]?.queues[0]) {
+          span.setAttribute("winning_env", envQueues[0].envId);
+          span.setAttribute(
+            "winning_org",
+            this.options.keys.orgIdFromQueue(envQueues[0].queues[0])
+          );
+        }
+
+        return envQueues;
+      }
+    );
+  }
+
+  #shuffleQueuesByEnv(snapshot: FairQueueSnapshot): Array<EnvQueues> {
+    const envs = Object.keys(snapshot.envs);
+    const biases = this.options.biases ?? defaultBiases;
+
+    if (biases.concurrencyLimitBias === 0 && biases.availableCapacityBias === 0) {
+      const shuffledEnvs = this.#shuffle(envs);
+      return this.#orderQueuesByEnvs(shuffledEnvs, snapshot);
+    }
+
+    // Find the maximum concurrency limit for normalization
+    const maxLimit = Math.max(...envs.map((envId) => snapshot.envs[envId].concurrency.limit));
+
+    // Calculate weights for each environment
+    const weightedEnvs: WeightedEnv[] = envs.map((envId) => {
+      const env = snapshot.envs[envId];
+
+      // Start with base weight of 1
+      let weight = 1;
+
+      // Add normalized concurrency limit bias if configured
+      if (biases.concurrencyLimitBias > 0) {
+        const normalizedLimit = env.concurrency.limit / maxLimit;
+        // Square or cube the bias to make it more pronounced at higher values
+        weight *= 1 + Math.pow(normalizedLimit * biases.concurrencyLimitBias, 2);
+      }
+
+      // Add available capacity bias if configured
+      if (biases.availableCapacityBias > 0) {
+        const usedCapacityPercentage = env.concurrency.current / env.concurrency.limit;
+        const availableCapacityBonus = 1 - usedCapacityPercentage;
+        // Square or cube the bias to make it more pronounced at higher values
+        weight *= 1 + Math.pow(availableCapacityBonus * biases.availableCapacityBias, 2);
+      }
+
+      return { envId, weight };
+    });
+
+    const shuffledEnvs = this.#weightedShuffle(weightedEnvs);
+    return this.#orderQueuesByEnvs(shuffledEnvs, snapshot);
+  }
+
+  #weightedShuffle(weightedItems: WeightedEnv[]): string[] {
+    const totalWeight = weightedItems.reduce((sum, item) => sum + item.weight, 0);
+    const result: string[] = [];
+    const items = [...weightedItems];
+
+    while (items.length > 0) {
+      let random = this._rng() * totalWeight;
+      let index = 0;
+
+      // Find item based on weighted random selection
+      while (random > 0 && index < items.length) {
+        random -= items[index].weight;
+        index++;
+      }
+      index = Math.max(0, index - 1);
+
+      // Add selected item to result and remove from items
+      result.push(items[index].envId);
+      items.splice(index, 1);
+    }
+
+    return result;
+  }
+
+  #orderQueuesByEnvs(envs: string[], snapshot: FairQueueSnapshot): Array<EnvQueues> {
+    const queuesByEnv = snapshot.queues.reduce(
+      (acc, queue) => {
+        if (!acc[queue.env]) {
+          acc[queue.env] = [];
+        }
+        acc[queue.env].push(queue);
+        return acc;
+      },
+      {} as Record<string, Array<FairQueue>>
+    );
+
+    return envs.reduce((acc, envId) => {
+      if (queuesByEnv[envId]) {
+        // Get ordered queues for this env
+        const orderedQueues = this.#weightedRandomQueueOrder(queuesByEnv[envId]);
+
+        // Apply queue limit if maximumQueuePerEnvCount is set
+        const limitedQueues = this.options.maximumQueuePerEnvCount
+          ? orderedQueues.slice(0, this.options.maximumQueuePerEnvCount)
+          : orderedQueues;
+
+        // Only add the env if it has queues
+        if (limitedQueues.length > 0) {
+          acc.push({
+            envId,
+            queues: limitedQueues.map((queue) => queue.id),
+          });
+        }
+      }
+      return acc;
+    }, [] as Array<EnvQueues>);
+  }
+
+  #weightedRandomQueueOrder(queues: FairQueue[]): FairQueue[] {
+    if (queues.length <= 1) return queues;
+
+    const biases = this.options.biases ?? defaultBiases;
+
+    // When queueAgeRandomization is 0, use strict age-based ordering
+    if (biases.queueAgeRandomization === 0) {
+      return [...queues].sort((a, b) => b.age - a.age);
+    }
+
+    // Find the maximum age for normalization
+    const maxAge = Math.max(...queues.map((q) => q.age));
+
+    // Calculate weights for each queue
+    const weightedQueues: WeightedQueue[] = queues.map((queue) => {
+      // Normalize age to be between 0 and 1
+      const normalizedAge = queue.age / maxAge;
+
+      // Calculate weight: combine base weight with configurable age influence
+      const baseWeight = 1;
+      const weight = baseWeight + normalizedAge * biases.queueAgeRandomization;
+
+      return { queue, weight };
+    });
+
+    // Perform weighted random selection for ordering
+    const result: FairQueue[] = [];
+    let remainingQueues = [...weightedQueues];
+    let totalWeight = remainingQueues.reduce((sum, wq) => sum + wq.weight, 0);
+
+    while (remainingQueues.length > 0) {
+      let random = this._rng() * totalWeight;
+      let index = 0;
+
+      // Find queue based on weighted random selection
+      while (random > 0 && index < remainingQueues.length) {
+        random -= remainingQueues[index].weight;
+        index++;
+      }
+      index = Math.max(0, index - 1);
+
+      // Add selected queue to result and remove from remaining
+      result.push(remainingQueues[index].queue);
+      totalWeight -= remainingQueues[index].weight;
+      remainingQueues.splice(index, 1);
+    }
+
+    return result;
+  }
+
+  #shuffle<T>(array: Array<T>): Array<T> {
+    let currentIndex = array.length;
+    let temporaryValue;
+    let randomIndex;
+
+    const newArray = [...array];
+
+    while (currentIndex !== 0) {
+      randomIndex = Math.floor(this._rng() * currentIndex);
+      currentIndex -= 1;
+
+      temporaryValue = newArray[currentIndex];
+      newArray[currentIndex] = newArray[randomIndex];
+      newArray[randomIndex] = temporaryValue;
+    }
+
+    return newArray;
+  }
+
+  async #createQueueSnapshot(parentQueue: string, consumerId: string): Promise<FairQueueSnapshot> {
+    return await startSpan(this.options.tracer, "createQueueSnapshot", async (span) => {
+      span.setAttribute("consumer_id", consumerId);
+      span.setAttribute("parent_queue", parentQueue);
+
+      if (
+        typeof this.options.reuseSnapshotCount === "number" &&
+        this.options.reuseSnapshotCount > 0
+      ) {
+        const key = `${parentQueue}:${consumerId}`;
+        const reusedSnapshot = this._reusedSnapshotForConsumer.get(key);
+
+        if (reusedSnapshot) {
+          if (reusedSnapshot.reuseCount < this.options.reuseSnapshotCount) {
+            span.setAttribute("reused_snapshot", true);
+
+            this._reusedSnapshotForConsumer.set(key, {
+              snapshot: reusedSnapshot.snapshot,
+              reuseCount: reusedSnapshot.reuseCount + 1,
+            });
+
+            return reusedSnapshot.snapshot;
+          } else {
+            this._reusedSnapshotForConsumer.delete(key);
+          }
+        }
+      }
+
+      span.setAttribute("reused_snapshot", false);
+
+      const now = Date.now();
+
+      let queues = await this.#allChildQueuesByScore(parentQueue, consumerId, now);
+
+      span.setAttribute("parent_queue_count", queues.length);
+
+      if (queues.length === 0) {
+        return emptyFairQueueSnapshot;
+      }
+
+      // Apply env selection if maximumEnvCount is specified
+      let selectedEnvIds: Set<string>;
+      if (this.options.maximumEnvCount && this.options.maximumEnvCount > 0) {
+        selectedEnvIds = this.#selectTopEnvs(queues, this.options.maximumEnvCount);
+        // Filter queues to only include selected envs
+        queues = queues.filter((queue) => selectedEnvIds.has(queue.env));
+
+        span.setAttribute("selected_env_count", selectedEnvIds.size);
+      }
+
+      span.setAttribute("selected_queue_count", queues.length);
+
+      const envIds = new Set<string>();
+
+      for (const queue of queues) {
+        envIds.add(queue.env);
+      }
+
+      const envs = await Promise.all(
+        Array.from(envIds).map(async (envId) => {
+          return { id: envId, concurrency: await this.#getEnvConcurrency(envId) };
+        })
+      );
+
+      const envsAtFullConcurrency = envs.filter(
+        (env) => env.concurrency.current >= env.concurrency.limit + env.concurrency.reserve
+      );
+
+      const envIdsAtFullConcurrency = new Set(envsAtFullConcurrency.map((env) => env.id));
+
+      const envsSnapshot = envs.reduce(
+        (acc, env) => {
+          if (!envIdsAtFullConcurrency.has(env.id)) {
+            acc[env.id] = env;
+          }
+          return acc;
+        },
+        {} as Record<string, { concurrency: FairQueueConcurrency }>
+      );
+
+      span.setAttributes({
+        env_count: envs.length,
+        envs_at_full_concurrency_count: envsAtFullConcurrency.length,
+      });
+
+      const queuesSnapshot = queues.filter((queue) => !envIdsAtFullConcurrency.has(queue.env));
+
+      const snapshot = {
+        id: randomUUID(),
+        envs: envsSnapshot,
+        queues: queuesSnapshot,
+      };
+
+      if (
+        typeof this.options.reuseSnapshotCount === "number" &&
+        this.options.reuseSnapshotCount > 0
+      ) {
+        this._reusedSnapshotForConsumer.set(`${parentQueue}:${consumerId}`, {
+          snapshot,
+          reuseCount: 0,
+        });
+      }
+
+      return snapshot;
+    });
+  }
+
+  #selectTopEnvs(queues: FairQueue[], maximumEnvCount: number): Set<string> {
+    // Group queues by env
+    const queuesByEnv = queues.reduce(
+      (acc, queue) => {
+        if (!acc[queue.env]) {
+          acc[queue.env] = [];
+        }
+        acc[queue.env].push(queue);
+        return acc;
+      },
+      {} as Record<string, FairQueue[]>
+    );
+
+    // Calculate average age for each env
+    const envAverageAges = Object.entries(queuesByEnv).map(([envId, envQueues]) => {
+      const averageAge = envQueues.reduce((sum, q) => sum + q.age, 0) / envQueues.length;
+      return { envId, averageAge };
+    });
+
+    // Perform weighted shuffle based on average ages
+    const maxAge = Math.max(...envAverageAges.map((e) => e.averageAge));
+    const weightedEnvs = envAverageAges.map((env) => ({
+      envId: env.envId,
+      weight: env.averageAge / maxAge, // Normalize weights
+    }));
+
+    // Select top N envs using weighted shuffle
+    const selectedEnvs = new Set<string>();
+    let remainingEnvs = [...weightedEnvs];
+    let totalWeight = remainingEnvs.reduce((sum, env) => sum + env.weight, 0);
+
+    while (selectedEnvs.size < maximumEnvCount && remainingEnvs.length > 0) {
+      let random = this._rng() * totalWeight;
+      let index = 0;
+
+      while (random > 0 && index < remainingEnvs.length) {
+        random -= remainingEnvs[index].weight;
+        index++;
+      }
+      index = Math.max(0, index - 1);
+
+      selectedEnvs.add(remainingEnvs[index].envId);
+      totalWeight -= remainingEnvs[index].weight;
+      remainingEnvs.splice(index, 1);
+    }
+
+    return selectedEnvs;
+  }
+
+  async #getEnvConcurrency(envId: string): Promise<FairQueueConcurrency> {
+    return await startSpan(this.options.tracer, "getEnvConcurrency", async (span) => {
+      span.setAttribute("env_id", envId);
+
+      const [currentValue, limitValue, reserveValue] = await Promise.all([
+        this.#getEnvCurrentConcurrency(envId),
+        this.#getEnvConcurrencyLimit(envId),
+        this.#getEnvReserveConcurrency(envId),
+      ]);
+
+      span.setAttribute("current_value", currentValue);
+      span.setAttribute("limit_value", limitValue);
+      span.setAttribute("reserve_value", reserveValue);
+
+      return { current: currentValue, limit: limitValue, reserve: reserveValue };
+    });
+  }
+
+  async #allChildQueuesByScore(
+    parentQueue: string,
+    consumerId: string,
+    now: number
+  ): Promise<Array<FairQueue>> {
+    return await startSpan(this.options.tracer, "allChildQueuesByScore", async (span) => {
+      span.setAttribute("consumer_id", consumerId);
+      span.setAttribute("parent_queue", parentQueue);
+
+      const valuesWithScores = await this.options.redis.zrangebyscore(
+        parentQueue,
+        "-inf",
+        now,
+        "WITHSCORES",
+        "LIMIT",
+        0,
+        this.options.parentQueueLimit
+      );
+
+      const result: Array<FairQueue> = [];
+
+      for (let i = 0; i < valuesWithScores.length; i += 2) {
+        result.push({
+          id: valuesWithScores[i],
+          age: now - Number(valuesWithScores[i + 1]),
+          env: this.options.keys.envIdFromQueue(valuesWithScores[i]),
+          org: this.options.keys.orgIdFromQueue(valuesWithScores[i]),
+        });
+      }
+
+      span.setAttribute("queue_count", result.length);
+
+      if (result.length === this.options.parentQueueLimit) {
+        span.setAttribute("parent_queue_limit_reached", true);
+      }
+
+      return result;
+    });
+  }
+
+  async #getEnvConcurrencyLimit(envId: string) {
+    return await startSpan(this.options.tracer, "getEnvConcurrencyLimit", async (span) => {
+      span.setAttribute("env_id", envId);
+
+      const key = this.options.keys.envConcurrencyLimitKey(envId);
+
+      const result = await this._cache.concurrencyLimit.swr(key, async () => {
+        const value = await this.options.redis.get(key);
+
+        if (!value) {
+          return this.options.defaultEnvConcurrency;
+        }
+
+        return Number(value);
+      });
+
+      return result.val ?? this.options.defaultEnvConcurrency;
+    });
+  }
+
+  async #getEnvCurrentConcurrency(envId: string) {
+    return await startSpan(this.options.tracer, "getEnvCurrentConcurrency", async (span) => {
+      span.setAttribute("env_id", envId);
+
+      const key = this.options.keys.envCurrentConcurrencyKey(envId);
+
+      const result = await this.options.redis.scard(key);
+
+      span.setAttribute("current_value", result);
+
+      return result;
+    });
+  }
+
+  async #getEnvReserveConcurrency(envId: string) {
+    return await startSpan(this.options.tracer, "getEnvReserveConcurrency", async (span) => {
+      span.setAttribute("env_id", envId);
+
+      const key = this.options.keys.envReserveConcurrencyKey(envId);
+
+      const result = await this.options.redis.scard(key);
+
+      span.setAttribute("current_value", result);
+
+      return result;
+    });
+  }
+}
+
+export class NoopFairDequeuingStrategy implements MarQSFairDequeueStrategy {
+  async distributeFairQueuesFromParentQueue(
+    parentQueue: string,
+    consumerId: string
+  ): Promise<Array<EnvQueues>> {
+    return [];
+  }
+}

apps/webapp/app/v3/marqs/marqsKeyProducer.ts

@@ -0,0 +1,287 @@
+import type { MarQSKeyProducer, MarQSKeyProducerEnv, QueueDescriptor } from "./types";
+
+const constants = {
+  SHARED_QUEUE: "sharedQueue",
+  SHARED_WORKER_QUEUE: "sharedWorkerQueue",
+  CURRENT_CONCURRENCY_PART: "currentConcurrency",
+  CONCURRENCY_LIMIT_PART: "concurrency",
+  DISABLED_CONCURRENCY_LIMIT_PART: "disabledConcurrency",
+  ENV_PART: "env",
+  ORG_PART: "org",
+  QUEUE_PART: "queue",
+  CONCURRENCY_KEY_PART: "ck",
+  MESSAGE_PART: "message",
+  RESERVE_CONCURRENCY_PART: "reserveConcurrency",
+} as const;
+
+const ORG_REGEX = /org:([^:]+):/;
+const ENV_REGEX = /env:([^:]+):/;
+const QUEUE_REGEX = /queue:([^:]+)(?::|$)/;
+const CONCURRENCY_KEY_REGEX = /ck:([^:]+)(?::|$)/;
+
+export class MarQSShortKeyProducer implements MarQSKeyProducer {
+  constructor(private _prefix: string) {}
+
+  sharedQueueScanPattern() {
+    return `${this._prefix}*${constants.SHARED_QUEUE}`;
+  }
+
+  queueCurrentConcurrencyScanPattern() {
+    return `${this._prefix}${constants.ORG_PART}:*:${constants.ENV_PART}:*:queue:*:${constants.CURRENT_CONCURRENCY_PART}`;
+  }
+
+  stripKeyPrefix(key: string): string {
+    if (key.startsWith(this._prefix)) {
+      return key.slice(this._prefix.length);
+    }
+
+    return key;
+  }
+
+  queueConcurrencyLimitKey(env: MarQSKeyProducerEnv, queue: string) {
+    return [this.queueKey(env, queue), constants.CONCURRENCY_LIMIT_PART].join(":");
+  }
+
+  envConcurrencyLimitKey(envId: string): string;
+  envConcurrencyLimitKey(env: MarQSKeyProducerEnv): string;
+  envConcurrencyLimitKey(envOrId: MarQSKeyProducerEnv | string): string {
+    return [
+      this.envKeySection(typeof envOrId === "string" ? envOrId : envOrId.id),
+      constants.CONCURRENCY_LIMIT_PART,
+    ].join(":");
+  }
+
+  queueKey(orgId: string, envId: string, queue: string, concurrencyKey?: string): string;
+  queueKey(env: MarQSKeyProducerEnv, queue: string, concurrencyKey?: string): string;
+  queueKey(
+    envOrOrgId: MarQSKeyProducerEnv | string,
+    queueOrEnvId: string,
+    queueOrConcurrencyKey: string,
+    concurrencyKeyOrPriority?: string | number
+  ): string {
+    if (typeof envOrOrgId === "string") {
+      return [
+        this.orgKeySection(envOrOrgId),
+        this.envKeySection(queueOrEnvId),
+        this.queueSection(queueOrConcurrencyKey),
+      ]
+        .concat(
+          typeof concurrencyKeyOrPriority === "string"
+            ? this.concurrencyKeySection(concurrencyKeyOrPriority)
+            : []
+        )
+        .join(":");
+    } else {
+      return [
+        this.orgKeySection(envOrOrgId.organizationId),
+        this.envKeySection(envOrOrgId.id),
+        this.queueSection(queueOrEnvId),
+      ]
+        .concat(queueOrConcurrencyKey ? this.concurrencyKeySection(queueOrConcurrencyKey) : [])
+        .join(":");
+    }
+  }
+
+  queueKeyFromQueue(queue: string): string {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return this.queueKey(
+      descriptor.organization,
+      descriptor.environment,
+      descriptor.name,
+      descriptor.concurrencyKey
+    );
+  }
+
+  envSharedQueueKey(env: MarQSKeyProducerEnv) {
+    if (env.type === "DEVELOPMENT") {
+      return [
+        this.orgKeySection(env.organizationId),
+        this.envKeySection(env.id),
+        constants.SHARED_QUEUE,
+      ].join(":");
+    }
+
+    return this.sharedQueueKey();
+  }
+
+  sharedQueueKey(): string {
+    return constants.SHARED_QUEUE;
+  }
+
+  sharedWorkerQueueKey(): string {
+    return constants.SHARED_WORKER_QUEUE;
+  }
+
+  queueConcurrencyLimitKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return this.queueConcurrencyLimitKeyFromDescriptor(descriptor);
+  }
+
+  queueCurrentConcurrencyKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+    return this.currentConcurrencyKeyFromDescriptor(descriptor);
+  }
+
+  queueReserveConcurrencyKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return this.queueReserveConcurrencyKeyFromDescriptor(descriptor);
+  }
+
+  queueCurrentConcurrencyKey(
+    env: MarQSKeyProducerEnv,
+    queue: string,
+    concurrencyKey?: string
+  ): string {
+    return [this.queueKey(env, queue, concurrencyKey), constants.CURRENT_CONCURRENCY_PART].join(
+      ":"
+    );
+  }
+
+  envConcurrencyLimitKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return `${constants.ENV_PART}:${descriptor.environment}:${constants.CONCURRENCY_LIMIT_PART}`;
+  }
+
+  envCurrentConcurrencyKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return `${constants.ENV_PART}:${descriptor.environment}:${constants.CURRENT_CONCURRENCY_PART}`;
+  }
+
+  envReserveConcurrencyKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return this.envReserveConcurrencyKey(descriptor.environment);
+  }
+
+  envReserveConcurrencyKey(envId: string): string {
+    return `${constants.ENV_PART}:${this.shortId(envId)}:${constants.RESERVE_CONCURRENCY_PART}`;
+  }
+
+  envCurrentConcurrencyKey(envId: string): string;
+  envCurrentConcurrencyKey(env: MarQSKeyProducerEnv): string;
+  envCurrentConcurrencyKey(envOrId: MarQSKeyProducerEnv | string): string {
+    return [
+      this.envKeySection(typeof envOrId === "string" ? envOrId : envOrId.id),
+      constants.CURRENT_CONCURRENCY_PART,
+    ].join(":");
+  }
+
+  envQueueKeyFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return `${constants.ENV_PART}:${descriptor.environment}:${constants.QUEUE_PART}`;
+  }
+
+  envQueueKey(env: MarQSKeyProducerEnv): string {
+    return [constants.ENV_PART, this.shortId(env.id), constants.QUEUE_PART].join(":");
+  }
+
+  messageKey(messageId: string) {
+    return `${constants.MESSAGE_PART}:${messageId}`;
+  }
+
+  nackCounterKey(messageId: string): string {
+    return `${constants.MESSAGE_PART}:${messageId}:nacks`;
+  }
+
+  orgIdFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return descriptor.organization;
+  }
+
+  envIdFromQueue(queue: string) {
+    const descriptor = this.queueDescriptorFromQueue(queue);
+
+    return descriptor.environment;
+  }
+
+  queueDescriptorFromQueue(queue: string): QueueDescriptor {
+    const match = queue.match(QUEUE_REGEX);
+
+    if (!match) {
+      throw new Error(`Invalid queue: ${queue}, no queue name found`);
+    }
+
+    const [, queueName] = match;
+
+    const envMatch = queue.match(ENV_REGEX);
+
+    if (!envMatch) {
+      throw new Error(`Invalid queue: ${queue}, no environment found`);
+    }
+
+    const [, envId] = envMatch;
+
+    const orgMatch = queue.match(ORG_REGEX);
+
+    if (!orgMatch) {
+      throw new Error(`Invalid queue: ${queue}, no organization found`);
+    }
+
+    const [, orgId] = orgMatch;
+
+    const concurrencyKeyMatch = queue.match(CONCURRENCY_KEY_REGEX);
+
+    const concurrencyKey = concurrencyKeyMatch ? concurrencyKeyMatch[1] : undefined;
+
+    return {
+      name: queueName,
+      environment: envId,
+      organization: orgId,
+      concurrencyKey,
+    };
+  }
+
+  private shortId(id: string) {
+    // Return the last 12 characters of the id
+    return id.slice(-12);
+  }
+
+  private envKeySection(envId: string) {
+    return `${constants.ENV_PART}:${this.shortId(envId)}`;
+  }
+
+  private orgKeySection(orgId: string) {
+    return `${constants.ORG_PART}:${this.shortId(orgId)}`;
+  }
+
+  private queueSection(queue: string) {
+    return `${constants.QUEUE_PART}:${queue}`;
+  }
+
+  private concurrencyKeySection(concurrencyKey: string) {
+    return `${constants.CONCURRENCY_KEY_PART}:${concurrencyKey}`;
+  }
+
+  private currentConcurrencyKeyFromDescriptor(descriptor: QueueDescriptor) {
+    return [
+      this.queueKey(
+        descriptor.organization,
+        descriptor.environment,
+        descriptor.name,
+        descriptor.concurrencyKey
+      ),
+      constants.CURRENT_CONCURRENCY_PART,
+    ].join(":");
+  }
+
+  private queueReserveConcurrencyKeyFromDescriptor(descriptor: QueueDescriptor) {
+    return [
+      this.queueKey(descriptor.organization, descriptor.environment, descriptor.name),
+      constants.RESERVE_CONCURRENCY_PART,
+    ].join(":");
+  }
+
+  private queueConcurrencyLimitKeyFromDescriptor(descriptor: QueueDescriptor) {
+    return [
+      this.queueKey(descriptor.organization, descriptor.environment, descriptor.name),
+      constants.CONCURRENCY_LIMIT_PART,
+    ].join(":");
+  }
+}

apps/webapp/app/v3/marqs/types.ts

@@ -0,0 +1,111 @@
+import type { RuntimeEnvironmentType } from "@trigger.dev/database";
+import { z } from "zod";
+
+export type QueueRange = { offset: number; count: number };
+
+export type QueueDescriptor = {
+  organization: string;
+  environment: string;
+  name: string;
+  concurrencyKey?: string;
+};
+
+export type MarQSKeyProducerEnv = {
+  id: string;
+  organizationId: string;
+  type: RuntimeEnvironmentType;
+};
+
+export interface MarQSKeyProducer {
+  queueConcurrencyLimitKey(env: MarQSKeyProducerEnv, queue: string): string;
+
+  envConcurrencyLimitKey(envId: string): string;
+  envConcurrencyLimitKey(env: MarQSKeyProducerEnv): string;
+
+  envCurrentConcurrencyKey(envId: string): string;
+  envCurrentConcurrencyKey(env: MarQSKeyProducerEnv): string;
+
+  envReserveConcurrencyKey(envId: string): string;
+
+  queueKey(orgId: string, envId: string, queue: string, concurrencyKey?: string): string;
+  queueKey(env: MarQSKeyProducerEnv, queue: string, concurrencyKey?: string): string;
+
+  queueKeyFromQueue(queue: string): string;
+
+  envQueueKey(env: MarQSKeyProducerEnv): string;
+  envSharedQueueKey(env: MarQSKeyProducerEnv): string;
+  sharedQueueKey(): string;
+  sharedQueueScanPattern(): string;
+  sharedWorkerQueueKey(): string;
+  queueCurrentConcurrencyScanPattern(): string;
+  queueConcurrencyLimitKeyFromQueue(queue: string): string;
+  queueCurrentConcurrencyKeyFromQueue(queue: string): string;
+  queueCurrentConcurrencyKey(
+    env: MarQSKeyProducerEnv,
+    queue: string,
+    concurrencyKey?: string
+  ): string;
+  envConcurrencyLimitKeyFromQueue(queue: string): string;
+  envCurrentConcurrencyKeyFromQueue(queue: string): string;
+  envReserveConcurrencyKeyFromQueue(queue: string): string;
+  envQueueKeyFromQueue(queue: string): string;
+  messageKey(messageId: string): string;
+  nackCounterKey(messageId: string): string;
+  stripKeyPrefix(key: string): string;
+  orgIdFromQueue(queue: string): string;
+  envIdFromQueue(queue: string): string;
+
+  queueReserveConcurrencyKeyFromQueue(queue: string): string;
+  queueDescriptorFromQueue(queue: string): QueueDescriptor;
+}
+
+export type EnvQueues = {
+  envId: string;
+  queues: string[];
+};
+
+const MarQSPriorityLevel = z.enum(["resume", "retry"]);
+
+export type MarQSPriorityLevel = z.infer<typeof MarQSPriorityLevel>;
+
+export interface MarQSFairDequeueStrategy {
+  distributeFairQueuesFromParentQueue(
+    parentQueue: string,
+    consumerId: string
+  ): Promise<Array<EnvQueues>>;
+}
+
+export const MessagePayload = z.object({
+  version: z.literal("1"),
+  data: z.record(z.unknown()),
+  queue: z.string(),
+  messageId: z.string(),
+  timestamp: z.number(),
+  parentQueue: z.string(),
+  concurrencyKey: z.string().optional(),
+  priority: MarQSPriorityLevel.optional(),
+  availableAt: z.number().optional(),
+  enqueueMethod: z.enum(["enqueue", "requeue", "replace"]).default("enqueue"),
+});
+
+export type MessagePayload = z.infer<typeof MessagePayload>;
+
+export interface MessageQueueSubscriber {
+  messageEnqueued(message: MessagePayload): Promise<void>;
+  messageDequeued(message: MessagePayload): Promise<void>;
+  messageAcked(message: MessagePayload): Promise<void>;
+  messageNacked(message: MessagePayload): Promise<void>;
+  messageReplaced(message: MessagePayload): Promise<void>;
+  messageRequeued(message: MessagePayload): Promise<void>;
+}
+
+export interface VisibilityTimeoutStrategy {
+  startHeartbeat(messageId: string, timeoutInMs: number): Promise<void>;
+  heartbeat(messageId: string, timeoutInMs: number): Promise<void>;
+  cancelHeartbeat(messageId: string): Promise<void>;
+}
+
+export type EnqueueMessageReserveConcurrencyOptions = {
+  messageId: string;
+  recursiveQueue: boolean;
+};

apps/webapp/app/v3/marqs/v3VisibilityTimeout.server.ts

@@ -0,0 +1,39 @@
+import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server";
+import { TaskRunHeartbeatFailedService } from "../taskRunHeartbeatFailed.server";
+import type { VisibilityTimeoutStrategy } from "./types";
+
+export class V3GraphileVisibilityTimeout implements VisibilityTimeoutStrategy {
+  async startHeartbeat(messageId: string, timeoutInMs: number): Promise<void> {
+    await TaskRunHeartbeatFailedService.enqueue(messageId, new Date(Date.now() + timeoutInMs));
+  }
+
+  async heartbeat(messageId: string, timeoutInMs: number): Promise<void> {
+    await TaskRunHeartbeatFailedService.enqueue(messageId, new Date(Date.now() + timeoutInMs));
+  }
+
+  async cancelHeartbeat(messageId: string): Promise<void> {
+    await TaskRunHeartbeatFailedService.dequeue(messageId);
+  }
+}
+
+export class V3LegacyRunEngineWorkerVisibilityTimeout implements VisibilityTimeoutStrategy {
+  async startHeartbeat(messageId: string, timeoutInMs: number): Promise<void> {
+    await legacyRunEngineWorker.enqueue({
+      id: `heartbeat:${messageId}`,
+      job: "runHeartbeat",
+      payload: { runId: messageId },
+      availableAt: new Date(Date.now() + timeoutInMs),
+    });
+  }
+
+  async heartbeat(messageId: string, timeoutInMs: number): Promise<void> {
+    await legacyRunEngineWorker.reschedule(
+      `heartbeat:${messageId}`,
+      new Date(Date.now() + timeoutInMs)
+    );
+  }
+
+  async cancelHeartbeat(messageId: string): Promise<void> {
+    await legacyRunEngineWorker.ack(`heartbeat:${messageId}`);
+  }
+}

apps/webapp/app/v3/models/workerDeployment.server.ts

@@ -0,0 +1,299 @@
+import type { Prettify } from "@trigger.dev/core";
+import type {
+  BackgroundWorker,
+  PrismaClientOrTransaction,
+  RunEngineVersion,
+  WorkerDeploymentType,
+} from "@trigger.dev/database";
+import {
+  CURRENT_DEPLOYMENT_LABEL,
+  CURRENT_UNMANAGED_DEPLOYMENT_LABEL,
+} from "@trigger.dev/core/v3/isomorphic";
+import type { Prisma } from "~/db.server";
+import { prisma } from "~/db.server";
+import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+
+export type CurrentWorkerDeployment = Prettify<
+  NonNullable<Awaited<ReturnType<typeof findCurrentWorkerDeployment>>>
+>;
+
+export type BackgroundWorkerTaskSlim = Prisma.BackgroundWorkerTaskGetPayload<{
+  select: {
+    id: true;
+    friendlyId: true;
+    slug: true;
+    filePath: true;
+    exportName: true;
+    triggerSource: true;
+    machineConfig: true;
+    maxDurationInSeconds: true;
+  };
+}>;
+
+type WorkerDeploymentWithWorkerTasks = Prisma.WorkerDeploymentGetPayload<{
+  select: {
+    id: true;
+    imageReference: true;
+    version: true;
+    worker: {
+      select: {
+        id: true;
+        friendlyId: true;
+        version: true;
+        sdkVersion: true;
+        cliVersion: true;
+        supportsLazyAttempts: true;
+        engine: true;
+        tasks: {
+          select: {
+            id: true;
+            friendlyId: true;
+            slug: true;
+            filePath: true;
+            exportName: true;
+            triggerSource: true;
+            machineConfig: true;
+            maxDurationInSeconds: true;
+            queueConfig: true;
+            queueId: true;
+            payloadSchema: true;
+            fileId: true;
+          };
+        };
+      };
+    };
+  };
+}>;
+
+/**
+ * Finds the current worker deployment for a given environment.
+ *
+ * @param environmentId - The ID of the environment to find the current worker deployment for.
+ * @param label - The label of the current worker deployment to find.
+ * @param type - The type of worker deployment to find. If the current deployment is NOT of this type,
+ *   we will return the latest deployment of the given type.
+ */
+export async function findCurrentWorkerDeployment({
+  environmentId,
+  label = CURRENT_DEPLOYMENT_LABEL,
+  type,
+  prismaClient,
+}: {
+  environmentId: string;
+  label?: string;
+  type?: WorkerDeploymentType;
+  prismaClient?: PrismaClientOrTransaction;
+}): Promise<WorkerDeploymentWithWorkerTasks | undefined> {
+  const $prisma = prismaClient ?? prisma;
+
+  const promotion = await $prisma.workerDeploymentPromotion.findFirst({
+    where: {
+      environmentId,
+      label,
+    },
+    select: {
+      deployment: {
+        select: {
+          id: true,
+          imageReference: true,
+          version: true,
+          type: true,
+          worker: {
+            select: {
+              id: true,
+              friendlyId: true,
+              version: true,
+              sdkVersion: true,
+              cliVersion: true,
+              supportsLazyAttempts: true,
+              tasks: true,
+              engine: true,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!promotion) {
+    return undefined;
+  }
+
+  if (!type) {
+    return promotion.deployment;
+  }
+
+  if (promotion.deployment.type === type) {
+    return promotion.deployment;
+  }
+
+  // We need to get the latest deployment of the given type
+  const latestDeployment = await prisma.workerDeployment.findFirst({
+    where: {
+      environmentId,
+      type,
+    },
+    orderBy: {
+      id: "desc",
+    },
+    select: {
+      id: true,
+      imageReference: true,
+      version: true,
+      type: true,
+      worker: {
+        select: {
+          id: true,
+          friendlyId: true,
+          version: true,
+          sdkVersion: true,
+          cliVersion: true,
+          supportsLazyAttempts: true,
+          tasks: true,
+          engine: true,
+        },
+      },
+    },
+  });
+
+  if (!latestDeployment) {
+    return undefined;
+  }
+
+  return latestDeployment;
+}
+
+export async function getCurrentWorkerDeploymentEngineVersion(
+  environmentId: string,
+  label = CURRENT_DEPLOYMENT_LABEL
+): Promise<RunEngineVersion | undefined> {
+  const promotion = await prisma.workerDeploymentPromotion.findFirst({
+    where: {
+      environmentId,
+      label,
+    },
+    select: {
+      deployment: {
+        select: {
+          type: true,
+        },
+      },
+    },
+  });
+
+  if (typeof promotion?.deployment.type === "string") {
+    return promotion.deployment.type === "V1" ? "V1" : "V2";
+  }
+
+  return undefined;
+}
+
+export async function findCurrentUnmanagedWorkerDeployment(
+  environmentId: string
+): Promise<WorkerDeploymentWithWorkerTasks | undefined> {
+  return await findCurrentWorkerDeployment({
+    environmentId,
+    label: CURRENT_UNMANAGED_DEPLOYMENT_LABEL,
+    type: "UNMANAGED",
+  });
+}
+
+export async function findCurrentWorkerFromEnvironment(
+  environment: Pick<AuthenticatedEnvironment, "id" | "type">,
+  prismaClient: PrismaClientOrTransaction = prisma,
+  label = CURRENT_DEPLOYMENT_LABEL
+): Promise<Pick<
+  BackgroundWorker,
+  "id" | "friendlyId" | "version" | "sdkVersion" | "cliVersion" | "supportsLazyAttempts" | "engine"
+> | null> {
+  if (environment.type === "DEVELOPMENT") {
+    const latestDevWorker = await prismaClient.backgroundWorker.findFirst({
+      where: {
+        runtimeEnvironmentId: environment.id,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+    return latestDevWorker;
+  } else {
+    const deployment = await findCurrentWorkerDeployment({
+      environmentId: environment.id,
+      label,
+      prismaClient,
+    });
+    return deployment?.worker ?? null;
+  }
+}
+
+export async function findCurrentUnmanagedWorkerFromEnvironment(
+  environment: Pick<AuthenticatedEnvironment, "id" | "type">,
+  prismaClient: PrismaClientOrTransaction = prisma
+): Promise<Pick<
+  BackgroundWorker,
+  "id" | "friendlyId" | "version" | "sdkVersion" | "cliVersion" | "supportsLazyAttempts"
+> | null> {
+  if (environment.type === "DEVELOPMENT") {
+    return null;
+  }
+
+  return await findCurrentWorkerFromEnvironment(
+    environment,
+    prismaClient,
+    CURRENT_UNMANAGED_DEPLOYMENT_LABEL
+  );
+}
+
+export async function getWorkerDeploymentFromWorker(
+  workerId: string
+): Promise<WorkerDeploymentWithWorkerTasks | undefined> {
+  const worker = await prisma.backgroundWorker.findFirst({
+    where: {
+      id: workerId,
+    },
+    include: {
+      deployment: true,
+      tasks: true,
+    },
+  });
+
+  if (!worker?.deployment) {
+    return;
+  }
+
+  const { deployment, ...workerWithoutDeployment } = worker;
+
+  return {
+    ...deployment,
+    worker: workerWithoutDeployment,
+  };
+}
+
+export async function getWorkerDeploymentFromWorkerTask(
+  workerTaskId: string
+): Promise<WorkerDeploymentWithWorkerTasks | undefined> {
+  const workerTask = await prisma.backgroundWorkerTask.findFirst({
+    where: {
+      id: workerTaskId,
+    },
+    include: {
+      worker: {
+        include: {
+          deployment: true,
+          tasks: true,
+        },
+      },
+    },
+  });
+
+  if (!workerTask?.worker.deployment) {
+    return;
+  }
+
+  const { deployment, ...workerWithoutDeployment } = workerTask.worker;
+
+  return {
+    ...deployment,
+    worker: workerWithoutDeployment,
+  };
+}

apps/webapp/app/v3/mollifier/applyMetadataMutation.server.ts

@@ -0,0 +1,206 @@
+import { applyMetadataOperations } from "@trigger.dev/core/v3";
+import type { FlushedRunMetadata } from "@trigger.dev/core/v3/schemas";
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { logger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+
+// On `applied` we surface the parent/root friendlyIds captured during
+// the snapshot read. Callers that fan parent/root metadata operations
+// out to their respective runs can use these without a second
+// `findRunByIdWithMollifierFallback` round trip — and, more importantly,
+// without racing the drainer's terminal-failure path (which atomically
+// DELetes the entry hash). Without these on the outcome the second
+// read can come back null mid-route, silently dropping the caller's
+// parentOperations / rootOperations after the primary mutation already
+// landed on the snapshot.
+//
+// FriendlyIds (not internal cuids) because the consuming
+// `routeOperationsToRun` helper gates on the `run_…` prefix to decide
+// whether to attempt the buffer fallback; cuids would skip that path.
+// The snapshot's `parentTaskRunId` / `rootTaskRunId` are engine-side
+// cuids, so we convert via `RunId.toFriendlyId` here — identical to
+// what `readFallback.server.ts` does when assembling its SyntheticRun.
+export type ApplyMetadataMutationOutcome =
+  | {
+      kind: "applied";
+      newMetadata: Record<string, unknown>;
+      parentTaskRunFriendlyId: string | undefined;
+      rootTaskRunFriendlyId: string | undefined;
+    }
+  | { kind: "not_found" }
+  | { kind: "busy" }
+  | { kind: "version_exhausted" }
+  // Mirrors the PG-side `MetadataTooLargeError` (status 413). Carries
+  // the limit + observed size so the route can produce a useful body.
+  | { kind: "metadata_too_large"; maximumSize: number; observedSize: number };
+
+// Apply a metadata PUT (body.metadata replace AND/OR body.operations
+// deltas) to a buffered run's snapshot. Mirrors the PG-side
+// `UpdateMetadataService.#updateRunMetadataWithOperations` retry loop:
+// read snapshot → apply operations in JS → CAS-write back with the
+// observed `metadataVersion`. Retries on conflict; bounded by
+// `maxRetries`. The Lua CAS is the atomicity primitive — concurrent
+// callers never lose an increment / append / set.
+export async function applyMetadataMutationToBufferedRun(input: {
+  runId: string;
+  // Env+org scoping closes a cross-environment write gap on the buffer
+  // path: the route's PG path is already env-scoped via Prisma filters,
+  // and this helper now enforces the same isolation before any buffer
+  // write so a caller authed in env A can't mutate a buffered run that
+  // belongs to env B.
+  environmentId: string;
+  organizationId: string;
+  // Byte-size cap on the resulting metadata payload, mirroring the
+  // PG-side `UpdateMetadataService.maximumSize` (sourced from
+  // `env.TASK_RUN_METADATA_MAXIMUM_SIZE`). Required so the buffer path
+  // doesn't silently allow writes the PG path would have rejected.
+  maximumSize: number;
+  body: Pick<FlushedRunMetadata, "metadata" | "operations">;
+  buffer?: MollifierBuffer | null;
+  maxRetries?: number;
+  // Jittered conflict-backoff envelope: random in [0, base + attempt * step) ms.
+  backoffBaseMs?: number;
+  backoffStepMs?: number;
+}): Promise<ApplyMetadataMutationOutcome> {
+  const buffer = input.buffer ?? getMollifierBuffer();
+  if (!buffer) return { kind: "not_found" };
+
+  // Default retry budget tuned for buffered-window concurrency. The
+  // PG-side `UpdateMetadataService` uses 3, which is fine when the only
+  // writer is the executing task itself. For a buffered run the writers
+  // are external API callers, and N parallel writers exhaust 3 retries
+  // quickly under contention. Bumping to 12 covers ~50-way concurrency
+  // with sub-percent failure probability; the cost is bounded (each
+  // retry is one Redis Lua call ~1ms).
+  const maxRetries = input.maxRetries ?? 12;
+  const backoffBaseMs = input.backoffBaseMs ?? 5;
+  const backoffStepMs = input.backoffStepMs ?? 5;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    const entry = await buffer.getEntry(input.runId);
+    if (!entry) return { kind: "not_found" };
+    // Env+org check: an entry from a different env is treated as a
+    // miss (not 403) so existence in other envs doesn't leak.
+    if (entry.envId !== input.environmentId || entry.orgId !== input.organizationId) {
+      return { kind: "not_found" };
+    }
+    if (entry.status !== "QUEUED" || entry.materialised) {
+      return { kind: "busy" };
+    }
+
+    const snapshot = JSON.parse(entry.payload) as Record<string, unknown>;
+    const currentMetadataType =
+      typeof snapshot.metadataType === "string" ? snapshot.metadataType : "application/json";
+
+    // Capture parent/root ids during this read so the caller can fan
+    // parent/root operations out without a second buffer.getEntry. If
+    // the drainer's terminal-failure path runs between our CAS-write
+    // below and the route's follow-up, the entry hash would be DELd
+    // and a second read would return null — silently dropping the
+    // caller's `body.parentOperations` / `body.rootOperations`. The ids
+    // themselves are immutable for a run, so capturing them on any
+    // loop iteration is fine.
+    const snapshotParentTaskRunInternalId =
+      typeof snapshot.parentTaskRunId === "string" ? snapshot.parentTaskRunId : undefined;
+    const snapshotParentTaskRunFriendlyId = snapshotParentTaskRunInternalId
+      ? RunId.toFriendlyId(snapshotParentTaskRunInternalId)
+      : undefined;
+    const snapshotRootTaskRunInternalId =
+      typeof snapshot.rootTaskRunId === "string" ? snapshot.rootTaskRunId : undefined;
+    const snapshotRootTaskRunFriendlyId = snapshotRootTaskRunInternalId
+      ? RunId.toFriendlyId(snapshotRootTaskRunInternalId)
+      : undefined;
+
+    // Match PG semantics: `body.operations` and `body.metadata` are
+    // mutually exclusive on a single request. The PG service
+    // (`UpdateMetadataService.#updateRunMetadata`) branches on
+    // `Array.isArray(body.operations)` — if operations are present it
+    // applies them on top of the EXISTING metadata and ignores
+    // `body.metadata` entirely; otherwise `body.metadata` is the new
+    // full value. Doing both here would make a request like
+    // `{ metadata: {b:2}, operations: [set c=3] }` produce
+    // `{b:2,c:3}` on the buffer vs `{a:1,c:3}` on PG, which silently
+    // changes semantics across the buffered/materialised boundary.
+    const parseSnapshotMetadata = (): Record<string, unknown> => {
+      if (typeof snapshot.metadata !== "string") return {};
+      try {
+        return JSON.parse(snapshot.metadata) as Record<string, unknown>;
+      } catch {
+        return {};
+      }
+    };
+
+    let metadataObject: Record<string, unknown>;
+    // Use `Array.isArray` (the PG service's predicate) instead of a
+    // truthy length check. For `{ metadata, operations: [] }` PG sees
+    // Array.isArray([])=true and no-ops on existing metadata; a
+    // `.length` check would treat the empty array as falsy and fall
+    // through to the `body.metadata` branch, replacing metadata —
+    // exactly the cross-boundary drift the comment above warns
+    // against.
+    if (Array.isArray(input.body.operations)) {
+      // Operations take precedence: apply on top of existing snapshot
+      // metadata; ignore `body.metadata` to match PG behaviour.
+      metadataObject = applyMetadataOperations(
+        parseSnapshotMetadata(),
+        input.body.operations
+      ).newMetadata;
+    } else if (input.body.metadata !== undefined) {
+      // No operations — full replace.
+      metadataObject = input.body.metadata as Record<string, unknown>;
+    } else {
+      // Neither — write back existing snapshot metadata (no-op shape).
+      metadataObject = parseSnapshotMetadata();
+    }
+
+    const newMetadataStr = JSON.stringify(metadataObject);
+
+    // Size cap — match PG (`handleMetadataPacket` throws
+    // `MetadataTooLargeError` (413) when the JSON-encoded packet
+    // exceeds the configured cap). Reject in-loop, before CAS, so a
+    // single oversize write doesn't churn the retry budget.
+    const observedSize = Buffer.byteLength(newMetadataStr, "utf8");
+    if (observedSize > input.maximumSize) {
+      return {
+        kind: "metadata_too_large",
+        maximumSize: input.maximumSize,
+        observedSize,
+      };
+    }
+
+    const cas = await buffer.casSetMetadata({
+      runId: input.runId,
+      expectedVersion: entry.metadataVersion,
+      newMetadata: newMetadataStr,
+      newMetadataType: currentMetadataType,
+    });
+
+    if (cas.kind === "applied") {
+      return {
+        kind: "applied",
+        newMetadata: metadataObject,
+        parentTaskRunFriendlyId: snapshotParentTaskRunFriendlyId,
+        rootTaskRunFriendlyId: snapshotRootTaskRunFriendlyId,
+      };
+    }
+    if (cas.kind === "not_found") return { kind: "not_found" };
+    if (cas.kind === "busy") return { kind: "busy" };
+    // version_conflict — another caller wrote between our read + CAS.
+    // Small jittered backoff so a thundering herd of N retriers doesn't
+    // all re-read + re-CAS at exactly the same moment.
+    logger.debug("applyMetadataMutationToBufferedRun: version_conflict, retrying", {
+      runId: input.runId,
+      attempt,
+      observedVersion: entry.metadataVersion,
+      currentVersion: cas.currentVersion,
+    });
+    const backoffMs = Math.floor(Math.random() * (backoffBaseMs + attempt * backoffStepMs));
+    await new Promise((resolve) => setTimeout(resolve, backoffMs));
+  }
+
+  logger.warn("applyMetadataMutationToBufferedRun: retries exhausted", {
+    runId: input.runId,
+    maxRetries,
+  });
+  return { kind: "version_exhausted" };
+}

apps/webapp/app/v3/mollifier/bufferedTriggerPayload.server.ts

@@ -0,0 +1,107 @@
+import type { TriggerTaskRequestBody } from "@trigger.dev/core/v3";
+import type { TriggerTaskServiceOptions } from "~/v3/services/triggerTask.server";
+
+// Canonical payload shape written to the mollifier buffer when the gate
+// decides to mollify a trigger. At this stage the call site ALSO calls
+// engine.trigger directly (dual-write), so this is currently an
+// audit/preview record. A later change makes the buffer the primary write
+// path: the drainer's handler reads this payload and replays it through
+// engine.trigger to create the run in Postgres, and read-fallback
+// endpoints synthesise a Run view from it while it is still QUEUED.
+//
+// CONTRACT: this shape must contain everything the drainer-replay needs to
+// reconstruct an equivalent engine.trigger call. Today it is emitted to
+// logs; later it is serialised into Redis and rebuilt on the drain side.
+// Keep it serialisable — no functions, no class instances.
+export type BufferedTriggerPayload = {
+  runFriendlyId: string;
+
+  // Routing identifiers — let the drainer re-fetch full AuthenticatedEnvironment
+  // at replay time rather than embedding it in the payload.
+  envId: string;
+  envType: string;
+  envSlug: string;
+  orgId: string;
+  orgSlug: string;
+  projectId: string;
+  projectRef: string;
+
+  // Task identifier — looked up against the locked BackgroundWorkerTask
+  // at replay time to recover task-defaults.
+  taskId: string;
+
+  // Customer-supplied trigger body (payload, options, context).
+  body: TriggerTaskRequestBody;
+
+  // Resolved values from upstream concerns. The drainer should NOT re-resolve
+  // these — that would create a second idempotency-key check, etc.
+  idempotencyKey: string | null;
+  idempotencyKeyExpiresAt: string | null;
+  tags: string[];
+
+  // Parent/root linkage for nested triggers.
+  parentRunFriendlyId: string | null;
+
+  // Trace context — propagates the original triggering span across the
+  // buffer→drain boundary so the run's lifecycle stays under one trace.
+  traceContext: Record<string, unknown>;
+
+  // Annotations + service options that influence routing/replay.
+  triggerSource: string;
+  triggerAction: string;
+  serviceOptions: TriggerTaskServiceOptions;
+
+  // Wall-clock instants relevant to the run.
+  createdAt: string;
+};
+
+// Assemble the canonical payload from the inputs available at the point
+// `evaluateGate` returns "mollify" in `RunEngineTriggerTaskService.call`.
+// All fields must be derivable from data already in scope at that call site;
+// nothing should require an extra DB lookup.
+export function buildBufferedTriggerPayload(input: {
+  runFriendlyId: string;
+  taskId: string;
+  envId: string;
+  envType: string;
+  envSlug: string;
+  orgId: string;
+  orgSlug: string;
+  projectId: string;
+  projectRef: string;
+  body: TriggerTaskRequestBody;
+  idempotencyKey: string | null;
+  idempotencyKeyExpiresAt: Date | null;
+  tags: string[];
+  parentRunFriendlyId: string | null;
+  traceContext: Record<string, unknown>;
+  triggerSource: string;
+  triggerAction: string;
+  serviceOptions: TriggerTaskServiceOptions;
+  createdAt: Date;
+}): BufferedTriggerPayload {
+  return {
+    runFriendlyId: input.runFriendlyId,
+    envId: input.envId,
+    envType: input.envType,
+    envSlug: input.envSlug,
+    orgId: input.orgId,
+    orgSlug: input.orgSlug,
+    projectId: input.projectId,
+    projectRef: input.projectRef,
+    taskId: input.taskId,
+    body: input.body,
+    idempotencyKey: input.idempotencyKey,
+    idempotencyKeyExpiresAt:
+      input.idempotencyKey && input.idempotencyKeyExpiresAt
+        ? input.idempotencyKeyExpiresAt.toISOString()
+        : null,
+    tags: input.tags,
+    parentRunFriendlyId: input.parentRunFriendlyId,
+    traceContext: input.traceContext,
+    triggerSource: input.triggerSource,
+    triggerAction: input.triggerAction,
+    serviceOptions: input.serviceOptions,
+    createdAt: input.createdAt.toISOString(),
+  };
+}

apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts

@@ -0,0 +1,218 @@
+import { randomUUID } from "node:crypto";
+import type {
+  IdempotencyClaimResult,
+  IdempotencyLookupInput,
+  MollifierBuffer,
+} from "@trigger.dev/redis-worker";
+import { logger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+
+// Tunables. The TTL on the claim key is bounded by typical trigger-pipeline
+// dwell; long enough that a slow PG insert doesn't expire mid-flight,
+// short enough that a crashed claimant unblocks waiters quickly.
+export const DEFAULT_CLAIM_TTL_SECONDS = 30;
+// safetyNetMs caps how long a waiter blocks before returning timed_out.
+// Matches the mutateWithFallback safety net so SDK retry policies don't
+// have to special-case this path.
+export const DEFAULT_CLAIM_WAIT_MS = 5_000;
+export const DEFAULT_CLAIM_POLL_MS = 25;
+
+export type ClaimOrAwaitOutcome =
+  // We own the claim. `token` MUST be passed to publishClaim/releaseClaim
+  // so the buffer can compare-and-act against our ownership marker — a
+  // late release from a previous claimant whose TTL expired cannot
+  // erase our slot.
+  | { kind: "claimed"; token: string }
+  | { kind: "resolved"; runId: string } // someone else's runId; caller returns isCached:true
+  | { kind: "timed_out" };
+
+export type ClaimOrAwaitInput = IdempotencyLookupInput & {
+  ttlSeconds?: number;
+  safetyNetMs?: number;
+  pollStepMs?: number;
+  abortSignal?: AbortSignal;
+  // Test injection.
+  buffer?: MollifierBuffer | null;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+  // Test override for the ownership-token generator. Defaults to
+  // `crypto.randomUUID()`. Tests pass a deterministic value so they
+  // can assert publish/release pass-through.
+  generateToken?: () => string;
+};
+
+// Pre-gate Redis claim. All same-key triggers serialise through here
+// before the trigger pipeline runs. Returning `resolved` short-circuits
+// the trigger entirely — the caller responds with the cached runId.
+// Returning `claimed` means we own the claim and MUST publish the
+// winning runId on success (`publishClaim`) or release the claim on
+// failure (`releaseClaim`).
+//
+// Failure modes:
+// - Redis down at claim time: returns `claimed` (fail open, no
+//   coordination). Customer is no worse than today's race; the
+//   PG unique constraint is the eventual arbiter.
+// - Claimant crashes mid-pipeline: claim TTL expires, waiters
+//   eventually time out, SDK retries.
+// - PG/buffer publish failure: waiters time out and SDK retries; next
+//   attempt sees the eventual PG/buffer state via existing
+//   IdempotencyKeyConcern PG-first lookup.
+export async function claimOrAwait(input: ClaimOrAwaitInput): Promise<ClaimOrAwaitOutcome> {
+  const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
+  if (!buffer) {
+    // Mollifier disabled / buffer construction failed. Fall open —
+    // caller proceeds with the trigger pipeline (PG unique constraint
+    // backstop). The token is never read in this case (publish/release
+    // are buffer-null no-ops downstream), so we skip the default
+    // `randomUUID()` to keep the mollifier-OFF hot path allocation-free
+    // for idempotency-keyed triggers — `triggerTask` is the
+    // highest-throughput code path in the system. A test-injected
+    // generator is still honoured for deterministic assertions.
+    return { kind: "claimed", token: input.generateToken ? input.generateToken() : "" };
+  }
+  const generateToken = input.generateToken ?? randomUUID;
+  // Generate the ownership token up front so the retry loop reuses it
+  // — we're the same logical claimant across attempts; only the slot
+  // owner changes between releases.
+  const token = generateToken();
+  const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS;
+  const safetyNetMs = input.safetyNetMs ?? DEFAULT_CLAIM_WAIT_MS;
+  const pollStepMs = input.pollStepMs ?? DEFAULT_CLAIM_POLL_MS;
+  const now = input.now ?? Date.now;
+  const sleep = input.sleep ?? defaultSleep;
+
+  const lookupInput: IdempotencyLookupInput = {
+    envId: input.envId,
+    taskIdentifier: input.taskIdentifier,
+    idempotencyKey: input.idempotencyKey,
+  };
+
+  // Initial claim attempt. Most production-path calls resolve here on
+  // the first call (either we win, or the key is already resolved from
+  // a prior burst).
+  let result: IdempotencyClaimResult;
+  try {
+    result = await buffer.claimIdempotency({ ...lookupInput, token, ttlSeconds });
+  } catch (err) {
+    logger.warn("idempotency claim failed (fail-open)", {
+      envId: input.envId,
+      taskIdentifier: input.taskIdentifier,
+      err: err instanceof Error ? err.message : String(err),
+    });
+    return { kind: "claimed", token };
+  }
+
+  if (result.kind === "claimed") return { kind: "claimed", token };
+  if (result.kind === "resolved") return result;
+
+  // result.kind === "pending" — wait/poll loop. May see the value flip
+  // to "resolved" (winner published), the key vanish (winner released
+  // on error → retry claim), or stay "pending" until the safety net.
+  const deadline = now() + safetyNetMs;
+  while (now() < deadline) {
+    if (input.abortSignal?.aborted) return { kind: "timed_out" };
+    await sleep(pollStepMs);
+
+    let current: IdempotencyClaimResult | null;
+    try {
+      current = await buffer.readClaim(lookupInput);
+    } catch (err) {
+      // Transient read failure — keep polling until deadline.
+      logger.warn("idempotency claim read failed mid-poll", {
+        err: err instanceof Error ? err.message : String(err),
+      });
+      continue;
+    }
+
+    if (current === null) {
+      // Claimant released on error. Re-attempt the claim — one of the
+      // waiters will win, the rest see "pending" again. Reuse our token:
+      // we're still the same logical claimant, just contending for a
+      // freshly empty slot.
+      try {
+        const retry = await buffer.claimIdempotency({ ...lookupInput, token, ttlSeconds });
+        if (retry.kind === "claimed") return { kind: "claimed", token };
+        if (retry.kind === "resolved") return retry;
+        // "pending" again → keep polling.
+      } catch (err) {
+        logger.warn("idempotency claim retry failed", {
+          err: err instanceof Error ? err.message : String(err),
+        });
+        return { kind: "claimed", token };
+      }
+      continue;
+    }
+    if (current.kind === "resolved") return current;
+    // current.kind === "pending" → keep polling.
+  }
+  return { kind: "timed_out" };
+}
+
+// Publish the winning runId so waiters resolve. Best-effort: failure
+// here means waiters will time out and the SDK will retry, which will
+// then find the row via the existing IdempotencyKeyConcern PG-first
+// check.
+export async function publishClaim(input: {
+  envId: string;
+  taskIdentifier: string;
+  idempotencyKey: string;
+  // Ownership token from the `claimed` outcome. Buffer compare-and-sets
+  // on this so a publish from a stale claimant (TTL expired, another
+  // claimant moved in) is a no-op rather than overwriting their claim.
+  token: string;
+  runId: string;
+  ttlSeconds?: number;
+  buffer?: MollifierBuffer | null;
+}): Promise<void> {
+  const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
+  if (!buffer) return;
+  const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS;
+  try {
+    await buffer.publishClaim({
+      envId: input.envId,
+      taskIdentifier: input.taskIdentifier,
+      idempotencyKey: input.idempotencyKey,
+      token: input.token,
+      runId: input.runId,
+      ttlSeconds,
+    });
+  } catch (err) {
+    logger.warn("idempotency claim publish failed", {
+      envId: input.envId,
+      taskIdentifier: input.taskIdentifier,
+      err: err instanceof Error ? err.message : String(err),
+    });
+  }
+}
+
+// Release on pipeline failure. Best-effort. If the DEL fails, the claim
+// TTL is the safety net — waiters time out, SDK retries.
+export async function releaseClaim(input: {
+  envId: string;
+  taskIdentifier: string;
+  idempotencyKey: string;
+  // Ownership token from the `claimed` outcome. Buffer compare-and-
+  // deletes on this so a release from a stale claimant whose TTL
+  // expired can't wipe a new owner's claim.
+  token: string;
+  buffer?: MollifierBuffer | null;
+}): Promise<void> {
+  const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer;
+  if (!buffer) return;
+  try {
+    await buffer.releaseClaim({
+      envId: input.envId,
+      taskIdentifier: input.taskIdentifier,
+      idempotencyKey: input.idempotencyKey,
+      token: input.token,
+    });
+  } catch (err) {
+    logger.warn("idempotency claim release failed", {
+      err: err instanceof Error ? err.message : String(err),
+    });
+  }
+}
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

apps/webapp/app/v3/mollifier/mollifierBuffer.server.ts

@@ -0,0 +1,35 @@
+import { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+
+// DI seam type for consumers (e.g. triggerTask.server.ts) that need a
+// nullable buffer accessor at construction time.
+export type MollifierGetBuffer = () => MollifierBuffer | null;
+
+function initializeMollifierBuffer(): MollifierBuffer {
+  logger.debug("Initializing mollifier buffer", {
+    host: env.TRIGGER_MOLLIFIER_REDIS_HOST,
+  });
+
+  return new MollifierBuffer({
+    redisOptions: {
+      keyPrefix: "",
+      host: env.TRIGGER_MOLLIFIER_REDIS_HOST,
+      port: env.TRIGGER_MOLLIFIER_REDIS_PORT,
+      username: env.TRIGGER_MOLLIFIER_REDIS_USERNAME,
+      password: env.TRIGGER_MOLLIFIER_REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.TRIGGER_MOLLIFIER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+    ackGraceTtlSeconds: env.TRIGGER_MOLLIFIER_ACK_GRACE_TTL_SECONDS,
+    maxRetriesPerRequest: env.TRIGGER_MOLLIFIER_REDIS_MAX_RETRIES_PER_REQUEST,
+    reconnectStepMs: env.TRIGGER_MOLLIFIER_REDIS_RECONNECT_STEP_MS,
+    reconnectMaxMs: env.TRIGGER_MOLLIFIER_REDIS_RECONNECT_MAX_MS,
+  });
+}
+
+export function getMollifierBuffer(): MollifierBuffer | null {
+  if (env.TRIGGER_MOLLIFIER_ENABLED !== "1") return null;
+  return singleton("mollifierBuffer", initializeMollifierBuffer);
+}

apps/webapp/app/v3/mollifier/mollifierDrainer.server.ts

@@ -0,0 +1,105 @@
+import { MollifierDrainer } from "@trigger.dev/redis-worker";
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { engine as runEngine } from "~/v3/runEngine.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+import {
+  createDrainerHandler,
+  createDrainerTerminalFailureHandler,
+  isRetryablePgError,
+} from "./mollifierDrainerHandler.server";
+import type { MollifierSnapshot } from "./mollifierSnapshot.server";
+
+// Distinct error class for the deterministic "fail loud at boot" throws
+// below. The bootstrap in `mollifierDrainerWorker.server.ts` catches
+// transient/init errors and logs them so an unrelated Redis blip doesn't
+// crash the webapp, but it RETHROWS this class — a misconfigured
+// shutdown timeout or missing buffer is a deploy-time mistake that
+// should fail health checks and roll back, not silently disable a
+// half-rolled-out feature.
+//
+// The `name` getter is set explicitly so cross-realm `instanceof` checks
+// (e.g. when Remix dev hot-reloads the module and the consumer keeps a
+// reference to the old class) can fall back to `error.name === ...` and
+// still recognise the marker.
+export class MollifierConfigurationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "MollifierConfigurationError";
+  }
+}
+
+function initializeMollifierDrainer(): MollifierDrainer<MollifierSnapshot> {
+  const buffer = getMollifierBuffer();
+  if (!buffer) {
+    // Unreachable in normal config: getMollifierDrainer() gates on the
+    // same env flag as getMollifierBuffer(). If we hit this, fail loud
+    // — the operator has set TRIGGER_MOLLIFIER_ENABLED=1 on a worker pod but
+    // the buffer can't initialise (e.g. TRIGGER_MOLLIFIER_REDIS_HOST resolves
+    // to nothing). Crashing surfaces the misconfig immediately rather
+    // than silently leaving entries un-drained.
+    throw new MollifierConfigurationError(
+      "MollifierDrainer initialised without a buffer — env vars inconsistent"
+    );
+  }
+
+  // Validate BEFORE start() so a misconfigured shutdown timeout fails
+  // loud at module-load time and the singleton is never cached. If start()
+  // ran first and the throw propagated out, the loop would already be
+  // polling with no SIGTERM handler registered by the caller — exactly
+  // the failure mode the validation is supposed to prevent.
+  //
+  // The SIGTERM handler in mollifierDrainerWorker.server.ts is sync fire-and-forget:
+  // `drainer.stop({ timeoutMs })` returns a promise that keeps the event
+  // loop alive, but in cluster mode the primary runs its own
+  // GRACEFUL_SHUTDOWN_TIMEOUT and will call `process.exit(0)`
+  // independently. If the drainer's deadline exceeds the primary's, the
+  // drainer is cut off mid-wait — "log a warning on timeout" turns into
+  // "hard exit with no log". 1s margin gives the primary room to finish
+  // its own teardown after the drainer settles.
+  const shutdownMarginMs = env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_MARGIN_MS;
+  if (
+    env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS >=
+    env.GRACEFUL_SHUTDOWN_TIMEOUT - shutdownMarginMs
+  ) {
+    throw new MollifierConfigurationError(
+      `TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS (${env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS}) must be at least ${shutdownMarginMs}ms below GRACEFUL_SHUTDOWN_TIMEOUT (${env.GRACEFUL_SHUTDOWN_TIMEOUT}); otherwise the primary's hard exit shadows the drainer's deadline.`
+    );
+  }
+
+  logger.debug("Initializing mollifier drainer", {
+    concurrency: env.TRIGGER_MOLLIFIER_DRAIN_CONCURRENCY,
+    maxAttempts: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ATTEMPTS,
+    drainBatchSize: env.TRIGGER_MOLLIFIER_DRAIN_BATCH_SIZE,
+  });
+
+  const drainer = new MollifierDrainer<MollifierSnapshot>({
+    buffer,
+    handler: createDrainerHandler({ engine: runEngine, prisma }),
+    onTerminalFailure: createDrainerTerminalFailureHandler({ engine: runEngine, prisma }),
+    concurrency: env.TRIGGER_MOLLIFIER_DRAIN_CONCURRENCY,
+    maxAttempts: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ATTEMPTS,
+    maxOrgsPerTick: env.TRIGGER_MOLLIFIER_DRAIN_MAX_ORGS_PER_TICK,
+    drainBatchSize: env.TRIGGER_MOLLIFIER_DRAIN_BATCH_SIZE,
+    pollIntervalMs: env.TRIGGER_MOLLIFIER_DRAIN_POLL_INTERVAL_MS,
+    maxBackoffMs: env.TRIGGER_MOLLIFIER_DRAIN_MAX_BACKOFF_MS,
+    backoffFloorMs: env.TRIGGER_MOLLIFIER_DRAIN_BACKOFF_FLOOR_MS,
+    isRetryable: isRetryablePgError,
+  });
+
+  return drainer;
+}
+
+// Returns a configured-but-stopped drainer. Callers MUST register their
+// SIGTERM / SIGINT shutdown handlers before invoking `drainer.start()` —
+// see `apps/webapp/app/v3/mollifierDrainerWorker.server.ts`. Starting
+// inside the singleton factory would put the polling loop ahead of
+// handler registration, leaving a narrow window where a SIGTERM landing
+// between `start()` and `process.once("SIGTERM", ...)` would skip the
+// graceful stop. The split is intentional.
+export function getMollifierDrainer(): MollifierDrainer<MollifierSnapshot> | null {
+  if (env.TRIGGER_MOLLIFIER_ENABLED !== "1") return null;
+  return singleton("mollifierDrainer", initializeMollifierDrainer);
+}

apps/webapp/app/v3/mollifier/mollifierDrainerHandler.server.ts

@@ -0,0 +1,401 @@
+import { context, trace, TraceFlags } from "@opentelemetry/api";
+import type { RunEngine } from "@internal/run-engine";
+import type { PrismaClientOrTransaction } from "@trigger.dev/database";
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import type {
+  MollifierDrainerHandler,
+  MollifierDrainerTerminalFailureHandler,
+} from "@trigger.dev/redis-worker";
+import { logger } from "~/services/logger.server";
+import { recordRunDebugLog } from "~/v3/eventRepository/index.server";
+import { PerformTaskRunAlertsService } from "~/v3/services/alerts/performTaskRunAlerts.server";
+import { startSpan } from "~/v3/tracing.server";
+import type { MollifierSnapshot } from "./mollifierSnapshot.server";
+
+const tracer = trace.getTracer("mollifier-drainer");
+
+export function isRetryablePgError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  const msg = err.message ?? "";
+  // Prisma surfaces P1001 ("Can't reach database server") via two
+  // different error classes — `PrismaClientKnownRequestError` exposes
+  // it as `err.code`, `PrismaClientInitializationError` exposes it as
+  // `err.errorCode`. Check both so reconnection-time errors retry
+  // regardless of which class fires.
+  const code = (err as { code?: string }).code;
+  const errorCode = (err as { errorCode?: string }).errorCode;
+  if (code === "P2024") return true;
+  if (code === "P1001" || errorCode === "P1001") return true;
+  if (msg.includes("Can't reach database server")) return true;
+  if (msg.includes("Connection lost")) return true;
+  if (msg.includes("ECONNRESET")) return true;
+  return false;
+}
+
+export function createDrainerHandler(deps: {
+  engine: RunEngine;
+  prisma: PrismaClientOrTransaction;
+}): MollifierDrainerHandler<MollifierSnapshot> {
+  return async (input) => {
+    const dwellMs = Date.now() - input.createdAt.getTime();
+
+    // Re-attach to the trace started by the caller's mollifier.queued span
+    // (its traceId + spanId were captured into the snapshot at buffer time).
+    // Without this the drainer would emit mollifier.drained in a brand-new
+    // trace and the engine.trigger instrumentation would inherit an empty
+    // active context — leaving the run-detail page with only the root span.
+    const snapshotTraceId =
+      typeof input.payload.traceId === "string" ? input.payload.traceId : undefined;
+    const snapshotSpanId =
+      typeof input.payload.spanId === "string" ? input.payload.spanId : undefined;
+
+    const parentContext =
+      snapshotTraceId && snapshotSpanId
+        ? trace.setSpanContext(context.active(), {
+            traceId: snapshotTraceId,
+            spanId: snapshotSpanId,
+            traceFlags: TraceFlags.SAMPLED,
+            isRemote: true,
+          })
+        : context.active();
+
+    // Cancel-wins-over-trigger. If a cancel API call landed on this
+    // entry while it was QUEUED, the snapshot carries `cancelledAt` +
+    // `cancelReason`. Skip the normal materialise path and write a
+    // CANCELED PG row directly. The `runCancelled` bus emit is
+    // suppressed here because a buffered-only run never had a primary
+    // trace event written for it — the runCancelled handler's
+    // `cancelRunEvent` lookup would fail and log noise per cancel.
+    const cancelledAtStr =
+      typeof input.payload.cancelledAt === "string" ? input.payload.cancelledAt : undefined;
+    if (cancelledAtStr) {
+      const cancelReason =
+        typeof input.payload.cancelReason === "string"
+          ? input.payload.cancelReason
+          : "Canceled by user";
+      await context.with(parentContext, async () => {
+        await startSpan(tracer, "mollifier.drained.cancelled", async (span) => {
+          span.setAttribute("mollifier.drained", true);
+          span.setAttribute("mollifier.dwell_ms", dwellMs);
+          span.setAttribute("mollifier.attempts", input.attempts);
+          span.setAttribute("mollifier.run_friendly_id", input.runId);
+          span.setAttribute("mollifier.cancel_bifurcation", true);
+          span.setAttribute("taskRunId", input.runId);
+          try {
+            await deps.engine.createCancelledRun(
+              {
+                snapshot: input.payload as any,
+                cancelledAt: new Date(cancelledAtStr),
+                cancelReason,
+                emitRunCancelledEvent: false,
+              },
+              deps.prisma
+            );
+          } catch (err) {
+            // createCancelledRun throws a conflict when the normal trigger
+            // replay path won the race and already materialised a live
+            // (non-CANCELED) row for this friendlyId. Its contract leaves
+            // the resolution to us: honour the cancel by actually
+            // cancelling the now-live run. Letting the conflict propagate
+            // would instead reach the drainer's terminal-failure path
+            // (isRetryablePgError() is false for it), buffer.fail() the
+            // entry, and silently lose the cancellation while the run
+            // keeps executing.
+            const isConflict =
+              err instanceof Error && err.message.startsWith("createCancelledRun conflict");
+            if (!isConflict) {
+              // Mirror the SYSTEM_FAILURE fallback the non-cancelled
+              // trigger path uses below. Without this branch, a
+              // non-retryable createCancelledRun failure rethrows, the
+              // drainer's onTerminalFailure handler skips because it
+              // gates on `cause === "max-attempts-exhausted"` (and the
+              // outer drainer classifies non-retryable failures with
+              // `cause: "non-retryable"`), and buffer.fail() deletes
+              // the entry — leaving NO PG row. The cancellation
+              // disappears silently from the customer's dashboard.
+              // Writing a SYSTEM_FAILURE row gives the run a terminal,
+              // visible state.
+              if (isRetryablePgError(err)) {
+                throw err;
+              }
+              span.setAttribute(
+                "mollifier.cancel_terminal_failure_reason",
+                err instanceof Error ? err.message : String(err)
+              );
+              try {
+                const wrote = await writeMollifierTerminalFailureRow(deps, {
+                  friendlyId: input.runId,
+                  snapshot: input.payload as Record<string, unknown>,
+                  reason: err instanceof Error ? err.message : String(err),
+                });
+                if (wrote) return;
+              } catch (writeErr) {
+                if (isRetryablePgError(writeErr)) {
+                  span.setAttribute("mollifier.cancel_terminal_write_retryable", true);
+                  throw writeErr;
+                }
+                span.setAttribute(
+                  "mollifier.cancel_terminal_write_error",
+                  writeErr instanceof Error ? writeErr.message : String(writeErr)
+                );
+              }
+              throw err;
+            }
+            span.setAttribute("mollifier.cancel_conflict", true);
+            const friendlyId =
+              typeof input.payload.friendlyId === "string" ? input.payload.friendlyId : input.runId;
+            await deps.engine.cancelRun({
+              runId: RunId.fromFriendlyId(friendlyId),
+              completedAt: new Date(cancelledAtStr),
+              reason: cancelReason,
+            });
+          }
+        });
+      });
+      return;
+    }
+
+    await context.with(parentContext, async () => {
+      await startSpan(tracer, "mollifier.drained", async (span) => {
+        span.setAttribute("mollifier.drained", true);
+        span.setAttribute("mollifier.dwell_ms", dwellMs);
+        span.setAttribute("mollifier.attempts", input.attempts);
+        span.setAttribute("mollifier.run_friendly_id", input.runId);
+        span.setAttribute("taskRunId", input.runId);
+
+        let triggerSucceeded = false;
+        try {
+          await deps.engine.trigger(input.payload as any, deps.prisma);
+          triggerSucceeded = true;
+        } catch (err) {
+          // The retryable-PG class re-throws so the drainer's outer
+          // worker loop can `buffer.requeue` (handled in
+          // `MollifierDrainer.drainOne`). For non-retryable failures we
+          // write a terminal SYSTEM_FAILURE row to PG via the engine's
+          // existing `createFailedTaskRun` (used by batch-trigger for
+          // the same purpose) so the customer sees the run in their
+          // dashboard / SDK instead of silently losing it when the
+          // buffer entry TTLs out. If THAT insert also fails (PG truly
+          // unreachable), rethrow so the drainer's outer catch falls
+          // through to its existing `buffer.fail` terminal-marker path.
+          if (isRetryablePgError(err)) {
+            throw err;
+          }
+          const reason = err instanceof Error ? err.message : String(err);
+          span.setAttribute("mollifier.terminal_failure_reason", reason);
+          try {
+            const wrote = await writeMollifierTerminalFailureRow(deps, {
+              friendlyId: input.runId,
+              snapshot: input.payload as Record<string, unknown>,
+              reason,
+            });
+            if (!wrote) {
+              // Snapshot too malformed to even construct a TaskRun row.
+              // Drainer's outer catch will buffer.fail this entry.
+              throw err;
+            }
+          } catch (writeErr) {
+            // The terminal SYSTEM_FAILURE write itself failed. If it
+            // failed because PG is transiently unreachable, rethrow the
+            // *write* error so the drainer requeues — buffer.fail()ing on
+            // the original non-retryable error would lose the run with no
+            // PG row ever landing. Once PG recovers the requeued entry
+            // writes its failure row and the customer sees it.
+            if (isRetryablePgError(writeErr)) {
+              span.setAttribute("mollifier.terminal_write_retryable", true);
+              throw writeErr;
+            }
+            // PG reachable but the write was rejected for another reason
+            // (genuinely bad snapshot). Rethrow the original trigger error
+            // so the drainer falls back to buffer.fail.
+            span.setAttribute(
+              "mollifier.terminal_write_error",
+              writeErr instanceof Error ? writeErr.message : String(writeErr)
+            );
+            throw err;
+          }
+        }
+
+        // Admin-only audit trail emitted once engine.trigger has
+        // landed a PG row. `recordRunDebugLog` flips this to the
+        // admin-gated debug kind (TaskEventKind.LOG in the PG store /
+        // DEBUG_EVENT in the ClickHouse store) which the trace view +
+        // logs download already strip for non-admins
+        // (`eventRepository.server.ts:108`,
+        // `resources.runs.$runParam.logs.download.ts:118`).
+        //
+        // Placement: emit as a zero-duration marker AT materialisation
+        // time, not as a back-dated bar spanning the buffered window.
+        // `engine.trigger` rewrites the run's root span at
+        // materialisation (it adopts the synth root via traceId/spanId
+        // carryover but updates start_time to "now"), so the trace
+        // renderer treats materialisation time as t=0. A back-dated
+        // event with startTime = bufferedAt would land before that t=0
+        // and get clipped from the tree. Same pattern as the
+        // `[engine] QUEUED` markers. The window itself is preserved
+        // in metadata so admins can read it off the span detail pane.
+        //
+        // Best-effort: `recordRunDebugLog` has its own try/catch and
+        // returns a result, so it never throws into the materialisation
+        // path. Failures are logged but not surfaced because the
+        // customer-visible run has already landed.
+        if (triggerSucceeded) {
+          const debugResult = await recordRunDebugLog(
+            RunId.fromFriendlyId(input.runId),
+            `Mollifier buffered ${dwellMs}ms before materialising`,
+            {
+              attributes: {
+                runId: input.runId,
+                metadata: {
+                  "mollifier.bufferedAt": input.createdAt.toISOString(),
+                  "mollifier.materialisedAt": new Date().toISOString(),
+                  "mollifier.dwellMs": dwellMs,
+                  "mollifier.attempts": input.attempts,
+                },
+              },
+              parentId: snapshotSpanId,
+            }
+          );
+          if (!debugResult.success && debugResult.code !== "RUN_NOT_FOUND") {
+            logger.warn("mollifier drainer: failed to record admin debug log", {
+              runId: input.runId,
+              code: debugResult.code,
+            });
+          }
+        }
+      });
+    });
+  };
+}
+
+// Shared SYSTEM_FAILURE construction used by both terminal paths:
+//   - non-retryable failure inside the handler (above)
+//   - retryable failure after maxAttempts inside the drainer's
+//     `processEntry` (via `createDrainerTerminalFailureHandler`)
+//
+// Suppresses `runFailed` and enqueues the alert manually — the engine's
+// `runFailed` handler calls `completeFailedRunEvent`, which looks up
+// the run's primary span. Buffered-only runs never had a primary trace
+// event written (the mollifier gate intercepts BEFORE
+// `repository.traceEvent` runs), so the lookup always fails and the
+// handler logs a systematic `[runFailed] Failed to complete failed
+// run event` error per terminal failure. `TriggerFailedTaskService`
+// handles the identical situation the same way (see triggerFailedTask
+// .server.ts:212 and 324) — pass `emitRunFailedEvent: false` to the
+// engine and call `PerformTaskRunAlertsService.enqueue(...)` directly
+// so customers' ERROR channels still fire. Alert enqueue is
+// best-effort; an alert-side failure is logged but does not bubble up
+// (the SYSTEM_FAILURE row landing is the load-bearing customer-visible
+// outcome).
+//
+// Returns the new `TaskRun` on success or `null` when the snapshot was
+// so malformed it couldn't even produce an environment — caller decides
+// whether to escalate that to `buffer.fail` directly. Throws on any
+// other failure so the drainer's retryable/non-retryable disposition
+// logic can own the decision.
+async function writeMollifierTerminalFailureRow(
+  deps: { engine: RunEngine; prisma: PrismaClientOrTransaction },
+  args: { friendlyId: string; snapshot: Record<string, unknown>; reason: string }
+) {
+  const { snapshot } = args;
+  const env = snapshot.environment as
+    | {
+        id: string;
+        type: any;
+        project: { id: string };
+        organization: { id: string };
+      }
+    | undefined;
+  if (!env) return null;
+  // Extract batch association from the snapshot if present. Without this
+  // a SYSTEM_FAILURE row for a buffered batch child won't be linked to
+  // its batch, and the batch parent's completion tracking can hang
+  // indefinitely waiting on a child that landed but isn't visible to
+  // the batch.
+  const rawBatch = snapshot.batch;
+  const batch =
+    rawBatch &&
+    typeof rawBatch === "object" &&
+    "id" in rawBatch &&
+    typeof (rawBatch as { id: unknown }).id === "string" &&
+    "index" in rawBatch &&
+    typeof (rawBatch as { index: unknown }).index === "number"
+      ? (rawBatch as { id: string; index: number })
+      : undefined;
+  const failedRun = await deps.engine.createFailedTaskRun({
+    friendlyId: args.friendlyId,
+    environment: env,
+    taskIdentifier: String(snapshot.taskIdentifier ?? ""),
+    payload: typeof snapshot.payload === "string" ? snapshot.payload : undefined,
+    payloadType: typeof snapshot.payloadType === "string" ? snapshot.payloadType : undefined,
+    error: {
+      type: "STRING_ERROR",
+      raw: `Mollifier drainer terminal failure: ${args.reason}`,
+    },
+    parentTaskRunId:
+      typeof snapshot.parentTaskRunId === "string" ? snapshot.parentTaskRunId : undefined,
+    rootTaskRunId: typeof snapshot.rootTaskRunId === "string" ? snapshot.rootTaskRunId : undefined,
+    depth: typeof snapshot.depth === "number" ? snapshot.depth : 0,
+    resumeParentOnCompletion: snapshot.resumeParentOnCompletion === true,
+    batch,
+    traceId: typeof snapshot.traceId === "string" ? snapshot.traceId : undefined,
+    spanId: typeof snapshot.spanId === "string" ? snapshot.spanId : undefined,
+    taskEventStore:
+      typeof snapshot.taskEventStore === "string" ? snapshot.taskEventStore : undefined,
+    queue: typeof snapshot.queue === "string" ? snapshot.queue : undefined,
+    lockedQueueId: typeof snapshot.lockedQueueId === "string" ? snapshot.lockedQueueId : undefined,
+    emitRunFailedEvent: false,
+  });
+  // Alerts side of `runFailed` — the engine emit was suppressed above
+  // so we don't create an orphan trace event; enqueue the alert
+  // directly so customers' ERROR channels still see the failure.
+  // Best-effort, mirroring TriggerFailedTaskService.
+  try {
+    await PerformTaskRunAlertsService.enqueue(failedRun.id);
+  } catch (alertsError) {
+    logger.warn("writeMollifierTerminalFailureRow: alert enqueue failed", {
+      friendlyId: args.friendlyId,
+      error: alertsError instanceof Error ? alertsError.message : String(alertsError),
+    });
+  }
+  return failedRun;
+}
+
+// Drainer-side terminal-failure callback. Fires from
+// `MollifierDrainer.processEntry` BEFORE `buffer.fail()` on any path
+// where the in-handler write didn't already land — currently the
+// `cause: "max-attempts-exhausted"` case for retryable PG errors. Writes
+// the same SYSTEM_FAILURE row the non-retryable handler path writes
+// inline (via the shared `writeMollifierTerminalFailureRow` helper) so
+// the customer-visible behaviour is identical regardless of how the
+// failure was classified.
+//
+// Re-throws retryable PG errors so the drainer requeues — buffer.fail()ing
+// here would still lose the run if PG is genuinely unreachable. Throwing
+// anything else falls through to buffer.fail to avoid an infinite loop on
+// a genuinely bad snapshot (the drainer logs it).
+export function createDrainerTerminalFailureHandler(deps: {
+  engine: RunEngine;
+  prisma: PrismaClientOrTransaction;
+}): MollifierDrainerTerminalFailureHandler<MollifierSnapshot> {
+  return async (input) => {
+    // The handler's own non-retryable terminal path has already written
+    // the SYSTEM_FAILURE row before it throws non-retryable. Only the
+    // retryable-exhausted path reaches us with no row written yet — gate
+    // on `cause` to avoid double-writing for non-retryable failures.
+    if (input.cause !== "max-attempts-exhausted") return;
+    await startSpan(tracer, "mollifier.drained.terminal_failure", async (span) => {
+      span.setAttribute("mollifier.drained", false);
+      span.setAttribute("mollifier.attempts", input.attempts);
+      span.setAttribute("mollifier.run_friendly_id", input.runId);
+      span.setAttribute("mollifier.terminal_failure_cause", input.cause);
+      span.setAttribute("mollifier.terminal_failure_reason", input.error.message);
+      span.setAttribute("taskRunId", input.runId);
+      await writeMollifierTerminalFailureRow(deps, {
+        friendlyId: input.runId,
+        snapshot: input.payload as Record<string, unknown>,
+        reason: input.error.message,
+      });
+    });
+  };
+}

apps/webapp/app/v3/mollifier/mollifierDrainingGauge.server.ts

@@ -0,0 +1,65 @@
+import { logger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+import { reportDrainingCount } from "./mollifierTelemetry.server";
+
+// How often we ZCARD the draining-tracker set. Each poll is a single
+// O(1) Redis call, so cadence is bounded by "how fresh do we want the
+// gauge?" rather than cost. 15s gives a tight-enough window to spot a
+// brief OOM-induced spike without burning RTTs, and lines up well with
+// typical Prometheus scrape intervals.
+const POLL_INTERVAL_MS = 15_000;
+
+let intervalHandle: ReturnType<typeof setInterval> | null = null;
+
+// Polls `mollifier:draining` cardinality on an interval and feeds the
+// gauge in `mollifierTelemetry.server.ts`. Started from the drainer
+// worker bootstrap (alongside `drainer.start()`) so it runs on the same
+// pods that actually pop/ack entries — observability is colocated with
+// the lifecycle.
+//
+// Idempotent: a second call is a no-op (Remix dev hot-reload re-runs
+// the bootstrap; the existing interval keeps ticking).
+export function startMollifierDrainingGauge(
+  opts: {
+    intervalMs?: number;
+    getBuffer?: typeof getMollifierBuffer;
+  } = {}
+): void {
+  if (intervalHandle !== null) return;
+
+  const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
+  const getBuffer = opts.getBuffer ?? getMollifierBuffer;
+
+  // Fire one poll immediately so the gauge populates before the first
+  // scrape rather than reading 0 for a full interval after boot.
+  const tick = async () => {
+    const buffer = getBuffer();
+    if (!buffer) return;
+    try {
+      const count = await buffer.getDrainingCount();
+      reportDrainingCount(count);
+    } catch (err) {
+      // Transient Redis blip — don't tank the loop, just leave the
+      // gauge at its last-known value. A sustained Redis outage will
+      // surface via the drainer's own alerts long before this gauge
+      // staleness becomes a primary signal.
+      logger.warn("Mollifier draining gauge poll failed; keeping previous value", { err });
+    }
+  };
+
+  void tick();
+  // unref so the interval doesn't keep the process alive past
+  // graceful shutdown — the gauge is best-effort, not a flush boundary.
+  intervalHandle = setInterval(() => {
+    void tick();
+  }, intervalMs);
+  intervalHandle.unref?.();
+}
+
+// Test seam. Production code never calls this; lifecycle is implicitly
+// process-end.
+export function stopMollifierDrainingGauge(): void {
+  if (intervalHandle === null) return;
+  clearInterval(intervalHandle);
+  intervalHandle = null;
+}

apps/webapp/app/v3/mollifier/mollifierGate.server.ts

@@ -0,0 +1,241 @@
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+import { createRealTripEvaluator } from "./mollifierTripEvaluator.server";
+import {
+  recordDecision,
+  type DecisionOutcome,
+  type RecordDecisionOptions,
+} from "./mollifierTelemetry.server";
+
+// `count` is the fleet-wide fixed-window counter for the env (INCR with a
+// PEXPIRE armed on the first tick of each window — see
+// `mollifierEvaluateTrip` in `packages/redis-worker/src/mollifier/buffer.ts`).
+// All webapp replicas pointing at the same Redis share the key
+// `mollifier:rate:${envId}`, so the threshold is the fleet-wide ceiling
+// rather than a per-instance one. At a window boundary an env can briefly
+// admit up to ~2x threshold across the fleet before tripping (fixed-window
+// not sliding-window). The tripped marker is refreshed on every overage
+// call, so a sustained burst holds the divert state until the rate falls
+// below threshold within a window.
+export type TripDecision =
+  | { divert: false }
+  | {
+      divert: true;
+      reason: "per_env_rate";
+      count: number;
+      threshold: number;
+      windowMs: number;
+      holdMs: number;
+    };
+
+export type GateOutcome =
+  | { action: "pass_through" }
+  | { action: "mollify"; decision: Extract<TripDecision, { divert: true }> }
+  | { action: "shadow_log"; decision: Extract<TripDecision, { divert: true }> };
+
+export type GateInputs = {
+  envId: string;
+  orgId: string;
+  taskId: string;
+  // Org-scoped flag overrides — taken from `Organization.featureFlags` on the
+  // AuthenticatedEnvironment at the call site. The repo-wide `flag()` helper
+  // queries the global `FeatureFlag` table; passing per-org overrides lets the
+  // mollifier opt in a single org without touching the global row, matching
+  // the pattern used by `canAccessAi`, `canAccessPrivateConnections`, and the
+  // compute-template beta gate.
+  orgFeatureFlags: Record<string, unknown> | null;
+  // Trigger options that drive the debounce / OTU / triggerAndWait
+  // bypasses. The mollify path can't
+  // serialise stateful callbacks (debounce), can't safely break OTU's
+  // synchronous-rejection contract, and shouldn't intercept single
+  // triggerAndWait (batchTriggerAndWait still funnels through per item).
+  options?: {
+    debounce?: unknown;
+    oneTimeUseToken?: string;
+    parentTaskRunId?: string;
+    resumeParentOnCompletion?: boolean;
+  };
+};
+
+export type TripEvaluator = (inputs: GateInputs) => Promise<TripDecision>;
+
+// DI seam type for consumers (e.g. triggerTask.server.ts) that inject the
+// gate at construction time. Deliberately narrower than `evaluateGate`'s
+// real signature — no `deps` param — because consumers only call it with
+// inputs and rely on the module-level defaults.
+export type MollifierEvaluateGate = (inputs: GateInputs) => Promise<GateOutcome>;
+
+export type GateDependencies = {
+  isMollifierEnabled: () => boolean;
+  isShadowModeOn: () => boolean;
+  resolveOrgFlag: (inputs: GateInputs) => Promise<boolean>;
+  evaluator: TripEvaluator;
+  logShadow: (inputs: GateInputs, decision: Extract<TripDecision, { divert: true }>) => void;
+  logMollified: (inputs: GateInputs, decision: Extract<TripDecision, { divert: true }>) => void;
+  recordDecision: (outcome: DecisionOutcome, opts: RecordDecisionOptions) => void;
+};
+
+// `options` is a thunk so env reads happen per-evaluation, not at module load.
+// Don't "simplify" to a plain object — dynamic config relies on the
+// gate observing whichever env values are live at trigger time.
+const defaultEvaluator = createRealTripEvaluator({
+  getBuffer: () => getMollifierBuffer(),
+  options: () => ({
+    windowMs: env.TRIGGER_MOLLIFIER_TRIP_WINDOW_MS,
+    threshold: env.TRIGGER_MOLLIFIER_TRIP_THRESHOLD,
+    holdMs: env.TRIGGER_MOLLIFIER_HOLD_MS,
+  }),
+});
+
+function logDivertDecision(
+  message: "mollifier.would_mollify" | "mollifier.mollified",
+  inputs: GateInputs,
+  decision: Extract<TripDecision, { divert: true }>
+): void {
+  logger.debug(message, {
+    envId: inputs.envId,
+    orgId: inputs.orgId,
+    taskId: inputs.taskId,
+    reason: decision.reason,
+    count: decision.count,
+    threshold: decision.threshold,
+    windowMs: decision.windowMs,
+    holdMs: decision.holdMs,
+  });
+}
+
+// Resolve the per-org mollifier flag purely from the in-memory
+// `Organization.featureFlags` JSON. No DB query — `triggerTask` is the
+// trigger hot path and the webapp CLAUDE.md forbids adding Prisma calls
+// there. The fleet-wide kill switch lives in `TRIGGER_MOLLIFIER_ENABLED`; rollout
+// is per-org via the JSON, matching the pattern used by `canAccessAi`,
+// `hasComputeAccess`, etc. There is no global `FeatureFlag` table read
+// in this path by design.
+export function makeResolveMollifierFlag(): (inputs: GateInputs) => Promise<boolean> {
+  return (inputs) => {
+    const override = inputs.orgFeatureFlags?.[FEATURE_FLAG.mollifierEnabled];
+    if (override !== undefined) {
+      const parsed = FeatureFlagCatalog[FEATURE_FLAG.mollifierEnabled].safeParse(override);
+      if (parsed.success) {
+        return Promise.resolve(parsed.data);
+      }
+    }
+    return Promise.resolve(false);
+  };
+}
+
+const resolveMollifierFlag = makeResolveMollifierFlag();
+
+export const defaultGateDependencies: GateDependencies = {
+  isMollifierEnabled: () => env.TRIGGER_MOLLIFIER_ENABLED === "1",
+  isShadowModeOn: () => env.TRIGGER_MOLLIFIER_SHADOW_MODE === "1",
+  resolveOrgFlag: resolveMollifierFlag,
+  evaluator: defaultEvaluator,
+  logShadow: (inputs, decision) => logDivertDecision("mollifier.would_mollify", inputs, decision),
+  logMollified: (inputs, decision) => logDivertDecision("mollifier.mollified", inputs, decision),
+  recordDecision,
+};
+
+export async function evaluateGate(
+  inputs: GateInputs,
+  deps: Partial<GateDependencies> = {}
+): Promise<GateOutcome> {
+  const d = { ...defaultGateDependencies, ...deps };
+
+  // Resolve the per-org flag up front so every decision below — including
+  // the bypasses — can be labelled enrolled vs not on the
+  // `mollifier.decisions` counter. Fail open: a transient error must not
+  // block triggers. The resolver is purely in-memory (reads
+  // `Organization.featureFlags`); it adds no DB round-trip to the hot path.
+  let orgFlagEnabled: boolean;
+  try {
+    orgFlagEnabled = await d.resolveOrgFlag(inputs);
+  } catch (error) {
+    logger.warn("mollifier.resolve_org_flag_failed", {
+      envId: inputs.envId,
+      orgId: inputs.orgId,
+      taskId: inputs.taskId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    orgFlagEnabled = false;
+  }
+  // Passed to every `recordDecision`. `org` only becomes a label for the
+  // (operationally capped) enrolled cohort — the guard is in
+  // `decisionLabels`, so passing orgId unconditionally here is safe.
+  const labels: RecordDecisionOptions = { enrolled: orgFlagEnabled, orgId: inputs.orgId };
+
+  // Debounce bypass. onDebounced is a closure over webapp state and
+  // can't be snapshotted into the buffer for drainer replay. Skip before the
+  // trip evaluator so debounce traffic is never counted against the rate.
+  if (inputs.options?.debounce) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+  // OneTimeUseToken bypass. OTU is a security feature on the PUBLIC_JWT
+  // auth path; its synchronous-rejection contract is materially worse to
+  // break than the idempotency-key contract.
+  if (inputs.options?.oneTimeUseToken) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+  // Single triggerAndWait bypass. batchTriggerAndWait still funnels
+  // through TriggerTaskService.call per item so the dominant burst pattern
+  // remains covered.
+  if (inputs.options?.parentTaskRunId && inputs.options?.resumeParentOnCompletion) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+
+  if (!d.isMollifierEnabled()) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+
+  const shadowOn = d.isShadowModeOn();
+
+  if (!orgFlagEnabled && !shadowOn) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+
+  // Fail open on evaluator errors too. The default `createRealTripEvaluator`
+  // catches its own errors and returns `{ divert: false }`, but injected or
+  // future evaluators may not — keep the contract symmetric with the org
+  // flag resolution above so the trigger hot path can never be broken by a
+  // gate-internal failure.
+  //
+  // Note: the evaluator INCRs the per-env Redis counter (`mollifier:rate:${envId}`)
+  // in *both* shadow-only and flag-on modes — shadow mode is observation-only at
+  // the user-visible level (no diversion), but not Redis-passive. It has to write
+  // because the threshold is computed from a counter, and a counter that doesn't
+  // increment isn't a counter. There's no cross-org bleed: `RuntimeEnvironment`
+  // is 1:1 with `Organization`, so the per-env counter is effectively per-org.
+  let decision: TripDecision;
+  try {
+    decision = await d.evaluator(inputs);
+  } catch (error) {
+    logger.warn("mollifier.evaluator_failed", {
+      envId: inputs.envId,
+      orgId: inputs.orgId,
+      taskId: inputs.taskId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    decision = { divert: false };
+  }
+  if (!decision.divert) {
+    d.recordDecision("pass_through", labels);
+    return { action: "pass_through" };
+  }
+
+  if (orgFlagEnabled) {
+    d.logMollified(inputs, decision);
+    d.recordDecision("mollify", { ...labels, reason: decision.reason });
+    return { action: "mollify", decision };
+  }
+
+  d.logShadow(inputs, decision);
+  d.recordDecision("shadow_log", { ...labels, reason: decision.reason });
+  return { action: "shadow_log", decision };
+}

apps/webapp/app/v3/mollifier/mollifierMollify.server.ts

@@ -0,0 +1,97 @@
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { serialiseMollifierSnapshot, type MollifierSnapshot } from "./mollifierSnapshot.server";
+import type { TripDecision } from "./mollifierGate.server";
+
+export type MollifyNotice = {
+  code: "mollifier.queued";
+  message: string;
+  docs: string;
+};
+
+export type MollifySyntheticResult = {
+  // `id` is the canonical TaskRun primary key derived from `friendlyId`
+  // via `RunId.fromFriendlyId`. Downstream consumers in the trigger
+  // route — notably `saveRequestIdempotency` — index the request-
+  // idempotency cache by this id; without it the cache stores
+  // `undefined` and Prisma's `findFirst({ where: { id: undefined } })`
+  // on retry strips the predicate and returns an arbitrary TaskRun
+  // (potential cross-tenant leak). Always populated.
+  //
+  // `spanId` is the root-span id allocated at gate-accept time and
+  // stored in the snapshot. Callers like the dashboard's Test action
+  // use it to build a `v3RunSpanPath` URL that auto-opens the right
+  // details panel — without it, the buffered run lands on the
+  // run-detail page with no span selected (parity gap with PG runs).
+  run: { id: string; friendlyId: string; spanId: string };
+  error: undefined;
+  // The race-loser path: if accept's SETNX hit an existing
+  // buffered run with the same (env, task, idempotencyKey), the
+  // response echoes the winner's runId with isCached=true. The
+  // mollifier-queued notice is only attached for the happy accept.
+  isCached: boolean;
+  notice?: MollifyNotice;
+};
+
+const NOTICE: MollifyNotice = {
+  code: "mollifier.queued",
+  message: "Trigger accepted into burst buffer. Consider batchTrigger for fan-outs of 100+.",
+  docs: "https://trigger.dev/docs/management/tasks/batch-trigger",
+};
+
+export async function mollifyTrigger(args: {
+  runFriendlyId: string;
+  environmentId: string;
+  organizationId: string;
+  engineTriggerInput: MollifierSnapshot;
+  decision: Extract<TripDecision, { divert: true }>;
+  buffer: MollifierBuffer;
+  // Optional idempotency context. When both are passed, accept SETNXes
+  // the lookup so the buffered window participates in trigger-time
+  // dedup symmetrically with PG.
+  idempotencyKey?: string;
+  taskIdentifier?: string;
+}): Promise<MollifySyntheticResult> {
+  const result = await args.buffer.accept({
+    runId: args.runFriendlyId,
+    envId: args.environmentId,
+    orgId: args.organizationId,
+    payload: serialiseMollifierSnapshot(args.engineTriggerInput),
+    idempotencyKey: args.idempotencyKey,
+    taskIdentifier: args.taskIdentifier,
+  });
+
+  if (result.kind === "duplicate_idempotency") {
+    // Race loser. Echo the winner's runId so the SDK's response shape
+    // matches PG-side idempotency cache hits. The winner's spanId isn't
+    // readily available without a second buffer fetch; an empty string
+    // causes `v3RunSpanPath` to omit the `?span=` param, which matches
+    // current behaviour for cached PG responses.
+    return {
+      run: {
+        id: RunId.fromFriendlyId(result.existingRunId),
+        friendlyId: result.existingRunId,
+        spanId: "",
+      },
+      error: undefined,
+      isCached: true,
+    };
+  }
+
+  // Both "accepted" and "duplicate_run_id" produce the same customer-
+  // visible response: a buffered-trigger acknowledgement. The duplicate
+  // runId case is unreachable in practice (runIds are server-generated
+  // and unique) but is silently idempotent at the buffer layer either way.
+  const rawSpanId = args.engineTriggerInput.spanId;
+  const spanId = typeof rawSpanId === "string" ? rawSpanId : "";
+  return {
+    run: {
+      id: RunId.fromFriendlyId(args.runFriendlyId),
+      friendlyId: args.runFriendlyId,
+      spanId,
+    },
+    error: undefined,
+    isCached: false,
+    notice: NOTICE,
+  };
+}

apps/webapp/app/v3/mollifier/mollifierSnapshot.server.ts

@@ -0,0 +1,16 @@
+import { serialiseSnapshot, deserialiseSnapshot } from "@trigger.dev/redis-worker";
+
+// MollifierSnapshot is the JSON-serialisable shape of the input that would be
+// passed to engine.trigger(). The drainer deserialises and replays it.
+// Kept as Record<string, unknown> at this layer — the engine.trigger call site
+// casts it to the engine's typed input. This keeps the mollifier subdirectory
+// from depending on @internal/run-engine internals.
+export type MollifierSnapshot = Record<string, unknown>;
+
+export function serialiseMollifierSnapshot(input: MollifierSnapshot): string {
+  return serialiseSnapshot(input);
+}
+
+export function deserialiseMollifierSnapshot(serialised: string): MollifierSnapshot {
+  return deserialiseSnapshot<MollifierSnapshot>(serialised);
+}

apps/webapp/app/v3/mollifier/mollifierStaleSweep.server.ts

@@ -0,0 +1,252 @@
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { logger as defaultLogger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+import { type StaleSweepStateStore } from "./mollifierStaleSweepState.server";
+import {
+  recordStaleEntry as defaultRecordStaleEntry,
+  reportStaleEntrySnapshot as defaultReportStaleEntrySnapshot,
+} from "./mollifierTelemetry.server";
+
+// One pass of the sweep scans a bounded slice of orgs from the buffer's
+// queue LIST, identified by a durable cursor in Redis. Per-env entry
+// scan is also bounded so a single pathological env can't extend the
+// pass.
+const DEFAULT_MAX_ENTRIES_PER_ENV = 1000;
+// Max orgs visited per tick. Together with `maxEntriesPerEnv` this
+// caps Redis traffic per pass. One "cycle" (visiting every org once)
+// takes `ceil(N_orgs / cap)` ticks, after which the cursor wraps and a
+// fresh org list is taken.
+const DEFAULT_MAX_ORGS_PER_PASS = 100;
+
+export type StaleSweepConfig = {
+  // Entries whose dwell exceeds this threshold are flagged stale. Set
+  // it well below `entryTtlSeconds * 1000` so ops have lead time before
+  // TTL-induced silent loss; the default (half of entryTtlSeconds)
+  // matches the cadence in the plan doc.
+  staleThresholdMs: number;
+  maxEntriesPerEnv?: number;
+  // Hard cap on orgs visited per tick. Bounds the per-pass Redis traffic
+  // and wall-time. Default 100 — at typical fleet sizes one or two
+  // ticks cover everyone; under incident-scale fan-out a full cycle
+  // takes a handful of ticks (~minutes) which is still well below the
+  // staleness signal latency that ops cares about.
+  maxOrgsPerPass?: number;
+};
+
+export type StaleSweepDeps = {
+  getBuffer?: () => MollifierBuffer | null;
+  // Durable cursor + per-env counts hash. Required: the sweep is
+  // useless without persistent state across ticks. The webapp wires up
+  // a real `MollifierStaleSweepState`; tests pass one constructed
+  // against the test container.
+  state: StaleSweepStateStore;
+  // No `envId` arg — `envId` is a high-cardinality metric attribute and
+  // is intentionally not emitted as a metric label. The structured warn
+  // log below carries envId for forensic drill-down.
+  recordStaleEntry?: () => void;
+  reportStaleEntrySnapshot?: (snapshot: Map<string, number>) => void;
+  logger?: { warn: (message: string, fields: Record<string, unknown>) => void };
+  now?: () => number;
+};
+
+export type StaleSweepResult = {
+  orgsScanned: number;
+  envsScanned: number;
+  entriesScanned: number;
+  staleCount: number;
+};
+
+// Walks a bounded slice of `orgs → envs → entries`, emitting an OTel
+// counter tick and a structured warning log for each buffer entry whose
+// dwell exceeds the stale threshold. Read-only on the buffer's own
+// state; writes only to the sweep's three dedicated keys
+// (`mollifier:stale_sweep:*`). The sweep does NOT remove or salvage
+// buffer entries; that decision is deferred to a separate retention-
+// policy change. The signal here exists so ops sees the drainer falling
+// behind well before TTL-induced loss kicks in.
+//
+// Sharding contract:
+// - Cursor starts at 0. On cursor=0 the org list is refreshed by
+//   snapshotting `buffer.listOrgs()` into the durable LIST — that is
+//   the cycle's frozen view of orgs to visit.
+// - Each tick consumes up to `maxOrgsPerPass` orgs from the LIST,
+//   advances the cursor, and persists.
+// - When the cursor reaches the end of the LIST it wraps to 0; the next
+//   tick rebuilds the org list, capturing any orgs that joined the
+//   buffer mid-cycle.
+// - The per-env counts HASH carries over across ticks: an env visited
+//   on tick N and not revisited until tick N+M keeps its last-known
+//   stale count in the gauge for that window. This is the price of
+//   sharding — accepted because the alternative (re-scan everything
+//   every tick) does not bound work.
+export async function runStaleSweepOnce(
+  config: StaleSweepConfig,
+  deps: StaleSweepDeps
+): Promise<StaleSweepResult> {
+  const getBuffer = deps.getBuffer ?? getMollifierBuffer;
+  const recordStale = deps.recordStaleEntry ?? defaultRecordStaleEntry;
+  const reportSnapshot = deps.reportStaleEntrySnapshot ?? defaultReportStaleEntrySnapshot;
+  const log = deps.logger ?? defaultLogger;
+  const now = (deps.now ?? Date.now)();
+  const maxEntries = config.maxEntriesPerEnv ?? DEFAULT_MAX_ENTRIES_PER_ENV;
+  const maxOrgsPerPass = config.maxOrgsPerPass ?? DEFAULT_MAX_ORGS_PER_PASS;
+
+  const buffer = getBuffer();
+  if (!buffer) {
+    // Replace any previous snapshot with empty so a previously-paging
+    // env doesn't stay latched if mollifier is turned off mid-flight.
+    // Also clear the durable state so a re-enable starts from a clean
+    // slate instead of resuming on a stale cursor.
+    await deps.state.clearAll();
+    reportSnapshot(new Map());
+    return { orgsScanned: 0, envsScanned: 0, entriesScanned: 0, staleCount: 0 };
+  }
+
+  let cursor = await deps.state.readCursor();
+  if (cursor === 0) {
+    // Fresh cycle — capture the current set of orgs into the frozen
+    // LIST. Any orgs that join after this snapshot wait until the next
+    // cycle to be visited. Acceptable for an observational sweep; the
+    // staleness signal would only fire on entries that have been
+    // dwelling for `staleThresholdMs` anyway, so they're not new.
+    const orgs = await buffer.listOrgs();
+    await deps.state.rebuildOrgList(orgs);
+  }
+
+  const { orgs: slice, total } = await deps.state.readOrgListSlice(cursor, maxOrgsPerPass);
+
+  let envsScanned = 0;
+  let entriesScanned = 0;
+  let staleCount = 0;
+
+  for (const orgId of slice) {
+    const envs = await buffer.listEnvsForOrg(orgId);
+    for (const envId of envs) {
+      envsScanned += 1;
+      let envStale = 0;
+      const entries = await buffer.listEntriesForEnv(envId, maxEntries);
+      for (const entry of entries) {
+        entriesScanned += 1;
+        const dwellMs = now - entry.createdAt.getTime();
+        if (dwellMs > config.staleThresholdMs) {
+          recordStale();
+          log.warn("mollifier.stale_entry", {
+            runId: entry.runId,
+            envId,
+            orgId,
+            dwellMs,
+            staleThresholdMs: config.staleThresholdMs,
+          });
+          envStale += 1;
+        }
+      }
+      // Persist the per-env count to the durable hash. HSET when stale
+      // > 0, HDEL when it dropped back to zero — the hash is the source
+      // of truth for the gauge snapshot below.
+      await deps.state.setEnvStaleCount(envId, envStale);
+      // Track that this env was visited during the current cycle. The
+      // reconcile step at cycle wrap uses this to HDEL counts hash
+      // entries for envs that fully drained mid-cycle (they disappear
+      // from listEnvsForOrg, so the inner loop above never reaches them
+      // and never HDELs their hash field — without reconcile the gauge
+      // would stay elevated forever).
+      await deps.state.markEnvVisited(envId);
+      staleCount += envStale;
+    }
+  }
+
+  // Advance the cursor. If the slice consumed the end of the LIST, wrap
+  // to 0 so the next tick rebuilds the org list and starts a new cycle.
+  const advanced = cursor + slice.length;
+  const wrapped = advanced >= total;
+  const newCursor = wrapped ? 0 : advanced;
+  await deps.state.writeCursor(newCursor);
+
+  if (wrapped) {
+    // Cycle ended. HDEL any env still in the counts hash that didn't
+    // appear in any tick of the just-completed cycle — these are envs
+    // that fully drained from the buffer mid-cycle and would otherwise
+    // hold their stale gauge value forever. Also DELs the visited set
+    // so the next cycle starts clean.
+    await deps.state.reconcileVisited();
+  }
+
+  // Emit the snapshot from the durable hash, which carries values for
+  // envs visited in earlier ticks too. This is what makes the gauge
+  // stable across ticks (and across webapp restarts).
+  const snapshot = await deps.state.readAllEnvStaleCounts();
+  reportSnapshot(snapshot);
+
+  return { orgsScanned: slice.length, envsScanned, entriesScanned, staleCount };
+}
+
+export type StaleSweepIntervalHandle = {
+  stop: () => Promise<void>;
+};
+
+// Production wrapper: schedule `runStaleSweepOnce` on a fixed interval.
+// One pass at a time — if a sweep is still running when the timer fires
+// the next tick is skipped (a backed-up Redis would otherwise queue
+// overlapping sweeps that all log the same stale entries).
+export function startStaleSweepInterval(
+  config: StaleSweepConfig & { intervalMs: number },
+  deps: StaleSweepDeps
+): StaleSweepIntervalHandle {
+  let stopped = false;
+  let inFlight = false;
+  // Tracks the current tick so `stop()` can await it before closing the
+  // state's Redis client. Without this, a tick that's already past the
+  // `stopped` guard at entry would continue making `state.*` calls
+  // against an ioredis client that `stop()` has already `quit()`ed,
+  // raising errors that the tick's own try/catch then logs as
+  // `mollifier.stale_sweep.failed` warnings — spurious noise on every
+  // graceful shutdown.
+  let currentTick: Promise<void> | null = null;
+
+  const tick = async () => {
+    if (stopped || inFlight) return;
+    inFlight = true;
+    const run = (async () => {
+      try {
+        await runStaleSweepOnce(config, deps);
+      } catch (err) {
+        const log = deps.logger ?? defaultLogger;
+        log.warn("mollifier.stale_sweep.failed", {
+          err: err instanceof Error ? err.message : String(err),
+        });
+      } finally {
+        inFlight = false;
+        currentTick = null;
+      }
+    })();
+    currentTick = run;
+    await run;
+  };
+
+  const timer = setInterval(() => {
+    void tick();
+  }, config.intervalMs);
+
+  return {
+    stop: async () => {
+      stopped = true;
+      clearInterval(timer);
+      // Drain any tick that started before `stopped` flipped. Its
+      // `state.*` calls must land before we close the Redis client.
+      if (currentTick) {
+        try {
+          await currentTick;
+        } catch {
+          // tick has its own catch — this await is just to ensure
+          // ordering, not to surface errors that have already been
+          // logged inside the tick.
+        }
+      }
+      // Close the state's underlying resource. The `close()` method is
+      // part of the `StaleSweepStateStore` contract — production's
+      // `MollifierStaleSweepState` shuts down its ioredis client; fake
+      // test states implement a no-op.
+      await deps.state.close();
+    },
+  };
+}

apps/webapp/app/v3/mollifier/mollifierStaleSweepState.server.ts

@@ -0,0 +1,189 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
+
+// Durable per-tick state for the sharded stale sweep. Four Redis keys,
+// all in the `mollifier:` namespace alongside the buffer's own state:
+//
+//   mollifier:stale_sweep:cursor    STRING  next position in org_list (0 = fresh cycle)
+//   mollifier:stale_sweep:org_list  LIST    org IDs frozen at the start of the cycle
+//   mollifier:stale_sweep:counts    HASH    envId -> last-known stale count
+//   mollifier:stale_sweep:visited   SET     envIds visited during the current cycle
+//
+// The state survives webapp restarts: a restarted process picks up the
+// cursor where the previous one left off and re-emits the last-known
+// gauge values immediately, rather than blinking to zero until the next
+// cycle visits each env.
+//
+// The `visited` set exists to GC the `counts` hash at cycle wrap: an env
+// that drains completely between sweep ticks disappears from
+// `buffer.listEnvsForOrg`, so the sweep's inner loop never revisits it
+// and never HDELs its counts entry. Without the visited-set GC the
+// counts hash retains the env's last-known stale count forever and the
+// gauge stays permanently elevated. At cursor wrap we diff the hash
+// against the cycle's visited set and HDEL the difference.
+//
+// Storage is owned by this class rather than added to MollifierBuffer
+// because the keys are sweep-internal — the buffer abstracts the
+// drainer/queue state, this abstracts sweep state. They share a
+// namespace prefix but no API surface.
+
+export interface StaleSweepStateStore {
+  readCursor(): Promise<number>;
+  writeCursor(value: number): Promise<void>;
+  /** Replaces the cycle's frozen org_list. Called at cursor=0. */
+  rebuildOrgList(orgs: string[]): Promise<void>;
+  /** Returns up to `count` org IDs starting at `start`, plus the LIST's total length. */
+  readOrgListSlice(start: number, count: number): Promise<{ orgs: string[]; total: number }>;
+  /** HSET when count > 0, HDEL when count === 0 (so the snapshot reflects current truth). */
+  setEnvStaleCount(envId: string, count: number): Promise<void>;
+  readAllEnvStaleCounts(): Promise<Map<string, number>>;
+  /** SADD `envId` to the current cycle's visited set. Called once per env scanned per tick. */
+  markEnvVisited(envId: string): Promise<void>;
+  /**
+   * HDEL every env in the counts hash that is NOT in the visited set, then
+   * DEL the visited set. Called when the cursor wraps (cycle ends) so
+   * envs that fully drained mid-cycle get cleaned out of the gauge.
+   */
+  reconcileVisited(): Promise<void>;
+  clearAll(): Promise<void>;
+  close(): Promise<void>;
+}
+
+const CURSOR_KEY = "mollifier:stale_sweep:cursor";
+const ORG_LIST_KEY = "mollifier:stale_sweep:org_list";
+const COUNTS_KEY = "mollifier:stale_sweep:counts";
+const VISITED_KEY = "mollifier:stale_sweep:visited";
+
+export class MollifierStaleSweepState implements StaleSweepStateStore {
+  private readonly redis: Redis;
+  private readonly logger: Logger;
+
+  constructor(options: {
+    redisOptions: RedisOptions;
+    logger?: Logger;
+    maxRetriesPerRequest?: number;
+  }) {
+    this.logger = options.logger ?? new Logger("MollifierStaleSweepState", "debug");
+    this.redis = createRedisClient(
+      { ...options.redisOptions, maxRetriesPerRequest: options.maxRetriesPerRequest ?? 20 },
+      {
+        onError: (error) => {
+          this.logger.error("MollifierStaleSweepState redis client error:", { error });
+        },
+      }
+    );
+  }
+
+  async readCursor(): Promise<number> {
+    const raw = await this.redis.get(CURSOR_KEY);
+    if (raw === null) return 0;
+    const n = Number.parseInt(raw, 10);
+    return Number.isFinite(n) && n >= 0 ? n : 0;
+  }
+
+  async writeCursor(value: number): Promise<void> {
+    await this.redis.set(CURSOR_KEY, String(value));
+  }
+
+  async rebuildOrgList(orgs: string[]): Promise<void> {
+    // DEL + RPUSH in a pipeline — close enough to atomic for an
+    // observational sweep (the inFlight guard at startStaleSweepInterval
+    // serialises sweep passes; nothing else writes these keys).
+    const pipeline = this.redis.pipeline();
+    pipeline.del(ORG_LIST_KEY);
+    if (orgs.length > 0) {
+      pipeline.rpush(ORG_LIST_KEY, ...orgs);
+    }
+    await pipeline.exec();
+  }
+
+  async readOrgListSlice(start: number, count: number): Promise<{ orgs: string[]; total: number }> {
+    const pipeline = this.redis.pipeline();
+    pipeline.lrange(ORG_LIST_KEY, start, start + count - 1);
+    pipeline.llen(ORG_LIST_KEY);
+    const results = await pipeline.exec();
+    // `pipeline.exec()` returning null is the abort-on-broken-pipe path.
+    // Surface it as a thrown error — the previous `return { orgs: [], total: 0 }`
+    // looked indistinguishable from a genuinely empty org list to the
+    // caller (`runStaleSweepOnce`), which then wrote cursor=0, reconciled
+    // visited envs against the empty result, and cleared the stale-entry
+    // gauge. That hid real Redis problems and silenced the alerts the
+    // sweep exists to raise.
+    if (!results) {
+      throw new Error("MollifierStaleSweepState.readOrgListSlice: pipeline.exec returned null");
+    }
+    const [lrangeErr, lrangeRes] = results[0] as [Error | null, string[] | null];
+    const [llenErr, llenRes] = results[1] as [Error | null, number | null];
+    if (lrangeErr || llenErr) {
+      this.logger.error("MollifierStaleSweepState.readOrgListSlice failed", {
+        lrangeErr: lrangeErr?.message,
+        llenErr: llenErr?.message,
+      });
+      // Same reasoning as the null-result path above — propagate the
+      // failure so the sweep's interval wrapper records a failed cycle
+      // and the durable cursor / counts hash stay untouched.
+      throw lrangeErr ?? llenErr ?? new Error("MollifierStaleSweepState.readOrgListSlice failed");
+    }
+    return { orgs: lrangeRes ?? [], total: llenRes ?? 0 };
+  }
+
+  async setEnvStaleCount(envId: string, count: number): Promise<void> {
+    if (count > 0) {
+      await this.redis.hset(COUNTS_KEY, envId, String(count));
+    } else {
+      await this.redis.hdel(COUNTS_KEY, envId);
+    }
+  }
+
+  async readAllEnvStaleCounts(): Promise<Map<string, number>> {
+    const raw = await this.redis.hgetall(COUNTS_KEY);
+    const out = new Map<string, number>();
+    for (const [envId, value] of Object.entries(raw)) {
+      const n = Number.parseInt(value, 10);
+      if (Number.isFinite(n)) out.set(envId, n);
+    }
+    return out;
+  }
+
+  async markEnvVisited(envId: string): Promise<void> {
+    await this.redis.sadd(VISITED_KEY, envId);
+  }
+
+  async reconcileVisited(): Promise<void> {
+    // HKEYS + SMEMBERS in a pipeline, then HDEL the difference locally.
+    // For typical fleet sizes (counts and visited both bounded by the
+    // count of buffered envs) this is well within a single RTT plus one
+    // small HDEL.
+    const pipeline = this.redis.pipeline();
+    pipeline.hkeys(COUNTS_KEY);
+    pipeline.smembers(VISITED_KEY);
+    const results = await pipeline.exec();
+    if (!results) return;
+    const [hkeysErr, hkeysRes] = results[0] as [Error | null, string[] | null];
+    const [smembersErr, smembersRes] = results[1] as [Error | null, string[] | null];
+    if (hkeysErr || smembersErr) {
+      this.logger.error("MollifierStaleSweepState.reconcileVisited failed", {
+        hkeysErr: hkeysErr?.message,
+        smembersErr: smembersErr?.message,
+      });
+      return;
+    }
+    const hashEnvs = hkeysRes ?? [];
+    const visited = new Set(smembersRes ?? []);
+    const orphans = hashEnvs.filter((envId) => !visited.has(envId));
+    const cleanup = this.redis.pipeline();
+    if (orphans.length > 0) {
+      cleanup.hdel(COUNTS_KEY, ...orphans);
+    }
+    cleanup.del(VISITED_KEY);
+    await cleanup.exec();
+  }
+
+  async clearAll(): Promise<void> {
+    await this.redis.del(CURSOR_KEY, ORG_LIST_KEY, COUNTS_KEY, VISITED_KEY);
+  }
+
+  async close(): Promise<void> {
+    await this.redis.quit();
+  }
+}

apps/webapp/app/v3/mollifier/mollifierTelemetry.server.ts

@@ -0,0 +1,152 @@
+import { getMeter } from "@internal/tracing";
+
+const meter = getMeter("mollifier");
+
+export const mollifierDecisionsCounter = meter.createCounter("mollifier.decisions", {
+  description: "Count of mollifier gate decisions by outcome",
+});
+
+export type DecisionOutcome = "pass_through" | "shadow_log" | "mollify";
+export type DecisionReason = "per_env_rate";
+
+export type RecordDecisionOptions = {
+  reason?: DecisionReason;
+  // Whether the org has the per-org mollifier flag enabled. Emitted as the
+  // bounded `enrolled` label so we can see how often enrolled orgs pass
+  // through instead of mollifying — the whole point of this instrumentation.
+  enrolled: boolean;
+  // Org id, attached as the `org` label ONLY when `enrolled` is true. The
+  // enrolled cohort is capped operationally (<= 10 orgs), so this stays
+  // low-cardinality. It must NEVER be attached for non-enrolled orgs — that
+  // would fan the metric out across every org id in production (unbounded;
+  // the same high-cardinality ban that keeps envId/orgId off the other
+  // mollifier metrics). The guard lives in `decisionLabels`, so callers can
+  // pass orgId unconditionally.
+  orgId?: string;
+};
+
+// Pure: builds the metric label set for a gate decision. Extracted from
+// `recordDecision` so the org-only-when-enrolled cardinality guard is
+// unit-testable without standing up an OTel meter.
+export function decisionLabels(
+  outcome: DecisionOutcome,
+  opts: RecordDecisionOptions
+): Record<string, string> {
+  return {
+    outcome,
+    enrolled: opts.enrolled ? "true" : "false",
+    ...(opts.reason ? { reason: opts.reason } : {}),
+    ...(opts.enrolled && opts.orgId ? { org: opts.orgId } : {}),
+  };
+}
+
+export function recordDecision(outcome: DecisionOutcome, opts: RecordDecisionOptions): void {
+  mollifierDecisionsCounter.add(1, decisionLabels(outcome, opts));
+}
+
+// Counts subscriptions hitting `/realtime/v1/runs/<id>` for a run that
+// lives only in the mollifier buffer (no PG row yet). The route opens
+// the Electric stream anyway so the eventual drainer-INSERT propagates
+// to the client; this counter is the signal of how often customers
+// subscribe inside the buffered window.
+export const realtimeBufferedSubscriptionsCounter = meter.createCounter(
+  "mollifier.realtime_subscriptions.buffered",
+  {
+    description:
+      "Realtime subscriptions opened against a runId that exists only in the mollifier buffer",
+  }
+);
+
+// No `envId` attribute — `envId` is a banned high-cardinality metric
+// label per the repo's OTel rules. The structured warn log emitted
+// alongside the counter tick (in `mollifierStaleSweep.server.ts`)
+// carries the envId / orgId / runId for forensic drill-down; the
+// metric stays an aggregate.
+export function recordRealtimeBufferedSubscription(): void {
+  realtimeBufferedSubscriptionsCounter.add(1);
+}
+
+// Counts buffer entries that have been waiting in the queue ZSET longer
+// than the configured stale threshold. Useful for historical "stale
+// events over time" views, but not directly alertable on its own — a
+// single stuck entry observed by N sweep ticks adds N to the counter,
+// so `rate()` over an alerting window reflects (entries × ticks), not
+// "entries that are stale right now".
+export const staleEntriesCounter = meter.createCounter("mollifier.stale_entries", {
+  description: "Mollifier buffer entries whose dwell exceeds the stale threshold (per sweep pass)",
+});
+
+// No `envId` attribute — see comment above.
+export function recordStaleEntry(): void {
+  staleEntriesCounter.add(1);
+}
+
+// Alertable signal: the total count of stale entries observed by the
+// latest sweep. The sweep snapshots the full picture on each pass so
+// the gauge drops back to 0 when the drainer catches up instead of
+// staying latched. Recommended alert:
+//   mollifier_stale_entries_current > 0 for 5m
+export const staleEntriesGauge = meter.createObservableGauge("mollifier.stale_entries.current", {
+  description:
+    "Buffer entries whose dwell exceeds the stale threshold, as observed by the latest sweep pass",
+});
+
+let latestStaleTotal = 0;
+
+export function reportStaleEntrySnapshot(snapshot: Map<string, number>): void {
+  // Sum across envs. Per-env breakdown is intentionally NOT emitted as
+  // a metric label (high-cardinality); the structured warn log lines
+  // from the sweep carry per-env detail for ops to drill down.
+  let total = 0;
+  for (const count of snapshot.values()) {
+    total += count;
+  }
+  latestStaleTotal = total;
+}
+
+meter.addBatchObservableCallback(
+  (result) => {
+    result.observe(staleEntriesGauge, latestStaleTotal);
+  },
+  [staleEntriesGauge]
+);
+
+// Observability gauge for entries currently in DRAINING state — popped
+// by the drainer but not yet acked/failed/requeued. Backed by the
+// `mollifier:draining` ZSET (see `MollifierBuffer.getDrainingCount`)
+// and polled by the loop in `mollifierDrainingGaugeLoop.server.ts`.
+//
+// Useful for:
+//   - "Is anything mid-drain right now?" panels
+//   - Post-crash forensics ("how many entries got stranded by that ECS OOM?")
+//   - Alerting: a sustained non-zero with no drainer progress is a stall
+//
+// No `envId` attribute — same high-cardinality constraint as the other
+// mollifier gauges. The per-entry hash carries env/org for drill-down.
+export const drainingCountGauge = meter.createObservableGauge("mollifier.draining.current", {
+  description:
+    "Mollifier buffer entries currently in DRAINING state (popped but not yet acked/failed/requeued)",
+});
+
+let latestDrainingCount = 0;
+
+export function reportDrainingCount(count: number): void {
+  latestDrainingCount = count;
+}
+
+meter.addBatchObservableCallback(
+  (result) => {
+    result.observe(drainingCountGauge, latestDrainingCount);
+  },
+  [drainingCountGauge]
+);
+
+// Electric SQL's shape-stream protocol adds a `handle=` query param on
+// every reconnect after the initial GET. Gating the realtime-buffered
+// log/counter on its absence keeps the signal at one tick per
+// subscription instead of one tick per ~20s live-poll iteration —
+// without it the counter would over-count by the long-poll factor.
+export function isInitialBufferedSubscriptionRequest(url: string | URL): boolean {
+  const u = typeof url === "string" ? new URL(url) : url;
+  return !u.searchParams.has("handle");
+}

apps/webapp/app/v3/mollifier/mollifierTripEvaluator.server.ts

@@ -0,0 +1,47 @@
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { logger } from "~/services/logger.server";
+import type { GateInputs, TripDecision, TripEvaluator } from "./mollifierGate.server";
+
+export type TripEvaluatorOptions = {
+  windowMs: number;
+  threshold: number;
+  holdMs: number;
+};
+
+export type CreateRealTripEvaluatorDeps = {
+  getBuffer: () => MollifierBuffer | null;
+  options: () => TripEvaluatorOptions;
+};
+
+export function createRealTripEvaluator(deps: CreateRealTripEvaluatorDeps): TripEvaluator {
+  return async (inputs: GateInputs): Promise<TripDecision> => {
+    const buffer = deps.getBuffer();
+    if (!buffer) return { divert: false };
+
+    const opts = deps.options();
+
+    try {
+      const { tripped, count } = await buffer.evaluateTrip(inputs.envId, opts);
+      if (!tripped) return { divert: false };
+
+      return {
+        divert: true,
+        reason: "per_env_rate",
+        count,
+        threshold: opts.threshold,
+        windowMs: opts.windowMs,
+        holdMs: opts.holdMs,
+      };
+    } catch (err) {
+      // Deliberate: no error counter here. Shadow mode means a silent miss is
+      // harmless — fail-open is the safe direction. The error log + Sentry
+      // capture is sufficient operability while this runs in shadow mode. Revisit
+      // once buffer writes are the primary path and a missed evaluation has cost.
+      logger.error("mollifier trip evaluator: fail-open on error", {
+        envId: inputs.envId,
+        err: err instanceof Error ? err.message : String(err),
+      });
+      return { divert: false };
+    }
+  };
+}

apps/webapp/app/v3/mollifier/mutateWithFallback.server.ts

@@ -0,0 +1,229 @@
+import type {
+  BufferEntry,
+  MollifierBuffer,
+  MutateSnapshotResult,
+  SnapshotPatch,
+} from "@trigger.dev/redis-worker";
+import type { TaskRun } from "@trigger.dev/database";
+import type { PrismaClientOrTransaction, PrismaReplicaClient } from "~/db.server";
+import { prisma, $replica } from "~/db.server";
+import { runStore } from "~/v3/runStore.server";
+import { logger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+
+// Wait/retry knobs. Exported for tests.
+export const DEFAULT_SAFETY_NET_MS = 2_000;
+// Initial gap between buffer polls; grows by BACKOFF_FACTOR up to
+// DEFAULT_MAX_POLL_STEP_MS so a slow drain doesn't poll at a tight fixed
+// cadence for the whole safety-net budget.
+export const DEFAULT_POLL_STEP_MS = 20;
+export const DEFAULT_MAX_POLL_STEP_MS = 250;
+export const DEFAULT_BACKOFF_FACTOR = 1.7;
+
+export type MutateWithFallbackInput<TResponse> = {
+  runId: string;
+  environmentId: string;
+  organizationId: string;
+  bufferPatch: SnapshotPatch;
+  // Called when a PG row exists (either replica-hit or post-wait writer-hit).
+  // Receives the full TaskRun shape and returns the customer-visible body.
+  pgMutation: (pgRow: TaskRun) => Promise<TResponse>;
+  // Called when the patch landed cleanly on the buffer snapshot. The
+  // drainer will see the patched payload on its next pop. Receives the
+  // pre-mutation snapshot entry (the one fetched for the env auth
+  // check above) so the caller can compute response details that
+  // depend on the prior state — e.g. the tags route needs to dedup
+  // against the existing tags to report an accurate `newTags` count
+  // matching the PG path, without an extra Redis round-trip.
+  // `bufferEntry` is `null` in the rare race where the entry didn't
+  // exist at pre-check time but appeared before `mutateSnapshot`.
+  synthesisedResponse: (ctx: { bufferEntry: BufferEntry | null }) => TResponse | Promise<TResponse>;
+  // Called when the buffer rejected the patch as invalid (e.g. an
+  // `append_tags` patch carrying `maxTags` would exceed the cap). Required
+  // only by callers that send a rejectable patch; the helper throws if the
+  // buffer reports a rejection and no builder was supplied. Receives the
+  // same `bufferEntry` context as `synthesisedResponse` so a rejection
+  // message can reference the prior state if useful.
+  rejectedResponse?: (ctx: { bufferEntry: BufferEntry | null }) => TResponse | Promise<TResponse>;
+  abortSignal?: AbortSignal;
+  // Override defaults for tests.
+  safetyNetMs?: number;
+  pollStepMs?: number;
+  maxPollStepMs?: number;
+  backoffFactor?: number;
+  // Test injection.
+  getBuffer?: () => MollifierBuffer | null;
+  prismaWriter?: PrismaClientOrTransaction;
+  prismaReplica?: PrismaReplicaClient;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => number;
+  // Jitter source; defaults to Math.random. Inject `() => 0` for
+  // deterministic poll timing in tests.
+  random?: () => number;
+};
+
+export type MutateWithFallbackOutcome<TResponse> =
+  | { kind: "pg"; response: TResponse }
+  | { kind: "snapshot"; response: TResponse }
+  | { kind: "rejected"; response: TResponse }
+  | { kind: "not_found" }
+  | { kind: "timed_out" };
+
+// PG-first → buffer mutateSnapshot → wait-and-bounce. The
+// caller decides how to translate the outcome into an HTTP response —
+// this helper never throws Response objects so it remains route-agnostic
+// and unit-testable in isolation.
+export async function mutateWithFallback<TResponse>(
+  input: MutateWithFallbackInput<TResponse>
+): Promise<MutateWithFallbackOutcome<TResponse>> {
+  const replica = input.prismaReplica ?? $replica;
+  const writer = input.prismaWriter ?? prisma;
+  const buffer = (input.getBuffer ?? getMollifierBuffer)();
+  const sleep = input.sleep ?? defaultSleep;
+  const now = input.now ?? Date.now;
+
+  // Path 1 — PG is already canonical.
+  const replicaRow = await findRunInPg(replica, input.runId, input.environmentId);
+  if (replicaRow) {
+    const response = await input.pgMutation(replicaRow);
+    return { kind: "pg", response };
+  }
+
+  if (!buffer) {
+    // No buffer configured (mollifier disabled or boot-time error). The
+    // pre-PR mutation routes read from the writer directly, so a freshly-
+    // created PG row was always visible regardless of replication lag.
+    // Now that the read moved to the replica (line 87) for the offload,
+    // a `!buffer` short-circuit would regress: a real PG row + replica
+    // lag would return 404. Mirror the writer-disambiguation block below
+    // (line 148, the buffer-says-not-found path) so degraded mode
+    // (mollifier disabled) still matches pre-PR mutation behaviour.
+    const writerRow = await findRunInPg(writer, input.runId, input.environmentId);
+    if (writerRow) {
+      const response = await input.pgMutation(writerRow);
+      return { kind: "pg", response };
+    }
+    return { kind: "not_found" };
+  }
+
+  // Env-scoped authorization for the buffer path. The replica/writer
+  // lookups above are already env-scoped via findRunInPg; this closes
+  // the same gap on the buffer side so a caller authed in env A can't
+  // mutate a buffered run that belongs to env B (or a different org)
+  // by guessing its friendlyId. Non-atomic w.r.t. the mutateSnapshot
+  // call below, but the TOCTOU is benign: runIds are globally unique,
+  // so a cross-env entry can't suddenly appear after a same-env check.
+  // A genuinely-missing entry (entry === null) falls through and is
+  // handled by the existing not_found / writer-recovery path below.
+  const entryForAuth = await buffer.getEntry(input.runId);
+  if (
+    entryForAuth &&
+    (entryForAuth.envId !== input.environmentId || entryForAuth.orgId !== input.organizationId)
+  ) {
+    // Hide existence on env mismatch: return not_found, same shape as
+    // a true miss, rather than 403 which would leak that the runId
+    // exists in some other env.
+    return { kind: "not_found" };
+  }
+
+  // Path 2 — buffer snapshot mutation.
+  const result: MutateSnapshotResult = await buffer.mutateSnapshot(input.runId, input.bufferPatch);
+
+  if (result === "applied_to_snapshot") {
+    return {
+      kind: "snapshot",
+      response: await input.synthesisedResponse({ bufferEntry: entryForAuth }),
+    };
+  }
+
+  if (result === "limit_exceeded") {
+    // The buffer refused the patch (e.g. tag cap). Nothing was written.
+    // Surface the caller's rejection body; a missing builder means the
+    // caller sent a rejectable patch without handling the rejection.
+    if (!input.rejectedResponse) {
+      throw new Error(
+        "mutateWithFallback: buffer returned 'limit_exceeded' but no rejectedResponse was provided"
+      );
+    }
+    return {
+      kind: "rejected",
+      response: await input.rejectedResponse({ bufferEntry: entryForAuth }),
+    };
+  }
+
+  if (result === "not_found") {
+    // Disambiguate a genuine 404 from a replica-lag miss: ask the writer
+    // directly. If the row just appeared post-drain we route through the
+    // PG mutation path.
+    const writerRow = await findRunInPg(writer, input.runId, input.environmentId);
+    if (writerRow) {
+      const response = await input.pgMutation(writerRow);
+      return { kind: "pg", response };
+    }
+    return { kind: "not_found" };
+  }
+
+  // result === "busy" — the entry is mid-handoff (DRAINING) or already
+  // materialised. We do NOT poll the primary for the row to appear: that
+  // piles read load onto the writer at exactly the moment mollifier exists
+  // to shed it. Instead we watch the buffer entry itself (cheap Redis
+  // reads). The drainer writes the PG row BEFORE it acks (sets
+  // `materialised`) or fails (deletes the entry), so the entry's own state
+  // is an authoritative, already-in-Redis signal for "is the row in PG
+  // yet?". Only once it resolves do we touch the primary — exactly once,
+  // for the real mutation.
+  const safetyNetMs = input.safetyNetMs ?? DEFAULT_SAFETY_NET_MS;
+  const maxPollStepMs = input.maxPollStepMs ?? DEFAULT_MAX_POLL_STEP_MS;
+  const backoffFactor = input.backoffFactor ?? DEFAULT_BACKOFF_FACTOR;
+  const random = input.random ?? Math.random;
+  const deadline = now() + safetyNetMs;
+  let step = input.pollStepMs ?? DEFAULT_POLL_STEP_MS;
+
+  while (now() < deadline) {
+    if (input.abortSignal?.aborted) {
+      return { kind: "timed_out" };
+    }
+
+    const entry = await buffer.getEntry(input.runId);
+    // Resolved when the entry is gone (`fail` deleted it after writing a
+    // terminal SYSTEM_FAILURE row) or materialised (`ack` after a
+    // successful trigger / cancel write). In both cases the PG row is now
+    // committed on the primary, so read it once and route through the
+    // canonical PG mutation path.
+    if (entry === null || entry.materialised === true) {
+      const row = await findRunInPg(writer, input.runId, input.environmentId);
+      if (row) {
+        const response = await input.pgMutation(row);
+        return { kind: "pg", response };
+      }
+      // Entry gone with no PG row: the drainer's terminal write itself
+      // failed (PG unreachable). Nothing to mutate.
+      return { kind: "not_found" };
+    }
+    // Still QUEUED (requeued after a retryable drain error) or DRAINING —
+    // the run hasn't reached PG. Back off with jitter so concurrent
+    // waiters on the same draining run don't requery in lockstep.
+    if (now() >= deadline) break;
+    const jittered = step + Math.floor(random() * step);
+    await sleep(jittered);
+    step = Math.min(Math.ceil(step * backoffFactor), maxPollStepMs);
+  }
+
+  logger.warn("mollifier mutate-with-fallback: drainer resolution timed out", {
+    runId: input.runId,
+    safetyNetMs,
+  });
+  return { kind: "timed_out" };
+}
+
+async function findRunInPg(
+  client: PrismaClientOrTransaction | PrismaReplicaClient,
+  friendlyId: string,
+  environmentId: string
+): Promise<TaskRun | null> {
+  return runStore.findRun({ friendlyId, runtimeEnvironmentId: environmentId }, client);
+}
+
+function defaultSleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

apps/webapp/app/v3/mollifier/readFallback.server.ts

@@ -0,0 +1,265 @@
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import { IdempotencyKeyOptionsSchema } from "@trigger.dev/core/v3/schemas";
+import type { z } from "zod";
+import { logger } from "~/services/logger.server";
+import { deserialiseMollifierSnapshot } from "./mollifierSnapshot.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+
+export type ReadFallbackInput = {
+  runId: string;
+  environmentId: string;
+  organizationId: string;
+};
+
+export type SyntheticRun = {
+  // Snapshot-derived TaskRun primary key. Used by ReplayTaskRunService
+  // for logging and by callers passing this object where a TaskRun is
+  // expected (cast). Derived deterministically from `friendlyId`.
+  id: string;
+  friendlyId: string;
+  status: "QUEUED" | "FAILED" | "CANCELED";
+  // Set when the customer cancelled the run via the dashboard or API
+  // while it was buffered. The drainer's cancel bifurcation reads this
+  // on next pop and writes a CANCELED PG row directly (skipping
+  // materialisation). Reflected back into the UI by the synthesised
+  // SpanRun so the run-detail page shows the cancelled state even before
+  // the drainer materialises it.
+  cancelledAt: Date | undefined;
+  cancelReason: string | undefined;
+  // Reschedule patch (`set_delay`) writes `delayUntil` into the snapshot.
+  // Surfacing it on SyntheticRun lets the retrieve-run shape reflect the
+  // pending delay before the drainer materialises the PG row.
+  delayUntil: Date | undefined;
+  taskIdentifier: string | undefined;
+  createdAt: Date;
+
+  payload: unknown;
+  payloadType: string | undefined;
+  metadata: unknown;
+  metadataType: string | undefined;
+  // Seed-metadata mirrors what `triggerTask.server.ts` writes into the
+  // snapshot: the original metadataPacket data preserved separately from
+  // any later customer mutations. ReplayTaskRunService uses these to
+  // rebuild the replay's metadata.
+  seedMetadata: string | undefined;
+  seedMetadataType: string | undefined;
+
+  idempotencyKey: string | undefined;
+  // Surfaced for the cached-hit expiration check in IdempotencyKeyConcern.
+  // The PG-resident path enforces this (clears key, allows new run when
+  // expired). For buffered runs the snapshot carries the same field — we
+  // expose it here so the cached-hit branch can apply the same check
+  // rather than indefinitely returning the buffered run's id.
+  idempotencyKeyExpiresAt: Date | undefined;
+  // `{ key, scope }` object form, matching how the SDK serialises and PG
+  // stores it. Previously typed as `string[]` (legacy/incorrect — Prisma
+  // is `Json?` carrying the schema-shaped object). `getUserProvidedIdempotencyKey`
+  // and `extractIdempotencyKeyScope` both parse via the same Zod schema;
+  // they returned `undefined` for the array-shape, which silently
+  // demoted the response to surface the hash instead of the user-
+  // provided key for buffered runs — a contract divergence from
+  // PG-resident runs. See the regression test in `mollifierReadFallback.test.ts`.
+  idempotencyKeyOptions: z.infer<typeof IdempotencyKeyOptionsSchema> | undefined;
+  isTest: boolean;
+  depth: number;
+  ttl: string | undefined;
+  tags: string[];
+  // Mirror of `tags` under the PG field name. ReplayTaskRunService reads
+  // `existingTaskRun.runTags`; both names are kept here so a synthetic
+  // run can be passed wherever the PG-shape `runTags` is expected.
+  runTags: string[];
+  lockedToVersion: string | undefined;
+  resumeParentOnCompletion: boolean;
+  parentTaskRunId: string | undefined;
+
+  // Allocated at gate-accept time and embedded in the snapshot so the run's
+  // trace is continuous from QUEUED-in-buffer through executing post-drain.
+  traceId: string | undefined;
+  spanId: string | undefined;
+  parentSpanId: string | undefined;
+
+  // Replay-relevant fields populated from the engine-trigger snapshot.
+  // ReplayTaskRunService reads each of these from the existing TaskRun;
+  // when the original lives in the buffer we synthesise them here.
+  runtimeEnvironmentId: string | undefined;
+  engine: "V2";
+  workerQueue: string | undefined;
+  region: string | undefined;
+  queue: string | undefined;
+  concurrencyKey: string | undefined;
+  machinePreset: string | undefined;
+  realtimeStreamsVersion: string | undefined;
+
+  // Additional snapshot-sourced fields used when synthesising a SpanRun
+  // for the dashboard's right-side details panel. All optional because
+  // older snapshots may not carry them.
+  maxAttempts: number | undefined;
+  maxDurationInSeconds: number | undefined;
+  replayedFromTaskRunFriendlyId: string | undefined;
+  annotations: unknown;
+  traceContext: unknown;
+  scheduleId: string | undefined;
+  batchId: string | undefined;
+  parentTaskRunFriendlyId: string | undefined;
+  rootTaskRunFriendlyId: string | undefined;
+
+  error?: { code: string; message: string };
+};
+
+export type ReadFallbackDeps = {
+  getBuffer?: () => MollifierBuffer | null;
+};
+
+function asString(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function asStringArray(value: unknown): string[] {
+  return Array.isArray(value) && value.every((v) => typeof v === "string")
+    ? (value as string[])
+    : [];
+}
+
+function asDate(value: unknown): Date | undefined {
+  const raw = asString(value);
+  if (!raw) return undefined;
+  const parsed = new Date(raw);
+  return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+}
+
+// Snapshot ids are written by engine.trigger as INTERNAL ids (cuids); the
+// SyntheticRun contract exposes friendlyIds. `RunId.toFriendlyId` is
+// already used for the synthetic run's own id (line 155); reuse it for
+// parent/root so consumers see the same shape as the PG path.
+function internalRunIdToFriendlyId(internalId: string | undefined): string | undefined {
+  if (!internalId) return undefined;
+  return RunId.toFriendlyId(internalId);
+}
+
+export async function findRunByIdWithMollifierFallback(
+  input: ReadFallbackInput,
+  deps: ReadFallbackDeps = {}
+): Promise<SyntheticRun | null> {
+  const buffer = (deps.getBuffer ?? getMollifierBuffer)();
+  if (!buffer) return null;
+
+  try {
+    const entry = await buffer.getEntry(input.runId);
+    if (!entry) return null;
+
+    if (entry.envId !== input.environmentId || entry.orgId !== input.organizationId) {
+      logger.warn("mollifier read-fallback auth mismatch", {
+        runId: input.runId,
+        callerEnvId: input.environmentId,
+        callerOrgId: input.organizationId,
+      });
+      return null;
+    }
+
+    const snapshot = deserialiseMollifierSnapshot(entry.payload);
+    // Parse via the canonical schema (`{ key: string, scope: "run" |
+    // "attempt" | "global" }`) rather than the legacy Array.isArray
+    // check. The SDK and Prisma both store this as an object; the array
+    // form never matches, so a buffered run's response previously fell
+    // back to the server-side hash in `getUserProvidedIdempotencyKey`
+    // instead of the customer-supplied key — diverging from how
+    // materialised runs render the same field.
+    const idempotencyKeyOptionsParsed = IdempotencyKeyOptionsSchema.safeParse(
+      snapshot.idempotencyKeyOptions
+    );
+    const idempotencyKeyOptions = idempotencyKeyOptionsParsed.success
+      ? idempotencyKeyOptionsParsed.data
+      : undefined;
+
+    const tags = asStringArray(snapshot.tags);
+    const environment =
+      snapshot.environment && typeof snapshot.environment === "object"
+        ? (snapshot.environment as Record<string, unknown>)
+        : undefined;
+
+    const cancelledAt = asDate(snapshot.cancelledAt);
+    const cancelReason = asString(snapshot.cancelReason);
+    let status: SyntheticRun["status"] = "QUEUED";
+    if (cancelledAt) {
+      status = "CANCELED";
+    } else if (entry.status === "FAILED") {
+      status = "FAILED";
+    }
+    const delayUntil = asDate(snapshot.delayUntil);
+
+    return {
+      id: RunId.fromFriendlyId(entry.runId),
+      friendlyId: entry.runId,
+      status,
+      cancelledAt,
+      cancelReason,
+      delayUntil,
+      taskIdentifier: asString(snapshot.taskIdentifier),
+      createdAt: entry.createdAt,
+
+      payload: snapshot.payload,
+      payloadType: asString(snapshot.payloadType),
+      metadata: snapshot.metadata,
+      metadataType: asString(snapshot.metadataType),
+      seedMetadata: asString(snapshot.seedMetadata),
+      seedMetadataType: asString(snapshot.seedMetadataType),
+
+      idempotencyKey: asString(snapshot.idempotencyKey),
+      idempotencyKeyExpiresAt: asDate(snapshot.idempotencyKeyExpiresAt),
+      idempotencyKeyOptions,
+      isTest: snapshot.isTest === true,
+      depth: typeof snapshot.depth === "number" ? snapshot.depth : 0,
+      ttl: asString(snapshot.ttl),
+      tags,
+      runTags: tags,
+      lockedToVersion: asString(snapshot.taskVersion),
+      resumeParentOnCompletion: snapshot.resumeParentOnCompletion === true,
+      parentTaskRunId: asString(snapshot.parentTaskRunId),
+
+      traceId: asString(snapshot.traceId),
+      spanId: asString(snapshot.spanId),
+      parentSpanId: asString(snapshot.parentSpanId),
+
+      runtimeEnvironmentId: asString(environment?.id) ?? entry.envId,
+      engine: "V2",
+      workerQueue: asString(snapshot.workerQueue),
+      region: asString(snapshot.region),
+      queue: asString(snapshot.queue),
+      concurrencyKey: asString(snapshot.concurrencyKey),
+      machinePreset: asString(snapshot.machine),
+      realtimeStreamsVersion: asString(snapshot.realtimeStreamsVersion),
+
+      maxAttempts: typeof snapshot.maxAttempts === "number" ? snapshot.maxAttempts : undefined,
+      maxDurationInSeconds:
+        typeof snapshot.maxDurationInSeconds === "number"
+          ? snapshot.maxDurationInSeconds
+          : undefined,
+      replayedFromTaskRunFriendlyId: asString(snapshot.replayedFromTaskRunFriendlyId),
+      annotations: snapshot.annotations,
+      traceContext: snapshot.traceContext,
+      scheduleId: asString(snapshot.scheduleId),
+      // The engine.trigger input embeds the batch as `{ id, index }` (see
+      // triggerTask.server.ts #buildEngineTriggerInput), not as a flat
+      // `batchId`. The nested `id` is the batch's internal cuid — the same
+      // value PG stores in `TaskRun.batchId` — so callers reconstruct the
+      // friendly id via `BatchId.toFriendlyId` exactly as the PG path does.
+      batchId: asString((snapshot.batch as { id?: unknown } | undefined)?.id),
+      // The snapshot only carries the INTERNAL parent/root ids
+      // (`parentTaskRunId` / `rootTaskRunId` — what engine.trigger consumes),
+      // not the friendlyIds the SyntheticRun contract expects. Convert
+      // internal → friendly here so consumers don't have to special-case
+      // the buffered path.
+      parentTaskRunFriendlyId: internalRunIdToFriendlyId(asString(snapshot.parentTaskRunId)),
+      rootTaskRunFriendlyId: internalRunIdToFriendlyId(asString(snapshot.rootTaskRunId)),
+
+      error: entry.lastError,
+    };
+  } catch (err) {
+    logger.error("mollifier read-fallback errored — fail-open to null", {
+      runId: input.runId,
+      err: err instanceof Error ? err.message : String(err),
+    });
+    return null;
+  }
+}

apps/webapp/app/v3/mollifier/resolveRunForMutation.server.ts

@@ -0,0 +1,73 @@
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import type { PrismaClientOrTransaction, PrismaReplicaClient } from "~/db.server";
+import { $replica as defaultReplica, prisma as defaultWriter } from "~/db.server";
+import { runStore } from "~/v3/runStore.server";
+import { getMollifierBuffer as defaultGetBuffer } from "./mollifierBuffer.server";
+
+// Discriminated-union resolver used by mutation routes' `findResource`.
+// The route builder treats a null return from `findResource` as a 404
+// BEFORE the action handler runs (`apiBuilder.server.ts:321`), so we
+// must check BOTH the PG canonical store and the mollifier buffer here
+// — otherwise a buffered run can't be cancelled / mutated even though
+// the underlying mutateWithFallback flow would handle it correctly.
+//
+// (Regression: before extracting this helper the cancel route had
+// `findResource: async () => null`, which made every cancel 404 before
+// the action ran. The helper makes the lookup unit-testable.)
+export type ResolvedRunForMutation =
+  | { source: "pg"; friendlyId: string }
+  | { source: "buffer"; friendlyId: string };
+
+export type ResolveRunForMutationDeps = {
+  prismaReplica?: PrismaReplicaClient;
+  prismaWriter?: PrismaClientOrTransaction;
+  getBuffer?: () => MollifierBuffer | null;
+};
+
+export async function resolveRunForMutation(input: {
+  runParam: string;
+  environmentId: string;
+  organizationId: string;
+  deps?: ResolveRunForMutationDeps;
+}): Promise<ResolvedRunForMutation | null> {
+  const replica = input.deps?.prismaReplica ?? defaultReplica;
+  const writer = input.deps?.prismaWriter ?? defaultWriter;
+  const getBuffer = input.deps?.getBuffer ?? defaultGetBuffer;
+
+  const pgRun = await runStore.findRun(
+    { friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId },
+    { select: { friendlyId: true } },
+    replica
+  );
+  if (pgRun) return { source: "pg", friendlyId: pgRun.friendlyId };
+
+  const buffer = getBuffer();
+
+  if (buffer) {
+    const entry = await buffer.getEntry(input.runParam);
+    if (entry && entry.envId === input.environmentId && entry.orgId === input.organizationId) {
+      return { source: "buffer", friendlyId: input.runParam };
+    }
+  }
+
+  // Replica + buffer both missed. Before declaring "not found" (which the
+  // route builder converts to a hard 404 *before* the action handler runs,
+  // so the downstream `mutateWithFallback` writer-recovery never gets a
+  // chance to fire), do one final probe against the writer. This catches
+  // two cases:
+  //   1. Replica lag on a freshly-created PG row.
+  //   2. A buffered run that materialised in the window between the
+  //      replica read and our buffer check (the entry was ack'd and the
+  //      hash is mid-grace-TTL but our getEntry returned null due to
+  //      lookup-by-friendlyId timing).
+  // Without this, the resolver returns null in degraded states that the
+  // downstream mutateWithFallback flow would otherwise handle correctly.
+  const writerRun = await runStore.findRun(
+    { friendlyId: input.runParam, runtimeEnvironmentId: input.environmentId },
+    { select: { friendlyId: true } },
+    writer
+  );
+  if (writerRun) return { source: "pg", friendlyId: writerRun.friendlyId };
+
+  return null;
+}

apps/webapp/app/v3/mollifier/syntheticApiResponses.server.ts

@@ -0,0 +1,73 @@
+import type { SyntheticRun } from "./readFallback.server";
+
+// Buffered runs have no execution data — the drainer hasn't materialised
+// the PG row and the worker hasn't started. The SDK-facing read routes
+// still need to return a span/trace shape that satisfies their response
+// schemas; these helpers build that minimal shape from the buffered
+// SyntheticRun.
+//
+// CANCELED and FAILED are terminal states: a FAILED buffered run is
+// errored (drainer exhausted retries or the gate rejected it) and must
+// not signal "still in progress." The flags below mirror
+// syntheticTrace.server.ts so the SDK contract stays consistent across
+// the three read paths (spans, trace, dashboard trace presenter).
+
+function deriveTerminalFlags(status: SyntheticRun["status"]): {
+  isError: boolean;
+  isPartial: boolean;
+  isCancelled: boolean;
+} {
+  const isCancelled = status === "CANCELED";
+  const isFailed = status === "FAILED";
+  return {
+    isError: isFailed,
+    isPartial: !isCancelled && !isFailed,
+    isCancelled,
+  };
+}
+
+// Body for GET /api/v1/runs/:runId/spans/:spanId when the run is buffered
+// and `:spanId` has already been verified against `buffered.spanId` by the
+// route. Pure function so the route layer just authenticates, resolves
+// the run, validates the spanId, and forwards the buffered run here.
+export function buildSyntheticSpanDetailBody(buffered: SyntheticRun) {
+  const flags = deriveTerminalFlags(buffered.status);
+  return {
+    spanId: buffered.spanId,
+    parentId: buffered.parentSpanId ?? null,
+    runId: buffered.friendlyId,
+    message: buffered.taskIdentifier ?? "",
+    ...flags,
+    level: "TRACE" as const,
+    startTime: buffered.createdAt,
+    durationMs: 0,
+  };
+}
+
+// Body for GET /api/v1/runs/:runId/trace when the run is buffered.
+// Returns the `{ trace: { traceId, rootSpan } }` envelope expected by the
+// SDK's RetrieveRunTraceResponseBody schema.
+export function buildSyntheticTraceBody(buffered: SyntheticRun) {
+  const flags = deriveTerminalFlags(buffered.status);
+  return {
+    trace: {
+      traceId: buffered.traceId ?? "",
+      rootSpan: {
+        id: buffered.spanId ?? "",
+        runId: buffered.friendlyId,
+        data: {
+          message: buffered.taskIdentifier ?? "",
+          taskSlug: buffered.taskIdentifier ?? undefined,
+          events: [] as unknown[],
+          startTime: buffered.createdAt,
+          duration: 0,
+          ...flags,
+          level: "TRACE" as const,
+          queueName: buffered.queue ?? undefined,
+          machinePreset: buffered.machinePreset ?? undefined,
+        },
+        children: [] as unknown[],
+      },
+    },
+  };
+}

apps/webapp/app/v3/mollifier/syntheticRedirectInfo.server.ts

@@ -0,0 +1,119 @@
+import type { MollifierBuffer } from "@trigger.dev/redis-worker";
+import type { PrismaClientOrTransaction } from "@trigger.dev/database";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import { getMollifierBuffer } from "./mollifierBuffer.server";
+// Use the webapp-side wrapper (not `deserialiseSnapshot` from
+// @trigger.dev/redis-worker directly) so this file shares a single
+// deserialisation path with readFallback.server.ts. The two are
+// behaviourally identical today (both wrap `JSON.parse`), but pinning
+// the shared helper keeps the two read-side modules from drifting if
+// snapshot encoding ever changes.
+import { deserialiseMollifierSnapshot } from "./mollifierSnapshot.server";
+
+// Validated subset of a mollifier snapshot — just the fields needed to
+// rebuild a canonical run-detail URL for a buffered run. Anything else
+// in the payload is ignored. `safeParse` against this schema replaces
+// the ad-hoc `as Record<string, unknown>` + `typeof === "string"` checks
+// that the redirect path used to do by hand; missing or wrong-typed
+// fields collapse into a single `parsed.success === false` branch.
+const BufferedSnapshotSchema = z.object({
+  spanId: z.string().optional(),
+  environment: z.object({
+    slug: z.string(),
+    project: z.object({ slug: z.string() }),
+    organization: z.object({ slug: z.string() }),
+  }),
+});
+
+export type BufferedRunRedirectInfo = {
+  organizationSlug: string;
+  projectSlug: string;
+  environmentSlug: string;
+  spanId: string | undefined;
+};
+
+export type FindBufferedRunRedirectInfoDeps = {
+  getBuffer?: () => MollifierBuffer | null;
+  prismaClient?: PrismaClientOrTransaction;
+};
+
+// Resolve the org/project/env slugs needed to build the canonical run-detail
+// URL for a buffered run. Used by the short-URL redirect routes
+// (`runs.$runParam`, `@.runs.$runParam`, `projects.v3.$projectRef.runs.$runParam`)
+// so a customer clicking the trigger-API-returned run link doesn't 404
+// during the buffered window.
+//
+// Authorisation: PG query confirms the requesting user belongs to the
+// organisation the buffer entry says owns the run. Without this check a
+// known runId would leak slugs.
+export async function findBufferedRunRedirectInfo(
+  args: {
+    runFriendlyId: string;
+    userId: string;
+    // Admin impersonation paths bypass org-membership; mirrors the existing
+    // PG-side admin route behaviour (`@.runs.$runParam` doesn't filter by
+    // org membership in the PG query either).
+    skipOrgMembershipCheck?: boolean;
+  },
+  deps: FindBufferedRunRedirectInfoDeps = {}
+): Promise<BufferedRunRedirectInfo | null> {
+  const buffer = (deps.getBuffer ?? getMollifierBuffer)();
+  const prismaClient = deps.prismaClient ?? prisma;
+  if (!buffer) return null;
+
+  let entry;
+  try {
+    entry = await buffer.getEntry(args.runFriendlyId);
+  } catch (err) {
+    logger.warn("buffered redirect: buffer.getEntry failed", {
+      runFriendlyId: args.runFriendlyId,
+      err: err instanceof Error ? err.message : String(err),
+    });
+    return null;
+  }
+  if (!entry) return null;
+
+  if (!args.skipOrgMembershipCheck) {
+    const member = await prismaClient.orgMember.findFirst({
+      where: { userId: args.userId, organizationId: entry.orgId },
+      select: { id: true },
+    });
+    if (!member) return null;
+  }
+
+  let raw: unknown;
+  try {
+    raw = deserialiseMollifierSnapshot(entry.payload);
+  } catch (err) {
+    logger.warn("buffered redirect: snapshot deserialise failed", {
+      runFriendlyId: args.runFriendlyId,
+      err: err instanceof Error ? err.message : String(err),
+    });
+    return null;
+  }
+
+  const parsed = BufferedSnapshotSchema.safeParse(raw);
+  if (!parsed.success) {
+    // Either the snapshot is from a different writer that doesn't carry
+    // environment slugs (in which case we genuinely can't build a URL)
+    // or a buffer-format drift snuck through. Log at debug; the caller
+    // 404s and the user sees the standard not-found page, not a 500.
+    logger.debug("buffered redirect: snapshot shape mismatch", {
+      runFriendlyId: args.runFriendlyId,
+      issues: parsed.error.issues.map((issue) => ({
+        path: issue.path.join("."),
+        code: issue.code,
+      })),
+    });
+    return null;
+  }
+
+  return {
+    organizationSlug: parsed.data.environment.organization.slug,
+    projectSlug: parsed.data.environment.project.slug,
+    environmentSlug: parsed.data.environment.slug,
+    spanId: parsed.data.spanId,
+  };
+}

apps/webapp/app/v3/mollifier/syntheticReplayTaskRun.server.ts

@@ -0,0 +1,51 @@
+import type { TaskRun } from "@trigger.dev/database";
+import type { SyntheticRun } from "./readFallback.server";
+
+export type SyntheticReplayTaskRun = TaskRun & {
+  project: { slug: string; organization: { slug: string } };
+  runtimeEnvironment: { slug: string };
+};
+
+// Adapt a buffered-run snapshot into the TaskRun-shaped input that
+// `ReplayTaskRunService.call` expects. ReplayTaskRunService builds the
+// new run's traceparent as `00-${existingTaskRun.traceId}-${existingTaskRun.spanId}-01`
+// without guarding for undefined, so a synthetic with missing traceId
+// or spanId (older snapshots — both fields are documented optional on
+// `SyntheticRun`) would produce `00-undefined-undefined-01`, an invalid
+// W3C traceparent that OTel silently drops, severing the replay's trace
+// link to the original run.
+//
+// Returns null when those fields are missing — the caller surfaces this
+// as "Run not found" so the customer retries once the drainer has
+// materialised the PG row, where traceId/spanId are guaranteed present.
+export function buildSyntheticReplayTaskRun(args: {
+  synthetic: SyntheticRun;
+  envRow: {
+    slug: string;
+    project: { slug: string; organization: { slug: string } };
+  };
+}): SyntheticReplayTaskRun | null {
+  const { synthetic, envRow } = args;
+  if (!synthetic.traceId || !synthetic.spanId) return null;
+  return {
+    // The double `as unknown as TaskRun` cast is load-bearing — a direct
+    // `synthetic as TaskRun` won't compile. `SyntheticRun` carries the
+    // subset of fields that `ReplayTaskRunService.call` actually reads
+    // (the contract is enumerated on the SyntheticRun type comment in
+    // readFallback.server.ts), but its shape is not structurally
+    // assignable to the full Prisma `TaskRun` row: optional vs required
+    // fields diverge, several PG columns (number, batchId variants,
+    // status enum widening) are deliberately absent or narrower on the
+    // synthetic. Routing it through `unknown` is the explicit "we know
+    // this is a subset, we've audited which fields are read" signal,
+    // and the traceId/spanId guard above prevents the only field
+    // ReplayTaskRunService consumes that would corrupt downstream
+    // behaviour (the OTel traceparent) when undefined.
+    ...(synthetic as unknown as TaskRun),
+    project: {
+      slug: envRow.project.slug,
+      organization: { slug: envRow.project.organization.slug },
+    },
+    runtimeEnvironment: { slug: envRow.slug },
+  };
+}

apps/webapp/app/v3/mollifier/syntheticRunHeader.server.ts

@@ -0,0 +1,75 @@
+import type { SyntheticRun } from "./readFallback.server";
+
+// Synthesise the run-detail page's `run` header shape (the NavBar +
+// status badge + Cancel-button gate) from a buffered run snapshot. The
+// shape matches `RunPresenter.getRun`'s `runData` — keep this in sync
+// when fields are added there.
+//
+// CANCELED and FAILED state is reflected back from
+// `SyntheticRun.cancelledAt` / `status` so terminal buffered runs show
+// the correct status in the NavBar + isFinished:true (which collapses
+// the Cancel button on the page header) before the drainer materialises
+// the PG row. This mirrors what `buildSyntheticSpanRun` does for the
+// right-side details panel — the SyntheticRun.cancelledAt contract
+// comment in readFallback.server.ts names this exact UI surface.
+//
+// FAILED status maps to `SYSTEM_FAILURE` to match the drainer's
+// non-retryable terminal path, which is what `buildSyntheticSpanRun`
+// uses too. Symmetric across the header + span-detail panel so an
+// admin doesn't see "Pending" + "FAILED" simultaneously on the same
+// run.
+export function buildSyntheticRunHeader(args: {
+  run: SyntheticRun;
+  environment: {
+    id: string;
+    organizationId: string;
+    type: "PRODUCTION" | "DEVELOPMENT" | "STAGING" | "PREVIEW";
+    slug: string;
+  };
+}) {
+  const { run, environment } = args;
+  const isCancelled = run.status === "CANCELED";
+  const isFailed = run.status === "FAILED";
+
+  return {
+    // `id` mirrors RunPresenter.getRun's runData (the PG path), which
+    // is the internal cuid — not the friendlyId. SyntheticRun.id is
+    // already the cuid (RunId.fromFriendlyId(entry.runId) in
+    // readFallback.server.ts) so the admin debug tooltip on the run
+    // detail page shows the same format for buffered + materialised
+    // runs.
+    id: run.id,
+    number: 1,
+    friendlyId: run.friendlyId,
+    traceId: run.traceId ?? "",
+    spanId: run.spanId ?? "",
+    status: isCancelled
+      ? ("CANCELED" as const)
+      : isFailed
+        ? ("SYSTEM_FAILURE" as const)
+        : ("PENDING" as const),
+    isFinished: isCancelled || isFailed,
+    startedAt: null,
+    // Symmetric with `buildSyntheticSpanRun` and the
+    // `ApiRetrieveRunPresenter` synth path. The run-detail route
+    // derives `isCompleted` from `completedAt !== null` and gates SSE
+    // live-reloading on it (`route.tsx:459`, `:551`); leaving
+    // `completedAt` null for FAILED would keep a terminal buffered run
+    // live-reloading forever. PG-resident SYSTEM_FAILURE rows always
+    // have completedAt set, so fall back to createdAt (the buffer
+    // entry has no separate failedAt — closest proxy for when the
+    // terminal state landed).
+    completedAt: run.cancelledAt ?? (isFailed ? run.createdAt : null),
+    logsDeletedAt: null,
+    rootTaskRun: null,
+    parentTaskRun: null,
+    environment: {
+      id: environment.id,
+      organizationId: environment.organizationId,
+      type: environment.type,
+      slug: environment.slug,
+      userId: undefined,
+      userName: undefined,
+    },
+  };
+}

apps/webapp/app/v3/mollifier/syntheticSpanRun.server.ts

@@ -0,0 +1,203 @@
+import { prettyPrintPacket, RunAnnotations } from "@trigger.dev/core/v3";
+import { getMaxDuration } from "@trigger.dev/core/v3/isomorphic";
+import {
+  extractIdempotencyKeyScope,
+  getUserProvidedIdempotencyKey,
+} from "@trigger.dev/core/v3/serverOnly";
+import { MachinePresetName } from "@trigger.dev/core/v3/schemas";
+import type { SpanRun } from "~/presenters/v3/SpanPresenter.server";
+import type { SyntheticRun } from "./readFallback.server";
+
+// `SyntheticRun.machinePreset` is sourced from the snapshot payload as
+// a plain string, but `SpanRun.machinePreset` is the narrowed enum.
+// Validate against the canonical enum so an unknown / stale preset
+// string collapses to undefined rather than fighting the type checker.
+function narrowMachinePreset(value: string | undefined): SpanRun["machinePreset"] {
+  if (value === undefined) return undefined;
+  const parsed = MachinePresetName.safeParse(value);
+  return parsed.success ? parsed.data : undefined;
+}
+
+// Synthesise a SpanRun-shaped object from a buffered run so the run-detail
+// page's right-side details panel renders identically to a PG-resident
+// run. The shape matches `SpanPresenter.getRun`'s return value;
+// buffered-irrelevant fields (output, attempts, schedule, session,
+// region, batch) are filled with sensible defaults, while terminal state
+// (CANCELED / FAILED) is reflected into `status`, `isFinished`, `isError`
+// and `error` so a finished buffered run does not render as PENDING.
+//
+// Pretty-printing for payload and metadata mirrors SpanPresenter so the
+// UI receives data in the same shape. Buffered runs cannot use the
+// `application/store` packet path (no R2 object yet) so we treat raw
+// snapshot fields as inline packets.
+export async function buildSyntheticSpanRun(args: {
+  run: SyntheticRun;
+  environment: {
+    id: string;
+    slug: string;
+    type: "PRODUCTION" | "DEVELOPMENT" | "STAGING" | "PREVIEW";
+  };
+}): Promise<SpanRun> {
+  const { run, environment } = args;
+
+  const payload =
+    typeof run.payload !== "undefined" && run.payload !== null
+      ? await prettyPrintPacket(run.payload, run.payloadType ?? undefined)
+      : undefined;
+
+  // Nullish check, not truthy — matches the payload branch above so an
+  // intentionally-empty packet (e.g. metadata: "") still gets handed to
+  // `prettyPrintPacket` and renders consistently. A truthy check would
+  // drop the empty-string case and the two paths would diverge.
+  const metadata =
+    typeof run.metadata !== "undefined" && run.metadata !== null
+      ? await prettyPrintPacket(run.metadata, run.metadataType, {
+          filteredKeys: ["$$streams", "$$streamsVersion", "$$streamsBaseUrl"],
+        })
+      : undefined;
+
+  const idempotencyShape = {
+    idempotencyKey: run.idempotencyKey ?? null,
+    idempotencyKeyExpiresAt: null,
+    idempotencyKeyOptions: run.idempotencyKeyOptions ?? null,
+  };
+
+  const idempotencyKey = getUserProvidedIdempotencyKey(idempotencyShape);
+  const idempotencyKeyScope = extractIdempotencyKeyScope(idempotencyShape);
+  const idempotencyKeyStatus: SpanRun["idempotencyKeyStatus"] = idempotencyKey
+    ? "active"
+    : idempotencyKeyScope
+      ? "inactive"
+      : undefined;
+
+  const taskKind = RunAnnotations.safeParse(run.annotations).data?.taskKind;
+  const isAgentRun = taskKind === "AGENT";
+  const isScheduled = taskKind === "SCHEDULED";
+
+  const queueName = run.queue ?? "task/";
+  const isCancelled = run.status === "CANCELED";
+  const isFailed = run.status === "FAILED";
+
+  // The run-detail panel derives terminal/error state from `status`,
+  // `isFinished` and `isError` (SpanPresenter.getRun -> isFinalRunStatus /
+  // isFailedRunStatus). Buffered FAILED runs surface as SYSTEM_FAILURE to
+  // match ApiRetrieveRunPresenter.bufferedStatusToTaskRunStatus; both
+  // CANCELED and SYSTEM_FAILURE are final run statuses, and SYSTEM_FAILURE
+  // is also a failed status.
+  const status: SpanRun["status"] = isCancelled
+    ? "CANCELED"
+    : isFailed
+      ? "SYSTEM_FAILURE"
+      : "PENDING";
+
+  // Mirror ApiRetrieveRunPresenter's STRING_ERROR synthesis so the panel
+  // shows why a buffered run failed instead of an empty error block.
+  const error: SpanRun["error"] =
+    isFailed && run.error
+      ? { type: "STRING_ERROR", raw: `${run.error.code}: ${run.error.message}` }
+      : undefined;
+
+  return {
+    id: run.id,
+    friendlyId: run.friendlyId,
+    status,
+    statusReason: isCancelled
+      ? (run.cancelReason ?? undefined)
+      : isFailed
+        ? (run.error?.message ?? undefined)
+        : undefined,
+    createdAt: run.createdAt,
+    startedAt: null,
+    executedAt: null,
+    updatedAt: run.cancelledAt ?? run.createdAt,
+    delayUntil: run.delayUntil ?? null,
+    expiredAt: null,
+    // Symmetric with `ApiRetrieveRunPresenter` — FAILED buffered runs
+    // must surface a non-null `completedAt` so the run-detail panel
+    // (and any caller checking `isFinished && completedAt`) doesn't
+    // render a finished run with no completion timestamp. PG-resident
+    // SYSTEM_FAILURE rows always have completedAt set; the buffer
+    // entry has no separate failedAt, so we fall back to createdAt
+    // as the best proxy for when the terminal state landed.
+    completedAt: run.cancelledAt ?? (isFailed ? run.createdAt : null),
+    logsDeletedAt: null,
+    ttl: run.ttl ?? null,
+    taskIdentifier: run.taskIdentifier ?? "",
+    version: undefined,
+    sdkVersion: undefined,
+    runtime: undefined,
+    runtimeVersion: undefined,
+    isTest: run.isTest,
+    replayedFromTaskRunFriendlyId: run.replayedFromTaskRunFriendlyId ?? null,
+    environmentId: environment.id,
+    idempotencyKey,
+    idempotencyKeyExpiresAt: null,
+    idempotencyKeyScope,
+    idempotencyKeyStatus,
+    debounce: null,
+    schedule: undefined,
+    queue: {
+      name: queueName,
+      isCustomQueue: !queueName.startsWith("task/"),
+      concurrencyKey: run.concurrencyKey ?? null,
+    },
+    tags: run.runTags,
+    baseCostInCents: 0,
+    costInCents: 0,
+    totalCostInCents: 0,
+    usageDurationMs: 0,
+    isFinished: isCancelled || isFailed,
+    isRunning: false,
+    isError: isFailed,
+    isAgentRun,
+    isScheduled,
+    payload,
+    payloadType: run.payloadType ?? "application/json",
+    output: undefined,
+    outputType: "application/json",
+    error,
+    // The snapshot only carries the root/parent friendly IDs, not the
+    // spanId or taskIdentifier that SpanPresenter sources from the joined
+    // PG rows. Emitting them with empty-string stubs renders a blank task
+    // name and a misleading `?span=` jump target, so we omit the
+    // relationships until the drainer materialises the row (a transient
+    // window). Top-level buffered runs have no relationships regardless.
+    relationships: {
+      root: undefined,
+      parent: undefined,
+    },
+    context: JSON.stringify(
+      {
+        task: {
+          id: run.taskIdentifier ?? "",
+        },
+        run: {
+          id: run.friendlyId,
+          createdAt: run.createdAt,
+          isTest: run.isTest,
+        },
+        environment: {
+          id: environment.id,
+          slug: environment.slug,
+          type: environment.type,
+        },
+      },
+      null,
+      2
+    ),
+    metadata,
+    maxDurationInSeconds: getMaxDuration(run.maxDurationInSeconds),
+    batch: undefined,
+    session: undefined,
+    engine: "V2",
+    region: null,
+    workerQueue: run.workerQueue ?? "",
+    traceId: run.traceId ?? "",
+    spanId: run.spanId ?? "",
+    isCached: false,
+    isBuffered: true,
+    machinePreset: narrowMachinePreset(run.machinePreset),
+    taskEventStore: "taskEvent",
+    externalTraceId: undefined,
+  };
+}

apps/webapp/app/v3/mollifier/syntheticTrace.server.ts

@@ -0,0 +1,90 @@
+import { millisecondsToNanoseconds } from "@trigger.dev/core/v3";
+import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
+import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
+import type { SpanSummary } from "~/v3/eventRepository/eventRepository.types";
+import type { SyntheticRun } from "./readFallback.server";
+
+// Build a single-span trace for a buffered run so the run-detail page
+// renders a meaningful timeline before the drainer materialises the
+// row. Mirrors the shape produced by `RunPresenter` when its trace
+// store lookup returns no spans, so the dashboard consumer treats the
+// buffered run identically to a freshly enqueued PG run that hasn't
+// emitted any events yet.
+export function buildSyntheticTraceForBufferedRun(run: SyntheticRun) {
+  const spanId = run.spanId ?? "";
+  const isCancelled = run.status === "CANCELED";
+  const isFailed = run.status === "FAILED";
+  const span: SpanSummary = {
+    id: spanId,
+    parentId: run.parentSpanId,
+    runId: run.friendlyId,
+    data: {
+      message: run.taskIdentifier ?? "Task",
+      style: { icon: "task", variant: "primary" },
+      events: [],
+      startTime: run.createdAt,
+      duration: 0,
+      isError: isFailed,
+      // CANCELED and FAILED are terminal; only a still-queued buffered run
+      // is partial. A partial failed span would otherwise render as
+      // "executing" forever in the timeline.
+      isPartial: !isCancelled && !isFailed,
+      isCancelled,
+      isDebug: false,
+      level: "TRACE",
+    },
+  };
+
+  const tree = createTreeFromFlatItems([span], spanId);
+  const treeRootStartTimeMs = tree?.data.startTime.getTime() ?? 0;
+  const totalDuration = Math.max(tree?.data.duration ?? 0, millisecondsToNanoseconds(1));
+
+  const events = tree
+    ? flattenTree(tree).map((n) => {
+        const offset = millisecondsToNanoseconds(n.data.startTime.getTime() - treeRootStartTimeMs);
+        // Mirror RunPresenter: raw span events stay server-side, only
+        // timelineEvents ship to the client.
+        const { events: spanEvents, ...data } = n.data;
+        return {
+          ...n,
+          data: {
+            ...data,
+            timelineEvents: createTimelineSpanEventsFromSpanEvents(
+              spanEvents,
+              false,
+              treeRootStartTimeMs
+            ),
+            duration: n.data.isPartial ? null : n.data.duration,
+            offset,
+            isRoot: n.id === spanId,
+            // Synthetic traces represent buffered/queued/canceled runs that
+            // haven't executed yet — they can't be agent runs. Keeping this
+            // field present (instead of omitting it) keeps the trace shape
+            // structurally identical to `RunPresenter`'s, which downstream
+            // consumers like the agent-icon check in runs/$runParam/route
+            // require to typecheck against the JsonifyObject union.
+            isAgentRun: false,
+          },
+        };
+      })
+    : [];
+
+  return {
+    // Matches RunPresenter's derivation: failed root span -> "failed",
+    // otherwise a terminal (non-partial) span -> "completed", else
+    // "executing". CANCELED is terminal-but-not-error, so "completed".
+    rootSpanStatus: (isFailed ? "failed" : isCancelled ? "completed" : "executing") as
+      | "executing"
+      | "completed"
+      | "failed",
+    events,
+    duration: totalDuration,
+    rootStartedAt: tree?.data.startTime,
+    startedAt: null,
+    queuedDuration: undefined,
+    overridesBySpanId: undefined,
+    linkedRunIdBySpanId: {} as Record<string, string>,
+    isTruncated: false,
+    missingAnchor: false,
+  };
+}

apps/webapp/app/v3/mollifierDrainerWorker.server.ts

@@ -0,0 +1,132 @@
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { signalsEmitter } from "~/services/signals.server";
+import {
+  getMollifierDrainer,
+  MollifierConfigurationError,
+} from "./mollifier/mollifierDrainer.server";
+import { startMollifierDrainingGauge } from "./mollifier/mollifierDrainingGauge.server";
+
+declare global {
+  // eslint-disable-next-line no-var
+  var __mollifierShutdownRegistered__: boolean | undefined;
+}
+
+/**
+ * Bootstraps the mollifier drainer.
+ *
+ * Two-step lifecycle:
+ *   1. Construct the drainer via the gated singleton in
+ *      `mollifierDrainer.server.ts`. That factory validates the
+ *      shutdown-timeout reconciliation against `GRACEFUL_SHUTDOWN_TIMEOUT`
+ *      and throws BEFORE returning if it's misconfigured; the returned
+ *      drainer is configured-but-stopped.
+ *   2. Register SIGTERM/SIGINT shutdown handlers, then call
+ *      `drainer.start()`. Doing this in the bootstrap (and not in the
+ *      factory) guarantees a signal landing during boot can never find
+ *      the polling loop running without a graceful-stop path.
+ *
+ * The drainer is intentionally NOT wired through `~/services/worker.server`
+ * — that file is the legacy ZodWorker / graphile-worker setup. The
+ * mollifier drainer is a custom polling loop over `MollifierBuffer`, not
+ * a graphile-worker job, so it gets its own lifecycle file alongside the
+ * redis-worker workers (`commonWorker`, `alertsWorker`,
+ * `batchTriggerWorker`).
+ *
+ * Gating order:
+ *   - `TRIGGER_MOLLIFIER_DRAINER_ENABLED !== "1"`  → early return. Unset defaults
+ *     to `TRIGGER_MOLLIFIER_ENABLED`, so single-container self-hosters still get
+ *     the drainer for free with one flag. In multi-replica deployments,
+ *     set this to "0" explicitly on every replica except the dedicated
+ *     drainer service so the polling loop doesn't race across replicas.
+ *   - `TRIGGER_MOLLIFIER_ENABLED !== "1"`  → `getMollifierDrainer()` returns null
+ *     and the bootstrap is a no-op. `TRIGGER_MOLLIFIER_ENABLED` remains the
+ *     master kill switch; the new flag only controls WHICH replicas
+ *     run the drainer when the system is on.
+ */
+export function initMollifierDrainerWorker(
+  opts: {
+    // Test seams. Production callers pass nothing; the defaults read the
+    // live env and resolve the live singleton. Tests inject overrides so
+    // the misconfig-rethrow / transient-swallow branches can be driven
+    // without manipulating module-level env state.
+    isEnabled?: () => boolean;
+    getDrainer?: typeof getMollifierDrainer;
+  } = {}
+): void {
+  const isEnabled = opts.isEnabled ?? (() => env.TRIGGER_MOLLIFIER_DRAINER_ENABLED === "1");
+  const getDrainer = opts.getDrainer ?? getMollifierDrainer;
+
+  if (!isEnabled()) {
+    return;
+  }
+
+  try {
+    const drainer = getDrainer();
+    if (drainer && !global.__mollifierShutdownRegistered__) {
+      // `__mollifierShutdownRegistered__` guards against double-register
+      // on dev hot-reloads (this bootstrap is called from
+      // entry.server.tsx, which Remix dev re-evaluates on every change).
+      // Same guard owns both the handler registration and the start()
+      // call so the two never get out of sync.
+      //
+      // Registers through `signalsEmitter` (the webapp-wide singleton in
+      // `~/services/signals.server`) rather than `process.once` directly:
+      //  - matches the codebase convention (runsReplicationInstance,
+      //    llmPricingRegistry, dynamicFlushScheduler etc. all listen on
+      //    the same emitter);
+      //  - `.on` (not `.once`) means a second SIGTERM still reaches us if
+      //    the orchestrator delivers more than one signal before SIGKILL;
+      //  - if SIGTERM lands in the gap between this listener attaching
+      //    and `drainer.start()` below, the first invocation no-ops
+      //    (stop() returns early because the drainer isn't running yet)
+      //    but the listener stays attached for a subsequent signal,
+      //    rather than being consumed by `once`.
+      const stopDrainer = () => {
+        drainer
+          .stop({ timeoutMs: env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS })
+          .catch((error) => {
+            logger.error("Failed to stop mollifier drainer", { error });
+          });
+      };
+      signalsEmitter.on("SIGTERM", stopDrainer);
+      signalsEmitter.on("SIGINT", stopDrainer);
+      global.__mollifierShutdownRegistered__ = true;
+      drainer.start();
+      // Spin up the observability-only gauge poller for the
+      // `mollifier:draining` ZSET cardinality. Colocated with the
+      // drainer because that's the loop creating the DRAINING entries
+      // — same pod, same Redis client lifecycle. Idempotent + unref'd
+      // so it's safe under dev hot-reload and doesn't block shutdown.
+      startMollifierDrainingGauge({
+        intervalMs: env.TRIGGER_MOLLIFIER_DRAINING_GAUGE_INTERVAL_MS,
+      });
+    }
+  } catch (error) {
+    // Deterministic misconfig (shutdown-timeout vs GRACEFUL_SHUTDOWN_TIMEOUT,
+    // missing buffer client) is a deploy-time mistake the operator must
+    // see immediately — rethrow so the process crashes, health checks
+    // fail, and the orchestrator rolls the deploy back. The drainer is currently
+    // monitoring-only and the silent-fallback was tempting, but later phases
+    // make the drainer the source of truth for diverted triggers, where a
+    // silently-disabled drainer means data loss. Better to fail loud now
+    // than retrofit later.
+    //
+    // We accept both `instanceof` and `error.name === ...` so Remix dev
+    // hot-reload (where the consumer can hold a stale class reference)
+    // still recognises the marker.
+    if (
+      error instanceof MollifierConfigurationError ||
+      (error instanceof Error && error.name === "MollifierConfigurationError")
+    ) {
+      logger.error("Mollifier drainer misconfiguration — failing loud", {
+        error: error.message,
+      });
+      throw error;
+    }
+    // Anything else (transient Redis blip, unexpected runtime error) is
+    // logged but kept non-fatal — the rest of the webapp shouldn't go
+    // down because the buffer's Redis cluster is briefly unreachable.
+    logger.error("Failed to initialise mollifier drainer", { error });
+  }
+}

apps/webapp/app/v3/mollifierStaleSweepWorker.server.ts

@@ -0,0 +1,76 @@
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { signalsEmitter } from "~/services/signals.server";
+import {
+  startStaleSweepInterval,
+  type StaleSweepIntervalHandle,
+} from "./mollifier/mollifierStaleSweep.server";
+import { MollifierStaleSweepState } from "./mollifier/mollifierStaleSweepState.server";
+
+declare global {
+  // eslint-disable-next-line no-var
+  var __mollifierStaleSweepRegistered__: boolean | undefined;
+  // eslint-disable-next-line no-var
+  var __mollifierStaleSweepHandle__: StaleSweepIntervalHandle | undefined;
+}
+
+/**
+ * Bootstraps the mollifier stale-entry sweep.
+ *
+ * Independent of the drainer — its purpose is to alert when entries are
+ * piling up despite the drainer being supposedly healthy, so it runs
+ * any time the mollifier itself is enabled (gated separately from
+ * `TRIGGER_MOLLIFIER_DRAINER_ENABLED`). The sweep is read-only: it
+ * counts and logs stale entries but does not remove or salvage them.
+ *
+ * The Remix dev server re-evaluates `entry.server.tsx` on every change,
+ * so the registration guard + handle cache make the bootstrap
+ * idempotent across hot reloads.
+ */
+export function initMollifierStaleSweepWorker(): void {
+  if (env.TRIGGER_MOLLIFIER_STALE_SWEEP_ENABLED !== "1") return;
+  if (global.__mollifierStaleSweepRegistered__) return;
+
+  logger.debug("Initializing mollifier stale-entry sweep", {
+    intervalMs: env.TRIGGER_MOLLIFIER_STALE_SWEEP_INTERVAL_MS,
+    staleThresholdMs: env.TRIGGER_MOLLIFIER_STALE_SWEEP_THRESHOLD_MS,
+  });
+
+  // Construct the sweep's durable-state Redis client using the same
+  // mollifier-Redis credentials as the buffer. Keeping this client
+  // separate from the buffer's own client keeps state ownership clean:
+  // the buffer abstracts queue/entry state, this abstracts sweep state.
+  const state = new MollifierStaleSweepState({
+    redisOptions: {
+      keyPrefix: "",
+      host: env.TRIGGER_MOLLIFIER_REDIS_HOST,
+      port: env.TRIGGER_MOLLIFIER_REDIS_PORT,
+      username: env.TRIGGER_MOLLIFIER_REDIS_USERNAME,
+      password: env.TRIGGER_MOLLIFIER_REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.TRIGGER_MOLLIFIER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+    maxRetriesPerRequest: env.TRIGGER_MOLLIFIER_REDIS_MAX_RETRIES_PER_REQUEST,
+  });
+
+  const handle = startStaleSweepInterval(
+    {
+      intervalMs: env.TRIGGER_MOLLIFIER_STALE_SWEEP_INTERVAL_MS,
+      staleThresholdMs: env.TRIGGER_MOLLIFIER_STALE_SWEEP_THRESHOLD_MS,
+      maxEntriesPerEnv: env.TRIGGER_MOLLIFIER_STALE_SWEEP_MAX_ENTRIES_PER_ENV,
+      maxOrgsPerPass: env.TRIGGER_MOLLIFIER_STALE_SWEEP_MAX_ORGS_PER_PASS,
+    },
+    { state }
+  );
+
+  // `handle.stop` is now async (it closes the Redis client). The signals
+  // emitter swallows promise rejections from listeners, so wrap it in a
+  // void-returning shim to be explicit about discarding the promise.
+  const onShutdown = (): void => {
+    void handle.stop();
+  };
+  signalsEmitter.on("SIGTERM", onShutdown);
+  signalsEmitter.on("SIGINT", onShutdown);
+  global.__mollifierStaleSweepRegistered__ = true;
+  global.__mollifierStaleSweepHandle__ = handle;
+}

apps/webapp/app/v3/objectStore.server.ts

@@ -0,0 +1,439 @@
+import { json } from "@remix-run/server-runtime";
+import { type IOPacket } from "@trigger.dev/core/v3";
+import { env } from "~/env.server";
+import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { ServiceValidationError } from "~/v3/services/common.server";
+import { singleton } from "~/utils/singleton";
+import {
+  normalizeObjectStoreLogicalKeyPathname,
+  ObjectStoreClient,
+  type ObjectStoreClientConfig,
+} from "./objectStoreClient.server";
+
+/**
+ * Parsed storage URI with optional protocol prefix
+ * @example { protocol: "s3", path: "run_abc/payload.json" }
+ * @example { protocol: undefined, path: "batch_123/item_0/payload.json" } // legacy, uses default
+ */
+export type ParsedStorageUri = {
+  protocol?: string;
+  path: string;
+};
+
+/**
+ * Parse a storage URI into protocol and path components
+ * @param uri Storage URI, optionally prefixed with protocol (e.g., "s3://path" or "path")
+ * @returns Parsed components { protocol?, path }
+ */
+export function parseStorageUri(uri: string): ParsedStorageUri {
+  const match = uri.match(/^([a-z0-9]+):\/\/(.+)$/);
+  if (match) {
+    return {
+      protocol: match[1],
+      path: match[2],
+    };
+  }
+  return {
+    protocol: undefined,
+    path: uri,
+  };
+}
+
+/**
+ * Format a storage URI with optional protocol prefix
+ * @param path Storage path
+ * @param protocol Optional protocol to prefix (e.g., "s3", "r2")
+ * @returns Formatted URI (e.g., "s3://path" or "path")
+ */
+export function formatStorageUri(path: string, protocol?: string): string {
+  if (protocol) {
+    return `${protocol}://${path}`;
+  }
+  return path;
+}
+
+export const INVALID_PACKET_STORAGE_PATH = "Invalid packet storage path";
+
+export type PacketPresignFailure = {
+  success: false;
+  error: string;
+  status?: number;
+};
+
+const PACKET_RELATIVE_PATH_BASE = "/__packet_base__";
+
+function throwInvalidPacketStoragePath(): never {
+  throw new ServiceValidationError(INVALID_PACKET_STORAGE_PATH, 400);
+}
+
+function assertRawPacketRelativePathSegments(path: string): void {
+  if (!path || path.includes("\\") || path.startsWith("/")) {
+    throwInvalidPacketStoragePath();
+  }
+
+  for (const segment of path.split("/")) {
+    if (segment === "" || segment === "." || segment === "..") {
+      throwInvalidPacketStoragePath();
+    }
+
+    if (segment.includes("%")) {
+      let decoded: string;
+      try {
+        decoded = decodeURIComponent(segment);
+      } catch {
+        throwInvalidPacketStoragePath();
+      }
+
+      if (decoded === "." || decoded === ".." || decoded.includes("/")) {
+        throwInvalidPacketStoragePath();
+      }
+    }
+  }
+}
+
+/**
+ * Normalize a packet-relative path using the same URL pathname resolution as object-store clients.
+ */
+export function normalizePacketRelativePath(path: string): string {
+  const url = new URL("https://trigger.invalid");
+  url.pathname = `${PACKET_RELATIVE_PATH_BASE}/${path.replace(/^\/+/, "")}`;
+
+  const prefix = `${PACKET_RELATIVE_PATH_BASE}/`;
+  if (!url.pathname.startsWith(prefix)) {
+    throwInvalidPacketStoragePath();
+  }
+
+  return url.pathname.slice(prefix.length);
+}
+
+/**
+ * Ensure a full logical object-store key resolves under the packet prefix after URL normalization.
+ */
+export function assertPacketObjectStoreKeyUnderPrefix(key: string, packetPrefix: string): void {
+  const normalizedKeyPath = normalizeObjectStoreLogicalKeyPathname(key);
+  const normalizedPrefixPath = normalizeObjectStoreLogicalKeyPathname(packetPrefix);
+
+  if (
+    normalizedKeyPath !== normalizedPrefixPath &&
+    !normalizedKeyPath.startsWith(`${normalizedPrefixPath}/`)
+  ) {
+    throwInvalidPacketStoragePath();
+  }
+}
+
+/**
+ * Validate a packet-relative path and return the canonical form used for object-store keys.
+ */
+export function resolveSafePacketRelativePath(path: string): string {
+  assertRawPacketRelativePathSegments(path);
+  const normalized = normalizePacketRelativePath(path);
+  assertRawPacketRelativePathSegments(normalized);
+  return normalized;
+}
+
+/**
+ * Reject path traversal and other unsafe packet-relative storage paths before
+ * building object-store keys or presigned URLs.
+ */
+export function assertSafePacketRelativePath(path: string): void {
+  resolveSafePacketRelativePath(path);
+}
+
+function buildPacketObjectStoreKey(
+  projectRef: string,
+  envSlug: string,
+  relativePath: string
+): string {
+  const safeRelativePath = resolveSafePacketRelativePath(relativePath);
+  const prefix = `packets/${projectRef}/${envSlug}`;
+  const key = `${prefix}/${safeRelativePath}`;
+  assertPacketObjectStoreKeyUnderPrefix(key, prefix);
+  return key;
+}
+
+/** JSON response for packet presign failures (400 client error vs 500 internal). */
+export function jsonPacketPresignFailure(failure: PacketPresignFailure) {
+  const status = failure.status ?? 500;
+  if (status === 400) {
+    return json({ error: failure.error }, { status: 400 });
+  }
+
+  return json({ error: `Failed to generate presigned URL: ${failure.error}` }, { status: 500 });
+}
+
+/**
+ * Get object storage configuration for a given protocol.
+ * Returns a config if baseUrl is set, even without explicit credentials —
+ * in that case the AWS credential chain (ECS task role, EC2 IMDS, etc.) is used,
+ * and OBJECT_STORE_BUCKET must also be set.
+ */
+function getObjectStoreConfig(protocol?: string): ObjectStoreClientConfig | undefined {
+  if (protocol) {
+    // Named provider (e.g., OBJECT_STORE_S3_*)
+    const prefix = `OBJECT_STORE_${protocol.toUpperCase()}_`;
+    const baseUrl = process.env[`${prefix}BASE_URL`];
+    if (!baseUrl) return undefined;
+
+    return {
+      baseUrl,
+      bucket: process.env[`${prefix}BUCKET`] || undefined,
+      accessKeyId: process.env[`${prefix}ACCESS_KEY_ID`] || undefined,
+      secretAccessKey: process.env[`${prefix}SECRET_ACCESS_KEY`] || undefined,
+      region: process.env[`${prefix}REGION`] || undefined,
+      service: process.env[`${prefix}SERVICE`] || undefined,
+    };
+  }
+
+  // Default provider (backward compatible)
+  if (!env.OBJECT_STORE_BASE_URL) {
+    return undefined;
+  }
+
+  return {
+    baseUrl: env.OBJECT_STORE_BASE_URL,
+    bucket: env.OBJECT_STORE_BUCKET || undefined,
+    accessKeyId: env.OBJECT_STORE_ACCESS_KEY_ID || undefined,
+    secretAccessKey: env.OBJECT_STORE_SECRET_ACCESS_KEY || undefined,
+    region: env.OBJECT_STORE_REGION || undefined,
+    service: env.OBJECT_STORE_SERVICE || undefined,
+  };
+}
+
+/**
+ * Object storage client registry. Maps protocol name to ObjectStoreClient singleton.
+ * ObjectStoreClient internally uses either aws4fetch (static credentials) or the
+ * AWS SDK S3Client (IAM credential chain), selected at creation time.
+ */
+const objectStoreClients = singleton(
+  "objectStoreClients",
+  () => new Map<string, ObjectStoreClient>()
+);
+
+function getObjectStoreClient(protocol?: string): ObjectStoreClient | undefined {
+  const config = getObjectStoreConfig(protocol);
+  if (!config) return undefined;
+
+  // Key includes baseUrl so that config changes (e.g. different containers in tests)
+  // always produce a fresh client while production usage (stable env) is effectively
+  // a per-protocol singleton.
+  const cacheKey = `${protocol ?? "default"}:${config.baseUrl}`;
+  if (objectStoreClients.has(cacheKey)) {
+    return objectStoreClients.get(cacheKey);
+  }
+
+  const client = ObjectStoreClient.create(config);
+  objectStoreClients.set(cacheKey, client);
+  return client;
+}
+
+export function hasObjectStoreClient(): boolean {
+  const defaultConfig = getObjectStoreConfig();
+  const protocolConfig = env.OBJECT_STORE_DEFAULT_PROTOCOL
+    ? getObjectStoreConfig(env.OBJECT_STORE_DEFAULT_PROTOCOL)
+    : undefined;
+  return !!(defaultConfig || protocolConfig);
+}
+
+export async function uploadPacketToObjectStore(
+  filename: string,
+  data: ReadableStream | string,
+  contentType: string,
+  environment: AuthenticatedEnvironment,
+  storageProtocol?: string
+): Promise<string> {
+  const protocol = storageProtocol || env.OBJECT_STORE_DEFAULT_PROTOCOL;
+  const client = getObjectStoreClient(protocol);
+
+  if (!client) {
+    throw new Error(`Object store is not configured for protocol: ${protocol || "default"}`);
+  }
+
+  const { path } = parseStorageUri(filename);
+  const safePath = resolveSafePacketRelativePath(path);
+  const key = buildPacketObjectStoreKey(
+    environment.project.externalRef,
+    environment.slug,
+    safePath
+  );
+
+  logger.debug("Uploading to object store", { key, protocol: protocol || "default" });
+
+  await client.putObject(key, data, contentType);
+
+  // Return canonical storage URI (path only in the key; protocol prefix applied here)
+  return formatStorageUri(safePath, protocol);
+}
+
+export async function downloadPacketFromObjectStore(
+  packet: IOPacket,
+  environment: AuthenticatedEnvironment
+): Promise<IOPacket> {
+  if (packet.dataType !== "application/store") {
+    return packet;
+  }
+
+  // There shouldn't be an offloaded packet with undefined data…
+  if (!packet.data) {
+    logger.error("Object store packet has undefined data", { packet, environment });
+    return {
+      dataType: "application/json",
+      data: undefined,
+    };
+  }
+
+  const { protocol, path } = parseStorageUri(packet.data);
+  const key = buildPacketObjectStoreKey(environment.project.externalRef, environment.slug, path);
+
+  const client = getObjectStoreClient(protocol);
+
+  if (!client) {
+    throw new Error(`Object store is not configured for protocol: ${protocol || "default"}`);
+  }
+
+  logger.debug("Downloading from object store", { key, protocol: protocol || "default" });
+
+  const data = await client.getObject(key);
+
+  return { data, dataType: "application/json" };
+}
+
+export type GeneratePacketPresignOptions = {
+  /**
+   * When true (v1 packet PUT only), unprefixed keys use the legacy default object store only.
+   * When false/omitted (v2 packet PUT), unprefixed keys also use OBJECT_STORE_DEFAULT_PROTOCOL.
+   * Ignored for GET — reads never infer protocol from env for unprefixed keys.
+   */
+  forceNoPrefix?: boolean;
+};
+
+/**
+ * Resolve object-store protocol for packet presigns.
+ * GET: never apply OBJECT_STORE_DEFAULT_PROTOCOL to unprefixed keys.
+ * PUT: optional forceNoPrefix for v1 legacy upload behavior.
+ */
+export function resolveStoreProtocolForPacketPresign(
+  filename: string,
+  method: "PUT" | "GET",
+  forceNoPrefix?: boolean
+): { path: string; storeProtocol: string | undefined } {
+  const { protocol: explicitProtocol, path } = parseStorageUri(filename);
+
+  if (method === "GET") {
+    return { path, storeProtocol: explicitProtocol };
+  }
+
+  if (explicitProtocol !== undefined) {
+    return { path, storeProtocol: explicitProtocol };
+  }
+
+  if (forceNoPrefix) {
+    return { path, storeProtocol: undefined };
+  }
+
+  return { path, storeProtocol: env.OBJECT_STORE_DEFAULT_PROTOCOL };
+}
+
+export async function generatePresignedRequest(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT",
+  options?: GeneratePacketPresignOptions
+): Promise<
+  | PacketPresignFailure
+  | {
+      success: true;
+      request: Request;
+      /** Canonical pointer for IOPacket.data (PUT only). */
+      storagePath?: string;
+    }
+> {
+  const { path, storeProtocol } = resolveStoreProtocolForPacketPresign(
+    filename,
+    method,
+    options?.forceNoPrefix
+  );
+
+  let safePath: string;
+  try {
+    safePath = resolveSafePacketRelativePath(path);
+  } catch (error) {
+    if (error instanceof ServiceValidationError) {
+      return {
+        success: false,
+        error: error.message,
+        status: error.status ?? 400,
+      };
+    }
+
+    throw error;
+  }
+
+  const config = getObjectStoreConfig(storeProtocol);
+  if (!config?.baseUrl) {
+    return {
+      success: false,
+      error: `Object store is not configured for protocol: ${storeProtocol || "default"}`,
+    };
+  }
+
+  const client = getObjectStoreClient(storeProtocol);
+  if (!client) {
+    return {
+      success: false,
+      error: `Object store is not configured for protocol: ${storeProtocol || "default"}`,
+    };
+  }
+
+  const key = buildPacketObjectStoreKey(projectRef, envSlug, safePath);
+
+  try {
+    const url = await client.presign(key, method, 300); // 5 minutes
+
+    logger.debug("Generated presigned URL", {
+      url,
+      projectRef,
+      envSlug,
+      filename,
+      protocol: storeProtocol || "default",
+    });
+
+    const storagePath = method === "PUT" ? formatStorageUri(safePath, storeProtocol) : undefined;
+
+    return {
+      success: true,
+      request: new Request(url, { method }),
+      storagePath,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export async function generatePresignedUrl(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT",
+  options?: GeneratePacketPresignOptions
+): Promise<PacketPresignFailure | { success: true; url: string; storagePath?: string }> {
+  const signed = await generatePresignedRequest(projectRef, envSlug, filename, method, options);
+
+  if (!signed.success) {
+    return {
+      success: false,
+      error: signed.error,
+      status: signed.status,
+    };
+  }
+
+  return {
+    success: true,
+    url: signed.request.url,
+    storagePath: signed.storagePath,
+  };
+}

apps/webapp/app/v3/objectStoreClient.server.ts

@@ -0,0 +1,218 @@
+import { AwsClient } from "aws4fetch";
+import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+
+/**
+ * Normalize a logical object-store key the same way Aws4FetchClient assigns URL pathnames.
+ * Decodes percent-escapes and resolves `.` / `..` segments before the request is signed.
+ */
+export function normalizeObjectStoreLogicalKeyPathname(logicalKey: string): string {
+  const url = new URL("https://trigger.invalid");
+  url.pathname = `/${logicalKey.replace(/^\/+/, "")}`;
+  return url.pathname;
+}
+
+interface IObjectStoreClient {
+  putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string>;
+  getObject(key: string): Promise<string>;
+  presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string>;
+}
+
+type Aws4FetchConfig = {
+  baseUrl: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  region?: string;
+  service?: string;
+};
+
+class Aws4FetchClient implements IObjectStoreClient {
+  private readonly awsClient: AwsClient;
+
+  constructor(private readonly config: Aws4FetchConfig) {
+    this.awsClient = new AwsClient({
+      accessKeyId: config.accessKeyId,
+      secretAccessKey: config.secretAccessKey,
+      region: config.region,
+      // We set the default value to "s3" in the schema to enhance interoperability with various
+      // S3-compatible services. Setting this env var to an empty string restores the old behaviour.
+      service: config.service || undefined,
+    });
+  }
+
+  private buildUrl(key: string): string {
+    const url = new URL(this.config.baseUrl);
+    url.pathname = normalizeObjectStoreLogicalKeyPathname(key);
+    return url.toString();
+  }
+
+  async putObject(
+    key: string,
+    body: ReadableStream | string,
+    contentType: string
+  ): Promise<string> {
+    const objectUrl = this.buildUrl(key);
+    const response = await this.awsClient.fetch(objectUrl, {
+      method: "PUT",
+      headers: { "Content-Type": contentType },
+      body,
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to upload to object store: ${response.statusText}`);
+    }
+    return objectUrl;
+  }
+
+  async getObject(key: string): Promise<string> {
+    const response = await this.awsClient.fetch(this.buildUrl(key));
+    if (!response.ok) {
+      throw new Error(`Failed to download from object store: ${response.statusText}`);
+    }
+    return response.text();
+  }
+
+  async presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string> {
+    const url = new URL(this.config.baseUrl);
+    url.pathname = normalizeObjectStoreLogicalKeyPathname(key);
+    url.searchParams.set("X-Amz-Expires", String(expiresIn));
+
+    const signed = await this.awsClient.sign(new Request(url, { method }), {
+      aws: { signQuery: true },
+    });
+    return signed.url;
+  }
+}
+
+type AwsSdkConfig = {
+  bucket: string;
+  baseUrl: string;
+  region?: string;
+};
+
+class AwsSdkClient implements IObjectStoreClient {
+  private readonly s3Client: S3Client;
+
+  constructor(private readonly config: AwsSdkConfig) {
+    this.s3Client = new S3Client({
+      endpoint: config.baseUrl,
+      forcePathStyle: true,
+      ...(config.region ? { region: config.region } : {}),
+    });
+  }
+
+  /**
+   * Callers use a single logical key (same as aws4fetch path: `bucket/object/...`).
+   * S3 APIs take Bucket + Key where Key must not repeat the bucket name.
+   */
+  private toS3ObjectKey(logicalKey: string): string {
+    const prefix = `${this.config.bucket}/`;
+    if (logicalKey.startsWith(prefix)) {
+      return logicalKey.slice(prefix.length);
+    }
+    return logicalKey;
+  }
+
+  private logicalObjectUrl(logicalKey: string): string {
+    const url = new URL(this.config.baseUrl);
+    url.pathname = normalizeObjectStoreLogicalKeyPathname(logicalKey);
+    return url.href;
+  }
+
+  async putObject(
+    key: string,
+    body: ReadableStream | string,
+    contentType: string
+  ): Promise<string> {
+    const s3Key = this.toS3ObjectKey(key);
+    await this.s3Client.send(
+      new PutObjectCommand({
+        Bucket: this.config.bucket,
+        Key: s3Key,
+        Body: body,
+        ContentType: contentType,
+      })
+    );
+    return this.logicalObjectUrl(key);
+  }
+
+  async getObject(key: string): Promise<string> {
+    const s3Key = this.toS3ObjectKey(key);
+    const response = await this.s3Client.send(
+      new GetObjectCommand({ Bucket: this.config.bucket, Key: s3Key })
+    );
+    if (!response.Body) {
+      throw new Error(`Empty response body from object store for key: ${key}`);
+    }
+    return response.Body.transformToString();
+  }
+
+  async presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string> {
+    const s3Key = this.toS3ObjectKey(key);
+    const command =
+      method === "PUT"
+        ? new PutObjectCommand({ Bucket: this.config.bucket, Key: s3Key })
+        : new GetObjectCommand({ Bucket: this.config.bucket, Key: s3Key });
+
+    return getSignedUrl(this.s3Client, command, { expiresIn });
+  }
+}
+
+export type ObjectStoreClientConfig = {
+  baseUrl: string;
+  bucket?: string;
+  accessKeyId?: string;
+  secretAccessKey?: string;
+  region?: string;
+  service?: string;
+};
+
+export class ObjectStoreClient implements IObjectStoreClient {
+  private constructor(
+    private readonly impl: IObjectStoreClient,
+    /** When set, logical keys may start with `${bucket}/…`; AwsSdkClient strips that prefix for S3 APIs. */
+    readonly bucket: string | undefined
+  ) {}
+
+  static create(config: ObjectStoreClientConfig): ObjectStoreClient {
+    if (config.accessKeyId && config.secretAccessKey) {
+      return new ObjectStoreClient(
+        new Aws4FetchClient({
+          baseUrl: config.baseUrl,
+          accessKeyId: config.accessKeyId,
+          secretAccessKey: config.secretAccessKey,
+          region: config.region,
+          service: config.service,
+        }),
+        config.bucket
+      );
+    }
+
+    // IAM credential chain — AWS SDK S3Client handles credential refresh automatically
+    if (!config.bucket) {
+      throw new Error(
+        "OBJECT_STORE_BUCKET is required when not using access key credentials (IAM mode)"
+      );
+    }
+
+    return new ObjectStoreClient(
+      new AwsSdkClient({
+        bucket: config.bucket,
+        baseUrl: config.baseUrl,
+        region: config.region,
+      }),
+      config.bucket
+    );
+  }
+
+  putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string> {
+    return this.impl.putObject(key, body, contentType);
+  }
+
+  getObject(key: string): Promise<string> {
+    return this.impl.getObject(key);
+  }
+
+  presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string> {
+    return this.impl.presign(key, method, expiresIn);
+  }
+}

apps/webapp/app/v3/otlpExporter.server.ts

@@ -0,0 +1,463 @@
+import type { Tracer } from "@opentelemetry/api";
+import { trace } from "@opentelemetry/api";
+import { SemanticInternalAttributes } from "@trigger.dev/core/v3";
+import type {
+  ExportLogsServiceRequest,
+  ExportMetricsServiceRequest,
+  ExportTraceServiceRequest,
+  ResourceMetrics,
+} from "@trigger.dev/otlp-importer";
+import {
+  ExportLogsServiceResponse,
+  ExportMetricsServiceResponse,
+  ExportTraceServiceResponse,
+} from "@trigger.dev/otlp-importer";
+import type { MetricsV1Input } from "@internal/clickhouse";
+import { getMeter, type Counter, type Histogram, type Meter } from "@internal/tracing";
+import { logger } from "~/services/logger.server";
+import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server";
+import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
+import { eventRepository } from "./eventRepository/eventRepository.server";
+import type { CreateEventInput, IEventRepository } from "./eventRepository/eventRepository.types";
+import { startSpan } from "./tracing.server";
+import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server";
+import { env } from "~/env.server";
+import { singleton } from "~/utils/singleton";
+import {
+  convertLogsToCreateableEvents,
+  convertMetricsToClickhouseRows,
+  convertSpansToCreateableEvents,
+  isBoolValue,
+  isStringValue,
+} from "./otlpTransform.server";
+import os from "node:os";
+import { getOtlpWorkerPool } from "./otlpWorkerPool.server";
+import {
+  llmPricingRegistry,
+  subscribeToPricingReload,
+  waitForLlmPricingReady,
+} from "./llmPricingRegistry.server";
+
+// When enabled, decode+convert+enrich run in a worker pool; the main thread keeps the single
+// consolidated insert path (batching/part-count unchanged). Off = today's single-thread path.
+export const otlpTransformWorkerPoolEnabled = env.OTEL_TRANSFORM_WORKER_POOL_ENABLED;
+
+// Always at least 1 worker: a 0/negative override must not silently disable the pool while the
+// flag is on (that would hang every raw-export request on an empty pool).
+const OTEL_TRANSFORM_WORKER_POOL_SIZE = Math.max(
+  1,
+  env.OTEL_TRANSFORM_WORKER_POOL_SIZE ?? (os.cpus()?.length ?? 2) - 2
+);
+
+type OTLPExporterConfig = {
+  clickhouseFactory: ClickhouseFactory;
+  verbose: boolean;
+  spanAttributeValueLengthLimit: number;
+  // Inject in tests; defaults to the global provider. Instruments are no-op when metrics are off.
+  meter?: Meter;
+};
+
+class OTLPExporter {
+  private _tracer: Tracer;
+  private readonly _clickhouseFactory: ClickhouseFactory;
+  private readonly _verbose: boolean;
+  private readonly _spanAttributeValueLengthLimit: number;
+  #pricingSubscribed = false;
+
+  private readonly _meter: Meter;
+  private readonly _ingestRequests: Counter;
+  private readonly _ingestBytes: Counter;
+  private readonly _ingestDuration: Histogram;
+  private readonly _eventsProduced: Counter;
+  private readonly _metricRowsProduced: Counter;
+
+  constructor(config: OTLPExporterConfig) {
+    this._tracer = trace.getTracer("otlp-exporter");
+    this._clickhouseFactory = config.clickhouseFactory;
+    this._verbose = config.verbose;
+    this._spanAttributeValueLengthLimit = config.spanAttributeValueLengthLimit;
+
+    this._meter = config.meter ?? getMeter("ingest");
+    this._ingestRequests = this._meter.createCounter("ingest.requests", {
+      description: "OTLP export calls received, by signal / path / outcome",
+      unit: "requests",
+    });
+    this._ingestBytes = this._meter.createCounter("ingest.bytes", {
+      description: "Compressed protobuf payload bytes received",
+      unit: "By",
+    });
+    this._ingestDuration = this._meter.createHistogram("ingest.duration", {
+      description: "End-to-end time to handle an OTLP export call",
+      unit: "ms",
+    });
+    this._eventsProduced = this._meter.createCounter("ingest.events.produced", {
+      description: "Task events produced from spans/logs after filter + convert",
+      unit: "events",
+    });
+    this._metricRowsProduced = this._meter.createCounter("ingest.metric_rows.produced", {
+      description: "ClickHouse metric rows produced from OTLP metrics",
+      unit: "rows",
+    });
+  }
+
+  // One helper for both the worker (mode="worker") and inline (mode="inline") paths. Called once
+  // per export call, so building the small attribute objects here is not a hot-path allocation.
+  #recordIngest(kind: string, mode: string, outcome: string, startedAt: number): void {
+    this._ingestRequests.add(1, { kind, mode, outcome });
+    this._ingestDuration.record(Date.now() - startedAt, { kind, mode });
+  }
+
+  async exportTraces(request: ExportTraceServiceRequest): Promise<ExportTraceServiceResponse> {
+    return await startSpan(this._tracer, "exportTraces", async (span) => {
+      const startedAt = Date.now();
+      try {
+        this.#logExportTracesVerbose(request);
+
+        const eventsWithStores = this.#filterResourceSpans(request.resourceSpans).flatMap(
+          (resourceSpan) => {
+            return convertSpansToCreateableEvents(
+              resourceSpan,
+              this._spanAttributeValueLengthLimit,
+              env.EVENT_REPOSITORY_DEFAULT_STORE
+            );
+          }
+        );
+
+        const eventCount = await this.#exportEvents(eventsWithStores);
+
+        span.setAttribute("event_count", eventCount);
+        this._eventsProduced.add(eventCount, { kind: "traces" });
+        this.#recordIngest("traces", "inline", "ok", startedAt);
+
+        return ExportTraceServiceResponse.create();
+      } catch (error) {
+        this.#recordIngest("traces", "inline", "error", startedAt);
+        throw error;
+      }
+    });
+  }
+
+  async exportMetrics(request: ExportMetricsServiceRequest): Promise<ExportMetricsServiceResponse> {
+    return await startSpan(this._tracer, "exportMetrics", async (span) => {
+      const startedAt = Date.now();
+      try {
+        const rows = this.#filterResourceMetrics(request.resourceMetrics).flatMap(
+          (resourceMetrics) =>
+            convertMetricsToClickhouseRows(resourceMetrics, this._spanAttributeValueLengthLimit)
+        );
+
+        span.setAttribute("metric_row_count", rows.length);
+
+        if (rows.length > 0) {
+          await this.#exportMetricRows(rows);
+        }
+
+        this._metricRowsProduced.add(rows.length, { kind: "metrics" });
+        this.#recordIngest("metrics", "inline", "ok", startedAt);
+
+        return ExportMetricsServiceResponse.create();
+      } catch (error) {
+        this.#recordIngest("metrics", "inline", "error", startedAt);
+        throw error;
+      }
+    });
+  }
+
+  async exportLogs(request: ExportLogsServiceRequest): Promise<ExportLogsServiceResponse> {
+    return await startSpan(this._tracer, "exportLogs", async (span) => {
+      const startedAt = Date.now();
+      try {
+        this.#logExportLogsVerbose(request);
+
+        const eventsWithStores = this.#filterResourceLogs(request.resourceLogs).flatMap(
+          (resourceLog) => {
+            return convertLogsToCreateableEvents(
+              resourceLog,
+              this._spanAttributeValueLengthLimit,
+              env.EVENT_REPOSITORY_DEFAULT_STORE
+            );
+          }
+        );
+
+        const eventCount = await this.#exportEvents(eventsWithStores);
+
+        span.setAttribute("event_count", eventCount);
+        this._eventsProduced.add(eventCount, { kind: "logs" });
+        this.#recordIngest("logs", "inline", "ok", startedAt);
+
+        return ExportLogsServiceResponse.create();
+      } catch (error) {
+        this.#recordIngest("logs", "inline", "error", startedAt);
+        throw error;
+      }
+    });
+  }
+
+  async exportTracesRaw(payload: Uint8Array): Promise<void> {
+    await this.#exportRawEvents("traces", payload);
+  }
+
+  async exportLogsRaw(payload: Uint8Array): Promise<void> {
+    await this.#exportRawEvents("logs", payload);
+  }
+
+  async exportMetricsRaw(payload: Uint8Array): Promise<void> {
+    await startSpan(this._tracer, "exportMetricsRaw", async (span) => {
+      const startedAt = Date.now();
+      this._ingestBytes.add(payload.byteLength, { kind: "metrics" });
+      try {
+        const pool = await this.#pool();
+        const { rows } = await pool.runTransform("metrics", payload, this.#transformConfig());
+        span.setAttribute("metric_row_count", rows.length);
+        if (rows.length > 0) {
+          await this.#exportMetricRows(rows);
+        }
+        this._metricRowsProduced.add(rows.length, { kind: "metrics" });
+        this.#recordIngest("metrics", "worker", "ok", startedAt);
+      } catch (error) {
+        this.#recordIngest("metrics", "worker", "error", startedAt);
+        throw error;
+      }
+    });
+  }
+
+  async #exportRawEvents(kind: "traces" | "logs", payload: Uint8Array): Promise<void> {
+    await startSpan(
+      this._tracer,
+      kind === "traces" ? "exportTracesRaw" : "exportLogsRaw",
+      async (span) => {
+        const startedAt = Date.now();
+        this._ingestBytes.add(payload.byteLength, { kind });
+        try {
+          const pool = await this.#pool();
+          const { eventsWithStores } = await pool.runTransform(
+            kind,
+            payload,
+            this.#transformConfig()
+          );
+          const eventCount = await this.#exportEvents(eventsWithStores, true);
+          span.setAttribute("event_count", eventCount);
+          this._eventsProduced.add(eventCount, { kind });
+          this.#recordIngest(kind, "worker", "ok", startedAt);
+        } catch (error) {
+          this.#recordIngest(kind, "worker", "error", startedAt);
+          throw error;
+        }
+      }
+    );
+  }
+
+  #transformConfig() {
+    return {
+      spanAttributeValueLengthLimit: this._spanAttributeValueLengthLimit,
+      defaultEventStore: env.EVENT_REPOSITORY_DEFAULT_STORE,
+    };
+  }
+
+  async #pool() {
+    await waitForLlmPricingReady();
+    const models =
+      llmPricingRegistry && llmPricingRegistry.isLoaded ? llmPricingRegistry.toSerializable() : [];
+    const pool = getOtlpWorkerPool(
+      OTEL_TRANSFORM_WORKER_POOL_SIZE,
+      models,
+      env.OTEL_TRANSFORM_WORKER_PATH,
+      this._meter
+    );
+    if (!this.#pricingSubscribed) {
+      this.#pricingSubscribed = true;
+      // Re-broadcast pricing to workers on every registry reload so their cost math stays fresh.
+      subscribeToPricingReload((updated) => pool.broadcastPricing(updated));
+    }
+    return pool;
+  }
+
+  async #exportEvents(
+    eventsWithStores: { events: Array<CreateEventInput>; taskEventStore: string }[],
+    alreadyEnriched = false
+  ) {
+    if (!alreadyEnriched) {
+      await waitForLlmPricingReady();
+    }
+
+    // Group by unique event repositories
+    const routeCache = new Map<string, { key: string; repository: IEventRepository }>();
+    const groups = new Map<string, { repository: IEventRepository; events: CreateEventInput[] }>();
+    for (const { events, taskEventStore } of eventsWithStores) {
+      for (const event of events) {
+        const routeKey = `${event.organizationId}\0${taskEventStore}`;
+        let resolved = routeCache.get(routeKey);
+        if (!resolved) {
+          // Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
+          // The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
+          if (taskEventStore !== "clickhouse" && taskEventStore !== "clickhouse_v2") {
+            // Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
+            // The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
+            resolved = { key: "postgres:default", repository: eventRepository };
+          } else {
+            resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
+              taskEventStore,
+              event.organizationId
+            );
+          }
+          routeCache.set(routeKey, resolved);
+        }
+
+        let group = groups.get(resolved.key);
+        if (!group) {
+          group = { repository: resolved.repository, events: [] };
+          groups.set(resolved.key, group);
+        }
+        group.events.push(event);
+      }
+    }
+
+    let eventCount = 0;
+
+    for (const [repoKey, { repository, events }] of groups) {
+      const enrichedEvents = alreadyEnriched ? events : enrichCreatableEvents(events);
+
+      this.#logEventsVerbose(enrichedEvents, `exportEvents ${repoKey}`);
+
+      eventCount += enrichedEvents.length;
+
+      repository.insertMany(enrichedEvents);
+    }
+
+    return eventCount;
+  }
+
+  async #exportMetricRows(rows: MetricsV1Input[]): Promise<void> {
+    const routeCache = new Map<string, { key: string; repository: IEventRepository }>();
+    const groups = new Map<string, { repository: IEventRepository; rows: MetricsV1Input[] }>();
+
+    for (const row of rows) {
+      const routeKey = row.organization_id;
+      let resolved = routeCache.get(routeKey);
+      if (!resolved) {
+        resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
+          "clickhouse_v2",
+          row.organization_id
+        );
+        routeCache.set(routeKey, resolved);
+      }
+
+      let group = groups.get(resolved.key);
+      if (!group) {
+        group = { repository: resolved.repository, rows: [] };
+        groups.set(resolved.key, group);
+      }
+      group.rows.push(row);
+    }
+
+    for (const [, { repository, rows: groupedRows }] of groups) {
+      repository.insertManyMetrics(groupedRows);
+    }
+  }
+
+  #logEventsVerbose(events: CreateEventInput[], prefix: string) {
+    if (!this._verbose) return;
+
+    events.forEach((event) => {
+      logger.debug(`Exporting ${prefix} event`, { event });
+    });
+  }
+
+  #logExportTracesVerbose(request: ExportTraceServiceRequest) {
+    if (!this._verbose) return;
+
+    logger.debug("Exporting traces", {
+      resourceSpans: request.resourceSpans.length,
+      totalSpans: request.resourceSpans.reduce(
+        (acc, resourceSpan) => acc + resourceSpan.scopeSpans.length,
+        0
+      ),
+    });
+  }
+
+  #logExportLogsVerbose(request: ExportLogsServiceRequest) {
+    if (!this._verbose) return;
+
+    logger.debug("Exporting logs", {
+      resourceLogs: request.resourceLogs.length,
+      totalLogs: request.resourceLogs.reduce(
+        (acc, resourceLog) =>
+          acc +
+          resourceLog.scopeLogs.reduce((acc, scopeLog) => acc + scopeLog.logRecords.length, 0),
+        0
+      ),
+    });
+  }
+
+  #filterResourceSpans(
+    resourceSpans: ExportTraceServiceRequest["resourceSpans"]
+  ): ExportTraceServiceRequest["resourceSpans"] {
+    return resourceSpans.filter((resourceSpan) => {
+      const triggerAttribute = resourceSpan.resource?.attributes.find(
+        (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+      );
+
+      const executionEnvironmentAttribute = resourceSpan.resource?.attributes.find(
+        (attribute) => attribute.key === SemanticInternalAttributes.EXECUTION_ENVIRONMENT
+      );
+
+      if (!triggerAttribute && !executionEnvironmentAttribute) {
+        logger.debug("Skipping resource span without trigger attribute", {
+          attributes: resourceSpan.resource?.attributes,
+          spans: resourceSpan.scopeSpans.flatMap((scopeSpan) => scopeSpan.spans),
+        });
+
+        return true; // go ahead and let this resource span through
+      }
+
+      const executionEnvironment = isStringValue(executionEnvironmentAttribute?.value)
+        ? executionEnvironmentAttribute.value.stringValue
+        : undefined;
+
+      if (executionEnvironment === "trigger") {
+        return true; // go ahead and let this resource span through
+      }
+
+      return isBoolValue(triggerAttribute?.value) ? triggerAttribute.value.boolValue : false;
+    });
+  }
+
+  #filterResourceLogs(
+    resourceLogs: ExportLogsServiceRequest["resourceLogs"]
+  ): ExportLogsServiceRequest["resourceLogs"] {
+    return resourceLogs.filter((resourceLog) => {
+      const attribute = resourceLog.resource?.attributes.find(
+        (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+      );
+
+      if (!attribute) return false;
+
+      return isBoolValue(attribute.value) ? attribute.value.boolValue : false;
+    });
+  }
+
+  #filterResourceMetrics(resourceMetrics: ResourceMetrics[]): ResourceMetrics[] {
+    return resourceMetrics.filter((rm) => {
+      const triggerAttribute = rm.resource?.attributes.find(
+        (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+      );
+
+      if (!triggerAttribute) return false;
+
+      return isBoolValue(triggerAttribute.value) ? triggerAttribute.value.boolValue : false;
+    });
+  }
+}
+
+export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter);
+
+async function initializeOTLPExporter() {
+  await clickhouseFactory.isReady();
+  return new OTLPExporter({
+    clickhouseFactory,
+    verbose: process.env.OTLP_EXPORTER_VERBOSE === "1",
+    spanAttributeValueLengthLimit: process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT
+      ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10)
+      : 8192,
+  });
+}

apps/webapp/app/v3/otlpTransform.server.ts

@@ -0,0 +1,930 @@
+// Worker-safe OTLP transform: no server singletons (env/clickhouse/repository/prisma).
+import { SemanticInternalAttributes } from "@trigger.dev/core/v3";
+import type {
+  AnyValue,
+  KeyValue,
+  ResourceLogs,
+  ResourceMetrics,
+  ResourceSpans,
+  Span,
+  Span_Event,
+} from "@trigger.dev/otlp-importer";
+import { SeverityNumber, Span_SpanKind, Status_StatusCode } from "@trigger.dev/otlp-importer";
+import type { MetricsV1Input } from "@internal/clickhouse";
+import { generateSpanId } from "./eventRepository/common.server";
+import type {
+  CreatableEventKind,
+  CreatableEventStatus,
+  CreateEventInput,
+} from "./eventRepository/eventRepository.types";
+
+// Filters mirror OTLPExporter's #filterResource* methods, minus the debug logging, so a
+// worker can run them without the server logger.
+export function filterResourceSpans(resourceSpans: ResourceSpans[]): ResourceSpans[] {
+  return resourceSpans.filter((resourceSpan) => {
+    const triggerAttribute = resourceSpan.resource?.attributes.find(
+      (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+    );
+    const executionEnvironmentAttribute = resourceSpan.resource?.attributes.find(
+      (attribute) => attribute.key === SemanticInternalAttributes.EXECUTION_ENVIRONMENT
+    );
+
+    if (!triggerAttribute && !executionEnvironmentAttribute) return true;
+
+    const executionEnvironment = isStringValue(executionEnvironmentAttribute?.value)
+      ? executionEnvironmentAttribute.value.stringValue
+      : undefined;
+    if (executionEnvironment === "trigger") return true;
+
+    return isBoolValue(triggerAttribute?.value) ? triggerAttribute.value.boolValue : false;
+  });
+}
+
+export function filterResourceLogs(resourceLogs: ResourceLogs[]): ResourceLogs[] {
+  return resourceLogs.filter((resourceLog) => {
+    const attribute = resourceLog.resource?.attributes.find(
+      (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+    );
+    if (!attribute) return false;
+    return isBoolValue(attribute.value) ? attribute.value.boolValue : false;
+  });
+}
+
+export function filterResourceMetrics(resourceMetrics: ResourceMetrics[]): ResourceMetrics[] {
+  return resourceMetrics.filter((rm) => {
+    const triggerAttribute = rm.resource?.attributes.find(
+      (attribute) => attribute.key === SemanticInternalAttributes.TRIGGER
+    );
+    if (!triggerAttribute) return false;
+    return isBoolValue(triggerAttribute.value) ? triggerAttribute.value.boolValue : false;
+  });
+}
+
+export function convertLogsToCreateableEvents(
+  resourceLog: ResourceLogs,
+  spanAttributeValueLengthLimit: number,
+  defaultEventStore: string
+): { events: Array<CreateEventInput>; taskEventStore: string } {
+  const resourceAttributes = resourceLog.resource?.attributes ?? [];
+
+  const resourceProperties = extractEventProperties(resourceAttributes);
+
+  const userDefinedResourceAttributes = truncateAttributes(
+    convertKeyValueItemsToMap(resourceAttributes ?? [], [], undefined, [
+      SemanticInternalAttributes.USAGE,
+      SemanticInternalAttributes.SPAN,
+      SemanticInternalAttributes.METADATA,
+      SemanticInternalAttributes.STYLE,
+      SemanticInternalAttributes.METRIC_EVENTS,
+      SemanticInternalAttributes.TRIGGER,
+      "process",
+      "sdk",
+      "service",
+      "ctx",
+      "cli",
+      "cloud",
+    ]),
+    spanAttributeValueLengthLimit
+  );
+
+  const taskEventStore =
+    extractStringAttribute(resourceAttributes, [SemanticInternalAttributes.TASK_EVENT_STORE]) ??
+    defaultEventStore;
+
+  const events = resourceLog.scopeLogs.flatMap((scopeLog) => {
+    return scopeLog.logRecords
+      .map((log) => {
+        const logLevel = logLevelToEventLevel(log.severityNumber);
+
+        if (!log.traceId || !log.spanId) {
+          return;
+        }
+
+        const logProperties = extractEventProperties(
+          log.attributes ?? [],
+          SemanticInternalAttributes.METADATA
+        );
+
+        const properties =
+          truncateAttributes(
+            convertKeyValueItemsToMap(log.attributes ?? [], [], undefined, [
+              SemanticInternalAttributes.USAGE,
+              SemanticInternalAttributes.SPAN,
+              SemanticInternalAttributes.METADATA,
+              SemanticInternalAttributes.STYLE,
+              SemanticInternalAttributes.METRIC_EVENTS,
+              SemanticInternalAttributes.TRIGGER,
+            ]),
+            spanAttributeValueLengthLimit
+          ) ?? {};
+
+        return {
+          traceId: binaryToHex(log.traceId),
+          spanId: generateSpanId(),
+          parentId: binaryToHex(log.spanId),
+          message: isStringValue(log.body)
+            ? log.body.stringValue.slice(0, 4096)
+            : `${log.severityText} log`,
+          isPartial: false,
+          kind: "INTERNAL" as const,
+          level: logLevelToEventLevel(log.severityNumber),
+          isError: logLevel === "ERROR",
+          status: logLevelToEventStatus(log.severityNumber),
+          startTime: log.timeUnixNano,
+          properties,
+          resourceProperties: userDefinedResourceAttributes,
+          style: convertKeyValueItemsToMap(
+            pickAttributes(log.attributes ?? [], SemanticInternalAttributes.STYLE),
+            []
+          ),
+          metadata: logProperties.metadata ?? resourceProperties.metadata ?? {},
+          environmentId:
+            logProperties.environmentId ?? resourceProperties.environmentId ?? "unknown",
+          environmentType: "DEVELOPMENT" as const, // We've deprecated this but we need to keep it for backwards compatibility
+          organizationId:
+            logProperties.organizationId ?? resourceProperties.organizationId ?? "unknown",
+          projectId: logProperties.projectId ?? resourceProperties.projectId ?? "unknown",
+          runId: logProperties.runId ?? resourceProperties.runId ?? "unknown",
+          taskSlug: logProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
+          machineId: logProperties.machineId ?? resourceProperties.machineId,
+          attemptNumber:
+            extractNumberAttribute(
+              log.attributes ?? [],
+              [SemanticInternalAttributes.METADATA, SemanticInternalAttributes.ATTEMPT_NUMBER].join(
+                "."
+              )
+            ) ?? resourceProperties.attemptNumber,
+        };
+      })
+      .filter(Boolean);
+  });
+
+  return { events, taskEventStore };
+}
+
+export function convertSpansToCreateableEvents(
+  resourceSpan: ResourceSpans,
+  spanAttributeValueLengthLimit: number,
+  defaultEventStore: string
+): { events: Array<CreateEventInput>; taskEventStore: string } {
+  const resourceAttributes = resourceSpan.resource?.attributes ?? [];
+
+  const resourceProperties = extractEventProperties(resourceAttributes);
+
+  const userDefinedResourceAttributes = truncateAttributes(
+    convertKeyValueItemsToMap(resourceAttributes ?? [], [], undefined, [
+      SemanticInternalAttributes.USAGE,
+      SemanticInternalAttributes.SPAN,
+      SemanticInternalAttributes.METADATA,
+      SemanticInternalAttributes.STYLE,
+      SemanticInternalAttributes.METRIC_EVENTS,
+      SemanticInternalAttributes.TRIGGER,
+      "process",
+      "sdk",
+      "service",
+      "ctx",
+      "cli",
+      "cloud",
+    ]),
+    spanAttributeValueLengthLimit
+  );
+
+  const taskEventStore =
+    extractStringAttribute(resourceAttributes, [SemanticInternalAttributes.TASK_EVENT_STORE]) ??
+    defaultEventStore;
+
+  const events = resourceSpan.scopeSpans.flatMap((scopeSpan) => {
+    return scopeSpan.spans
+      .map((span) => {
+        const isPartial = isPartialSpan(span);
+
+        if (!span.traceId || !span.spanId) {
+          return;
+        }
+
+        const spanProperties = extractEventProperties(
+          span.attributes ?? [],
+          SemanticInternalAttributes.METADATA
+        );
+
+        const runTags = extractArrayAttribute(
+          span.attributes ?? [],
+          SemanticInternalAttributes.RUN_TAGS
+        );
+
+        const properties =
+          truncateAttributes(
+            convertKeyValueItemsToMap(span.attributes ?? [], [], undefined, [
+              SemanticInternalAttributes.USAGE,
+              SemanticInternalAttributes.SPAN,
+              SemanticInternalAttributes.METADATA,
+              SemanticInternalAttributes.STYLE,
+              SemanticInternalAttributes.METRIC_EVENTS,
+              SemanticInternalAttributes.TRIGGER,
+            ]),
+            spanAttributeValueLengthLimit
+          ) ?? {};
+
+        return {
+          traceId: binaryToHex(span.traceId),
+          spanId: isPartial
+            ? extractStringAttribute(
+                span?.attributes ?? [],
+                SemanticInternalAttributes.SPAN_ID,
+                binaryToHex(span.spanId)
+              )
+            : binaryToHex(span.spanId),
+          parentId: binaryToHex(span.parentSpanId),
+          message: span.name,
+          isPartial,
+          isError: span.status?.code === Status_StatusCode.ERROR,
+          kind: spanKindToEventKind(span.kind),
+          level: "TRACE" as const,
+          status: spanStatusToEventStatus(span.status),
+          startTime: span.startTimeUnixNano,
+          events: spanEventsToEventEvents(span.events ?? []),
+          duration: span.endTimeUnixNano - span.startTimeUnixNano,
+          properties,
+          resourceProperties: userDefinedResourceAttributes,
+          style: convertKeyValueItemsToMap(
+            pickAttributes(span.attributes ?? [], SemanticInternalAttributes.STYLE),
+            []
+          ),
+          metadata: spanProperties.metadata ?? resourceProperties.metadata ?? {},
+          environmentId:
+            spanProperties.environmentId ?? resourceProperties.environmentId ?? "unknown",
+          environmentType: "DEVELOPMENT" as const,
+          organizationId:
+            spanProperties.organizationId ?? resourceProperties.organizationId ?? "unknown",
+          projectId: spanProperties.projectId ?? resourceProperties.projectId ?? "unknown",
+          runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown",
+          taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
+          machineId: spanProperties.machineId ?? resourceProperties.machineId,
+          runTags,
+          attemptNumber:
+            extractNumberAttribute(
+              span.attributes ?? [],
+              [SemanticInternalAttributes.METADATA, SemanticInternalAttributes.ATTEMPT_NUMBER].join(
+                "."
+              )
+            ) ?? resourceProperties.attemptNumber,
+        };
+      })
+      .filter(Boolean);
+  });
+
+  return { events, taskEventStore };
+}
+
+function floorToTenSecondBucket(timeUnixNano: bigint | number): string {
+  const epochMs = Number(BigInt(timeUnixNano) / BigInt(1_000_000));
+  const flooredMs = Math.floor(epochMs / 10_000) * 10_000;
+  const date = new Date(flooredMs);
+  // Format as ClickHouse DateTime: YYYY-MM-DD HH:MM:SS
+  return date
+    .toISOString()
+    .replace("T", " ")
+    .replace(/\.\d{3}Z$/, "");
+}
+
+export function convertMetricsToClickhouseRows(
+  resourceMetrics: ResourceMetrics,
+  spanAttributeValueLengthLimit: number
+): MetricsV1Input[] {
+  const resourceAttributes = resourceMetrics.resource?.attributes ?? [];
+  const resourceProperties = extractEventProperties(resourceAttributes);
+
+  const organizationId = resourceProperties.organizationId ?? "unknown";
+  const projectId = resourceProperties.projectId ?? "unknown";
+  const environmentId = resourceProperties.environmentId ?? "unknown";
+  const resourceCtx = {
+    taskSlug: resourceProperties.taskSlug,
+    runId: resourceProperties.runId,
+    attemptNumber: resourceProperties.attemptNumber,
+    machineId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.MACHINE_ID),
+    workerId: extractStringAttribute(resourceAttributes, SemanticInternalAttributes.WORKER_ID),
+    workerVersion: extractStringAttribute(
+      resourceAttributes,
+      SemanticInternalAttributes.WORKER_VERSION
+    ),
+  };
+
+  const rows: MetricsV1Input[] = [];
+
+  for (const scopeMetrics of resourceMetrics.scopeMetrics) {
+    for (const metric of scopeMetrics.metrics) {
+      const metricName = metric.name;
+
+      // Process gauge data points
+      if (metric.gauge) {
+        for (const dp of metric.gauge.dataPoints) {
+          const value: number =
+            dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "gauge",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+
+      // Process sum data points
+      if (metric.sum) {
+        for (const dp of metric.sum.dataPoints) {
+          const value: number =
+            dp.asDouble !== undefined ? dp.asDouble : dp.asInt !== undefined ? Number(dp.asInt) : 0;
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "sum",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+
+      // Process histogram data points
+      if (metric.histogram) {
+        for (const dp of metric.histogram.dataPoints) {
+          const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
+          const count = Number(dp.count);
+          const sum = dp.sum ?? 0;
+
+          rows.push({
+            organization_id: organizationId,
+            project_id: projectId,
+            environment_id: environmentId,
+            metric_name: metricName,
+            metric_type: "histogram",
+            metric_subject: resolved.machineId ?? "unknown",
+            bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
+            value: count > 0 ? sum / count : 0,
+            attributes: resolved.attributes,
+          });
+        }
+      }
+    }
+  }
+
+  return rows;
+}
+
+// Prefixes injected by TaskContextMetricExporter — these are extracted into
+// the nested `trigger` key and should not appear as top-level user attributes.
+const INTERNAL_METRIC_ATTRIBUTE_PREFIXES = ["ctx.", "worker."];
+
+interface ResourceContext {
+  taskSlug: string | undefined;
+  runId: string | undefined;
+  attemptNumber: number | undefined;
+  machineId: string | undefined;
+  workerId: string | undefined;
+  workerVersion: string | undefined;
+}
+
+function resolveDataPointContext(
+  dpAttributes: KeyValue[],
+  resourceCtx: ResourceContext
+): {
+  machineId: string | undefined;
+  attributes: Record<string, unknown>;
+} {
+  const runId =
+    resourceCtx.runId ?? extractStringAttribute(dpAttributes, SemanticInternalAttributes.RUN_ID);
+  const taskSlug =
+    resourceCtx.taskSlug ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.TASK_SLUG);
+  const attemptNumber =
+    resourceCtx.attemptNumber ??
+    extractNumberAttribute(dpAttributes, SemanticInternalAttributes.ATTEMPT_NUMBER);
+  const machineId =
+    resourceCtx.machineId ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.MACHINE_ID);
+  const workerId =
+    resourceCtx.workerId ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_ID);
+  const workerVersion =
+    resourceCtx.workerVersion ??
+    extractStringAttribute(dpAttributes, SemanticInternalAttributes.WORKER_VERSION);
+  const machineName = extractStringAttribute(
+    dpAttributes,
+    SemanticInternalAttributes.MACHINE_PRESET_NAME
+  );
+  const environmentType = extractStringAttribute(
+    dpAttributes,
+    SemanticInternalAttributes.ENVIRONMENT_TYPE
+  );
+
+  // Build the trigger context object with only defined values
+  const trigger: Record<string, string | number> = {};
+  if (runId) trigger.run_id = runId;
+  if (taskSlug) trigger.task_slug = taskSlug;
+  if (attemptNumber !== undefined) trigger.attempt_number = attemptNumber;
+  if (machineId) trigger.machine_id = machineId;
+  if (machineName) trigger.machine_name = machineName;
+  if (workerId) trigger.worker_id = workerId;
+  if (workerVersion) trigger.worker_version = workerVersion;
+  if (environmentType) trigger.environment_type = environmentType;
+
+  // Build user attributes, filtering out internal ctx/worker keys
+  const result: Record<string, unknown> = {};
+
+  if (Object.keys(trigger).length > 0) {
+    result.trigger = trigger;
+  }
+
+  for (const attr of dpAttributes) {
+    if (INTERNAL_METRIC_ATTRIBUTE_PREFIXES.some((prefix) => attr.key.startsWith(prefix))) {
+      continue;
+    }
+
+    if (isStringValue(attr.value)) {
+      result[attr.key] = attr.value.stringValue;
+    } else if (isIntValue(attr.value)) {
+      result[attr.key] = Number(attr.value.intValue);
+    } else if (isDoubleValue(attr.value)) {
+      result[attr.key] = attr.value.doubleValue;
+    } else if (isBoolValue(attr.value)) {
+      result[attr.key] = attr.value.boolValue;
+    }
+  }
+
+  return { machineId, attributes: result };
+}
+
+function extractEventProperties(attributes: KeyValue[], prefix?: string) {
+  return {
+    metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]),
+    environmentId: extractStringAttribute(attributes, [
+      prefix,
+      SemanticInternalAttributes.ENVIRONMENT_ID,
+    ]),
+    organizationId: extractStringAttribute(attributes, [
+      prefix,
+      SemanticInternalAttributes.ORGANIZATION_ID,
+    ]),
+    projectId: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.PROJECT_ID]),
+    runId: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.RUN_ID]),
+    attemptNumber: extractNumberAttribute(attributes, [
+      prefix,
+      SemanticInternalAttributes.ATTEMPT_NUMBER,
+    ]),
+    taskSlug: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.TASK_SLUG]),
+    machineId: extractStringAttribute(attributes, [prefix, SemanticInternalAttributes.MACHINE_ID]),
+  };
+}
+
+function pickAttributes(attributes: KeyValue[], prefix: string): KeyValue[] {
+  return attributes
+    .filter((attribute) => attribute.key.startsWith(prefix))
+    .map((attribute) => {
+      return {
+        key: attribute.key.replace(`${prefix}.`, ""),
+        value: attribute.value,
+      };
+    });
+}
+
+function convertKeyValueItemsToMap(
+  attributes: KeyValue[],
+  filteredKeys: string[] = [],
+  prefix?: string,
+  filteredPrefixes: string[] = []
+): Record<string, string | number | boolean | undefined> | undefined {
+  if (!attributes) return;
+  if (!attributes.length) return;
+
+  let filteredAttributes = attributes.filter((attribute) => !filteredKeys.includes(attribute.key));
+
+  if (!filteredAttributes.length) return;
+
+  if (filteredPrefixes.length) {
+    filteredAttributes = filteredAttributes.filter(
+      (attribute) => !filteredPrefixes.some((prefix) => attribute.key.startsWith(prefix))
+    );
+  }
+
+  if (!filteredAttributes.length) return;
+
+  const result = filteredAttributes.reduce(
+    (map: Record<string, string | number | boolean | undefined>, attribute) => {
+      map[`${prefix ? `${prefix}.` : ""}${attribute.key}`] = isStringValue(attribute.value)
+        ? attribute.value.stringValue
+        : isIntValue(attribute.value)
+          ? Number(attribute.value.intValue)
+          : isDoubleValue(attribute.value)
+            ? attribute.value.doubleValue
+            : isBoolValue(attribute.value)
+              ? attribute.value.boolValue
+              : isBytesValue(attribute.value)
+                ? binaryToHex(attribute.value.bytesValue)
+                : isArrayValue(attribute.value)
+                  ? serializeArrayValue(attribute.value.arrayValue!.values)
+                  : undefined;
+
+      return map;
+    },
+    {}
+  );
+
+  return result;
+}
+
+function convertSelectedKeyValueItemsToMap(
+  attributes: KeyValue[],
+  selectedPrefixes: string[] = [],
+  prefix?: string
+): Record<string, string | number | boolean | undefined> | undefined {
+  if (!attributes) return;
+  if (!attributes.length) return;
+
+  let selectedAttributes = attributes.filter((attribute) =>
+    selectedPrefixes.some((prefix) => attribute.key.startsWith(prefix))
+  );
+
+  if (!selectedAttributes.length) return;
+
+  const result = selectedAttributes.reduce(
+    (map: Record<string, string | number | boolean | undefined>, attribute) => {
+      map[`${prefix ? `${prefix}.` : ""}${attribute.key}`] = isStringValue(attribute.value)
+        ? attribute.value.stringValue
+        : isIntValue(attribute.value)
+          ? Number(attribute.value.intValue)
+          : isDoubleValue(attribute.value)
+            ? attribute.value.doubleValue
+            : isBoolValue(attribute.value)
+              ? attribute.value.boolValue
+              : isBytesValue(attribute.value)
+                ? binaryToHex(attribute.value.bytesValue)
+                : isArrayValue(attribute.value)
+                  ? serializeArrayValue(attribute.value.arrayValue!.values)
+                  : undefined;
+
+      return map;
+    },
+    {}
+  );
+
+  return result;
+}
+function spanEventsToEventEvents(events: Span_Event[]): CreateEventInput["events"] {
+  return events.map((event) => {
+    return {
+      name: event.name,
+      time: convertUnixNanoToDate(event.timeUnixNano),
+      properties: convertKeyValueItemsToMap(event.attributes ?? []),
+    };
+  });
+}
+
+function spanStatusToEventStatus(status: Span["status"]): CreatableEventStatus {
+  if (!status) return "UNSET";
+
+  switch (status.code) {
+    case Status_StatusCode.OK: {
+      return "OK";
+    }
+    case Status_StatusCode.ERROR: {
+      return "ERROR";
+    }
+    case Status_StatusCode.UNSET: {
+      return "UNSET";
+    }
+    default: {
+      return "UNSET";
+    }
+  }
+}
+
+function spanKindToEventKind(kind: Span["kind"]): CreatableEventKind {
+  switch (kind) {
+    case Span_SpanKind.CLIENT: {
+      return "CLIENT";
+    }
+    case Span_SpanKind.SERVER: {
+      return "SERVER";
+    }
+    case Span_SpanKind.CONSUMER: {
+      return "CONSUMER";
+    }
+    case Span_SpanKind.PRODUCER: {
+      return "PRODUCER";
+    }
+    default: {
+      return "INTERNAL";
+    }
+  }
+}
+
+function logLevelToEventLevel(level: SeverityNumber): CreateEventInput["level"] {
+  switch (level) {
+    case SeverityNumber.TRACE:
+    case SeverityNumber.TRACE2:
+    case SeverityNumber.TRACE3:
+    case SeverityNumber.TRACE4: {
+      return "TRACE";
+    }
+    case SeverityNumber.DEBUG:
+    case SeverityNumber.DEBUG2:
+    case SeverityNumber.DEBUG3:
+    case SeverityNumber.DEBUG4: {
+      return "DEBUG";
+    }
+    case SeverityNumber.INFO:
+    case SeverityNumber.INFO2:
+    case SeverityNumber.INFO3:
+    case SeverityNumber.INFO4: {
+      return "INFO";
+    }
+    case SeverityNumber.WARN:
+    case SeverityNumber.WARN2:
+    case SeverityNumber.WARN3:
+    case SeverityNumber.WARN4: {
+      return "WARN";
+    }
+    case SeverityNumber.ERROR:
+    case SeverityNumber.ERROR2:
+    case SeverityNumber.ERROR3:
+    case SeverityNumber.ERROR4: {
+      return "ERROR";
+    }
+    case SeverityNumber.FATAL:
+    case SeverityNumber.FATAL2:
+    case SeverityNumber.FATAL3:
+    case SeverityNumber.FATAL4: {
+      return "ERROR";
+    }
+    default: {
+      return "INFO";
+    }
+  }
+}
+
+function logLevelToEventStatus(level: SeverityNumber): CreatableEventStatus {
+  switch (level) {
+    case SeverityNumber.TRACE:
+    case SeverityNumber.TRACE2:
+    case SeverityNumber.TRACE3:
+    case SeverityNumber.TRACE4: {
+      return "OK";
+    }
+    case SeverityNumber.DEBUG:
+    case SeverityNumber.DEBUG2:
+    case SeverityNumber.DEBUG3:
+    case SeverityNumber.DEBUG4: {
+      return "OK";
+    }
+    case SeverityNumber.INFO:
+    case SeverityNumber.INFO2:
+    case SeverityNumber.INFO3:
+    case SeverityNumber.INFO4: {
+      return "OK";
+    }
+    case SeverityNumber.WARN:
+    case SeverityNumber.WARN2:
+    case SeverityNumber.WARN3:
+    case SeverityNumber.WARN4: {
+      return "OK";
+    }
+    case SeverityNumber.ERROR:
+    case SeverityNumber.ERROR2:
+    case SeverityNumber.ERROR3:
+    case SeverityNumber.ERROR4: {
+      return "ERROR";
+    }
+    case SeverityNumber.FATAL:
+    case SeverityNumber.FATAL2:
+    case SeverityNumber.FATAL3:
+    case SeverityNumber.FATAL4: {
+      return "ERROR";
+    }
+    default: {
+      return "OK";
+    }
+  }
+}
+
+function convertUnixNanoToDate(unixNano: bigint | number): Date {
+  return new Date(Number(BigInt(unixNano) / BigInt(1_000_000)));
+}
+
+function extractStringAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>
+): string | undefined;
+function extractStringAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>,
+  fallback: string
+): string;
+function extractStringAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>,
+  fallback?: string
+): string | undefined {
+  const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name;
+
+  const attribute = attributes.find((attribute) => attribute.key === key);
+
+  if (!attribute) return fallback;
+
+  return isStringValue(attribute?.value) ? attribute.value.stringValue : fallback;
+}
+
+function extractNumberAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>
+): number | undefined;
+function extractNumberAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>,
+  fallback: number
+): number;
+function extractNumberAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>,
+  fallback?: number
+): number | undefined {
+  const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name;
+
+  const attribute = attributes.find((attribute) => attribute.key === key);
+
+  if (!attribute) return fallback;
+
+  return isIntValue(attribute?.value) ? Number(attribute.value.intValue) : fallback;
+}
+
+function extractArrayAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>
+): string[] | undefined {
+  const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name;
+
+  const attribute = attributes.find((attribute) => attribute.key === key);
+
+  if (!attribute?.value?.arrayValue?.values) return undefined;
+
+  return attribute.value.arrayValue.values
+    .filter((v): v is { stringValue: string } => isStringValue(v))
+    .map((v) => v.stringValue);
+}
+
+function isPartialSpan(span: Span): boolean {
+  if (!span.attributes) return false;
+
+  const attribute = span.attributes.find(
+    (attribute) => attribute.key === SemanticInternalAttributes.SPAN_PARTIAL
+  );
+
+  if (!attribute) return false;
+
+  return isBoolValue(attribute.value) ? attribute.value.boolValue : false;
+}
+
+export function isBoolValue(value: AnyValue | undefined): value is { boolValue: boolean } {
+  if (!value) return false;
+
+  return typeof value.boolValue === "boolean";
+}
+
+export function isStringValue(value: AnyValue | undefined): value is { stringValue: string } {
+  if (!value) return false;
+
+  return typeof value.stringValue === "string";
+}
+
+function isIntValue(value: AnyValue | undefined): value is { intValue: bigint } {
+  if (!value) return false;
+
+  return typeof value.intValue === "number" || typeof value.intValue === "bigint";
+}
+
+function isDoubleValue(value: AnyValue | undefined): value is { doubleValue: number } {
+  if (!value) return false;
+
+  return typeof value.doubleValue === "number";
+}
+
+function isBytesValue(value: AnyValue | undefined): value is { bytesValue: Buffer } {
+  if (!value) return false;
+
+  return Buffer.isBuffer(value.bytesValue);
+}
+
+function isArrayValue(
+  value: AnyValue | undefined
+): value is { arrayValue: { values: AnyValue[] } } {
+  if (!value) return false;
+
+  return value.arrayValue != null && Array.isArray(value.arrayValue.values);
+}
+
+/**
+ * Serialize an OTEL array value into a JSON string.
+ * For arrays of strings, produces a JSON array: `["item1","item2"]`
+ * For mixed types, extracts primitives and serializes.
+ */
+function serializeArrayValue(values: AnyValue[]): string {
+  const items = values.map((v) => {
+    if (isStringValue(v)) return v.stringValue;
+    if (isIntValue(v)) return Number(v.intValue);
+    if (isDoubleValue(v)) return v.doubleValue;
+    if (isBoolValue(v)) return v.boolValue;
+    return null;
+  });
+
+  return JSON.stringify(items);
+}
+
+function binaryToHex(buffer: Buffer | string): string;
+function binaryToHex(buffer: Buffer | string | undefined): string | undefined;
+function binaryToHex(buffer: Buffer | string | undefined): string | undefined {
+  if (!buffer) return undefined;
+  if (typeof buffer === "string") return buffer;
+
+  return Buffer.from(Array.from(buffer)).toString("hex");
+}
+
+function truncateAttributes(
+  attributes: Record<string, string | number | boolean | undefined> | undefined,
+  maximumLength: number = 1024
+): Record<string, string | number | boolean | undefined> | undefined {
+  if (!attributes) return undefined;
+
+  const truncatedAttributes: Record<string, string | number | boolean | undefined> = {};
+
+  for (const [key, value] of Object.entries(attributes)) {
+    if (!key) continue;
+
+    if (typeof value === "string") {
+      truncatedAttributes[key] = truncateAndDetectUnpairedSurrogate(value, maximumLength);
+    } else {
+      truncatedAttributes[key] = value;
+    }
+  }
+
+  return truncatedAttributes;
+}
+
+function truncateAndDetectUnpairedSurrogate(str: string, maximumLength: number): string {
+  const truncatedString = smartTruncateString(str, maximumLength);
+
+  if (hasUnpairedSurrogateAtEnd(truncatedString)) {
+    return smartTruncateString(truncatedString, [...truncatedString].length - 1);
+  }
+
+  return truncatedString;
+}
+
+const ASCII_ONLY_REGEX = /^[\p{ASCII}]*$/u;
+
+function smartTruncateString(str: string, maximumLength: number): string {
+  if (!str) return "";
+  if (str.length <= maximumLength) return str;
+
+  const checkLength = Math.min(str.length, maximumLength * 2 + 2);
+
+  if (ASCII_ONLY_REGEX.test(str.slice(0, checkLength))) {
+    return str.slice(0, maximumLength);
+  }
+
+  return [...str.slice(0, checkLength)].slice(0, maximumLength).join("");
+}
+
+function hasUnpairedSurrogateAtEnd(str: string): boolean {
+  if (str.length === 0) return false;
+
+  const lastCode = str.charCodeAt(str.length - 1);
+
+  // Check if last character is an unpaired high surrogate
+  if (lastCode >= 0xd800 && lastCode <= 0xdbff) {
+    return true; // High surrogate at end = unpaired
+  }
+
+  // Check if last character is an unpaired low surrogate
+  if (lastCode >= 0xdc00 && lastCode <= 0xdfff) {
+    // Low surrogate is only valid if preceded by high surrogate
+    if (str.length === 1) return true; // Single low surrogate
+
+    const secondLastCode = str.charCodeAt(str.length - 2);
+    if (secondLastCode < 0xd800 || secondLastCode > 0xdbff) {
+      return true; // Low surrogate not preceded by high surrogate
+    }
+  }
+
+  return false;
+}

apps/webapp/app/v3/otlpTransformWorker.ts

@@ -0,0 +1,106 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { ModelPricingRegistry } from "@internal/llm-model-catalog";
+import type { LlmModelWithPricing } from "@internal/llm-model-catalog";
+import {
+  ExportLogsServiceRequest,
+  ExportMetricsServiceRequest,
+  ExportTraceServiceRequest,
+} from "@trigger.dev/otlp-importer";
+import {
+  convertLogsToCreateableEvents,
+  convertMetricsToClickhouseRows,
+  convertSpansToCreateableEvents,
+  filterResourceLogs,
+  filterResourceMetrics,
+  filterResourceSpans,
+} from "./otlpTransform.server";
+import { enrichCreatableEvents, setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server";
+
+type TransformTask = {
+  id: number;
+  kind: "traces" | "logs" | "metrics";
+  payload: Uint8Array;
+  spanAttributeValueLengthLimit: number;
+  defaultEventStore: string;
+};
+
+type PricingUpdate = { type: "pricing"; models: LlmModelWithPricing[] };
+
+// The main thread is the only DB reader; it broadcasts the compiled model rows here.
+const registry = new ModelPricingRegistry();
+setLlmPricingRegistry(registry);
+
+function applyPricing(models: LlmModelWithPricing[]) {
+  registry.loadFromModels(models);
+}
+
+if (Array.isArray(workerData?.pricingModels)) {
+  applyPricing(workerData.pricingModels);
+}
+
+function runTask(task: TransformTask) {
+  const bytes = new Uint8Array(task.payload);
+
+  if (task.kind === "traces") {
+    const request = ExportTraceServiceRequest.decode(bytes);
+    const eventsWithStores = filterResourceSpans(request.resourceSpans).flatMap((resourceSpan) =>
+      convertSpansToCreateableEvents(
+        resourceSpan,
+        task.spanAttributeValueLengthLimit,
+        task.defaultEventStore
+      )
+    );
+    for (const group of eventsWithStores) {
+      group.events = enrichCreatableEvents(group.events);
+    }
+    return { eventsWithStores };
+  }
+
+  if (task.kind === "logs") {
+    const request = ExportLogsServiceRequest.decode(bytes);
+    const eventsWithStores = filterResourceLogs(request.resourceLogs).flatMap((resourceLog) =>
+      convertLogsToCreateableEvents(
+        resourceLog,
+        task.spanAttributeValueLengthLimit,
+        task.defaultEventStore
+      )
+    );
+    for (const group of eventsWithStores) {
+      group.events = enrichCreatableEvents(group.events);
+    }
+    return { eventsWithStores };
+  }
+
+  const request = ExportMetricsServiceRequest.decode(bytes);
+  const rows = filterResourceMetrics(request.resourceMetrics).flatMap((resourceMetrics) =>
+    convertMetricsToClickhouseRows(resourceMetrics, task.spanAttributeValueLengthLimit)
+  );
+  return { rows };
+}
+
+if (!parentPort) {
+  throw new Error("otlpTransformWorker must be run as a worker thread");
+}
+
+parentPort.on("message", (message: TransformTask | PricingUpdate) => {
+  if ("type" in message && message.type === "pricing") {
+    applyPricing(message.models);
+    return;
+  }
+
+  const task = message as TransformTask;
+  try {
+    // The worker has no MeterProvider, so it can't emit metrics itself. It measures its own
+    // compute time (decode + convert + enrich) and hands it back for the main thread to record.
+    const startedAt = performance.now();
+    const result = runTask(task);
+    const computeMs = performance.now() - startedAt;
+    parentPort!.postMessage({ id: task.id, ok: true, result, computeMs });
+  } catch (error) {
+    parentPort!.postMessage({
+      id: task.id,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+});

apps/webapp/app/v3/otlpWorkerPool.server.ts

@@ -0,0 +1,349 @@
+import { Worker } from "node:worker_threads";
+import path from "node:path";
+import {
+  getMeter,
+  type Counter,
+  type Histogram,
+  type Meter,
+  type ObservableGauge,
+} from "@internal/tracing";
+import { logger } from "~/services/logger.server";
+import { signalsEmitter } from "~/services/signals.server";
+import { singleton } from "~/utils/singleton";
+
+export type TransformKind = "traces" | "logs" | "metrics";
+
+type TaskMessage = {
+  id: number;
+  kind: TransformKind;
+  payload: Uint8Array;
+  spanAttributeValueLengthLimit: number;
+  defaultEventStore: string;
+};
+
+type Task = {
+  message: TaskMessage;
+  transfer: ArrayBuffer[];
+  resolve: (r: any) => void;
+  reject: (e: Error) => void;
+  timer: NodeJS.Timeout;
+  worker?: Worker;
+  // Wall-clock stamp at enqueue; the task-duration histogram measures enqueue -> terminal state
+  // (queue wait + worker compute), so the gap from the worker-reported compute time is queue wait.
+  enqueuedAt: number;
+};
+
+type ReapReason = "error" | "exit" | "timeout";
+
+const TASK_TIMEOUT_MS = 30_000;
+const MAX_QUEUE_DEPTH = 2_000;
+const RESPAWN_BASE_MS = 500;
+const RESPAWN_MAX_MS = 30_000;
+const SHUTDOWN_DRAIN_MS = 5_000;
+
+// Hand-rolled worker_threads pool: one in-flight task per worker so CPU-bound transforms run
+// fully in parallel. The main thread stays the only DB reader and broadcasts pricing to workers.
+export class OtlpWorkerPool {
+  private readonly workers: Worker[] = [];
+  private readonly idle: Worker[] = [];
+  private readonly queue: number[] = [];
+  private readonly tasks = new Map<number, Task>();
+  private readonly busyByWorker = new Map<Worker, number>();
+  private nextId = 1;
+  private consecutiveFailures = 0;
+  private isShuttingDown = false;
+  private latestPricingModels: unknown[];
+
+  // Pre-allocated per-kind {kind} attribute objects so the per-task record path never allocates.
+  private readonly _kindAttrs: Record<TransformKind, { kind: TransformKind }> = {
+    traces: { kind: "traces" },
+    logs: { kind: "logs" },
+    metrics: { kind: "metrics" },
+  };
+  private _taskDurationHistogram?: Histogram;
+  private _computeDurationHistogram?: Histogram;
+  private _tasksCounter?: Counter;
+  private _respawnsCounter?: Counter;
+
+  constructor(
+    private readonly size: number,
+    private readonly workerPath: string,
+    pricingModels: unknown[],
+    meter?: Meter
+  ) {
+    this.latestPricingModels = pricingModels;
+    this.#setupOtelMetrics(meter);
+    for (let i = 0; i < size; i++) this.spawn();
+    logger.info("OtlpWorkerPool started", { size, workerPath });
+  }
+
+  #setupOtelMetrics(meterOverride: Meter | undefined): void {
+    const meter = meterOverride ?? getMeter("ingest");
+
+    this._taskDurationHistogram = meter.createHistogram("ingest.worker_pool.task.duration", {
+      description: "Enqueue-to-completion time for a transform task (queue wait + worker compute)",
+      unit: "ms",
+    });
+    this._computeDurationHistogram = meter.createHistogram("ingest.worker_pool.compute.duration", {
+      description: "Worker-reported compute time (decode + convert + enrich)",
+      unit: "ms",
+    });
+    this._tasksCounter = meter.createCounter("ingest.worker_pool.tasks", {
+      description: "Transform tasks by terminal outcome",
+      unit: "tasks",
+    });
+    this._respawnsCounter = meter.createCounter("ingest.worker_pool.respawns", {
+      description: "Worker respawns by reason",
+      unit: "respawns",
+    });
+
+    // Pull-based gauges: read at export time only, zero hot-path cost.
+    const queueDepthGauge: ObservableGauge = meter.createObservableGauge(
+      "ingest.worker_pool.queue_depth",
+      { description: "Tasks queued and awaiting a free worker", unit: "tasks" }
+    );
+    const workersGauge: ObservableGauge = meter.createObservableGauge(
+      "ingest.worker_pool.workers",
+      {
+        description: "Pool workers by state (alive workers, idle workers)",
+        unit: "workers",
+      }
+    );
+
+    meter.addBatchObservableCallback(
+      (result) => {
+        result.observe(queueDepthGauge, this.queue.length);
+        result.observe(workersGauge, this.workers.length, { state: "alive" });
+        result.observe(workersGauge, this.idle.length, { state: "idle" });
+      },
+      [queueDepthGauge, workersGauge]
+    );
+  }
+
+  #recordTaskEnd(task: Task, outcome: string, computeMs?: number): void {
+    this._taskDurationHistogram?.record(
+      Date.now() - task.enqueuedAt,
+      this._kindAttrs[task.message.kind]
+    );
+    this._tasksCounter?.add(1, { kind: task.message.kind, outcome });
+    if (computeMs !== undefined) {
+      this._computeDurationHistogram?.record(computeMs, this._kindAttrs[task.message.kind]);
+    }
+  }
+
+  private spawn() {
+    const worker = new Worker(this.workerPath, {
+      workerData: { pricingModels: this.latestPricingModels },
+    });
+
+    worker.on(
+      "message",
+      (msg: { id: number; ok: boolean; result?: any; error?: string; computeMs?: number }) => {
+        if (this.workers.indexOf(worker) === -1) return; // late message from an already-reaped worker
+        this.consecutiveFailures = 0;
+        this.busyByWorker.delete(worker);
+        const task = this.tasks.get(msg.id);
+        if (task) {
+          clearTimeout(task.timer);
+          this.tasks.delete(msg.id);
+          if (msg.ok) {
+            this.#recordTaskEnd(task, "ok", msg.computeMs);
+            task.resolve(msg.result);
+          } else {
+            this.#recordTaskEnd(task, "error", msg.computeMs);
+            task.reject(new Error(msg.error ?? "otlp worker error"));
+          }
+        }
+        this.release(worker);
+      }
+    );
+
+    worker.on("error", (error) => {
+      logger.error("OtlpWorkerPool worker error", { error: error.message });
+      this.reap(worker, error, "error");
+    });
+
+    worker.on("exit", (code) => {
+      // Any exit means this worker is gone, including a clean exit while it held a task; reap()
+      // no-ops if the worker was already removed (e.g. error fired first).
+      this.reap(worker, new Error(`otlp worker exited with code ${code}`), "exit");
+    });
+
+    this.workers.push(worker);
+    this.idle.push(worker);
+  }
+
+  // On crash/timeout: fail the worker's in-flight task (if still pending), drop the worker, and
+  // respawn with exponential backoff so a persistently failing worker can't tight-loop.
+  private reap(worker: Worker, error: Error, reason: ReapReason) {
+    const wi = this.workers.indexOf(worker);
+    if (wi === -1) return; // already reaped (error + exit can both fire for one crash)
+    this.workers.splice(wi, 1);
+
+    const ii = this.idle.indexOf(worker);
+    if (ii !== -1) this.idle.splice(ii, 1);
+
+    const inFlightId = this.busyByWorker.get(worker);
+    this.busyByWorker.delete(worker);
+    if (inFlightId !== undefined) {
+      const task = this.tasks.get(inFlightId);
+      if (task) {
+        clearTimeout(task.timer);
+        this.tasks.delete(inFlightId);
+        // A timed-out task already recorded its own end + was removed from the map, so this only
+        // fires for a crash that killed a task mid-flight.
+        this.#recordTaskEnd(task, "crash");
+        task.reject(error);
+      }
+    }
+
+    this._respawnsCounter?.add(1, { reason });
+    void worker.terminate().catch(() => {});
+    this.scheduleRespawn();
+  }
+
+  private scheduleRespawn() {
+    if (this.isShuttingDown) return;
+    if (this.workers.length >= this.size) return;
+    const delay = Math.min(RESPAWN_BASE_MS * 2 ** this.consecutiveFailures, RESPAWN_MAX_MS);
+    this.consecutiveFailures++;
+    setTimeout(() => {
+      if (this.isShuttingDown) return;
+      if (this.workers.length < this.size) this.spawn();
+      this.drain();
+    }, delay);
+  }
+
+  private release(worker: Worker) {
+    this.idle.push(worker);
+    this.drain();
+  }
+
+  private drain() {
+    while (this.queue.length > 0 && this.idle.length > 0) {
+      const worker = this.idle.pop()!;
+      const id = this.queue.shift()!;
+      const task = this.tasks.get(id);
+      if (!task) continue;
+      task.worker = worker;
+      this.busyByWorker.set(worker, id);
+      worker.postMessage(task.message, task.transfer);
+    }
+  }
+
+  private onTimeout(id: number) {
+    const task = this.tasks.get(id);
+    if (!task) return;
+    this.tasks.delete(id);
+    this.#recordTaskEnd(task, "timeout");
+    const err = new Error(`otlp worker task timed out after ${TASK_TIMEOUT_MS}ms`);
+    if (task.worker) {
+      // Dispatched to a stuck worker: reap it. The task is already removed, so reap won't
+      // double-reject.
+      this.reap(task.worker, err, "timeout");
+    } else {
+      const qi = this.queue.indexOf(id);
+      if (qi !== -1) this.queue.splice(qi, 1);
+    }
+    task.reject(err);
+  }
+
+  runTransform(
+    kind: TransformKind,
+    payload: Uint8Array,
+    config: { spanAttributeValueLengthLimit: number; defaultEventStore: string }
+  ): Promise<any> {
+    if (this.isShuttingDown) {
+      return Promise.reject(new Error("otlp worker pool is shutting down"));
+    }
+    if (this.queue.length >= MAX_QUEUE_DEPTH) {
+      this._tasksCounter?.add(1, { kind, outcome: "rejected" });
+      return Promise.reject(new Error("otlp worker pool queue is full"));
+    }
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => this.onTimeout(id), TASK_TIMEOUT_MS);
+      this.tasks.set(id, {
+        message: {
+          id,
+          kind,
+          payload,
+          spanAttributeValueLengthLimit: config.spanAttributeValueLengthLimit,
+          defaultEventStore: config.defaultEventStore,
+        },
+        // Zero-copy the payload into the worker; the request owns a fresh ArrayBuffer.
+        transfer: [payload.buffer as ArrayBuffer],
+        resolve,
+        reject,
+        timer,
+        enqueuedAt: Date.now(),
+      });
+      this.queue.push(id);
+      this.drain();
+    });
+  }
+
+  broadcastPricing(models: unknown[]) {
+    this.latestPricingModels = models;
+    for (const worker of this.workers) {
+      worker.postMessage({ type: "pricing", models });
+    }
+    logger.info("OtlpWorkerPool broadcast pricing", {
+      models: models.length,
+      workers: this.workers.length,
+    });
+  }
+
+  get queueDepth() {
+    return this.queue.length;
+  }
+
+  // Stop taking new work, let in-flight tasks finish (bounded), then terminate every worker.
+  // Terminated workers fire "exit", but reap() no-ops on an already-removed worker, and the
+  // isShuttingDown guard stops any pending respawn, so shutdown is quiet.
+  async shutdown(): Promise<void> {
+    if (this.isShuttingDown) return;
+    this.isShuttingDown = true;
+
+    logger.info("OtlpWorkerPool shutting down", {
+      workers: this.workers.length,
+      inFlight: this.tasks.size,
+    });
+
+    const deadline = Date.now() + SHUTDOWN_DRAIN_MS;
+    while (this.tasks.size > 0 && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+
+    const workers = this.workers.splice(0);
+    this.idle.length = 0;
+    this.queue.length = 0;
+    this.busyByWorker.clear();
+    // Reject anything that didn't drain within the deadline.
+    for (const [, task] of this.tasks) {
+      clearTimeout(task.timer);
+      task.reject(new Error("otlp worker pool shutting down"));
+    }
+    this.tasks.clear();
+    await Promise.all(workers.map((worker) => worker.terminate().catch(() => {})));
+  }
+}
+
+export function getOtlpWorkerPool(
+  size: number,
+  pricingModels: unknown[],
+  workerPath?: string,
+  meter?: Meter
+): OtlpWorkerPool {
+  // singleton() stores on globalThis so the pool (and its worker threads) survive Remix HMR in dev
+  // rather than leaking an orphaned pool + workers on every reload.
+  return singleton("otlpWorkerPool", () => {
+    const resolvedPath = workerPath ?? path.join(process.cwd(), "build", "otlpTransformWorker.cjs");
+    const created = new OtlpWorkerPool(size, resolvedPath, pricingModels, meter);
+    // Drain + terminate workers on shutdown so they aren't force-killed mid-task (which would
+    // churn respawns). The main thread stays the only DB writer, so inserts are unaffected.
+    signalsEmitter.on("SIGTERM", () => void created.shutdown());
+    signalsEmitter.on("SIGINT", () => void created.shutdown());
+    return created;
+  });
+}

apps/webapp/app/v3/outOfEntitlementError.server.ts

@@ -0,0 +1,6 @@
+export class OutOfEntitlementError extends Error {
+  constructor() {
+    super("You can't trigger a task because you have run out of credits.");
+    this.name = "OutOfEntitlementError";
+  }
+}

apps/webapp/app/v3/querySchemas.ts

@@ -0,0 +1,991 @@
+import { column, type BucketThreshold, type TableSchema } from "@internal/tsql";
+import { z } from "zod";
+import { autoFormatSQL } from "~/components/code/TSQLEditor";
+import { runFriendlyStatus, runStatusTitleFromStatus } from "~/components/runs/v3/TaskRunStatus";
+import { logger } from "~/services/logger.server";
+
+export const QueryScopeSchema = z.enum(["organization", "project", "environment"]);
+export type QueryScope = z.infer<typeof QueryScopeSchema>;
+
+/**
+ * Environment type values
+ */
+const ENVIRONMENT_TYPES = ["PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW"] as const;
+
+/**
+ * Machine preset values
+ */
+const MACHINE_PRESETS = [
+  "micro",
+  "small-1x",
+  "small-2x",
+  "medium-1x",
+  "medium-2x",
+  "large-1x",
+  "large-2x",
+] as const;
+
+/**
+ * Schema definition for the runs table (trigger_dev.task_runs_v2)
+ */
+export const runsSchema: TableSchema = {
+  name: "runs",
+  clickhouseName: "trigger_dev.task_runs_v2",
+  description: "Task runs - stores all task execution records",
+  timeConstraint: "triggered_at",
+  useFinal: true,
+  tenantColumns: {
+    organizationId: "organization_id",
+    projectId: "project_id",
+    environmentId: "environment_id",
+  },
+  requiredFilters: [{ column: "engine", value: "V2" }],
+  columns: {
+    run_id: {
+      name: "run_id",
+      clickhouseName: "friendly_id",
+      ...column("String", {
+        description:
+          "A unique ID for a run. They always start with `run_`, e.g., run_cm1a2b3c4d5e6f7g8h9i",
+        customRenderType: "runId",
+        example: "run_cm1a2b3c4d5e6f7g8h9i",
+        coreColumn: true,
+      }),
+    },
+    environment: {
+      name: "environment",
+      clickhouseName: "environment_id",
+      ...column("String", { description: "The environment slug", example: "prod" }),
+      fieldMapping: "environment",
+      customRenderType: "environment",
+    },
+    project: {
+      name: "project",
+      clickhouseName: "project_id",
+      ...column("String", {
+        description: "The project reference, they always start with `proj_`.",
+        example: "proj_howcnaxbfxdmwmxazktx",
+      }),
+      fieldMapping: "project",
+      customRenderType: "project",
+    },
+    environment_type: {
+      name: "environment_type",
+      ...column("LowCardinality(String)", {
+        description: "Environment type",
+        allowedValues: [...ENVIRONMENT_TYPES],
+        customRenderType: "environmentType",
+        example: "PRODUCTION",
+      }),
+    },
+    attempt_count: {
+      name: "attempt_count",
+      clickhouseName: "attempt",
+      ...column("UInt8", {
+        description: "Number of attempts (starts at 1)",
+        example: "1",
+        customRenderType: "number",
+      }),
+    },
+    status: {
+      name: "status",
+      ...column("LowCardinality(String)", {
+        description: "Run status",
+        allowedValues: [...runFriendlyStatus],
+        valueMap: runStatusTitleFromStatus,
+        customRenderType: "runStatus",
+        example: "Completed",
+        coreColumn: true,
+      }),
+    },
+    is_finished: {
+      name: "is_finished",
+      ...column("UInt8", {
+        description:
+          "Whether the run is finished. This includes failed and successful runs. (0 or 1)",
+        example: "0",
+      }),
+      expression:
+        "if(status IN ('COMPLETED_SUCCESSFULLY', 'COMPLETED_WITH_ERRORS', 'CANCELED', 'TIMED_OUT', 'CRASHED', 'SYSTEM_FAILURE', 'EXPIRED', 'PAUSED'), true, false)",
+    },
+
+    // Task & queue
+    task_identifier: {
+      name: "task_identifier",
+      ...column("String", {
+        description: "Task identifier/slug",
+        example: "my-background-task",
+        coreColumn: true,
+      }),
+    },
+    queue: {
+      name: "queue",
+      ...column("String", {
+        description: "Queue name",
+        example: "task/my-background-task",
+        customRenderType: "queue",
+      }),
+    },
+
+    batch_id: {
+      name: "batch_id",
+      ...column("String", {
+        description: "Batch ID (if part of a batch)",
+        example: "batch_5678efgh",
+        expression: "if(batch_id = '', NULL, 'batch_' || batch_id)",
+      }),
+      whereTransform: (value: string) => value.replace(/^batch_/, ""),
+    },
+
+    // Related runs
+    root_run_id: {
+      name: "root_run_id",
+      ...column("String", {
+        description: "Root run ID (for child runs)",
+        example: "run_cm1a2b3c4d5e6f7g8h9i",
+        customRenderType: "runId",
+        expression: "if(root_run_id = '', NULL, 'run_' || root_run_id)",
+      }),
+      whereTransform: (value: string) => value.replace(/^run_/, ""),
+    },
+    parent_run_id: {
+      name: "parent_run_id",
+      ...column("String", {
+        description: "Parent run ID (for child runs)",
+        example: "run_cm1a2b3c4d5e6f7g8h9i",
+        customRenderType: "runId",
+        expression: "if(parent_run_id = '', NULL, 'run_' || parent_run_id)",
+      }),
+      whereTransform: (value: string) => value.replace(/^run_/, ""),
+    },
+    depth: {
+      name: "depth",
+      ...column("UInt8", { description: "Nesting depth (0 for root runs)", example: "0" }),
+    },
+    is_root_run: {
+      name: "is_root_run",
+      ...column("UInt8", { description: "Whether this is a root run (0 or 1)", example: "0" }),
+      expression: "if(depth = 0, true, false)",
+    },
+    is_child_run: {
+      name: "is_child_run",
+      ...column("UInt8", { description: "Whether this is a child run (0 or 1)", example: "0" }),
+      expression: "if(depth > 0, true, false)",
+    },
+
+    idempotency_key: {
+      name: "idempotency_key",
+      clickhouseName: "idempotency_key_user",
+      ...column("String", {
+        description: "Idempotency key (available from 4.3.3)",
+        example: "user-123-action-456",
+      }),
+    },
+    idempotency_key_scope: {
+      name: "idempotency_key_scope",
+      ...column("String", {
+        description:
+          "The idempotency key scope determines whether a task should be considered unique within a parent run, a specific attempt, or globally. An empty value means there's no idempotency key set (available from 4.3.3).",
+        example: "run",
+        allowedValues: ["global", "run", "attempt"],
+      }),
+    },
+    region: {
+      name: "region",
+      clickhouseName: "region",
+      ...column("String", {
+        description: "Region",
+        example: "us-east-1",
+      }),
+      // No whereTransform: the expression drives WHERE too, so pre-region rows still match.
+      expression:
+        "multiIf(region != '', region, startsWith(worker_queue, 'cm'), NULL, worker_queue)",
+    },
+
+    // Timing
+    triggered_at: {
+      name: "triggered_at",
+      clickhouseName: "created_at",
+      ...column("DateTime64", {
+        description: "When the run was triggered.",
+        example: "2024-01-15 09:30:00.000",
+        coreColumn: true,
+      }),
+    },
+    queued_at: {
+      name: "queued_at",
+      ...column("Nullable(DateTime64)", {
+        description:
+          "When the run was added to the queue. This is normally the same time as the triggered_at time, unless a delay is passed in or it's a scheduled run.",
+        example: "2024-01-15 09:30:01.000",
+      }),
+    },
+    dequeued_at: {
+      name: "dequeued_at",
+      clickhouseName: "started_at",
+      ...column("Nullable(DateTime64)", {
+        description:
+          "When the run was dequeued for execution. This happens when there is available concurrency to execute your run.",
+        example: "2024-01-15 09:30:01.000",
+      }),
+    },
+    executed_at: {
+      name: "executed_at",
+      ...column("Nullable(DateTime64)", {
+        description: "When execution of the run began.",
+        example: "2024-01-15 09:30:01.500",
+      }),
+    },
+    completed_at: {
+      name: "completed_at",
+      ...column("Nullable(DateTime64)", {
+        description: "When the run completed",
+        example: "2024-01-15 09:30:05.000",
+      }),
+    },
+    delay_until: {
+      name: "delay_until",
+      ...column("Nullable(DateTime64)", {
+        description: "Delayed execution until this time",
+        example: "2024-01-15 10:00:00.000",
+      }),
+    },
+    has_delay: {
+      name: "has_delay",
+      ...column("UInt8", { description: "Whether the run had a delay passed in", example: "1" }),
+      expression: "if(isNotNull(delay_until), true, false)",
+    },
+    expired_at: {
+      name: "expired_at",
+      ...column("Nullable(DateTime64)", {
+        description:
+          'If there was a TTL on the run, this is when the run "expired". By default dev runs have a TTL of 10 minutes.',
+        example: "2024-01-15 09:35:00.000",
+      }),
+    },
+    ttl: {
+      name: "ttl",
+      clickhouseName: "expiration_ttl",
+      ...column("String", {
+        description: "The TTL string for expiration – by default dev runs have a TTL of '10m'.",
+        example: "10m",
+      }),
+    },
+
+    // Useful time periods
+    execution_duration: {
+      name: "execution_duration",
+      ...column("Nullable(Int64)", {
+        description:
+          "The time between starting to execute and completing. This includes any time spent waiting (it is not compute time, use `usage_duration` for that).",
+        customRenderType: "duration",
+        example: "4000",
+      }),
+      expression: "dateDiff('millisecond', executed_at, completed_at)",
+    },
+    total_duration: {
+      name: "total_duration",
+      ...column("Nullable(Int64)", {
+        description:
+          "The time between being triggered and completing (if it has). This includes any time spent waiting (it is not compute time, use `usage_duration` for that).",
+        customRenderType: "duration",
+        example: "4000",
+      }),
+      expression: "dateDiff('millisecond', created_at, completed_at)",
+    },
+    queued_duration: {
+      name: "queued_duration",
+      ...column("Nullable(Int64)", {
+        description:
+          "The time between being queued and dequeued. Remember you need enough available concurrency for runs to be dequeued and start executing.",
+        customRenderType: "duration",
+        example: "4000",
+      }),
+      expression: "dateDiff('millisecond', queued_at, started_at)",
+    },
+
+    // Cost & usage
+    usage_duration: {
+      name: "usage_duration",
+      clickhouseName: "usage_duration_ms",
+      ...column("UInt32", {
+        description: "Compute usage duration in milliseconds.",
+        customRenderType: "duration",
+        example: "3500",
+      }),
+    },
+    compute_cost: {
+      name: "compute_cost",
+      ...column("Float64", {
+        description: "Compute cost in dollars",
+        customRenderType: "costInDollars",
+        example: "0.000676",
+      }),
+      expression: "cost_in_cents / 100.0",
+    },
+    invocation_cost: {
+      name: "invocation_cost",
+      ...column("Float64", {
+        description: "Invocation cost in dollars – the cost to start a run.",
+        customRenderType: "costInDollars",
+        example: "0.000025",
+      }),
+      expression: "base_cost_in_cents / 100.0",
+    },
+    total_cost: {
+      name: "total_cost",
+      ...column("Float64", {
+        description: "Total cost in dollars (compute_cost + invocation_cost)",
+        customRenderType: "costInDollars",
+        example: "0.000701",
+      }),
+      expression: "(cost_in_cents + base_cost_in_cents) / 100.0",
+    },
+
+    // Output & error (JSON columns)
+    // For JSON columns, NULL checks are transformed to check for empty object '{}'
+    // So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'`
+    // textColumn uses the pre-materialized text columns for better performance
+    // dataPrefix handles the internal {"data": ...} wrapper transparently
+    output: {
+      name: "output",
+      ...column("JSON", {
+        description: "The data you returned from the task.",
+        example: '{"result": "success"}',
+      }),
+      nullValue: "'{}'", // Transform NULL checks to compare against empty object
+      textColumn: "output_text", // Use output_text for full JSON value queries
+      dataPrefix: "data", // Internal data is wrapped in {"data": ...}
+    },
+    error: {
+      name: "error",
+      ...column("JSON", {
+        description:
+          "If a run completely failed (after all attempts) then this error will be populated.",
+        example: '{"message": "Task failed"}',
+      }),
+      nullValue: "'{}'", // Transform NULL checks to compare against empty object
+      textColumn: "error_text", // Use error_text for full JSON value queries
+      dataPrefix: "data", // Internal data is wrapped in {"data": ...}
+    },
+
+    // Tags & versions
+    tags: {
+      name: "tags",
+      ...column("Array(String)", {
+        description: "Tags you have added to the run.",
+        customRenderType: "tags",
+        example: '["user:123", "priority:high"]',
+      }),
+    },
+    task_version: {
+      name: "task_version",
+      ...column("String", {
+        description: "The version of your code in reverse date format.",
+        example: "20240115.1",
+      }),
+    },
+    sdk_version: {
+      name: "sdk_version",
+      ...column("String", {
+        description: "The SDK package version for this run.",
+        example: "3.3.0",
+      }),
+    },
+    cli_version: {
+      name: "cli_version",
+      ...column("String", {
+        description: "The CLI package version for this run.",
+        example: "3.3.0",
+      }),
+    },
+    machine: {
+      name: "machine",
+      clickhouseName: "machine_preset",
+      ...column("LowCardinality(String)", {
+        description: "The machine that the run executed on.",
+        allowedValues: [...MACHINE_PRESETS],
+        customRenderType: "machine",
+        example: "small-1x",
+      }),
+    },
+    is_test: {
+      name: "is_test",
+      ...column("UInt8", { description: "Whether this is a test run (0 or 1)", example: "0" }),
+      expression: "if(is_test > 0, true, false)",
+    },
+    is_warm_start: {
+      name: "is_warm_start",
+      ...column("Nullable(UInt8)", {
+        description: "Whether this run used a warm start vs a cold start.",
+        example: "1",
+      }),
+    },
+    concurrency_key: {
+      name: "concurrency_key",
+      ...column("String", {
+        description: "The concurrency key you passed in when triggering the run.",
+        example: "user:1234567",
+      }),
+    },
+    max_duration: {
+      name: "max_duration",
+      clickhouseName: "max_duration_in_seconds",
+      ...column("Nullable(UInt32)", {
+        description:
+          "The maximum allowed compute duration for this run in seconds. If the run exceeds this duration, the run will fail with an error. Can be set on an individual task, in the trigger.config, or per-run when triggering.",
+        example: "300",
+        customRenderType: "durationSeconds",
+      }),
+    },
+    bulk_action_group_ids: {
+      name: "bulk_action_group_ids",
+      ...column("Array(String)", {
+        description: "Any bulk actions that operated on this run.",
+        example: '["bulk_12345678", "bulk_34567890"]',
+        whereTransform: (value: string) => {
+          logger.log(`WHERE TRANSFORM: ${value}`);
+          return value.replace(/^bulk_/, "");
+        },
+      }),
+    },
+  },
+};
+
+/**
+ * Schema definition for the metrics table (trigger_dev.metrics_v1)
+ */
+export const metricsSchema: TableSchema = {
+  name: "metrics",
+  clickhouseName: "trigger_dev.metrics_v1",
+  description: "Host and runtime metrics collected during task execution",
+  timeConstraint: "bucket_start",
+  tenantColumns: {
+    organizationId: "organization_id",
+    projectId: "project_id",
+    environmentId: "environment_id",
+  },
+  columns: {
+    environment: {
+      name: "environment",
+      clickhouseName: "environment_id",
+      ...column("String", { description: "The environment slug", example: "prod" }),
+      fieldMapping: "environment",
+      customRenderType: "environment",
+    },
+    project: {
+      name: "project",
+      clickhouseName: "project_id",
+      ...column("String", {
+        description: "The project reference, they always start with `proj_`.",
+        example: "proj_howcnaxbfxdmwmxazktx",
+      }),
+      fieldMapping: "project",
+      customRenderType: "project",
+    },
+    metric_name: {
+      name: "metric_name",
+      ...column("LowCardinality(String)", {
+        description: "The name of the metric (e.g. process.cpu.utilization, system.memory.usage)",
+        example: "process.cpu.utilization",
+        coreColumn: true,
+      }),
+    },
+    metric_type: {
+      name: "metric_type",
+      ...column("LowCardinality(String)", {
+        description: "The type of metric",
+        allowedValues: ["gauge", "sum", "histogram"],
+        example: "gauge",
+      }),
+    },
+    machine_id: {
+      name: "machine_id",
+      clickhouseName: "metric_subject",
+      ...column("String", {
+        description: "The machine ID that produced this metric",
+        example: "machine-abc123",
+      }),
+    },
+    bucket_start: {
+      name: "bucket_start",
+      ...column("DateTime", {
+        description: "The start of the 10-second aggregation bucket",
+        example: "2024-01-15 09:30:00",
+        coreColumn: true,
+      }),
+    },
+    metric_value: {
+      name: "metric_value",
+      clickhouseName: "value",
+      ...column("Float64", {
+        description: "The metric value",
+        example: "0.75",
+        coreColumn: true,
+      }),
+    },
+
+    // Attributes (JSON column for user-defined and system attributes)
+    attributes: {
+      name: "attributes",
+      ...column("JSON", {
+        description: "JSON attributes attached to the metric data point.",
+        example: '{"region": "us-east-1"}',
+      }),
+    },
+
+    // Trigger context columns (from attributes.trigger.* JSON subpaths)
+    run_id: {
+      name: "run_id",
+      ...column("String", {
+        description: "The run ID associated with this metric",
+        customRenderType: "runId",
+        example: "run_cm1a2b3c4d5e6f7g8h9i",
+        coreColumn: true,
+      }),
+      expression: "attributes.trigger.run_id",
+    },
+    task_identifier: {
+      name: "task_identifier",
+      ...column("String", {
+        description: "Task identifier/slug",
+        example: "my-background-task",
+        coreColumn: true,
+      }),
+      expression: "attributes.trigger.task_slug",
+    },
+    attempt_number: {
+      name: "attempt_number",
+      ...column("UInt64", {
+        description: "The attempt number for this metric",
+        example: "1",
+      }),
+      expression: "attributes.trigger.attempt_number",
+    },
+    machine_name: {
+      name: "machine_name",
+      ...column("String", {
+        description: "The machine preset used for execution",
+        allowedValues: [...MACHINE_PRESETS],
+        example: "small-1x",
+      }),
+      expression: "attributes.trigger.machine_name",
+    },
+    environment_type: {
+      name: "environment_type",
+      ...column("String", {
+        description: "Environment type",
+        allowedValues: [...ENVIRONMENT_TYPES],
+        customRenderType: "environmentType",
+        example: "PRODUCTION",
+      }),
+      expression: "attributes.trigger.environment_type",
+    },
+    worker_id: {
+      name: "worker_id",
+      ...column("String", {
+        description: "The worker ID that produced this metric",
+        customRenderType: "deploymentId",
+        example: "deployment_cm1a2b3c4d5e",
+      }),
+      expression: "attributes.trigger.worker_id",
+    },
+    worker_version: {
+      name: "worker_version",
+      ...column("String", {
+        description: "The worker version that produced this metric",
+        example: "20240115.1",
+      }),
+      expression: "attributes.trigger.worker_version",
+    },
+  },
+  timeBucketThresholds: [
+    // Metrics are pre-aggregated into 10-second buckets, so 10s is the most granular interval.
+    // All thresholds are shifted coarser compared to the runs table defaults.
+    { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } },
+    { maxRangeSeconds: 12 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
+    { maxRangeSeconds: 2 * 24 * 60 * 60, interval: { value: 5, unit: "MINUTE" } },
+    { maxRangeSeconds: 7 * 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } },
+    { maxRangeSeconds: 30 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } },
+    { maxRangeSeconds: 90 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } },
+    { maxRangeSeconds: 180 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } },
+    { maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } },
+  ] satisfies BucketThreshold[],
+};
+
+/**
+ * All available schemas for the query editor
+ */
+/**
+ * Schema definition for the llm_metrics table (trigger_dev.llm_metrics_v1)
+ */
+export const llmMetricsSchema: TableSchema = {
+  name: "llm_metrics",
+  clickhouseName: "trigger_dev.llm_metrics_v1",
+  description: "LLM metrics: token usage, cost, performance, and behavior from GenAI spans",
+  timeConstraint: "start_time",
+  tenantColumns: {
+    organizationId: "organization_id",
+    projectId: "project_id",
+    environmentId: "environment_id",
+  },
+  columns: {
+    environment: {
+      name: "environment",
+      clickhouseName: "environment_id",
+      ...column("String", { description: "The environment slug", example: "prod" }),
+      fieldMapping: "environment",
+      customRenderType: "environment",
+    },
+    project: {
+      name: "project",
+      clickhouseName: "project_id",
+      ...column("String", {
+        description: "The project reference, they always start with `proj_`.",
+        example: "proj_howcnaxbfxdmwmxazktx",
+      }),
+      fieldMapping: "project",
+      customRenderType: "project",
+    },
+    run_id: {
+      name: "run_id",
+      ...column("String", {
+        description: "The run ID",
+        customRenderType: "runId",
+        coreColumn: true,
+      }),
+    },
+    trace_id: {
+      name: "trace_id",
+      ...column("String", {
+        description: "The trace ID",
+      }),
+    },
+    span_id: {
+      name: "span_id",
+      ...column("String", {
+        description: "The span ID",
+      }),
+    },
+    task_identifier: {
+      name: "task_identifier",
+      ...column("LowCardinality(String)", {
+        description: "The task identifier",
+        example: "my-task",
+        coreColumn: true,
+      }),
+    },
+    gen_ai_system: {
+      name: "gen_ai_system",
+      ...column("LowCardinality(String)", {
+        description: "AI provider (e.g. openai, anthropic)",
+        example: "openai",
+        coreColumn: true,
+      }),
+    },
+    request_model: {
+      name: "request_model",
+      ...column("String", {
+        description: "The model name requested",
+        example: "gpt-4o",
+      }),
+    },
+    response_model: {
+      name: "response_model",
+      ...column("String", {
+        description: "The model name returned by the provider",
+        example: "gpt-4o-2024-08-06",
+        coreColumn: true,
+      }),
+    },
+    operation_id: {
+      name: "operation_id",
+      ...column("LowCardinality(String)", {
+        description: "Operation type (e.g. ai.streamText.doStream, ai.generateText.doGenerate)",
+        example: "ai.streamText.doStream",
+      }),
+    },
+    finish_reason: {
+      name: "finish_reason",
+      ...column("LowCardinality(String)", {
+        description: "Why the LLM stopped generating (e.g. stop, tool-calls, length)",
+        example: "stop",
+        coreColumn: true,
+      }),
+    },
+    cost_source: {
+      name: "cost_source",
+      ...column("LowCardinality(String)", {
+        description: "Where cost data came from (registry, gateway, openrouter)",
+        example: "registry",
+      }),
+    },
+    input_tokens: {
+      name: "input_tokens",
+      ...column("UInt64", {
+        description: "Number of input tokens",
+        example: "702",
+      }),
+    },
+    output_tokens: {
+      name: "output_tokens",
+      ...column("UInt64", {
+        description: "Number of output tokens",
+        example: "22",
+      }),
+    },
+    total_tokens: {
+      name: "total_tokens",
+      ...column("UInt64", {
+        description: "Total token count",
+        example: "724",
+      }),
+    },
+    cached_read_tokens: {
+      name: "cached_read_tokens",
+      ...column("UInt64", {
+        description:
+          "Input tokens served from the provider's prompt cache (cheaper than regular input tokens). Supported by Anthropic and OpenAI.",
+        example: "8200",
+      }),
+      expression: "usage_details['input_cached_tokens']",
+    },
+    cache_creation_tokens: {
+      name: "cache_creation_tokens",
+      ...column("UInt64", {
+        description:
+          "Input tokens written to create a new prompt cache entry. Supported by Anthropic.",
+        example: "1751",
+      }),
+      expression: "usage_details['cache_creation_input_tokens']",
+    },
+    reasoning_tokens: {
+      name: "reasoning_tokens",
+      ...column("UInt64", {
+        description:
+          "Tokens used for chain-of-thought reasoning (e.g. OpenAI o-series, DeepSeek R1). These count toward output but are not visible in the response.",
+        example: "512",
+      }),
+      expression: "usage_details['reasoning_tokens']",
+    },
+    input_cost: {
+      name: "input_cost",
+      ...column("Decimal64(12)", {
+        description: "Input cost in USD (from pricing registry)",
+        customRenderType: "costInDollars",
+      }),
+    },
+    output_cost: {
+      name: "output_cost",
+      ...column("Decimal64(12)", {
+        description: "Output cost in USD (from pricing registry)",
+        customRenderType: "costInDollars",
+      }),
+    },
+    total_cost: {
+      name: "total_cost",
+      ...column("Decimal64(12)", {
+        description: "Total cost in USD",
+        customRenderType: "costInDollars",
+        coreColumn: true,
+      }),
+    },
+    cached_read_cost: {
+      name: "cached_read_cost",
+      ...column("Decimal64(12)", {
+        description:
+          "Cost of cached input tokens (discounted vs regular input). Only present when the pricing tier has a separate cached input price.",
+        customRenderType: "costInDollars",
+      }),
+      expression: "cost_details['input_cached_tokens']",
+    },
+    cache_creation_cost: {
+      name: "cache_creation_cost",
+      ...column("Decimal64(12)", {
+        description: "Cost of tokens written to create a prompt cache entry.",
+        customRenderType: "costInDollars",
+      }),
+      expression: "cost_details['cache_creation_input_tokens']",
+    },
+    provider_cost: {
+      name: "provider_cost",
+      ...column("Decimal64(12)", {
+        description: "Provider-reported cost in USD (from gateway or openrouter)",
+        customRenderType: "costInDollars",
+      }),
+    },
+    ms_to_first_chunk: {
+      name: "ms_to_first_chunk",
+      ...column("Float64", {
+        description: "Time to first chunk in milliseconds (TTFC)",
+        example: "245.3",
+        coreColumn: true,
+      }),
+    },
+    tokens_per_second: {
+      name: "tokens_per_second",
+      ...column("Float64", {
+        description: "Average output tokens per second",
+        example: "72.5",
+      }),
+    },
+    pricing_tier_name: {
+      name: "pricing_tier_name",
+      ...column("LowCardinality(String)", {
+        description: "The matched pricing tier name",
+        example: "Standard",
+      }),
+    },
+    start_time: {
+      name: "start_time",
+      ...column("DateTime64(9)", {
+        description: "When the LLM call started",
+        coreColumn: true,
+      }),
+    },
+    duration: {
+      name: "duration",
+      ...column("UInt64", {
+        description: "Span duration in nanoseconds",
+        customRenderType: "durationNs",
+      }),
+    },
+    prompt_slug: {
+      name: "prompt_slug",
+      ...column("LowCardinality(String)", {
+        description: "The managed prompt slug used for this LLM call",
+        example: "customer-support",
+        coreColumn: true,
+      }),
+    },
+    prompt_version: {
+      name: "prompt_version",
+      ...column("UInt32", {
+        description: "The managed prompt version number used for this LLM call",
+        example: "3",
+      }),
+    },
+    metadata: {
+      name: "metadata",
+      ...column("Map(LowCardinality(String), String)", {
+        description:
+          "Key-value metadata from run tags (key:value format) and AI SDK telemetry metadata. Access keys with dot notation (metadata.userId) or bracket syntax (metadata['userId']).",
+        example: "{'userId':'user_123','org':'acme'}",
+      }),
+    },
+  },
+};
+
+/**
+ * Schema definition for the llm_models table (trigger_dev.llm_model_aggregates_v1)
+ * Global table — no tenant columns. Contains anonymized cross-tenant model performance data.
+ */
+export const llmModelsSchema: TableSchema = {
+  name: "llm_models",
+  clickhouseName: "trigger_dev.llm_model_aggregates_v1",
+  description:
+    "Cross-tenant model performance aggregates: calls, cost, latency, and throughput per model per minute. No tenant-specific data.",
+  timeConstraint: "minute",
+  // No tenantColumns — this is a global table with anonymized data
+  columns: {
+    response_model: {
+      name: "response_model",
+      ...column("String", {
+        description: "The model name as returned by the provider",
+        example: "gpt-4o-2024-08-06",
+        coreColumn: true,
+      }),
+    },
+    base_response_model: {
+      name: "base_response_model",
+      ...column("String", {
+        description: "The base model name with dated variants grouped",
+        example: "gpt-4o",
+        coreColumn: true,
+      }),
+    },
+    gen_ai_system: {
+      name: "gen_ai_system",
+      ...column("String", {
+        description: "The AI provider system identifier",
+        example: "openai.responses",
+        coreColumn: true,
+      }),
+    },
+    minute: {
+      name: "minute",
+      ...column("DateTime", {
+        description: "Aggregation time bucket (per minute)",
+        coreColumn: true,
+      }),
+    },
+    call_count: {
+      name: "call_count",
+      ...column("UInt64", {
+        description: "Number of LLM calls in this time bucket",
+        coreColumn: true,
+      }),
+    },
+    total_input_tokens: {
+      name: "total_input_tokens",
+      ...column("UInt64", {
+        description: "Total input tokens consumed",
+      }),
+    },
+    total_output_tokens: {
+      name: "total_output_tokens",
+      ...column("UInt64", {
+        description: "Total output tokens generated",
+      }),
+    },
+    total_cost: {
+      name: "total_cost",
+      ...column("Float64", {
+        description: "Total cost in USD",
+        customRenderType: "costInDollars",
+        coreColumn: true,
+      }),
+    },
+    // Aggregate state columns — use quantilesMerge() in queries to extract values
+    // Example: quantilesMerge(0.5)(ttfc_quantiles)[1] AS ttfc_p50
+    ttfc_quantiles: {
+      name: "ttfc_quantiles",
+      ...column("String", {
+        description:
+          "Time to first chunk quantile state. Use quantilesMerge(0.5)(ttfc_quantiles)[1] AS ttfc_p50 in queries.",
+        example: "quantilesMerge(0.5)(ttfc_quantiles)[1]",
+      }),
+    },
+    tps_quantiles: {
+      name: "tps_quantiles",
+      ...column("String", {
+        description:
+          "Tokens per second quantile state. Use quantilesMerge(0.5)(tps_quantiles)[1] AS tps_p50 in queries.",
+        example: "quantilesMerge(0.5)(tps_quantiles)[1]",
+      }),
+    },
+    duration_quantiles: {
+      name: "duration_quantiles",
+      ...column("String", {
+        description:
+          "Duration quantile state. Use quantilesMerge(0.5)(duration_quantiles)[1] AS duration_p50 in queries.",
+        example: "quantilesMerge(0.5)(duration_quantiles)[1]",
+      }),
+    },
+  },
+};
+
+export const querySchemas: TableSchema[] = [
+  runsSchema,
+  metricsSchema,
+  llmMetricsSchema,
+  llmModelsSchema,
+];
+
+/**
+ * Default query for the query editor
+ */
+export const defaultQuery = autoFormatSQL(`SELECT run_id, task_identifier, triggered_at, status
+FROM runs
+ORDER BY triggered_at DESC
+LIMIT 100`);

apps/webapp/app/v3/queueSizeLimits.server.ts

@@ -0,0 +1,42 @@
+import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { env } from "~/env.server";
+import type { MarQS } from "./marqs/index.server";
+
+export type QueueSizeGuardResult = {
+  isWithinLimits: boolean;
+  maximumSize?: number;
+  queueSize?: number;
+};
+
+export async function guardQueueSizeLimitsForEnv(
+  environment: AuthenticatedEnvironment,
+  marqs?: MarQS,
+  itemsToAdd: number = 1
+): Promise<QueueSizeGuardResult> {
+  const maximumSize = getMaximumSizeForEnvironment(environment);
+
+  if (typeof maximumSize === "undefined") {
+    return { isWithinLimits: true };
+  }
+
+  if (!marqs) {
+    return { isWithinLimits: true, maximumSize };
+  }
+
+  const queueSize = await marqs.lengthOfEnvQueue(environment);
+  const projectedSize = queueSize + itemsToAdd;
+
+  return {
+    isWithinLimits: projectedSize <= maximumSize,
+    maximumSize,
+    queueSize,
+  };
+}
+
+function getMaximumSizeForEnvironment(environment: AuthenticatedEnvironment): number | undefined {
+  if (environment.type === "DEVELOPMENT") {
+    return environment.organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE;
+  } else {
+    return environment.organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE;
+  }
+}

apps/webapp/app/v3/regionAccess.server.ts

@@ -0,0 +1,50 @@
+import { type Prisma, type WorkloadType } from "@trigger.dev/database";
+import { type PrismaClientOrTransaction } from "~/db.server";
+import { FEATURE_FLAG } from "./featureFlags";
+import { makeFlag } from "./featureFlags.server";
+
+/**
+ * Resolves whether an org has compute access based on feature flags.
+ */
+export async function resolveComputeAccess(
+  prisma: PrismaClientOrTransaction,
+  orgFeatureFlags: unknown
+): Promise<boolean> {
+  const flag = makeFlag(prisma);
+  return flag({
+    key: FEATURE_FLAG.hasComputeAccess,
+    defaultValue: false,
+    overrides: (orgFeatureFlags as Record<string, unknown>) ?? {},
+  });
+}
+
+/**
+ * Builds a visibility filter for non-admin, non-allowlisted users.
+ * Without compute access, MICROVM regions are excluded entirely.
+ * With compute access, hidden flag works normally (existing behavior).
+ */
+export function defaultVisibilityFilter(
+  hasComputeAccess: boolean
+): Prisma.WorkerInstanceGroupWhereInput {
+  if (hasComputeAccess) {
+    return { hidden: false };
+  }
+
+  return { hidden: false, workloadType: { not: "MICROVM" } };
+}
+
+/**
+ * Whether a region is accessible given compute access.
+ * MICROVM regions require compute access; all other types pass through.
+ */
+export function isComputeRegionAccessible(
+  region: { workloadType: WorkloadType },
+  hasComputeAccess: boolean
+): boolean {
+  if (region.workloadType !== "MICROVM") {
+    return true;
+  }
+
+  // Allow access to any MICROVM region if the org has compute access
+  return hasComputeAccess;
+}

apps/webapp/app/v3/registryConfig.server.ts

@@ -0,0 +1,38 @@
+import { env } from "~/env.server";
+
+export type RegistryConfig = {
+  host: string;
+  username?: string;
+  password?: string;
+  namespace: string;
+  ecrTags?: string;
+  ecrAssumeRoleArn?: string;
+  ecrAssumeRoleExternalId?: string;
+  ecrDefaultRepositoryPolicy?: string;
+};
+
+export function getRegistryConfig(isV4Deployment: boolean): RegistryConfig {
+  if (isV4Deployment) {
+    return {
+      host: env.V4_DEPLOY_REGISTRY_HOST,
+      username: env.V4_DEPLOY_REGISTRY_USERNAME,
+      password: env.V4_DEPLOY_REGISTRY_PASSWORD,
+      namespace: env.V4_DEPLOY_REGISTRY_NAMESPACE,
+      ecrTags: env.V4_DEPLOY_REGISTRY_ECR_TAGS,
+      ecrAssumeRoleArn: env.V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN,
+      ecrAssumeRoleExternalId: env.V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID,
+      ecrDefaultRepositoryPolicy: env.V4_DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY,
+    };
+  }
+
+  return {
+    host: env.DEPLOY_REGISTRY_HOST,
+    username: env.DEPLOY_REGISTRY_USERNAME,
+    password: env.DEPLOY_REGISTRY_PASSWORD,
+    namespace: env.DEPLOY_REGISTRY_NAMESPACE,
+    ecrTags: env.DEPLOY_REGISTRY_ECR_TAGS,
+    ecrAssumeRoleArn: env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN,
+    ecrAssumeRoleExternalId: env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID,
+    ecrDefaultRepositoryPolicy: env.DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY,
+  };
+}

apps/webapp/app/v3/remoteImageBuilder.server.ts

@@ -0,0 +1,87 @@
+import { depot } from "@depot/sdk-node";
+import { type ExternalBuildData } from "@trigger.dev/core/v3";
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import pRetry from "p-retry";
+import { logger } from "~/services/logger.server";
+
+// Just the project columns this module reads — keeps the signature
+// compatible with both the full Prisma `Project` payload and the slim
+// `AuthenticatedEnvironment["project"]` shape.
+type ProjectForBuilder = {
+  id: string;
+  externalRef: string;
+  builderProjectId: string | null;
+};
+
+export async function createRemoteImageBuild(
+  project: ProjectForBuilder
+): Promise<ExternalBuildData | undefined> {
+  if (!remoteBuildsEnabled()) {
+    return;
+  }
+
+  const builderProjectId = await createBuilderProjectIfNotExists(project);
+
+  const result = await pRetry(
+    () =>
+      depot.build.v1.BuildService.createBuild(
+        { projectId: builderProjectId },
+        {
+          headers: {
+            Authorization: `Bearer ${env.DEPOT_TOKEN}`,
+          },
+        }
+      ),
+    {
+      retries: 3,
+      minTimeout: 200,
+      maxTimeout: 2000,
+      onFailedAttempt: (error) => {
+        logger.error("Failed attempt to create remote Depot build", { error });
+      },
+    }
+  );
+
+  return {
+    projectId: builderProjectId,
+    buildToken: result.buildToken,
+    buildId: result.buildId,
+  };
+}
+
+async function createBuilderProjectIfNotExists(project: ProjectForBuilder) {
+  if (project.builderProjectId) {
+    return project.builderProjectId;
+  }
+
+  const result = await depot.core.v1.ProjectService.createProject(
+    {
+      name: `${env.APP_ENV} ${project.externalRef}`,
+      organizationId: env.DEPOT_ORG_ID,
+      regionId: env.DEPOT_REGION,
+    },
+    {
+      headers: {
+        Authorization: `Bearer ${env.DEPOT_TOKEN}`,
+      },
+    }
+  );
+
+  if (!result.project) {
+    throw new Error("Failed to create builder project");
+  }
+
+  await prisma.project.update({
+    where: { id: project.id },
+    data: {
+      builderProjectId: result.project.projectId,
+    },
+  });
+
+  return result.project.projectId;
+}
+
+export function remoteBuildsEnabled() {
+  return env.DEPOT_TOKEN && env.DEPOT_ORG_ID && env.DEPOT_REGION;
+}

apps/webapp/app/v3/replayTask.ts

@@ -0,0 +1,48 @@
+import { z } from "zod";
+import { RunOptionsData } from "./testTask";
+
+export const ReplayRunData = z
+  .object({
+    environment: z.string().optional(),
+    payload: z
+      .string()
+      .optional()
+      .transform((val, ctx) => {
+        if (!val) {
+          return "{}";
+        }
+
+        try {
+          JSON.parse(val);
+          return val;
+        } catch {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "Payload must be a valid JSON string",
+          });
+          return z.NEVER;
+        }
+      }),
+    metadata: z
+      .string()
+      .optional()
+      .transform((val, ctx) => {
+        if (!val) {
+          return {};
+        }
+
+        try {
+          return JSON.parse(val);
+        } catch {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "Metadata must be a valid JSON string",
+          });
+          return z.NEVER;
+        }
+      }),
+    failedRedirect: z.string(),
+  })
+  .and(RunOptionsData);
+
+export type ReplayRunData = z.infer<typeof ReplayRunData>;

apps/webapp/app/v3/runEngine.server.ts

@@ -0,0 +1,251 @@
+import { RunEngine } from "@internal/run-engine";
+import { $replica, prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { createBatchGlobalRateLimiter } from "~/runEngine/concerns/batchGlobalRateLimiter.server";
+import { SCHEDULED_WORKER_QUEUE_SUFFIX } from "~/runEngine/concerns/workerQueueSplit.server";
+import { logger } from "~/services/logger.server";
+import { defaultMachine, getCurrentPlan } from "~/services/platform.v3.server";
+import { singleton } from "~/utils/singleton";
+import { allMachines } from "./machinePresets.server";
+import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.server";
+import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server";
+import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server";
+import { runStore } from "./runStore.server";
+import { meter, tracer } from "./tracer.server";
+
+export const engine = singleton("RunEngine", createRunEngine);
+
+export type { RunEngine };
+
+function createRunEngine() {
+  const engine = new RunEngine({
+    prisma,
+    readOnlyPrisma: $replica,
+    crossSeamGuard: pickRunOpsStoreForCompletion,
+    // Inject the shared run-store singleton so the engine and the webapp presenters/
+    // services route through ONE store. When split is off this is the same passthrough
+    // PostgresRunStore the engine would have defaulted to, so behavior is unchanged.
+    store: runStore,
+    controlPlaneResolver: runEngineControlPlaneResolver,
+    logLevel: env.RUN_ENGINE_WORKER_LOG_LEVEL,
+    treatProductionExecutionStallsAsOOM:
+      env.RUN_ENGINE_TREAT_PRODUCTION_EXECUTION_STALLS_AS_OOM === "1",
+    readReplicaSnapshotsSinceEnabled: env.RUN_ENGINE_READ_REPLICA_SNAPSHOTS_SINCE_ENABLED === "1",
+    readReplicaSnapshotsSinceRetryDelay: {
+      minMs: env.RUN_ENGINE_SNAPSHOTS_SINCE_REPLICA_RETRY_MIN_MS,
+      maxMs: env.RUN_ENGINE_SNAPSHOTS_SINCE_REPLICA_RETRY_MAX_MS,
+    },
+    worker: {
+      disabled: env.RUN_ENGINE_WORKER_ENABLED === "0",
+      workers: env.RUN_ENGINE_WORKER_COUNT,
+      tasksPerWorker: env.RUN_ENGINE_TASKS_PER_WORKER,
+      pollIntervalMs: env.RUN_ENGINE_WORKER_POLL_INTERVAL,
+      immediatePollIntervalMs: env.RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL,
+      limit: env.RUN_ENGINE_WORKER_CONCURRENCY_LIMIT,
+      shutdownTimeoutMs: env.RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS,
+      redis: {
+        keyPrefix: "engine:",
+        port: env.RUN_ENGINE_WORKER_REDIS_PORT ?? undefined,
+        host: env.RUN_ENGINE_WORKER_REDIS_HOST ?? undefined,
+        username: env.RUN_ENGINE_WORKER_REDIS_USERNAME ?? undefined,
+        password: env.RUN_ENGINE_WORKER_REDIS_PASSWORD ?? undefined,
+        enableAutoPipelining: true,
+        ...(env.RUN_ENGINE_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+      },
+    },
+    machines: {
+      defaultMachine,
+      machines: allMachines(),
+      baseCostInCents: env.CENTS_PER_RUN,
+    },
+    queue: {
+      defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT,
+      defaultEnvConcurrencyBurstFactor: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_BURST_FACTOR,
+      logLevel: env.RUN_ENGINE_RUN_QUEUE_LOG_LEVEL,
+      redis: {
+        keyPrefix: "engine:",
+        port: env.RUN_ENGINE_RUN_QUEUE_REDIS_PORT ?? undefined,
+        host: env.RUN_ENGINE_RUN_QUEUE_REDIS_HOST ?? undefined,
+        username: env.RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME ?? undefined,
+        password: env.RUN_ENGINE_RUN_QUEUE_REDIS_PASSWORD ?? undefined,
+        enableAutoPipelining: true,
+        ...(env.RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+      },
+      queueSelectionStrategyOptions: {
+        parentQueueLimit: env.RUN_ENGINE_PARENT_QUEUE_LIMIT,
+        biases: {
+          concurrencyLimitBias: env.RUN_ENGINE_CONCURRENCY_LIMIT_BIAS,
+          availableCapacityBias: env.RUN_ENGINE_AVAILABLE_CAPACITY_BIAS,
+          queueAgeRandomization: env.RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS,
+        },
+        reuseSnapshotCount: env.RUN_ENGINE_REUSE_SNAPSHOT_COUNT,
+        maximumEnvCount: env.RUN_ENGINE_MAXIMUM_ENV_COUNT,
+        tracer,
+      },
+      shardCount: env.RUN_ENGINE_RUN_QUEUE_SHARD_COUNT,
+      processWorkerQueueDebounceMs: env.RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS,
+      dequeueBlockingTimeoutSeconds: env.RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS,
+      masterQueueConsumersIntervalMs: env.RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS,
+      masterQueueConsumersDisabled: env.RUN_ENGINE_WORKER_ENABLED === "0",
+      masterQueueCooloffPeriodMs: env.RUN_ENGINE_MASTER_QUEUE_COOLOFF_PERIOD_MS,
+      masterQueueCooloffCountThreshold: env.RUN_ENGINE_MASTER_QUEUE_COOLOFF_COUNT_THRESHOLD,
+      masterQueueConsumerDequeueCount: env.RUN_ENGINE_MASTER_QUEUE_CONSUMER_DEQUEUE_COUNT,
+      concurrencySweeper: {
+        scanSchedule: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_SCHEDULE,
+        processMarkedSchedule: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE,
+        scanJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS,
+        processMarkedJitterInMs: env.RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS,
+      },
+      ttlSystem: {
+        disabled: env.RUN_ENGINE_TTL_SYSTEM_DISABLED,
+        consumersDisabled: env.RUN_ENGINE_TTL_CONSUMERS_DISABLED,
+        shardCount: env.RUN_ENGINE_TTL_SYSTEM_SHARD_COUNT,
+        pollIntervalMs: env.RUN_ENGINE_TTL_SYSTEM_POLL_INTERVAL_MS,
+        batchSize: env.RUN_ENGINE_TTL_SYSTEM_BATCH_SIZE,
+        workerConcurrency: env.RUN_ENGINE_TTL_WORKER_CONCURRENCY,
+        batchMaxSize: env.RUN_ENGINE_TTL_WORKER_BATCH_MAX_SIZE,
+        batchMaxWaitMs: env.RUN_ENGINE_TTL_WORKER_BATCH_MAX_WAIT_MS,
+      },
+    },
+    runLock: {
+      redis: {
+        keyPrefix: "engine:",
+        port: env.RUN_ENGINE_RUN_LOCK_REDIS_PORT ?? undefined,
+        host: env.RUN_ENGINE_RUN_LOCK_REDIS_HOST ?? undefined,
+        username: env.RUN_ENGINE_RUN_LOCK_REDIS_USERNAME ?? undefined,
+        password: env.RUN_ENGINE_RUN_LOCK_REDIS_PASSWORD ?? undefined,
+        enableAutoPipelining: true,
+        ...(env.RUN_ENGINE_RUN_LOCK_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+      },
+      duration: env.RUN_ENGINE_RUN_LOCK_DURATION,
+      automaticExtensionThreshold: env.RUN_ENGINE_RUN_LOCK_AUTOMATIC_EXTENSION_THRESHOLD,
+      retryConfig: {
+        maxAttempts: env.RUN_ENGINE_RUN_LOCK_MAX_RETRIES,
+        baseDelay: env.RUN_ENGINE_RUN_LOCK_BASE_DELAY,
+        maxDelay: env.RUN_ENGINE_RUN_LOCK_MAX_DELAY,
+        backoffMultiplier: env.RUN_ENGINE_RUN_LOCK_BACKOFF_MULTIPLIER,
+        jitterFactor: env.RUN_ENGINE_RUN_LOCK_JITTER_FACTOR,
+        maxTotalWaitTime: env.RUN_ENGINE_RUN_LOCK_MAX_TOTAL_WAIT_TIME,
+      },
+    },
+    tracer,
+    meter,
+    workerQueueObserver: {
+      enabled: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_ENABLED === "1",
+      intervalMs: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_INTERVAL_MS,
+      // Also observe the scheduled split variant of each worker queue. The suffix
+      // naming convention lives in the webapp, so it is passed in here.
+      additionalQueueSuffixes: [SCHEDULED_WORKER_QUEUE_SUFFIX],
+      excludedCloudProviders: env.RUN_ENGINE_WORKER_QUEUE_OBSERVER_EXCLUDED_CLOUD_PROVIDERS.split(
+        ","
+      )
+        .map((provider) => provider.trim())
+        .filter(Boolean),
+    },
+    defaultMaxTtl: env.RUN_ENGINE_DEFAULT_MAX_TTL,
+    heartbeatTimeoutsMs: {
+      PENDING_EXECUTING: env.RUN_ENGINE_TIMEOUT_PENDING_EXECUTING,
+      PENDING_CANCEL: env.RUN_ENGINE_TIMEOUT_PENDING_CANCEL,
+      EXECUTING: env.RUN_ENGINE_TIMEOUT_EXECUTING,
+      EXECUTING_WITH_WAITPOINTS: env.RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS,
+      SUSPENDED: env.RUN_ENGINE_TIMEOUT_SUSPENDED,
+    },
+    suspendedHeartbeatRetriesConfig: {
+      maxCount: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_COUNT,
+      maxDelayMs: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_DELAY_MS,
+      initialDelayMs: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_INITIAL_DELAY_MS,
+      factor: env.RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_FACTOR,
+    },
+    retryWarmStartThresholdMs: env.RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS,
+    pendingVersionRunIdLookup: runEnginePendingVersionLookup,
+    billing: {
+      getCurrentPlan: async (orgId: string) => {
+        const plan = await getCurrentPlan(orgId);
+
+        // This only happens when there's no billing service running or on errors
+        if (!plan) {
+          logger.warn("engine.getCurrentPlan: no plan", { orgId });
+          return {
+            isPaying: true,
+            type: "paid", // default to paid
+          };
+        }
+
+        // This shouldn't happen
+        if (!plan.v3Subscription) {
+          logger.warn("engine.getCurrentPlan: no v3 subscription", { orgId });
+          return {
+            isPaying: false,
+            type: "free",
+          };
+        }
+
+        // Neither should this
+        if (!plan.v3Subscription.plan) {
+          logger.warn("engine.getCurrentPlan: no v3 subscription plan", { orgId });
+          return {
+            isPaying: plan.v3Subscription.isPaying,
+            type: plan.v3Subscription.isPaying ? "paid" : "free",
+          };
+        }
+
+        // This is the normal case when the billing service is running
+        return {
+          isPaying: plan.v3Subscription.isPaying,
+          type: plan.v3Subscription.plan.type,
+          hasPrivateLink: plan.v3Subscription.plan.limits.hasPrivateNetworking ?? false,
+        };
+      },
+    },
+    // BatchQueue with DRR scheduling for fair batch processing
+    // Consumers are controlled by options.worker.disabled (same as main worker)
+    batchQueue: {
+      redis: {
+        keyPrefix: "engine:",
+        port: env.BATCH_TRIGGER_WORKER_REDIS_PORT ?? undefined,
+        host: env.BATCH_TRIGGER_WORKER_REDIS_HOST ?? undefined,
+        username: env.BATCH_TRIGGER_WORKER_REDIS_USERNAME ?? undefined,
+        password: env.BATCH_TRIGGER_WORKER_REDIS_PASSWORD ?? undefined,
+        enableAutoPipelining: true,
+        ...(env.BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+      },
+      drr: {
+        quantum: env.BATCH_QUEUE_DRR_QUANTUM,
+        maxDeficit: env.BATCH_QUEUE_MAX_DEFICIT,
+        masterQueueLimit: env.BATCH_QUEUE_MASTER_QUEUE_LIMIT,
+      },
+      shardCount: env.BATCH_QUEUE_SHARD_COUNT,
+      workerQueueBlockingTimeoutSeconds: env.BATCH_QUEUE_WORKER_QUEUE_ENABLED
+        ? env.BATCH_QUEUE_WORKER_QUEUE_TIMEOUT_SECONDS
+        : undefined,
+      consumerCount: env.BATCH_QUEUE_CONSUMER_COUNT,
+      consumerIntervalMs: env.BATCH_QUEUE_CONSUMER_INTERVAL_MS,
+      consumerEnabled: env.BATCH_QUEUE_WORKER_ENABLED,
+      // Default processing concurrency when no specific limit is set
+      // This is overridden per-batch based on the plan type at batch creation
+      defaultConcurrency: env.BATCH_CONCURRENCY_LIMIT_DEFAULT,
+      // Optional global rate limiter - limits max items/sec processed across all consumers
+      globalRateLimiter: env.BATCH_QUEUE_GLOBAL_RATE_LIMIT
+        ? createBatchGlobalRateLimiter(env.BATCH_QUEUE_GLOBAL_RATE_LIMIT)
+        : undefined,
+      // Worker queue depth cap - prevents unbounded growth protecting visibility timeouts
+      workerQueueMaxDepth: env.BATCH_QUEUE_WORKER_QUEUE_MAX_DEPTH,
+      retry: {
+        maxAttempts: 6,
+        minTimeoutInMs: 1_000,
+        maxTimeoutInMs: 30_000,
+        factor: 2,
+        randomize: true,
+      },
+    },
+    // Debounce configuration
+    debounce: {
+      maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS,
+      quantizeNewDelayUntilMs: env.RUN_ENGINE_DEBOUNCE_QUANTIZE_NEW_DELAY_UNTIL_MS,
+      fastPathSkipEnabled: env.RUN_ENGINE_DEBOUNCE_FAST_PATH_SKIP_ENABLED === "1",
+      useReplicaForFastPathRead: env.RUN_ENGINE_DEBOUNCE_USE_REPLICA_FOR_FAST_PATH_READ === "1",
+    },
+  });
+
+  return engine;
+}

apps/webapp/app/v3/runEngineHandlersShared.server.ts

@@ -0,0 +1,223 @@
+/**
+ * Pure, store-routing helpers extracted from runEngineHandlers.server.ts so they
+ * are testable without constructing the engine (importing that module pulls in the
+ * whole webapp service graph). The handlers wire the production defaults; tests
+ * inject per-container stores/replicas, so these helpers never import db.server.
+ */
+import type { CompleteBatchResult } from "@internal/run-engine";
+import type { RunStore } from "@internal/run-store";
+import type { BatchTaskRunStatus, Prisma } from "@trigger.dev/database";
+import type { PrismaClient, PrismaReplicaClient } from "~/db.server";
+import { logger } from "~/services/logger.server";
+import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server";
+
+export type EventReadDeps = {
+  store: RunStore;
+  newReplica: PrismaReplicaClient;
+  legacyReplica: PrismaReplicaClient;
+  splitEnabled: boolean;
+  // Pure boundary forwarded to read-through; production leaves it undefined
+  // so the read-through layer uses its own wired default. Tests inject a fake.
+  isPastRetention?: (runId: string) => boolean;
+};
+
+/**
+ * Resolve a TaskRun for an event-bus enrichment read through the run-ops
+ * read-through layer. The store stays the read mechanism (the
+ * closures call `store.findRun(...)`); read-through only chooses which replica.
+ * Returns null when not-found / past-retention. Passthrough in single-DB.
+ */
+export async function readRunForEvent<S extends Prisma.TaskRunSelect>(
+  runId: string,
+  environmentId: string,
+  select: S,
+  deps: EventReadDeps
+): Promise<Prisma.TaskRunGetPayload<{ select: S }> | null> {
+  const result = await readThroughRun<Prisma.TaskRunGetPayload<{ select: S }>>({
+    runId,
+    environmentId,
+    readNew: (client) => deps.store.findRun({ id: runId }, { select }, client),
+    readLegacy: (replica) => deps.store.findRun({ id: runId }, { select }, replica),
+    deps: {
+      newClient: deps.newReplica,
+      legacyReplica: deps.legacyReplica,
+      splitEnabled: deps.splitEnabled,
+      isPastRetention: deps.isPastRetention,
+    },
+  });
+
+  return result.source === "not-found" || result.source === "past-retention" ? null : result.value;
+}
+
+/**
+ * Reproduces the `findRunOrThrow` not-found-as-error semantics the 6 throwing
+ * read sites rely on (a missing run throws, which their `tryCatch` turns into
+ * the existing error-log + early-return — never a silent no-op).
+ */
+export async function readRunForEventOrThrow<S extends Prisma.TaskRunSelect>(
+  runId: string,
+  environmentId: string,
+  select: S,
+  deps: EventReadDeps
+): Promise<Prisma.TaskRunGetPayload<{ select: S }>> {
+  const run = await readRunForEvent(runId, environmentId, select, deps);
+  if (!run) {
+    throw new Error("Task run not found");
+  }
+  return run;
+}
+
+/**
+ * Resolve which run-ops writer physically owns the `BatchTaskRun` row for
+ * `batchId` by probing where the row lives, so the batch-completion txn commits
+ * on a single run-ops DB. Length classification is INVALID here: a batch id may
+ * be a run-ops id (cut-over orgs) or a cuid (and cuid-shaped ids can be backfilled
+ * onto NEW), so id-shape does not reliably indicate the row's actual residency.
+ * The existence probe is the correct signal.
+ */
+export async function resolveBatchRunOpsWriter(
+  batchId: string,
+  deps: {
+    newReplica: PrismaReplicaClient;
+    newWriter: PrismaClient;
+    legacyWriter: PrismaClient;
+  }
+): Promise<PrismaClient> {
+  const onNew = await deps.newReplica.batchTaskRun.findFirst({
+    where: { id: batchId },
+    select: { id: true },
+  });
+  return onNew ? deps.newWriter : deps.legacyWriter;
+}
+
+/**
+ * errorCode returned by the batch process-item callback when the trigger was
+ * rejected because the environment's queue is at its maximum size. The
+ * BatchQueue (via `skipRetries`) short-circuits retries for this code, and the
+ * batch completion callback collapses per-item errors into a single aggregate
+ * `BatchTaskRunError` row instead of writing one per item.
+ */
+export const QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE = "QUEUE_SIZE_LIMIT_EXCEEDED";
+
+export type BatchCompletionDeps = {
+  splitEnabled: boolean;
+  newReplica: PrismaReplicaClient;
+  newWriter: PrismaClient;
+  legacyWriter: PrismaClient;
+  tryCompleteBatch: (batchId: string) => Promise<unknown>;
+};
+
+/**
+ * Routes the batch-completion transaction (BatchTaskRun update + BatchTaskRunError
+ * createMany — both run-ops tables) onto the run-ops writer that physically owns
+ * the BatchTaskRun row for `batchId`, so the whole txn commits on a single DB. The
+ * transaction body is unchanged from before the split; only the client changes.
+ */
+export async function handleBatchCompletion(
+  result: CompleteBatchResult,
+  deps: BatchCompletionDeps
+) {
+  const { batchId, runIds, successfulRunCount, failedRunCount, failures } = result;
+
+  // Determine final status
+  let status: BatchTaskRunStatus;
+  if (failedRunCount > 0 && successfulRunCount === 0) {
+    status = "ABORTED";
+  } else if (failedRunCount > 0) {
+    status = "PARTIAL_FAILED";
+  } else {
+    status = "PENDING"; // All runs created, waiting for completion
+  }
+
+  // Always probe residency — never special-case on splitEnabled (see commit msg).
+  const runOpsWriter = await resolveBatchRunOpsWriter(batchId, {
+    newReplica: deps.newReplica,
+    newWriter: deps.newWriter,
+    legacyWriter: deps.legacyWriter,
+  });
+
+  try {
+    // Use a transaction to ensure atomicity of batch update and error record creation
+    // skipDuplicates handles idempotency when callback is retried (relies on unique constraint)
+    await runOpsWriter.$transaction(async (tx) => {
+      // Update BatchTaskRun
+      await tx.batchTaskRun.update({
+        where: { id: batchId },
+        data: {
+          status,
+          runIds,
+          successfulRunCount,
+          failedRunCount,
+          completedAt: status === "ABORTED" ? new Date() : undefined,
+          processingCompletedAt: new Date(),
+        },
+      });
+
+      // Create error records if there were failures.
+      //
+      // Fast-path for queue-size-limit overload: when every failure is the
+      // same QUEUE_SIZE_LIMIT_EXCEEDED error, collapse them into a single
+      // aggregate row instead of writing one per item. This keeps the DB
+      // write volume bounded to O(batches) instead of O(items) when a noisy
+      // tenant fills their queue and all of their batches start bouncing.
+      if (failures.length > 0) {
+        const allQueueSizeLimit = failures.every(
+          (f) => f.errorCode === QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE
+        );
+
+        if (allQueueSizeLimit) {
+          const sample = failures[0]!;
+          await tx.batchTaskRunError.createMany({
+            data: [
+              {
+                batchTaskRunId: batchId,
+                // Use the first item's index as a stable anchor for the
+                // (batchTaskRunId, index) unique constraint so callback
+                // retries remain idempotent.
+                index: sample.index,
+                taskIdentifier: sample.taskIdentifier,
+                payload: sample.payload,
+                options: sample.options as Prisma.InputJsonValue | undefined,
+                error: `${sample.error} (${failures.length} items in this batch failed with the same error)`,
+                errorCode: sample.errorCode,
+              },
+            ],
+            skipDuplicates: true,
+          });
+        } else {
+          await tx.batchTaskRunError.createMany({
+            data: failures.map((failure) => ({
+              batchTaskRunId: batchId,
+              index: failure.index,
+              taskIdentifier: failure.taskIdentifier,
+              payload: failure.payload,
+              options: failure.options as Prisma.InputJsonValue | undefined,
+              error: failure.error,
+              errorCode: failure.errorCode,
+            })),
+            skipDuplicates: true,
+          });
+        }
+      }
+    });
+
+    // Try to complete the batch (handles waitpoint completion if all runs are done)
+    if (status !== "ABORTED") {
+      await deps.tryCompleteBatch(batchId);
+    }
+
+    logger.info("Batch completion handled", {
+      batchId,
+      status,
+      successfulRunCount,
+      failedRunCount,
+    });
+  } catch (error) {
+    logger.error("Failed to handle batch completion", {
+      batchId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    // Re-throw to preserve Redis data for retry (BatchQueue expects errors to propagate)
+    throw error;
+  }
+}

apps/webapp/app/v3/runEnginePendingVersionLookup.server.ts

@@ -0,0 +1,24 @@
+import { type PendingVersionRunIdLookup } from "@internal/run-engine";
+import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { ClickhousePendingVersionLookup } from "./services/clickhousePendingVersionLookup.server";
+
+/**
+ * Lookup used by `@internal/run-engine`'s `PendingVersionSystem` to find
+ * `PENDING_VERSION` TaskRun ids via ClickHouse, removing the need for
+ * Postgres index #13 (`TaskRun_status_runtimeEnvironmentId_createdAt_id_idx`).
+ *
+ * Resolves the ClickHouse client per call via {@link clickhouseFactory}
+ * using the `"engine"` client type, configured by `RUN_ENGINE_CLICKHOUSE_*`
+ * env vars and routed per-organization for customers with HIPAA / data
+ * sovereignty data stores.
+ */
+export const runEnginePendingVersionLookup = singleton(
+  "runEnginePendingVersionLookup",
+  initializeRunEnginePendingVersionLookup
+);
+
+function initializeRunEnginePendingVersionLookup(): PendingVersionRunIdLookup {
+  return new ClickhousePendingVersionLookup({ clickhouseFactory, logger });
+}

apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.test.ts

@@ -0,0 +1,237 @@
+import { describe, expect, it } from "vitest";
+import {
+  ControlPlaneCache,
+  type ResolvedAuthenticatedEnv,
+  type ResolvedEnv,
+  type ResolvedRunLockedWorker,
+  type ResolvedWorkerVersion,
+} from "./controlPlaneCache.server";
+
+// Minimal, structurally-irrelevant stand-ins: the cache stores and returns opaque values by
+// reference, so these only need to be distinguishable objects — the slot types are exercised for
+// key routing, not field shape.
+const anEnv = { id: "env_1", organizationId: "org_1" } as unknown as ResolvedEnv;
+const aVersion = { worker: { id: "bw_1" } } as unknown as ResolvedWorkerVersion;
+const anAuthEnv = {
+  id: "env_1",
+  slug: "prod",
+  organizationId: "org_1",
+} as unknown as ResolvedAuthenticatedEnv;
+const aLockedWorker = { lockedBy: null, lockedToVersion: null } as ResolvedRunLockedWorker;
+
+describe("ControlPlaneCache", () => {
+  it("round-trips a value through every slot", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("env_1", anEnv);
+    cache.setWorkerVersion("env_1:current", aVersion);
+    cache.setEnvExists("env_1", true);
+    cache.setAuthEnv("env_1", anAuthEnv);
+    cache.setLockedWorker("bw_1:v_1", aLockedWorker);
+
+    expect(cache.getEnv("env_1")).toBe(anEnv);
+    expect(cache.getWorkerVersion("env_1:current")).toBe(aVersion);
+    expect(cache.getEnvExists("env_1")).toBe(true);
+    expect(cache.getAuthEnv("env_1")).toBe(anAuthEnv);
+    expect(cache.getLockedWorker("bw_1:v_1")).toBe(aLockedWorker);
+  });
+
+  it("returns undefined for a key that was never set, in every slot", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    expect(cache.getEnv("missing")).toBeUndefined();
+    expect(cache.getWorkerVersion("missing")).toBeUndefined();
+    expect(cache.getEnvExists("missing")).toBeUndefined();
+    expect(cache.getAuthEnv("missing")).toBeUndefined();
+    expect(cache.getLockedWorker("missing")).toBeUndefined();
+  });
+
+  it("distinguishes a cached null (confirmed absence) from an unset miss", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    expect(cache.getEnv("env_2")).toBeUndefined();
+    cache.setEnv("env_2", null);
+    expect(cache.getEnv("env_2")).toBeNull();
+
+    expect(cache.getAuthEnv("env_2")).toBeUndefined();
+    cache.setAuthEnv("env_2", null);
+    expect(cache.getAuthEnv("env_2")).toBeNull();
+
+    expect(cache.getWorkerVersion("env_2:current")).toBeUndefined();
+    cache.setWorkerVersion("env_2:current", null);
+    expect(cache.getWorkerVersion("env_2:current")).toBeNull();
+
+    expect(cache.getLockedWorker("_:_")).toBeUndefined();
+    cache.setLockedWorker("_:_", null);
+    expect(cache.getLockedWorker("_:_")).toBeNull();
+  });
+
+  it("caches a false env-existence result distinctly from an unset miss", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    expect(cache.getEnvExists("env_3")).toBeUndefined();
+    cache.setEnvExists("env_3", false);
+    expect(cache.getEnvExists("env_3")).toBe(false);
+  });
+
+  it("invalidateEnv forces the next getEnv to miss", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("env_4", anEnv);
+    expect(cache.getEnv("env_4")).toBe(anEnv);
+
+    cache.invalidateEnv("env_4");
+    expect(cache.getEnv("env_4")).toBeUndefined();
+  });
+
+  it("makes a re-setEnv after invalidation readable again", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+    const replacement = { id: "env_5b" } as unknown as ResolvedEnv;
+
+    cache.setEnv("env_5", anEnv);
+    cache.invalidateEnv("env_5");
+    expect(cache.getEnv("env_5")).toBeUndefined();
+
+    cache.setEnv("env_5", replacement);
+    expect(cache.getEnv("env_5")).toBe(replacement);
+  });
+
+  it("invalidateEnv is scoped to its own id", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+    const other = { id: "env_keep" } as unknown as ResolvedEnv;
+
+    cache.setEnv("env_drop", anEnv);
+    cache.setEnv("env_keep", other);
+    cache.invalidateEnv("env_drop");
+
+    expect(cache.getEnv("env_drop")).toBeUndefined();
+    expect(cache.getEnv("env_keep")).toBe(other);
+  });
+
+  it("does not collide keys across slots for the same id", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("x", anEnv);
+    cache.setEnvExists("x", true);
+    cache.setAuthEnv("x", anAuthEnv);
+
+    expect(cache.getEnv("x")).toBe(anEnv);
+    expect(cache.getEnvExists("x")).toBe(true);
+    expect(cache.getAuthEnv("x")).toBe(anAuthEnv);
+
+    // Invalidating the env slot leaves the sibling slots for the same id intact.
+    cache.invalidateEnv("x");
+    expect(cache.getEnv("x")).toBeUndefined();
+    expect(cache.getEnvExists("x")).toBe(true);
+    expect(cache.getAuthEnv("x")).toBe(anAuthEnv);
+  });
+
+  it("evicts the oldest entry once maxEntries is exceeded", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 2 });
+
+    cache.setEnv("first", { id: "first" } as unknown as ResolvedEnv);
+    cache.setEnv("second", { id: "second" } as unknown as ResolvedEnv);
+    cache.setEnv("third", { id: "third" } as unknown as ResolvedEnv);
+
+    expect(cache.getEnv("first")).toBeUndefined();
+    expect(cache.getEnv("second")).toMatchObject({ id: "second" });
+    expect(cache.getEnv("third")).toMatchObject({ id: "third" });
+  });
+
+  it("treats a zero-TTL entry as immediately expired", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 0, maxEntries: 100 });
+
+    cache.setEnv("env_ttl", anEnv);
+    expect(cache.getEnv("env_ttl")).toBeUndefined();
+  });
+
+  it("invalidateEnvironment forces the next env/authEnv/envExists read to miss", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("env_6", anEnv);
+    cache.setAuthEnv("env_6", anAuthEnv);
+    cache.setEnvExists("env_6", true);
+    expect(cache.getEnv("env_6")).toBe(anEnv);
+    expect(cache.getAuthEnv("env_6")).toBe(anAuthEnv);
+    expect(cache.getEnvExists("env_6")).toBe(true);
+
+    cache.invalidateEnvironment("env_6");
+
+    expect(cache.getEnv("env_6")).toBeUndefined();
+    expect(cache.getAuthEnv("env_6")).toBeUndefined();
+    expect(cache.getEnvExists("env_6")).toBeUndefined();
+  });
+
+  it("invalidateEnvironment is scoped to its own id", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+    const keepEnv = { id: "env_keep", organizationId: "org_1" } as unknown as ResolvedEnv;
+
+    cache.setEnv("env_drop", anEnv);
+    cache.setEnv("env_keep", keepEnv);
+    cache.invalidateEnvironment("env_drop");
+
+    expect(cache.getEnv("env_drop")).toBeUndefined();
+    expect(cache.getEnv("env_keep")).toBe(keepEnv);
+  });
+
+  it("invalidateOrganization drops env/authEnv rows for that org across every env id", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+    const envA = { id: "env_a", organizationId: "org_1" } as unknown as ResolvedEnv;
+    const envB = { id: "env_b", organizationId: "org_1" } as unknown as ResolvedEnv;
+    const authA = {
+      id: "env_a",
+      slug: "a",
+      organizationId: "org_1",
+    } as unknown as ResolvedAuthenticatedEnv;
+
+    cache.setEnv("env_a", envA);
+    cache.setEnv("env_b", envB);
+    cache.setAuthEnv("env_a", authA);
+    expect(cache.getEnv("env_a")).toBe(envA);
+    expect(cache.getEnv("env_b")).toBe(envB);
+    expect(cache.getAuthEnv("env_a")).toBe(authA);
+
+    cache.invalidateOrganization("org_1");
+
+    // Every env/authEnv row for org_1 misses — no reverse org->env index required.
+    expect(cache.getEnv("env_a")).toBeUndefined();
+    expect(cache.getEnv("env_b")).toBeUndefined();
+    expect(cache.getAuthEnv("env_a")).toBeUndefined();
+  });
+
+  it("invalidateOrganization does not affect a different org's cached envs", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+    const otherOrgEnv = { id: "env_other", organizationId: "org_2" } as unknown as ResolvedEnv;
+
+    cache.setEnv("env_1", anEnv); // org_1
+    cache.setEnv("env_other", otherOrgEnv); // org_2
+
+    cache.invalidateOrganization("org_1");
+
+    expect(cache.getEnv("env_1")).toBeUndefined();
+    expect(cache.getEnv("env_other")).toBe(otherOrgEnv);
+  });
+
+  it("re-setting an env after an org invalidation makes it readable again", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("env_1", anEnv);
+    cache.invalidateOrganization("org_1");
+    expect(cache.getEnv("env_1")).toBeUndefined();
+
+    // A write after the bump stamps the new org epoch, so it reads back.
+    cache.setEnv("env_1", anEnv);
+    expect(cache.getEnv("env_1")).toBe(anEnv);
+  });
+
+  it("a cached null env survives an org invalidation (a confirmed absence carries no org)", () => {
+    const cache = new ControlPlaneCache({ ttlMs: 60_000, maxEntries: 100 });
+
+    cache.setEnv("env_absent", null);
+    expect(cache.getEnv("env_absent")).toBeNull();
+
+    cache.invalidateOrganization("org_1");
+
+    expect(cache.getEnv("env_absent")).toBeNull();
+  });
+});

apps/webapp/app/v3/runOpsMigration/controlPlaneCache.server.ts

@@ -0,0 +1,244 @@
+import type {
+  BackgroundWorker,
+  BackgroundWorkerTask,
+  Prisma,
+  RuntimeEnvironmentType,
+  TaskQueue,
+  WorkerDeployment,
+} from "@trigger.dev/database";
+import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
+import type { AuthenticatedEnvironment } from "@trigger.dev/core/v3/auth/environment";
+
+/**
+ * Cache policy + invalidation for the cross-DB control-plane resolver.
+ *
+ * One-way dependency: this module is imported by `controlPlaneResolver.server.ts`;
+ * it must NEVER import the resolver. The shared `Resolved*` return types live here
+ * so both files reference an identical definition (the resolver re-exports them for
+ * consumers).
+ *
+ * Invalidation note: the underlying `BoundedTtlCache` exposes no public `delete`, so
+ * explicit invalidation is implemented with a per-key epoch map. A write stamps the
+ * stored value with the key's current epoch; a read returns the value only if its
+ * stamped epoch still matches the current epoch, otherwise it is treated as a miss.
+ * `invalidate*` bumps the key's epoch, forcing the next read to miss. (If a future
+ * rebase gives `BoundedTtlCache` a public `delete`, prefer it and drop the epoch map.)
+ *
+ * Two invalidation scopes: `invalidateEnvironment(id)` bumps every env-keyed slot for one
+ * env; `invalidateOrganization(orgId)` bumps a per-org epoch that env/authEnv values are
+ * also stamped with at write time (no reverse org->env index needed), so all of that org's
+ * cached env/authEnv rows miss on the next read.
+ */
+
+export const DEFAULT_CP_CACHE_TTL_MS = 30_000;
+export const DEFAULT_CP_CACHE_MAX_ENTRIES = 10_000;
+
+export type ResolvedEnv = {
+  id: string;
+  type: RuntimeEnvironmentType;
+  projectId: string;
+  organizationId: string;
+  archivedAt: Date | null;
+  // The parent env's type, or null when this env has no parent. Alerts compute
+  // `parentEnvironmentType ?? type` (byte-identical to `parentEnvironment?.type ?? type`).
+  parentEnvironmentType: RuntimeEnvironmentType | null;
+  // Concurrency + nested ids the run-engine ControlPlaneResolver adapter maps to
+  // `ResolvedEngineEnv` (a MinimalAuthenticatedEnvironment superset). Existing app consumers
+  // ignore these additive fields.
+  maximumConcurrencyLimit: number;
+  concurrencyLimitBurstFactor: Prisma.Decimal;
+};
+
+/** Mirrors `WorkerDeploymentWithWorkerTasks` in `dequeueSystem.ts` exactly. */
+export type ResolvedWorkerVersion = {
+  worker: BackgroundWorker;
+  tasks: BackgroundWorkerTask[];
+  queues: TaskQueue[];
+  deployment: WorkerDeployment | null;
+};
+
+// The canonical authenticated-environment shape (slug/type/project/organization/orgMember/…)
+// PLUS the `git` JSON column the run-engine runAttemptSystem reads. `AuthenticatedEnvironment`
+// does not carry `git`, so the intersection adds it; this matches the run-engine
+// `ResolvedAuthenticatedEnv` so the engine adapter can delegate to this cached slot.
+export type ResolvedAuthenticatedEnv = AuthenticatedEnvironment & { git: Prisma.JsonValue | null };
+
+/**
+ * The slim `lockedBy` (BackgroundWorkerTask) + `lockedToVersion` (BackgroundWorker, with its
+ * WorkerDeployment) shape — the UNION of every field webapp run sites read off these two
+ * cross-DB worker relations. Each field is optional because a run may be locked to a version
+ * but not a task (or neither); resolvers return only what exists.
+ */
+export type ResolvedRunLockedWorker = {
+  lockedBy: {
+    id: string;
+    filePath: string;
+    exportName: string | null;
+    slug: string;
+    machineConfig: Prisma.JsonValue | null;
+    worker: {
+      id: string;
+      version: string;
+      sdkVersion: string;
+      cliVersion: string;
+      supportsLazyAttempts: boolean;
+      deployment: {
+        friendlyId: string;
+        shortCode: string;
+        version: string;
+        runtime: string | null;
+        runtimeVersion: string | null;
+        git: Prisma.JsonValue | null;
+      } | null;
+    };
+  } | null;
+  lockedToVersion: {
+    version: string;
+    sdkVersion: string;
+    runtime: string | null;
+    runtimeVersion: string | null;
+    supportsLazyAttempts: boolean;
+  } | null;
+};
+
+// `orgEpoch` is stamped only on slots that embed org config (env/authEnv); undefined slots
+// are exempt from the org-epoch check.
+type Stamped<V> = { value: V; epoch: number; orgEpoch?: number };
+
+export class ControlPlaneCache {
+  readonly #env: BoundedTtlCache<Stamped<ResolvedEnv | null>>;
+  readonly #version: BoundedTtlCache<Stamped<ResolvedWorkerVersion | null>>;
+  readonly #envExists: BoundedTtlCache<Stamped<boolean>>;
+  readonly #authEnv: BoundedTtlCache<Stamped<ResolvedAuthenticatedEnv | null>>;
+  readonly #lockedWorker: BoundedTtlCache<Stamped<ResolvedRunLockedWorker | null>>;
+
+  // Explicit invalidation: bumping a key's (or org's) epoch forces the next read to miss.
+  readonly #epochs = new Map<string, number>();
+  readonly #orgEpochs = new Map<string, number>();
+
+  constructor(opts?: { ttlMs?: number; maxEntries?: number }) {
+    const ttl = opts?.ttlMs ?? DEFAULT_CP_CACHE_TTL_MS;
+    const max = opts?.maxEntries ?? DEFAULT_CP_CACHE_MAX_ENTRIES;
+    this.#env = new BoundedTtlCache(ttl, max);
+    this.#version = new BoundedTtlCache(ttl, max);
+    this.#envExists = new BoundedTtlCache(ttl, max);
+    this.#authEnv = new BoundedTtlCache(ttl, max);
+    this.#lockedWorker = new BoundedTtlCache(ttl, max);
+  }
+
+  #epoch(key: string): number {
+    return this.#epochs.get(key) ?? 0;
+  }
+
+  #orgEpoch(orgId: string): number {
+    return this.#orgEpochs.get(orgId) ?? 0;
+  }
+
+  #read<V>(cache: BoundedTtlCache<Stamped<V>>, key: string, orgId?: string): V | undefined {
+    const entry = cache.get(key);
+    if (entry === undefined || entry.epoch !== this.#epoch(key)) {
+      return undefined;
+    }
+    if (orgId !== undefined && entry.orgEpoch !== this.#orgEpoch(orgId)) {
+      return undefined;
+    }
+    return entry.value;
+  }
+
+  #write<V>(cache: BoundedTtlCache<Stamped<V>>, key: string, value: V, orgId?: string): void {
+    cache.set(key, {
+      value,
+      epoch: this.#epoch(key),
+      orgEpoch: orgId !== undefined ? this.#orgEpoch(orgId) : undefined,
+    });
+  }
+
+  #bump(key: string): void {
+    this.#epochs.set(key, this.#epoch(key) + 1);
+  }
+
+  getEnv(id: string): (ResolvedEnv | null) | undefined {
+    const entry = this.#env.get(`env:${id}`);
+    if (entry === undefined || entry.epoch !== this.#epoch(`env:${id}`)) {
+      return undefined;
+    }
+    // A cached null (or an entry written without an org) carries no org, so it can never be
+    // stale against an org write.
+    if (
+      entry.value !== null &&
+      entry.value.organizationId &&
+      entry.orgEpoch !== this.#orgEpoch(entry.value.organizationId)
+    ) {
+      return undefined;
+    }
+    return entry.value;
+  }
+  setEnv(id: string, value: ResolvedEnv | null): void {
+    this.#write(this.#env, `env:${id}`, value, value?.organizationId);
+  }
+  invalidateEnv(id: string): void {
+    this.#bump(`env:${id}`);
+  }
+
+  // worker version: key = `${environmentId}:${backgroundWorkerId ?? "current"}`
+  getWorkerVersion(key: string): (ResolvedWorkerVersion | null) | undefined {
+    return this.#read(this.#version, `version:${key}`);
+  }
+  setWorkerVersion(key: string, value: ResolvedWorkerVersion | null): void {
+    this.#write(this.#version, `version:${key}`, value);
+  }
+
+  // env existence (boolean; for the dropped-FK replacement check)
+  getEnvExists(id: string): boolean | undefined {
+    return this.#read(this.#envExists, `envExists:${id}`);
+  }
+  setEnvExists(id: string, exists: boolean): void {
+    this.#write(this.#envExists, `envExists:${id}`, exists);
+  }
+
+  // full authenticated environment (toAuthenticated shape)
+  getAuthEnv(id: string): (ResolvedAuthenticatedEnv | null) | undefined {
+    const entry = this.#authEnv.get(`authEnv:${id}`);
+    if (entry === undefined || entry.epoch !== this.#epoch(`authEnv:${id}`)) {
+      return undefined;
+    }
+    if (
+      entry.value !== null &&
+      entry.value.organizationId &&
+      entry.orgEpoch !== this.#orgEpoch(entry.value.organizationId)
+    ) {
+      return undefined;
+    }
+    return entry.value;
+  }
+  setAuthEnv(id: string, value: ResolvedAuthenticatedEnv | null): void {
+    this.#write(this.#authEnv, `authEnv:${id}`, value, value?.organizationId);
+  }
+
+  /**
+   * Invalidate every env-keyed slot for a single environment. Call this from a control-plane
+   * write that mutates one env's config (pause/resume, archive, concurrency/burst-factor).
+   */
+  invalidateEnvironment(id: string): void {
+    this.#bump(`env:${id}`);
+    this.#bump(`authEnv:${id}`);
+    this.#bump(`envExists:${id}`);
+  }
+
+  /**
+   * Invalidate every cached env/authEnv row belonging to an organization. Call this from a
+   * control-plane write that mutates org-level config (feature flags, org concurrency, runs
+   * enable/disable, rate limits) — it affects the org object embedded in each of the org's envs.
+   */
+  invalidateOrganization(orgId: string): void {
+    this.#orgEpochs.set(orgId, this.#orgEpoch(orgId) + 1);
+  }
+
+  // run-locked worker (lockedBy + lockedToVersion); key = `${lockedById ?? "_"}:${lockedToVersionId ?? "_"}`
+  getLockedWorker(key: string): (ResolvedRunLockedWorker | null) | undefined {
+    return this.#read(this.#lockedWorker, `lockedWorker:${key}`);
+  }
+  setLockedWorker(key: string, value: ResolvedRunLockedWorker | null): void {
+    this.#write(this.#lockedWorker, `lockedWorker:${key}`, value);
+  }
+}

apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts

@@ -0,0 +1,463 @@
+import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic";
+import type {
+  PrismaClient,
+  PrismaReplicaClient,
+  RuntimeEnvironmentType,
+} from "@trigger.dev/database";
+import { prisma, $replica } from "~/db.server";
+import { env } from "~/env.server";
+import {
+  ControlPlaneCache,
+  DEFAULT_CP_CACHE_MAX_ENTRIES,
+  DEFAULT_CP_CACHE_TTL_MS,
+  type ResolvedAuthenticatedEnv,
+  type ResolvedEnv,
+  type ResolvedWorkerVersion,
+  type ResolvedRunLockedWorker,
+} from "./controlPlaneCache.server";
+import { authIncludeWithParent, toAuthenticated } from "~/models/runtimeEnvironment.server";
+
+/**
+ * App-level control-plane resolution + cache layer. Replaces the run-ops -> control-plane
+ * Prisma joins (env/project/org, the pinned/current worker version + its tasks/queues, the
+ * TaskQueue, the TaskSchedule friendlyId mapping) with cached lookups against the
+ * control-plane client, so the split (cross-DB) hot path avoids a cross-WAN round-trip per
+ * resolution.
+ *
+ * Split ON (cloud): cache-first reads against the control-plane replica; `null` is cached as
+ * a confirmed absence. Split OFF (self-host/local/CI): plain Prisma join against the single
+ * control-plane client on every call, NO cache — byte-identical to today's inline join.
+ *
+ * The split gate is a SYNCHRONOUS `splitEnabled: () => boolean` injected at construction; the
+ * resolver never awaits the async `isSplitEnabled()` (that gate is reserved for the boot
+ * sentinel). Tests inject testcontainer clients + a sync predicate; only the module-level
+ * singleton at the bottom reads from `db.server.ts` / `env.server.ts`.
+ *
+ * Scope boundary: this unit owns ONLY control-plane resolution (env, worker version,
+ * env existence). The run-ops batchId friendlyId->id resolution belongs to the
+ * run-ops read path (the unit owning `runsRepository.server.ts`); do not duplicate it here.
+ */
+
+export { ResolvedEnv, ResolvedWorkerVersion };
+export type { ResolvedAuthenticatedEnv, ResolvedRunLockedWorker };
+
+/** Thrown by `assertEnvExists` when a referenced control-plane env does not exist. */
+export class ControlPlaneReferenceError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ControlPlaneReferenceError";
+  }
+}
+
+export type ControlPlaneResolverOptions = {
+  controlPlanePrimary: PrismaClient;
+  controlPlaneReplica: PrismaReplicaClient;
+  cache: ControlPlaneCache;
+  splitEnabled: () => boolean;
+};
+
+type CpClient = PrismaClient | PrismaReplicaClient;
+
+function workerVersionKey(
+  environmentId: string,
+  backgroundWorkerId: string | undefined,
+  type: RuntimeEnvironmentType | undefined
+): string {
+  return `${environmentId}:${backgroundWorkerId ?? "current"}:${type ?? "any"}`;
+}
+
+function lockedWorkerKey(lockedById?: string | null, lockedToVersionId?: string | null): string {
+  return `${lockedById ?? "_"}:${lockedToVersionId ?? "_"}`;
+}
+
+export class ControlPlaneResolver {
+  private readonly controlPlanePrimary: PrismaClient;
+  private readonly controlPlaneReplica: PrismaReplicaClient;
+  private readonly cache: ControlPlaneCache;
+  private readonly splitEnabled: () => boolean;
+
+  constructor(opts: ControlPlaneResolverOptions) {
+    this.controlPlanePrimary = opts.controlPlanePrimary;
+    this.controlPlaneReplica = opts.controlPlaneReplica;
+    this.cache = opts.cache;
+    this.splitEnabled = opts.splitEnabled;
+  }
+
+  async resolveEnv(environmentId: string): Promise<ResolvedEnv | null> {
+    if (!this.splitEnabled()) {
+      return this.#queryEnv(this.controlPlanePrimary, environmentId);
+    }
+
+    const cached = this.cache.getEnv(environmentId);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    const resolved = await this.#queryEnv(this.controlPlaneReplica, environmentId);
+    this.cache.setEnv(environmentId, resolved);
+    return resolved;
+  }
+
+  async #queryEnv(client: CpClient, environmentId: string): Promise<ResolvedEnv | null> {
+    const env = await client.runtimeEnvironment.findFirst({
+      where: { id: environmentId },
+      select: {
+        id: true,
+        type: true,
+        projectId: true,
+        archivedAt: true,
+        maximumConcurrencyLimit: true,
+        concurrencyLimitBurstFactor: true,
+        project: { select: { organizationId: true } },
+        parentEnvironment: { select: { type: true } },
+      },
+    });
+
+    if (!env) {
+      return null;
+    }
+
+    return {
+      id: env.id,
+      type: env.type,
+      projectId: env.projectId,
+      organizationId: env.project.organizationId,
+      archivedAt: env.archivedAt,
+      parentEnvironmentType: env.parentEnvironment?.type ?? null,
+      maximumConcurrencyLimit: env.maximumConcurrencyLimit,
+      concurrencyLimitBurstFactor: env.concurrencyLimitBurstFactor,
+    };
+  }
+
+  async resolveAuthenticatedEnv(environmentId: string): Promise<ResolvedAuthenticatedEnv | null> {
+    if (!this.splitEnabled()) {
+      return this.#queryAuthenticatedEnv(this.controlPlanePrimary, environmentId);
+    }
+
+    const cached = this.cache.getAuthEnv(environmentId);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    const resolved = await this.#queryAuthenticatedEnv(this.controlPlaneReplica, environmentId);
+    this.cache.setAuthEnv(environmentId, resolved);
+    return resolved;
+  }
+
+  async #queryAuthenticatedEnv(
+    client: CpClient,
+    environmentId: string
+  ): Promise<ResolvedAuthenticatedEnv | null> {
+    const env = await client.runtimeEnvironment.findFirst({
+      where: { id: environmentId },
+      include: authIncludeWithParent,
+    });
+
+    if (!env) {
+      return null;
+    }
+
+    // `authIncludeWithParent` returns all RuntimeEnvironment scalars on the row (including
+    // `git`), so we map the auth shape via toAuthenticated() and add `git` from the same row.
+    return { ...toAuthenticated(env), git: env.git };
+  }
+
+  async resolveRunLockedWorker(args: {
+    lockedById?: string | null;
+    lockedToVersionId?: string | null;
+  }): Promise<ResolvedRunLockedWorker | null> {
+    const { lockedById, lockedToVersionId } = args;
+
+    if (!this.splitEnabled()) {
+      return this.#queryRunLockedWorker(this.controlPlanePrimary, lockedById, lockedToVersionId);
+    }
+
+    const key = lockedWorkerKey(lockedById, lockedToVersionId);
+    const cached = this.cache.getLockedWorker(key);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    const resolved = await this.#queryRunLockedWorker(
+      this.controlPlaneReplica,
+      lockedById,
+      lockedToVersionId
+    );
+    this.cache.setLockedWorker(key, resolved);
+    return resolved;
+  }
+
+  async #queryRunLockedWorker(
+    client: CpClient,
+    lockedById?: string | null,
+    lockedToVersionId?: string | null
+  ): Promise<ResolvedRunLockedWorker | null> {
+    const lockedByRow = lockedById
+      ? await client.backgroundWorkerTask.findFirst({
+          where: { id: lockedById },
+          select: {
+            id: true,
+            filePath: true,
+            exportName: true,
+            slug: true,
+            machineConfig: true,
+            worker: {
+              select: {
+                id: true,
+                version: true,
+                sdkVersion: true,
+                cliVersion: true,
+                supportsLazyAttempts: true,
+                deployment: {
+                  select: {
+                    friendlyId: true,
+                    shortCode: true,
+                    version: true,
+                    runtime: true,
+                    runtimeVersion: true,
+                    git: true,
+                  },
+                },
+              },
+            },
+          },
+        })
+      : null;
+
+    const lockedToVersionRow = lockedToVersionId
+      ? await client.backgroundWorker.findFirst({
+          where: { id: lockedToVersionId },
+          select: {
+            version: true,
+            sdkVersion: true,
+            runtime: true,
+            runtimeVersion: true,
+            supportsLazyAttempts: true,
+          },
+        })
+      : null;
+
+    return {
+      lockedBy: lockedByRow,
+      lockedToVersion: lockedToVersionRow,
+    };
+  }
+
+  async resolveWorkerVersion(args: {
+    environmentId: string;
+    backgroundWorkerId?: string;
+    /**
+     * When provided, the full run-engine dequeue dispatch is used (DEV resolves the most-recent
+     * worker; deployed resolves the promoted MANAGED deployment with the latest-v2 fallback).
+     * When omitted, the original app behavior applies (worker-by-id, else current promotion).
+     */
+    type?: RuntimeEnvironmentType;
+  }): Promise<ResolvedWorkerVersion | null> {
+    const { environmentId, backgroundWorkerId, type } = args;
+
+    if (!this.splitEnabled()) {
+      return this.#queryWorkerVersion(
+        this.controlPlanePrimary,
+        environmentId,
+        backgroundWorkerId,
+        type
+      );
+    }
+
+    const key = workerVersionKey(environmentId, backgroundWorkerId, type);
+    const cached = this.cache.getWorkerVersion(key);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    const resolved = await this.#queryWorkerVersion(
+      this.controlPlaneReplica,
+      environmentId,
+      backgroundWorkerId,
+      type
+    );
+    this.cache.setWorkerVersion(key, resolved);
+    return resolved;
+  }
+
+  async #queryWorkerVersion(
+    client: CpClient,
+    environmentId: string,
+    backgroundWorkerId?: string,
+    type?: RuntimeEnvironmentType
+  ): Promise<ResolvedWorkerVersion | null> {
+    // Full run-engine dequeue dispatch (mirrors dequeueSystem's four helpers) when the env type is
+    // known. DEVELOPMENT envs resolve by most-recent worker; deployed envs resolve the promoted
+    // MANAGED deployment.
+    if (type === "DEVELOPMENT") {
+      return backgroundWorkerId
+        ? this.#queryWorkerById(client, backgroundWorkerId)
+        : this.#queryMostRecentWorker(client, environmentId);
+    }
+
+    if (backgroundWorkerId) {
+      const worker = await client.backgroundWorker.findFirst({
+        where: { id: backgroundWorkerId },
+        include: { deployment: true, tasks: true, queues: true },
+      });
+
+      if (!worker) {
+        return null;
+      }
+
+      return {
+        worker,
+        tasks: worker.tasks,
+        queues: worker.queues,
+        deployment: worker.deployment,
+      };
+    }
+
+    // Deployed env, no workerId: resolve the currently-promoted deployment's worker. When `type`
+    // is known (engine dispatch) apply the MANAGED guard + latest-v2 fallback that the run-engine
+    // path requires; without `type` keep the original app behavior (return the promoted worker).
+    const promotion = await client.workerDeploymentPromotion.findFirst({
+      where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL },
+      include: {
+        deployment: {
+          include: { worker: { include: { tasks: true, queues: true } } },
+        },
+      },
+    });
+
+    if (!promotion?.deployment.worker) {
+      return null;
+    }
+
+    if (type === undefined || promotion.deployment.type === "MANAGED") {
+      return {
+        worker: promotion.deployment.worker,
+        tasks: promotion.deployment.worker.tasks,
+        queues: promotion.deployment.worker.queues,
+        deployment: promotion.deployment,
+      };
+    }
+
+    // Engine dispatch only: the promoted deployment is not run-engine v2; fall back to the latest
+    // MANAGED deployment.
+    const latestV2Deployment = await client.workerDeployment.findFirst({
+      where: { environmentId, type: "MANAGED" },
+      orderBy: { id: "desc" },
+      include: { worker: { include: { tasks: true, queues: true } } },
+    });
+
+    if (!latestV2Deployment?.worker) {
+      return null;
+    }
+
+    return {
+      worker: latestV2Deployment.worker,
+      tasks: latestV2Deployment.worker.tasks,
+      queues: latestV2Deployment.worker.queues,
+      deployment: latestV2Deployment,
+    };
+  }
+
+  async #queryWorkerById(
+    client: CpClient,
+    workerId: string
+  ): Promise<ResolvedWorkerVersion | null> {
+    const worker = await client.backgroundWorker.findFirst({
+      where: { id: workerId },
+      include: { deployment: true, tasks: true, queues: true },
+      orderBy: { id: "desc" },
+    });
+
+    if (!worker) {
+      return null;
+    }
+
+    return { worker, tasks: worker.tasks, queues: worker.queues, deployment: worker.deployment };
+  }
+
+  async #queryMostRecentWorker(
+    client: CpClient,
+    environmentId: string
+  ): Promise<ResolvedWorkerVersion | null> {
+    const worker = await client.backgroundWorker.findFirst({
+      where: { runtimeEnvironmentId: environmentId },
+      include: { tasks: true, queues: true },
+      orderBy: { id: "desc" },
+    });
+
+    if (!worker) {
+      return null;
+    }
+
+    return { worker, tasks: worker.tasks, queues: worker.queues, deployment: null };
+  }
+
+  async assertEnvExists(environmentId: string): Promise<void> {
+    if (!this.splitEnabled()) {
+      // Split OFF = single DB, so run and env are co-located and there is no FK/check
+      // to replace (matches main). Skip the hot-path read entirely.
+      return;
+    }
+
+    const cached = this.cache.getEnvExists(environmentId);
+    if (cached !== undefined) {
+      if (!cached) {
+        throw new ControlPlaneReferenceError(
+          `Referenced environment does not exist: ${environmentId}`
+        );
+      }
+      return;
+    }
+
+    const exists = await this.#queryEnvExists(this.controlPlaneReplica, environmentId);
+    this.cache.setEnvExists(environmentId, exists);
+    if (!exists) {
+      throw new ControlPlaneReferenceError(
+        `Referenced environment does not exist: ${environmentId}`
+      );
+    }
+  }
+
+  async #queryEnvExists(client: CpClient, environmentId: string): Promise<boolean> {
+    const env = await client.runtimeEnvironment.findFirst({
+      where: { id: environmentId },
+      select: { id: true },
+    });
+    return env !== null;
+  }
+
+  /**
+   * Drop cached control-plane rows for one environment after a control-plane write to that
+   * env's config. A no-op when split is OFF (nothing is cached), so it is always safe to call.
+   */
+  invalidateEnvironment(environmentId: string): void {
+    this.cache.invalidateEnvironment(environmentId);
+  }
+
+  /**
+   * Drop cached env/authEnv rows for every environment of an organization after a
+   * control-plane write to that org's config. Safe under split OFF (no cache).
+   */
+  invalidateOrganization(organizationId: string): void {
+    this.cache.invalidateOrganization(organizationId);
+  }
+}
+
+// Module-level singleton: wires the real control-plane clients + env split predicate.
+// The control-plane writer/replica are the unchanged `prisma` / `$replica` exports. The
+// split decision is a boot constant derived once from the env predicate (same one the
+// run-ops topology factory uses); the async isSplitEnabled() distinct-DB sentinel is enforced
+// at boot elsewhere and is never awaited on a resolver hot path.
+const SPLIT_ENABLED =
+  env.RUN_OPS_SPLIT_ENABLED && !!env.RUN_OPS_DATABASE_URL && !!env.RUN_OPS_LEGACY_DATABASE_URL;
+
+export const controlPlaneResolver = new ControlPlaneResolver({
+  controlPlanePrimary: prisma,
+  controlPlaneReplica: $replica,
+  // Relax the cache via config. Unset env knobs -> built-in defaults (byte-identical).
+  cache: new ControlPlaneCache({
+    ttlMs: env.CONTROL_PLANE_CACHE_TTL_MS ?? DEFAULT_CP_CACHE_TTL_MS,
+    maxEntries: env.CONTROL_PLANE_CACHE_MAX_ENTRIES ?? DEFAULT_CP_CACHE_MAX_ENTRIES,
+  }),
+  splitEnabled: () => SPLIT_ENABLED,
+});

apps/webapp/app/v3/runOpsMigration/crossSeamGuard.server.ts

@@ -0,0 +1,93 @@
+import { ownerEngine } from "@trigger.dev/core/v3/isomorphic";
+import { isSplitEnabled } from "./splitMode.server";
+import type {
+  CrossSeamGuardDecision,
+  CrossSeamGuardInput,
+  RunOpsResidency,
+  StoreTarget,
+  UnblockRouteKind,
+} from "./types";
+
+const KNOWN_ROUTE_KINDS: ReadonlySet<UnblockRouteKind> = new Set<UnblockRouteKind>([
+  "MANUAL",
+  "DATETIME",
+  "RESUME_TOKEN",
+  "IDEMPOTENCY_REUSE",
+  "RUN",
+]);
+
+// There is NO default store: an unrecognised route is a loud failure.
+function assertKnownRouteKind(routeKind: UnblockRouteKind): void {
+  if (!KNOWN_ROUTE_KINDS.has(routeKind)) {
+    throw new Error(`Unknown unblock routeKind: ${JSON.stringify(routeKind)}`);
+  }
+}
+
+function storeForResidency(residency: RunOpsResidency): StoreTarget {
+  return residency === "NEW" ? "new" : "legacy";
+}
+
+/**
+ * Pin precedence (deterministic, documented order):
+ *   1. non-tree-owned          (treeOwnerResidency === "LEGACY")
+ *   2. cross-tree-idempotency  (isCrossTreeIdempotency === true)
+ *   3. legacy-parent-descendant (hasLegacyParent === true)
+ * Any hit overrides the store to "legacy"; the waitpoint's own residency is
+ * preserved on the decision so callers/metrics can see "NEW pinned to legacy".
+ */
+function applyPinningRules(
+  input: CrossSeamGuardInput
+): CrossSeamGuardDecision["pinnedReason"] | undefined {
+  if (input.treeOwnerResidency === "LEGACY") return "non-tree-owned";
+  if (input.isCrossTreeIdempotency === true) return "cross-tree-idempotency";
+  if (input.hasLegacyParent === true) return "legacy-parent-descendant";
+  return undefined;
+}
+
+/**
+ * Pure store-selection core. No env import, no I/O — driven exhaustively by the
+ * downstream proof harness via the optional `classify` seam.
+ */
+export function selectStoreForWaitpoint(
+  input: CrossSeamGuardInput,
+  deps?: { classify?: (id: string) => RunOpsResidency }
+): CrossSeamGuardDecision {
+  assertKnownRouteKind(input.routeKind);
+
+  const classify = deps?.classify ?? ownerEngine;
+
+  const residency: RunOpsResidency = classify(input.waitpointId);
+
+  const pinnedReason = applyPinningRules(input);
+  const store: StoreTarget = pinnedReason ? "legacy" : storeForResidency(residency);
+
+  return {
+    store,
+    residency,
+    routeKind: input.routeKind,
+    ...(pinnedReason ? { pinnedReason } : {}),
+  };
+}
+
+/**
+ * Pure flag-aware core. In single-DB mode "legacy" IS the single store, so we
+ * return it WITHOUT ever consulting the classifier (off in single-DB). When
+ * split is on, delegate to the pure selection core.
+ */
+export function computeStoreForCompletion(
+  input: CrossSeamGuardInput,
+  opts: { splitEnabled: boolean; classify?: (id: string) => RunOpsResidency }
+): CrossSeamGuardDecision {
+  if (opts.splitEnabled === false) {
+    return { store: "legacy", residency: "LEGACY", routeKind: input.routeKind };
+  }
+  return selectStoreForWaitpoint(input, { classify: opts.classify });
+}
+
+/** Thin server entry the waitpoint-completion consumers call. */
+export async function pickRunOpsStoreForCompletion(
+  input: CrossSeamGuardInput
+): Promise<CrossSeamGuardDecision> {
+  const splitEnabled = await isSplitEnabled();
+  return computeStoreForCompletion(input, { splitEnabled });
+}

apps/webapp/app/v3/runOpsMigration/distinctDbSentinel.server.ts

@@ -0,0 +1,55 @@
+import { PrismaClient } from "@trigger.dev/database";
+
+type DatabaseFingerprint = { systemIdentifier: string; databaseName: string };
+
+async function readDatabaseFingerprint(url: string): Promise<DatabaseFingerprint> {
+  const client = new PrismaClient({ datasources: { db: { url } } });
+  try {
+    const rows = await client.$queryRawUnsafe<
+      Array<{ system_identifier: string; database_name: string }>
+    >(
+      "SELECT system_identifier::text AS system_identifier, current_database() AS database_name FROM pg_control_system()"
+    );
+    const row = rows[0];
+    if (!row) {
+      throw new Error("distinct-db sentinel: pg_control_system() returned no rows");
+    }
+    return { systemIdentifier: row.system_identifier, databaseName: row.database_name };
+  } finally {
+    await client.$disconnect();
+  }
+}
+
+export async function probeDistinctDatabases(
+  legacyUrl: string,
+  newUrl: string,
+  opts?: { logger?: { warn: (msg: string, meta?: Record<string, unknown>) => void } }
+): Promise<{ distinct: true } | { distinct: false; reason: string }> {
+  try {
+    const [legacy, next] = await Promise.all([
+      readDatabaseFingerprint(legacyUrl),
+      readDatabaseFingerprint(newUrl),
+    ]);
+    const sameCluster = legacy.systemIdentifier === next.systemIdentifier;
+    const sameDb = sameCluster && legacy.databaseName === next.databaseName;
+    // Same-cluster-different-database policy: two databases inside the SAME cluster
+    // (same system identifier, different current_database()) are reported distinct: true.
+    // That is acceptable — they are genuinely separate Postgres databases with separate
+    // WAL-visible state for our purposes, and the Cloud topology always uses separate
+    // clusters anyway. A stricter "must be a different cluster" policy would gate on
+    // sameCluster alone; that is flagged as an open question, not decided here.
+    if (sameDb) {
+      const reason =
+        "run-ops legacy and new URLs resolve to the SAME physical database " +
+        `(systemIdentifier=${legacy.systemIdentifier}, database=${legacy.databaseName}); ` +
+        "refusing to enable split — pooler/replica likely.";
+      opts?.logger?.warn(reason);
+      return { distinct: false, reason };
+    }
+    return { distinct: true };
+  } catch (error) {
+    const reason = `distinct-db sentinel probe failed; failing closed (single-DB). ${String(error)}`;
+    opts?.logger?.warn(reason, { error });
+    return { distinct: false, reason };
+  }
+}