chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,105 @@
+import { Logger } from "@trigger.dev/core/logger";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { RunEngineBatchTriggerService } from "~/runEngine/services/batchTrigger.server";
+import { logger } from "~/services/logger.server";
+import { singleton } from "~/utils/singleton";
+import { BatchTriggerV3Service } from "./services/batchTriggerV3.server";
+// Import engine to ensure it's initialized (which initializes BatchQueue for v2 batches)
+import { engine } from "./runEngine.server";
+
+/**
+ * Legacy batch trigger worker for processing v3 and run engine v1 batches.
+ *
+ * NOTE: Run Engine v2 batches (batchVersion: "runengine:v2") use the new BatchQueue
+ * system with Deficit Round Robin scheduling, which is encapsulated within the RunEngine.
+ * See runEngine.server.ts for the configuration.
+ *
+ * This worker is kept for backwards compatibility with:
+ * - v3 batches (batchVersion: "v3") - handled by BatchTriggerV3Service
+ * - Run Engine v1 batches (batchVersion: "runengine:v1") - handled by RunEngineBatchTriggerService
+ */
+function initializeWorker() {
+  // Ensure the engine (and its BatchQueue) is initialized
+  void engine;
+  const redisOptions = {
+    keyPrefix: "batch-trigger:worker:",
+    host: env.BATCH_TRIGGER_WORKER_REDIS_HOST,
+    port: env.BATCH_TRIGGER_WORKER_REDIS_PORT,
+    username: env.BATCH_TRIGGER_WORKER_REDIS_USERNAME,
+    password: env.BATCH_TRIGGER_WORKER_REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  };
+
+  logger.debug(
+    `👨‍🏭 Initializing batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}`
+  );
+
+  const worker = new RedisWorker({
+    name: "batch-trigger-worker",
+    redisOptions,
+    catalog: {
+      "v3.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+        }),
+        visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+      "runengine.processBatchTaskRun": {
+        schema: z.object({
+          batchId: z.string(),
+          processingId: z.string(),
+          range: z.object({ start: z.number().int(), count: z.number().int() }),
+          attemptCount: z.number().int(),
+          strategy: z.enum(["sequential", "parallel"]),
+          parentRunId: z.string().optional(),
+          resumeParentOnCompletion: z.boolean().optional(),
+        }),
+        visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS,
+        retry: {
+          maxAttempts: 5,
+        },
+      },
+    },
+    concurrency: {
+      workers: env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS,
+      tasksPerWorker: env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER,
+      limit: env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT,
+    },
+    pollIntervalMs: env.BATCH_TRIGGER_WORKER_POLL_INTERVAL,
+    immediatePollIntervalMs: env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.BATCH_TRIGGER_WORKER_SHUTDOWN_TIMEOUT_MS,
+    logger: new Logger("BatchTriggerWorker", env.BATCH_TRIGGER_WORKER_LOG_LEVEL),
+    jobs: {
+      "v3.processBatchTaskRun": async ({ payload }) => {
+        const service = new BatchTriggerV3Service(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+      "runengine.processBatchTaskRun": async ({ payload }) => {
+        const service = new RunEngineBatchTriggerService(payload.strategy);
+        await service.processBatchTaskRun(payload);
+      },
+    },
+  });
+
+  if (env.BATCH_TRIGGER_WORKER_ENABLED === "true") {
+    logger.debug(
+      `👨‍🏭 Starting batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}, pollInterval = ${env.BATCH_TRIGGER_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT}`
+    );
+
+    worker.start();
+  }
+
+  return worker;
+}
+
+export const batchTriggerWorker = singleton("batchTriggerWorker", initializeWorker);