import { Logger } from "@trigger.dev/core/logger"; import { Worker as RedisWorker } from "@trigger.dev/redis-worker"; import { z } from "zod"; import { env } from "~/env.server"; import { RunEngineBatchTriggerService } from "~/runEngine/services/batchTrigger.server"; import { logger } from "~/services/logger.server"; import { singleton } from "~/utils/singleton"; import { BatchTriggerV3Service } from "./services/batchTriggerV3.server"; // Import engine to ensure it's initialized (which initializes BatchQueue for v2 batches) import { engine } from "./runEngine.server"; /** * Legacy batch trigger worker for processing v3 and run engine v1 batches. * * NOTE: Run Engine v2 batches (batchVersion: "runengine:v2") use the new BatchQueue * system with Deficit Round Robin scheduling, which is encapsulated within the RunEngine. * See runEngine.server.ts for the configuration. * * This worker is kept for backwards compatibility with: * - v3 batches (batchVersion: "v3") - handled by BatchTriggerV3Service * - Run Engine v1 batches (batchVersion: "runengine:v1") - handled by RunEngineBatchTriggerService */ function initializeWorker() { // Ensure the engine (and its BatchQueue) is initialized void engine; const redisOptions = { keyPrefix: "batch-trigger:worker:", host: env.BATCH_TRIGGER_WORKER_REDIS_HOST, port: env.BATCH_TRIGGER_WORKER_REDIS_PORT, username: env.BATCH_TRIGGER_WORKER_REDIS_USERNAME, password: env.BATCH_TRIGGER_WORKER_REDIS_PASSWORD, enableAutoPipelining: true, ...(env.BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), }; logger.debug( `👨‍🏭 Initializing batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}` ); const worker = new RedisWorker({ name: "batch-trigger-worker", redisOptions, catalog: { "v3.processBatchTaskRun": { schema: z.object({ batchId: z.string(), processingId: z.string(), range: z.object({ start: z.number().int(), count: z.number().int() }), attemptCount: z.number().int(), strategy: z.enum(["sequential", "parallel"]), }), visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS, retry: { maxAttempts: 5, }, }, "runengine.processBatchTaskRun": { schema: z.object({ batchId: z.string(), processingId: z.string(), range: z.object({ start: z.number().int(), count: z.number().int() }), attemptCount: z.number().int(), strategy: z.enum(["sequential", "parallel"]), parentRunId: z.string().optional(), resumeParentOnCompletion: z.boolean().optional(), }), visibilityTimeoutMs: env.BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS, retry: { maxAttempts: 5, }, }, }, concurrency: { workers: env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS, tasksPerWorker: env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER, limit: env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT, }, pollIntervalMs: env.BATCH_TRIGGER_WORKER_POLL_INTERVAL, immediatePollIntervalMs: env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL, shutdownTimeoutMs: env.BATCH_TRIGGER_WORKER_SHUTDOWN_TIMEOUT_MS, logger: new Logger("BatchTriggerWorker", env.BATCH_TRIGGER_WORKER_LOG_LEVEL), jobs: { "v3.processBatchTaskRun": async ({ payload }) => { const service = new BatchTriggerV3Service(payload.strategy); await service.processBatchTaskRun(payload); }, "runengine.processBatchTaskRun": async ({ payload }) => { const service = new RunEngineBatchTriggerService(payload.strategy); await service.processBatchTaskRun(payload); }, }, }); if (env.BATCH_TRIGGER_WORKER_ENABLED === "true") { logger.debug( `👨‍🏭 Starting batch trigger worker at host ${env.BATCH_TRIGGER_WORKER_REDIS_HOST}, pollInterval = ${env.BATCH_TRIGGER_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT}` ); worker.start(); } return worker; } export const batchTriggerWorker = singleton("batchTriggerWorker", initializeWorker);