import { sanitizeQueueName } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server"; import type { LockedBackgroundWorker, QueueManager, QueueProperties, QueueValidationResult, TriggerTaskRequest, } from "../types"; import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server"; import type { RunEngine } from "~/v3/runEngine.server"; import { env } from "~/env.server"; import { tryCatch } from "@trigger.dev/core/v3"; import { ServiceValidationError } from "~/v3/services/common.server"; import { isInfrastructureError } from "~/utils/prismaErrors"; import { createCache, createLRUMemoryStore, DefaultStatefulContext, Namespace, } from "@internal/cache"; import { singleton } from "~/utils/singleton"; import type { TaskMetadataCache, TaskMetadataEntry } from "~/services/taskMetadataCache.server"; import { taskMetadataCacheInstance } from "~/services/taskMetadataCacheInstance.server"; import { recordTaskMetaResolve, type TaskMetaResolveSource, } from "~/services/taskMetadataCacheTelemetry.server"; // LRU cache for environment queue sizes to reduce Redis calls const queueSizeCache = singleton("queueSizeCache", () => { const ctx = new DefaultStatefulContext(); const memory = createLRUMemoryStore(env.QUEUE_SIZE_CACHE_MAX_SIZE, "queue-size-cache"); return createCache({ queueSize: new Namespace(ctx, { stores: [memory], fresh: env.QUEUE_SIZE_CACHE_TTL_MS, stale: env.QUEUE_SIZE_CACHE_TTL_MS + 1000, }), }); }); /** * Extract the queue name from a queue option that may be: * - An object with a string `name` property: { name: "queue-name" } * - A double-wrapped object (bug case): { name: { name: "queue-name", ... } } * * This handles the case where the SDK accidentally double-wraps the queue * option when it's already an object with a name property. */ function extractQueueName(queue: { name?: unknown } | undefined): string | undefined { if (!queue?.name) { return undefined; } // Normal case: queue.name is a string if (typeof queue.name === "string") { return queue.name; } // Double-wrapped case: queue.name is an object with its own name property if (typeof queue.name === "object" && queue.name !== null && "name" in queue.name) { const innerName = (queue.name as { name: unknown }).name; if (typeof innerName === "string") { return innerName; } } return undefined; } export class DefaultQueueManager implements QueueManager { private readonly replicaPrisma: PrismaClientOrTransaction; private readonly taskMetaCache: TaskMetadataCache; constructor( private readonly prisma: PrismaClientOrTransaction, private readonly engine: RunEngine, replicaPrisma?: PrismaClientOrTransaction, taskMetaCache: TaskMetadataCache = taskMetadataCacheInstance ) { this.replicaPrisma = replicaPrisma ?? prisma; this.taskMetaCache = taskMetaCache; } async resolveQueueProperties( request: TriggerTaskRequest, lockedBackgroundWorker?: LockedBackgroundWorker ): Promise { let queueName: string; let lockedQueueId: string | undefined; let taskTtl: string | null | undefined; let taskKind: string | undefined; // Determine queue name based on lockToVersion and provided options if (lockedBackgroundWorker) { // Task is locked to a specific worker version const specifiedQueueName = extractQueueName(request.body.options?.queue); if (specifiedQueueName) { // A specific queue name is provided, validate it exists for the locked worker. // Pre-existing query — not cached because TaskQueue rows can be added or // removed independently of BackgroundWorkerTask, and a stale "queue exists" // claim would silently route to the wrong queue. const specifiedQueue = await this.prisma.taskQueue.findFirst({ where: { name: specifiedQueueName, runtimeEnvironmentId: request.environment.id, workers: { some: { id: lockedBackgroundWorker.id } }, }, }); if (!specifiedQueue) { throw new ServiceValidationError( `Specified queue '${specifiedQueueName}' not found or not associated with locked version '${ lockedBackgroundWorker.version ?? "" }'.` ); } // Use the validated queue name directly queueName = specifiedQueue.name; lockedQueueId = specifiedQueue.id; // Pull `triggerSource` (for `taskKind` annotation) and `ttl` from cache. // On cache hit this is 0 PG queries; on miss the helper falls back to // a BackgroundWorkerTask lookup and back-fills the cache. // // If the task slug isn't on this locked worker version, we tolerate // the missing row and fall through with `taskKind = undefined` // (coalesced to "STANDARD" downstream) and `taskTtl = undefined`. // This matches main's pre-PR behavior — the no-override branch below // still throws because there's no queue to route to in that case, // but here the caller already named the queue. const lockedMeta = await this.resolveLockedTaskMetadata( lockedBackgroundWorker.id, request.environment.id, request.taskId ); if (request.body.options?.ttl === undefined) { taskTtl = lockedMeta?.ttl ?? undefined; } taskKind = lockedMeta?.triggerSource; } else { // No queue override - resolve default queue + TTL + triggerSource via cache, // falling back to a single BackgroundWorkerTask lookup on miss. const lockedMeta = await this.resolveLockedTaskMetadata( lockedBackgroundWorker.id, request.environment.id, request.taskId ); if (!lockedMeta) { throw new ServiceValidationError( `Task '${request.taskId}' not found on locked version '${ lockedBackgroundWorker.version ?? "" }'.` ); } taskTtl = lockedMeta.ttl; if (!lockedMeta.queueName) { // This case should ideally be prevented by earlier checks or schema constraints, // but handle it defensively. logger.error("Task found on locked version, but has no associated queue record", { taskId: request.taskId, workerId: lockedBackgroundWorker.id, version: lockedBackgroundWorker.version, }); throw new ServiceValidationError( `Default queue configuration for task '${request.taskId}' missing on locked version '${ lockedBackgroundWorker.version ?? "" }'.` ); } // Use the task's default queue name queueName = lockedMeta.queueName; lockedQueueId = lockedMeta.queueId ?? undefined; taskKind = lockedMeta.triggerSource; } } else { // Task is not locked to a specific version, use regular logic if (request.body.options?.lockToVersion) { // This should only happen if the findFirst failed, indicating the version doesn't exist throw new ServiceValidationError( `Task locked to version '${request.body.options.lockToVersion}', but no worker found with that version.` ); } // Get queue name using the helper for non-locked case (handles provided name or finds default) const taskInfo = await this.getTaskQueueInfo(request); queueName = taskInfo.queueName; taskTtl = taskInfo.taskTtl; taskKind = taskInfo.taskKind; } // Sanitize the final determined queue name once const sanitizedQueueName = sanitizeQueueName(queueName); // Check that the queuename is not an empty string if (!sanitizedQueueName) { queueName = sanitizeQueueName(`task/${request.taskId}`); // Fallback if sanitization results in empty } else { queueName = sanitizedQueueName; } return { queueName, lockedQueueId, taskTtl, taskKind, }; } private async getTaskQueueInfo( request: TriggerTaskRequest ): Promise<{ queueName: string; taskTtl?: string | null; taskKind?: string | undefined }> { const { taskId, environment, body } = request; const { queue } = body.options ?? {}; // Use extractQueueName to handle double-wrapped queue objects const overriddenQueueName = extractQueueName(queue); const defaultQueueName = `task/${taskId}`; // Resolve the current worker's task metadata via cache (HGET on warm path, // BackgroundWorkerTask findFirst + cache back-fill on miss). When this hits, // both the queue-override + TTL caller and the default-queue caller satisfy // their full result without any database query. const meta = await this.resolveCurrentTaskMetadata(environment, taskId); if (overriddenQueueName) { // Caller already named the queue. We only need triggerSource (for taskKind) // and ttl (for the call site to coalesce against body.options.ttl). return { queueName: overriddenQueueName, taskTtl: meta?.ttl ?? undefined, taskKind: meta?.triggerSource, }; } if (!meta) { logger.debug("Failed to get queue name: No worker or task found", { taskId, environmentId: environment.id, }); return { queueName: defaultQueueName, taskTtl: undefined }; } if (!meta.queueName) { logger.debug("Failed to get queue name: No queue found", { taskId, environmentId: environment.id, }); return { queueName: defaultQueueName, taskTtl: meta.ttl, taskKind: meta.triggerSource }; } return { queueName: meta.queueName, taskTtl: meta.ttl, taskKind: meta.triggerSource }; } /** * Resolve task metadata for a locked-version trigger. Reads from the * `task-meta:by-worker:{workerId}` Redis hash; falls back to a single * BackgroundWorkerTask findFirst on miss and back-fills the cache. * * Returns null when no BackgroundWorkerTask row exists. */ private async resolveLockedTaskMetadata( workerId: string, environmentId: string, slug: string ): Promise { const cached = await this.taskMetaCache.getByWorker(workerId, slug); if (cached) { recordTaskMetaResolve("locked", "cache"); return cached; } // Cache miss. Read the row from the replica first; if the replica comes // back empty, re-check the writer before concluding the task is missing. // The locked worker itself was just resolved on the writer (see // triggerTask.server.ts), so a replica that returns no row here is stale, // not authoritative. Trusting a stale-replica negative throws a // non-retryable "not found on locked version" for a task that is in fact // registered. The writer read only runs on this rare miss-then-empty path, // never on the hot path. let row = await this.findLockedTaskRow(this.replicaPrisma, workerId, environmentId, slug); let source: TaskMetaResolveSource = "replica"; if (!row && this.replicaPrisma !== this.prisma) { row = await this.findLockedTaskRow(this.prisma, workerId, environmentId, slug); if (row) { source = "writer"; logger.warn("Locked task metadata missing on replica but found on writer", { workerId, environmentId, slug, }); } } if (!row) { recordTaskMetaResolve("locked", "miss"); return null; } recordTaskMetaResolve("locked", source); const entry: TaskMetadataEntry = { slug, ttl: row.ttl, triggerSource: row.triggerSource, queueId: row.queue?.id ?? null, queueName: row.queue?.name ?? "", }; // Fire-and-forget back-fill — `setByWorker` upserts the single field and // refreshes the hash TTL. Errors are logged inside the cache and swallowed. void this.taskMetaCache.setByWorker(workerId, entry); return entry; } private findLockedTaskRow( client: PrismaClientOrTransaction, workerId: string, environmentId: string, slug: string ) { return client.backgroundWorkerTask.findFirst({ where: { workerId, runtimeEnvironmentId: environmentId, slug }, select: { ttl: true, triggerSource: true, queue: { select: { id: true, name: true } }, }, }); } /** * Resolve task metadata for a non-locked trigger. Reads from the * `task-meta:env:{envId}` Redis hash; falls back to * findCurrentWorkerFromEnvironment + a single BackgroundWorkerTask findFirst * on miss and back-fills both keyspaces. * * Returns null when no current worker or task can be resolved. */ private async resolveCurrentTaskMetadata( environment: AuthenticatedEnvironment, slug: string ): Promise { const cached = await this.taskMetaCache.getCurrent(environment.id, slug); if (cached) { recordTaskMetaResolve("current", "cache"); return cached; } // Cold cache: discover the current worker for the env. Replica is fine — // the adjacent BackgroundWorkerTask lookup below uses `replicaPrisma` too // (replica lag for "just deployed" is bounded the same way for both // queries; reading from the writer here would only widen the window). const worker = await findCurrentWorkerFromEnvironment(environment, this.replicaPrisma); if (!worker) { recordTaskMetaResolve("current", "miss"); return null; } const row = await this.replicaPrisma.backgroundWorkerTask.findFirst({ where: { workerId: worker.id, runtimeEnvironmentId: environment.id, slug }, select: { ttl: true, triggerSource: true, queue: { select: { id: true, name: true } }, }, }); if (!row) { recordTaskMetaResolve("current", "miss"); return null; } recordTaskMetaResolve("current", "replica"); const entry: TaskMetadataEntry = { slug, ttl: row.ttl, triggerSource: row.triggerSource, queueId: row.queue?.id ?? null, queueName: row.queue?.name ?? "", }; // Fire-and-forget back-fill — atomically upserts the slug into both // keyspaces so a subsequent locked-or-not trigger hits the cache. The // env-keyspace TTL is preserved (promotion owns it); the by-worker TTL // is refreshed (sliding window keeps active workers warm). void this.taskMetaCache.setByCurrentWorker(environment.id, worker.id, entry); return entry; } async validateQueueLimits( environment: AuthenticatedEnvironment, queueName: string, itemsToAdd?: number ): Promise { const queueSizeGuard = await guardQueueSizeLimitsForQueue( this.engine, environment, queueName, itemsToAdd ); logger.debug("Queue size guard result", { queueSizeGuard, queueName, environment: { id: environment.id, type: environment.type, organization: environment.organization, project: environment.project, }, }); return { ok: queueSizeGuard.isWithinLimits, maximumSize: queueSizeGuard.maximumSize ?? 0, queueSize: queueSizeGuard.queueSize ?? 0, }; } async getWorkerQueue( environment: AuthenticatedEnvironment, regionOverride?: string ): Promise<{ masterQueue: string; enableFastPath: boolean } | undefined> { if (environment.type === "DEVELOPMENT") { return { masterQueue: environment.id, enableFastPath: true }; } const workerGroupService = new WorkerGroupService({ prisma: this.prisma, engine: this.engine, }); const [error, workerGroup] = await tryCatch( workerGroupService.getDefaultWorkerGroupForProject({ projectId: environment.projectId, regionOverride, }) ); if (error) { // getDefaultWorkerGroupForProject queries the writer DB. A Prisma // infrastructure error (e.g. P1001 "Can't reach database server", whose // message carries the DB hostname) must NOT be promoted into a // client-facing ServiceValidationError: that leaks internal infra detail // to the API client (the SDK echoes it into the run view) and // mis-classifies a transient outage as a non-retryable 422. Let it // propagate to the route's generic 500 handler (scrubbed + retryable); // only wrap genuine domain failures. if (isInfrastructureError(error)) { throw error; } throw new ServiceValidationError(error.message); } if (!workerGroup) { throw new ServiceValidationError("No worker group found"); } return { masterQueue: workerGroup.masterQueue, enableFastPath: workerGroup.enableFastPath, }; } } export function getMaximumSizeForEnvironment( environment: AuthenticatedEnvironment ): number | undefined { if (environment.type === "DEVELOPMENT") { return environment.organization.maximumDevQueueSize ?? env.MAXIMUM_DEV_QUEUE_SIZE; } else { return environment.organization.maximumDeployedQueueSize ?? env.MAXIMUM_DEPLOYED_QUEUE_SIZE; } } async function guardQueueSizeLimitsForQueue( engine: RunEngine, environment: AuthenticatedEnvironment, queueName: string, itemsToAdd: number = 1 ) { const maximumSize = getMaximumSizeForEnvironment(environment); if (typeof maximumSize === "undefined") { return { isWithinLimits: true }; } const queueSize = await getCachedQueueSize(engine, environment, queueName); const projectedSize = queueSize + itemsToAdd; return { isWithinLimits: projectedSize <= maximumSize, maximumSize, queueSize, }; } async function getCachedQueueSize( engine: RunEngine, environment: AuthenticatedEnvironment, queueName: string ): Promise { if (!env.QUEUE_SIZE_CACHE_ENABLED) { return engine.lengthOfQueue(environment, queueName); } const cacheKey = `${environment.id}:${queueName}`; const result = await queueSizeCache.queueSize.swr(cacheKey, async () => { return engine.lengthOfQueue(environment, queueName); }); return result.val ?? 0; }