chore: import upstream snapshot with attribution

apps/sim/lib/core/async-jobs/backends/database.ts

@@ -0,0 +1,327 @@
+import { asyncJobs, db } from '@sim/db'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateShortId } from '@sim/utils/id'
+import { eq, sql } from 'drizzle-orm'
+import {
+  type EnqueueOptions,
+  JOB_STATUS,
+  type Job,
+  type JobMetadata,
+  type JobQueueBackend,
+  type JobStatus,
+  type JobType,
+} from '@/lib/core/async-jobs/types'
+
+const logger = createLogger('DatabaseJobQueue')
+
+type AsyncJobRow = typeof asyncJobs.$inferSelect
+type Runner = NonNullable<EnqueueOptions['runner']>
+
+function rowToJob(row: AsyncJobRow): Job {
+  return {
+    id: row.id,
+    type: row.type as JobType,
+    payload: row.payload,
+    status: row.status as JobStatus,
+    createdAt: row.createdAt,
+    startedAt: row.startedAt ?? undefined,
+    completedAt: row.completedAt ?? undefined,
+    attempts: row.attempts,
+    maxAttempts: row.maxAttempts,
+    error: row.error ?? undefined,
+    output: row.output as unknown,
+    metadata: (row.metadata ?? {}) as JobMetadata,
+  }
+}
+
+const inlineAbortControllers = new Map<string, AbortController>()
+
+/**
+ * Per-cancel-key abort controllers for the `batchEnqueueAndWait` direct-call
+ * path. Distinct from `inlineAbortControllers` (which keys by jobId) — this
+ * map keys by the domain `cancelKey` callers pass in, since the await-blocking
+ * path skips `async_jobs` entirely and has no jobId to cancel by.
+ */
+const inlineCancelKeyControllers = new Map<string, AbortController>()
+
+interface Semaphore {
+  limit: number
+  available: number
+  waiters: Array<() => void>
+}
+const semaphores = new Map<string, Semaphore>()
+
+async function acquireSlot(key: string, limit: number): Promise<void> {
+  let s = semaphores.get(key)
+  if (!s) {
+    s = { limit, available: limit, waiters: [] }
+    semaphores.set(key, s)
+  }
+  if (s.available > 0) {
+    s.available -= 1
+    return
+  }
+  await new Promise<void>((resolve) => s.waiters.push(resolve))
+}
+
+function releaseSlot(key: string): void {
+  const s = semaphores.get(key)
+  if (!s) return
+  const next = s.waiters.shift()
+  if (next) {
+    next()
+    return
+  }
+  s.available += 1
+  if (s.available === s.limit) {
+    semaphores.delete(key)
+  }
+}
+
+export class DatabaseJobQueue implements JobQueueBackend {
+  async enqueue<TPayload>(
+    type: JobType,
+    payload: TPayload,
+    options?: EnqueueOptions
+  ): Promise<string> {
+    const jobId = options?.jobId ?? `run_${generateShortId(20)}`
+    const now = new Date()
+
+    await db
+      .insert(asyncJobs)
+      .values({
+        id: jobId,
+        type,
+        payload: payload as Record<string, unknown>,
+        status: JOB_STATUS.PENDING,
+        createdAt: now,
+        runAt:
+          options?.delayMs && options.delayMs > 0 ? new Date(now.getTime() + options.delayMs) : now,
+        attempts: 0,
+        maxAttempts: options?.maxAttempts ?? 3,
+        metadata: (options?.metadata ?? {}) as Record<string, unknown>,
+        updatedAt: now,
+      })
+      .onConflictDoNothing()
+
+    logger.debug('Enqueued job', { jobId, type })
+    if (options?.runner) {
+      this.runInline(
+        type,
+        jobId,
+        payload,
+        options.runner,
+        options.concurrencyKey,
+        options.concurrencyLimit
+      )
+    }
+    return jobId
+  }
+
+  async batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    const now = new Date()
+    const rows = items.map(({ payload, options }) => ({
+      id: `run_${generateShortId(20)}`,
+      type,
+      payload: payload as Record<string, unknown>,
+      status: JOB_STATUS.PENDING,
+      createdAt: now,
+      attempts: 0,
+      maxAttempts: options?.maxAttempts ?? 3,
+      metadata: (options?.metadata ?? {}) as Record<string, unknown>,
+      updatedAt: now,
+    }))
+
+    await db.insert(asyncJobs).values(rows)
+
+    logger.debug('Batch-enqueued jobs', { count: rows.length, type })
+
+    for (let i = 0; i < items.length; i++) {
+      const { payload, options } = items[i]
+      if (options?.runner) {
+        this.runInline(
+          type,
+          rows[i].id,
+          payload,
+          options.runner,
+          options.concurrencyKey,
+          options.concurrencyLimit
+        )
+      }
+    }
+
+    return rows.map((r) => r.id)
+  }
+
+  /** Skips `async_jobs` entirely — ids are returned empty since callers can't
+   *  look up rows that don't exist. Cancel goes through `cancelByKey`. */
+  async batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    const tracked: Array<{ key: string; controller: AbortController }> = []
+    const runs = items.map((item) => {
+      const runner = item.options?.runner
+      if (!runner) return Promise.resolve()
+      const controller = new AbortController()
+      const cancelKey = item.options?.cancelKey
+      if (cancelKey) {
+        inlineCancelKeyControllers.set(cancelKey, controller)
+        tracked.push({ key: cancelKey, controller })
+      }
+      return runner(item.payload, controller.signal).catch((err) => {
+        logger.error(`[${type}] Inline run failed`, {
+          cancelKey,
+          error: toError(err).message,
+        })
+      })
+    })
+    try {
+      await Promise.all(runs)
+    } finally {
+      // Compare-and-delete guards against a re-enqueue under the same key
+      // racing with our cleanup.
+      for (const t of tracked) {
+        if (inlineCancelKeyControllers.get(t.key) === t.controller) {
+          inlineCancelKeyControllers.delete(t.key)
+        }
+      }
+    }
+    return items.map(() => '')
+  }
+
+  async getJob(jobId: string): Promise<Job | null> {
+    const [row] = await db.select().from(asyncJobs).where(eq(asyncJobs.id, jobId)).limit(1)
+
+    return row ? rowToJob(row) : null
+  }
+
+  async startJob(jobId: string): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.PROCESSING,
+        startedAt: now,
+        attempts: sql`${asyncJobs.attempts} + 1`,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Started job', { jobId })
+  }
+
+  async completeJob(jobId: string, output: unknown): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.COMPLETED,
+        completedAt: now,
+        output: output as Record<string, unknown>,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Completed job', { jobId })
+  }
+
+  async markJobFailed(jobId: string, error: string): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.FAILED,
+        completedAt: now,
+        error,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Marked job as failed', { jobId })
+  }
+
+  async cancelJob(jobId: string): Promise<void> {
+    // Abort any in-process inline execution first so the running workflow
+    // observes the signal and stops mid-flight. Then mark the row failed so
+    // any future poller skips it.
+    const controller = inlineAbortControllers.get(jobId)
+    let aborted = false
+    if (controller) {
+      controller.abort('Cancelled')
+      inlineAbortControllers.delete(jobId)
+      aborted = true
+    }
+
+    const now = new Date()
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.FAILED,
+        completedAt: now,
+        error: 'Cancelled',
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Marked job as cancelled (DB queue)', { jobId, abortedInline: aborted })
+  }
+
+  cancelByKey(cancelKey: string): boolean {
+    const controller = inlineCancelKeyControllers.get(cancelKey)
+    if (!controller) return false
+    controller.abort('Cancelled')
+    inlineCancelKeyControllers.delete(cancelKey)
+    return true
+  }
+
+  /**
+   * Fire-and-forget IIFE that owns the lifecycle for an inline job: registers
+   * the abort controller (so `cancelJob` can interrupt mid-flight), acquires
+   * a concurrency slot if `concurrencyKey` is set, drives
+   * `startJob → runner → completeJob | markJobFailed`.
+   */
+  private runInline<TPayload>(
+    type: JobType,
+    jobId: string,
+    payload: TPayload,
+    runner: Runner,
+    concurrencyKey?: string,
+    concurrencyLimit?: number
+  ): void {
+    const abortController = new AbortController()
+    inlineAbortControllers.set(jobId, abortController)
+    void (async () => {
+      if (concurrencyKey && concurrencyLimit && concurrencyLimit > 0) {
+        await acquireSlot(concurrencyKey, concurrencyLimit)
+      }
+      try {
+        await this.startJob(jobId)
+        await runner(payload, abortController.signal)
+        await this.completeJob(jobId, null)
+      } catch (err) {
+        const message = toError(err).message
+        logger.error(`[${type}] Inline job ${jobId} failed`, { error: message })
+        try {
+          await this.markJobFailed(jobId, message)
+        } catch (markErr) {
+          logger.error(`[${type}] Failed to mark job ${jobId} as failed`, { markErr })
+        }
+      } finally {
+        inlineAbortControllers.delete(jobId)
+        if (concurrencyKey && concurrencyLimit && concurrencyLimit > 0) {
+          releaseSlot(concurrencyKey)
+        }
+      }
+    })()
+  }
+}

apps/sim/lib/core/async-jobs/backends/trigger-dev.ts

@@ -0,0 +1,260 @@
+import { createLogger } from '@sim/logger'
+import { taskContext } from '@trigger.dev/core/v3'
+import { runs, type TriggerOptions, tasks } from '@trigger.dev/sdk'
+import { resolveTriggerRegion } from '@/lib/core/async-jobs/region'
+import {
+  type EnqueueOptions,
+  JOB_STATUS,
+  type Job,
+  type JobMetadata,
+  type JobQueueBackend,
+  type JobStatus,
+  type JobType,
+} from '@/lib/core/async-jobs/types'
+
+const logger = createLogger('TriggerDevJobQueue')
+
+/**
+ * Maps trigger.dev task IDs to our JobType
+ */
+const JOB_TYPE_TO_TASK_ID: Record<JobType, string> = {
+  'workflow-execution': 'workflow-execution',
+  'schedule-execution': 'schedule-execution',
+  'webhook-execution': 'webhook-execution',
+  'tiktok-webhook-ingress': 'tiktok-webhook-ingress',
+  'resume-execution': 'resume-execution',
+  'workflow-group-cell': 'workflow-group-cell',
+  'cleanup-logs': 'cleanup-logs',
+  'cleanup-soft-deletes': 'cleanup-soft-deletes',
+  'cleanup-tasks': 'cleanup-tasks',
+  'run-data-drain': 'run-data-drain',
+}
+
+/**
+ * Maps trigger.dev run status to our JobStatus
+ */
+function mapTriggerDevStatus(status: string): JobStatus {
+  switch (status) {
+    case 'QUEUED':
+    case 'WAITING_FOR_DEPLOY':
+      return JOB_STATUS.PENDING
+    case 'EXECUTING':
+    case 'RESCHEDULED':
+    case 'FROZEN':
+      return JOB_STATUS.PROCESSING
+    case 'COMPLETED':
+      return JOB_STATUS.COMPLETED
+    case 'CANCELED':
+    case 'FAILED':
+    case 'CRASHED':
+    case 'INTERRUPTED':
+    case 'SYSTEM_FAILURE':
+    case 'EXPIRED':
+      return JOB_STATUS.FAILED
+    default:
+      return JOB_STATUS.PENDING
+  }
+}
+
+/**
+ * Adapter that wraps the trigger.dev SDK to conform to JobQueueBackend interface.
+ */
+export class TriggerDevJobQueue implements JobQueueBackend {
+  async enqueue<TPayload>(
+    type: JobType,
+    payload: TPayload,
+    options?: EnqueueOptions
+  ): Promise<string> {
+    const taskId = JOB_TYPE_TO_TASK_ID[type]
+    if (!taskId) {
+      throw new Error(`Unknown job type: ${type}`)
+    }
+
+    const enrichedPayload =
+      options?.metadata && typeof payload === 'object' && payload !== null
+        ? { ...payload, ...options.metadata }
+        : payload
+
+    const tags = buildTags(options)
+    const triggerOptions: TriggerOptions = {}
+    if (tags.length > 0) triggerOptions.tags = tags
+    if (options?.concurrencyKey) triggerOptions.concurrencyKey = options.concurrencyKey
+    if (options?.jobId) {
+      triggerOptions.idempotencyKey = options.jobId
+      triggerOptions.idempotencyKeyTTL = '14d'
+    }
+    if (options?.delayMs && options.delayMs > 0) {
+      triggerOptions.delay = new Date(Date.now() + options.delayMs)
+    }
+    triggerOptions.region = await resolveTriggerRegion()
+    const handle = await tasks.trigger(taskId, enrichedPayload, triggerOptions)
+
+    logger.debug('Enqueued job via trigger.dev', { jobId: handle.id, type, taskId, tags })
+    return handle.id
+  }
+
+  async batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    // tasks.batchTrigger returns only a batchId, not per-item run IDs, so we
+    // can't use it when callers need to track individual runs (e.g. table cell
+    // tasks need per-row jobIds for cancellation). Sequential `tasks.trigger`
+    // gives us per-item IDs and naturally preserves input order in the queue.
+    const ids: string[] = []
+    for (const { payload, options } of items) {
+      const id = await this.enqueue(type, payload, options)
+      ids.push(id)
+    }
+    return ids
+  }
+
+  async batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    // The SDK's checkpoint-and-resume requires task runtime context. The only
+    // caller (`dispatcherStep` invoked by `tableRunDispatcherTask.run`) is
+    // always inside a task; check defensively so misuse fails at the boundary
+    // instead of as a confusing SDK internal error.
+    if (!taskContext.isInsideTask) {
+      throw new Error(
+        'batchEnqueueAndWait requires trigger.dev task runtime context — call from within a registered task'
+      )
+    }
+
+    const taskId = JOB_TYPE_TO_TASK_ID[type]
+    if (!taskId) throw new Error(`Unknown job type: ${type}`)
+
+    const region = await resolveTriggerRegion()
+    const batchItems = items.map(({ payload, options }) => {
+      const enrichedPayload =
+        options?.metadata && typeof payload === 'object' && payload !== null
+          ? { ...payload, ...options.metadata }
+          : payload
+      const tags = buildTags(options)
+      const batchItem: {
+        payload: unknown
+        options?: { concurrencyKey?: string; tags?: string[]; region?: string }
+      } = { payload: enrichedPayload }
+      const batchOpts: { concurrencyKey?: string; tags?: string[]; region?: string } = { region }
+      if (options?.concurrencyKey) batchOpts.concurrencyKey = options.concurrencyKey
+      if (tags.length > 0) batchOpts.tags = tags
+      batchItem.options = batchOpts
+      return batchItem
+    })
+
+    const result = await tasks.batchTriggerAndWait(taskId, batchItems)
+    logger.debug('batchTriggerAndWait completed', {
+      type,
+      taskId,
+      runCount: result.runs.length,
+    })
+    return result.runs.map((r) => r.id)
+  }
+
+  async getJob(jobId: string): Promise<Job | null> {
+    try {
+      const run = await runs.retrieve(jobId)
+
+      const payload = run.payload as Record<string, unknown>
+      const metadata: JobMetadata = {
+        workflowId: payload?.workflowId as string | undefined,
+        userId: payload?.userId as string | undefined,
+        correlation:
+          payload?.correlation && typeof payload.correlation === 'object'
+            ? (payload.correlation as JobMetadata['correlation'])
+            : undefined,
+      }
+
+      return {
+        id: jobId,
+        type: run.taskIdentifier as JobType,
+        payload: run.payload,
+        status: mapTriggerDevStatus(run.status),
+        createdAt: run.createdAt ? new Date(run.createdAt) : new Date(),
+        startedAt: run.startedAt ? new Date(run.startedAt) : undefined,
+        completedAt: run.finishedAt ? new Date(run.finishedAt) : undefined,
+        attempts: run.attemptCount ?? 1,
+        maxAttempts: 3,
+        error: run.error?.message,
+        output: run.output as unknown,
+        metadata,
+      }
+    } catch (error) {
+      const isNotFound =
+        (error instanceof Error && error.message.toLowerCase().includes('not found')) ||
+        (error && typeof error === 'object' && 'status' in error && error.status === 404)
+
+      if (isNotFound) {
+        logger.debug('Job not found in trigger.dev', { jobId })
+        return null
+      }
+
+      logger.error('Failed to get job from trigger.dev', { jobId, error })
+      throw error
+    }
+  }
+
+  async startJob(_jobId: string): Promise<void> {}
+
+  async completeJob(_jobId: string, _output: unknown): Promise<void> {}
+
+  async markJobFailed(_jobId: string, _error: string): Promise<void> {}
+
+  async cancelJob(jobId: string): Promise<void> {
+    try {
+      await runs.cancel(jobId)
+      logger.debug('Cancelled trigger.dev run', { jobId })
+    } catch (error) {
+      const isNotFound =
+        (error instanceof Error && error.message.toLowerCase().includes('not found')) ||
+        (error && typeof error === 'object' && 'status' in error && error.status === 404)
+      if (isNotFound) {
+        logger.debug('Cancel target not found in trigger.dev (already finished?)', { jobId })
+        return
+      }
+      logger.error('Failed to cancel trigger.dev run', { jobId, error })
+      throw error
+    }
+  }
+
+  cancelByKey(_cancelKey: string): boolean {
+    // No in-process AbortControllers to abort — trigger.dev runs are cancelled
+    // by jobId or via tag sweep (see `cancelCellRunsByTags`). Callers that
+    // need both surfaces should fan out themselves.
+    return false
+  }
+}
+
+/**
+ * Derives trigger.dev tags from job type, metadata, and explicit tags.
+ * Tags follow the `namespace:value` convention for consistent filtering.
+ * Max 10 tags per run, each max 128 chars.
+ */
+function buildTags(options?: EnqueueOptions): string[] {
+  const tags: string[] = []
+  const meta = options?.metadata
+
+  if (meta?.workspaceId) tags.push(`workspaceId:${meta.workspaceId}`)
+  if (meta?.workflowId) tags.push(`workflowId:${meta.workflowId}`)
+  if (meta?.userId) tags.push(`userId:${meta.userId}`)
+
+  if (meta?.correlation) {
+    const c = meta.correlation
+    tags.push(`source:${c.source}`)
+    if (c.webhookId) tags.push(`webhookId:${c.webhookId}`)
+    if (c.scheduleId) tags.push(`scheduleId:${c.scheduleId}`)
+    if (c.provider) tags.push(`provider:${c.provider}`)
+  }
+
+  if (options?.tags) {
+    for (const tag of options.tags) {
+      if (!tags.includes(tag)) tags.push(tag)
+    }
+  }
+
+  return tags.slice(0, 10)
+}

apps/sim/lib/core/async-jobs/config.ts

@@ -0,0 +1,101 @@
+import { createLogger } from '@sim/logger'
+import { taskContext } from '@trigger.dev/core/v3'
+import type { AsyncBackendType, JobQueueBackend } from '@/lib/core/async-jobs/types'
+import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
+
+const logger = createLogger('AsyncJobsConfig')
+
+let cachedBackend: JobQueueBackend | null = null
+let cachedBackendType: AsyncBackendType | null = null
+let cachedInlineBackend: JobQueueBackend | null = null
+
+/**
+ * Determines which async backend to use based on environment configuration.
+ * Falls back to the database backend when trigger.dev isn't enabled — except
+ * when this process IS a trigger.dev worker (`taskContext.isInsideTask`), in
+ * which case the SDK runtime is available regardless of env vars and we
+ * always want to enqueue back through trigger.dev. Without this carve-out, a
+ * worker pod missing `TRIGGER_DEV_ENABLED=true` silently routes cell jobs to
+ * the database backend that nothing's draining.
+ */
+export function getAsyncBackendType(): AsyncBackendType {
+  if (isTriggerDevEnabled || taskContext.isInsideTask) {
+    return 'trigger-dev'
+  }
+
+  return 'database'
+}
+
+/**
+ * Gets the job queue backend singleton.
+ * Creates the appropriate backend based on environment configuration.
+ */
+export async function getJobQueue(): Promise<JobQueueBackend> {
+  if (cachedBackend) {
+    return cachedBackend
+  }
+
+  const type = getAsyncBackendType()
+
+  switch (type) {
+    case 'trigger-dev': {
+      const { TriggerDevJobQueue } = await import('@/lib/core/async-jobs/backends/trigger-dev')
+      cachedBackend = new TriggerDevJobQueue()
+      break
+    }
+    case 'database': {
+      const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
+      cachedBackend = new DatabaseJobQueue()
+      break
+    }
+  }
+
+  cachedBackendType = type
+  logger.info(`Async job backend initialized: ${type}`)
+
+  if (!cachedBackend) {
+    throw new Error(`Failed to initialize async backend: ${type}`)
+  }
+
+  return cachedBackend
+}
+
+/**
+ * Gets the current backend type (for logging/debugging)
+ */
+export function getCurrentBackendType(): AsyncBackendType | null {
+  return cachedBackendType
+}
+
+/**
+ * Gets a job queue backend that bypasses Trigger.dev (Database only).
+ * Used for execution paths that must avoid Trigger.dev cold starts.
+ */
+export async function getInlineJobQueue(): Promise<JobQueueBackend> {
+  if (cachedInlineBackend) {
+    return cachedInlineBackend
+  }
+
+  const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
+  cachedInlineBackend = new DatabaseJobQueue()
+
+  logger.info('Inline job backend initialized: database')
+  return cachedInlineBackend
+}
+
+/**
+ * Checks if jobs should be executed inline in-process.
+ * Database fallback is the only mode that still relies on inline execution.
+ */
+export function shouldExecuteInline(): boolean {
+  return getAsyncBackendType() === 'database'
+}
+
+/**
+ * Resets the cached backend (useful for testing)
+ */
+export function resetJobQueueCache(): void {
+  cachedBackend = null
+  cachedBackendType = null
+  cachedInlineBackend = null
+}

apps/sim/lib/core/async-jobs/index.ts

@@ -0,0 +1,14 @@
+export {
+  getAsyncBackendType,
+  getCurrentBackendType,
+  getInlineJobQueue,
+  getJobQueue,
+  resetJobQueueCache,
+  shouldExecuteInline,
+} from './config'
+export {
+  JOB_MAX_LIFETIME_SECONDS,
+  JOB_RETENTION_HOURS,
+  JOB_RETENTION_SECONDS,
+  JOB_STATUS,
+} from './types'

apps/sim/lib/core/async-jobs/region.test.ts

@@ -0,0 +1,42 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockIsFeatureEnabled } = vi.hoisted(() => ({
+  mockIsFeatureEnabled: vi.fn(),
+}))
+
+vi.mock('@/lib/core/config/feature-flags', () => ({
+  isFeatureEnabled: mockIsFeatureEnabled,
+}))
+
+import {
+  resolveTriggerRegion,
+  TRIGGER_REGION_EU_CENTRAL,
+  TRIGGER_REGION_US_EAST,
+} from '@/lib/core/async-jobs/region'
+
+describe('resolveTriggerRegion', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('returns eu-central-1 when the flag is enabled', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(true)
+    expect(await resolveTriggerRegion()).toBe(TRIGGER_REGION_EU_CENTRAL)
+    expect(mockIsFeatureEnabled).toHaveBeenCalledWith('trigger-eu-region')
+  })
+
+  it('returns us-east-1 when the flag is disabled', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(false)
+    expect(await resolveTriggerRegion()).toBe(TRIGGER_REGION_US_EAST)
+  })
+
+  it('evaluates globally, passing no gating context', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(false)
+    await resolveTriggerRegion()
+    expect(mockIsFeatureEnabled).toHaveBeenCalledTimes(1)
+    expect(mockIsFeatureEnabled.mock.calls[0]).toEqual(['trigger-eu-region'])
+  })
+})

apps/sim/lib/core/async-jobs/region.ts

@@ -0,0 +1,21 @@
+import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
+
+/** Default Trigger.dev region — the project default when the eu-central flag is off. */
+export const TRIGGER_REGION_US_EAST = 'us-east-1'
+
+/** Target region when the `trigger-eu-region` flag is enabled. */
+export const TRIGGER_REGION_EU_CENTRAL = 'eu-central-1'
+
+/**
+ * Resolve which Trigger.dev region a run should execute in. Gated globally by the
+ * `trigger-eu-region` feature flag (all-or-nothing — no per-user/org targeting):
+ * `eu-central-1` when enabled, otherwise `us-east-1`.
+ *
+ * The result is passed as the `region` option to `tasks.trigger` / `batchTrigger`,
+ * overriding the project's dashboard default per run.
+ */
+export async function resolveTriggerRegion(): Promise<string> {
+  return (await isFeatureEnabled('trigger-eu-region'))
+    ? TRIGGER_REGION_EU_CENTRAL
+    : TRIGGER_REGION_US_EAST
+}

apps/sim/lib/core/async-jobs/types.test.ts

@@ -0,0 +1,32 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { JOB_MAX_LIFETIME_SECONDS, JOB_RETENTION_HOURS, JOB_RETENTION_SECONDS } from './types'
+
+describe('Job retention constants', () => {
+  it.concurrent('JOB_RETENTION_HOURS should be 24', async () => {
+    expect(JOB_RETENTION_HOURS).toBe(24)
+  })
+
+  it.concurrent('JOB_RETENTION_SECONDS should be derived from JOB_RETENTION_HOURS', async () => {
+    expect(JOB_RETENTION_SECONDS).toBe(JOB_RETENTION_HOURS * 60 * 60)
+  })
+
+  it.concurrent('JOB_RETENTION_SECONDS should equal 86400 (24 hours)', async () => {
+    expect(JOB_RETENTION_SECONDS).toBe(86400)
+  })
+
+  it.concurrent('constants should be consistent with each other', async () => {
+    const hoursToSeconds = JOB_RETENTION_HOURS * 60 * 60
+    expect(JOB_RETENTION_SECONDS).toBe(hoursToSeconds)
+  })
+
+  it.concurrent(
+    'JOB_MAX_LIFETIME_SECONDS should be greater than JOB_RETENTION_SECONDS',
+    async () => {
+      expect(JOB_MAX_LIFETIME_SECONDS).toBeGreaterThan(JOB_RETENTION_SECONDS)
+      expect(JOB_MAX_LIFETIME_SECONDS).toBe(48 * 60 * 60)
+    }
+  )
+})

apps/sim/lib/core/async-jobs/types.ts

@@ -0,0 +1,189 @@
+/**
+ * Types and constants for the async job queue system
+ */
+
+/** Retention period for completed/failed jobs (in hours) */
+export const JOB_RETENTION_HOURS = 24
+
+/** Retention period for completed/failed jobs (in seconds, for Redis TTL) */
+export const JOB_RETENTION_SECONDS = JOB_RETENTION_HOURS * 60 * 60
+
+/** Max lifetime for jobs in Redis (in seconds) - cleanup for stuck pending/processing jobs */
+export const JOB_MAX_LIFETIME_SECONDS = 48 * 60 * 60
+
+export const JOB_STATUS = {
+  PENDING: 'pending',
+  PROCESSING: 'processing',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+} as const
+
+export type JobStatus = (typeof JOB_STATUS)[keyof typeof JOB_STATUS]
+
+export type JobType =
+  | 'workflow-execution'
+  | 'schedule-execution'
+  | 'webhook-execution'
+  | 'tiktok-webhook-ingress'
+  | 'resume-execution'
+  | 'workflow-group-cell'
+  | 'cleanup-logs'
+  | 'cleanup-soft-deletes'
+  | 'cleanup-tasks'
+  | 'run-data-drain'
+
+export type AsyncExecutionCorrelationSource = 'workflow' | 'schedule' | 'webhook'
+
+export interface AsyncExecutionCorrelation {
+  executionId: string
+  requestId: string
+  source: AsyncExecutionCorrelationSource
+  workflowId: string
+  triggerType?: string
+  webhookId?: string
+  scheduleId?: string
+  path?: string
+  provider?: string
+  scheduledFor?: string
+}
+
+export interface Job<TPayload = unknown, TOutput = unknown> {
+  id: string
+  type: JobType
+  payload: TPayload
+  status: JobStatus
+  createdAt: Date
+  startedAt?: Date
+  completedAt?: Date
+  attempts: number
+  maxAttempts: number
+  error?: string
+  output?: TOutput
+  metadata: JobMetadata
+}
+
+export interface JobMetadata {
+  workflowId?: string
+  workspaceId?: string
+  userId?: string
+  correlation?: AsyncExecutionCorrelation
+  [key: string]: unknown
+}
+
+export interface EnqueueOptions {
+  maxAttempts?: number
+  metadata?: JobMetadata
+  jobId?: string
+  priority?: number
+  name?: string
+  delayMs?: number
+  tags?: string[]
+  /**
+   * Combined with the task's `queue.concurrencyLimit`, caps parallel runs
+   * sharing this key. Trigger.dev enforces server-side; the database backend
+   * enforces in-process via a FIFO semaphore.
+   */
+  concurrencyKey?: string
+  /**
+   * Per-key concurrency cap. Database backend only — trigger.dev reads this
+   * from the task definition (`queue.concurrencyLimit`).
+   */
+  concurrencyLimit?: number
+  /**
+   * Job body invoked when the queue backend lacks an external worker.
+   * Trigger.dev ignores this (its workers execute the task definition);
+   * the database backend kicks it off as a fire-and-forget IIFE so the
+   * row drives through `processing → completed | failed`. Receives the
+   * payload and an `AbortSignal` driven by `cancelJob`.
+   */
+  runner?: <TPayload>(payload: TPayload, signal: AbortSignal) => Promise<void>
+  /**
+   * Stable identity for cancellation lookups on the database backend's
+   * `batchEnqueueAndWait` path (which skips `async_jobs` entirely, so there
+   * is no jobId to cancel by). Lets callers map a domain identity (e.g.
+   * `tableId:rowId:groupId`) to the in-flight `AbortController`. Ignored
+   * by trigger.dev — runs there are cancelled by tag or jobId.
+   */
+  cancelKey?: string
+}
+
+/**
+ * Backend interface for job queue implementations.
+ * All backends must implement this interface.
+ */
+export interface JobQueueBackend {
+  /**
+   * Add a job to the queue
+   */
+  enqueue<TPayload>(type: JobType, payload: TPayload, options?: EnqueueOptions): Promise<string>
+
+  /**
+   * Enqueue multiple jobs as a single batch. Returns one jobId per item, in
+   * input order. Backends preserve input order in queue dispatch (trigger.dev
+   * via tasks.batchTrigger, database via a single multi-row INSERT).
+   */
+  batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]>
+
+  /**
+   * Enqueue a batch and block until every job has reached a terminal state
+   * (completed, failed, or cancelled). The caller — typically a dispatcher
+   * walking work in windows — uses this to gate window N+1 on window N's
+   * completion.
+   *
+   * Backend implementations:
+   * - Trigger.dev: wraps `tasks.batchTriggerAndWait`. MUST be called from
+   *   inside a registered trigger.dev task (the SDK's checkpoint-and-resume
+   *   requires task runtime context). Backends guard with
+   *   `taskContext.isInsideTask` and throw a clear error otherwise.
+   * - Database (in-process): bypasses `async_jobs` entirely. Since the
+   *   caller is awaiting in-process, the row would serve no live purpose
+   *   (no cross-process recovery, no by-id lookup, no semaphore needed —
+   *   window size IS the concurrency cap). Calls the runner directly via
+   *   `Promise.all` and resolves on the runner's exit.
+   */
+  batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]>
+
+  /**
+   * Get a job by ID
+   */
+  getJob(jobId: string): Promise<Job | null>
+
+  /**
+   * Mark a job as started/processing
+   */
+  startJob(jobId: string): Promise<void>
+
+  /**
+   * Mark a job as completed with output
+   */
+  completeJob(jobId: string, output: unknown): Promise<void>
+
+  /**
+   * Mark a job as failed with error message
+   */
+  markJobFailed(jobId: string, error: string): Promise<void>
+
+  /**
+   * Request cancellation of a queued or running job. Best-effort: backends should
+   * fail loudly if the underlying provider rejects, but a missing/unknown jobId
+   * should resolve quietly so callers can drive cancel from possibly-stale state.
+   */
+  cancelJob(jobId: string): Promise<void>
+
+  /**
+   * Cancel an in-flight job by its `cancelKey` (the domain identity callers
+   * stamped on enqueue via `EnqueueOptions.cancelKey`). Used by
+   * `batchEnqueueAndWait` paths that skip per-job ids; the trigger.dev
+   * backend has no in-process AbortControllers to abort and returns `false`.
+   * Returns `true` if a matching controller was found and aborted.
+   */
+  cancelByKey(cancelKey: string): boolean
+}
+
+export type AsyncBackendType = 'trigger-dev' | 'database'