chore: import upstream snapshot with attribution

apps/sim/background/async-execution-correlation.test.ts

@@ -0,0 +1,113 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  describeRetryableInfrastructureError,
+  isRetryableInfrastructureError,
+} from '@/lib/core/errors/retryable-infrastructure'
+import {
+  buildScheduleCorrelation,
+  scheduleExecutionTaskOptions,
+} from '@/background/schedule-execution'
+import { buildWebhookCorrelation } from '@/background/webhook-execution'
+import { buildWorkflowCorrelation } from '@/background/workflow-execution'
+
+describe('async execution correlation fallbacks', () => {
+  it('falls back for legacy workflow payloads missing correlation fields', () => {
+    const correlation = buildWorkflowCorrelation({
+      workflowId: 'workflow-1',
+      userId: 'user-1',
+      triggerType: 'api',
+      executionId: 'execution-legacy',
+    })
+
+    expect(correlation).toEqual({
+      executionId: 'execution-legacy',
+      requestId: 'executio',
+      source: 'workflow',
+      workflowId: 'workflow-1',
+      triggerType: 'api',
+    })
+  })
+
+  it('falls back for legacy schedule payloads missing preassigned request id', () => {
+    const correlation = buildScheduleCorrelation({
+      scheduleId: 'schedule-1',
+      workflowId: 'workflow-1',
+      executionId: 'schedule-exec-1',
+      now: '2025-01-01T00:00:00.000Z',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+    })
+
+    expect(correlation).toEqual({
+      executionId: 'schedule-exec-1',
+      requestId: 'schedule',
+      source: 'schedule',
+      workflowId: 'workflow-1',
+      scheduleId: 'schedule-1',
+      triggerType: 'schedule',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+    })
+  })
+
+  it('caps schedule execution concurrency at the task queue', () => {
+    expect(scheduleExecutionTaskOptions).toMatchObject({
+      queue: {
+        name: 'schedule-execution',
+        concurrencyLimit: 30,
+      },
+    })
+  })
+
+  it('classifies retryable driver causes without treating every failed query as retryable', () => {
+    const driverError = Object.assign(new Error('remaining connection slots are reserved'), {
+      code: '53300',
+    })
+    const drizzleError = new Error('Failed query: select * from "environment"', {
+      cause: driverError,
+    })
+
+    expect(isRetryableInfrastructureError(drizzleError)).toBe(true)
+    expect(describeRetryableInfrastructureError(drizzleError)).toEqual(
+      expect.objectContaining({
+        code: '53300',
+        message: 'remaining connection slots are reserved',
+      })
+    )
+    expect(
+      isRetryableInfrastructureError(new Error('remaining connection slots are reserved'))
+    ).toBe(false)
+    expect(
+      isRetryableInfrastructureError(
+        Object.assign(new Error('connect failed'), { code: 'ETIMEDOUT' })
+      )
+    ).toBe(true)
+    expect(isRetryableInfrastructureError(new Error('Failed query: syntax error'))).toBe(false)
+  })
+
+  it('falls back for legacy webhook payloads missing preassigned fields', () => {
+    const correlation = buildWebhookCorrelation({
+      webhookId: 'webhook-1',
+      workflowId: 'workflow-1',
+      userId: 'user-1',
+      executionId: 'webhook-exec-1',
+      provider: 'slack',
+      body: {},
+      headers: {},
+      path: 'incoming/slack',
+    })
+
+    expect(correlation).toEqual({
+      executionId: 'webhook-exec-1',
+      requestId: 'webhook-',
+      source: 'webhook',
+      workflowId: 'workflow-1',
+      webhookId: 'webhook-1',
+      path: 'incoming/slack',
+      provider: 'slack',
+      triggerType: 'webhook',
+    })
+  })
+})

apps/sim/background/async-preprocessing-correlation.test.ts

@@ -0,0 +1,319 @@
+/**
+ * @vitest-environment node
+ */
+
+import {
+  dbChainMock,
+  dbChainMockFns,
+  executionPreprocessingMock,
+  executionPreprocessingMockFns,
+  LoggingSessionMock,
+  loggingSessionMock,
+  resetDbChainMock,
+  workflowsPersistenceUtilsMock,
+  workflowsPersistenceUtilsMockFns,
+} from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockTask, mockExecuteWorkflowCore, mockGetScheduleTimeValues, mockGetSubBlockValue } =
+  vi.hoisted(() => ({
+    mockTask: vi.fn((config) => config),
+    mockExecuteWorkflowCore: vi.fn(),
+    mockGetScheduleTimeValues: vi.fn(),
+    mockGetSubBlockValue: vi.fn(),
+  }))
+
+const mockPreprocessExecution = executionPreprocessingMockFns.mockPreprocessExecution
+const mockLoadDeployedWorkflowState = workflowsPersistenceUtilsMockFns.mockLoadDeployedWorkflowState
+
+vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
+
+vi.mock('@sim/db', () => ({
+  ...dbChainMock,
+  workflow: {},
+  workflowSchedule: {},
+}))
+
+vi.mock('drizzle-orm', () => ({
+  eq: vi.fn(),
+  and: vi.fn(),
+  isNull: vi.fn(),
+  sql: Object.assign(vi.fn(), { raw: vi.fn() }),
+}))
+
+vi.mock('@/lib/execution/preprocessing', () => executionPreprocessingMock)
+
+vi.mock('@/lib/logs/execution/logging-session', () => loggingSessionMock)
+
+vi.mock('@/lib/core/execution-limits', () => ({
+  createTimeoutAbortController: vi.fn(() => ({
+    signal: undefined,
+    cleanup: vi.fn(),
+    isTimedOut: vi.fn().mockReturnValue(false),
+    timeoutMs: undefined,
+  })),
+  getTimeoutErrorMessage: vi.fn(),
+}))
+
+vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({
+  buildTraceSpans: vi.fn(() => ({ traceSpans: [] })),
+}))
+
+vi.mock('@/lib/workflows/executor/execution-core', () => ({
+  executeWorkflowCore: mockExecuteWorkflowCore,
+  wasExecutionFinalizedByCore: vi.fn().mockReturnValue(false),
+}))
+
+vi.mock('@/lib/workflows/executor/human-in-the-loop-manager', () => ({
+  PauseResumeManager: {
+    persistPauseResult: vi.fn(),
+    processQueuedResumes: vi.fn(),
+  },
+}))
+
+vi.mock('@/lib/workflows/persistence/utils', () => workflowsPersistenceUtilsMock)
+
+vi.mock('@/lib/workflows/schedules/utils', () => ({
+  calculateNextRunTime: vi.fn(),
+  getScheduleTimeValues: mockGetScheduleTimeValues,
+  getSubBlockValue: mockGetSubBlockValue,
+}))
+
+vi.mock('@/executor/execution/snapshot', () => ({
+  ExecutionSnapshot: vi.fn(),
+}))
+
+vi.mock('@/executor/utils/errors', () => ({
+  hasExecutionResult: vi.fn().mockReturnValue(false),
+}))
+
+import { executeScheduleJob } from './schedule-execution'
+import { executeWorkflowJob } from './workflow-execution'
+
+describe('async preprocessing correlation threading', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    dbChainMockFns.limit.mockResolvedValue([
+      {
+        id: 'schedule-1',
+        workflowId: 'workflow-1',
+        status: 'active',
+        archivedAt: null,
+        lastQueuedAt: new Date('2025-01-01T00:00:00.000Z'),
+      },
+    ])
+    mockLoadDeployedWorkflowState.mockResolvedValue({
+      blocks: {
+        'schedule-block': {
+          type: 'schedule',
+        },
+      },
+      edges: [],
+      loops: {},
+      parallels: {},
+      deploymentVersionId: 'deployment-1',
+    })
+    mockGetSubBlockValue.mockReturnValue('daily')
+    mockGetScheduleTimeValues.mockReturnValue({ timezone: 'UTC' })
+  })
+
+  it('does not pre-start workflow logging before core execution', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: true,
+      actorUserId: 'actor-1',
+      workflowRecord: {
+        id: 'workflow-1',
+        userId: 'owner-1',
+        workspaceId: 'workspace-1',
+        variables: {},
+      },
+      executionTimeout: {},
+    })
+    mockExecuteWorkflowCore.mockResolvedValueOnce({
+      success: true,
+      status: 'success',
+      output: { ok: true },
+      metadata: { duration: 10, userId: 'actor-1' },
+    })
+
+    await executeWorkflowJob({
+      workflowId: 'workflow-1',
+      userId: 'user-1',
+      triggerType: 'api',
+      executionId: 'execution-1',
+      requestId: 'request-1',
+    })
+
+    const loggingSession = LoggingSessionMock.mock.results[0]?.value
+    expect(loggingSession).toBeDefined()
+    expect(loggingSession.safeStart).not.toHaveBeenCalled()
+    expect(mockExecuteWorkflowCore).toHaveBeenCalledWith(
+      expect.objectContaining({
+        loggingSession,
+      })
+    )
+  })
+
+  it('does not pre-start schedule logging before core execution', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: true,
+      actorUserId: 'actor-2',
+      workflowRecord: {
+        id: 'workflow-1',
+        userId: 'owner-1',
+        workspaceId: 'workspace-1',
+        variables: {},
+      },
+      executionTimeout: {},
+    })
+    mockExecuteWorkflowCore.mockResolvedValueOnce({
+      success: true,
+      status: 'success',
+      output: { ok: true },
+      metadata: { duration: 12, userId: 'actor-2' },
+    })
+
+    await executeScheduleJob({
+      scheduleId: 'schedule-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-2',
+      requestId: 'request-2',
+      now: '2025-01-01T00:00:00.000Z',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+    })
+
+    const loggingSession = LoggingSessionMock.mock.results[0]?.value
+    expect(loggingSession).toBeDefined()
+    expect(loggingSession.safeStart).not.toHaveBeenCalled()
+    expect(mockExecuteWorkflowCore).toHaveBeenCalledWith(
+      expect.objectContaining({
+        loggingSession,
+      })
+    )
+  })
+
+  it('passes workflow correlation into preprocessing', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: false,
+      error: { message: 'preprocessing failed', statusCode: 500, logCreated: true },
+    })
+
+    await expect(
+      executeWorkflowJob({
+        workflowId: 'workflow-1',
+        userId: 'user-1',
+        triggerType: 'api',
+        executionId: 'execution-1',
+        requestId: 'request-1',
+      })
+    ).rejects.toThrow('preprocessing failed')
+
+    expect(mockPreprocessExecution).toHaveBeenCalledWith(
+      expect.objectContaining({
+        triggerData: {
+          correlation: {
+            executionId: 'execution-1',
+            requestId: 'request-1',
+            source: 'workflow',
+            workflowId: 'workflow-1',
+            triggerType: 'api',
+          },
+        },
+      })
+    )
+  })
+
+  it('passes schedule correlation into preprocessing', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: false,
+      error: { message: 'auth failed', statusCode: 401, logCreated: true },
+    })
+
+    await executeScheduleJob({
+      scheduleId: 'schedule-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-2',
+      requestId: 'request-2',
+      now: '2025-01-01T00:00:00.000Z',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+    })
+
+    expect(mockPreprocessExecution).toHaveBeenCalledWith(
+      expect.objectContaining({
+        triggerData: {
+          correlation: {
+            executionId: 'execution-2',
+            requestId: 'request-2',
+            source: 'schedule',
+            workflowId: 'workflow-1',
+            scheduleId: 'schedule-1',
+            triggerType: 'schedule',
+            scheduledFor: '2025-01-01T00:00:00.000Z',
+          },
+        },
+      })
+    )
+  })
+
+  it('increments infrastructure retry count for retryable schedule preprocessing failures', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: false,
+      error: {
+        message: 'database unavailable',
+        statusCode: 500,
+        logCreated: true,
+        retryable: true,
+        cause: { code: '53300' },
+      },
+    })
+
+    await executeScheduleJob({
+      scheduleId: 'schedule-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-retry',
+      requestId: 'request-retry',
+      now: '2025-01-01T00:00:00.000Z',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+      infraRetryCount: 2,
+    })
+
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({
+        lastQueuedAt: null,
+        infraRetryCount: 3,
+      })
+    )
+  })
+
+  it('moves exhausted infrastructure retries onto the normal failure path', async () => {
+    mockPreprocessExecution.mockResolvedValueOnce({
+      success: false,
+      error: {
+        message: 'database unavailable',
+        statusCode: 500,
+        logCreated: true,
+        retryable: true,
+        cause: { code: '53300' },
+      },
+    })
+
+    await executeScheduleJob({
+      scheduleId: 'schedule-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-retry-exhausted',
+      requestId: 'request-retry-exhausted',
+      now: '2025-01-01T00:00:00.000Z',
+      scheduledFor: '2025-01-01T00:00:00.000Z',
+      infraRetryCount: 10,
+    })
+
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({
+        lastQueuedAt: null,
+        lastFailedAt: expect.any(Date),
+        infraRetryCount: 0,
+      })
+    )
+  })
+})

apps/sim/background/cleanup-logs.test.ts

@@ -0,0 +1,311 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+interface CleanupRow {
+  id: string
+  files: unknown
+}
+
+interface CapturedBatchDeleteOptions {
+  selectChunk: (chunkIds: string[], limit: number) => Promise<unknown>
+  onBatch?: (rows: CleanupRow[]) => Promise<void>
+  batchSize?: number
+  maxBatches?: number
+  totalRowLimit?: number
+}
+
+const {
+  mockAnd,
+  mockBatchDeleteByWorkspaceAndTimestamp,
+  mockChunkedBatchDelete,
+  mockDeleteFileMetadata,
+  mockDeleteFiles,
+  mockEq,
+  mockExecute,
+  mockFrom,
+  mockInArray,
+  mockIsNull,
+  mockLeftJoin,
+  mockLimit,
+  mockLt,
+  mockMarkLargeValuesDeleted,
+  mockNotInArray,
+  mockOr,
+  mockOrderBy,
+  mockPruneLargeValueMetadata,
+  mockSelect,
+  mockTask,
+  mockWhere,
+} = vi.hoisted(() => {
+  const mockLimit = vi.fn(async () => [])
+  const mockOrderBy = vi.fn(() => ({ limit: mockLimit }))
+  const mockWhere = vi.fn(() => ({ limit: mockLimit, orderBy: mockOrderBy }))
+  const mockLeftJoin = vi.fn(() => ({ where: mockWhere }))
+  const mockFrom = vi.fn(() => ({ leftJoin: mockLeftJoin, where: mockWhere }))
+  const mockSelect = vi.fn(() => ({ from: mockFrom }))
+
+  return {
+    mockAnd: vi.fn((...args: unknown[]) => ({ op: 'and', args })),
+    mockBatchDeleteByWorkspaceAndTimestamp: vi.fn(async () => ({
+      table: 'job',
+      deleted: 0,
+      failed: 0,
+    })),
+    mockChunkedBatchDelete: vi.fn(),
+    mockDeleteFileMetadata: vi.fn(async () => true),
+    mockDeleteFiles: vi.fn(async () => ({ deleted: 2, failed: [] })),
+    mockEq: vi.fn((...args: unknown[]) => ({ op: 'eq', args })),
+    mockExecute: vi.fn(),
+    mockFrom,
+    mockInArray: vi.fn((...args: unknown[]) => ({ op: 'inArray', args })),
+    mockIsNull: vi.fn((...args: unknown[]) => ({ op: 'isNull', args })),
+    mockLeftJoin,
+    mockLimit,
+    mockLt: vi.fn((...args: unknown[]) => ({ op: 'lt', args })),
+    mockMarkLargeValuesDeleted: vi.fn(async () => undefined),
+    mockNotInArray: vi.fn((...args: unknown[]) => ({ op: 'notInArray', args })),
+    mockOr: vi.fn((...args: unknown[]) => ({ op: 'or', args })),
+    mockOrderBy,
+    mockPruneLargeValueMetadata: vi.fn(async () => ({
+      referencesDeleted: 0,
+      dependenciesDeleted: 0,
+      tombstonesDeleted: 0,
+    })),
+    mockSelect,
+    mockTask: vi.fn((config: unknown) => config),
+    mockWhere,
+  }
+})
+
+vi.mock('@sim/db', () => ({
+  db: {
+    execute: mockExecute,
+    select: mockSelect,
+  },
+}))
+
+vi.mock('@sim/db/schema', () => ({
+  executionLargeValueDependencies: {
+    childKey: 'executionLargeValueDependencies.childKey',
+    parentKey: 'executionLargeValueDependencies.parentKey',
+    workspaceId: 'executionLargeValueDependencies.workspaceId',
+  },
+  executionLargeValueReferences: {
+    executionId: 'executionLargeValueReferences.executionId',
+    key: 'executionLargeValueReferences.key',
+    source: 'executionLargeValueReferences.source',
+  },
+  executionLargeValues: {
+    createdAt: 'executionLargeValues.createdAt',
+    deletedAt: 'executionLargeValues.deletedAt',
+    key: 'executionLargeValues.key',
+    workspaceId: 'executionLargeValues.workspaceId',
+  },
+  jobExecutionLogs: {
+    startedAt: 'jobExecutionLogs.startedAt',
+    workspaceId: 'jobExecutionLogs.workspaceId',
+  },
+  pausedExecutions: {
+    executionId: 'pausedExecutions.executionId',
+    status: 'pausedExecutions.status',
+  },
+  workspaceFiles: {
+    context: 'workspaceFiles.context',
+    deletedAt: 'workspaceFiles.deletedAt',
+    key: 'workspaceFiles.key',
+    uploadedAt: 'workspaceFiles.uploadedAt',
+    workspaceId: 'workspaceFiles.workspaceId',
+  },
+  workflowExecutionLogs: {
+    executionData: 'workflowExecutionLogs.executionData',
+    executionId: 'workflowExecutionLogs.executionId',
+    files: 'workflowExecutionLogs.files',
+    id: 'workflowExecutionLogs.id',
+    startedAt: 'workflowExecutionLogs.startedAt',
+    workspaceId: 'workflowExecutionLogs.workspaceId',
+  },
+}))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: vi.fn(() => ({
+    error: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+  })),
+}))
+
+vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
+
+vi.mock('drizzle-orm', () => ({
+  and: mockAnd,
+  asc: vi.fn((column: unknown) => ({ op: 'asc', column })),
+  eq: mockEq,
+  inArray: mockInArray,
+  isNull: mockIsNull,
+  lt: mockLt,
+  notInArray: mockNotInArray,
+  or: mockOr,
+  sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values })),
+}))
+
+vi.mock('@/lib/cleanup/batch-delete', () => ({
+  batchDeleteByWorkspaceAndTimestamp: mockBatchDeleteByWorkspaceAndTimestamp,
+  chunkArray: (items: string[], size: number) => {
+    const chunks: string[][] = []
+    for (let index = 0; index < items.length; index += size) {
+      chunks.push(items.slice(index, index + size))
+    }
+    return chunks
+  },
+  chunkedBatchDelete: mockChunkedBatchDelete,
+}))
+
+vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({
+  LIVE_PAUSED_REFERENCE_STATUSES: ['paused', 'partially_resumed', 'cancelling'],
+  markLargeValuesDeleted: mockMarkLargeValuesDeleted,
+  pruneLargeValueMetadata: mockPruneLargeValueMetadata,
+  unreferencedLargeValuePredicate: vi.fn(() => ({ op: 'unreferencedLargeValuePredicate' })),
+}))
+
+vi.mock('@/lib/logs/execution/snapshot/service', () => ({
+  snapshotService: {
+    cleanupOrphanedSnapshots: vi.fn(async () => 0),
+  },
+}))
+
+vi.mock('@/lib/uploads', () => ({
+  isUsingCloudStorage: vi.fn(() => true),
+  StorageService: {
+    deleteFiles: mockDeleteFiles,
+  },
+}))
+
+vi.mock('@/lib/uploads/server/metadata', () => ({
+  deleteFileMetadata: mockDeleteFileMetadata,
+}))
+
+import { cleanupLogsTask, runCleanupLogs } from '@/background/cleanup-logs'
+
+describe('cleanup logs worker', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockChunkedBatchDelete.mockImplementation(async (options: CapturedBatchDeleteOptions) => {
+      await options.selectChunk(['workspace-1'], 500)
+      await options.onBatch?.([
+        {
+          id: 'log-1',
+          files: [
+            { key: 'execution-file-a' },
+            { key: 'execution-file-a' },
+            { key: 'execution-file-b' },
+          ],
+        },
+      ])
+      return { table: 'workflow_execution_logs', deleted: 1, failed: 0 }
+    })
+  })
+
+  it('cleans logs without selecting execution_data or scanning refs', async () => {
+    await runCleanupLogs({
+      label: 'free/1',
+      plan: 'free',
+      retentionHours: 720,
+      workspaceIds: ['workspace-1'],
+    })
+
+    expect(mockChunkedBatchDelete).toHaveBeenCalledWith(
+      expect.objectContaining({
+        batchSize: 500,
+        maxBatches: 50,
+        totalRowLimit: 25_000,
+      })
+    )
+    expect(mockSelect).toHaveBeenCalledWith({
+      id: 'workflowExecutionLogs.id',
+      files: 'workflowExecutionLogs.files',
+    })
+    expect(mockExecute).not.toHaveBeenCalled()
+    expect(mockDeleteFiles).toHaveBeenCalledWith(
+      ['execution-file-a', 'execution-file-b'],
+      'execution'
+    )
+    expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(2)
+    expect(mockPruneLargeValueMetadata).toHaveBeenCalledWith(
+      expect.objectContaining({ workspaceIds: ['workspace-1'] })
+    )
+    expect(mockBatchDeleteByWorkspaceAndTimestamp).toHaveBeenCalledOnce()
+  })
+
+  it('does not count large values as deleted when deleted_at marking fails', async () => {
+    const largeValueKey =
+      'execution/workspace-1/workflow-1/execution-1/large-value-lv_abcdefghijkl.json'
+    mockLimit.mockResolvedValueOnce([]).mockResolvedValueOnce([{ key: largeValueKey }])
+    mockDeleteFiles
+      .mockResolvedValueOnce({ deleted: 2, failed: [] })
+      .mockResolvedValueOnce({ deleted: 1, failed: [] })
+    mockMarkLargeValuesDeleted.mockRejectedValueOnce(new Error('db unavailable'))
+
+    await runCleanupLogs({
+      label: 'free/1',
+      plan: 'free',
+      retentionHours: 720,
+      workspaceIds: ['workspace-1'],
+    })
+
+    expect(mockMarkLargeValuesDeleted).toHaveBeenCalledWith([largeValueKey])
+    expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(2)
+  })
+
+  it('cleans legacy large values from file metadata without selecting execution_data', async () => {
+    const legacyKey =
+      'execution/workspace-1/workflow-1/execution-1/large-value-lv_abcdefghijkl.json'
+    mockLimit
+      .mockResolvedValueOnce([])
+      .mockResolvedValueOnce([])
+      .mockResolvedValueOnce([{ key: legacyKey }])
+    mockDeleteFiles
+      .mockResolvedValueOnce({ deleted: 2, failed: [] })
+      .mockResolvedValueOnce({ deleted: 1, failed: [] })
+
+    await runCleanupLogs({
+      label: 'free/1',
+      plan: 'free',
+      retentionHours: 720,
+      workspaceIds: ['workspace-1'],
+    })
+
+    expect(mockSelect).toHaveBeenCalledWith({
+      id: 'workflowExecutionLogs.id',
+      files: 'workflowExecutionLogs.files',
+    })
+    expect(mockSelect).not.toHaveBeenCalledWith(
+      expect.objectContaining({ executionData: expect.anything() })
+    )
+    const legacyWhereArgs = mockAnd.mock.calls
+      .flat()
+      .filter((arg): arg is { strings: string[] } => {
+        return (
+          typeof arg === 'object' &&
+          arg !== null &&
+          Array.isArray((arg as { strings?: unknown }).strings)
+        )
+      })
+      .map((arg) => arg.strings.join(' '))
+      .join(' ')
+    expect(legacyWhereArgs).toContain('FROM ')
+    expect(legacyWhereArgs).toContain("ref.source = 'execution_log'")
+    expect(legacyWhereArgs).toContain("ref.source = 'paused_snapshot'")
+    expect(legacyWhereArgs).toContain('dependency.child_key')
+    expect(mockDeleteFiles).toHaveBeenLastCalledWith([legacyKey], 'execution')
+    expect(mockDeleteFileMetadata).toHaveBeenCalledWith(legacyKey)
+  })
+
+  it('caps Trigger.dev concurrency for log cleanup tasks', () => {
+    expect(cleanupLogsTask).toMatchObject({
+      queue: { concurrencyLimit: 2 },
+    })
+  })
+})

apps/sim/background/cleanup-logs.ts

@@ -0,0 +1,483 @@
+import { db } from '@sim/db'
+import {
+  executionLargeValueDependencies,
+  executionLargeValueReferences,
+  executionLargeValues,
+  jobExecutionLogs,
+  pausedExecutions,
+  workflowExecutionLogs,
+  workspaceFiles,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { and, asc, eq, inArray, isNull, lt, notInArray, or, sql } from 'drizzle-orm'
+import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher'
+import {
+  batchDeleteByWorkspaceAndTimestamp,
+  chunkArray,
+  chunkedBatchDelete,
+  type TableCleanupResult,
+} from '@/lib/cleanup/batch-delete'
+import {
+  LIVE_PAUSED_REFERENCE_STATUSES,
+  markLargeValuesDeleted,
+  pruneLargeValueMetadata,
+  unreferencedLargeValuePredicate,
+} from '@/lib/execution/payloads/large-value-metadata'
+import { snapshotService } from '@/lib/logs/execution/snapshot/service'
+import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
+import { deleteFileMetadata } from '@/lib/uploads/server/metadata'
+
+const logger = createLogger('CleanupLogs')
+
+interface FileDeleteStats {
+  filesTotal: number
+  filesDeleted: number
+  filesDeleteFailed: number
+}
+
+const WORKFLOW_LOG_CLEANUP_BATCH_SIZE = 500
+const WORKFLOW_LOG_CLEANUP_MAX_BATCHES = 50
+const WORKFLOW_LOG_CLEANUP_ROW_LIMIT =
+  WORKFLOW_LOG_CLEANUP_BATCH_SIZE * WORKFLOW_LOG_CLEANUP_MAX_BATCHES
+const LOG_CLEANUP_CONCURRENCY_LIMIT = 2
+const LARGE_VALUE_CLEANUP_BATCH_SIZE = 500
+const LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT = 5_000
+const LARGE_VALUE_CLEANUP_GRACE_HOURS = 7 * 24
+const LEGACY_LARGE_VALUE_CLEANUP_GRACE_HOURS = 30 * 24
+const LARGE_VALUE_TOMBSTONE_RETENTION_HOURS = 30 * 24
+
+async function deleteExecutionFiles(files: unknown, stats: FileDeleteStats): Promise<void> {
+  if (!isUsingCloudStorage() || !files || !Array.isArray(files)) return
+
+  const keys = Array.from(
+    new Set(files.filter((f) => f && typeof f === 'object' && f.key).map((f) => f.key as string))
+  )
+  stats.filesTotal += keys.length
+  if (keys.length === 0) return
+
+  let result: Awaited<ReturnType<typeof StorageService.deleteFiles>>
+  try {
+    result = await StorageService.deleteFiles(keys, 'execution')
+  } catch (error) {
+    stats.filesDeleteFailed += keys.length
+    logger.error('Failed to bulk delete execution files:', { error })
+    return
+  }
+
+  const failedKeys = new Set(result.failed.map(({ key }) => key))
+  stats.filesDeleted += result.deleted
+  stats.filesDeleteFailed += result.failed.length
+
+  for (const { key, error } of result.failed) {
+    logger.error(`Failed to delete file ${key}:`, { error })
+  }
+  for (const key of keys) {
+    if (failedKeys.has(key)) continue
+    try {
+      await deleteFileMetadata(key)
+    } catch (metadataError) {
+      stats.filesDeleteFailed++
+      logger.error(`Failed to delete file metadata ${key}:`, { metadataError })
+    }
+  }
+}
+
+interface LargeValueCleanupStats {
+  largeValuesTotal: number
+  largeValuesDeleted: number
+  largeValuesDeleteFailed: number
+}
+
+async function deleteLargeValueKeys(keys: string[]): Promise<{ deleted: number; failed: number }> {
+  if (!isUsingCloudStorage() || keys.length === 0) {
+    return { deleted: 0, failed: 0 }
+  }
+
+  let result: Awaited<ReturnType<typeof StorageService.deleteFiles>>
+  try {
+    result = await StorageService.deleteFiles(keys, 'execution')
+  } catch (error) {
+    logger.error('Failed to bulk delete large execution values:', { error })
+    return { deleted: 0, failed: keys.length }
+  }
+
+  const failedKeys = new Set(result.failed.map(({ key }) => key))
+  const deletedKeys = keys.filter((key) => !failedKeys.has(key))
+
+  if (deletedKeys.length > 0) {
+    try {
+      await markLargeValuesDeleted(deletedKeys)
+    } catch (error) {
+      logger.error('Failed to mark large execution values as deleted:', { error })
+      return { deleted: 0, failed: result.failed.length + deletedKeys.length }
+    }
+  }
+
+  for (const { key, error } of result.failed) {
+    logger.error(`Failed to delete large execution value ${key}:`, { error })
+  }
+
+  for (const key of deletedKeys) {
+    try {
+      await deleteFileMetadata(key)
+    } catch (metadataError) {
+      logger.error(`Failed to delete large execution value metadata ${key}:`, { metadataError })
+    }
+  }
+
+  return { deleted: deletedKeys.length, failed: result.failed.length }
+}
+
+async function cleanupLargeExecutionValues(
+  workspaceIds: string[],
+  retentionDate: Date,
+  label: string
+): Promise<LargeValueCleanupStats> {
+  const stats: LargeValueCleanupStats = {
+    largeValuesTotal: 0,
+    largeValuesDeleted: 0,
+    largeValuesDeleteFailed: 0,
+  }
+  if (workspaceIds.length === 0) return stats
+
+  const largeValueRetentionDate = new Date(
+    retentionDate.getTime() - LARGE_VALUE_CLEANUP_GRACE_HOURS * 60 * 60 * 1000
+  )
+  const workspaceChunks = chunkArray(workspaceIds, 50)
+  let attempted = 0
+
+  for (const chunkIds of workspaceChunks) {
+    while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) {
+      const limit = Math.min(
+        LARGE_VALUE_CLEANUP_BATCH_SIZE,
+        LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted
+      )
+      const rows = await db
+        .select({ key: executionLargeValues.key })
+        .from(executionLargeValues)
+        .where(
+          and(
+            inArray(executionLargeValues.workspaceId, chunkIds),
+            isNull(executionLargeValues.deletedAt),
+            lt(executionLargeValues.createdAt, largeValueRetentionDate),
+            unreferencedLargeValuePredicate()
+          )
+        )
+        .orderBy(
+          asc(executionLargeValues.workspaceId),
+          asc(executionLargeValues.createdAt),
+          asc(executionLargeValues.key)
+        )
+        .limit(limit)
+
+      if (rows.length === 0) break
+
+      const keys = rows.map((row) => row.key)
+      stats.largeValuesTotal += keys.length
+      attempted += keys.length
+      const result = await deleteLargeValueKeys(keys)
+      stats.largeValuesDeleted += result.deleted
+      stats.largeValuesDeleteFailed += result.failed
+
+      if (result.deleted === 0) {
+        break
+      }
+    }
+
+    if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) break
+  }
+
+  logger.info(
+    `[${label}/execution_large_values] Complete: ${stats.largeValuesDeleted}/${stats.largeValuesTotal} deleted, ${stats.largeValuesDeleteFailed} failed`
+  )
+
+  return stats
+}
+
+async function cleanupLegacyLargeExecutionValues(
+  workspaceIds: string[],
+  retentionDate: Date,
+  label: string
+): Promise<LargeValueCleanupStats> {
+  const stats: LargeValueCleanupStats = {
+    largeValuesTotal: 0,
+    largeValuesDeleted: 0,
+    largeValuesDeleteFailed: 0,
+  }
+  if (workspaceIds.length === 0) return stats
+
+  const legacyRetentionDate = new Date(
+    retentionDate.getTime() - LEGACY_LARGE_VALUE_CLEANUP_GRACE_HOURS * 60 * 60 * 1000
+  )
+  const workspaceChunks = chunkArray(workspaceIds, 50)
+  let attempted = 0
+
+  for (const chunkIds of workspaceChunks) {
+    while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) {
+      const limit = Math.min(
+        LARGE_VALUE_CLEANUP_BATCH_SIZE,
+        LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted
+      )
+      const rows = await db
+        .select({ key: workspaceFiles.key })
+        .from(workspaceFiles)
+        .where(
+          and(
+            inArray(workspaceFiles.workspaceId, chunkIds),
+            eq(workspaceFiles.context, 'execution'),
+            isNull(workspaceFiles.deletedAt),
+            lt(workspaceFiles.uploadedAt, legacyRetentionDate),
+            sql`${workspaceFiles.key} LIKE 'execution/%/%/%/large-value-lv_%.json'`,
+            sql`NOT EXISTS (
+              SELECT 1
+              FROM ${executionLargeValues} AS registered_value
+              WHERE registered_value.key = ${workspaceFiles.key}
+            )`,
+            sql`NOT EXISTS (
+              SELECT 1
+              FROM ${executionLargeValueReferences} AS ref
+              WHERE ref.key = ${workspaceFiles.key}
+                AND (
+                  (
+                    ref.source = 'execution_log'
+                    AND EXISTS (
+                      SELECT 1
+                      FROM ${workflowExecutionLogs} AS ref_wel
+                      WHERE ref_wel.execution_id = ref.execution_id
+                    )
+                  )
+                  OR (
+                    ref.source = 'paused_snapshot'
+                    AND EXISTS (
+                      SELECT 1
+                      FROM ${pausedExecutions} AS ref_pe
+                      WHERE ref_pe.execution_id = ref.execution_id
+                        AND ref_pe.status = ANY(${LIVE_PAUSED_REFERENCE_STATUSES}::text[])
+                    )
+                  )
+                )
+            )`,
+            sql`NOT EXISTS (
+              SELECT 1
+              FROM ${executionLargeValueDependencies} AS dependency
+              INNER JOIN ${executionLargeValues} AS parent_value
+                ON parent_value.key = dependency.parent_key
+               AND parent_value.deleted_at IS NULL
+              WHERE dependency.child_key = ${workspaceFiles.key}
+                AND dependency.workspace_id = ${workspaceFiles.workspaceId}
+                AND (
+                  EXISTS (
+                    SELECT 1
+                    FROM ${workflowExecutionLogs} AS parent_owner_wel
+                    WHERE parent_owner_wel.execution_id = parent_value.owner_execution_id
+                  )
+                  OR EXISTS (
+                    SELECT 1
+                    FROM ${pausedExecutions} AS parent_owner_pe
+                    WHERE parent_owner_pe.execution_id = parent_value.owner_execution_id
+                      AND parent_owner_pe.status = ANY(${LIVE_PAUSED_REFERENCE_STATUSES}::text[])
+                  )
+                  OR EXISTS (
+                    SELECT 1
+                    FROM ${executionLargeValueReferences} AS parent_ref
+                    WHERE parent_ref.key = parent_value.key
+                      AND (
+                        (
+                          parent_ref.source = 'execution_log'
+                          AND EXISTS (
+                            SELECT 1
+                            FROM ${workflowExecutionLogs} AS parent_ref_wel
+                            WHERE parent_ref_wel.execution_id = parent_ref.execution_id
+                          )
+                        )
+                        OR (
+                          parent_ref.source = 'paused_snapshot'
+                          AND EXISTS (
+                            SELECT 1
+                            FROM ${pausedExecutions} AS parent_ref_pe
+                            WHERE parent_ref_pe.execution_id = parent_ref.execution_id
+                              AND parent_ref_pe.status = ANY(${LIVE_PAUSED_REFERENCE_STATUSES}::text[])
+                          )
+                        )
+                      )
+                  )
+                )
+            )`,
+            sql`NOT EXISTS (
+              SELECT 1
+              FROM ${workflowExecutionLogs} AS owner_wel
+              WHERE owner_wel.execution_id = split_part(${workspaceFiles.key}, '/', 4)
+            )`,
+            sql`NOT EXISTS (
+              SELECT 1
+              FROM ${pausedExecutions} AS pe
+              WHERE pe.execution_id = split_part(${workspaceFiles.key}, '/', 4)
+                AND pe.status = ANY(${LIVE_PAUSED_REFERENCE_STATUSES}::text[])
+            )`
+          )
+        )
+        .orderBy(
+          asc(workspaceFiles.workspaceId),
+          asc(workspaceFiles.uploadedAt),
+          asc(workspaceFiles.key)
+        )
+        .limit(limit)
+
+      if (rows.length === 0) break
+
+      const keys = rows.map((row) => row.key)
+      stats.largeValuesTotal += keys.length
+      attempted += keys.length
+      const result = await deleteLargeValueKeys(keys)
+      stats.largeValuesDeleted += result.deleted
+      stats.largeValuesDeleteFailed += result.failed
+
+      if (result.deleted === 0) {
+        break
+      }
+    }
+
+    if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) break
+  }
+
+  logger.info(
+    `[${label}/legacy_execution_large_values] Complete: ${stats.largeValuesDeleted}/${stats.largeValuesTotal} deleted, ${stats.largeValuesDeleteFailed} failed`
+  )
+
+  return stats
+}
+
+async function cleanupLargeValueMetadata(workspaceIds: string[], label: string): Promise<void> {
+  try {
+    const tombstonesDeletedBefore = new Date(
+      Date.now() - LARGE_VALUE_TOMBSTONE_RETENTION_HOURS * 60 * 60 * 1000
+    )
+    const result = await pruneLargeValueMetadata({ workspaceIds, tombstonesDeletedBefore })
+    logger.info(
+      `[${label}/execution_large_value_metadata] Pruned ${result.referencesDeleted} stale references, ${result.dependenciesDeleted} dependencies, ${result.tombstonesDeleted} tombstones`
+    )
+  } catch (error) {
+    logger.error(`[${label}/execution_large_value_metadata] Failed to prune metadata`, { error })
+  }
+}
+
+async function cleanupWorkflowExecutionLogs(
+  workspaceIds: string[],
+  retentionDate: Date,
+  label: string
+): Promise<TableCleanupResult & FileDeleteStats> {
+  const fileStats: FileDeleteStats = {
+    filesTotal: 0,
+    filesDeleted: 0,
+    filesDeleteFailed: 0,
+  }
+
+  const dbStats = await chunkedBatchDelete({
+    tableDef: workflowExecutionLogs,
+    workspaceIds,
+    tableName: `${label}/workflow_execution_logs`,
+    selectChunk: (chunkIds, limit) =>
+      db
+        .select({
+          id: workflowExecutionLogs.id,
+          files: workflowExecutionLogs.files,
+        })
+        .from(workflowExecutionLogs)
+        .leftJoin(
+          pausedExecutions,
+          eq(pausedExecutions.executionId, workflowExecutionLogs.executionId)
+        )
+        .where(
+          and(
+            inArray(workflowExecutionLogs.workspaceId, chunkIds),
+            lt(workflowExecutionLogs.startedAt, retentionDate),
+            or(
+              isNull(pausedExecutions.status),
+              notInArray(pausedExecutions.status, [...LIVE_PAUSED_REFERENCE_STATUSES])
+            )
+          )
+        )
+        .limit(limit),
+    onBatch: async (rows) => {
+      for (const row of rows) {
+        await deleteExecutionFiles(row.files, fileStats)
+      }
+    },
+    batchSize: WORKFLOW_LOG_CLEANUP_BATCH_SIZE,
+    maxBatches: WORKFLOW_LOG_CLEANUP_MAX_BATCHES,
+    totalRowLimit: WORKFLOW_LOG_CLEANUP_ROW_LIMIT,
+  })
+
+  return { ...dbStats, ...fileStats }
+}
+
+async function cleanupFreePlanOrphanedSnapshots(retentionHours: number): Promise<void> {
+  try {
+    const retentionDays = Math.floor(retentionHours / 24)
+    const snapshotsCleaned = await snapshotService.cleanupOrphanedSnapshots(retentionDays + 1)
+    logger.info(`Cleaned up ${snapshotsCleaned} orphaned snapshots`)
+  } catch (snapshotError) {
+    logger.error('Error cleaning up orphaned snapshots:', { snapshotError })
+  }
+}
+
+export async function runCleanupLogs(payload: CleanupJobPayload): Promise<void> {
+  const startTime = Date.now()
+  const { workspaceIds, retentionHours, label, plan, runGlobalHousekeeping } = payload
+
+  const retentionDate = new Date(Date.now() - retentionHours * 60 * 60 * 1000)
+
+  if (workspaceIds.length === 0) {
+    logger.info(`[${label}] No workspaces to process`)
+    if (runGlobalHousekeeping && plan === 'free') {
+      await cleanupFreePlanOrphanedSnapshots(retentionHours)
+    }
+    return
+  }
+
+  logger.info(
+    `[${label}] Cleaning ${workspaceIds.length} workspaces, cutoff: ${retentionDate.toISOString()}`
+  )
+
+  const workflowResults = await cleanupWorkflowExecutionLogs(workspaceIds, retentionDate, label)
+  logger.info(
+    `[${label}] workflow_execution_logs files: ${workflowResults.filesDeleted}/${workflowResults.filesTotal} deleted, ${workflowResults.filesDeleteFailed} failed`
+  )
+  const largeValueResults = await cleanupLargeExecutionValues(workspaceIds, retentionDate, label)
+  logger.info(
+    `[${label}] execution_large_values: ${largeValueResults.largeValuesDeleted}/${largeValueResults.largeValuesTotal} deleted, ${largeValueResults.largeValuesDeleteFailed} failed`
+  )
+  const legacyLargeValueResults = await cleanupLegacyLargeExecutionValues(
+    workspaceIds,
+    retentionDate,
+    label
+  )
+  logger.info(
+    `[${label}] legacy_execution_large_values: ${legacyLargeValueResults.largeValuesDeleted}/${legacyLargeValueResults.largeValuesTotal} deleted, ${legacyLargeValueResults.largeValuesDeleteFailed} failed`
+  )
+  await cleanupLargeValueMetadata(workspaceIds, label)
+
+  await batchDeleteByWorkspaceAndTimestamp({
+    tableDef: jobExecutionLogs,
+    workspaceIdCol: jobExecutionLogs.workspaceId,
+    timestampCol: jobExecutionLogs.startedAt,
+    workspaceIds,
+    retentionDate,
+    tableName: `${label}/job_execution_logs`,
+  })
+
+  if (runGlobalHousekeeping && plan === 'free') {
+    await cleanupFreePlanOrphanedSnapshots(retentionHours)
+  }
+
+  const timeElapsed = (Date.now() - startTime) / 1000
+  logger.info(`[${label}] Job completed in ${timeElapsed.toFixed(2)}s`)
+}
+
+export const cleanupLogsTask = task({
+  id: 'cleanup-logs',
+  machine: 'large-1x',
+  queue: { concurrencyLimit: LOG_CLEANUP_CONCURRENCY_LIMIT },
+  run: runCleanupLogs,
+})

apps/sim/background/cleanup-soft-deletes.test.ts

@@ -0,0 +1,163 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockBatchDeleteByWorkspaceAndTimestamp,
+  mockDeleteFileMetadata,
+  mockDeleteFiles,
+  mockDeleteRowsById,
+  mockIsUsingCloudStorage,
+  mockLimit,
+  mockOrderBy,
+  mockPrepareChatCleanup,
+  mockSelect,
+  mockSelectRowsByIdChunks,
+  mockTask,
+  mockWhere,
+} = vi.hoisted(() => {
+  const mockLimit = vi.fn(async () => [] as Array<{ key: string }>)
+  const mockOrderBy = vi.fn(() => ({ limit: mockLimit }))
+  const mockWhere = vi.fn(() => ({ orderBy: mockOrderBy, limit: mockLimit }))
+  const mockFrom = vi.fn(() => ({
+    where: mockWhere,
+    leftJoin: vi.fn(() => ({ where: mockWhere })),
+  }))
+  const mockSelect = vi.fn(() => ({ from: mockFrom }))
+
+  return {
+    mockBatchDeleteByWorkspaceAndTimestamp: vi.fn(async () => ({ deleted: 0, failed: 0 })),
+    mockDeleteFileMetadata: vi.fn(async () => true),
+    mockDeleteFiles: vi.fn(async () => ({ deleted: 0, failed: [] as Array<{ key: string }> })),
+    mockDeleteRowsById: vi.fn(async () => ({ deleted: 0, failed: 0 })),
+    mockIsUsingCloudStorage: vi.fn(() => true),
+    mockLimit,
+    mockOrderBy,
+    mockPrepareChatCleanup: vi.fn(async () => ({ execute: vi.fn(async () => undefined) })),
+    mockSelect,
+    mockSelectRowsByIdChunks: vi.fn(async () => [] as unknown[]),
+    mockTask: vi.fn((config: unknown) => config),
+    mockWhere,
+  }
+})
+
+vi.mock('@sim/db', () => ({ db: { select: mockSelect } }))
+
+vi.mock('@sim/db/schema', () => {
+  const table = (cols: string[]) =>
+    Object.fromEntries(cols.map((c) => [c, `col.${c}`])) as Record<string, string>
+  const wsFileCols = ['id', 'key', 'context', 'workspaceId', 'deletedAt', 'uploadedAt']
+  const softCols = ['id', 'archivedAt', 'deletedAt', 'workspaceId']
+  return {
+    copilotChats: table(['id', 'workflowId']),
+    document: table(['id', 'storageKey', 'knowledgeBaseId']),
+    knowledgeBase: table(softCols),
+    mcpServers: table(softCols),
+    memory: table(softCols),
+    userTableDefinitions: table(softCols),
+    workflow: table(softCols),
+    workflowFolder: table(softCols),
+    workflowMcpServer: table(softCols),
+    workspaceFile: table(wsFileCols),
+    workspaceFiles: table(wsFileCols),
+  }
+})
+
+vi.mock('@sim/logger', () => ({
+  createLogger: vi.fn(() => ({ error: vi.fn(), info: vi.fn(), warn: vi.fn() })),
+}))
+
+vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
+
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn((...args: unknown[]) => ({ op: 'and', args })),
+  asc: vi.fn((column: unknown) => ({ op: 'asc', column })),
+  eq: vi.fn((...args: unknown[]) => ({ op: 'eq', args })),
+  inArray: vi.fn((...args: unknown[]) => ({ op: 'inArray', args })),
+  isNotNull: vi.fn((...args: unknown[]) => ({ op: 'isNotNull', args })),
+  isNull: vi.fn((...args: unknown[]) => ({ op: 'isNull', args })),
+  lt: vi.fn((...args: unknown[]) => ({ op: 'lt', args })),
+  sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values })),
+}))
+
+vi.mock('@/lib/cleanup/batch-delete', () => ({
+  batchDeleteByWorkspaceAndTimestamp: mockBatchDeleteByWorkspaceAndTimestamp,
+  chunkArray: (items: string[], size: number) => {
+    const chunks: string[][] = []
+    for (let index = 0; index < items.length; index += size) {
+      chunks.push(items.slice(index, index + size))
+    }
+    return chunks
+  },
+  deleteRowsById: mockDeleteRowsById,
+  selectRowsByIdChunks: mockSelectRowsByIdChunks,
+}))
+
+vi.mock('@/lib/cleanup/chat-cleanup', () => ({ prepareChatCleanup: mockPrepareChatCleanup }))
+
+vi.mock('@/lib/uploads', () => ({
+  isUsingCloudStorage: mockIsUsingCloudStorage,
+  StorageService: { deleteFiles: mockDeleteFiles },
+}))
+
+vi.mock('@/lib/uploads/server/metadata', () => ({ deleteFileMetadata: mockDeleteFileMetadata }))
+
+import { runCleanupSoftDeletes } from '@/background/cleanup-soft-deletes'
+
+const basePayload = {
+  label: 'free/1',
+  plan: 'free' as const,
+  retentionHours: 720,
+  workspaceIds: ['ws-1'],
+}
+
+describe('cleanup soft deletes — orphan KB binding sweep', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockIsUsingCloudStorage.mockReturnValue(true)
+    mockLimit.mockResolvedValue([])
+  })
+
+  it('soft-deletes abandoned KB bindings and removes their storage objects', async () => {
+    mockLimit
+      .mockResolvedValueOnce([{ key: 'kb/orphan-1' }, { key: 'kb/orphan-2' }])
+      .mockResolvedValueOnce([])
+
+    await runCleanupSoftDeletes(basePayload)
+
+    expect(mockDeleteFiles).toHaveBeenCalledWith(['kb/orphan-1', 'kb/orphan-2'], 'knowledge-base')
+    expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-1')
+    expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-2')
+    expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(2)
+  })
+
+  it('still removes bindings but skips object deletion without cloud storage', async () => {
+    mockIsUsingCloudStorage.mockReturnValue(false)
+    mockLimit.mockResolvedValueOnce([{ key: 'kb/orphan-1' }]).mockResolvedValueOnce([])
+
+    await runCleanupSoftDeletes(basePayload)
+
+    expect(mockDeleteFiles).not.toHaveBeenCalled()
+    expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-1')
+  })
+
+  it('stops the batch loop when binding deletion makes no progress', async () => {
+    mockLimit.mockResolvedValue([{ key: 'kb/stuck' }])
+    mockDeleteFileMetadata.mockRejectedValue(new Error('db down'))
+
+    await runCleanupSoftDeletes(basePayload)
+
+    // One batch attempted, then the no-progress guard breaks the loop.
+    expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not run the sweep when there are no workspaces', async () => {
+    await runCleanupSoftDeletes({ ...basePayload, workspaceIds: [] })
+
+    expect(mockSelect).not.toHaveBeenCalled()
+    expect(mockDeleteFiles).not.toHaveBeenCalled()
+    expect(mockDeleteFileMetadata).not.toHaveBeenCalled()
+  })
+})

apps/sim/background/cleanup-soft-deletes.ts

@@ -0,0 +1,366 @@
+import { db } from '@sim/db'
+import {
+  copilotChats,
+  document,
+  knowledgeBase,
+  mcpServers,
+  memory,
+  userTableDefinitions,
+  workflow,
+  workflowFolder,
+  workflowMcpServer,
+  workspaceFile,
+  workspaceFiles,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { and, asc, eq, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm'
+import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher'
+import {
+  batchDeleteByWorkspaceAndTimestamp,
+  chunkArray,
+  deleteRowsById,
+  selectRowsByIdChunks,
+} from '@/lib/cleanup/batch-delete'
+import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
+import type { StorageContext } from '@/lib/uploads'
+import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
+import { deleteFileMetadata } from '@/lib/uploads/server/metadata'
+
+const logger = createLogger('CleanupSoftDeletes')
+
+const KB_ORPHAN_BINDING_BATCH_SIZE = 500
+const KB_ORPHAN_BINDING_TOTAL_LIMIT = 5_000
+/**
+ * Grace window before an unreferenced KB binding is swept. Comfortably longer
+ * than any presign → upload → document-insert flow, so an in-flight upload is
+ * never mistaken for an abandoned one.
+ */
+const KB_ORPHAN_BINDING_GRACE_HOURS = 7 * 24
+const KB_ORPHAN_BINDING_WORKSPACE_CHUNK = 50
+
+interface WorkspaceFileScope {
+  /** Rows from `workspace_file` (singular, legacy workspace-context only). */
+  legacyRows: Array<{ id: string; key: string }>
+  /** Rows from `workspace_files` (plural, multi-context). */
+  multiContextRows: Array<{ id: string; key: string; context: StorageContext }>
+}
+
+/**
+ * Select every soft-deleted file row that's eligible for permanent removal.
+ * Returned once and reused for both S3 deletion and DB deletion so the external
+ * cleanup cannot drift from the row-level cleanup.
+ */
+async function selectExpiredWorkspaceFiles(
+  workspaceIds: string[],
+  retentionDate: Date
+): Promise<WorkspaceFileScope> {
+  const [legacyRows, multiContextRows] = await Promise.all([
+    selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) =>
+      db
+        .select({ id: workspaceFile.id, key: workspaceFile.key })
+        .from(workspaceFile)
+        .where(
+          and(
+            inArray(workspaceFile.workspaceId, chunkIds),
+            isNotNull(workspaceFile.deletedAt),
+            lt(workspaceFile.deletedAt, retentionDate)
+          )
+        )
+        .limit(chunkLimit)
+    ),
+    selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) =>
+      db
+        .select({
+          id: workspaceFiles.id,
+          key: workspaceFiles.key,
+          context: workspaceFiles.context,
+        })
+        .from(workspaceFiles)
+        .where(
+          and(
+            inArray(workspaceFiles.workspaceId, chunkIds),
+            isNotNull(workspaceFiles.deletedAt),
+            lt(workspaceFiles.deletedAt, retentionDate)
+          )
+        )
+        .limit(chunkLimit)
+    ),
+  ])
+
+  return {
+    legacyRows,
+    multiContextRows: multiContextRows.map((r) => ({
+      id: r.id,
+      key: r.key,
+      context: r.context as StorageContext,
+    })),
+  }
+}
+
+async function cleanupWorkspaceFileStorage(
+  scope: WorkspaceFileScope
+): Promise<{ filesDeleted: number; filesFailed: number }> {
+  const stats = { filesDeleted: 0, filesFailed: 0 }
+  if (!isUsingCloudStorage()) return stats
+
+  const keysByContext = new Map<StorageContext, string[]>()
+  for (const r of scope.legacyRows) {
+    const bucket = keysByContext.get('workspace')
+    if (bucket) bucket.push(r.key)
+    else keysByContext.set('workspace', [r.key])
+  }
+  for (const r of scope.multiContextRows) {
+    const bucket = keysByContext.get(r.context)
+    if (bucket) bucket.push(r.key)
+    else keysByContext.set(r.context, [r.key])
+  }
+
+  for (const [context, keys] of keysByContext) {
+    const result = await StorageService.deleteFiles(keys, context)
+    stats.filesDeleted += result.deleted
+    stats.filesFailed += result.failed.length
+    for (const { key, error } of result.failed) {
+      logger.error(`Failed to delete storage file ${key} (context: ${context}):`, { error })
+    }
+  }
+
+  return stats
+}
+
+/**
+ * Tables cleaned by the generic workspace-scoped batched DELETE. Tables whose
+ * hard-delete triggers external side effects (workflow → copilot chats cascade,
+ * workspace files → S3 storage) are handled explicitly so the SELECT that drives
+ * the external cleanup and the SELECT that drives the DB delete see the same rows.
+ */
+const CLEANUP_TARGETS = [
+  {
+    table: workflowFolder,
+    softDeleteCol: workflowFolder.archivedAt,
+    wsCol: workflowFolder.workspaceId,
+    name: 'workflowFolder',
+  },
+  {
+    table: knowledgeBase,
+    softDeleteCol: knowledgeBase.deletedAt,
+    wsCol: knowledgeBase.workspaceId,
+    name: 'knowledgeBase',
+  },
+  {
+    table: userTableDefinitions,
+    softDeleteCol: userTableDefinitions.archivedAt,
+    wsCol: userTableDefinitions.workspaceId,
+    name: 'userTableDefinitions',
+  },
+  { table: memory, softDeleteCol: memory.deletedAt, wsCol: memory.workspaceId, name: 'memory' },
+  {
+    table: mcpServers,
+    softDeleteCol: mcpServers.deletedAt,
+    wsCol: mcpServers.workspaceId,
+    name: 'mcpServers',
+  },
+  {
+    table: workflowMcpServer,
+    softDeleteCol: workflowMcpServer.deletedAt,
+    wsCol: workflowMcpServer.workspaceId,
+    name: 'workflowMcpServer',
+  },
+] as const
+
+/**
+ * Sweep abandoned knowledge-base ownership bindings. The presigned upload flow
+ * writes a `workspace_files` binding when it hands out an upload URL, before the
+ * object is stored and before any document is created. If the upload is never
+ * completed, that binding is orphaned — no `document.storageKey` ever references
+ * its key. Such bindings are inert (read access requires a live document, and
+ * the move re-point only follows referenced keys), but they accumulate, so we
+ * drop the best-effort object and soft-delete the binding once they are older
+ * than the grace window.
+ */
+async function cleanupOrphanedKnowledgeBaseBindings(
+  workspaceIds: string[],
+  label: string
+): Promise<{ total: number; deleted: number; failed: number }> {
+  const stats = { total: 0, deleted: 0, failed: 0 }
+  if (workspaceIds.length === 0) return stats
+
+  const orphanCutoff = new Date(Date.now() - KB_ORPHAN_BINDING_GRACE_HOURS * 60 * 60 * 1000)
+
+  for (const chunkIds of chunkArray(workspaceIds, KB_ORPHAN_BINDING_WORKSPACE_CHUNK)) {
+    let attempted = 0
+    while (attempted < KB_ORPHAN_BINDING_TOTAL_LIMIT) {
+      const limit = Math.min(
+        KB_ORPHAN_BINDING_BATCH_SIZE,
+        KB_ORPHAN_BINDING_TOTAL_LIMIT - attempted
+      )
+      const rows = await db
+        .select({ key: workspaceFiles.key })
+        .from(workspaceFiles)
+        .where(
+          and(
+            inArray(workspaceFiles.workspaceId, chunkIds),
+            eq(workspaceFiles.context, 'knowledge-base'),
+            isNull(workspaceFiles.deletedAt),
+            lt(workspaceFiles.uploadedAt, orphanCutoff),
+            sql`NOT EXISTS (
+              SELECT 1 FROM ${document} AS doc
+              WHERE doc.storage_key = ${workspaceFiles.key}
+            )`
+          )
+        )
+        .orderBy(asc(workspaceFiles.uploadedAt), asc(workspaceFiles.key))
+        .limit(limit)
+
+      if (rows.length === 0) break
+
+      const keys = rows.map((row) => row.key)
+      stats.total += keys.length
+      attempted += keys.length
+
+      if (isUsingCloudStorage()) {
+        const result = await StorageService.deleteFiles(keys, 'knowledge-base')
+        stats.failed += result.failed.length
+        for (const { key, error } of result.failed) {
+          logger.error(`[${label}] Failed to delete orphan KB object ${key}:`, { error })
+        }
+      }
+
+      let deletedThisBatch = 0
+      for (const key of keys) {
+        try {
+          await deleteFileMetadata(key)
+          deletedThisBatch++
+        } catch (error) {
+          stats.failed++
+          logger.error(`[${label}] Failed to delete orphan KB binding ${key}:`, { error })
+        }
+      }
+      stats.deleted += deletedThisBatch
+
+      // No progress (every delete failed) — stop rather than reselect the same rows.
+      if (deletedThisBatch === 0) break
+    }
+  }
+
+  logger.info(
+    `[${label}/kb_orphan_bindings] Complete: ${stats.deleted}/${stats.total} bindings cleaned, ${stats.failed} failed`
+  )
+  return stats
+}
+
+export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise<void> {
+  const startTime = Date.now()
+  const { workspaceIds, retentionHours, label } = payload
+
+  if (workspaceIds.length === 0) {
+    logger.info(`[${label}] No workspaces to process`)
+    return
+  }
+
+  const retentionDate = new Date(Date.now() - retentionHours * 60 * 60 * 1000)
+  logger.info(
+    `[${label}] Processing ${workspaceIds.length} workspaces, cutoff: ${retentionDate.toISOString()}`
+  )
+
+  // Select workflows + files once. These sets drive BOTH external cleanup
+  // (chats + S3) AND the DB deletes below — selecting twice could return
+  // different subsets above the LIMIT cap and orphan or prematurely purge data.
+  const [doomedWorkflows, fileScope] = await Promise.all([
+    selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) =>
+      db
+        .select({ id: workflow.id })
+        .from(workflow)
+        .where(
+          and(
+            inArray(workflow.workspaceId, chunkIds),
+            isNotNull(workflow.archivedAt),
+            lt(workflow.archivedAt, retentionDate)
+          )
+        )
+        .limit(chunkLimit)
+    ),
+    selectExpiredWorkspaceFiles(workspaceIds, retentionDate),
+  ])
+
+  const doomedWorkflowIds = doomedWorkflows.map((w) => w.id)
+  let chatCleanup: { execute: () => Promise<void> } | null = null
+
+  if (doomedWorkflowIds.length > 0) {
+    const doomedChats = await selectRowsByIdChunks(doomedWorkflowIds, (chunkIds, chunkLimit) =>
+      db
+        .select({ id: copilotChats.id })
+        .from(copilotChats)
+        .where(inArray(copilotChats.workflowId, chunkIds))
+        .limit(chunkLimit)
+    )
+
+    const doomedChatIds = doomedChats.map((c) => c.id)
+    if (doomedChatIds.length > 0) {
+      chatCleanup = await prepareChatCleanup(doomedChatIds, label)
+    }
+  }
+
+  const fileStats = await cleanupWorkspaceFileStorage(fileScope)
+
+  let totalDeleted = 0
+
+  // Delete the workflow + file rows using the exact IDs we already selected.
+  const workflowResult = await deleteRowsById(
+    workflow,
+    workflow.id,
+    doomedWorkflowIds,
+    `${label}/workflow`
+  )
+  totalDeleted += workflowResult.deleted
+
+  const legacyFileResult = await deleteRowsById(
+    workspaceFile,
+    workspaceFile.id,
+    fileScope.legacyRows.map((r) => r.id),
+    `${label}/workspaceFile`
+  )
+  totalDeleted += legacyFileResult.deleted
+
+  const multiContextFileResult = await deleteRowsById(
+    workspaceFiles,
+    workspaceFiles.id,
+    fileScope.multiContextRows.map((r) => r.id),
+    `${label}/workspaceFiles`
+  )
+  totalDeleted += multiContextFileResult.deleted
+
+  for (const target of CLEANUP_TARGETS) {
+    const result = await batchDeleteByWorkspaceAndTimestamp({
+      tableDef: target.table,
+      workspaceIdCol: target.wsCol,
+      timestampCol: target.softDeleteCol,
+      workspaceIds,
+      retentionDate,
+      tableName: `${label}/${target.name}`,
+      requireTimestampNotNull: true,
+    })
+    totalDeleted += result.deleted
+  }
+
+  const orphanBindingStats = await cleanupOrphanedKnowledgeBaseBindings(workspaceIds, label)
+
+  logger.info(
+    `[${label}] Complete: ${totalDeleted} rows deleted, ${fileStats.filesDeleted} files cleaned, ${orphanBindingStats.deleted} orphan KB bindings cleaned`
+  )
+
+  // Clean up copilot backend + chat storage files after DB rows are gone
+  if (chatCleanup) {
+    await chatCleanup.execute()
+  }
+
+  const timeElapsed = (Date.now() - startTime) / 1000
+  logger.info(`[${label}] Job completed in ${timeElapsed.toFixed(2)}s`)
+}
+
+export const cleanupSoftDeletesTask = task({
+  id: 'cleanup-soft-deletes',
+  machine: 'large-1x',
+  queue: { concurrencyLimit: 5 },
+  run: runCleanupSoftDeletes,
+})

apps/sim/background/cleanup-tasks.ts

@@ -0,0 +1,162 @@
+import { db } from '@sim/db'
+import {
+  copilotAsyncToolCalls,
+  copilotChats,
+  copilotFeedback,
+  copilotRunCheckpoints,
+  copilotRuns,
+  mothershipInboxTask,
+} from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { and, inArray, lt } from 'drizzle-orm'
+import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher'
+import {
+  batchDeleteByWorkspaceAndTimestamp,
+  deleteRowsById,
+  selectRowsByIdChunks,
+  type TableCleanupResult,
+} from '@/lib/cleanup/batch-delete'
+import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
+
+const logger = createLogger('CleanupTasks')
+
+/**
+ * Delete copilot run checkpoints and async tool calls via join through copilotRuns.
+ * These tables don't have a direct workspaceId — we find qualifying run IDs first.
+ */
+const RUN_CHILD_TABLES = [
+  {
+    table: copilotRunCheckpoints,
+    runIdCol: copilotRunCheckpoints.runId,
+    name: 'copilotRunCheckpoints',
+  },
+  {
+    table: copilotAsyncToolCalls,
+    runIdCol: copilotAsyncToolCalls.runId,
+    name: 'copilotAsyncToolCalls',
+  },
+] as const
+
+async function cleanupRunChildren(
+  workspaceIds: string[],
+  retentionDate: Date,
+  label: string
+): Promise<TableCleanupResult[]> {
+  if (workspaceIds.length === 0) return []
+
+  const runIds = await selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) =>
+    db
+      .select({ id: copilotRuns.id })
+      .from(copilotRuns)
+      .where(
+        and(inArray(copilotRuns.workspaceId, chunkIds), lt(copilotRuns.updatedAt, retentionDate))
+      )
+      .limit(chunkLimit)
+  )
+
+  if (runIds.length === 0) {
+    return RUN_CHILD_TABLES.map((t) => ({ table: `${label}/${t.name}`, deleted: 0, failed: 0 }))
+  }
+
+  const ids = runIds.map((r) => r.id)
+
+  return Promise.all(
+    RUN_CHILD_TABLES.map((t) => deleteRowsById(t.table, t.runIdCol, ids, `${label}/${t.name}`))
+  )
+}
+
+export async function runCleanupTasks(payload: CleanupJobPayload): Promise<void> {
+  const startTime = Date.now()
+  const { workspaceIds, retentionHours, label } = payload
+
+  if (workspaceIds.length === 0) {
+    logger.info(`[${label}] No workspaces to process`)
+    return
+  }
+
+  const retentionDate = new Date(Date.now() - retentionHours * 60 * 60 * 1000)
+  logger.info(
+    `[${label}] Processing ${workspaceIds.length} workspaces, cutoff: ${retentionDate.toISOString()}`
+  )
+
+  const doomedChats = await selectRowsByIdChunks(workspaceIds, (chunkIds, chunkLimit) =>
+    db
+      .select({ id: copilotChats.id })
+      .from(copilotChats)
+      .where(
+        and(inArray(copilotChats.workspaceId, chunkIds), lt(copilotChats.updatedAt, retentionDate))
+      )
+      .limit(chunkLimit)
+  )
+
+  const doomedChatIds = doomedChats.map((c) => c.id)
+
+  // Prepare chat cleanup (collect file keys + copilot backend call) BEFORE DB deletion
+  const chatCleanup = await prepareChatCleanup(doomedChatIds, label)
+
+  // Delete run children first (checkpoints, tool calls) since they reference runs
+  const runChildResults = await cleanupRunChildren(workspaceIds, retentionDate, label)
+  for (const r of runChildResults) {
+    if (r.deleted > 0) logger.info(`[${r.table}] ${r.deleted} deleted`)
+  }
+
+  // Delete feedback — no direct workspaceId, reuse chat IDs collected above
+  const feedbackResult = await deleteRowsById(
+    copilotFeedback,
+    copilotFeedback.chatId,
+    doomedChatIds,
+    `${label}/copilotFeedback`
+  )
+
+  // Delete copilot runs (has workspaceId directly, cascades checkpoints)
+  const runsResult = await batchDeleteByWorkspaceAndTimestamp({
+    tableDef: copilotRuns,
+    workspaceIdCol: copilotRuns.workspaceId,
+    timestampCol: copilotRuns.updatedAt,
+    workspaceIds,
+    retentionDate,
+    tableName: `${label}/copilotRuns`,
+  })
+
+  // Delete copilot chats using the exact IDs collected above so the chat
+  // cleanup (S3 + copilot backend) and the DB delete can never disagree.
+  const chatsResult = await deleteRowsById(
+    copilotChats,
+    copilotChats.id,
+    doomedChatIds,
+    `${label}/copilotChats`
+  )
+
+  // Delete mothership inbox tasks (has workspaceId directly)
+  const inboxResult = await batchDeleteByWorkspaceAndTimestamp({
+    tableDef: mothershipInboxTask,
+    workspaceIdCol: mothershipInboxTask.workspaceId,
+    timestampCol: mothershipInboxTask.createdAt,
+    workspaceIds,
+    retentionDate,
+    tableName: `${label}/mothershipInboxTask`,
+  })
+
+  const totalDeleted =
+    runChildResults.reduce((s, r) => s + r.deleted, 0) +
+    feedbackResult.deleted +
+    runsResult.deleted +
+    chatsResult.deleted +
+    inboxResult.deleted
+
+  logger.info(`[${label}] Complete: ${totalDeleted} total rows deleted`)
+
+  // Clean up copilot backend + storage files after DB rows are gone
+  await chatCleanup.execute()
+
+  const timeElapsed = (Date.now() - startTime) / 1000
+  logger.info(`Task cleanup completed in ${timeElapsed.toFixed(2)}s`)
+}
+
+export const cleanupTasksTask = task({
+  id: 'cleanup-tasks',
+  machine: 'large-1x',
+  queue: { concurrencyLimit: 5 },
+  run: runCleanupTasks,
+})

apps/sim/background/concurrency-limits.ts

@@ -0,0 +1,21 @@
+import { env, envNumber } from '@/lib/core/config/env'
+
+/** Per-task Trigger.dev concurrency caps. Bound heavy DB tasks so unbounded fan-out can't saturate the pool. */
+
+export const WORKFLOW_EXECUTION_CONCURRENCY_LIMIT = envNumber(
+  env.WORKFLOW_EXECUTION_CONCURRENCY_LIMIT,
+  75,
+  { min: 1, integer: true }
+)
+
+export const WEBHOOK_EXECUTION_CONCURRENCY_LIMIT = envNumber(
+  env.WEBHOOK_EXECUTION_CONCURRENCY_LIMIT,
+  75,
+  { min: 1, integer: true }
+)
+
+export const RESUME_EXECUTION_CONCURRENCY_LIMIT = envNumber(
+  env.RESUME_EXECUTION_CONCURRENCY_LIMIT,
+  50,
+  { min: 1, integer: true }
+)

apps/sim/background/fork-content-copy.ts

@@ -0,0 +1,25 @@
+import { task } from '@trigger.dev/sdk'
+import {
+  type ForkContentCopyPayload,
+  runForkContentCopy,
+} from '@/ee/workspace-forking/lib/copy/content-copy-runner'
+
+/**
+ * Trigger.dev wrapper for the post-fork heavy-content copy (table rows, KB
+ * documents + embeddings, file blobs). Backgrounding keeps the fork request fast
+ * and lets the copy survive app deploys. `maxAttempts: 1` — the copy is
+ * non-transactional best-effort (per-row inserts with fresh ids), so a blind
+ * re-run would duplicate rows; a partial failure simply leaves the fork's content
+ * incomplete (the workflows themselves committed synchronously).
+ */
+export const forkContentCopyTask = task({
+  id: 'fork-content-copy',
+  retry: { maxAttempts: 1 },
+  queue: {
+    name: 'fork-content-copy',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: ForkContentCopyPayload) => {
+    await runForkContentCopy(payload)
+  },
+})

apps/sim/background/knowledge-connector-sync.ts

@@ -0,0 +1,54 @@
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { executeSync } from '@/lib/knowledge/connectors/sync-engine'
+
+const logger = createLogger('TriggerKnowledgeConnectorSync')
+
+export type ConnectorSyncPayload = {
+  connectorId: string
+  fullSync?: boolean
+  requestId: string
+}
+
+export const knowledgeConnectorSync = task({
+  id: 'knowledge-connector-sync',
+  maxDuration: 1800,
+  machine: 'large-1x',
+  retry: {
+    maxAttempts: 3,
+    factor: 2,
+    minTimeoutInMs: 5000,
+    maxTimeoutInMs: 30000,
+  },
+  queue: {
+    concurrencyLimit: 5,
+    name: 'connector-sync-queue',
+  },
+  run: async (payload: ConnectorSyncPayload) => {
+    const { connectorId, fullSync, requestId } = payload
+
+    logger.info(`[${requestId}] Starting connector sync: ${connectorId}`)
+
+    try {
+      const result = await executeSync(connectorId, { fullSync })
+
+      logger.info(`[${requestId}] Connector sync completed`, {
+        connectorId,
+        added: result.docsAdded,
+        updated: result.docsUpdated,
+        deleted: result.docsDeleted,
+        unchanged: result.docsUnchanged,
+        failed: result.docsFailed,
+      })
+
+      return {
+        success: !result.error,
+        connectorId,
+        ...result,
+      }
+    } catch (error) {
+      logger.error(`[${requestId}] Connector sync failed: ${connectorId}`, error)
+      throw error
+    }
+  },
+})

apps/sim/background/knowledge-processing.ts

@@ -0,0 +1,59 @@
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { env, envNumber } from '@/lib/core/config/env'
+import { processDocumentAsync } from '@/lib/knowledge/documents/service'
+
+const logger = createLogger('TriggerKnowledgeProcessing')
+
+export type DocumentProcessingPayload = {
+  knowledgeBaseId: string
+  documentId: string
+  docData: {
+    filename: string
+    fileUrl: string
+    fileSize: number
+    mimeType: string
+  }
+  processingOptions: {
+    recipe?: string
+    lang?: string
+  }
+  requestId: string
+}
+
+export const processDocument = task({
+  id: 'knowledge-process-document',
+  maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
+  machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
+  retry: {
+    maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
+    factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
+    minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
+    maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
+  },
+  queue: {
+    concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
+    name: 'document-processing-queue',
+  },
+  run: async (payload: DocumentProcessingPayload) => {
+    const { knowledgeBaseId, documentId, docData, processingOptions, requestId } = payload
+
+    logger.info(`[${requestId}] Starting Trigger.dev processing for document: ${docData.filename}`)
+
+    try {
+      await processDocumentAsync(knowledgeBaseId, documentId, docData, processingOptions)
+
+      logger.info(`[${requestId}] Successfully processed document: ${docData.filename}`)
+
+      return {
+        success: true,
+        documentId,
+        filename: docData.filename,
+        processingTime: Date.now(),
+      }
+    } catch (error) {
+      logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
+      throw error
+    }
+  },
+})

apps/sim/background/lifecycle-email.ts

@@ -0,0 +1,66 @@
+import { db } from '@sim/db'
+import { user } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { eq } from 'drizzle-orm'
+import { getEmailSubject, renderOnboardingFollowupEmail } from '@/components/emails'
+import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
+import { checkEnterprisePlan } from '@/lib/billing/subscriptions/utils'
+import { sendEmail } from '@/lib/messaging/email/mailer'
+import { getHelpEmailAddress, getPersonalEmailFrom } from '@/lib/messaging/email/utils'
+import { LIFECYCLE_EMAIL_TASK_ID, type LifecycleEmailType } from '@/lib/messaging/lifecycle'
+
+const logger = createLogger('LifecycleEmail')
+
+interface LifecycleEmailParams {
+  userId: string
+  type: LifecycleEmailType
+}
+
+async function sendLifecycleEmail({ userId, type }: LifecycleEmailParams): Promise<void> {
+  const [userData] = await db.select().from(user).where(eq(user.id, userId)).limit(1)
+
+  if (!userData?.email) {
+    logger.warn('[lifecycle-email] User not found or has no email', { userId, type })
+    return
+  }
+
+  const subscription = await getHighestPrioritySubscription(userId)
+  if (checkEnterprisePlan(subscription)) {
+    logger.info('[lifecycle-email] Skipping lifecycle email for enterprise user', { userId, type })
+    return
+  }
+
+  const { from } = getPersonalEmailFrom()
+  const replyTo = getHelpEmailAddress()
+
+  let html: string
+
+  switch (type) {
+    case 'onboarding-followup':
+      html = await renderOnboardingFollowupEmail(userData.name || undefined)
+      break
+    default:
+      logger.warn('[lifecycle-email] Unknown lifecycle email type', { type })
+      return
+  }
+
+  await sendEmail({
+    to: userData.email,
+    subject: getEmailSubject(type),
+    html,
+    from,
+    replyTo,
+    emailType: 'notifications',
+  })
+
+  logger.info('[lifecycle-email] Sent lifecycle email', { userId, type })
+}
+
+export const lifecycleEmailTask = task({
+  id: LIFECYCLE_EMAIL_TASK_ID,
+  retry: { maxAttempts: 2 },
+  run: async (params: LifecycleEmailParams) => {
+    await sendLifecycleEmail(params)
+  },
+})

apps/sim/background/mothership-inbox-execution.ts

@@ -0,0 +1,28 @@
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { executeInboxTask } from '@/lib/mothership/inbox/executor'
+
+const logger = createLogger('MothershipInboxExecution')
+
+export interface MothershipInboxExecutionParams {
+  taskId: string
+}
+
+export const mothershipInboxExecution = task({
+  id: 'mothership-inbox-execution',
+  machine: { preset: 'medium-1x' },
+  retry: {
+    maxAttempts: 2,
+    minTimeoutInMs: 5000,
+    maxTimeoutInMs: 30000,
+    factor: 2,
+  },
+  run: async (params: MothershipInboxExecutionParams) => {
+    logger.info('Starting inbox task execution', { taskId: params.taskId })
+
+    await executeInboxTask(params.taskId)
+
+    logger.info('Inbox task execution completed', { taskId: params.taskId })
+    return { success: true, taskId: params.taskId }
+  },
+})

apps/sim/background/resume-execution.ts

@@ -0,0 +1,359 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { task } from '@trigger.dev/sdk'
+import { withCascadeLock } from '@/lib/table/cascade-lock'
+import { isExecCancelled } from '@/lib/table/deps'
+import type { RowData, RowExecutionMetadata } from '@/lib/table/types'
+import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
+import { RESUME_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits'
+
+const logger = createLogger('TriggerResumeExecution')
+
+export type ResumeExecutionPayload = {
+  resumeEntryId: string
+  resumeExecutionId: string
+  pausedExecutionId: string
+  contextId: string
+  resumeInput: unknown
+  userId: string
+  workflowId: string
+  parentExecutionId: string
+}
+
+export async function executeResumeJob(payload: ResumeExecutionPayload) {
+  const { resumeExecutionId, pausedExecutionId, contextId, workflowId, parentExecutionId } = payload
+
+  logger.info('Starting background resume execution', {
+    resumeExecutionId,
+    pausedExecutionId,
+    contextId,
+    workflowId,
+    parentExecutionId,
+  })
+
+  try {
+    const pausedExecution = await PauseResumeManager.getPausedExecutionById(pausedExecutionId)
+    if (!pausedExecution) {
+      throw new Error(`Paused execution not found: ${pausedExecutionId}`)
+    }
+
+    // If this paused execution belongs to a table cell, rehydrate the cell
+    // context so post-resume block outputs land on the same row + group as
+    // the original cell task. Without this, blocks that run after the human
+    // approves write nothing back to the table — the row silently truncates
+    // at the pause boundary.
+    const { findCellContextByExecutionId } = await import('@/lib/table/workflow-columns')
+    const cellContext = await findCellContextByExecutionId(parentExecutionId)
+
+    // A paused/awaiting table cell that was cancelled by "Stop all" must not
+    // resume — the cancel write is authoritative (matches the cell-write guard
+    // philosophy). Aborting here also stops the wasted compute the guard alone
+    // can't prevent. Read the cell's current exec and bail if cancelled.
+    if (cellContext) {
+      const { getRowById } = await import('@/lib/table/rows/service')
+      const cellRow = await getRowById(
+        cellContext.tableId,
+        cellContext.rowId,
+        cellContext.workspaceId
+      )
+      if (isExecCancelled(cellRow?.executions?.[cellContext.groupId])) {
+        logger.info('Skipping resume — table cell cancelled', {
+          tableId: cellContext.tableId,
+          rowId: cellContext.rowId,
+          groupId: cellContext.groupId,
+          parentExecutionId,
+        })
+        return {
+          success: false,
+          workflowId,
+          executionId: resumeExecutionId,
+          parentExecutionId,
+          status: 'cancelled' as const,
+          output: undefined,
+          executedAt: new Date().toISOString(),
+        }
+      }
+    }
+
+    const writers = cellContext
+      ? await buildResumeCellWriters(cellContext, parentExecutionId)
+      : null
+
+    // No cell context → plain resume, no lock, no cascade continuation.
+    if (!cellContext || !writers) {
+      const result = await PauseResumeManager.startResumeExecution({
+        resumeEntryId: payload.resumeEntryId,
+        resumeExecutionId: payload.resumeExecutionId,
+        pausedExecution,
+        contextId: payload.contextId,
+        resumeInput: payload.resumeInput,
+        userId: payload.userId,
+      })
+      logger.info('Background resume execution completed', {
+        resumeExecutionId,
+        workflowId,
+        success: result.success,
+        status: result.status,
+      })
+      return {
+        success: result.success,
+        workflowId,
+        executionId: resumeExecutionId,
+        parentExecutionId,
+        status: result.status,
+        output: result.output,
+        executedAt: new Date().toISOString(),
+      }
+    }
+
+    // Cell-context path: hold the row's cascade lock for the resume + any
+    // downstream cascade continuation. On lock contention, fall through to
+    // resume-only (the lock holder will pick up the resumed group's
+    // completion on its next eligibility scan).
+    const outcome = await withCascadeLock(
+      cellContext.tableId,
+      cellContext.rowId,
+      parentExecutionId,
+      async () => {
+        const result = await runResumeAndCellTerminal(payload, pausedExecution, writers)
+        if (result.status === 'paused') return result
+        await continueCascadeAfterResume(cellContext)
+        return result
+      }
+    )
+
+    let result
+    if (outcome.status === 'contended') {
+      logger.info(
+        `Resume cascade lock held — writing resumed group only (table=${cellContext.tableId} row=${cellContext.rowId} executionId=${parentExecutionId})`
+      )
+      result = await runResumeAndCellTerminal(payload, pausedExecution, writers)
+    } else {
+      result = outcome.result
+    }
+
+    logger.info('Background resume execution completed', {
+      resumeExecutionId,
+      workflowId,
+      success: result.success,
+      status: result.status,
+    })
+
+    return {
+      success: result.success,
+      workflowId,
+      executionId: resumeExecutionId,
+      parentExecutionId,
+      status: result.status,
+      output: result.output,
+      executedAt: new Date().toISOString(),
+    }
+  } catch (error) {
+    logger.error('Background resume execution failed', {
+      resumeExecutionId,
+      workflowId,
+      error: toError(error).message,
+    })
+    throw error
+  }
+}
+
+type CellWriters = {
+  cellOnBlockComplete: (blockId: string, output: unknown) => Promise<void>
+  writeCellTerminal: (
+    status: 'completed' | 'error' | 'paused',
+    error: string | null
+  ) => Promise<void>
+}
+
+async function buildResumeCellWriters(
+  cellContext: {
+    tableId: string
+    rowId: string
+    workspaceId: string
+    groupId: string
+    workflowId: string
+  },
+  parentExecutionId: string
+): Promise<CellWriters | null> {
+  const { getTableById } = await import('@/lib/table/service')
+  const { writeWorkflowGroupState, buildOutputsByBlockId } = await import('@/lib/table/cell-write')
+  const { pluckByPath } = await import('@/lib/table/pluck')
+
+  const table = await getTableById(cellContext.tableId)
+  const group = table?.schema.workflowGroups?.find((g) => g.id === cellContext.groupId)
+  if (!group) {
+    logger.warn('Cell context found but table or group missing — falling back to plain resume', {
+      parentExecutionId,
+      tableId: cellContext.tableId,
+      groupId: cellContext.groupId,
+    })
+    return null
+  }
+
+  const outputsByBlockId = buildOutputsByBlockId(group)
+  const accumulatedData: RowData = {}
+  const blockErrors: Record<string, string> = {}
+  const writeCtx = {
+    tableId: cellContext.tableId,
+    rowId: cellContext.rowId,
+    workspaceId: cellContext.workspaceId,
+    groupId: cellContext.groupId,
+    executionId: parentExecutionId,
+    requestId: `wfgrp-resume-${parentExecutionId}`,
+  }
+  let writeChain: Promise<void> = Promise.resolve()
+  let terminalWritten = false
+
+  const cellOnBlockComplete = async (blockId: string, output: unknown) => {
+    const outputs = outputsByBlockId.get(blockId)
+    if (!outputs) return
+    const blockResult =
+      output && typeof output === 'object' && 'output' in (output as object)
+        ? (output as { output: unknown }).output
+        : output
+    const errorMessage =
+      blockResult &&
+      typeof blockResult === 'object' &&
+      typeof (blockResult as { error?: unknown }).error === 'string'
+        ? (blockResult as { error: string }).error
+        : null
+    if (errorMessage) {
+      blockErrors[blockId] = errorMessage
+    } else {
+      for (const out of outputs) {
+        const plucked = pluckByPath(blockResult, out.path)
+        if (plucked === undefined) continue
+        accumulatedData[out.columnName] = plucked as RowData[string]
+      }
+    }
+    const dataSnapshot: RowData = { ...accumulatedData }
+    const blockErrorsSnapshot = { ...blockErrors }
+    writeChain = writeChain
+      .then(async () => {
+        if (terminalWritten) return
+        const partial: RowExecutionMetadata = {
+          status: 'running',
+          executionId: parentExecutionId,
+          jobId: null,
+          workflowId: cellContext.workflowId,
+          error: null,
+          blockErrors: blockErrorsSnapshot,
+        }
+        await writeWorkflowGroupState(writeCtx, {
+          executionState: partial,
+          dataPatch: dataSnapshot,
+        })
+      })
+      .catch((err) => {
+        logger.warn(
+          `Resume per-block partial write failed (table=${cellContext.tableId} row=${cellContext.rowId} group=${cellContext.groupId}):`,
+          err
+        )
+      })
+  }
+
+  const writeCellTerminal = async (
+    status: 'completed' | 'error' | 'paused',
+    error: string | null
+  ) => {
+    terminalWritten = true
+    await writeChain.catch(() => {})
+    // Paused → keep `pending` + sentinel jobId so eligibility predicates
+    // continue treating the row as in-flight while we wait on another
+    // pause. Mirrors the initial cell-task pause branch.
+    const terminal: RowExecutionMetadata =
+      status === 'paused'
+        ? {
+            status: 'pending',
+            executionId: parentExecutionId,
+            jobId: `paused-${parentExecutionId}`,
+            workflowId: cellContext.workflowId,
+            error: null,
+            blockErrors,
+          }
+        : {
+            status,
+            executionId: parentExecutionId,
+            jobId: null,
+            workflowId: cellContext.workflowId,
+            error,
+            runningBlockIds: [],
+            blockErrors,
+          }
+    await writeWorkflowGroupState(writeCtx, {
+      executionState: terminal,
+      dataPatch: accumulatedData,
+    })
+  }
+
+  return { cellOnBlockComplete, writeCellTerminal }
+}
+
+async function runResumeAndCellTerminal(
+  payload: ResumeExecutionPayload,
+  pausedExecution: Awaited<ReturnType<typeof PauseResumeManager.getPausedExecutionById>>,
+  writers: CellWriters
+): Promise<Awaited<ReturnType<typeof PauseResumeManager.startResumeExecution>>> {
+  if (!pausedExecution) throw new Error('Paused execution missing — already nulled by caller')
+  const result = await PauseResumeManager.startResumeExecution({
+    resumeEntryId: payload.resumeEntryId,
+    resumeExecutionId: payload.resumeExecutionId,
+    pausedExecution,
+    contextId: payload.contextId,
+    resumeInput: payload.resumeInput,
+    userId: payload.userId,
+    onBlockComplete: writers.cellOnBlockComplete,
+  })
+
+  if (result.status === 'paused') {
+    await writers.writeCellTerminal('paused', null)
+  } else if (result.success) {
+    await writers.writeCellTerminal('completed', null)
+  } else {
+    await writers.writeCellTerminal('error', result.error ?? 'Workflow execution failed')
+  }
+
+  return result
+}
+
+async function continueCascadeAfterResume(cellContext: {
+  tableId: string
+  rowId: string
+  workspaceId: string
+  groupId: string
+}): Promise<void> {
+  const { getTableById } = await import('@/lib/table/service')
+  const { getRowById } = await import('@/lib/table/rows/service')
+  const { pickNextEligibleGroupForRow } = await import('@/lib/table/workflow-columns')
+  const { runRowCascadeLoop } = await import('@/background/workflow-column-execution')
+
+  const freshTable = await getTableById(cellContext.tableId)
+  if (!freshTable) return
+  const freshRow = await getRowById(cellContext.tableId, cellContext.rowId, cellContext.workspaceId)
+  if (!freshRow) return
+  const next = pickNextEligibleGroupForRow(freshTable, freshRow, cellContext.groupId)
+  if (!next) return
+  await runRowCascadeLoop({
+    tableId: cellContext.tableId,
+    tableName: freshTable.name,
+    rowId: cellContext.rowId,
+    workspaceId: cellContext.workspaceId,
+    groupId: next.id,
+    workflowId: next.workflowId,
+    executionId: generateId(),
+  })
+}
+
+export const resumeExecutionTask = task({
+  id: 'resume-execution',
+  machine: 'medium-1x',
+  retry: {
+    maxAttempts: 1,
+  },
+  queue: {
+    concurrencyLimit: RESUME_EXECUTION_CONCURRENCY_LIMIT,
+  },
+  run: executeResumeJob,
+})

apps/sim/background/run-data-drain.ts

@@ -0,0 +1,14 @@
+import { task } from '@trigger.dev/sdk'
+import { runDrain } from '@/lib/data-drains/service'
+import type { RunTrigger } from '@/lib/data-drains/types'
+
+interface RunDataDrainPayload {
+  drainId: string
+  trigger: RunTrigger
+}
+
+export const runDataDrainTask = task({
+  id: 'run-data-drain',
+  run: async ({ drainId, trigger }: RunDataDrainPayload, { signal }) =>
+    runDrain(drainId, trigger, { signal }),
+})

apps/sim/background/table-backfill.ts

@@ -0,0 +1,21 @@
+import { task } from '@trigger.dev/sdk'
+import { runTableBackfill, type TableBackfillPayload } from '@/lib/table/backfill-runner'
+
+/**
+ * Trigger.dev wrapper around `runTableBackfill` (output-column backfill from saved execution
+ * logs). Retry-safe: re-plucking the same trace spans writes the same values, and
+ * `overwrite: false` passes skip already-filled cells. The `table_jobs` ownership gate stops a
+ * run that lost the job within one page.
+ */
+export const tableBackfillTask = task({
+  id: 'table-backfill',
+  machine: 'small-1x',
+  retry: { maxAttempts: 3 },
+  queue: {
+    name: 'table-backfill',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: TableBackfillPayload) => {
+    await runTableBackfill(payload)
+  },
+})

apps/sim/background/table-delete.ts

@@ -0,0 +1,38 @@
+import { task } from '@trigger.dev/sdk'
+import {
+  markTableDeleteFailed,
+  runTableDelete,
+  type TableDeletePayload,
+} from '@/lib/table/delete-runner'
+
+/**
+ * `TableDeletePayload` with the cutoff as an ISO string — task payloads cross a JSON boundary, so
+ * the Date is rehydrated in `run` rather than trusting payload serialization.
+ */
+export interface TableDeleteTaskPayload extends Omit<TableDeletePayload, 'cutoff'> {
+  cutoff: string
+}
+
+/**
+ * Trigger.dev wrapper around `runTableDelete`. Errors propagate out of `run` so the retry policy
+ * actually fires; the job is marked failed only in `onFailure`, after the final attempt. Retry-
+ * safe: the worker keysets by id with a `created_at <= cutoff` floor and batches are committed
+ * independently, so a retried attempt simply re-walks and deletes whatever remains. The
+ * `table_jobs` ownership gate stops a retried run that lost the job (canceled / janitor-failed)
+ * within one page.
+ */
+export const tableDeleteTask = task({
+  id: 'table-delete',
+  machine: 'small-1x',
+  retry: { maxAttempts: 3 },
+  queue: {
+    name: 'table-delete',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: TableDeleteTaskPayload) => {
+    await runTableDelete({ ...payload, cutoff: new Date(payload.cutoff) })
+  },
+  onFailure: async ({ payload, error }) => {
+    await markTableDeleteFailed(payload.tableId, payload.jobId, error)
+  },
+})

apps/sim/background/table-export.ts

@@ -0,0 +1,21 @@
+import { task } from '@trigger.dev/sdk'
+import { runTableExport, type TableExportPayload } from '@/lib/table/export-runner'
+
+/**
+ * Trigger.dev wrapper around `runTableExport`. Retry-safe: a retried attempt regenerates the file
+ * from scratch (failures abort/clean up their partial upload), and the `table_jobs` ownership gate
+ * stops a run that lost the job. The file streams to storage in bounded multipart chunks (no longer
+ * buffered whole), so `medium-1x` is now headroom rather than a hard requirement.
+ */
+export const tableExportTask = task({
+  id: 'table-export',
+  machine: 'medium-1x',
+  retry: { maxAttempts: 3 },
+  queue: {
+    name: 'table-export',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: TableExportPayload) => {
+    await runTableExport(payload)
+  },
+})

apps/sim/background/table-import.ts

@@ -0,0 +1,24 @@
+import { task } from '@trigger.dev/sdk'
+import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner'
+
+/**
+ * Trigger.dev wrapper around `runTableImport`. The job's lifecycle (claim, progress heartbeat,
+ * cancel, terminal state) lives in the `table_jobs` state machine, so the task is a thin shell:
+ * the worker's per-batch ownership gate stops it on cancel/supersede regardless of where it runs.
+ *
+ * `maxAttempts: 1` — a blind re-run would re-insert batches the failed attempt already committed
+ * (imports commit per batch with no rollback). A crashed import marks failed via the worker's own
+ * catch, or the stale-job janitor if the process died; the user retries the upload.
+ */
+export const tableImportTask = task({
+  id: 'table-import',
+  machine: 'small-1x',
+  retry: { maxAttempts: 1 },
+  queue: {
+    name: 'table-import',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: TableImportPayload) => {
+    await runTableImport(payload)
+  },
+})

apps/sim/background/table-run-dispatcher.ts

@@ -0,0 +1,37 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { task } from '@trigger.dev/sdk'
+import { runDispatcherToCompletion } from '@/lib/table/dispatcher'
+
+const logger = createLogger('TableRunDispatcherTask')
+
+export interface TableRunDispatcherPayload {
+  dispatchId: string
+}
+
+/**
+ * Trigger.dev wrapper around `dispatcherStep`. One task run holds the
+ * dispatcher loop for the dispatch's entire lifetime — each iteration
+ * processes a window of cells via `batchTriggerAndWait`, which checkpoints
+ * the parent via CRIU during the wait so we don't pay compute while cells
+ * execute. The cursor is persisted in DB; if this run crashes, trigger.dev
+ * retries and the next attempt resumes from the persisted cursor.
+ */
+export const tableRunDispatcherTask = task({
+  id: 'table-run-dispatcher',
+  machine: 'small-1x',
+  retry: { maxAttempts: 3 },
+  queue: {
+    name: 'table-run-dispatcher',
+    concurrencyLimit: 8,
+  },
+  run: async (payload: TableRunDispatcherPayload) => {
+    const { dispatchId } = payload
+    try {
+      await runDispatcherToCompletion(dispatchId)
+    } catch (err) {
+      logger.error(`[${dispatchId}] dispatcher loop failed`, { error: toError(err).message })
+      throw err
+    }
+  },
+})

apps/sim/background/table-update.ts

@@ -0,0 +1,38 @@
+import { task } from '@trigger.dev/sdk'
+import {
+  markTableUpdateFailed,
+  runTableUpdate,
+  type TableUpdatePayload,
+} from '@/lib/table/update-runner'
+
+/**
+ * `TableUpdatePayload` with the cutoff as an ISO string — task payloads cross a JSON boundary, so
+ * the Date is rehydrated in `run` rather than trusting payload serialization.
+ */
+export interface TableUpdateTaskPayload extends Omit<TableUpdatePayload, 'cutoff'> {
+  cutoff: string
+}
+
+/**
+ * Trigger.dev wrapper around `runTableUpdate`. Errors propagate out of `run` so the retry policy
+ * fires; the job is marked failed only in `onFailure`, after the final attempt. Retry-safe: the
+ * worker keysets by id with a `created_at <= cutoff` floor and the JSONB-merge patch is idempotent
+ * (re-applying the same patch to an already-patched row is a no-op), so a retried attempt re-walks
+ * and re-applies whatever remains. The `table_jobs` ownership gate stops a retried run that lost
+ * the job within one page.
+ */
+export const tableUpdateTask = task({
+  id: 'table-update',
+  machine: 'small-1x',
+  retry: { maxAttempts: 3 },
+  queue: {
+    name: 'table-update',
+    concurrencyLimit: 10,
+  },
+  run: async (payload: TableUpdateTaskPayload) => {
+    await runTableUpdate({ ...payload, cutoff: new Date(payload.cutoff) })
+  },
+  onFailure: async ({ payload, error }) => {
+    await markTableUpdateFailed(payload.tableId, payload.jobId, error)
+  },
+})

apps/sim/background/tiktok-webhook-ingress.test.ts

@@ -0,0 +1,109 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockDispatchResolvedWebhookTarget, mockFindTikTokWebhookTargets } = vi.hoisted(() => ({
+  mockDispatchResolvedWebhookTarget: vi.fn(),
+  mockFindTikTokWebhookTargets: vi.fn(),
+}))
+
+vi.mock('@trigger.dev/sdk', () => ({
+  task: vi.fn((config: unknown) => config),
+}))
+
+vi.mock('@/lib/webhooks/processor', () => ({
+  dispatchResolvedWebhookTarget: mockDispatchResolvedWebhookTarget,
+}))
+
+vi.mock('@/lib/webhooks/providers/tiktok-targets', () => ({
+  findTikTokWebhookTargets: mockFindTikTokWebhookTargets,
+}))
+
+import {
+  executeTikTokWebhookIngress,
+  type TikTokWebhookIngressPayload,
+} from '@/background/tiktok-webhook-ingress'
+
+const payload: TikTokWebhookIngressPayload = {
+  envelope: {
+    client_key: 'client-key',
+    event: 'post.publish.complete',
+    create_time: 1_725_000_000,
+    user_openid: 'act.user',
+    content: '{"publish_id":"publish-1"}',
+  },
+  headers: { 'content-type': 'application/json' },
+  requestId: 'request-1',
+  receivedAt: 1_725_000_000_000,
+}
+
+const targets = [
+  {
+    webhook: { id: 'webhook-1', path: 'tiktok', provider: 'tiktok' },
+    workflow: { id: 'workflow-1' },
+  },
+  {
+    webhook: { id: 'webhook-2', path: 'tiktok', provider: 'tiktok' },
+    workflow: { id: 'workflow-2' },
+  },
+]
+
+describe('executeTikTokWebhookIngress', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('acknowledges deliveries without active targets', async () => {
+    mockFindTikTokWebhookTargets.mockResolvedValue([])
+
+    await expect(executeTikTokWebhookIngress(payload)).resolves.toEqual({
+      ignored: 0,
+      processed: 0,
+      targetCount: 0,
+    })
+    expect(mockDispatchResolvedWebhookTarget).not.toHaveBeenCalled()
+  })
+
+  it('dispatches every resolved target and reports typed outcomes', async () => {
+    mockFindTikTokWebhookTargets.mockResolvedValue(targets)
+    mockDispatchResolvedWebhookTarget
+      .mockResolvedValueOnce({
+        outcome: 'queued',
+        reason: 'queued',
+        response: new Response(null, { status: 200 }),
+      })
+      .mockResolvedValueOnce({
+        outcome: 'ignored',
+        reason: 'event-mismatch',
+        response: new Response(null, { status: 200 }),
+      })
+
+    await expect(executeTikTokWebhookIngress(payload)).resolves.toEqual({
+      ignored: 1,
+      processed: 1,
+      targetCount: 2,
+    })
+    expect(mockDispatchResolvedWebhookTarget).toHaveBeenCalledTimes(2)
+  })
+
+  it('throws after a target failure so the durable ingress job retries', async () => {
+    mockFindTikTokWebhookTargets.mockResolvedValue(targets)
+    mockDispatchResolvedWebhookTarget
+      .mockResolvedValueOnce({
+        outcome: 'queued',
+        reason: 'queued',
+        response: new Response(null, { status: 200 }),
+      })
+      .mockResolvedValueOnce({
+        outcome: 'failed',
+        reason: 'queue-failed',
+        response: new Response(null, { status: 500 }),
+      })
+
+    await expect(executeTikTokWebhookIngress(payload)).rejects.toThrow(
+      'Failed to dispatch 1 of 2 TikTok webhook targets'
+    )
+  })
+})

apps/sim/background/tiktok-webhook-ingress.ts

@@ -0,0 +1,105 @@
+import { createLogger } from '@sim/logger'
+import { task } from '@trigger.dev/sdk'
+import { NextRequest } from 'next/server'
+import type { TikTokWebhookEnvelope } from '@/lib/api/contracts/webhooks'
+import { dispatchResolvedWebhookTarget } from '@/lib/webhooks/processor'
+import { findTikTokWebhookTargets } from '@/lib/webhooks/providers/tiktok-targets'
+
+const logger = createLogger('TikTokWebhookIngressTask')
+
+export const TIKTOK_WEBHOOK_INGRESS_CONCURRENCY_LIMIT = 50
+export const TIKTOK_WEBHOOK_INGRESS_MAX_ATTEMPTS = 3
+
+export interface TikTokWebhookIngressPayload {
+  envelope: TikTokWebhookEnvelope
+  headers: {
+    'content-type': string
+  }
+  requestId: string
+  receivedAt: number
+}
+
+export interface TikTokWebhookIngressResult {
+  ignored: number
+  processed: number
+  targetCount: number
+}
+
+/**
+ * Resolves and dispatches all active workflow targets for one verified TikTok delivery. Throwing
+ * after any retryable target failure lets Trigger.dev replay the fanout; workflow-level
+ * idempotency prevents already-queued targets from executing twice.
+ */
+export async function executeTikTokWebhookIngress(
+  payload: TikTokWebhookIngressPayload
+): Promise<TikTokWebhookIngressResult> {
+  const targets = await findTikTokWebhookTargets(payload.envelope.user_openid, payload.requestId)
+  if (targets.length === 0) {
+    logger.info(`[${payload.requestId}] No TikTok webhook targets found`, {
+      event: payload.envelope.event,
+      userOpenIdPrefix: payload.envelope.user_openid.slice(0, 12),
+    })
+    return { ignored: 0, processed: 0, targetCount: 0 }
+  }
+
+  const request = new NextRequest('http://internal/api/webhooks/tiktok', {
+    method: 'POST',
+    headers: payload.headers,
+    body: JSON.stringify(payload.envelope),
+  })
+
+  let ignored = 0
+  let processed = 0
+  let failed = 0
+
+  for (const { webhook, workflow } of targets) {
+    const result = await dispatchResolvedWebhookTarget(
+      webhook,
+      workflow,
+      payload.envelope,
+      request,
+      {
+        requestId: payload.requestId,
+        path: webhook.path ?? undefined,
+        receivedAt: payload.receivedAt,
+        triggerTimestampMs: payload.envelope.create_time * 1000,
+      }
+    )
+
+    if (result.outcome === 'queued') {
+      processed += 1
+    } else if (result.outcome === 'ignored') {
+      ignored += 1
+    } else {
+      failed += 1
+    }
+  }
+
+  if (failed > 0) {
+    throw new Error(`Failed to dispatch ${failed} of ${targets.length} TikTok webhook targets`)
+  }
+
+  logger.info(`[${payload.requestId}] TikTok webhook fanout completed`, {
+    event: payload.envelope.event,
+    ignored,
+    processed,
+    targetCount: targets.length,
+  })
+
+  return { ignored, processed, targetCount: targets.length }
+}
+
+export const tiktokWebhookIngressTask = task({
+  id: 'tiktok-webhook-ingress',
+  machine: 'small-1x',
+  retry: {
+    maxAttempts: TIKTOK_WEBHOOK_INGRESS_MAX_ATTEMPTS,
+    factor: 2,
+    minTimeoutInMs: 1000,
+    maxTimeoutInMs: 10_000,
+  },
+  queue: {
+    concurrencyLimit: TIKTOK_WEBHOOK_INGRESS_CONCURRENCY_LIMIT,
+  },
+  run: async (payload: TikTokWebhookIngressPayload) => executeTikTokWebhookIngress(payload),
+})

apps/sim/background/webhook-execution.test.ts

@@ -0,0 +1,222 @@
+/**
+ * @vitest-environment node
+ */
+
+import {
+  dbChainMock,
+  dbChainMockFns,
+  executionPreprocessingMock,
+  executionPreprocessingMockFns,
+  loggingSessionMock,
+  loggingSessionMockFns,
+} from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockResolveWebhookRecordProviderConfig,
+  mockExecuteWorkflowCore,
+  mockWasExecutionFinalizedByCore,
+  mockRecordException,
+  mockGetActiveSpan,
+} = vi.hoisted(() => ({
+  mockResolveWebhookRecordProviderConfig: vi.fn(),
+  mockExecuteWorkflowCore: vi.fn(),
+  mockWasExecutionFinalizedByCore: vi.fn(),
+  mockRecordException: vi.fn(),
+  mockGetActiveSpan: vi.fn(),
+}))
+
+vi.mock('@opentelemetry/api', () => ({
+  trace: { getActiveSpan: mockGetActiveSpan },
+}))
+
+vi.mock('@sim/db', () => dbChainMock)
+vi.mock('@/lib/execution/preprocessing', () => executionPreprocessingMock)
+vi.mock('@/lib/logs/execution/logging-session', () => loggingSessionMock)
+
+vi.mock('@/lib/webhooks/env-resolver', () => ({
+  resolveWebhookRecordProviderConfig: mockResolveWebhookRecordProviderConfig,
+}))
+
+vi.mock('@/lib/workflows/executor/execution-core', () => ({
+  executeWorkflowCore: mockExecuteWorkflowCore,
+  wasExecutionFinalizedByCore: mockWasExecutionFinalizedByCore,
+}))
+
+vi.mock('@/lib/core/idempotency', () => ({
+  IdempotencyService: { createWebhookIdempotencyKey: vi.fn(() => 'idempotency-key') },
+  webhookIdempotency: {
+    executeWithIdempotency: vi.fn(
+      (_provider: string, _key: string, operation: () => Promise<unknown>) => operation()
+    ),
+  },
+}))
+
+vi.mock('@/lib/workflows/persistence/utils', () => ({
+  loadDeployedWorkflowState: vi.fn(async () => ({
+    blocks: {},
+    edges: [],
+    loops: {},
+    parallels: {},
+    deploymentVersionId: 'deployment-1',
+  })),
+}))
+
+vi.mock('@/lib/webhooks/providers', () => ({
+  getProviderHandler: vi.fn(() => ({})),
+}))
+
+vi.mock('@/lib/logs/execution/trace-spans/trace-spans', () => ({
+  buildTraceSpans: vi.fn(() => ({ traceSpans: [] })),
+}))
+
+vi.mock('@/lib/core/execution-limits', () => ({
+  createTimeoutAbortController: vi.fn(() => ({
+    signal: new AbortController().signal,
+    cleanup: vi.fn(),
+    isTimedOut: () => false,
+    timeoutMs: 120_000,
+  })),
+  getTimeoutErrorMessage: vi.fn(() => 'timed out'),
+}))
+
+vi.mock('@/lib/workflows/executor/pause-persistence', () => ({
+  handlePostExecutionPauseState: vi.fn(),
+}))
+
+vi.mock('@/lib/webhooks/attachment-processor', () => ({
+  WebhookAttachmentProcessor: class {},
+}))
+
+vi.mock('@/app/api/auth/oauth/utils', () => ({
+  resolveOAuthAccountId: vi.fn(),
+}))
+
+vi.mock('@/executor/execution/snapshot', () => ({
+  ExecutionSnapshot: class {},
+}))
+
+vi.mock('@/tools/safe-assign', () => ({ safeAssign: vi.fn() }))
+
+vi.mock('@/blocks', () => ({ getBlock: vi.fn(() => null) }))
+
+vi.mock('@/triggers', () => ({
+  getTrigger: vi.fn(),
+  isTriggerValid: vi.fn(() => false),
+}))
+
+import { executeWebhookJob, resolveWebhookExecutionProviderConfig } from './webhook-execution'
+
+describe('resolveWebhookExecutionProviderConfig', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('returns the resolved webhook record when provider config resolution succeeds', async () => {
+    const webhookRecord = {
+      id: 'webhook-1',
+      providerConfig: {
+        botToken: '{{SLACK_BOT_TOKEN}}',
+      },
+    }
+    const resolvedWebhookRecord = {
+      ...webhookRecord,
+      providerConfig: {
+        botToken: 'xoxb-resolved',
+      },
+    }
+
+    mockResolveWebhookRecordProviderConfig.mockResolvedValue(resolvedWebhookRecord)
+
+    await expect(
+      resolveWebhookExecutionProviderConfig(webhookRecord, 'slack', 'user-1', 'workspace-1')
+    ).resolves.toEqual(resolvedWebhookRecord)
+
+    expect(mockResolveWebhookRecordProviderConfig).toHaveBeenCalledWith(
+      webhookRecord,
+      'user-1',
+      'workspace-1'
+    )
+  })
+
+  it('throws a contextual error when provider config resolution fails', async () => {
+    mockResolveWebhookRecordProviderConfig.mockRejectedValue(new Error('env lookup failed'))
+
+    await expect(
+      resolveWebhookExecutionProviderConfig(
+        {
+          id: 'webhook-1',
+          providerConfig: {
+            botToken: '{{SLACK_BOT_TOKEN}}',
+          },
+        },
+        'slack',
+        'user-1',
+        'workspace-1'
+      )
+    ).rejects.toThrow(
+      'Failed to resolve webhook provider config for slack webhook webhook-1: env lookup failed'
+    )
+  })
+})
+
+describe('executeWebhookJob fault vs error handling', () => {
+  const payload = {
+    webhookId: 'webhook-1',
+    workflowId: 'workflow-1',
+    userId: 'user-1',
+    executionId: 'execution-1',
+    requestId: 'request-1',
+    provider: 'gmail',
+    body: { message: 'hello' },
+    headers: {},
+    path: '/webhook',
+    workspaceId: 'workspace-1',
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    executionPreprocessingMockFns.mockPreprocessExecution.mockResolvedValue({
+      success: true,
+      workflowRecord: { workspaceId: 'workspace-1', userId: 'user-1', variables: {} },
+      executionTimeout: { async: 120_000 },
+    })
+    mockResolveWebhookRecordProviderConfig.mockImplementation(async (record) => record)
+    dbChainMockFns.limit.mockResolvedValue([{ id: 'webhook-1' }])
+    mockGetActiveSpan.mockReturnValue({ recordException: mockRecordException })
+  })
+
+  it('completes the run (does not throw) when the failure was finalized by core', async () => {
+    mockExecuteWorkflowCore.mockRejectedValue(
+      new Error('Gmail 2 is missing required fields: Label')
+    )
+    mockWasExecutionFinalizedByCore.mockReturnValue(true)
+
+    const result = await executeWebhookJob(payload)
+
+    expect(result).toMatchObject({
+      success: false,
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      provider: 'gmail',
+    })
+    expect(loggingSessionMockFns.mockWaitForPostExecution).toHaveBeenCalled()
+    // User/workflow errors are already recorded by core — the catch must not re-log them.
+    expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled()
+    // The error is still recorded on the run span so it stays visible in traces.
+    expect(mockRecordException).toHaveBeenCalledWith(
+      expect.objectContaining({ message: 'Gmail 2 is missing required fields: Label' })
+    )
+  })
+
+  it('faults the run (re-throws) when the failure was not finalized by core', async () => {
+    mockExecuteWorkflowCore.mockRejectedValue(new Error('Workflow state not found'))
+    mockWasExecutionFinalizedByCore.mockReturnValue(false)
+
+    await expect(executeWebhookJob(payload)).rejects.toThrow('Workflow state not found')
+    // waitForPostExecution must run on every path so the finalized-by-core signal is always reliable.
+    expect(loggingSessionMockFns.mockWaitForPostExecution).toHaveBeenCalled()
+    // Pipeline/infra errors are recorded here before re-throwing to fault the trigger.dev run.
+    expect(loggingSessionMockFns.mockSafeCompleteWithError).toHaveBeenCalled()
+  })
+})

apps/sim/background/webhook-execution.ts

@@ -0,0 +1,728 @@
+import { trace } from '@opentelemetry/api'
+import { db } from '@sim/db'
+import { account, webhook } from '@sim/db/schema'
+import { createLogger, runWithRequestContext } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { isRecordLike } from '@sim/utils/object'
+import { task } from '@trigger.dev/sdk'
+import { eq } from 'drizzle-orm'
+import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
+import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits'
+import { IdempotencyService, webhookIdempotency } from '@/lib/core/idempotency'
+import { preprocessExecution } from '@/lib/execution/preprocessing'
+import { LoggingSession } from '@/lib/logs/execution/logging-session'
+import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
+import {
+  type WebhookAttachment,
+  WebhookAttachmentProcessor,
+} from '@/lib/webhooks/attachment-processor'
+import { resolveWebhookRecordProviderConfig } from '@/lib/webhooks/env-resolver'
+import { getProviderHandler } from '@/lib/webhooks/providers'
+import {
+  executeWorkflowCore,
+  wasExecutionFinalizedByCore,
+} from '@/lib/workflows/executor/execution-core'
+import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
+import { loadDeployedWorkflowState } from '@/lib/workflows/persistence/utils'
+import { resolveOAuthAccountId } from '@/app/api/auth/oauth/utils'
+import { WEBHOOK_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits'
+import { getBlock } from '@/blocks'
+import { ExecutionSnapshot } from '@/executor/execution/snapshot'
+import type { ExecutionMetadata } from '@/executor/execution/types'
+import type { ExecutionResult } from '@/executor/types'
+import { hasExecutionResult } from '@/executor/utils/errors'
+import { safeAssign } from '@/tools/safe-assign'
+import { getTrigger, isTriggerValid } from '@/triggers'
+
+const logger = createLogger('TriggerWebhookExecution')
+
+type WebhookAttachmentInput = Omit<WebhookAttachment, 'data'> & { data: unknown }
+
+function isSerializedBuffer(value: unknown): value is { type: 'Buffer'; data: number[] } {
+  return isRecordLike(value) && value.type === 'Buffer' && Array.isArray(value.data)
+}
+
+function hasSupportedAttachmentData(value: unknown): boolean {
+  return (
+    Buffer.isBuffer(value) ||
+    typeof value === 'string' ||
+    value instanceof ArrayBuffer ||
+    ArrayBuffer.isView(value) ||
+    Array.isArray(value) ||
+    isSerializedBuffer(value)
+  )
+}
+
+function toAttachmentBuffer(data: unknown, name: string): Buffer {
+  if (Buffer.isBuffer(data)) {
+    return data
+  }
+
+  if (isSerializedBuffer(data)) {
+    return Buffer.from(data.data)
+  }
+
+  if (data instanceof ArrayBuffer) {
+    return Buffer.from(data)
+  }
+
+  if (ArrayBuffer.isView(data)) {
+    return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
+  }
+
+  if (Array.isArray(data)) {
+    return Buffer.from(data)
+  }
+
+  if (typeof data === 'string') {
+    const trimmed = data.trim()
+    if (trimmed.startsWith('data:')) {
+      const [, base64Data] = trimmed.split(',')
+      return Buffer.from(base64Data ?? '', 'base64')
+    }
+    return Buffer.from(trimmed, 'base64')
+  }
+
+  throw new Error(`Attachment '${name}' has unsupported data format`)
+}
+
+function isWebhookAttachmentInput(value: unknown): value is WebhookAttachmentInput {
+  if (!isRecordLike(value)) {
+    return false
+  }
+
+  return (
+    typeof value.name === 'string' &&
+    typeof value.size === 'number' &&
+    hasSupportedAttachmentData(value.data) &&
+    (value.contentType === undefined || typeof value.contentType === 'string') &&
+    (value.mimeType === undefined || typeof value.mimeType === 'string')
+  )
+}
+
+function normalizeWebhookAttachment(value: unknown): WebhookAttachment | null {
+  if (!isWebhookAttachmentInput(value)) {
+    return null
+  }
+
+  return {
+    name: value.name,
+    data: toAttachmentBuffer(value.data, value.name),
+    contentType: value.contentType,
+    mimeType: value.mimeType,
+    size: value.size,
+  }
+}
+
+function normalizeWebhookAttachments(value: unknown): WebhookAttachment[] {
+  if (!Array.isArray(value)) {
+    return []
+  }
+
+  return value.flatMap((attachment) => {
+    const normalized = normalizeWebhookAttachment(attachment)
+    return normalized ? [normalized] : []
+  })
+}
+
+export function buildWebhookCorrelation(
+  payload: WebhookExecutionPayload
+): AsyncExecutionCorrelation {
+  const executionId = payload.executionId || generateId()
+  const requestId = payload.requestId || payload.correlation?.requestId || executionId.slice(0, 8)
+
+  return {
+    executionId,
+    requestId,
+    source: 'webhook',
+    workflowId: payload.workflowId,
+    webhookId: payload.webhookId,
+    path: payload.path,
+    provider: payload.provider,
+    triggerType: payload.correlation?.triggerType || 'webhook',
+  }
+}
+
+/**
+ * Process trigger outputs based on their schema definitions.
+ * Finds outputs marked as 'file' or 'file[]' and uploads them to execution storage.
+ */
+async function processTriggerFileOutputs(
+  input: unknown,
+  triggerOutputs: Record<string, unknown>,
+  context: {
+    workspaceId: string
+    workflowId: string
+    executionId: string
+    requestId: string
+    userId?: string
+  },
+  path = ''
+): Promise<unknown> {
+  if (!input || typeof input !== 'object') {
+    return input
+  }
+
+  const processed = (Array.isArray(input) ? [] : {}) as Record<string, unknown>
+
+  for (const [key, value] of Object.entries(input)) {
+    const currentPath = path ? `${path}.${key}` : key
+    const outputDef = triggerOutputs[key] as Record<string, unknown> | undefined
+
+    if (outputDef?.type === 'file[]' && Array.isArray(value)) {
+      try {
+        processed[key] = await WebhookAttachmentProcessor.processAttachments(
+          normalizeWebhookAttachments(value),
+          context
+        )
+      } catch (error) {
+        processed[key] = []
+      }
+    } else if (outputDef?.type === 'file' && value) {
+      const attachment = normalizeWebhookAttachment(value)
+      if (!attachment) {
+        processed[key] = value
+        continue
+      }
+
+      try {
+        const [processedFile] = await WebhookAttachmentProcessor.processAttachments(
+          [attachment],
+          context
+        )
+        processed[key] = processedFile
+      } catch (error) {
+        logger.error(`[${context.requestId}] Error processing ${currentPath}:`, error)
+        processed[key] = value
+      }
+    } else if (
+      outputDef &&
+      typeof outputDef === 'object' &&
+      (outputDef.type === 'object' || outputDef.type === 'json') &&
+      outputDef.properties
+    ) {
+      processed[key] = await processTriggerFileOutputs(
+        value,
+        outputDef.properties as Record<string, unknown>,
+        context,
+        currentPath
+      )
+    } else if (outputDef && typeof outputDef === 'object' && !outputDef.type) {
+      processed[key] = await processTriggerFileOutputs(
+        value,
+        outputDef as Record<string, unknown>,
+        context,
+        currentPath
+      )
+    } else {
+      processed[key] = value
+    }
+  }
+
+  return processed
+}
+
+export type WebhookExecutionPayload = {
+  webhookId: string
+  workflowId: string
+  userId: string
+  executionId?: string
+  requestId?: string
+  correlation?: AsyncExecutionCorrelation
+  provider: string
+  body: unknown
+  headers: Record<string, string>
+  path: string
+  blockId?: string
+  workspaceId?: string
+  credentialId?: string
+  /** Epoch ms when the webhook HTTP request was first received (for dispatch-latency metrics). */
+  webhookReceivedAt?: number
+  /** Epoch ms of the originating provider interaction (e.g. Slack x-slack-request-timestamp). */
+  triggerTimestampMs?: number
+  /**
+   * Billing actor resolved by the webhook route, set ONLY for in-process inline
+   * execution that runs microseconds after resolution. The background pass reuses
+   * it to skip the redundant billed-account lookup. Deliberately absent on queued
+   * (Trigger.dev) and persisted payloads — a deferred run could outlive a
+   * billed-account change, so it re-resolves the current actor instead.
+   */
+  resolvedActorUserId?: string
+}
+
+export async function executeWebhookJob(payload: WebhookExecutionPayload) {
+  const correlation = buildWebhookCorrelation(payload)
+  const executionId = correlation.executionId
+  const requestId = correlation.requestId
+
+  return runWithRequestContext({ requestId }, async () => {
+    logger.info(`[${requestId}] Starting webhook execution`, {
+      webhookId: payload.webhookId,
+      workflowId: payload.workflowId,
+      provider: payload.provider,
+      userId: payload.userId,
+      executionId,
+    })
+
+    const idempotencyKey = IdempotencyService.createWebhookIdempotencyKey(
+      payload.webhookId,
+      payload.headers,
+      payload.body,
+      payload.provider
+    )
+
+    const runOperation = async () => {
+      return await executeWebhookJobInternal(payload, correlation)
+    }
+
+    return await webhookIdempotency.executeWithIdempotency(
+      payload.provider,
+      idempotencyKey,
+      runOperation
+    )
+  })
+}
+
+export async function resolveWebhookExecutionProviderConfig<
+  T extends { id: string; providerConfig?: unknown },
+>(
+  webhookRecord: T,
+  provider: string,
+  userId: string,
+  workspaceId?: string
+): Promise<T & { providerConfig: Record<string, unknown> }> {
+  try {
+    return await resolveWebhookRecordProviderConfig(webhookRecord, userId, workspaceId)
+  } catch (error) {
+    const errorMessage = toError(error).message
+    throw new Error(
+      `Failed to resolve webhook provider config for ${provider} webhook ${webhookRecord.id}: ${errorMessage}`
+    )
+  }
+}
+
+async function resolveCredentialAccountUserId(credentialId: string): Promise<string | undefined> {
+  const resolved = await resolveOAuthAccountId(credentialId)
+  if (!resolved) {
+    return undefined
+  }
+  const [credentialRecord] = await db
+    .select({ userId: account.userId })
+    .from(account)
+    .where(eq(account.id, resolved.accountId))
+    .limit(1)
+  return credentialRecord?.userId
+}
+
+/**
+ * Handle execution result status (timeout, pause, resume).
+ * Shared between all provider paths to eliminate duplication.
+ */
+async function handleExecutionResult(
+  executionResult: ExecutionResult,
+  ctx: {
+    loggingSession: LoggingSession
+    timeoutController: ReturnType<typeof createTimeoutAbortController>
+    requestId: string
+    executionId: string
+    workflowId: string
+  }
+) {
+  if (
+    executionResult.status === 'cancelled' &&
+    ctx.timeoutController.isTimedOut() &&
+    ctx.timeoutController.timeoutMs
+  ) {
+    const timeoutErrorMessage = getTimeoutErrorMessage(null, ctx.timeoutController.timeoutMs)
+    logger.info(`[${ctx.requestId}] Webhook execution timed out`, {
+      timeoutMs: ctx.timeoutController.timeoutMs,
+    })
+    await ctx.loggingSession.markAsFailed(timeoutErrorMessage)
+  } else {
+    await handlePostExecutionPauseState({
+      result: executionResult,
+      workflowId: ctx.workflowId,
+      executionId: ctx.executionId,
+      loggingSession: ctx.loggingSession,
+    })
+  }
+
+  await ctx.loggingSession.waitForPostExecution()
+}
+
+async function executeWebhookJobInternal(
+  payload: WebhookExecutionPayload,
+  correlation: AsyncExecutionCorrelation
+) {
+  const { executionId, requestId } = correlation
+  const loggingSession = new LoggingSession(
+    payload.workflowId,
+    executionId,
+    payload.provider,
+    requestId
+  )
+
+  const preprocessResult = await preprocessExecution({
+    workflowId: payload.workflowId,
+    userId: payload.userId,
+    triggerType: 'webhook',
+    executionId,
+    requestId,
+    triggerData: { correlation },
+    checkRateLimit: false,
+    checkDeployment: false,
+    skipUsageLimits: true,
+    workspaceId: payload.workspaceId,
+    loggingSession,
+    /**
+     * Reuse the route-resolved actor only for inline execution (set on the
+     * in-process payload). When absent — queued/Trigger.dev runs — preprocessing
+     * re-resolves the current billed account. Either way the ban and
+     * archived-workflow gates run fresh against the resolved actor.
+     */
+    resolvedActorUserId: payload.resolvedActorUserId,
+  })
+
+  if (!preprocessResult.success) {
+    throw new Error(preprocessResult.error?.message || 'Preprocessing failed in background job')
+  }
+
+  const { workflowRecord, executionTimeout } = preprocessResult
+  if (!workflowRecord) {
+    throw new Error(`Workflow ${payload.workflowId} not found during preprocessing`)
+  }
+
+  const workspaceId = workflowRecord.workspaceId
+  if (!workspaceId) {
+    throw new Error(`Workflow ${payload.workflowId} has no associated workspace`)
+  }
+
+  const workflowVariables = (workflowRecord.variables as Record<string, unknown>) || {}
+  const asyncTimeout = executionTimeout?.async ?? 120_000
+  const timeoutController = createTimeoutAbortController(asyncTimeout)
+
+  let deploymentVersionId: string | undefined
+
+  try {
+    const [workflowData, webhookRows, resolvedCredentialUserId] = await Promise.all([
+      loadDeployedWorkflowState(payload.workflowId, workspaceId),
+      db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1),
+      payload.credentialId
+        ? resolveCredentialAccountUserId(payload.credentialId)
+        : Promise.resolve(undefined),
+    ])
+    const credentialAccountUserId = resolvedCredentialUserId
+    if (payload.credentialId && !credentialAccountUserId) {
+      logger.warn(
+        `[${requestId}] Failed to resolve credential account for credential ${payload.credentialId}`
+      )
+    }
+
+    if (!workflowData) {
+      throw new Error(
+        'Workflow state not found. The workflow may not be deployed or the deployment data may be corrupted.'
+      )
+    }
+
+    const { blocks, edges, loops, parallels } = workflowData
+    deploymentVersionId =
+      'deploymentVersionId' in workflowData
+        ? (workflowData.deploymentVersionId as string)
+        : undefined
+
+    const handler = getProviderHandler(payload.provider)
+
+    let input: Record<string, unknown> | null = null
+    let skipMessage: string | undefined
+
+    const webhookRecord = webhookRows[0]
+    if (!webhookRecord) {
+      throw new Error(`Webhook record not found: ${payload.webhookId}`)
+    }
+
+    const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig(
+      webhookRecord,
+      payload.provider,
+      workflowRecord.userId,
+      workspaceId
+    )
+
+    if (handler.formatInput) {
+      const result = await handler.formatInput({
+        webhook: resolvedWebhookRecord,
+        workflow: { id: payload.workflowId, userId: payload.userId },
+        body: payload.body,
+        headers: payload.headers,
+        requestId,
+      })
+      input = result.input as Record<string, unknown> | null
+      skipMessage = result.skip?.message
+    } else {
+      input = payload.body as Record<string, unknown> | null
+    }
+
+    if (!input && handler.handleEmptyInput) {
+      const skipResult = handler.handleEmptyInput(requestId)
+      if (skipResult) {
+        skipMessage = skipResult.message
+      }
+    }
+
+    if (skipMessage) {
+      await loggingSession.safeStart({
+        userId: payload.userId,
+        workspaceId,
+        variables: {},
+        triggerData: {
+          isTest: false,
+          correlation,
+        },
+        deploymentVersionId,
+      })
+
+      await loggingSession.safeComplete({
+        endedAt: new Date().toISOString(),
+        totalDurationMs: 0,
+        finalOutput: { message: skipMessage },
+        traceSpans: [],
+      })
+
+      return {
+        success: true,
+        workflowId: payload.workflowId,
+        executionId,
+        output: { message: skipMessage },
+        executedAt: new Date().toISOString(),
+      }
+    }
+
+    if (input && payload.blockId && blocks[payload.blockId]) {
+      try {
+        const triggerBlock = blocks[payload.blockId]
+        const rawSelectedTriggerId = triggerBlock?.subBlocks?.selectedTriggerId?.value
+        const rawTriggerId = triggerBlock?.subBlocks?.triggerId?.value
+
+        let resolvedTriggerId = [rawSelectedTriggerId, rawTriggerId].find(
+          (candidate): candidate is string =>
+            typeof candidate === 'string' && isTriggerValid(candidate)
+        )
+
+        if (!resolvedTriggerId) {
+          const blockConfig = getBlock(triggerBlock.type)
+          if (blockConfig?.category === 'triggers' && isTriggerValid(triggerBlock.type)) {
+            resolvedTriggerId = triggerBlock.type
+          } else if (triggerBlock.triggerMode && blockConfig?.triggers?.enabled) {
+            const available = blockConfig.triggers?.available?.[0]
+            if (available && isTriggerValid(available)) {
+              resolvedTriggerId = available
+            }
+          }
+        }
+
+        if (resolvedTriggerId) {
+          const triggerConfig = getTrigger(resolvedTriggerId)
+
+          if (triggerConfig.outputs) {
+            const processedInput = await processTriggerFileOutputs(input, triggerConfig.outputs, {
+              workspaceId,
+              workflowId: payload.workflowId,
+              executionId,
+              requestId,
+              userId: payload.userId,
+            })
+            safeAssign(input, processedInput as Record<string, unknown>)
+          }
+        }
+      } catch (error) {
+        logger.error(`[${requestId}] Error processing trigger file outputs:`, error)
+      }
+    }
+
+    if (input && handler.processInputFiles && payload.blockId && blocks[payload.blockId]) {
+      try {
+        await handler.processInputFiles({
+          input,
+          blocks,
+          blockId: payload.blockId,
+          workspaceId,
+          workflowId: payload.workflowId,
+          executionId,
+          requestId,
+          userId: payload.userId,
+        })
+      } catch (error) {
+        logger.error(`[${requestId}] Error processing provider-specific files:`, error)
+      }
+    }
+
+    logger.info(`[${requestId}] Executing workflow for ${payload.provider} webhook`)
+
+    const metadata: ExecutionMetadata = {
+      requestId,
+      executionId,
+      workflowId: payload.workflowId,
+      workspaceId,
+      userId: payload.userId,
+      sessionUserId: undefined,
+      workflowUserId: workflowRecord.userId,
+      triggerType: payload.provider || 'webhook',
+      triggerBlockId: payload.blockId,
+      useDraftState: false,
+      startTime: new Date().toISOString(),
+      isClientSession: false,
+      credentialAccountUserId,
+      correlation,
+      workflowStateOverride: {
+        blocks,
+        edges,
+        loops: loops || {},
+        parallels: parallels || {},
+        deploymentVersionId,
+      },
+    }
+
+    const triggerInput = input || {}
+
+    /**
+     * Surface the pre-execution latency that per-block timings cannot see: the
+     * gap between webhook receipt and the first block running, and — for
+     * trigger_id-bound providers like Slack — the true age of the interaction
+     * against its 3s expiry window. Logged structured so it is queryable/alarmable.
+     */
+    if (payload.webhookReceivedAt !== undefined || payload.triggerTimestampMs !== undefined) {
+      const now = Date.now()
+      logger.info(`[${requestId}] Webhook dispatch latency`, {
+        workflowId: payload.workflowId,
+        provider: payload.provider,
+        dispatchLatencyMs:
+          payload.webhookReceivedAt !== undefined ? now - payload.webhookReceivedAt : undefined,
+        triggerAgeMs:
+          payload.triggerTimestampMs !== undefined ? now - payload.triggerTimestampMs : undefined,
+      })
+    }
+
+    const snapshot = new ExecutionSnapshot(
+      metadata,
+      workflowRecord,
+      triggerInput,
+      workflowVariables,
+      []
+    )
+
+    const executionResult = await executeWorkflowCore({
+      snapshot,
+      callbacks: {},
+      loggingSession,
+      includeFileBase64: false,
+      base64MaxBytes: undefined,
+      abortSignal: timeoutController.signal,
+    })
+
+    await handleExecutionResult(executionResult, {
+      loggingSession,
+      timeoutController,
+      requestId,
+      executionId,
+      workflowId: payload.workflowId,
+    })
+
+    logger.info(`[${requestId}] Webhook execution completed`, {
+      success: executionResult.success,
+      workflowId: payload.workflowId,
+      provider: payload.provider,
+    })
+
+    return {
+      success: executionResult.success,
+      workflowId: payload.workflowId,
+      executionId,
+      output: executionResult.output,
+      executedAt: new Date().toISOString(),
+      provider: payload.provider,
+    }
+  } catch (error: unknown) {
+    const errorMessage = toError(error).message
+    const errorStack = error instanceof Error ? error.stack : undefined
+
+    logger.error(`[${requestId}] Webhook execution failed`, {
+      error: errorMessage,
+      stack: errorStack,
+      workflowId: payload.workflowId,
+      provider: payload.provider,
+    })
+
+    // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the
+    // signal is reliable and the failure is fully persisted before we decide fault vs error.
+    await loggingSession.waitForPostExecution()
+
+    // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.)
+    // is finalized by core and already recorded in the execution logs. That is a user/workflow error,
+    // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors
+    // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below.
+    if (wasExecutionFinalizedByCore(error, executionId)) {
+      // Record the exception on the run span so it stays visible in traces without
+      // marking the span as ERROR — that status is what faults the trigger.dev run.
+      trace.getActiveSpan()?.recordException(toError(error))
+
+      return {
+        success: false,
+        workflowId: payload.workflowId,
+        executionId,
+        output: hasExecutionResult(error) ? error.executionResult.output : {},
+        executedAt: new Date().toISOString(),
+        provider: payload.provider,
+      }
+    }
+
+    try {
+      await loggingSession.safeStart({
+        userId: payload.userId,
+        workspaceId,
+        variables: {},
+        triggerData: {
+          isTest: false,
+          correlation,
+        },
+        deploymentVersionId,
+      })
+
+      const executionResult = hasExecutionResult(error)
+        ? error.executionResult
+        : {
+            success: false,
+            output: {},
+            logs: [],
+          }
+      const { traceSpans } = buildTraceSpans(executionResult)
+
+      await loggingSession.safeCompleteWithError({
+        endedAt: new Date().toISOString(),
+        totalDurationMs: 0,
+        error: {
+          message: errorMessage || 'Webhook execution failed',
+          stackTrace: errorStack,
+        },
+        traceSpans,
+      })
+    } catch (loggingError) {
+      logger.error(`[${requestId}] Failed to complete logging session`, loggingError)
+    }
+
+    throw error
+  } finally {
+    timeoutController.cleanup()
+  }
+}
+
+export const webhookExecution = task({
+  id: 'webhook-execution',
+  machine: 'medium-1x',
+  retry: {
+    maxAttempts: 1,
+  },
+  queue: {
+    concurrencyLimit: WEBHOOK_EXECUTION_CONCURRENCY_LIMIT,
+  },
+  run: async (payload: WebhookExecutionPayload) => executeWebhookJob(payload),
+})

apps/sim/background/workflow-column-execution.ts

@@ -0,0 +1,877 @@
+import { db } from '@sim/db'
+import { workflow as workflowTable } from '@sim/db/schema'
+import { createLogger, runWithRequestContext } from '@sim/logger'
+import { describeError, toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { generateId } from '@sim/utils/id'
+import { backoffWithJitter } from '@sim/utils/retry'
+import { task } from '@trigger.dev/sdk'
+import { eq } from 'drizzle-orm'
+import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
+import { isRetryableInfrastructureError } from '@/lib/core/errors/retryable-infrastructure'
+import { createTimeoutAbortController } from '@/lib/core/execution-limits'
+import { RateLimiter } from '@/lib/core/rate-limiter/rate-limiter'
+import { preprocessExecution } from '@/lib/execution/preprocessing'
+import { withCascadeLock } from '@/lib/table/cascade-lock'
+import { getColumnId } from '@/lib/table/column-keys'
+import { isExecCancelled } from '@/lib/table/deps'
+import { appendTableEvent } from '@/lib/table/events'
+import type {
+  RowData,
+  RowExecutionMetadata,
+  TableDefinition,
+  WorkflowGroup,
+} from '@/lib/table/types'
+import type { WorkflowGroupCellPayload } from '@/lib/table/workflow-columns'
+import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
+
+export type { WorkflowGroupCellPayload }
+
+const logger = createLogger('TriggerWorkflowGroupCell')
+
+/** Max rate-limit retry attempts per cell before giving up and writing a
+ *  re-runnable error. With `backoffWithJitter` (base 500ms, max 30s) this is
+ *  ~1–2 minutes of pacing — enough to ride out a transient burst without
+ *  stalling the dispatcher window indefinitely. */
+const RATE_LIMIT_MAX_ATTEMPTS = 6
+
+/** Cell-task entrypoint. Holds a per-row cascade lock so only one worker
+ *  advances a given row at a time; bails on contention. The held lock heart-
+ *  beats every 10s so a crashed pod releases within ~30s.
+ *
+ *  After the cascade finishes and the lock releases, re-checks for a runnable
+ *  queued marker that may have landed between the cascade's final
+ *  `pickNextEligibleGroupForRow` and the lock release (a window where a
+ *  contender bails on the still-held lock but we're already done). If one
+ *  appeared, re-acquire and drive it — this is the same task re-acquiring the
+ *  lock, NOT a queue re-enqueue or a timed poll, and it loops only while a
+ *  runnable group exists. */
+export async function executeWorkflowGroupCellJob(
+  payload: WorkflowGroupCellPayload,
+  signal?: AbortSignal
+) {
+  const { tableId, rowId, workspaceId } = payload
+  const { getTableById } = await import('@/lib/table/service')
+  const { getRowById } = await import('@/lib/table/rows/service')
+  const { pickNextEligibleGroupForRow } = await import('@/lib/table/workflow-columns')
+
+  let currentPayload = payload
+  while (true) {
+    if (signal?.aborted) break
+    const outcome = await withCascadeLock(tableId, rowId, currentPayload.executionId, () =>
+      runRowCascadeLoop(currentPayload, signal)
+    )
+    if (outcome.status === 'contended') {
+      // Another worker owns the row's cascade; it drains the queued marker.
+      logger.info(
+        `Cascade lock held — bailing (table=${tableId} row=${rowId} executionId=${currentPayload.executionId})`
+      )
+      break
+    }
+    // Usage limit hit mid-cascade: the dispatch is halted and no cell was
+    // marked, so stop re-driving this row.
+    if (outcome.result === 'blocked') break
+    if (signal?.aborted) break
+    const freshTable = await getTableById(tableId)
+    if (!freshTable) break
+    const freshRow = await getRowById(tableId, rowId, workspaceId)
+    if (!freshRow) break
+    const next = pickNextEligibleGroupForRow(freshTable, freshRow)
+    if (!next) break
+    // Only re-drive a genuine queued marker (an explicit run request whose
+    // cell-task bailed during our release window). The inner cascade loop has
+    // already drained every auto-eligible group, so re-driving a non-marker
+    // group here would re-run forever — e.g. a group that completed with empty
+    // outputs stays auto-eligible (the inner loop excludes it via
+    // `excludeGroupId`, but this outer pass has no such anchor).
+    const nextExec = freshRow.executions?.[next.id]
+    const hasQueuedMarker = nextExec?.status === 'pending' && nextExec.executionId == null
+    if (!hasQueuedMarker) break
+    currentPayload = {
+      ...currentPayload,
+      groupId: next.id,
+      workflowId: next.workflowId,
+      // Re-derive so a workflow group after an enrichment group doesn't keep a stale enrichmentId.
+      enrichmentId: next.enrichmentId,
+      executionId: generateId(),
+    }
+  }
+}
+
+/** Re-fetches the table schema each iteration so groups added DURING the
+ *  cascade become visible to the eligibility check. The resume worker must
+ *  already hold the row's cascade lock before calling. */
+export async function runRowCascadeLoop(
+  payload: WorkflowGroupCellPayload,
+  signal?: AbortSignal
+): Promise<'blocked' | undefined> {
+  const { tableId, rowId, workspaceId } = payload
+  const { getTableById } = await import('@/lib/table/service')
+  const { getRowById } = await import('@/lib/table/rows/service')
+  const { pickNextEligibleGroupForRow } = await import('@/lib/table/workflow-columns')
+
+  let currentGroupId = payload.groupId
+  let currentWorkflowId = payload.workflowId
+  // Fresh executionId per iteration: SQL guard rejects writes whose id ≠
+  // row.executions[gid].executionId, so we need a new claim per group.
+  let currentExecutionId = payload.executionId
+
+  while (true) {
+    if (signal?.aborted) break
+
+    const freshTable = await getTableById(tableId)
+    if (!freshTable) {
+      logger.warn(`Table ${tableId} vanished mid-cascade`)
+      break
+    }
+    const currentGroup = freshTable.schema.workflowGroups?.find((g) => g.id === currentGroupId)
+    if (!currentGroup) {
+      logger.warn(`Group ${currentGroupId} no longer exists on table ${tableId}`)
+      break
+    }
+
+    const result = await runWorkflowAndWriteTerminal(
+      {
+        ...payload,
+        groupId: currentGroupId,
+        workflowId: currentWorkflowId,
+        executionId: currentExecutionId,
+      },
+      signal,
+      freshTable,
+      currentGroup
+    )
+
+    if (result === 'paused') break
+    // Hard stop (e.g. usage limit): the dispatch was halted and no cell was
+    // marked. Propagate so the outer re-drive loop stops too — otherwise it
+    // would re-pick the still-pending queued marker and spin.
+    if (result === 'blocked') return 'blocked'
+
+    const freshRow = await getRowById(tableId, rowId, workspaceId)
+    if (!freshRow) break
+    const next = pickNextEligibleGroupForRow(freshTable, freshRow, currentGroupId)
+    if (!next) break
+    currentGroupId = next.id
+    currentWorkflowId = next.workflowId
+    currentExecutionId = generateId()
+  }
+  return undefined
+}
+
+/** Returns `'paused'` to signal the cascade loop must exit (resume worker
+ *  takes over) and `'blocked'` for a hard stop (usage limit — dispatch halted,
+ *  cell left unmarked). `'completed' | 'error'` keep the loop running. */
+async function runWorkflowAndWriteTerminal(
+  payload: WorkflowGroupCellPayload,
+  signal: AbortSignal | undefined,
+  table: TableDefinition,
+  group: WorkflowGroup
+): Promise<'completed' | 'error' | 'paused' | 'blocked'> {
+  const { tableId, tableName, rowId, groupId, workflowId, workspaceId, executionId, dispatchId } =
+    payload
+  // Read from the live `group`, not the payload: in a cascade the payload is the
+  // first group's snapshot, so a downstream group with a different version must
+  // use its own setting (same reason `workflowId` is re-derived per iteration).
+  const deploymentMode = group.deploymentMode
+  const requestId = `wfgrp-${executionId}`
+
+  return runWithRequestContext({ requestId }, async () => {
+    const { getRowById } = await import('@/lib/table/rows/service')
+    const { executeWorkflow } = await import('@/lib/workflows/executor/execute-workflow')
+    const { loadWorkflowFromNormalizedTables, loadDeployedWorkflowState } = await import(
+      '@/lib/workflows/persistence/utils'
+    )
+    const { writeWorkflowGroupState, markWorkflowGroupPickedUp, buildOutputsByBlockId } =
+      await import('@/lib/table/cell-write')
+    const { stashCellContextForResume } = await import('@/lib/table/workflow-columns')
+
+    const cellCtx = { tableId, rowId, workspaceId, groupId, executionId, requestId }
+    const writeState = (executionState: RowExecutionMetadata, dataPatch?: RowData) =>
+      writeWorkflowGroupState(cellCtx, { executionState, dataPatch })
+
+    /** Pre-execution cancellation guard: a cell cancelled while it sat in the
+     *  queue (e.g. trigger.dev concurrency backlog) must not run once it
+     *  dequeues. Reads the already-loaded row's exec — no extra query. */
+    const cancelledBeforeRun = (exec: RowExecutionMetadata | undefined): boolean => {
+      if (!isExecCancelled(exec)) return false
+      logger.info(
+        `Skipping cell — cancelled before execution (table=${tableId} row=${rowId} group=${groupId})`
+      )
+      return true
+    }
+
+    // Enrichment groups call a registry function directly instead of running a
+    // workflow, reusing the same pickup → run → terminal-write status flow. The
+    // `enrichmentId` guard ensures only true registry enrichments take this path
+    // — a group typed 'enrichment' without a registry id falls through to the
+    // workflow path rather than erroring.
+    if (group.type === 'enrichment' && group.enrichmentId) {
+      const { getEnrichment } = await import('@/enrichments/registry')
+      const { runEnrichment, skippedEnrichmentDetail } = await import('@/enrichments/run')
+      const enrichment = getEnrichment(group.enrichmentId)
+      // `tableRowExecutions.workflowId` is an opaque id for status; use the
+      // enrichment id for enrichment cells.
+      const statusId = group.enrichmentId ?? ''
+      if (!enrichment) {
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId: statusId,
+          error: `Unknown enrichment "${group.enrichmentId ?? ''}"`,
+        })
+        return 'error'
+      }
+
+      const row = await getRowById(tableId, rowId, workspaceId)
+      if (!row) {
+        logger.warn(`Row ${rowId} vanished before enrichment`)
+        return 'error'
+      }
+
+      if (cancelledBeforeRun(row.executions?.[groupId])) return 'error'
+
+      // Resolve the billing/enforcement actor: the user who triggered the run,
+      // else the workspace billed account (automated/auto-fire). Enrichment must
+      // never run unattributed — if neither resolves, fail the cell rather than
+      // running free and ungated.
+      const enrichmentActorUserId =
+        payload.triggeredByUserId ?? (await getWorkspaceBilledAccountUserId(workspaceId))
+      if (!enrichmentActorUserId) {
+        logger.error(
+          `No billing actor for enrichment — failing cell (table=${tableId} row=${rowId} group=${groupId})`
+        )
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId: statusId,
+          error: 'Unable to resolve a billing account for this enrichment.',
+        })
+        return 'error'
+      }
+
+      // Gate per-member + pooled usage before incurring hosted-key cost. Mirrors
+      // the workflow-group gate below: clear the cell's pre-stamp and signal the
+      // client to upgrade rather than marking the cell errored.
+      const usage = await checkActorUsageLimits(enrichmentActorUserId, workspaceId)
+      if (usage.isExceeded) {
+        logger.warn(
+          `Usage limit reached — halting enrichment (table=${tableId} row=${rowId} group=${groupId})`
+        )
+        const { updateRow } = await import('@/lib/table/rows/service')
+        await updateRow(
+          { tableId, rowId, data: {}, workspaceId, executionsPatch: { [groupId]: null } },
+          table,
+          requestId
+        ).catch((err) =>
+          logger.warn(`Failed to clear cell pre-stamp on usage limit`, {
+            error: toError(err).message,
+          })
+        )
+        let shouldEmit = true
+        if (dispatchId) {
+          const { completeDispatchIfActive } = await import('@/lib/table/dispatcher')
+          shouldEmit = await completeDispatchIfActive(dispatchId)
+        }
+        if (shouldEmit) {
+          await appendTableEvent({
+            kind: 'usageLimitReached',
+            tableId,
+            ...(dispatchId ? { dispatchId } : {}),
+            message: usage.message ?? 'Usage limit exceeded. Please upgrade your plan to continue.',
+          })
+        }
+        return 'blocked'
+      }
+
+      const pickedUp = await markWorkflowGroupPickedUp(cellCtx, {
+        workflowId: statusId,
+        jobId: null,
+      })
+      if (pickedUp === 'skipped') return 'error'
+
+      // Map table columns → enrichment input ids (skip this group's own outputs).
+      const ownOutputColumns = new Set(group.outputs.map((o) => o.columnName))
+      const enrichInputs: Record<string, unknown> = {}
+      for (const m of group.inputMappings ?? []) {
+        if (ownOutputColumns.has(m.columnName)) continue
+        enrichInputs[m.inputName] = row.data[m.columnName]
+      }
+
+      // Skip (don't error) rows missing a required input — common when a table
+      // is partially filled. Clear any prior output values so a stale result
+      // doesn't linger (and doesn't mark the group `completed`-and-filled, which
+      // would block the auto cascade from re-enriching once inputs return).
+      const isEmpty = (v: unknown) => v === undefined || v === null || v === ''
+      const missingRequired = enrichment.inputs.some(
+        (i) => i.required && isEmpty(enrichInputs[i.id])
+      )
+      if (missingRequired) {
+        const clearPatch: RowData = {}
+        for (const out of group.outputs) {
+          if (!isEmpty(row.data[out.columnName])) clearPatch[out.columnName] = ''
+        }
+        await writeState(
+          {
+            status: 'completed',
+            executionId,
+            jobId: null,
+            workflowId: statusId,
+            error: null,
+            enrichmentDetails: skippedEnrichmentDetail(enrichment),
+          },
+          clearPatch
+        )
+        return 'completed'
+      }
+
+      try {
+        if (signal?.aborted) {
+          await writeState({
+            status: 'error',
+            executionId,
+            jobId: null,
+            workflowId: statusId,
+            error: 'Cancelled',
+            enrichmentDetails: skippedEnrichmentDetail(enrichment, { aborted: true }),
+          })
+          return 'error'
+        }
+        const { result, cost, error, detail } = await runEnrichment(enrichment, enrichInputs, {
+          tableId,
+          rowId,
+          workspaceId,
+          signal,
+        })
+
+        // An abort during the cascade must not be recorded as a completed cell.
+        if (signal?.aborted) {
+          await writeState({
+            status: 'error',
+            executionId,
+            jobId: null,
+            workflowId: statusId,
+            error: 'Cancelled',
+            enrichmentDetails: detail,
+          })
+          return 'error'
+        }
+
+        // Every provider that ran errored (auth / rate-limit / outage) — surface
+        // it rather than writing a blank cell that looks like "no data found".
+        if (error) {
+          await writeState({
+            status: 'error',
+            executionId,
+            jobId: null,
+            workflowId: statusId,
+            error,
+            enrichmentDetails: detail,
+          })
+          return 'error'
+        }
+
+        // Bill the run's actor (triggerer, else workspace billed account) for any
+        // hosted-key cost the providers incurred. Billing failures must not error
+        // an otherwise-successful cell.
+        if (cost > 0) {
+          try {
+            const { recordUsage } = await import('@/lib/billing/core/usage-log')
+            await recordUsage({
+              userId: enrichmentActorUserId,
+              workspaceId,
+              executionId,
+              entries: [
+                {
+                  category: 'fixed',
+                  source: 'enrichment',
+                  description: enrichment.name,
+                  cost,
+                  sourceReference: `enrichment:${tableId}:${rowId}:${enrichment.id}`,
+                  metadata: { enrichmentId: enrichment.id, tableId, rowId },
+                },
+              ],
+            })
+          } catch (billingErr) {
+            logger.error('Failed to record enrichment usage', {
+              enrichmentId: enrichment.id,
+              cost,
+              error: toError(billingErr).message,
+            })
+          }
+        }
+
+        // Write every output column: the result value when present, else clear
+        // it. A partial/empty result must blank the columns it didn't fill so a
+        // re-run that finds less than before doesn't leave stale values.
+        const dataPatch: RowData = {}
+        for (const out of group.outputs) {
+          if (!out.outputId) continue
+          const value = result[out.outputId]
+          dataPatch[out.columnName] =
+            value === undefined || value === null ? '' : (value as RowData[string])
+        }
+        await writeState(
+          {
+            status: 'completed',
+            executionId,
+            jobId: null,
+            workflowId: statusId,
+            error: null,
+            enrichmentDetails: detail,
+          },
+          dataPatch
+        )
+        return 'completed'
+      } catch (err) {
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId: statusId,
+          error: toError(err).message,
+        })
+        return 'error'
+      }
+    }
+
+    const blockErrors: Record<string, string> = {}
+    let writeChain: Promise<void> = Promise.resolve()
+    let terminalWritten = false
+
+    try {
+      const [workflowRecord] = await db
+        .select()
+        .from(workflowTable)
+        .where(eq(workflowTable.id, workflowId))
+        .limit(1)
+
+      if (!workflowRecord) {
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId,
+          error: 'Workflow not found',
+        })
+        return 'error'
+      }
+
+      // `deployed` groups run the workflow's latest active deployment; `live`
+      // (default) runs the editable draft. A `deployed` group whose workflow
+      // has never been deployed fails the cell — no silent fallback to draft.
+      let normalizedData: Awaited<ReturnType<typeof loadWorkflowFromNormalizedTables>>
+      if (deploymentMode === 'deployed') {
+        try {
+          normalizedData = await loadDeployedWorkflowState(workflowId, workspaceId)
+        } catch (err) {
+          // Surface the real reason (missing deployment vs. transient DB/migration
+          // failure) rather than always claiming the workflow isn't deployed.
+          await writeState({
+            status: 'error',
+            executionId,
+            jobId: null,
+            workflowId,
+            error: toError(err).message,
+          })
+          return 'error'
+        }
+      } else {
+        normalizedData = await loadWorkflowFromNormalizedTables(workflowId)
+      }
+      const startBlock = normalizedData
+        ? Object.values(normalizedData.blocks).find((b) => b?.type === 'start_trigger')
+        : undefined
+      if (!startBlock) {
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId,
+          error: 'Workflow is missing a Start trigger',
+        })
+        return 'error'
+      }
+
+      const row = await getRowById(tableId, rowId, workspaceId)
+      if (!row) {
+        logger.warn(`Row ${rowId} vanished before execution`)
+        return 'error'
+      }
+
+      if (cancelledBeforeRun(row.executions?.[groupId])) return 'error'
+
+      // Billing / usage / timeout gate — route table cells through the same
+      // preprocessing every other trigger uses. Keep running draft
+      // (checkDeployment: false). Rate limiting is paced separately below so a
+      // retry doesn't re-run the (stable) billing/usage/subscription lookups.
+      // Failures are surfaced via cell state / SSE / dispatch halt, so suppress
+      // preprocessing's own execution-log writes.
+      // Attribute the run to the member who triggered it (manual run, row edit, or
+      // auto-fire from their own write) so the gate + cost land on their per-member
+      // meter — mirroring the enrichment branch. Falls back to the workspace billed
+      // account for genuinely actor-less runs.
+      const preprocess = await preprocessExecution({
+        workflowId,
+        executionId,
+        requestId,
+        workspaceId,
+        workflowRecord,
+        userId: payload.triggeredByUserId ?? workflowRecord.userId,
+        useAuthenticatedUserAsActor: Boolean(payload.triggeredByUserId),
+        triggerType: 'workflow',
+        checkDeployment: false,
+        checkRateLimit: false,
+        skipConcurrencyReservation: true,
+        logPreprocessingErrors: false,
+      })
+      if (!preprocess.success) {
+        // Usage/quota exhausted: retrying won't help. Halt the dispatch without
+        // marking any cell, and signal the client to upgrade.
+        if (preprocess.error?.statusCode === 402) {
+          logger.warn(
+            `Usage limit reached — halting dispatch (table=${tableId} row=${rowId} group=${groupId})`
+          )
+          // Don't leave the cell stuck on its `pending` pre-stamp. Clear this
+          // cell's exec so it reverts to un-run (no error/cancelled badge —
+          // matching "don't mark"; re-runnable after upgrade). Each blocked
+          // cell clears its own.
+          const { updateRow } = await import('@/lib/table/rows/service')
+          await updateRow(
+            { tableId, rowId, data: {}, workspaceId, executionsPatch: { [groupId]: null } },
+            table,
+            requestId
+          ).catch((err) =>
+            logger.warn(`Failed to clear cell pre-stamp on usage limit`, {
+              error: toError(err).message,
+            })
+          )
+          // With up to 20 concurrent cells all hitting the limit at once, only
+          // the cell that transitions the dispatch active→complete emits the
+          // event — otherwise the user sees a toast per in-flight cell. Cells
+          // with no owning dispatch (auto-fire) always emit.
+          let shouldEmit = true
+          if (dispatchId) {
+            const { completeDispatchIfActive } = await import('@/lib/table/dispatcher')
+            shouldEmit = await completeDispatchIfActive(dispatchId)
+          }
+          if (shouldEmit) {
+            await appendTableEvent({
+              kind: 'usageLimitReached',
+              tableId,
+              ...(dispatchId ? { dispatchId } : {}),
+              message:
+                preprocess.error?.message ??
+                'Usage limit exceeded. Please upgrade your plan to continue.',
+            })
+          }
+          return 'blocked'
+        }
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId,
+          error: preprocess.error?.message ?? 'Workflow could not start',
+        })
+        return 'error'
+      }
+
+      const actorUserId = preprocess.actorUserId ?? workflowRecord.userId
+      const asyncTimeoutMs = preprocess.executionTimeout?.async
+
+      // Rate-limit pacing: tables count against the async counter (background
+      // jobs). On a hit, wait & retry so the row still runs rather than being
+      // skipped — only this cheap check repeats. The waiting cell holds its
+      // concurrency slot, pacing the whole dispatch to the user's rate limit.
+      const rateLimiter = new RateLimiter()
+      for (let attempt = 1; ; attempt++) {
+        if (signal?.aborted) return 'error'
+        const rl = await rateLimiter.checkRateLimitWithSubscription(
+          actorUserId,
+          preprocess.userSubscription ?? null,
+          'workflow',
+          true
+        )
+        if (rl.allowed) break
+        if (attempt >= RATE_LIMIT_MAX_ATTEMPTS) {
+          await writeState({
+            status: 'error',
+            executionId,
+            jobId: null,
+            workflowId,
+            error: 'Rate limit exceeded — please retry later',
+          })
+          return 'error'
+        }
+        // Exponential backoff WITH jitter — pass null, not the bucket's
+        // resetAt. That reset time is shared across all waiters, and
+        // backoffWithJitter clamps a non-null hint to a fixed value with no
+        // jitter, so honoring it would wake all ~20 concurrent cells in
+        // lockstep and stampede the bucket. Jittered backoff spreads retries.
+        const waitMs = backoffWithJitter(attempt, null)
+        logger.info(
+          `Rate limited — waiting ${Math.round(waitMs)}ms before retry ${attempt + 1} (table=${tableId} row=${rowId} group=${groupId})`
+        )
+        await sleep(waitMs)
+        // Stop All can land mid-wait. On the trigger.dev backend `signal` never
+        // fires (cancelByKey is a no-op there), so re-check the DB tombstone and
+        // release this concurrency slot promptly instead of sleeping out the
+        // full retry budget.
+        const refreshed = await getRowById(tableId, rowId, workspaceId)
+        if (!refreshed || cancelledBeforeRun(refreshed.executions?.[groupId])) return 'error'
+      }
+
+      // SQL guard also rejects if a stop click stamped `cancelled` between this
+      // check and pickup.
+      const pickedUp = await markWorkflowGroupPickedUp(cellCtx, {
+        workflowId,
+        jobId: null,
+      })
+      if (pickedUp === 'skipped') return 'error'
+
+      // Output columns produced by THIS group are skipped on input — they're
+      // populated by the run we're starting. Other group's outputs ARE
+      // included (they're plain primitives in `row.data` thanks to the
+      // flattened schema).
+      // `inputRow` is name-keyed: the workflow author references columns by name
+      // in the Start block and downstream blocks, while stored `row.data` is
+      // id-keyed. Translate, skipping this group's own output columns.
+      const ownOutputColumnIds = new Set(group.outputs.map((o) => o.columnName))
+      const inputRow: Record<string, unknown> = {}
+      for (const col of table.schema.columns) {
+        const id = getColumnId(col)
+        if (ownOutputColumnIds.has(id)) continue
+        inputRow[col.name] = row.data[id]
+      }
+
+      const headers = table.schema.columns
+        .filter((c) => !ownOutputColumnIds.has(getColumnId(c)))
+        .map((c) => c.name)
+
+      // When the group has explicit input mappings, feed the workflow's
+      // Start-block fields from the mapped columns (`inputName ← row[columnId]`).
+      // Otherwise fall back to spreading every non-output column by name, so a
+      // Start field still resolves when it matches a column name. `row`/`rawRow`
+      // always carry the full (name-keyed) row for downstream reference.
+      const inputMappings = group.inputMappings ?? []
+      const mappedInputs: Record<string, unknown> = {}
+      for (const m of inputMappings) {
+        mappedInputs[m.inputName] = row.data[m.columnName]
+      }
+
+      const input = {
+        ...(inputMappings.length > 0 ? mappedInputs : inputRow),
+        row: inputRow,
+        rawRow: inputRow,
+        previousRow: null,
+        changedColumns: [],
+        rowId,
+        headers,
+        tableId,
+        tableName,
+        timestamp: new Date().toISOString(),
+      }
+
+      const { pluckByPath } = await import('@/lib/table/pluck')
+      const outputsByBlockId = buildOutputsByBlockId(group)
+
+      const accumulatedData: RowData = {}
+      const runningBlockIds = new Set<string>()
+
+      const schedulePartialWrite = () => {
+        if (terminalWritten) return
+        const dataSnapshot: RowData = { ...accumulatedData }
+        const blockErrorsSnapshot = { ...blockErrors }
+        const runningSnapshot = Array.from(runningBlockIds)
+        writeChain = writeChain
+          .then(async () => {
+            if (signal?.aborted) return
+            if (terminalWritten) return
+            await writeState(
+              {
+                status: 'running',
+                executionId,
+                jobId: null,
+                workflowId,
+                error: null,
+                runningBlockIds: runningSnapshot,
+                blockErrors: blockErrorsSnapshot,
+              },
+              dataSnapshot
+            )
+          })
+          .catch((err) => {
+            logger.warn(
+              `Per-block partial write failed (table=${tableId} row=${rowId} group=${groupId})`,
+              { cause: describeError(err), retryable: isRetryableInfrastructureError(err) }
+            )
+          })
+      }
+
+      const onBlockStart = async (blockId: string): Promise<void> => {
+        if (!outputsByBlockId.has(blockId)) return
+        runningBlockIds.add(blockId)
+        schedulePartialWrite()
+      }
+
+      const onBlockComplete = async (blockId: string, output: unknown): Promise<void> => {
+        const outputs = outputsByBlockId.get(blockId)
+        if (!outputs) return
+
+        const blockResult =
+          output && typeof output === 'object' && 'output' in (output as object)
+            ? (output as { output: unknown }).output
+            : output
+
+        const blockErrorMessage =
+          blockResult &&
+          typeof blockResult === 'object' &&
+          typeof (blockResult as { error?: unknown }).error === 'string'
+            ? (blockResult as { error: string }).error
+            : null
+
+        if (blockErrorMessage) {
+          blockErrors[blockId] = blockErrorMessage
+        } else {
+          for (const out of outputs) {
+            const plucked = pluckByPath(blockResult, out.path)
+            if (plucked === undefined) continue
+            accumulatedData[out.columnName] = plucked as RowData[string]
+          }
+        }
+        runningBlockIds.delete(blockId)
+        schedulePartialWrite()
+      }
+
+      // Enforce the per-plan execution timeout (from preprocessing), combined
+      // with the existing cancel signal so either a timeout or a Stop aborts.
+      const timeoutController = createTimeoutAbortController(asyncTimeoutMs)
+      const abortSignal = signal
+        ? AbortSignal.any([signal, timeoutController.signal])
+        : timeoutController.signal
+
+      let result: Awaited<ReturnType<typeof executeWorkflow>>
+      try {
+        result = await executeWorkflow(
+          {
+            id: workflowRecord.id,
+            // Workflow owner — drives personal env-var resolution + ownership.
+            userId: workflowRecord.userId,
+            workspaceId: workflowRecord.workspaceId,
+            variables: (workflowRecord.variables as Record<string, unknown> | null) ?? {},
+          },
+          requestId,
+          input,
+          // Billing/usage/rate actor — the workspace billed account.
+          actorUserId,
+          {
+            enabled: true,
+            executionMode: 'sync',
+            workflowTriggerType: 'table',
+            triggerBlockId: startBlock.id,
+            // `deployed` groups execute the latest active deployment; everything
+            // else runs the editable draft (the table default). Matches the
+            // state loaded above for start-block / output-block resolution.
+            useDraftState: deploymentMode !== 'deployed',
+            abortSignal,
+            onBlockStart,
+            onBlockComplete,
+          },
+          executionId
+        )
+      } finally {
+        timeoutController.cleanup()
+      }
+
+      terminalWritten = true
+      await writeChain.catch(() => {})
+
+      if (result.status === 'paused') {
+        await writeState(
+          {
+            status: 'pending',
+            executionId,
+            jobId: `paused-${executionId}`,
+            workflowId,
+            error: null,
+            runningBlockIds: [],
+            blockErrors,
+          },
+          accumulatedData
+        )
+        await stashCellContextForResume({
+          executionId,
+          tableId,
+          tableName,
+          rowId,
+          groupId,
+          workflowId,
+          workspaceId,
+        })
+        return 'paused'
+      }
+
+      await writeState(
+        {
+          status: result.success ? 'completed' : 'error',
+          executionId,
+          jobId: null,
+          workflowId,
+          error: result.success ? null : (result.error ?? 'Workflow execution failed'),
+          runningBlockIds: [],
+          blockErrors,
+        },
+        accumulatedData
+      )
+      return result.success ? 'completed' : 'error'
+    } catch (err) {
+      const message = toError(err).message
+      logger.error(
+        `Workflow group cell execution failed (table=${tableId} row=${rowId} group=${groupId})`,
+        {
+          error: message,
+          executionId,
+          cause: describeError(err),
+          retryable: isRetryableInfrastructureError(err),
+        }
+      )
+      terminalWritten = true
+      await writeChain.catch(() => {})
+      try {
+        await writeState({
+          status: 'error',
+          executionId,
+          jobId: null,
+          workflowId,
+          error: message,
+          runningBlockIds: [],
+          blockErrors,
+        })
+      } catch (writeErr) {
+        logger.error('Also failed to write error state', {
+          error: toError(writeErr).message,
+          cause: describeError(writeErr),
+          retryable: isRetryableInfrastructureError(writeErr),
+        })
+      }
+      return 'error'
+    }
+  })
+}
+
+export const workflowGroupCellTask = task({
+  id: 'workflow-group-cell',
+  machine: 'medium-1x',
+  retry: { maxAttempts: 1 },
+  // Combined with `concurrencyKey: tableId`, caps each table's sub-queue to
+  // 20 in-flight cell jobs while letting different tables run in parallel.
+  queue: {
+    name: 'workflow-group-cell',
+    concurrencyLimit: 20,
+  },
+  run: (payload: WorkflowGroupCellPayload, { signal }) =>
+    executeWorkflowGroupCellJob(payload, signal),
+})

apps/sim/background/workflow-execution.ts

@@ -0,0 +1,214 @@
+import { createLogger, runWithRequestContext } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { task } from '@trigger.dev/sdk'
+import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
+import { createTimeoutAbortController, getTimeoutErrorMessage } from '@/lib/core/execution-limits'
+import { preprocessExecution } from '@/lib/execution/preprocessing'
+import { LoggingSession } from '@/lib/logs/execution/logging-session'
+import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
+import { cleanupExecutionBase64Cache } from '@/lib/uploads/utils/user-file-base64.server'
+import {
+  executeWorkflowCore,
+  wasExecutionFinalizedByCore,
+} from '@/lib/workflows/executor/execution-core'
+import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
+import { WORKFLOW_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits'
+import { ExecutionSnapshot } from '@/executor/execution/snapshot'
+import type { ExecutionMetadata } from '@/executor/execution/types'
+import { hasExecutionResult } from '@/executor/utils/errors'
+import type { CoreTriggerType } from '@/stores/logs/filters/types'
+
+const logger = createLogger('TriggerWorkflowExecution')
+
+export function buildWorkflowCorrelation(
+  payload: WorkflowExecutionPayload
+): AsyncExecutionCorrelation {
+  const executionId = payload.executionId || generateId()
+  const requestId = payload.requestId || payload.correlation?.requestId || executionId.slice(0, 8)
+
+  return {
+    executionId,
+    requestId,
+    source: 'workflow',
+    workflowId: payload.workflowId,
+    triggerType: payload.triggerType || payload.correlation?.triggerType || 'api',
+  }
+}
+
+export type WorkflowExecutionPayload = {
+  workflowId: string
+  userId: string
+  workspaceId?: string
+  input?: any
+  triggerType?: CoreTriggerType
+  executionId?: string
+  requestId?: string
+  correlation?: AsyncExecutionCorrelation
+  metadata?: Record<string, any>
+  callChain?: string[]
+  executionMode?: 'sync' | 'stream' | 'async'
+}
+
+/**
+ * Background workflow execution job
+ * @see preprocessExecution For detailed information on preprocessing checks
+ * @see executeWorkflowCore For the core workflow execution logic
+ */
+export async function executeWorkflowJob(payload: WorkflowExecutionPayload) {
+  const workflowId = payload.workflowId
+  const correlation = buildWorkflowCorrelation(payload)
+  const executionId = correlation.executionId
+  const requestId = correlation.requestId
+
+  return runWithRequestContext({ requestId }, async () => {
+    logger.info(`[${requestId}] Starting workflow execution job: ${workflowId}`, {
+      userId: payload.userId,
+      triggerType: payload.triggerType,
+      executionId,
+    })
+
+    const triggerType = (correlation.triggerType || 'api') as CoreTriggerType
+    const loggingSession = new LoggingSession(workflowId, executionId, triggerType, requestId)
+
+    try {
+      const preprocessResult = await preprocessExecution({
+        workflowId: payload.workflowId,
+        userId: payload.userId,
+        triggerType: triggerType,
+        executionId: executionId,
+        requestId: requestId,
+        checkRateLimit: true,
+        checkDeployment: true,
+        loggingSession: loggingSession,
+        triggerData: { correlation },
+      })
+
+      if (!preprocessResult.success) {
+        logger.error(`[${requestId}] Preprocessing failed: ${preprocessResult.error?.message}`, {
+          workflowId,
+          statusCode: preprocessResult.error?.statusCode,
+        })
+
+        throw new Error(preprocessResult.error?.message || 'Preprocessing failed')
+      }
+
+      const actorUserId = preprocessResult.actorUserId!
+      const workspaceId = preprocessResult.workflowRecord?.workspaceId
+      if (!workspaceId) {
+        throw new Error(`Workflow ${workflowId} has no associated workspace`)
+      }
+
+      logger.info(`[${requestId}] Preprocessing passed. Using actor: ${actorUserId}`)
+
+      const workflow = preprocessResult.workflowRecord!
+
+      const metadata: ExecutionMetadata = {
+        requestId,
+        executionId,
+        workflowId,
+        workspaceId,
+        userId: actorUserId,
+        sessionUserId: undefined,
+        workflowUserId: workflow.userId,
+        triggerType: payload.triggerType || 'api',
+        useDraftState: false,
+        startTime: new Date().toISOString(),
+        isClientSession: false,
+        callChain: payload.callChain,
+        correlation,
+        executionMode: payload.executionMode ?? 'async',
+      }
+
+      const snapshot = new ExecutionSnapshot(
+        metadata,
+        workflow,
+        payload.input,
+        workflow.variables || {},
+        []
+      )
+
+      const timeoutController = createTimeoutAbortController(
+        preprocessResult.executionTimeout?.async
+      )
+
+      let result
+      try {
+        result = await executeWorkflowCore({
+          snapshot,
+          callbacks: {},
+          loggingSession,
+          includeFileBase64: true,
+          base64MaxBytes: undefined,
+          abortSignal: timeoutController.signal,
+        })
+      } finally {
+        timeoutController.cleanup()
+      }
+
+      if (
+        result.status === 'cancelled' &&
+        timeoutController.isTimedOut() &&
+        timeoutController.timeoutMs
+      ) {
+        const timeoutErrorMessage = getTimeoutErrorMessage(null, timeoutController.timeoutMs)
+        logger.info(`[${requestId}] Workflow execution timed out`, {
+          timeoutMs: timeoutController.timeoutMs,
+        })
+        await loggingSession.markAsFailed(timeoutErrorMessage)
+      } else {
+        await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession })
+      }
+
+      await loggingSession.waitForPostExecution()
+
+      logger.info(`[${requestId}] Workflow execution completed: ${workflowId}`, {
+        success: result.success,
+        executionTime: result.metadata?.duration,
+        executionId,
+      })
+
+      return {
+        success: result.success,
+        workflowId: payload.workflowId,
+        executionId,
+        output: result.output,
+        executedAt: new Date().toISOString(),
+        metadata: payload.metadata,
+      }
+    } catch (error: unknown) {
+      logger.error(`[${requestId}] Workflow execution failed: ${workflowId}`, {
+        error: toError(error).message,
+        executionId,
+      })
+
+      if (wasExecutionFinalizedByCore(error, executionId)) {
+        throw error
+      }
+
+      const executionResult = hasExecutionResult(error) ? error.executionResult : undefined
+      const { traceSpans } = executionResult ? buildTraceSpans(executionResult) : { traceSpans: [] }
+
+      await loggingSession.safeCompleteWithError({
+        error: {
+          message: toError(error).message,
+          stackTrace: error instanceof Error ? error.stack : undefined,
+        },
+        traceSpans,
+      })
+
+      throw error
+    } finally {
+      void cleanupExecutionBase64Cache(executionId)
+    }
+  })
+}
+
+export const workflowExecutionTask = task({
+  id: 'workflow-execution',
+  machine: 'medium-1x',
+  queue: {
+    concurrencyLimit: WORKFLOW_EXECUTION_CONCURRENCY_LIMIT,
+  },
+  run: executeWorkflowJob,
+})