chore: import upstream snapshot with attribution

apps/sim/lib/workflows/executor/execute-workflow.ts

@@ -0,0 +1,193 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { LoggingSession } from '@/lib/logs/execution/logging-session'
+import { captureServerEvent } from '@/lib/posthog/server'
+import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core'
+import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence'
+import { ExecutionSnapshot } from '@/executor/execution/snapshot'
+import type { ExecutionMetadata, SerializableExecutionState } from '@/executor/execution/types'
+import type { ExecutionResult, StreamingExecution } from '@/executor/types'
+
+const logger = createLogger('WorkflowExecution')
+
+export interface ExecuteWorkflowOptions {
+  enabled: boolean
+  selectedOutputs?: string[]
+  isSecureMode?: boolean
+  workflowTriggerType?: 'api' | 'chat' | 'copilot' | 'table'
+  /**
+   * If set, the executor enters the workflow at this block instead of resolving a Start block.
+   * Use for trigger-originated runs (webhooks, table triggers, schedules) where the entry point
+   * is the trigger block itself.
+   */
+  triggerBlockId?: string
+  onStream?: (streamingExec: StreamingExecution) => Promise<void>
+  /** Fires before each block runs; lets callers track per-block lifecycle (e.g. table-cell live state). */
+  onBlockStart?: (
+    blockId: string,
+    blockName: string,
+    blockType: string,
+    executionOrder: number
+  ) => Promise<void>
+  onBlockComplete?: (blockId: string, output: unknown) => Promise<void>
+  skipLoggingComplete?: boolean
+  includeFileBase64?: boolean
+  base64MaxBytes?: number
+  largeValueKeys?: string[]
+  fileKeys?: string[]
+  abortSignal?: AbortSignal
+  /** Use the live/draft workflow state instead of the deployed state. Used by copilot. */
+  useDraftState?: boolean
+  /** Stop execution after this block completes. Used for "run until block" feature. */
+  stopAfterBlockId?: string
+  /** Run-from-block configuration using a prior execution snapshot. */
+  runFromBlock?: {
+    startBlockId: string
+    sourceSnapshot: SerializableExecutionState
+    sourceExecutionId?: string
+  }
+  executionMode?: 'sync' | 'stream' | 'async'
+}
+
+export interface WorkflowInfo {
+  id: string
+  userId: string
+  workspaceId?: string | null
+  isDeployed?: boolean
+  variables?: Record<string, any>
+}
+
+export async function executeWorkflow(
+  workflow: WorkflowInfo,
+  requestId: string,
+  input: unknown | undefined,
+  actorUserId: string,
+  streamConfig?: ExecuteWorkflowOptions,
+  providedExecutionId?: string
+): Promise<ExecutionResult> {
+  if (!workflow.workspaceId) {
+    throw new Error(`Workflow ${workflow.id} has no workspaceId`)
+  }
+
+  const workflowId = workflow.id
+  const workspaceId = workflow.workspaceId
+  const executionId = providedExecutionId || generateId()
+  const triggerType = streamConfig?.workflowTriggerType || 'api'
+  const loggingSession = new LoggingSession(workflowId, executionId, triggerType, requestId)
+
+  try {
+    const metadata: ExecutionMetadata = {
+      requestId,
+      executionId,
+      workflowId,
+      workspaceId,
+      userId: actorUserId,
+      workflowUserId: workflow.userId,
+      triggerType,
+      triggerBlockId: streamConfig?.triggerBlockId,
+      useDraftState: streamConfig?.useDraftState ?? false,
+      startTime: new Date().toISOString(),
+      isClientSession: false,
+      largeValueExecutionIds: Array.from(new Set([executionId])),
+      largeValueKeys: streamConfig?.largeValueKeys,
+      fileKeys: streamConfig?.fileKeys,
+      executionMode: streamConfig?.executionMode,
+    }
+
+    const snapshot = new ExecutionSnapshot(
+      metadata,
+      workflow,
+      input,
+      workflow.variables || {},
+      streamConfig?.selectedOutputs || []
+    )
+
+    const executionStartMs = Date.now()
+
+    const result = await executeWorkflowCore({
+      snapshot,
+      callbacks: {
+        onStream: streamConfig?.onStream,
+        onBlockStart: streamConfig?.onBlockStart
+          ? async (
+              blockId: string,
+              blockName: string,
+              blockType: string,
+              executionOrder: number
+            ) => {
+              await streamConfig.onBlockStart!(blockId, blockName, blockType, executionOrder)
+            }
+          : undefined,
+        onBlockComplete: streamConfig?.onBlockComplete
+          ? async (blockId: string, _blockName: string, _blockType: string, output: unknown) => {
+              await streamConfig.onBlockComplete!(blockId, output)
+            }
+          : undefined,
+      },
+      loggingSession,
+      includeFileBase64: streamConfig?.includeFileBase64,
+      base64MaxBytes: streamConfig?.base64MaxBytes,
+      abortSignal: streamConfig?.abortSignal,
+      stopAfterBlockId: streamConfig?.stopAfterBlockId,
+      runFromBlock: streamConfig?.runFromBlock,
+    })
+
+    const blockTypes = [
+      ...new Set(
+        (result.logs ?? [])
+          .map((log) => log.blockType)
+          .filter((t): t is string => typeof t === 'string')
+      ),
+    ]
+    if (result.status !== 'paused') {
+      captureServerEvent(
+        actorUserId,
+        'workflow_executed',
+        {
+          workflow_id: workflowId,
+          workspace_id: workspaceId,
+          trigger_type: triggerType,
+          success: result.success,
+          block_count: result.logs?.length ?? 0,
+          block_types: blockTypes.join(','),
+          duration_ms: Date.now() - executionStartMs,
+        },
+        {
+          groups: { workspace: workspaceId },
+          setOnce: { first_execution_at: new Date().toISOString() },
+        }
+      )
+    }
+
+    await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession })
+
+    if (streamConfig?.skipLoggingComplete) {
+      return {
+        ...result,
+        _streamingMetadata: {
+          loggingSession,
+          processedInput: input,
+        },
+      }
+    }
+
+    return result
+  } catch (error: unknown) {
+    logger.error(`[${requestId}] Workflow execution failed:`, error)
+
+    captureServerEvent(
+      actorUserId,
+      'workflow_execution_failed',
+      {
+        workflow_id: workflow.id,
+        workspace_id: workspaceId,
+        trigger_type: streamConfig?.workflowTriggerType || 'api',
+        error_message: toError(error).message,
+      },
+      { groups: { workspace: workspaceId } }
+    )
+
+    throw error
+  }
+}

apps/sim/lib/workflows/executor/execution-core.ts

@@ -0,0 +1,897 @@
+/**
+ * Core workflow execution logic - shared by all execution paths
+ * This is the SINGLE source of truth for workflow execution
+ */
+
+import { db } from '@sim/db'
+import { organization, workspace } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { filterUndefined, isPlainRecord, isRecordLike } from '@sim/utils/object'
+import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks'
+import { eq } from 'drizzle-orm'
+import type { Edge } from 'reactflow'
+import { z } from 'zod'
+import { type EffectivePiiRedaction, resolveEffectivePiiRedaction } from '@/lib/billing/retention'
+import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils'
+import { clearExecutionCancellation } from '@/lib/execution/cancellation'
+import { warmLargeValueRefs } from '@/lib/execution/payloads/hydration'
+import { parseLargeExecutionValue } from '@/lib/execution/payloads/large-execution-value'
+import type { LoggingSession } from '@/lib/logs/execution/logging-session'
+import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values'
+import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction'
+import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
+import { getCustomBlockRowsForWorkspace } from '@/lib/workflows/custom-blocks/operations'
+import {
+  loadDeployedWorkflowState,
+  loadWorkflowFromNormalizedTables,
+} from '@/lib/workflows/persistence/utils'
+import { TriggerUtils } from '@/lib/workflows/triggers/triggers'
+import { updateWorkflowRunCounts } from '@/lib/workflows/utils'
+import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay'
+import { Executor } from '@/executor'
+import type { ExecutionSnapshot } from '@/executor/execution/snapshot'
+import type {
+  ChildWorkflowContext,
+  ContextExtensions,
+  ExecutionCallbacks,
+  IterationContext,
+  SerializableExecutionState,
+} from '@/executor/execution/types'
+import type { ExecutionResult, NormalizedBlockOutput } from '@/executor/types'
+import { hasExecutionResult } from '@/executor/utils/errors'
+import { buildParallelSentinelEndId, buildSentinelEndId } from '@/executor/utils/subflow-utils'
+import { Serializer } from '@/serializer'
+
+const logger = createLogger('ExecutionCore')
+
+const EnvVarsSchema = z.record(z.string(), z.string())
+
+/**
+ * Surfaces the underlying driver error from a wrapped error chain.
+ *
+ * Drizzle wraps the original `postgres`/Node driver error as `error.cause`,
+ * which the logger's Error serializer drops (it only emits own-enumerable
+ * keys). Walking the chain from `error` itself and preferring the first error
+ * carrying a `code` exposes the diagnostic fields — notably the Postgres
+ * `code` — that distinguish a connection drop (`08006`), a rejected connection
+ * (`53300`), and a statement timeout (`57014`) behind an opaque "Failed query"
+ * message. Starting at `error` also captures a bare driver error that reaches
+ * this path unwrapped; when no error in the chain carries a `code`, it falls
+ * back to the first wrapped cause (the top-level error is already logged on its
+ * own, so it is not echoed here).
+ */
+function describeErrorCause(error: unknown): Record<string, unknown> | undefined {
+  try {
+    let driver: (Error & Record<string, unknown>) | undefined
+    let current: unknown = error
+    for (let depth = 0; depth < 10 && current instanceof Error; depth++) {
+      const candidate = current as Error & Record<string, unknown>
+      if (candidate.code !== undefined) {
+        driver = candidate
+        break
+      }
+      if (depth === 1) driver = candidate
+      current = candidate.cause
+    }
+    if (!driver) return undefined
+    return filterUndefined({
+      name: driver.name,
+      message: driver.message,
+      code: driver.code,
+      severity: driver.severity,
+      detail: driver.detail,
+      routine: driver.routine,
+      errno: driver.errno,
+      syscall: driver.syscall,
+    })
+  } catch {
+    return undefined
+  }
+}
+
+export interface ExecuteWorkflowCoreOptions {
+  snapshot: ExecutionSnapshot
+  callbacks: ExecutionCallbacks
+  loggingSession: LoggingSession
+  skipLogCreation?: boolean
+  abortSignal?: AbortSignal
+  includeFileBase64?: boolean
+  base64MaxBytes?: number
+  stopAfterBlockId?: string
+  /** Run-from-block mode: execute starting from a specific block using cached upstream outputs */
+  runFromBlock?: {
+    startBlockId: string
+    sourceSnapshot: SerializableExecutionState
+    sourceExecutionId?: string
+  }
+}
+
+function parseVariableValueByType(value: unknown, type: string): unknown {
+  const refValue = parseLargeExecutionValue(value)
+  if (refValue !== undefined) {
+    return refValue
+  }
+
+  if (value === null || value === undefined) {
+    switch (type) {
+      case 'number':
+        return 0
+      case 'boolean':
+        return false
+      case 'array':
+        return []
+      case 'object':
+        return {}
+      default:
+        return ''
+    }
+  }
+
+  if (type === 'number') {
+    if (typeof value === 'number') return value
+    if (typeof value === 'string') {
+      const num = Number(value)
+      return Number.isNaN(num) ? 0 : num
+    }
+    return 0
+  }
+
+  if (type === 'boolean') {
+    if (typeof value === 'boolean') return value
+    if (typeof value === 'string') {
+      return value.toLowerCase() === 'true'
+    }
+    return Boolean(value)
+  }
+
+  if (type === 'array') {
+    if (Array.isArray(value)) return value
+    if (typeof value === 'string' && value.trim()) {
+      try {
+        return JSON.parse(value)
+      } catch {
+        return []
+      }
+    }
+    return []
+  }
+
+  if (type === 'object') {
+    if (isRecordLike(value)) return value
+    if (typeof value === 'string' && value.trim()) {
+      try {
+        return JSON.parse(value)
+      } catch {
+        return {}
+      }
+    }
+    return {}
+  }
+
+  // string or plain
+  return typeof value === 'string' ? value : String(value)
+}
+
+type ExecutionErrorWithFinalizationFlag = Error & {
+  executionFinalizedByCore?: boolean
+}
+
+export const FINALIZED_EXECUTION_ID_TTL_MS = 5 * 60 * 1000
+
+const finalizedExecutionIds = new Map<string, number>()
+
+function cleanupExpiredFinalizedExecutionIds(now = Date.now()): void {
+  for (const [executionId, expiresAt] of finalizedExecutionIds.entries()) {
+    if (expiresAt <= now) {
+      finalizedExecutionIds.delete(executionId)
+    }
+  }
+}
+
+function rememberFinalizedExecutionId(executionId: string): void {
+  const now = Date.now()
+
+  cleanupExpiredFinalizedExecutionIds(now)
+  finalizedExecutionIds.set(executionId, now + FINALIZED_EXECUTION_ID_TTL_MS)
+}
+
+async function clearExecutionCancellationSafely(
+  executionId: string,
+  requestId: string
+): Promise<void> {
+  try {
+    await clearExecutionCancellation(executionId)
+  } catch (error) {
+    logger.error(`[${requestId}] Failed to clear execution cancellation`, { error, executionId })
+  }
+}
+
+function markExecutionFinalizedByCore(error: unknown, executionId: string): void {
+  rememberFinalizedExecutionId(executionId)
+
+  if (error instanceof Error) {
+    ;(error as ExecutionErrorWithFinalizationFlag).executionFinalizedByCore = true
+  }
+}
+
+export function wasExecutionFinalizedByCore(error: unknown, executionId?: string): boolean {
+  cleanupExpiredFinalizedExecutionIds()
+
+  if (executionId && finalizedExecutionIds.has(executionId)) {
+    return true
+  }
+
+  return (
+    error instanceof Error &&
+    (error as ExecutionErrorWithFinalizationFlag).executionFinalizedByCore === true
+  )
+}
+
+async function finalizeExecutionOutcome(params: {
+  result: ExecutionResult
+  loggingSession: LoggingSession
+  executionId: string
+  requestId: string
+  workflowInput: unknown
+}): Promise<void> {
+  const { result, loggingSession, executionId, requestId, workflowInput } = params
+  const { traceSpans, totalDuration } = buildTraceSpans(result)
+  const endedAt = new Date().toISOString()
+
+  try {
+    try {
+      if (result.status === 'cancelled') {
+        await loggingSession.safeCompleteWithCancellation({
+          endedAt,
+          totalDurationMs: totalDuration || 0,
+          traceSpans: traceSpans || [],
+        })
+        return
+      }
+
+      if (result.status === 'paused') {
+        await loggingSession.safeCompleteWithPause({
+          endedAt,
+          totalDurationMs: totalDuration || 0,
+          traceSpans: traceSpans || [],
+          workflowInput,
+        })
+        return
+      }
+
+      await loggingSession.safeComplete({
+        endedAt,
+        totalDurationMs: totalDuration || 0,
+        finalOutput: result.output || {},
+        traceSpans: traceSpans || [],
+        workflowInput,
+        executionState: result.executionState,
+      })
+    } catch (error) {
+      logger.warn(`[${requestId}] Post-execution finalization failed`, {
+        executionId,
+        status: result.status,
+        error,
+      })
+    }
+  } finally {
+    await clearExecutionCancellationSafely(executionId, requestId)
+  }
+}
+
+async function finalizeExecutionError(params: {
+  error: unknown
+  loggingSession: LoggingSession
+  executionId: string
+  requestId: string
+}): Promise<boolean> {
+  const { error, loggingSession, executionId, requestId } = params
+  const executionResult = hasExecutionResult(error) ? error.executionResult : undefined
+  const { traceSpans } = executionResult ? buildTraceSpans(executionResult) : { traceSpans: [] }
+
+  try {
+    await loggingSession.safeCompleteWithError({
+      endedAt: new Date().toISOString(),
+      totalDurationMs: executionResult?.metadata?.duration || 0,
+      error: {
+        message: getErrorMessage(error, 'Execution failed'),
+        stackTrace: error instanceof Error ? error.stack : undefined,
+      },
+      traceSpans,
+    })
+
+    return loggingSession.hasCompleted()
+  } catch (postExecError) {
+    logger.error(`[${requestId}] Post-execution error logging failed`, {
+      error: postExecError,
+    })
+    return false
+  } finally {
+    await clearExecutionCancellationSafely(executionId, requestId)
+  }
+}
+
+/**
+ * Establish the custom-block registry overlay for the execution's organization,
+ * then run the core. Wrapping here — the shared choke point for the sync route and
+ * the background job — puts `custom_block_*` types in scope for serialization,
+ * execution, and any nested child-workflow serialization (ALS propagates to the
+ * whole async subtree).
+ */
+export async function executeWorkflowCore(
+  options: ExecuteWorkflowCoreOptions
+): Promise<ExecutionResult> {
+  const workspaceId = options.snapshot.metadata.workspaceId
+  const rows = workspaceId ? await getCustomBlockRowsForWorkspace(workspaceId) : []
+  return withCustomBlockOverlay(rows, () => executeWorkflowCoreImpl(options))
+}
+
+async function executeWorkflowCoreImpl(
+  options: ExecuteWorkflowCoreOptions
+): Promise<ExecutionResult> {
+  const {
+    snapshot,
+    callbacks,
+    loggingSession,
+    skipLogCreation,
+    abortSignal,
+    includeFileBase64,
+    base64MaxBytes,
+    stopAfterBlockId,
+    runFromBlock,
+  } = options
+  const { metadata, workflow, input, workflowVariables, selectedOutputs } = snapshot
+  const { requestId, workflowId, userId, triggerType, executionId, triggerBlockId, useDraftState } =
+    metadata
+  const { onBlockStart, onBlockComplete, onStream, onChildWorkflowInstanceReady } = callbacks
+
+  const providedWorkspaceId = metadata.workspaceId
+  if (!providedWorkspaceId) {
+    throw new Error(`Execution metadata missing workspaceId for workflow ${workflowId}`)
+  }
+
+  let processedInput = input || {}
+  let deploymentVersionId: string | undefined
+  let loggingStarted = false
+  const pendingLifecycleCallbacks = new Set<Promise<void>>()
+
+  const trackLifecycleCallback = (promise: Promise<void>) => {
+    pendingLifecycleCallbacks.add(promise)
+    void promise
+      .finally(() => {
+        pendingLifecycleCallbacks.delete(promise)
+      })
+      .catch(() => {})
+  }
+
+  const waitForLifecycleCallbacks = async () => {
+    while (pendingLifecycleCallbacks.size > 0) {
+      await Promise.allSettled([...pendingLifecycleCallbacks])
+    }
+  }
+
+  try {
+    const personalEnvUserId =
+      metadata.isClientSession && metadata.sessionUserId
+        ? metadata.sessionUserId
+        : metadata.workflowUserId
+
+    if (!personalEnvUserId) {
+      throw new Error('Missing workflowUserId in execution metadata')
+    }
+
+    /**
+     * Resolves the workflow state from the override, the draft tables, or the
+     * deployed snapshot. The async load (draft/deployed) has no data dependency
+     * on the environment load, so the two are awaited concurrently below.
+     */
+    const loadWorkflowState = async () => {
+      if (metadata.workflowStateOverride) {
+        const override = metadata.workflowStateOverride
+        logger.info(`[${requestId}] Using workflow state override (diff workflow execution)`, {
+          blocksCount: Object.keys(override.blocks).length,
+          edgesCount: override.edges.length,
+        })
+        return {
+          blocks: override.blocks,
+          edges: override.edges,
+          loops: override.loops || {},
+          parallels: override.parallels || {},
+          deploymentVersionId: override.deploymentVersionId,
+        }
+      }
+
+      if (useDraftState) {
+        const draftData = await loadWorkflowFromNormalizedTables(workflowId)
+
+        if (!draftData) {
+          throw new Error('Workflow not found or not yet saved')
+        }
+
+        logger.info(
+          `[${requestId}] Using draft workflow state from normalized tables (client execution)`
+        )
+        return {
+          blocks: draftData.blocks,
+          edges: draftData.edges,
+          loops: draftData.loops,
+          parallels: draftData.parallels,
+          deploymentVersionId: undefined,
+        }
+      }
+
+      const deployedData = await loadDeployedWorkflowState(workflowId)
+      logger.info(`[${requestId}] Using deployed workflow state (deployed execution)`)
+      return {
+        blocks: deployedData.blocks,
+        edges: deployedData.edges,
+        loops: deployedData.loops,
+        parallels: deployedData.parallels,
+        deploymentVersionId: deployedData.deploymentVersionId,
+      }
+    }
+
+    const [workflowState, env] = await Promise.all([
+      loadWorkflowState(),
+      getPersonalAndWorkspaceEnv(personalEnvUserId, providedWorkspaceId),
+    ])
+
+    const { blocks, loops, parallels } = workflowState
+    const edges: Edge[] = workflowState.edges
+    deploymentVersionId = workflowState.deploymentVersionId
+
+    const mergedStates = mergeSubblockStateWithValues(blocks)
+
+    const { personalEncrypted, workspaceEncrypted, personalDecrypted, workspaceDecrypted } = env
+
+    // Use encrypted values for logging (don't log decrypted secrets)
+    const variables = EnvVarsSchema.parse({ ...personalEncrypted, ...workspaceEncrypted })
+
+    // Use already-decrypted values for execution (no redundant decryption)
+    const decryptedEnvVars: Record<string, string> = { ...personalDecrypted, ...workspaceDecrypted }
+
+    loggingStarted = await loggingSession.safeStart({
+      userId,
+      workspaceId: providedWorkspaceId,
+      variables,
+      triggerData: metadata.correlation ? { correlation: metadata.correlation } : undefined,
+      skipLogCreation,
+      deploymentVersionId,
+      workflowState: { blocks, edges, loops, parallels },
+    })
+
+    // Use edges directly - trigger-to-trigger edges are prevented at creation time
+    const filteredEdges = edges
+
+    // Check if this is a resume execution before trigger resolution
+    const resumeFromSnapshot = metadata.resumeFromSnapshot === true
+    const resumePendingQueue = snapshot.state?.pendingQueue
+    const resumeRemainingEdges = snapshot.state?.remainingEdges
+    const resumeTerminalNoop = metadata.resumeTerminalNoop === true
+
+    let resolvedTriggerBlockId = triggerBlockId
+
+    // Resume executions derive their queue from the snapshot. Even an empty
+    // queue is meaningful: a terminal pause block has no downstream work.
+    if (
+      resumeFromSnapshot &&
+      (resumePendingQueue !== undefined || resumeRemainingEdges !== undefined || resumeTerminalNoop)
+    ) {
+      resolvedTriggerBlockId = undefined
+      logger.info(`[${requestId}] Skipping trigger resolution for resume execution`, {
+        pendingQueueLength: resumePendingQueue?.length ?? 0,
+        remainingEdgeCount: resumeRemainingEdges?.length ?? 0,
+        resumeTerminalNoop,
+      })
+    } else if (!triggerBlockId) {
+      const executionKind =
+        triggerType === 'api' || triggerType === 'chat'
+          ? (triggerType as 'api' | 'chat')
+          : triggerType === 'webhook' || triggerType === 'schedule'
+            ? 'external'
+            : 'manual'
+
+      const startBlock = TriggerUtils.findStartBlock(mergedStates, executionKind, false)
+
+      if (!startBlock) {
+        const errorMsg = 'No start block found. Add a start block to this workflow.'
+        logger.error(`[${requestId}] ${errorMsg}`)
+        throw new Error(errorMsg)
+      }
+
+      resolvedTriggerBlockId = startBlock.blockId
+      logger.info(`[${requestId}] Identified trigger block for ${executionKind} execution:`, {
+        blockId: resolvedTriggerBlockId,
+        blockType: startBlock.block.type,
+        path: startBlock.path,
+      })
+    }
+
+    // Serialize workflow
+    const serializedWorkflow = new Serializer().serializeWorkflow(
+      mergedStates,
+      filteredEdges,
+      loops,
+      parallels,
+      true
+    )
+
+    processedInput = input || {}
+
+    // Resolve stopAfterBlockId for loop/parallel containers to their sentinel-end IDs
+    let resolvedStopAfterBlockId = stopAfterBlockId
+    if (stopAfterBlockId) {
+      if (serializedWorkflow.loops?.[stopAfterBlockId]) {
+        resolvedStopAfterBlockId = buildSentinelEndId(stopAfterBlockId)
+      } else if (serializedWorkflow.parallels?.[stopAfterBlockId]) {
+        resolvedStopAfterBlockId = buildParallelSentinelEndId(stopAfterBlockId)
+      }
+    }
+
+    // Create and execute workflow with callbacks
+    if (resumeFromSnapshot) {
+      logger.info(`[${requestId}] Resume execution detected`, {
+        resumePendingQueue,
+        hasState: !!snapshot.state,
+        stateBlockStatesCount: snapshot.state
+          ? Object.keys(snapshot.state.blockStates || {}).length
+          : 0,
+        executedBlocksCount: snapshot.state?.executedBlocks?.length ?? 0,
+        useDraftState,
+      })
+    }
+
+    const wrappedOnBlockComplete = (
+      blockId: string,
+      blockName: string,
+      blockType: string,
+      output: {
+        input?: unknown
+        output: NormalizedBlockOutput
+        executionTime: number
+        startedAt: string
+        endedAt: string
+      },
+      iterationContext?: IterationContext,
+      childWorkflowContext?: ChildWorkflowContext
+    ) => {
+      let persistenceSucceeded = false
+      const persistencePromise = (async () => {
+        await loggingSession.onBlockComplete(blockId, blockName, blockType, output)
+        persistenceSucceeded = true
+      })().catch((error) => {
+        logger.warn(`[${requestId}] Block completion persistence failed`, {
+          executionId,
+          blockId,
+          blockType,
+          error,
+        })
+      })
+
+      const lifecyclePromise = (async () => {
+        await persistencePromise
+        if (!persistenceSucceeded || !onBlockComplete) return
+
+        try {
+          await onBlockComplete(
+            blockId,
+            blockName,
+            blockType,
+            output,
+            iterationContext,
+            childWorkflowContext
+          )
+        } catch (error) {
+          logger.warn(`[${requestId}] Block completion callback failed`, {
+            executionId,
+            blockId,
+            blockType,
+            error,
+          })
+        }
+      })()
+
+      trackLifecycleCallback(lifecyclePromise)
+      return persistencePromise
+    }
+
+    const wrappedOnBlockStart = (
+      blockId: string,
+      blockName: string,
+      blockType: string,
+      executionOrder: number,
+      iterationContext?: IterationContext,
+      childWorkflowContext?: ChildWorkflowContext
+    ) => {
+      let persistenceSucceeded = false
+      const persistencePromise = (async () => {
+        await loggingSession.onBlockStart(blockId, blockName, blockType, new Date().toISOString())
+        persistenceSucceeded = true
+      })().catch((error) => {
+        logger.warn(`[${requestId}] Block start persistence failed`, {
+          executionId,
+          blockId,
+          blockType,
+          error,
+        })
+      })
+
+      const lifecyclePromise = (async () => {
+        await persistencePromise
+        if (!persistenceSucceeded || !onBlockStart) return
+
+        try {
+          await onBlockStart(
+            blockId,
+            blockName,
+            blockType,
+            executionOrder,
+            iterationContext,
+            childWorkflowContext
+          )
+        } catch (error) {
+          logger.warn(`[${requestId}] Block start callback failed`, {
+            executionId,
+            blockId,
+            blockType,
+            error,
+          })
+        }
+      })()
+
+      trackLifecycleCallback(lifecyclePromise)
+      return persistencePromise
+    }
+
+    const largeValueExecutionIds = Array.from(
+      new Set(
+        [executionId, ...(metadata.largeValueExecutionIds ?? [])].filter((id): id is string =>
+          Boolean(id)
+        )
+      )
+    )
+    const largeValueKeys = metadata.largeValueKeys
+    const fileKeys = metadata.fileKeys
+    const allowLargeValueWorkflowScope =
+      metadata.allowLargeValueWorkflowScope === true ||
+      metadata.resumeFromSnapshot === true ||
+      Boolean(runFromBlock?.sourceSnapshot && !runFromBlock.sourceExecutionId)
+
+    // Resolve the org/workspace PII redaction policy once; serves both the input
+    // stage (below) and the block-outputs stage (threaded into the executor).
+    // Resolved from stored rules UNCONDITIONALLY — deliberately NOT gated on the
+    // `pii-redaction` feature flag. The flag gates configuration (the settings
+    // route); a transient/false flag read at execution time would skip masking
+    // and leak PII (fail-open). Stored rules are only writable by entitled orgs,
+    // so their presence is the source of truth; absence yields the disabled
+    // default (one indexed lookup, no masking cost for non-PII orgs).
+    const [row] = await db
+      .select({ orgSettings: organization.dataRetentionSettings })
+      .from(workspace)
+      .leftJoin(organization, eq(organization.id, workspace.organizationId))
+      .where(eq(workspace.id, providedWorkspaceId))
+      .limit(1)
+    const piiRedaction: EffectivePiiRedaction = resolveEffectivePiiRedaction({
+      orgSettings: row?.orgSettings,
+      workspaceId: providedWorkspaceId,
+    })
+
+    if (piiRedaction.input.enabled) {
+      // Redact the input before the workflow sees it. `onFailure: 'throw'` aborts
+      // the run (handled by the surrounding catch) rather than feeding a scrub
+      // marker into execution or leaking unredacted input. A large input may
+      // already be offloaded to a large-value ref (opaque to the string walk), so
+      // hydrate → mask → re-store refs first, then mask inline strings.
+      const inputOpts = {
+        entityTypes: piiRedaction.input.entityTypes,
+        language: piiRedaction.input.language,
+        onFailure: 'throw' as const,
+      }
+      processedInput = await redactLargeValueRefsInValue(processedInput, {
+        ...inputOpts,
+        store: {
+          workspaceId: providedWorkspaceId,
+          workflowId,
+          executionId,
+          userId: userId ?? undefined,
+        },
+      })
+      processedInput = await redactObjectStrings(processedInput, inputOpts)
+    }
+
+    if (piiRedaction.blockOutputs.enabled) {
+      // Resume / run-from-block restore prior block outputs into state. If those
+      // predate the blockOutputs stage being enabled, re-mask them so downstream
+      // blocks can't read unredacted PII from restored snapshot state. Masking is
+      // idempotent, so outputs already masked in the original run are unaffected.
+      //
+      // Two disjoint passes cover the whole state: `redactLargeValueRefsInValue`
+      // hydrates → masks → re-stores any value offloaded to large-value storage
+      // (>8MB refs the string walk treats as opaque), then `redactObjectStrings`
+      // masks the remaining inline string leaves. Both fail-fast (`throw`), so an
+      // unmaskable restored value aborts the resume rather than warming raw PII
+      // into `blockStates` for downstream blocks.
+      const blockOutputOpts = {
+        entityTypes: piiRedaction.blockOutputs.entityTypes,
+        language: piiRedaction.blockOutputs.language,
+        onFailure: 'throw' as const,
+      }
+      const largeRefOpts = {
+        ...blockOutputOpts,
+        store: {
+          workspaceId: providedWorkspaceId,
+          workflowId,
+          executionId,
+          userId: userId ?? undefined,
+        },
+      }
+      if (snapshot.state?.blockStates) {
+        const hydrated = await redactLargeValueRefsInValue(snapshot.state.blockStates, largeRefOpts)
+        snapshot.state.blockStates = await redactObjectStrings(hydrated, blockOutputOpts)
+      }
+      if (runFromBlock?.sourceSnapshot?.blockStates) {
+        const hydrated = await redactLargeValueRefsInValue(
+          runFromBlock.sourceSnapshot.blockStates,
+          largeRefOpts
+        )
+        runFromBlock.sourceSnapshot.blockStates = await redactObjectStrings(
+          hydrated,
+          blockOutputOpts
+        )
+      }
+    }
+
+    const contextExtensions: ContextExtensions = {
+      stream: !!onStream,
+      selectedOutputs,
+      executionId,
+      largeValueExecutionIds,
+      largeValueKeys,
+      fileKeys,
+      allowLargeValueWorkflowScope,
+      workspaceId: providedWorkspaceId,
+      userId,
+      isDeployedContext: !metadata.isClientSession,
+      enforceCredentialAccess: metadata.enforceCredentialAccess ?? false,
+      piiBlockOutputRedaction: piiRedaction.blockOutputs,
+      onBlockStart: wrappedOnBlockStart,
+      onBlockComplete: wrappedOnBlockComplete,
+      onStream,
+      resumeFromSnapshot,
+      resumePendingQueue,
+      remainingEdges: snapshot.state?.remainingEdges?.map((edge) => ({
+        source: edge.source,
+        target: edge.target,
+        sourceHandle: edge.sourceHandle ?? undefined,
+        targetHandle: edge.targetHandle ?? undefined,
+      })),
+      dagIncomingEdges: snapshot.state?.dagIncomingEdges,
+      snapshotState: snapshot.state,
+      metadata,
+      abortSignal,
+      includeFileBase64,
+      base64MaxBytes,
+      stopAfterBlockId: resolvedStopAfterBlockId,
+      onChildWorkflowInstanceReady,
+      callChain: metadata.callChain,
+    }
+
+    if (snapshot.state) {
+      await warmLargeValueRefs(snapshot.state, {
+        workspaceId: providedWorkspaceId,
+        workflowId,
+        executionId,
+        largeValueExecutionIds,
+        largeValueKeys,
+        fileKeys,
+        allowLargeValueWorkflowScope,
+        userId,
+      })
+    }
+    for (const variable of Object.values(workflowVariables)) {
+      if (
+        isPlainRecord(variable) &&
+        variable.value !== undefined &&
+        typeof variable.type === 'string'
+      ) {
+        variable.value = parseVariableValueByType(variable.value, variable.type)
+      }
+    }
+
+    const executorInstance = new Executor({
+      workflow: serializedWorkflow,
+      envVarValues: decryptedEnvVars,
+      workflowInput: processedInput,
+      workflowVariables,
+      contextExtensions,
+    })
+
+    const result = runFromBlock
+      ? ((await executorInstance.executeFromBlock(
+          workflowId,
+          runFromBlock.startBlockId,
+          runFromBlock.sourceSnapshot
+        )) as ExecutionResult)
+      : ((await executorInstance.execute(workflowId, resolvedTriggerBlockId)) as ExecutionResult)
+
+    await waitForLifecycleCallbacks()
+
+    loggingSession.setPostExecutionPromise(
+      (async () => {
+        try {
+          await finalizeExecutionOutcome({
+            result,
+            loggingSession,
+            executionId,
+            requestId,
+            workflowInput: processedInput,
+          })
+
+          if (result.success && result.status !== 'paused') {
+            try {
+              await updateWorkflowRunCounts(workflowId)
+            } catch (runCountError) {
+              logger.error(`[${requestId}] Failed to update run counts`, { error: runCountError })
+            }
+          }
+        } catch (postExecError) {
+          logger.error(`[${requestId}] Post-execution logging failed`, { error: postExecError })
+        }
+      })()
+    )
+
+    logger.info(`[${requestId}] Workflow execution completed`, {
+      success: result.success,
+      status: result.status,
+      duration: result.metadata?.duration,
+    })
+
+    return result
+  } catch (error: unknown) {
+    const errorCause = describeErrorCause(error)
+    logger.error(
+      `[${requestId}] Execution failed:`,
+      error,
+      ...(errorCause ? [{ cause: errorCause }] : [])
+    )
+
+    await waitForLifecycleCallbacks()
+
+    if (!loggingStarted) {
+      loggingStarted = await loggingSession.safeStart({
+        userId,
+        workspaceId: providedWorkspaceId,
+        variables: {},
+        triggerData: metadata.correlation ? { correlation: metadata.correlation } : undefined,
+        skipLogCreation,
+        deploymentVersionId,
+      })
+    }
+
+    loggingSession.setPostExecutionPromise(
+      (async () => {
+        try {
+          const finalized = loggingStarted
+            ? await finalizeExecutionError({
+                error,
+                loggingSession,
+                executionId,
+                requestId,
+              })
+            : false
+
+          if (finalized) {
+            markExecutionFinalizedByCore(error, executionId)
+          }
+        } catch (postExecError) {
+          logger.error(`[${requestId}] Post-execution error logging failed`, {
+            error: postExecError,
+          })
+        }
+      })()
+    )
+
+    throw error
+  }
+}

apps/sim/lib/workflows/executor/execution-events.ts

@@ -0,0 +1,503 @@
+import type {
+  ChildWorkflowContext,
+  IterationContext,
+  ParentIteration,
+} from '@/executor/execution/types'
+import type { BlockLog } from '@/executor/types'
+import type { SubflowType } from '@/stores/workflows/workflow/types'
+
+export type ExecutionEventType =
+  | 'execution:started'
+  | 'execution:completed'
+  | 'execution:paused'
+  | 'execution:error'
+  | 'execution:cancelled'
+  | 'block:started'
+  | 'block:completed'
+  | 'block:error'
+  | 'block:childWorkflowStarted'
+  | 'stream:chunk'
+  | 'stream:done'
+
+/**
+ * Base event structure for SSE
+ */
+interface BaseExecutionEvent {
+  type: ExecutionEventType
+  timestamp: string
+  executionId: string
+  eventId?: number
+}
+
+/**
+ * Execution started event
+ */
+interface ExecutionStartedEvent extends BaseExecutionEvent {
+  type: 'execution:started'
+  workflowId: string
+  data: {
+    startTime: string
+  }
+}
+
+/**
+ * Execution completed event
+ */
+interface ExecutionCompletedEvent extends BaseExecutionEvent {
+  type: 'execution:completed'
+  workflowId: string
+  data: {
+    success: boolean
+    output: any
+    duration: number
+    startTime: string
+    endTime: string
+    /** Authoritative per-block terminal states from the server's blockLogs. */
+    finalBlockLogs?: BlockLog[]
+  }
+}
+
+/**
+ * Execution paused event (HITL block waiting for human input)
+ */
+interface ExecutionPausedEvent extends BaseExecutionEvent {
+  type: 'execution:paused'
+  workflowId: string
+  data: {
+    output: any
+    duration: number
+    startTime: string
+    endTime: string
+    /** Authoritative per-block terminal states from the server's blockLogs. */
+    finalBlockLogs?: BlockLog[]
+  }
+}
+
+/**
+ * Execution error event
+ */
+interface ExecutionErrorEvent extends BaseExecutionEvent {
+  type: 'execution:error'
+  workflowId: string
+  data: {
+    error: string
+    duration: number
+    /** Authoritative per-block terminal states from the server's blockLogs. */
+    finalBlockLogs?: BlockLog[]
+  }
+}
+
+interface ExecutionCancelledEvent extends BaseExecutionEvent {
+  type: 'execution:cancelled'
+  workflowId: string
+  data: {
+    duration: number
+    /** Authoritative per-block terminal states from the server's blockLogs. */
+    finalBlockLogs?: BlockLog[]
+  }
+}
+
+/**
+ * Block started event
+ */
+interface BlockStartedEvent extends BaseExecutionEvent {
+  type: 'block:started'
+  workflowId: string
+  data: {
+    blockId: string
+    blockName: string
+    blockType: string
+    executionOrder: number
+    iterationCurrent?: number
+    iterationTotal?: number
+    iterationType?: SubflowType
+    iterationContainerId?: string
+    parentIterations?: ParentIteration[]
+    childWorkflowBlockId?: string
+    childWorkflowName?: string
+  }
+}
+
+/**
+ * Block completed event
+ */
+interface BlockCompletedEvent extends BaseExecutionEvent {
+  type: 'block:completed'
+  workflowId: string
+  data: {
+    blockId: string
+    blockName: string
+    blockType: string
+    input?: any
+    output: any
+    durationMs: number
+    startedAt: string
+    executionOrder: number
+    endedAt: string
+    iterationCurrent?: number
+    iterationTotal?: number
+    iterationType?: SubflowType
+    iterationContainerId?: string
+    parentIterations?: ParentIteration[]
+    childWorkflowBlockId?: string
+    childWorkflowName?: string
+    /** Per-invocation unique ID for correlating child block events with this workflow block. */
+    childWorkflowInstanceId?: string
+  }
+}
+
+/**
+ * Block error event
+ */
+interface BlockErrorEvent extends BaseExecutionEvent {
+  type: 'block:error'
+  workflowId: string
+  data: {
+    blockId: string
+    blockName: string
+    blockType: string
+    input?: any
+    error: string
+    durationMs: number
+    startedAt: string
+    executionOrder: number
+    endedAt: string
+    iterationCurrent?: number
+    iterationTotal?: number
+    iterationType?: SubflowType
+    iterationContainerId?: string
+    parentIterations?: ParentIteration[]
+    childWorkflowBlockId?: string
+    childWorkflowName?: string
+    /** Per-invocation unique ID for correlating child block events with this workflow block. */
+    childWorkflowInstanceId?: string
+  }
+}
+
+/**
+ * Block child workflow started event — fires when a workflow block generates its instanceId,
+ * before child execution begins. Allows clients to pre-associate the running entry with
+ * the instanceId so child block events can be correlated in real-time.
+ */
+interface BlockChildWorkflowStartedEvent extends BaseExecutionEvent {
+  type: 'block:childWorkflowStarted'
+  workflowId: string
+  data: {
+    blockId: string
+    childWorkflowInstanceId: string
+    iterationCurrent?: number
+    iterationTotal?: number
+    iterationType?: SubflowType
+    iterationContainerId?: string
+    parentIterations?: ParentIteration[]
+    childWorkflowBlockId?: string
+    childWorkflowName?: string
+    executionOrder?: number
+  }
+}
+
+/**
+ * Stream chunk event (for agent blocks)
+ */
+interface StreamChunkEvent extends BaseExecutionEvent {
+  type: 'stream:chunk'
+  workflowId: string
+  data: {
+    blockId: string
+    chunk: string
+  }
+}
+
+/**
+ * Stream done event
+ */
+interface StreamDoneEvent extends BaseExecutionEvent {
+  type: 'stream:done'
+  workflowId: string
+  data: {
+    blockId: string
+  }
+}
+
+/**
+ * Union type of all execution events
+ */
+export type ExecutionEvent =
+  | ExecutionStartedEvent
+  | ExecutionCompletedEvent
+  | ExecutionPausedEvent
+  | ExecutionErrorEvent
+  | ExecutionCancelledEvent
+  | BlockStartedEvent
+  | BlockCompletedEvent
+  | BlockErrorEvent
+  | BlockChildWorkflowStartedEvent
+  | StreamChunkEvent
+  | StreamDoneEvent
+
+export type ExecutionStartedData = ExecutionStartedEvent['data']
+export type ExecutionCompletedData = ExecutionCompletedEvent['data']
+export type ExecutionPausedData = ExecutionPausedEvent['data']
+export type ExecutionErrorData = ExecutionErrorEvent['data']
+export type ExecutionCancelledData = ExecutionCancelledEvent['data']
+export type BlockStartedData = BlockStartedEvent['data']
+export type BlockCompletedData = BlockCompletedEvent['data']
+export type BlockErrorData = BlockErrorEvent['data']
+export type BlockChildWorkflowStartedData = BlockChildWorkflowStartedEvent['data']
+export type StreamChunkData = StreamChunkEvent['data']
+export type StreamDoneData = StreamDoneEvent['data']
+
+/**
+ * Helper to create SSE formatted message
+ */
+export function formatSSEEvent(event: ExecutionEvent): string {
+  return `data: ${JSON.stringify(event)}\n\n`
+}
+
+/**
+ * Helper to encode SSE event as Uint8Array
+ */
+export function encodeSSEEvent(event: ExecutionEvent): Uint8Array {
+  return new TextEncoder().encode(formatSSEEvent(event))
+}
+
+/**
+ * Options for creating SSE execution callbacks
+ */
+interface SSECallbackOptions {
+  executionId: string
+  workflowId: string
+  controller: ReadableStreamDefaultController<Uint8Array>
+  isStreamClosed: () => boolean
+  setStreamClosed: () => void
+}
+
+/**
+ * Creates execution callbacks using a provided event sink.
+ */
+export function createExecutionCallbacks(options: {
+  executionId: string
+  workflowId: string
+  sendEvent: (event: ExecutionEvent) => void | Promise<void>
+}) {
+  const { executionId, workflowId, sendEvent } = options
+
+  const sendBufferedEvent = async (event: ExecutionEvent) => {
+    await sendEvent(event)
+  }
+
+  const onBlockStart = async (
+    blockId: string,
+    blockName: string,
+    blockType: string,
+    executionOrder: number,
+    iterationContext?: IterationContext,
+    childWorkflowContext?: ChildWorkflowContext
+  ) => {
+    await sendBufferedEvent({
+      type: 'block:started',
+      timestamp: new Date().toISOString(),
+      executionId,
+      workflowId,
+      data: {
+        blockId,
+        blockName,
+        blockType,
+        executionOrder,
+        ...(iterationContext && {
+          iterationCurrent: iterationContext.iterationCurrent,
+          iterationTotal: iterationContext.iterationTotal,
+          iterationType: iterationContext.iterationType,
+          iterationContainerId: iterationContext.iterationContainerId,
+          ...(iterationContext.parentIterations?.length && {
+            parentIterations: iterationContext.parentIterations,
+          }),
+        }),
+        ...(childWorkflowContext && {
+          childWorkflowBlockId: childWorkflowContext.parentBlockId,
+          childWorkflowName: childWorkflowContext.workflowName,
+        }),
+      },
+    })
+  }
+
+  const onBlockComplete = async (
+    blockId: string,
+    blockName: string,
+    blockType: string,
+    callbackData: {
+      input?: unknown
+      output: any
+      executionTime: number
+      startedAt: string
+      executionOrder: number
+      endedAt: string
+      childWorkflowInstanceId?: string
+    },
+    iterationContext?: IterationContext,
+    childWorkflowContext?: ChildWorkflowContext
+  ) => {
+    const hasError = callbackData.output?.error
+    const iterationData = iterationContext
+      ? {
+          iterationCurrent: iterationContext.iterationCurrent,
+          iterationTotal: iterationContext.iterationTotal,
+          iterationType: iterationContext.iterationType,
+          iterationContainerId: iterationContext.iterationContainerId,
+          ...(iterationContext.parentIterations?.length && {
+            parentIterations: iterationContext.parentIterations,
+          }),
+        }
+      : {}
+    const childWorkflowData = childWorkflowContext
+      ? {
+          childWorkflowBlockId: childWorkflowContext.parentBlockId,
+          childWorkflowName: childWorkflowContext.workflowName,
+        }
+      : {}
+
+    const instanceData = callbackData.childWorkflowInstanceId
+      ? { childWorkflowInstanceId: callbackData.childWorkflowInstanceId }
+      : {}
+
+    if (hasError) {
+      await sendBufferedEvent({
+        type: 'block:error',
+        timestamp: new Date().toISOString(),
+        executionId,
+        workflowId,
+        data: {
+          blockId,
+          blockName,
+          blockType,
+          input: callbackData.input,
+          error: callbackData.output.error,
+          durationMs: callbackData.executionTime || 0,
+          startedAt: callbackData.startedAt,
+          executionOrder: callbackData.executionOrder,
+          endedAt: callbackData.endedAt,
+          ...iterationData,
+          ...childWorkflowData,
+          ...instanceData,
+        },
+      })
+    } else {
+      await sendBufferedEvent({
+        type: 'block:completed',
+        timestamp: new Date().toISOString(),
+        executionId,
+        workflowId,
+        data: {
+          blockId,
+          blockName,
+          blockType,
+          input: callbackData.input,
+          output: callbackData.output,
+          durationMs: callbackData.executionTime || 0,
+          startedAt: callbackData.startedAt,
+          executionOrder: callbackData.executionOrder,
+          endedAt: callbackData.endedAt,
+          ...iterationData,
+          ...childWorkflowData,
+          ...instanceData,
+        },
+      })
+    }
+  }
+
+  const onStream = async (streamingExecution: unknown) => {
+    const streamingExec = streamingExecution as { stream: ReadableStream; execution: any }
+    const blockId = streamingExec.execution?.blockId
+    const reader = streamingExec.stream.getReader()
+    const decoder = new TextDecoder()
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+        const chunk = decoder.decode(value, { stream: true })
+        await sendBufferedEvent({
+          type: 'stream:chunk',
+          timestamp: new Date().toISOString(),
+          executionId,
+          workflowId,
+          data: { blockId, chunk },
+        })
+      }
+      await sendBufferedEvent({
+        type: 'stream:done',
+        timestamp: new Date().toISOString(),
+        executionId,
+        workflowId,
+        data: { blockId },
+      })
+    } finally {
+      try {
+        reader.releaseLock()
+      } catch {}
+    }
+  }
+
+  const onChildWorkflowInstanceReady = async (
+    blockId: string,
+    childWorkflowInstanceId: string,
+    iterationContext?: IterationContext,
+    executionOrder?: number,
+    childWorkflowContext?: ChildWorkflowContext
+  ) => {
+    await sendBufferedEvent({
+      type: 'block:childWorkflowStarted',
+      timestamp: new Date().toISOString(),
+      executionId,
+      workflowId,
+      data: {
+        blockId,
+        childWorkflowInstanceId,
+        ...(iterationContext && {
+          iterationCurrent: iterationContext.iterationCurrent,
+          iterationTotal: iterationContext.iterationTotal,
+          iterationType: iterationContext.iterationType,
+          iterationContainerId: iterationContext.iterationContainerId,
+          ...(iterationContext.parentIterations?.length && {
+            parentIterations: iterationContext.parentIterations,
+          }),
+        }),
+        ...(childWorkflowContext && {
+          childWorkflowBlockId: childWorkflowContext.parentBlockId,
+          childWorkflowName: childWorkflowContext.workflowName,
+        }),
+        ...(executionOrder !== undefined && { executionOrder }),
+      },
+    })
+  }
+
+  return {
+    sendEvent: sendBufferedEvent,
+    onBlockStart,
+    onBlockComplete,
+    onStream,
+    onChildWorkflowInstanceReady,
+  }
+}
+
+/**
+ * Creates SSE callbacks for workflow execution streaming
+ */
+export function createSSECallbacks(options: SSECallbackOptions) {
+  const { executionId, workflowId, controller, isStreamClosed, setStreamClosed } = options
+
+  const sendEvent = (event: ExecutionEvent) => {
+    if (isStreamClosed()) return
+    try {
+      controller.enqueue(encodeSSEEvent(event))
+    } catch {
+      setStreamClosed()
+    }
+  }
+
+  return createExecutionCallbacks({
+    executionId,
+    workflowId,
+    sendEvent,
+  })
+}

apps/sim/lib/workflows/executor/execution-state.ts

@@ -0,0 +1,115 @@
+import { db } from '@sim/db'
+import { workflowExecutionLogs } from '@sim/db/schema'
+import { and, desc, eq, sql } from 'drizzle-orm'
+import type { SerializableExecutionState } from '@/executor/execution/types'
+
+export interface ExecutionStateRecord {
+  executionId: string
+  state: SerializableExecutionState
+}
+
+function isSerializableExecutionState(value: unknown): value is SerializableExecutionState {
+  if (!value || typeof value !== 'object') return false
+  const state = value as Record<string, unknown>
+  return (
+    typeof state.blockStates === 'object' &&
+    Array.isArray(state.executedBlocks) &&
+    Array.isArray(state.blockLogs) &&
+    typeof state.decisions === 'object' &&
+    Array.isArray(state.completedLoops) &&
+    Array.isArray(state.activeExecutionPath)
+  )
+}
+
+function extractExecutionState(executionData: unknown): SerializableExecutionState | null {
+  if (!executionData || typeof executionData !== 'object') return null
+  const state = (executionData as Record<string, unknown>).executionState
+  return isSerializableExecutionState(state) ? state : null
+}
+
+export async function getExecutionState(
+  executionId: string
+): Promise<SerializableExecutionState | null> {
+  const [row] = await db
+    .select({ executionData: workflowExecutionLogs.executionData })
+    .from(workflowExecutionLogs)
+    .where(eq(workflowExecutionLogs.executionId, executionId))
+    .limit(1)
+
+  return extractExecutionState(row?.executionData)
+}
+
+export async function getExecutionStateForWorkflow(
+  executionId: string,
+  workflowId: string
+): Promise<SerializableExecutionState | null> {
+  const [row] = await db
+    .select({ executionData: workflowExecutionLogs.executionData })
+    .from(workflowExecutionLogs)
+    .where(
+      and(
+        eq(workflowExecutionLogs.executionId, executionId),
+        eq(workflowExecutionLogs.workflowId, workflowId)
+      )
+    )
+    .limit(1)
+
+  return extractExecutionState(row?.executionData)
+}
+
+/**
+ * Returns the workflow input recorded for a past execution so a new run can
+ * reuse it by reference. `found` distinguishes a missing execution from an
+ * execution that recorded no input.
+ */
+export async function getExecutionInputForWorkflow(
+  executionId: string,
+  workflowId: string
+): Promise<{ found: boolean; input?: unknown }> {
+  const [row] = await db
+    .select({ executionData: workflowExecutionLogs.executionData })
+    .from(workflowExecutionLogs)
+    .where(
+      and(
+        eq(workflowExecutionLogs.executionId, executionId),
+        eq(workflowExecutionLogs.workflowId, workflowId)
+      )
+    )
+    .limit(1)
+
+  if (!row) {
+    return { found: false }
+  }
+
+  const data = row.executionData as { workflowInput?: unknown } | null | undefined
+  return { found: true, input: data?.workflowInput }
+}
+
+export async function getLatestExecutionState(
+  workflowId: string
+): Promise<SerializableExecutionState | null> {
+  const record = await getLatestExecutionStateWithExecutionId(workflowId)
+  return record?.state ?? null
+}
+
+export async function getLatestExecutionStateWithExecutionId(
+  workflowId: string
+): Promise<ExecutionStateRecord | null> {
+  const [row] = await db
+    .select({
+      executionId: workflowExecutionLogs.executionId,
+      executionData: workflowExecutionLogs.executionData,
+    })
+    .from(workflowExecutionLogs)
+    .where(
+      and(
+        eq(workflowExecutionLogs.workflowId, workflowId),
+        sql`${workflowExecutionLogs.executionData} -> 'executionState' IS NOT NULL`
+      )
+    )
+    .orderBy(desc(workflowExecutionLogs.startedAt))
+    .limit(1)
+
+  const state = extractExecutionState(row?.executionData)
+  return row && state ? { executionId: row.executionId, state } : null
+}

apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts

@@ -0,0 +1,337 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+import {
+  PauseResumeManager,
+  updateResumeOutputInAggregationBuffers,
+} from '@/lib/workflows/executor/human-in-the-loop-manager'
+import type { SerializableExecutionState } from '@/executor/execution/types'
+import type { PausePoint, SerializedSnapshot } from '@/executor/types'
+
+function createExecutionState(): SerializableExecutionState {
+  return {
+    blockStates: {},
+    executedBlocks: [],
+    blockLogs: [],
+    decisions: { router: {}, condition: {} },
+    completedLoops: [],
+    activeExecutionPath: [],
+  }
+}
+
+describe('updateResumeOutputInAggregationBuffers', () => {
+  it('replaces a paused parallel branch placeholder with the resumed HITL output', () => {
+    const pausedOutput = {
+      response: { status: 'paused' },
+      _pauseMetadata: {
+        contextId: 'pause-context-1',
+        blockId: 'hitl₍1₎',
+      },
+    }
+    const siblingOutput = { value: 'already-complete' }
+    const mergedOutput = {
+      response: { data: { submission: { approved: true } } },
+      submission: { approved: true },
+      _resumed: true,
+    }
+    const state = createExecutionState()
+    state.parallelExecutions = {
+      'parallel-1': {
+        branchOutputs: {
+          0: [siblingOutput],
+          1: [pausedOutput],
+        },
+      },
+    }
+
+    updateResumeOutputInAggregationBuffers(
+      state,
+      'hitl₍1₎',
+      'hitl',
+      'pause-context-1',
+      mergedOutput
+    )
+
+    expect(state.parallelExecutions['parallel-1'].branchOutputs).toEqual({
+      0: [siblingOutput],
+      1: [mergedOutput],
+    })
+  })
+
+  it('does not replace unrelated paused parallel branch outputs', () => {
+    const unrelatedPausedOutput = {
+      response: { status: 'paused' },
+      _pauseMetadata: {
+        contextId: 'different-context',
+        blockId: 'hitl₍1₎',
+      },
+    }
+    const mergedOutput = {
+      response: { data: { submission: { approved: true } } },
+      submission: { approved: true },
+      _resumed: true,
+    }
+    const state = createExecutionState()
+    state.parallelExecutions = {
+      'parallel-1': {
+        branchOutputs: {
+          1: [unrelatedPausedOutput],
+        },
+      },
+    }
+
+    updateResumeOutputInAggregationBuffers(
+      state,
+      'hitl₍1₎',
+      'hitl',
+      'pause-context-1',
+      mergedOutput
+    )
+
+    expect(state.parallelExecutions['parallel-1'].branchOutputs).toEqual({
+      1: [unrelatedPausedOutput],
+    })
+  })
+
+  it('replaces paused loop iteration outputs using the resumed state block key', () => {
+    const pausedOutput = {
+      response: { status: 'paused' },
+      _pauseMetadata: {
+        contextId: 'pause-context-1',
+        blockId: 'hitl',
+      },
+    }
+    const unrelatedPausedOutput = {
+      response: { status: 'paused' },
+      _pauseMetadata: {
+        contextId: 'different-context',
+        blockId: 'hitl',
+      },
+    }
+    const siblingOutput = { value: 'already-complete' }
+    const mergedOutput = {
+      response: { data: { submission: { approved: true } } },
+      submission: { approved: true },
+      _resumed: true,
+    }
+    const state = createExecutionState()
+    state.loopExecutions = {
+      'loop-1': {
+        currentIterationOutputs: {
+          hitl: pausedOutput,
+          sibling: siblingOutput,
+        },
+      },
+      'loop-2': {
+        currentIterationOutputs: {
+          hitl: unrelatedPausedOutput,
+        },
+      },
+    }
+
+    updateResumeOutputInAggregationBuffers(
+      state,
+      'hitl₍1₎',
+      'hitl',
+      'pause-context-1',
+      mergedOutput
+    )
+
+    expect(state.loopExecutions['loop-1'].currentIterationOutputs).toEqual({
+      'hitl₍1₎': mergedOutput,
+      sibling: siblingOutput,
+    })
+    expect(state.loopExecutions['loop-2'].currentIterationOutputs).toEqual({
+      hitl: unrelatedPausedOutput,
+    })
+  })
+})
+
+describe('PauseResumeManager.getPauseContextDetail', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('does not duplicate a pause point large response payload between pausePoint and execution.pausePoints', async () => {
+    const largeDisplayValue = 'x'.repeat(50_000)
+
+    const row = {
+      id: 'paused-exec-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      status: 'paused',
+      pausedAt: null,
+      updatedAt: null,
+      expiresAt: null,
+      metadata: {},
+      executionSnapshot: { triggerIds: [] },
+      pausePoints: {
+        'ctx-1': {
+          contextId: 'ctx-1',
+          blockId: 'hitl-1',
+          resumeStatus: 'paused',
+          snapshotReady: true,
+          pauseKind: 'human',
+          registeredAt: '2026-07-02T00:00:00.000Z',
+          response: {
+            data: {
+              operation: 'human',
+              inputFormat: [{ id: 'field_0', name: 'approved', type: 'boolean', required: false }],
+              submission: null,
+              responseStructure: [
+                { name: 'ai_analysis', type: 'string', value: largeDisplayValue },
+              ],
+            },
+            status: 200,
+            headers: {},
+          },
+        },
+        'ctx-2': {
+          contextId: 'ctx-2',
+          blockId: 'hitl-2',
+          resumeStatus: 'paused',
+          snapshotReady: true,
+          pauseKind: 'human',
+          registeredAt: '2026-07-02T00:00:00.000Z',
+          response: {
+            data: { operation: 'human', inputFormat: [], submission: null },
+            status: 200,
+            headers: {},
+          },
+        },
+      },
+    }
+
+    dbChainMockFns.limit.mockResolvedValueOnce([row])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([])
+
+    const detail = await PauseResumeManager.getPauseContextDetail({
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      contextId: 'ctx-1',
+    })
+
+    expect(detail).not.toBeNull()
+    // The requested pause point keeps its full response payload.
+    expect(detail!.pausePoint.response.data.responseStructure[0].value).toBe(largeDisplayValue)
+    expect(detail!.pausePoint.contextId).toBe('ctx-1')
+
+    // `execution.pausePoints` must not re-embed the (potentially large)
+    // response payload — it's already available via `pausePoint` above.
+    for (const point of detail!.execution.pausePoints) {
+      expect(point.response?.data).toBeUndefined()
+    }
+    // Non-payload fields are still present on the execution's pause points.
+    expect(detail!.execution.pausePoints.map((p) => p.contextId).sort()).toEqual(['ctx-1', 'ctx-2'])
+    expect(detail!.execution.pausePoints.find((p) => p.contextId === 'ctx-1')?.resumeStatus).toBe(
+      'paused'
+    )
+  })
+
+  it('returns null when the pause context no longer exists', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'paused-exec-1',
+        workflowId: 'workflow-1',
+        executionId: 'execution-1',
+        status: 'paused',
+        pausedAt: null,
+        updatedAt: null,
+        expiresAt: null,
+        metadata: {},
+        executionSnapshot: { triggerIds: [] },
+        pausePoints: {
+          'ctx-1': {
+            contextId: 'ctx-1',
+            blockId: 'hitl-1',
+            resumeStatus: 'paused',
+            snapshotReady: true,
+            pauseKind: 'human',
+            registeredAt: '2026-07-02T00:00:00.000Z',
+            response: { data: { operation: 'human' }, status: 200, headers: {} },
+          },
+        },
+      },
+    ])
+    dbChainMockFns.orderBy.mockResolvedValueOnce([])
+
+    const detail = await PauseResumeManager.getPauseContextDetail({
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      contextId: 'missing-ctx',
+    })
+
+    expect(detail).toBeNull()
+  })
+})
+
+describe('PauseResumeManager.persistPauseResult metadata merge on re-pause', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('preserves the stashed cellContext when an existing paused row re-pauses (chained waits)', async () => {
+    const cellContext = {
+      tableId: 'table-1',
+      rowId: 'row-1',
+      workspaceId: 'workspace-1',
+      groupId: 'group-1',
+      workflowId: 'workflow-1',
+    }
+    const existingRow = {
+      id: 'paused-exec-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      status: 'partially_resumed',
+      pausePoints: {
+        'ctx-wait-1': { contextId: 'ctx-wait-1', blockId: 'wait1', resumeStatus: 'resuming' },
+      },
+      metadata: {
+        pauseScope: 'execution',
+        triggerIds: ['start'],
+        executorUserId: 'user-1',
+        cellContext,
+      },
+    }
+
+    // First `.limit(1)` resolves the select-for-update to the existing row,
+    // forcing persistPauseResult down the update (not insert) branch.
+    dbChainMockFns.limit.mockResolvedValueOnce([existingRow])
+
+    const snapshotSeed: SerializedSnapshot = { snapshot: '{}', triggerIds: [] }
+    const pausePoints: PausePoint[] = [
+      {
+        contextId: 'ctx-wait-2',
+        blockId: 'wait2',
+        pauseKind: 'time',
+        resumeAt: new Date(Date.now() + 60_000).toISOString(),
+        resumeStatus: 'paused',
+      } as PausePoint,
+    ]
+
+    await PauseResumeManager.persistPauseResult({
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      pausePoints,
+      snapshotSeed,
+      executorUserId: 'user-1',
+    })
+
+    const updateSetCall = dbChainMockFns.set.mock.calls.find(
+      ([arg]) => arg && typeof arg === 'object' && 'metadata' in (arg as Record<string, unknown>)
+    )
+    expect(updateSetCall).toBeDefined()
+
+    const updatedMetadata = (updateSetCall![0] as { metadata: Record<string, unknown> }).metadata
+    expect(updatedMetadata.cellContext).toEqual(cellContext)
+    expect(updatedMetadata.pauseScope).toBe('execution')
+    expect(updatedMetadata.executorUserId).toBe('user-1')
+  })
+})

apps/sim/lib/workflows/executor/pause-persistence.ts

@@ -0,0 +1,70 @@
+import { createLogger } from '@sim/logger'
+import { describeError, toError } from '@sim/utils/errors'
+import { isRetryableInfrastructureError } from '@/lib/core/errors/retryable-infrastructure'
+import type { LoggingSession } from '@/lib/logs/execution/logging-session'
+import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
+import type { ExecutionResult } from '@/executor/types'
+
+const logger = createLogger('PausePersistence')
+
+interface HandlePostExecutionPauseStateArgs {
+  result: ExecutionResult
+  workflowId: string
+  executionId: string
+  loggingSession: LoggingSession
+}
+
+/**
+ * Handles pause persistence and resume queue processing after `executeWorkflowCore` returns.
+ *
+ * Every caller of `executeWorkflowCore` must call this after execution completes
+ * to ensure HITL pause state is persisted to the database and queued resumes are drained.
+ *
+ * - If execution is paused with a valid snapshot: persists to `paused_executions` table
+ * - If execution is paused without a snapshot: marks execution as failed
+ * - If execution is not paused: processes any queued resume entries
+ */
+export async function handlePostExecutionPauseState({
+  result,
+  workflowId,
+  executionId,
+  loggingSession,
+}: HandlePostExecutionPauseStateArgs): Promise<void> {
+  if (result.status === 'paused') {
+    if (!result.snapshotSeed) {
+      logger.error('Missing snapshot seed for paused execution', { executionId })
+      await loggingSession.markAsFailed('Missing snapshot seed for paused execution')
+    } else {
+      try {
+        await PauseResumeManager.persistPauseResult({
+          workflowId,
+          executionId,
+          pausePoints: result.pausePoints || [],
+          snapshotSeed: result.snapshotSeed,
+          executorUserId: result.metadata?.userId,
+        })
+      } catch (pauseError) {
+        logger.error('Failed to persist pause result', {
+          executionId,
+          error: toError(pauseError).message,
+          cause: describeError(pauseError),
+          retryable: isRetryableInfrastructureError(pauseError),
+        })
+        await loggingSession.markAsFailed(
+          `Failed to persist pause state: ${toError(pauseError).message}`
+        )
+      }
+    }
+  } else {
+    try {
+      await PauseResumeManager.processQueuedResumes(executionId, workflowId)
+    } catch (resumeError) {
+      logger.error('Failed to process queued resumes', {
+        executionId,
+        error: toError(resumeError).message,
+        cause: describeError(resumeError),
+        retryable: isRetryableInfrastructureError(resumeError),
+      })
+    }
+  }
+}