chore: import upstream snapshot with attribution

apps/sim/lib/logs/execution/logger.test.ts

@@ -0,0 +1,795 @@
+import { envFlagsMock } from '@sim/testing'
+import { beforeEach, describe, expect, test, vi } from 'vitest'
+import { recordUsage } from '@/lib/billing/core/usage-log'
+import { ExecutionLogger } from '@/lib/logs/execution/logger'
+
+const dbSelectMock = vi.hoisted(() => vi.fn())
+const dbExecuteMock = vi.hoisted(() => vi.fn())
+const txUpdateMock = vi.hoisted(() =>
+  vi.fn(() => ({ set: () => ({ where: () => Promise.resolve() }) }))
+)
+
+vi.mock('@sim/db', () => {
+  // The reconcile runs inside db.transaction with an advisory lock. The tx
+  // shares dbSelectMock so the existing call-order seeding (call 1 = workflow
+  // row via .limit, call 2 = already-billed via .groupBy) still applies;
+  // tx.execute (set_config + pg_advisory_xact_lock) is a no-op; tx.update backs
+  // the exact cost_total refine.
+  const tx = {
+    select: dbSelectMock,
+    insert: vi.fn(),
+    update: txUpdateMock,
+    execute: dbExecuteMock,
+  }
+  return {
+    db: {
+      select: dbSelectMock,
+      insert: vi.fn(),
+      update: vi.fn(),
+      execute: dbExecuteMock,
+      transaction: vi.fn(async (cb: (txArg: typeof tx) => Promise<unknown>) => cb(tx)),
+    },
+  }
+})
+
+// Mock billing modules
+vi.mock('@/lib/billing/core/subscription', () => ({
+  getHighestPrioritySubscription: vi.fn(() => Promise.resolve(null)),
+}))
+
+vi.mock('@/lib/billing/core/usage', () => ({
+  checkUsageStatus: vi.fn(() =>
+    Promise.resolve({
+      usageData: { limit: 100, percentUsed: 50, currentUsage: 50 },
+    })
+  ),
+  getOrgUsageLimit: vi.fn(() => Promise.resolve({ limit: 1000 })),
+  maybeSendUsageThresholdEmail: vi.fn(() => Promise.resolve()),
+}))
+
+vi.mock('@/lib/billing/core/usage-log', () => ({
+  recordUsage: vi.fn(() => Promise.resolve()),
+  stableEventKey: vi.fn((parts: Record<string, unknown>) => JSON.stringify(parts)),
+  deriveBillingContext: vi.fn((userId: string) => ({
+    billingEntity: { type: 'user', id: userId },
+    billingPeriod: { start: new Date('2024-01-01'), end: new Date('2024-02-01') },
+  })),
+}))
+
+vi.mock('@/lib/billing/threshold-billing', () => ({
+  checkAndBillOverageThreshold: vi.fn(() => Promise.resolve()),
+}))
+
+vi.mock('@/lib/core/config/env-flags', () => envFlagsMock)
+
+// Mock security module
+vi.mock('@/lib/core/security/redaction', () => ({
+  redactApiKeys: vi.fn((data) => data),
+}))
+
+// Mock display filters
+vi.mock('@/lib/core/utils/display-filters', () => ({
+  filterForDisplay: vi.fn((data) => data),
+}))
+
+// Mock workspace event emission
+vi.mock('@/lib/workspace-events/emitter', () => ({
+  emitExecutionCompletedEvent: vi.fn(() => Promise.resolve()),
+}))
+
+// Mock snapshot service
+vi.mock('@/lib/logs/execution/snapshot/service', () => ({
+  snapshotService: {
+    createSnapshotWithDeduplication: vi.fn(() =>
+      Promise.resolve({
+        snapshot: {
+          id: 'snapshot-123',
+          workflowId: 'workflow-123',
+          stateHash: 'hash-123',
+          stateData: { blocks: {}, edges: [], loops: {}, parallels: {} },
+          createdAt: '2024-01-01T00:00:00.000Z',
+        },
+        isNew: true,
+      })
+    ),
+    getSnapshot: vi.fn(() =>
+      Promise.resolve({
+        id: 'snapshot-123',
+        workflowId: 'workflow-123',
+        stateHash: 'hash-123',
+        stateData: { blocks: {}, edges: [], loops: {}, parallels: {} },
+        createdAt: '2024-01-01T00:00:00.000Z',
+      })
+    ),
+  },
+}))
+
+describe('ExecutionLogger', () => {
+  let logger: ExecutionLogger
+
+  beforeEach(() => {
+    logger = new ExecutionLogger()
+    vi.clearAllMocks()
+  })
+
+  describe('class instantiation', () => {
+    test('should create logger instance', () => {
+      expect(logger).toBeDefined()
+      expect(logger).toBeInstanceOf(ExecutionLogger)
+    })
+  })
+
+  describe('interface implementation', () => {
+    test('should have startWorkflowExecution method', () => {
+      expect(typeof logger.startWorkflowExecution).toBe('function')
+    })
+
+    test('should have completeWorkflowExecution method', () => {
+      expect(typeof logger.completeWorkflowExecution).toBe('function')
+    })
+
+    test('should have getWorkflowExecution method', () => {
+      expect(typeof logger.getWorkflowExecution).toBe('function')
+    })
+
+    test('preserves correlation and diagnostics when execution completes', () => {
+      const loggerInstance = new ExecutionLogger() as any
+
+      const completedData = loggerInstance.buildCompletedExecutionData({
+        existingExecutionData: {
+          environment: {
+            variables: {},
+            workflowId: 'workflow-123',
+            executionId: 'execution-123',
+            userId: 'user-123',
+            workspaceId: 'workspace-123',
+          },
+          trigger: {
+            type: 'webhook',
+            source: 'webhook',
+            timestamp: '2025-01-01T00:00:00.000Z',
+            data: {
+              correlation: {
+                executionId: 'execution-123',
+                requestId: 'req-1234',
+                source: 'webhook',
+                workflowId: 'workflow-123',
+                webhookId: 'webhook-123',
+                path: 'incoming/slack',
+                triggerType: 'webhook',
+              },
+            },
+          },
+          lastStartedBlock: {
+            blockId: 'block-start',
+            blockName: 'Start',
+            blockType: 'agent',
+            startedAt: '2025-01-01T00:00:00.000Z',
+          },
+          lastCompletedBlock: {
+            blockId: 'block-end',
+            blockName: 'Finish',
+            blockType: 'api',
+            endedAt: '2025-01-01T00:00:05.000Z',
+            success: true,
+          },
+        },
+        traceSpans: [],
+        finalOutput: { ok: true },
+        finalizationPath: 'completed',
+        completionFailure: 'fallback failure',
+        executionCost: {
+          tokens: { input: 0, output: 0, total: 0 },
+          models: {},
+        },
+      })
+
+      expect(completedData.environment?.workflowId).toBe('workflow-123')
+      expect(completedData.trigger?.data?.correlation).toEqual({
+        executionId: 'execution-123',
+        requestId: 'req-1234',
+        source: 'webhook',
+        workflowId: 'workflow-123',
+        webhookId: 'webhook-123',
+        path: 'incoming/slack',
+        triggerType: 'webhook',
+      })
+      expect(completedData.correlation).toEqual(completedData.trigger?.data?.correlation)
+      expect(completedData.finalOutput).toEqual({ ok: true })
+      expect(completedData.lastStartedBlock?.blockId).toBe('block-start')
+      expect(completedData.lastCompletedBlock?.blockId).toBe('block-end')
+      expect(completedData.finalizationPath).toBe('completed')
+      expect(completedData.completionFailure).toBe('fallback failure')
+      expect(completedData.hasTraceSpans).toBe(false)
+      expect(completedData.traceSpanCount).toBe(0)
+    })
+
+    test('summarizes oversized execution data before storage', () => {
+      const loggerInstance = new ExecutionLogger() as any
+      const largePayload = 'x'.repeat(1_100_000)
+      const executionState = {
+        blockStates: {
+          blockA: {
+            output: { data: largePayload },
+            executed: true,
+            executionTime: 10,
+          },
+        },
+        executedBlocks: ['blockA'],
+        blockLogs: [
+          {
+            blockId: 'blockA',
+            blockName: 'HTTP',
+            blockType: 'api',
+            startedAt: '2025-01-01T00:00:00.000Z',
+            endedAt: '2025-01-01T00:00:01.000Z',
+            durationMs: 1000,
+            success: true,
+            executionOrder: 1,
+            input: { url: 'https://example.com/image.jpg', data: largePayload },
+            output: { data: largePayload },
+          },
+        ],
+        decisions: { router: {}, condition: {} },
+        completedLoops: [],
+        activeExecutionPath: [],
+      }
+
+      const completedData = loggerInstance.buildCompletedExecutionData({
+        traceSpans: [
+          {
+            id: 'workflow-execution',
+            name: 'Workflow Execution',
+            type: 'workflow',
+            duration: 1000,
+            startTime: '2025-01-01T00:00:00.000Z',
+            endTime: '2025-01-01T00:00:01.000Z',
+            status: 'success',
+            children: [
+              {
+                id: 'blockA-1',
+                name: 'HTTP',
+                type: 'api',
+                duration: 1000,
+                startTime: '2025-01-01T00:00:00.000Z',
+                endTime: '2025-01-01T00:00:01.000Z',
+                status: 'success',
+                blockId: 'blockA',
+                executionOrder: 1,
+                input: { url: 'https://example.com/image.jpg', data: largePayload },
+                output: { data: largePayload },
+              },
+            ],
+          },
+        ],
+        finalOutput: { data: largePayload },
+        executionState,
+        finalizationPath: 'completed',
+        executionCost: {
+          tokens: { input: 0, output: 0, total: 0 },
+          models: {},
+        },
+      })
+
+      const compacted = loggerInstance.compactExecutionDataForStorage(
+        completedData,
+        'execution-oversized'
+      )
+      const storedBytes = Buffer.byteLength(JSON.stringify(compacted), 'utf8')
+
+      expect(storedBytes).toBeLessThanOrEqual(3 * 1024 * 1024)
+      expect(compacted.executionDataTruncated).toBe(true)
+      expect(compacted.executionState).toBeUndefined()
+      expect(compacted.executionStateSummary).toEqual({
+        executedBlockCount: 1,
+        blockLogCount: 1,
+        completedLoopCount: 0,
+        activeExecutionPathLength: 0,
+        pendingQueueLength: 0,
+      })
+      expect(compacted.traceSpans?.[0]?.children?.[0]?.input).toEqual({
+        _truncated: true,
+        reason: 'execution_data_size_limit',
+        originalBytes: expect.any(Number),
+        summary: 'object with 2 keys',
+      })
+    })
+  })
+
+  describe('file extraction', () => {
+    test('should extract files from trace spans with files property', () => {
+      const loggerInstance = new ExecutionLogger()
+
+      // Access the private method through the class prototype
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const traceSpans = [
+        {
+          id: 'span-1',
+          output: {
+            files: [
+              {
+                id: 'file-1',
+                name: 'test.pdf',
+                size: 1024,
+                type: 'application/pdf',
+                url: 'https://example.com/file.pdf',
+                key: 'uploads/file.pdf',
+              },
+            ],
+          },
+        },
+      ]
+
+      const files = extractFilesMethod(traceSpans, null, null)
+      expect(files).toHaveLength(1)
+      expect(files[0].name).toBe('test.pdf')
+      expect(files[0].id).toBe('file-1')
+    })
+
+    test('should extract files from attachments property', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const traceSpans = [
+        {
+          id: 'span-1',
+          output: {
+            attachments: [
+              {
+                id: 'attach-1',
+                name: 'attachment.docx',
+                size: 2048,
+                type: 'application/docx',
+                url: 'https://example.com/attach.docx',
+                key: 'attachments/attach.docx',
+              },
+            ],
+          },
+        },
+      ]
+
+      const files = extractFilesMethod(traceSpans, null, null)
+      expect(files).toHaveLength(1)
+      expect(files[0].name).toBe('attachment.docx')
+    })
+
+    test('should deduplicate files with same ID', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const duplicateFile = {
+        id: 'file-1',
+        name: 'test.pdf',
+        size: 1024,
+        type: 'application/pdf',
+        url: 'https://example.com/file.pdf',
+        key: 'uploads/file.pdf',
+      }
+
+      const traceSpans = [
+        { id: 'span-1', output: { files: [duplicateFile] } },
+        { id: 'span-2', output: { files: [duplicateFile] } },
+      ]
+
+      const files = extractFilesMethod(traceSpans, null, null)
+      expect(files).toHaveLength(1)
+    })
+
+    test('should extract files from final output', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const finalOutput = {
+        files: [
+          {
+            id: 'output-file-1',
+            name: 'output.txt',
+            size: 512,
+            type: 'text/plain',
+            url: 'https://example.com/output.txt',
+            key: 'outputs/output.txt',
+          },
+        ],
+      }
+
+      const files = extractFilesMethod([], finalOutput, null)
+      expect(files).toHaveLength(1)
+      expect(files[0].name).toBe('output.txt')
+    })
+
+    test('should extract files from workflow input', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const workflowInput = {
+        files: [
+          {
+            id: 'input-file-1',
+            name: 'input.csv',
+            size: 256,
+            type: 'text/csv',
+            url: 'https://example.com/input.csv',
+            key: 'inputs/input.csv',
+          },
+        ],
+      }
+
+      const files = extractFilesMethod([], null, workflowInput)
+      expect(files).toHaveLength(1)
+      expect(files[0].name).toBe('input.csv')
+    })
+
+    test('should handle empty inputs', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const files = extractFilesMethod(undefined, undefined, undefined)
+      expect(files).toHaveLength(0)
+    })
+
+    test('should handle deeply nested file objects', () => {
+      const loggerInstance = new ExecutionLogger()
+      const extractFilesMethod = (loggerInstance as any).extractFilesFromExecution.bind(
+        loggerInstance
+      )
+
+      const traceSpans = [
+        {
+          id: 'span-1',
+          output: {
+            nested: {
+              deeply: {
+                files: [
+                  {
+                    id: 'nested-file-1',
+                    name: 'nested.json',
+                    size: 128,
+                    type: 'application/json',
+                    url: 'https://example.com/nested.json',
+                    key: 'nested/file.json',
+                  },
+                ],
+              },
+            },
+          },
+        },
+      ]
+
+      const files = extractFilesMethod(traceSpans, null, null)
+      expect(files).toHaveLength(1)
+      expect(files[0].name).toBe('nested.json')
+    })
+  })
+})
+
+describe('recordExecutionUsage boundary-delta reconciliation', () => {
+  let logger: any
+
+  beforeEach(() => {
+    logger = new ExecutionLogger() as any
+    vi.clearAllMocks()
+  })
+
+  const costSummary = (overrides: Record<string, unknown> = {}) => ({
+    totalCost: 0,
+    totalInputCost: 0,
+    totalOutputCost: 0,
+    totalTokens: 0,
+    totalPromptTokens: 0,
+    totalCompletionTokens: 0,
+    baseExecutionCharge: 0.005,
+    models: {},
+    charges: {},
+    ...overrides,
+  })
+
+  // db.select() is called twice in recordExecutionUsage: first the workflow row
+  // (terminated by .limit), then the already-billed usage_log rows (terminated
+  // by .groupBy). Return each in order.
+  const mockDb = (billedRows: Array<Record<string, unknown>>) => {
+    let call = 0
+    dbSelectMock.mockImplementation(() => {
+      call += 1
+      const rows = call === 1 ? [{ id: 'workflow-1', workspaceId: 'ws-1' }] : billedRows
+      const chain: any = {
+        from: () => chain,
+        where: () => chain,
+        limit: () => Promise.resolve(rows),
+        groupBy: () => Promise.resolve(rows),
+      }
+      return chain
+    })
+  }
+
+  const run = (
+    summary: ReturnType<typeof costSummary>,
+    billedRows: Array<Record<string, unknown>>
+  ) => {
+    mockDb(billedRows)
+    return logger.recordExecutionUsage('workflow-1', summary, 'api', 'exec-1', 'user-1')
+  }
+
+  const lastEntries = () => vi.mocked(recordUsage).mock.calls[0][0].entries
+
+  test('fresh completion records all targets (base fee + model) and returns the increment', async () => {
+    const recorded = await run(
+      costSummary({
+        models: {
+          'gpt-4o': {
+            total: 1,
+            input: 0.6,
+            output: 0.4,
+            tokens: { input: 10, output: 5, total: 15 },
+          },
+        },
+      }),
+      []
+    )
+
+    expect(recordUsage).toHaveBeenCalledTimes(1)
+    expect(lastEntries()).toEqual([
+      expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
+      expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 1 }),
+    ])
+    // Returns the amount recorded at this boundary (drives threshold-email math).
+    expect(recorded).toBeCloseTo(1.005, 8)
+    // cost_total is refined to the exact ledger sum inside the locked tx.
+    expect(txUpdateMock).toHaveBeenCalledTimes(1)
+  })
+
+  test('resume records only the increment over what is already billed', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': {
+            total: 3,
+            input: 1.8,
+            output: 1.2,
+            tokens: { input: 30, output: 15, total: 45 },
+          },
+        },
+      }),
+      [
+        { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+        { category: 'model', description: 'gpt-4o', cost: '1' },
+      ]
+    )
+
+    expect(recordUsage).toHaveBeenCalledTimes(1)
+    const entries = lastEntries()
+    expect(entries).toHaveLength(1)
+    expect(entries[0]).toEqual(
+      expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 2 })
+    )
+  })
+
+  test('returns only the post-resume increment (not the cumulative total)', async () => {
+    const recorded = await run(
+      costSummary({
+        models: {
+          'gpt-4o': {
+            total: 3,
+            input: 0,
+            output: 0,
+            tokens: { input: 0, output: 0, total: 0 },
+          },
+        },
+      }),
+      [
+        { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+        { category: 'model', description: 'gpt-4o', cost: '1' },
+      ]
+    )
+    // Cumulative is 3.005, but only the $2 increment was recorded here — the
+    // threshold-email math must not re-count the pre-pause $1.005.
+    expect(recorded).toBe(2)
+  })
+
+  test('forwards a pre-resolved billing context to recordUsage (skips re-lookup)', async () => {
+    mockDb([])
+    const billingContext = {
+      billingEntity: { type: 'user' as const, id: 'user-1' },
+      billingPeriod: { start: new Date('2026-01-01'), end: new Date('2026-02-01') },
+    }
+    await (logger as any).recordExecutionUsage(
+      'workflow-1',
+      costSummary({
+        models: {
+          'gpt-4o': {
+            total: 1,
+            input: 0,
+            output: 0,
+            tokens: { input: 0, output: 0, total: 0 },
+          },
+        },
+      }),
+      'api',
+      'exec-1',
+      'user-1',
+      billingContext
+    )
+    expect(vi.mocked(recordUsage).mock.calls[0][0]).toMatchObject({
+      billingEntity: { type: 'user', id: 'user-1' },
+      billingPeriod: billingContext.billingPeriod,
+    })
+  })
+
+  test('retry with everything already billed records nothing (idempotent)', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': {
+            total: 1,
+            input: 0.6,
+            output: 0.4,
+            tokens: { input: 10, output: 5, total: 15 },
+          },
+        },
+      }),
+      [
+        { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+        { category: 'model', description: 'gpt-4o', cost: '1' },
+      ]
+    )
+
+    expect(recordUsage).not.toHaveBeenCalled()
+  })
+
+  test('BYOK run records only the base fee (no zero-cost model rows)', async () => {
+    await run(costSummary({ models: {}, charges: {} }), [])
+
+    expect(recordUsage).toHaveBeenCalledTimes(1)
+    expect(lastEntries()).toEqual([
+      expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
+    ])
+  })
+
+  test('standalone hosted-tool charge reconciles as a tool row', async () => {
+    await run(costSummary({ charges: { 'Exa Search': { total: 0.02 } } }), [
+      { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+    ])
+
+    expect(lastEntries()).toEqual([
+      expect.objectContaining({ category: 'tool', description: 'Exa Search', cost: 0.02 }),
+    ])
+  })
+
+  test('two boundaries (pause then resume) bill the full run exactly once', async () => {
+    const model = (total: number) => ({
+      'gpt-4o': {
+        input: total * 0.6,
+        output: total * 0.4,
+        total,
+        tokens: { input: 10, output: 5, total: 15 },
+      },
+    })
+
+    // Boundary 1 (pause): nothing billed yet, partial cost.
+    await run(costSummary({ models: model(1) }), [])
+    const firstEntries = vi.mocked(recordUsage).mock.calls[0][0].entries
+
+    // Feed boundary 1's rows back as already-billed for boundary 2.
+    const billedAfterFirst = firstEntries.map((e: any) => ({
+      category: e.category,
+      description: e.description,
+      cost: String(e.cost),
+    }))
+
+    // Boundary 2 (resume terminal): same model, higher cumulative cost.
+    await run(costSummary({ models: model(3) }), billedAfterFirst)
+    const secondEntries = vi.mocked(recordUsage).mock.calls[1][0].entries
+
+    const ledgerTotal = [...firstEntries, ...secondEntries].reduce(
+      (sum: number, e: any) => sum + e.cost,
+      0
+    )
+    expect(ledgerTotal).toBeCloseTo(3.005, 8) // base 0.005 once + gpt-4o 3 total
+    // Base fee billed once (boundary 1 only); model increment only at boundary 2.
+    expect(firstEntries.some((e: any) => e.category === 'fixed')).toBe(true)
+    expect(secondEntries.some((e: any) => e.category === 'fixed')).toBe(false)
+    expect(secondEntries).toEqual([
+      expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 2 }),
+    ])
+  })
+
+  test('eventKey is scoped by billedBefore so cross-boundary increments do not collide', async () => {
+    const model = (total: number) => ({
+      'gpt-4o': { input: 0, output: 0, total, tokens: { input: 0, output: 0, total: 0 } },
+    })
+
+    await run(costSummary({ models: model(1) }), [])
+    const key0 = vi
+      .mocked(recordUsage)
+      .mock.calls[0][0].entries.find((e: any) => e.category === 'model')?.eventKey
+
+    await run(costSummary({ models: model(3) }), [
+      { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+      { category: 'model', description: 'gpt-4o', cost: '1' },
+    ])
+    const key1 = vi
+      .mocked(recordUsage)
+      .mock.calls[1][0].entries.find((e: any) => e.category === 'model')?.eventKey
+
+    expect(key0).toContain('"billedBefore":"0.00000000"')
+    expect(key1).toContain('"billedBefore":"1.00000000"')
+    expect(key0).not.toEqual(key1)
+  })
+
+  test('a decreased cumulative cost (negative delta) records nothing for that line', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': { input: 0, output: 0, total: 3, tokens: { input: 0, output: 0, total: 0 } },
+        },
+      }),
+      [
+        { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+        { category: 'model', description: 'gpt-4o', cost: '5' },
+      ]
+    )
+    expect(recordUsage).not.toHaveBeenCalled()
+  })
+
+  test('a model introduced only post-resume is billed in full; the already-billed model is skipped', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': { input: 0, output: 0, total: 1, tokens: { input: 0, output: 0, total: 0 } },
+          'claude-3': { input: 0, output: 0, total: 2, tokens: { input: 0, output: 0, total: 0 } },
+        },
+      }),
+      [
+        { category: 'fixed', description: 'execution_fee', cost: '0.005' },
+        { category: 'model', description: 'gpt-4o', cost: '1' },
+      ]
+    )
+    expect(lastEntries()).toEqual([
+      expect.objectContaining({ category: 'model', description: 'claude-3', cost: 2 }),
+    ])
+  })
+
+  test('zero-cost models and charges (BYOK) are filtered out, leaving only the base fee', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': { input: 0, output: 0, total: 0, tokens: { input: 0, output: 0, total: 0 } },
+        },
+        charges: { Exa: { total: 0 } },
+      }),
+      []
+    )
+    expect(lastEntries()).toEqual([
+      expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
+    ])
+  })
+
+  test('reconciles inside a transaction holding a per-execution advisory lock', async () => {
+    await run(
+      costSummary({
+        models: {
+          'gpt-4o': { input: 0, output: 0, total: 1, tokens: { input: 0, output: 0, total: 0 } },
+        },
+      }),
+      []
+    )
+
+    // set_config('lock_timeout') + pg_advisory_xact_lock both run on the tx.
+    expect(dbExecuteMock).toHaveBeenCalledTimes(2)
+    expect(recordUsage).toHaveBeenCalledTimes(1)
+    // The ledger INSERT participates in the locked transaction.
+    expect(vi.mocked(recordUsage).mock.calls[0][0]).toHaveProperty('tx')
+  })
+})

apps/sim/lib/logs/execution/logging-factory.test.ts

@@ -0,0 +1,715 @@
+import { workflowsPersistenceUtilsMock, workflowsPersistenceUtilsMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, test, vi } from 'vitest'
+import {
+  calculateCostSummary,
+  createEnvironmentObject,
+  createTriggerObject,
+} from '@/lib/logs/execution/logging-factory'
+
+/** Mock the billing constants */
+vi.mock('@/lib/billing/constants', () => ({
+  BASE_EXECUTION_CHARGE: 0.005,
+}))
+
+vi.mock('@/lib/workflows/persistence/utils', () => workflowsPersistenceUtilsMock)
+
+beforeEach(() => {
+  workflowsPersistenceUtilsMockFns.mockLoadDeployedWorkflowState.mockResolvedValue({
+    blocks: {},
+    edges: [],
+    loops: {},
+    parallels: {},
+  })
+  workflowsPersistenceUtilsMockFns.mockLoadWorkflowFromNormalizedTables.mockResolvedValue({
+    blocks: {},
+    edges: [],
+    loops: {},
+    parallels: {},
+  })
+})
+
+describe('createTriggerObject', () => {
+  test('should create a trigger object with basic type', () => {
+    const trigger = createTriggerObject('manual')
+
+    expect(trigger.type).toBe('manual')
+    expect(trigger.source).toBe('manual')
+    expect(trigger.timestamp).toBeDefined()
+    expect(new Date(trigger.timestamp).getTime()).not.toBeNaN()
+  })
+
+  test('should create a trigger object for api type', () => {
+    const trigger = createTriggerObject('api')
+
+    expect(trigger.type).toBe('api')
+    expect(trigger.source).toBe('api')
+  })
+
+  test('should create a trigger object for webhook type', () => {
+    const trigger = createTriggerObject('webhook')
+
+    expect(trigger.type).toBe('webhook')
+    expect(trigger.source).toBe('webhook')
+  })
+
+  test('should create a trigger object for schedule type', () => {
+    const trigger = createTriggerObject('schedule')
+
+    expect(trigger.type).toBe('schedule')
+    expect(trigger.source).toBe('schedule')
+  })
+
+  test('should create a trigger object for chat type', () => {
+    const trigger = createTriggerObject('chat')
+
+    expect(trigger.type).toBe('chat')
+    expect(trigger.source).toBe('chat')
+  })
+
+  test('should include additional data when provided', () => {
+    const additionalData = {
+      requestId: 'req-123',
+      headers: { 'x-custom': 'value' },
+    }
+
+    const trigger = createTriggerObject('api', additionalData)
+
+    expect(trigger.type).toBe('api')
+    expect(trigger.data).toEqual(additionalData)
+  })
+
+  test('should not include data property when additionalData is undefined', () => {
+    const trigger = createTriggerObject('manual')
+
+    expect(trigger.data).toBeUndefined()
+  })
+
+  test('should not include data property when additionalData is empty', () => {
+    const trigger = createTriggerObject('manual', undefined)
+
+    expect(trigger.data).toBeUndefined()
+  })
+})
+
+describe('createEnvironmentObject', () => {
+  test('should create an environment object with all fields', () => {
+    const env = createEnvironmentObject(
+      'workflow-123',
+      'execution-456',
+      'user-789',
+      'workspace-abc',
+      { API_KEY: 'secret', DEBUG: 'true' }
+    )
+
+    expect(env.workflowId).toBe('workflow-123')
+    expect(env.executionId).toBe('execution-456')
+    expect(env.userId).toBe('user-789')
+    expect(env.workspaceId).toBe('workspace-abc')
+    expect(env.variables).toEqual({ API_KEY: 'secret', DEBUG: 'true' })
+  })
+
+  test('should use empty string for optional userId', () => {
+    const env = createEnvironmentObject('workflow-123', 'execution-456')
+
+    expect(env.userId).toBe('')
+  })
+
+  test('should use empty string for optional workspaceId', () => {
+    const env = createEnvironmentObject('workflow-123', 'execution-456', 'user-789')
+
+    expect(env.workspaceId).toBe('')
+  })
+
+  test('should use empty object for optional variables', () => {
+    const env = createEnvironmentObject(
+      'workflow-123',
+      'execution-456',
+      'user-789',
+      'workspace-abc'
+    )
+
+    expect(env.variables).toEqual({})
+  })
+
+  test('should handle all optional parameters as undefined', () => {
+    const env = createEnvironmentObject('workflow-123', 'execution-456')
+
+    expect(env.workflowId).toBe('workflow-123')
+    expect(env.executionId).toBe('execution-456')
+    expect(env.userId).toBe('')
+    expect(env.workspaceId).toBe('')
+    expect(env.variables).toEqual({})
+  })
+})
+
+describe('calculateCostSummary', () => {
+  const BASE_EXECUTION_CHARGE = 0.005
+
+  test('should return base execution charge for empty trace spans', () => {
+    const result = calculateCostSummary([])
+
+    expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
+    expect(result.baseExecutionCharge).toBe(BASE_EXECUTION_CHARGE)
+    expect(result.totalInputCost).toBe(0)
+    expect(result.totalOutputCost).toBe(0)
+    expect(result.totalTokens).toBe(0)
+    expect(result.totalPromptTokens).toBe(0)
+    expect(result.totalCompletionTokens).toBe(0)
+    expect(result.models).toEqual({})
+  })
+
+  test('should return base execution charge for undefined trace spans', () => {
+    const result = calculateCostSummary(undefined as any)
+
+    expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
+  })
+
+  test('should calculate cost from single span with cost data', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        name: 'Agent Block',
+        type: 'agent',
+        model: 'gpt-4',
+        cost: {
+          input: 0.01,
+          output: 0.02,
+          total: 0.03,
+        },
+        tokens: {
+          input: 100,
+          output: 200,
+          total: 300,
+        },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
+    expect(result.totalInputCost).toBe(0.01)
+    expect(result.totalOutputCost).toBe(0.02)
+    expect(result.totalTokens).toBe(300)
+    expect(result.totalPromptTokens).toBe(100)
+    expect(result.totalCompletionTokens).toBe(200)
+    expect(result.models['gpt-4']).toBeDefined()
+    expect(result.models['gpt-4'].total).toBe(0.03)
+  })
+
+  test('should calculate cost from multiple spans', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        name: 'Agent Block 1',
+        type: 'agent',
+        model: 'gpt-4',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 100, output: 200, total: 300 },
+      },
+      {
+        id: 'span-2',
+        name: 'Agent Block 2',
+        type: 'agent',
+        model: 'gpt-3.5-turbo',
+        cost: { input: 0.001, output: 0.002, total: 0.003 },
+        tokens: { input: 50, output: 100, total: 150 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.033 + BASE_EXECUTION_CHARGE)
+    expect(result.totalInputCost).toBe(0.011)
+    expect(result.totalOutputCost).toBe(0.022)
+    expect(result.totalTokens).toBe(450)
+    expect(result.models['gpt-4']).toBeDefined()
+    expect(result.models['gpt-3.5-turbo']).toBeDefined()
+  })
+
+  test('should accumulate costs for same model across spans', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        model: 'gpt-4',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 100, output: 200, total: 300 },
+      },
+      {
+        id: 'span-2',
+        model: 'gpt-4',
+        cost: { input: 0.02, output: 0.04, total: 0.06 },
+        tokens: { input: 200, output: 400, total: 600 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.models['gpt-4'].input).toBe(0.03)
+    expect(result.models['gpt-4'].output).toBe(0.06)
+    expect(result.models['gpt-4'].total).toBe(0.09)
+    expect(result.models['gpt-4'].tokens.input).toBe(300)
+    expect(result.models['gpt-4'].tokens.output).toBe(600)
+    expect(result.models['gpt-4'].tokens.total).toBe(900)
+  })
+
+  test('should handle nested children with cost data', () => {
+    const traceSpans = [
+      {
+        id: 'parent-span',
+        name: 'Parent',
+        type: 'workflow',
+        children: [
+          {
+            id: 'child-span-1',
+            model: 'claude-3',
+            cost: { input: 0.005, output: 0.01, total: 0.015 },
+            tokens: { input: 50, output: 100, total: 150 },
+          },
+          {
+            id: 'child-span-2',
+            model: 'claude-3',
+            cost: { input: 0.005, output: 0.01, total: 0.015 },
+            tokens: { input: 50, output: 100, total: 150 },
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
+    expect(result.models['claude-3']).toBeDefined()
+    expect(result.models['claude-3'].total).toBe(0.03)
+  })
+
+  test('should handle deeply nested children', () => {
+    const traceSpans = [
+      {
+        id: 'level-1',
+        children: [
+          {
+            id: 'level-2',
+            children: [
+              {
+                id: 'level-3',
+                model: 'gpt-4',
+                cost: { input: 0.01, output: 0.02, total: 0.03 },
+                tokens: { input: 100, output: 200, total: 300 },
+              },
+            ],
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.models['gpt-4']).toBeDefined()
+  })
+
+  test('should handle prompt/completion token aliases', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        model: 'gpt-4',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { prompt: 100, completion: 200, total: 300 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalPromptTokens).toBe(100)
+    expect(result.totalCompletionTokens).toBe(200)
+  })
+
+  test('should skip spans without cost data', () => {
+    const traceSpans = [
+      {
+        id: 'span-without-cost',
+        name: 'Text Block',
+        type: 'text',
+      },
+      {
+        id: 'span-with-cost',
+        model: 'gpt-4',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 100, output: 200, total: 300 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(Object.keys(result.models)).toHaveLength(1)
+  })
+
+  test('should handle spans without model specified', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 100, output: 200, total: 300 },
+        // No model specified
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
+    // Should not add to models if model is not specified
+    expect(Object.keys(result.models)).toHaveLength(0)
+  })
+
+  test('should handle missing token fields gracefully', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        model: 'gpt-4',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        // tokens field is missing
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalTokens).toBe(0)
+    expect(result.totalPromptTokens).toBe(0)
+    expect(result.totalCompletionTokens).toBe(0)
+  })
+
+  test('should handle partial cost fields', () => {
+    const traceSpans = [
+      {
+        id: 'span-1',
+        model: 'gpt-4',
+        cost: { total: 0.03 }, // Only total specified
+        tokens: { total: 300 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
+    expect(result.totalInputCost).toBe(0)
+    expect(result.totalOutputCost).toBe(0)
+  })
+
+  test('BYOK regression: parent block cost is authoritative; model children are not double-counted', () => {
+    // Reproduces the BYOK billing leak: provider sets parent agent span's
+    // block-level cost to zero (BYOK suppression), but trace enrichers still
+    // wrote gross hosted cost into time-segment children. Before the fix this
+    // test would expect 0.03; after the fix the parent's zero is authoritative.
+    const traceSpans = [
+      {
+        id: 'agent-span',
+        type: 'agent',
+        model: 'claude-opus-4-6',
+        cost: { input: 0, output: 0, total: 0 },
+        tokens: { input: 68057, output: 1548, total: 69605 },
+        children: [
+          {
+            id: 'agent-span-segment-0',
+            type: 'model',
+            model: 'claude-opus-4-6',
+            cost: { input: 0.340285, output: 0.0387, total: 0.378985 },
+            tokens: { input: 68057, output: 1548, total: 69605 },
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
+    // Model is still tracked for token-usage display, but cost must be zero.
+    expect(result.models['claude-opus-4-6'].total).toBe(0)
+    expect(result.models['claude-opus-4-6'].input).toBe(0)
+    expect(result.models['claude-opus-4-6'].output).toBe(0)
+    expect(result.models['claude-opus-4-6'].tokens.input).toBe(68057)
+    expect(result.models['claude-opus-4-6'].tokens.output).toBe(1548)
+  })
+
+  test('non-BYOK still aggregates parent block cost correctly with model children present', () => {
+    // Same shape as the BYOK case but the parent carries the gross cost
+    // (typical hosted-key path). The parent's cost is counted once; model
+    // children are skipped to avoid double-counting.
+    const traceSpans = [
+      {
+        id: 'agent-span',
+        type: 'agent',
+        model: 'gpt-4o',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 1000, output: 2000, total: 3000 },
+        children: [
+          {
+            id: 'agent-span-segment-0',
+            type: 'model',
+            model: 'gpt-4o',
+            cost: { input: 0.01, output: 0.02, total: 0.03 },
+            tokens: { input: 1000, output: 2000, total: 3000 },
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
+    expect(result.models['gpt-4o'].total).toBe(0.03)
+  })
+
+  test('preserves parent toolCost while skipping model breakdown children', () => {
+    const traceSpans = [
+      {
+        id: 'agent-span',
+        type: 'agent',
+        model: 'gpt-4o',
+        cost: { input: 0.01, output: 0.02, toolCost: 0.015, total: 0.045 },
+        tokens: { input: 1000, output: 2000, total: 3000 },
+        children: [
+          {
+            id: 'agent-span-model-segment',
+            type: 'model',
+            model: 'gpt-4o',
+            cost: { input: 0.01, output: 0.02, total: 0.03 },
+            tokens: { input: 1000, output: 2000, total: 3000 },
+          },
+          {
+            id: 'agent-span-tool-segment',
+            type: 'tool',
+            name: 'firecrawl_scrape',
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.totalCost).toBe(0.045 + BASE_EXECUTION_CHARGE)
+    expect(result.models['gpt-4o'].total).toBe(0.045)
+    expect(result.models['gpt-4o'].toolCost).toBe(0.015)
+  })
+
+  test('records a standalone non-model billable span as a charge (closes the tool gap)', () => {
+    const traceSpans = [
+      {
+        id: 'exa-block',
+        name: 'Exa Search',
+        type: 'tool',
+        cost: { input: 0, output: 0, total: 0.01 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.charges['Exa Search']).toBeDefined()
+    expect(result.charges['Exa Search'].total).toBe(0.01)
+    expect(Object.keys(result.models)).toHaveLength(0)
+    // Ledger partition reconciles with the run total.
+    const ledgerSum =
+      result.baseExecutionCharge +
+      Object.values(result.models).reduce((s, m) => s + m.total, 0) +
+      Object.values(result.charges).reduce((s, c) => s + c.total, 0)
+    expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
+  })
+
+  test('does not double-count: agent-embedded tool stays in the model row, not charges', () => {
+    const traceSpans = [
+      {
+        id: 'agent-span',
+        name: 'Agent',
+        type: 'agent',
+        model: 'gpt-4o',
+        cost: { input: 0.01, output: 0.02, total: 0.045, toolCost: 0.015 },
+        tokens: { input: 1000, output: 2000, total: 3000 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(Object.keys(result.charges)).toHaveLength(0)
+    expect(result.models['gpt-4o'].total).toBe(0.045)
+    expect(result.models['gpt-4o'].toolCost).toBe(0.015)
+  })
+
+  test('mixed model + standalone tool run reconciles to total', () => {
+    const traceSpans = [
+      {
+        id: 'agent',
+        name: 'Agent',
+        type: 'agent',
+        model: 'gpt-4o',
+        cost: { input: 0.01, output: 0.02, total: 0.03 },
+        tokens: { input: 100, output: 200, total: 300 },
+      },
+      {
+        id: 'exa',
+        name: 'Exa Search',
+        type: 'tool',
+        cost: { input: 0, output: 0, total: 0.01 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.models['gpt-4o'].total).toBe(0.03)
+    expect(result.charges['Exa Search'].total).toBe(0.01)
+    const ledgerSum =
+      result.baseExecutionCharge +
+      Object.values(result.models).reduce((s, m) => s + m.total, 0) +
+      Object.values(result.charges).reduce((s, c) => s + c.total, 0)
+    expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
+  })
+
+  test('BYOK tool (no cost generated upstream) produces no charge row', () => {
+    const traceSpans = [
+      {
+        id: 'exa-byok',
+        name: 'Exa Search',
+        type: 'tool',
+        cost: { input: 0, output: 0, total: 0 },
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(Object.keys(result.charges)).toHaveLength(0)
+    expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
+  })
+
+  test('does not double-count the synthetic workflow root (aggregate cost over leaves)', () => {
+    // buildTraceSpans wraps every run in a synthetic { type: 'workflow' } root
+    // whose cost.total is the SUM of its leaves. Counting that root in addition
+    // to the leaves double-charges the run — the root must be a pass-through.
+    const traceSpans = [
+      {
+        id: 'workflow-execution',
+        name: 'Workflow Execution',
+        type: 'workflow',
+        cost: { total: 0.04 }, // == agent(0.03) + exa(0.01)
+        children: [
+          {
+            id: 'agent-1',
+            name: 'Agent',
+            type: 'agent',
+            model: 'gpt-4o',
+            cost: { input: 0.01, output: 0.02, total: 0.03 },
+            tokens: { input: 100, output: 200, total: 300 },
+          },
+          {
+            id: 'exa-1',
+            name: 'Exa Search',
+            type: 'tool',
+            cost: { input: 0, output: 0, total: 0.01 },
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    // The 0.04 root aggregate is NOT added on top of its leaves.
+    expect(result.charges['Workflow Execution']).toBeUndefined()
+    expect(result.models['gpt-4o'].total).toBe(0.03)
+    expect(result.charges['Exa Search'].total).toBe(0.01)
+    expect(result.totalCost).toBeCloseTo(0.04 + BASE_EXECUTION_CHARGE, 10)
+    const ledgerSum =
+      result.baseExecutionCharge +
+      Object.values(result.models).reduce((s, m) => s + m.total, 0) +
+      Object.values(result.charges).reduce((s, c) => s + c.total, 0)
+    expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
+  })
+
+  test('does not double-count nested sub-workflow roots', () => {
+    // A sub-workflow call nests another synthetic { type: 'workflow' } root
+    // (captureChildWorkflowLogs runs buildTraceSpans on the child). Both the
+    // outer root and the inner sub-workflow root carry aggregate costs; only the
+    // leaf agent inside should be billed.
+    const traceSpans = [
+      {
+        id: 'workflow-execution',
+        name: 'Workflow Execution',
+        type: 'workflow',
+        cost: { total: 0.03 },
+        children: [
+          {
+            id: 'subworkflow-root',
+            name: 'Workflow Execution',
+            type: 'workflow',
+            cost: { total: 0.03 },
+            children: [
+              {
+                id: 'child-agent',
+                name: 'Agent',
+                type: 'agent',
+                model: 'gpt-4o',
+                cost: { input: 0.01, output: 0.02, total: 0.03 },
+                tokens: { input: 100, output: 200, total: 300 },
+              },
+            ],
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.charges['Workflow Execution']).toBeUndefined()
+    expect(result.models['gpt-4o'].total).toBe(0.03)
+    expect(result.totalCost).toBeCloseTo(0.03 + BASE_EXECUTION_CHARGE, 10)
+  })
+
+  test('does not double-count deeply nested (3-level) sub-workflow roots', () => {
+    // A → B → C: each level is its own synthetic { type: 'workflow' } root with
+    // an aggregate cost. Only the leaf agents at the bottom must be billed, once.
+    const leafAgent = (id: string, model: string, total: number) => ({
+      id,
+      name: 'Agent',
+      type: 'agent',
+      model,
+      cost: { input: total / 3, output: (total * 2) / 3, total },
+      tokens: { input: 100, output: 200, total: 300 },
+    })
+
+    const traceSpans = [
+      {
+        id: 'root',
+        name: 'Workflow Execution',
+        type: 'workflow',
+        cost: { total: 0.06 }, // aggregate of everything below
+        children: [
+          leafAgent('parent-agent', 'gpt-4o', 0.02),
+          {
+            id: 'sub-a-root',
+            name: 'Workflow Execution',
+            type: 'workflow',
+            cost: { total: 0.04 },
+            children: [
+              leafAgent('a-agent', 'gpt-4o', 0.01),
+              {
+                id: 'sub-b-root',
+                name: 'Workflow Execution',
+                type: 'workflow',
+                cost: { total: 0.03 },
+                children: [leafAgent('b-agent', 'claude-sonnet-4-6', 0.03)],
+              },
+            ],
+          },
+        ],
+      },
+    ]
+
+    const result = calculateCostSummary(traceSpans)
+
+    expect(result.charges['Workflow Execution']).toBeUndefined()
+    // gpt-4o appears at two levels (0.02 + 0.01) and merges by model name.
+    expect(result.models['gpt-4o'].total).toBeCloseTo(0.03, 10)
+    expect(result.models['claude-sonnet-4-6'].total).toBeCloseTo(0.03, 10)
+    expect(result.totalCost).toBeCloseTo(0.06 + BASE_EXECUTION_CHARGE, 10)
+    const ledgerSum =
+      result.baseExecutionCharge +
+      Object.values(result.models).reduce((s, m) => s + m.total, 0) +
+      Object.values(result.charges).reduce((s, c) => s + c.total, 0)
+    expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
+  })
+})

apps/sim/lib/logs/execution/logging-factory.ts

@@ -0,0 +1,267 @@
+import { db, workflow } from '@sim/db'
+import { eq } from 'drizzle-orm'
+import { BASE_EXECUTION_CHARGE } from '@/lib/billing/constants'
+import type {
+  ExecutionEnvironment,
+  ExecutionTrigger,
+  TraceSpan,
+  WorkflowState,
+} from '@/lib/logs/types'
+import {
+  loadDeployedWorkflowState,
+  loadWorkflowFromNormalizedTables,
+} from '@/lib/workflows/persistence/utils'
+
+export function createTriggerObject(
+  type: ExecutionTrigger['type'],
+  additionalData?: Record<string, unknown>
+): ExecutionTrigger {
+  return {
+    type,
+    source: type,
+    timestamp: new Date().toISOString(),
+    ...(additionalData && { data: additionalData }),
+  }
+}
+
+export function createEnvironmentObject(
+  workflowId: string,
+  executionId: string,
+  userId?: string,
+  workspaceId?: string,
+  variables?: Record<string, string>
+): ExecutionEnvironment {
+  return {
+    variables: variables || {},
+    workflowId,
+    executionId,
+    userId: userId || '',
+    workspaceId: workspaceId || '',
+  }
+}
+
+export async function loadWorkflowStateForExecution(workflowId: string): Promise<WorkflowState> {
+  const [normalizedData, workflowRecord] = await Promise.all([
+    loadWorkflowFromNormalizedTables(workflowId),
+    db
+      .select({ variables: workflow.variables })
+      .from(workflow)
+      .where(eq(workflow.id, workflowId))
+      .limit(1)
+      .then((rows) => rows[0]),
+  ])
+
+  if (!normalizedData) {
+    throw new Error(
+      `Workflow ${workflowId} has no normalized data available. Ensure the workflow is properly saved to normalized tables.`
+    )
+  }
+
+  return {
+    blocks: normalizedData.blocks || {},
+    edges: normalizedData.edges || [],
+    loops: normalizedData.loops || {},
+    parallels: normalizedData.parallels || {},
+    variables: (workflowRecord?.variables as WorkflowState['variables']) || undefined,
+  }
+}
+
+/**
+ * Load deployed workflow state for logging purposes.
+ * This fetches the active deployment state, ensuring logs capture
+ * the exact state that was executed (not the live editor state).
+ */
+export async function loadDeployedWorkflowStateForLogging(
+  workflowId: string
+): Promise<WorkflowState> {
+  const deployedData = await loadDeployedWorkflowState(workflowId)
+
+  return {
+    blocks: deployedData.blocks || {},
+    edges: deployedData.edges || [],
+    loops: deployedData.loops || {},
+    parallels: deployedData.parallels || {},
+    variables: deployedData.variables as WorkflowState['variables'],
+  }
+}
+
+type CostTraceSpan = Pick<TraceSpan, 'cost' | 'model' | 'tokens'> & {
+  type?: TraceSpan['type']
+  name?: TraceSpan['name']
+  children?: CostTraceSpan[]
+}
+
+export interface CostSummaryModel {
+  input: number
+  output: number
+  total: number
+  toolCost?: number
+  tokens: { input: number; output: number; total: number }
+}
+
+/**
+ * Non-model billable charge (e.g. a standalone hosted-key tool block such as
+ * Exa/Tavily/falai run outside an agent). These spans contribute to the run's
+ * total cost but carry no `model`, so they live here rather than in `models`.
+ * Summed per span name so the ledger has one row per integration.
+ */
+export interface CostSummaryCharge {
+  total: number
+}
+
+export interface CostSummary {
+  totalCost: number
+  totalInputCost: number
+  totalOutputCost: number
+  totalTokens: number
+  totalPromptTokens: number
+  totalCompletionTokens: number
+  baseExecutionCharge: number
+  models: Record<string, CostSummaryModel>
+  /** Non-model billable charges keyed by span name (tool/integration costs). */
+  charges: Record<string, CostSummaryCharge>
+}
+
+type BillableTraceSpan = CostTraceSpan & { cost: NonNullable<TraceSpan['cost']> }
+
+function hasBillableCost(span: CostTraceSpan): span is BillableTraceSpan {
+  return span.cost !== undefined
+}
+
+function isModelBreakdownSpan(span: CostTraceSpan): boolean {
+  return span.type === 'model'
+}
+
+export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): CostSummary {
+  if (!traceSpans || traceSpans.length === 0) {
+    return {
+      totalCost: BASE_EXECUTION_CHARGE,
+      totalInputCost: 0,
+      totalOutputCost: 0,
+      totalTokens: 0,
+      totalPromptTokens: 0,
+      totalCompletionTokens: 0,
+      baseExecutionCharge: BASE_EXECUTION_CHARGE,
+      models: {},
+      charges: {},
+    }
+  }
+
+  /**
+   * Collects spans that contribute to the execution's billable cost.
+   *
+   * Rule: when a span has its own `cost` AND has child model segments, the
+   * parent's block-level cost is authoritative — skip the model children to
+   * avoid double-counting. The parent cost is set by the provider response
+   * (and is correctly zeroed by `executeProviderRequest` for BYOK calls);
+   * model children only carry per-segment cost from the trace enrichers,
+   * which is unaware of BYOK status. Non-model children are still visited
+   * so standalone nested costs remain billable.
+   *
+   * Spans without their own `cost` (e.g. parent workflow spans for
+   * subworkflow blocks) still recurse so nested billable spans are counted.
+   */
+  const collectCostSpans = (spans: CostTraceSpan[]): BillableTraceSpan[] => {
+    const costSpans: BillableTraceSpan[] = []
+
+    for (const span of spans) {
+      // `workflow`-typed spans are aggregate containers, not billable units: the
+      // synthetic "Workflow Execution" root (added to every run by
+      // buildTraceSpans) and any nested sub-workflow root carry a `cost.total`
+      // equal to the SUM of their descendants. Counting that aggregate in
+      // addition to the descendants double-charges the run, so treat these as
+      // pass-through: never count their own cost, always recurse into all
+      // children where the real billable leaves (agents, tools) live.
+      const isAggregateContainer = span.type === 'workflow'
+      const hasOwnCost = hasBillableCost(span)
+      const countOwnCost = hasOwnCost && !isAggregateContainer
+
+      if (countOwnCost) {
+        costSpans.push(span)
+      }
+
+      if (span.children && Array.isArray(span.children)) {
+        if (countOwnCost) {
+          // Authoritative leaf (e.g. an agent block whose block-level cost is set
+          // by the provider response and already accounts for its model
+          // segments): only recurse into non-model children to find further
+          // standalone billable units, skipping the model-breakdown duplicates.
+          const nonModelChildren = span.children.filter((child) => !isModelBreakdownSpan(child))
+          costSpans.push(...collectCostSpans(nonModelChildren))
+        } else {
+          // Container (workflow / sub-workflow root) or a no-cost parent: recurse
+          // into everything so nested billable leaves are counted exactly once.
+          costSpans.push(...collectCostSpans(span.children))
+        }
+      }
+    }
+
+    return costSpans
+  }
+
+  const costSpans = collectCostSpans(traceSpans)
+
+  let totalCost = 0
+  let totalInputCost = 0
+  let totalOutputCost = 0
+  let totalTokens = 0
+  let totalPromptTokens = 0
+  let totalCompletionTokens = 0
+  const models: Record<string, CostSummaryModel> = {}
+  const charges: Record<string, CostSummaryCharge> = {}
+
+  for (const span of costSpans) {
+    totalCost += span.cost.total || 0
+    totalInputCost += span.cost.input || 0
+    totalOutputCost += span.cost.output || 0
+    totalTokens += span.tokens?.total || 0
+    totalPromptTokens += span.tokens?.input ?? span.tokens?.prompt ?? 0
+    totalCompletionTokens += span.tokens?.output ?? span.tokens?.completion ?? 0
+
+    if (span.model) {
+      const model = span.model
+      if (!models[model]) {
+        models[model] = {
+          input: 0,
+          output: 0,
+          total: 0,
+          tokens: { input: 0, output: 0, total: 0 },
+        }
+      }
+      models[model].input += span.cost.input || 0
+      models[model].output += span.cost.output || 0
+      models[model].total += span.cost.total || 0
+      models[model].tokens.input += span.tokens?.input ?? span.tokens?.prompt ?? 0
+      models[model].tokens.output += span.tokens?.output ?? span.tokens?.completion ?? 0
+      models[model].tokens.total += span.tokens?.total || 0
+
+      if (span.cost.toolCost) {
+        models[model].toolCost = (models[model].toolCost || 0) + span.cost.toolCost
+      }
+    } else if ((span.cost.total || 0) > 0) {
+      // Non-model billable span (e.g. a standalone hosted-key tool block).
+      // These previously contributed to the run total but were never itemized
+      // in the ledger (the "standalone tool gap"). Key by span name so each
+      // integration gets a single, reconciling charge row.
+      const description = span.name || span.type || 'tool'
+      if (!charges[description]) {
+        charges[description] = { total: 0 }
+      }
+      charges[description].total += span.cost.total || 0
+    }
+  }
+
+  totalCost += BASE_EXECUTION_CHARGE
+
+  return {
+    totalCost,
+    totalInputCost,
+    totalOutputCost,
+    totalTokens,
+    totalPromptTokens,
+    totalCompletionTokens,
+    baseExecutionCharge: BASE_EXECUTION_CHARGE,
+    models,
+    charges,
+  }
+}

apps/sim/lib/logs/execution/logging-session.test.ts

@@ -0,0 +1,747 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const dbMocks = vi.hoisted(() => {
+  const selectLimit = vi.fn()
+  const selectWhere = vi.fn()
+  const selectFrom = vi.fn()
+  const select = vi.fn()
+  const updateWhere = vi.fn()
+  const updateSet = vi.fn()
+  const update = vi.fn()
+  const execute = vi.fn()
+  const eq = vi.fn()
+  const and = vi.fn((...args: unknown[]) => ({ type: 'and', args }))
+  const sql = vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values }))
+
+  select.mockReturnValue({ from: selectFrom })
+  selectFrom.mockReturnValue({ where: selectWhere })
+  selectWhere.mockReturnValue({ limit: selectLimit })
+
+  update.mockReturnValue({ set: updateSet })
+  updateSet.mockReturnValue({ where: updateWhere })
+
+  return {
+    select,
+    selectFrom,
+    selectWhere,
+    selectLimit,
+    update,
+    updateSet,
+    updateWhere,
+    execute,
+    eq,
+    and,
+    sql,
+  }
+})
+
+const {
+  completeWorkflowExecutionMock,
+  startWorkflowExecutionMock,
+  loadWorkflowStateForExecutionMock,
+} = vi.hoisted(() => ({
+  completeWorkflowExecutionMock: vi.fn(),
+  startWorkflowExecutionMock: vi.fn(),
+  loadWorkflowStateForExecutionMock: vi.fn(),
+}))
+
+vi.mock('@sim/db', () => ({
+  db: {
+    select: dbMocks.select,
+    update: dbMocks.update,
+    execute: dbMocks.execute,
+  },
+}))
+
+vi.mock('drizzle-orm', () => ({
+  eq: dbMocks.eq,
+  and: dbMocks.and,
+  sql: dbMocks.sql,
+}))
+
+vi.mock('@/lib/logs/execution/logger', () => ({
+  executionLogger: {
+    startWorkflowExecution: startWorkflowExecutionMock,
+    completeWorkflowExecution: completeWorkflowExecutionMock,
+  },
+}))
+
+const {
+  setLastStartedBlockMock,
+  setLastCompletedBlockMock,
+  getProgressMarkersMock,
+  clearProgressMarkersMock,
+} = vi.hoisted(() => ({
+  setLastStartedBlockMock: vi.fn().mockResolvedValue(false),
+  setLastCompletedBlockMock: vi.fn().mockResolvedValue(false),
+  getProgressMarkersMock: vi.fn().mockResolvedValue({}),
+  clearProgressMarkersMock: vi.fn().mockResolvedValue(undefined),
+}))
+
+vi.mock('@/lib/logs/execution/progress-markers', () => ({
+  setLastStartedBlock: setLastStartedBlockMock,
+  setLastCompletedBlock: setLastCompletedBlockMock,
+  getProgressMarkers: getProgressMarkersMock,
+  clearProgressMarkers: clearProgressMarkersMock,
+}))
+
+vi.mock('@/lib/logs/execution/logging-factory', () => ({
+  calculateCostSummary: vi.fn().mockReturnValue({
+    totalCost: 0,
+    totalInputCost: 0,
+    totalOutputCost: 0,
+    totalTokens: 0,
+    totalPromptTokens: 0,
+    totalCompletionTokens: 0,
+    baseExecutionCharge: 0,
+    models: {},
+  }),
+  createEnvironmentObject: vi.fn(),
+  createTriggerObject: vi.fn(),
+  loadDeployedWorkflowStateForLogging: vi.fn(),
+  loadWorkflowStateForExecution: loadWorkflowStateForExecutionMock,
+}))
+
+import { calculateCostSummary } from '@/lib/logs/execution/logging-factory'
+import { LoggingSession } from './logging-session'
+
+describe('LoggingSession start snapshots', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    startWorkflowExecutionMock.mockResolvedValue({})
+    loadWorkflowStateForExecutionMock.mockResolvedValue({
+      blocks: {
+        stale: {
+          id: 'stale',
+          type: 'function',
+          name: 'Stale',
+          position: { x: 0, y: 0 },
+          subBlocks: {},
+          outputs: {},
+          enabled: true,
+        },
+      },
+      edges: [],
+      loops: {},
+      parallels: {},
+    })
+  })
+
+  it('uses the executed workflow state override for execution snapshots', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'manual', 'req-1')
+    const executedWorkflowState = {
+      blocks: {
+        loop: {
+          id: 'loop',
+          type: 'loop',
+          name: 'Loop',
+          position: { x: 0, y: 0 },
+          subBlocks: {},
+          outputs: {},
+          enabled: true,
+        },
+        parallel: {
+          id: 'parallel',
+          type: 'parallel',
+          name: 'Parallel',
+          position: { x: 100, y: 80 },
+          subBlocks: {},
+          outputs: {},
+          enabled: true,
+          data: { parentId: 'loop', extent: 'parent' as const },
+        },
+      },
+      edges: [],
+      loops: { loop: { id: 'loop', nodes: ['parallel'], iterations: 1, loopType: 'for' as const } },
+      parallels: { parallel: { id: 'parallel', nodes: [], count: 1 } },
+    }
+
+    await session.start({
+      workspaceId: 'workspace-1',
+      workflowState: executedWorkflowState,
+    })
+
+    expect(loadWorkflowStateForExecutionMock).not.toHaveBeenCalled()
+    expect(startWorkflowExecutionMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        workflowState: executedWorkflowState,
+      })
+    )
+  })
+})
+
+describe('LoggingSession completion retries', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    dbMocks.selectLimit.mockResolvedValue([{ executionData: {} }])
+    dbMocks.updateWhere.mockResolvedValue(undefined)
+    dbMocks.execute.mockResolvedValue(undefined)
+  })
+
+  it('keeps completion best-effort when a later error completion retries after full completion and fallback both fail', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('success finalize failed'))
+      .mockRejectedValueOnce(new Error('cost only failed'))
+      .mockResolvedValueOnce({})
+
+    await expect(session.safeComplete({ finalOutput: { ok: true } })).resolves.toBeUndefined()
+
+    await expect(
+      session.safeCompleteWithError({
+        error: { message: 'fallback error finalize' },
+      })
+    ).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(3)
+    expect(session.hasCompleted()).toBe(true)
+  })
+
+  it('reuses the settled completion promise for repeated completion attempts', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('success finalize failed'))
+      .mockRejectedValueOnce(new Error('cost only failed'))
+
+    await expect(session.safeComplete({ finalOutput: { ok: true } })).resolves.toBeUndefined()
+    await expect(session.safeComplete({ finalOutput: { ok: true } })).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(2)
+  })
+
+  it('starts a new error completion attempt after a non-error completion and fallback both fail', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-3', 'api', 'req-1')
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('success finalize failed'))
+      .mockRejectedValueOnce(new Error('cost only failed'))
+      .mockResolvedValueOnce({})
+
+    await expect(session.safeComplete({ finalOutput: { ok: true } })).resolves.toBeUndefined()
+
+    await expect(
+      session.safeCompleteWithError({
+        error: { message: 'late error finalize' },
+      })
+    ).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(3)
+    expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        executionId: 'execution-3',
+        finalOutput: { error: 'late error finalize' },
+      })
+    )
+    expect(session.hasCompleted()).toBe(true)
+  })
+
+  it('preserves successful final output during fallback completion', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-5', 'api', 'req-1')
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('success finalize failed'))
+      .mockResolvedValueOnce({})
+
+    await expect(
+      session.safeComplete({ finalOutput: { ok: true, stage: 'done' } })
+    ).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        executionId: 'execution-5',
+        finalOutput: { ok: true, stage: 'done' },
+        finalizationPath: 'fallback_completed',
+      })
+    )
+  })
+
+  it('derives fallback cost from trace spans when the primary completion fails', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-6', 'api', 'req-1') as any
+
+    // Resume-accumulation is retired: the cost-only fallback now derives its
+    // cost summary from the in-memory trace spans (billing itself reconciles
+    // from the usage_log ledger in recordExecutionUsage). The primary complete()
+    // path consumes one calculateCostSummary call before it fails, so queue the
+    // same value twice (primary attempt + fallback).
+    const spanCostSummary = {
+      totalCost: 12,
+      totalInputCost: 5,
+      totalOutputCost: 7,
+      totalTokens: 24,
+      totalPromptTokens: 11,
+      totalCompletionTokens: 13,
+      baseExecutionCharge: 0,
+      models: {},
+      charges: {},
+    }
+    vi.mocked(calculateCostSummary)
+      .mockReturnValueOnce(spanCostSummary)
+      .mockReturnValueOnce(spanCostSummary)
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('success finalize failed'))
+      .mockResolvedValueOnce({})
+
+    const traceSpans = [
+      {
+        id: 'span-1',
+        name: 'Block A',
+        type: 'tool',
+        duration: 25,
+        startTime: '2026-03-13T10:00:00.000Z',
+        endTime: '2026-03-13T10:00:00.025Z',
+        status: 'success',
+      },
+    ] as any
+
+    await expect(
+      session.safeComplete({ finalOutput: { ok: true }, traceSpans })
+    ).resolves.toBeUndefined()
+
+    expect(calculateCostSummary).toHaveBeenLastCalledWith(traceSpans)
+    expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith(
+      expect.objectContaining({
+        executionId: 'execution-6',
+        finalizationPath: 'fallback_completed',
+        costSummary: expect.objectContaining({
+          totalCost: 12,
+          totalInputCost: 5,
+          totalOutputCost: 7,
+          totalTokens: 24,
+        }),
+      })
+    )
+  })
+
+  it('persists failed error semantics when completeWithError receives non-error trace spans', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-4', 'api', 'req-1')
+    const traceSpans = [
+      {
+        id: 'span-1',
+        name: 'Block A',
+        type: 'tool',
+        duration: 25,
+        startTime: '2026-03-13T10:00:00.000Z',
+        endTime: '2026-03-13T10:00:00.025Z',
+        status: 'success',
+      },
+    ]
+
+    completeWorkflowExecutionMock.mockResolvedValue({})
+
+    await expect(
+      session.safeCompleteWithError({
+        error: { message: 'persist me as failed' },
+        traceSpans,
+      })
+    ).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        executionId: 'execution-4',
+        finalOutput: { error: 'persist me as failed' },
+        traceSpans,
+        level: 'error',
+        status: 'failed',
+        finalizationPath: 'force_failed',
+        completionFailure: 'persist me as failed',
+      })
+    )
+  })
+
+  it('marks paused completions as completed and deduplicates later attempts', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    completeWorkflowExecutionMock.mockResolvedValue({})
+
+    await expect(
+      session.safeCompleteWithPause({
+        endedAt: new Date().toISOString(),
+        totalDurationMs: 10,
+        traceSpans: [],
+        workflowInput: { hello: 'world' },
+      })
+    ).resolves.toBeUndefined()
+
+    expect(session.hasCompleted()).toBe(true)
+
+    await expect(
+      session.safeCompleteWithError({
+        error: { message: 'should be ignored' },
+      })
+    ).resolves.toBeUndefined()
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('falls back to cost-only logging when paused completion fails', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-2', 'api', 'req-1')
+
+    completeWorkflowExecutionMock
+      .mockRejectedValueOnce(new Error('pause finalize failed'))
+      .mockResolvedValueOnce({})
+
+    await expect(
+      session.safeCompleteWithPause({
+        endedAt: new Date().toISOString(),
+        totalDurationMs: 10,
+        traceSpans: [],
+        workflowInput: { hello: 'world' },
+      })
+    ).resolves.toBeUndefined()
+
+    expect(session.hasCompleted()).toBe(true)
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledTimes(2)
+  })
+
+  it('persists last started block independently from cost accumulation', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.onBlockStart('block-1', 'Fetch', 'api', '2025-01-01T00:00:00.000Z')
+
+    expect(dbMocks.select).not.toHaveBeenCalled()
+    expect(dbMocks.execute).toHaveBeenCalledTimes(1)
+  })
+
+  it('enforces started marker monotonicity in the database write path', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.onBlockStart('block-1', 'Fetch', 'api', '2025-01-01T00:00:00.000Z')
+
+    expect(dbMocks.sql).toHaveBeenCalled()
+    expect(dbMocks.execute).toHaveBeenCalledTimes(1)
+  })
+
+  it('allows same-millisecond started markers to replace the prior marker', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.onBlockStart('block-1', 'Fetch', 'api', '2025-01-01T00:00:00.000Z')
+
+    const queryCall = dbMocks.sql.mock.calls.at(-1)
+    expect(queryCall).toBeDefined()
+
+    const [query] = queryCall!
+    expect(Array.from(query).join(' ')).toContain('<=')
+  })
+
+  it('persists last completed block for zero-cost outputs', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.onBlockComplete('block-2', 'Transform', 'function', {
+      endedAt: '2025-01-01T00:00:01.000Z',
+      output: { value: true },
+    })
+
+    expect(dbMocks.select).not.toHaveBeenCalled()
+    expect(dbMocks.execute).toHaveBeenCalledTimes(1)
+  })
+
+  it('allows same-millisecond completed markers to replace the prior marker', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.onBlockComplete('block-2', 'Transform', 'function', {
+      endedAt: '2025-01-01T00:00:01.000Z',
+      output: { value: true },
+    })
+
+    const queryCall = dbMocks.sql.mock.calls.at(-1)
+    expect(queryCall).toBeDefined()
+
+    const [query] = queryCall!
+    expect(Array.from(query).join(' ')).toContain('<=')
+  })
+
+  it('drains pending lifecycle writes before terminal completion', async () => {
+    let releasePersist: (() => void) | undefined
+    const persistPromise = new Promise<void>((resolve) => {
+      releasePersist = resolve
+    })
+
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') as any
+    session.persistLastStartedBlock = vi.fn(() => persistPromise)
+    session.complete = vi.fn().mockResolvedValue(undefined)
+
+    const startPromise = session.onBlockStart('block-1', 'Fetch', 'api', '2025-01-01T00:00:00.000Z')
+    const completionPromise = session.safeComplete({ finalOutput: { ok: true } })
+
+    await Promise.resolve()
+
+    expect(session.complete).not.toHaveBeenCalled()
+
+    releasePersist?.()
+
+    await startPromise
+    await completionPromise
+
+    expect(session.persistLastStartedBlock).toHaveBeenCalledTimes(1)
+    expect(session.complete).toHaveBeenCalledTimes(1)
+  })
+
+  it('drains fire-and-forget block-complete marker writes before terminal completion', async () => {
+    let releasePersist: (() => void) | undefined
+    const persistPromise = new Promise<void>((resolve) => {
+      releasePersist = resolve
+    })
+
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') as any
+    session.persistLastCompletedBlock = vi.fn(() => persistPromise)
+    session.complete = vi.fn().mockResolvedValue(undefined)
+
+    // onBlockComplete is now marker-only; its marker write is fire-and-forget
+    // but tracked, so terminal completion must drain it first.
+    void session.onBlockComplete('block-2', 'Transform', 'function', {
+      endedAt: '2025-01-01T00:00:01.000Z',
+      output: { value: true },
+    })
+
+    const completionPromise = session.safeComplete({ finalOutput: { ok: true } })
+
+    await Promise.resolve()
+
+    expect(session.complete).not.toHaveBeenCalled()
+
+    releasePersist?.()
+
+    await completionPromise
+
+    expect(session.persistLastCompletedBlock).toHaveBeenCalledTimes(1)
+    expect(session.complete).toHaveBeenCalledTimes(1)
+  })
+
+  it('keeps draining when new progress writes arrive during drain', async () => {
+    let releaseFirst: (() => void) | undefined
+    let releaseSecond: (() => void) | undefined
+    const firstPromise = new Promise<void>((resolve) => {
+      releaseFirst = resolve
+    })
+    const secondPromise = new Promise<void>((resolve) => {
+      releaseSecond = resolve
+    })
+
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') as any
+
+    void session.trackProgressWrite(firstPromise)
+
+    const drainPromise = session.drainPendingProgressWrites()
+
+    await Promise.resolve()
+
+    void session.trackProgressWrite(secondPromise)
+    releaseFirst?.()
+
+    await Promise.resolve()
+
+    let drained = false
+    void drainPromise.then(() => {
+      drained = true
+    })
+
+    await Promise.resolve()
+    expect(drained).toBe(false)
+
+    releaseSecond?.()
+    await drainPromise
+
+    expect(session.pendingProgressWrites.size).toBe(0)
+  })
+
+  it('marks pause completion as terminal and prevents duplicate pause finalization', async () => {
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') as any
+    session.completeExecutionWithFinalization = vi.fn().mockResolvedValue(undefined)
+
+    await session.completeWithPause({ workflowInput: { ok: true } })
+    await session.completeWithPause({ workflowInput: { ok: true } })
+
+    expect(session.completeExecutionWithFinalization).toHaveBeenCalledTimes(1)
+    expect(session.completed).toBe(true)
+    expect(session.completing).toBe(true)
+  })
+})
+
+describe('completeWithError cancelled-status guard', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    dbMocks.updateWhere.mockResolvedValue(undefined)
+    dbMocks.execute.mockResolvedValue(undefined)
+  })
+
+  it('skips writing failed and marks session complete when DB status is already cancelled', async () => {
+    dbMocks.selectLimit.mockResolvedValue([{ status: 'cancelled' }])
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.safeCompleteWithError({ error: { message: 'block errored mid-cancel' } })
+
+    expect(completeWorkflowExecutionMock).not.toHaveBeenCalled()
+    expect(session.hasCompleted()).toBe(true)
+  })
+
+  it('writes failed when DB status is running (no cancel in flight)', async () => {
+    dbMocks.selectLimit.mockResolvedValue([{ status: 'running' }])
+    completeWorkflowExecutionMock.mockResolvedValue({})
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.safeCompleteWithError({ error: { message: 'genuine block failure' } })
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'failed' })
+    )
+    expect(session.hasCompleted()).toBe(true)
+  })
+
+  it('writes failed when no execution log exists yet', async () => {
+    dbMocks.selectLimit.mockResolvedValue([])
+    completeWorkflowExecutionMock.mockResolvedValue({})
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.safeCompleteWithError({ error: { message: 'pre-log error' } })
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'failed' })
+    )
+  })
+
+  it('deduplicates all subsequent completion attempts after guard early-return', async () => {
+    dbMocks.selectLimit.mockResolvedValue([{ status: 'cancelled' }])
+    completeWorkflowExecutionMock.mockResolvedValue({})
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.safeCompleteWithError({ error: { message: 'error 1' } })
+    await session.safeCompleteWithError({ error: { message: 'error 2' } })
+    await session.safeComplete({ finalOutput: { ok: true } })
+
+    expect(completeWorkflowExecutionMock).not.toHaveBeenCalled()
+    expect(session.hasCompleted()).toBe(true)
+  })
+
+  it('falls through to cost-only fallback when the DB check itself throws', async () => {
+    dbMocks.selectLimit.mockRejectedValueOnce(new Error('DB connection lost'))
+    completeWorkflowExecutionMock.mockResolvedValue({})
+    const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1')
+
+    await session.safeCompleteWithError({ error: { message: 'block failed' } })
+
+    expect(completeWorkflowExecutionMock).toHaveBeenCalledWith(
+      expect.objectContaining({ finalizationPath: 'force_failed' })
+    )
+    expect(session.hasCompleted()).toBe(true)
+  })
+})
+
+describe('LoggingSession.markExecutionAsFailed workflowId scoping', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    dbMocks.updateWhere.mockResolvedValue(undefined)
+  })
+
+  it('scopes UPDATE by both executionId and workflowId', async () => {
+    await LoggingSession.markExecutionAsFailed('exec-1', undefined, undefined, 'wf-1')
+
+    expect(dbMocks.update).toHaveBeenCalledTimes(1)
+    expect(dbMocks.updateSet).toHaveBeenCalledTimes(1)
+    expect(dbMocks.updateWhere).toHaveBeenCalledTimes(1)
+
+    const whereArgs = dbMocks.updateWhere.mock.calls[0]
+    expect(whereArgs).toBeDefined()
+  })
+
+  it('instance markAsFailed forwards workflowId to the static method', async () => {
+    const updateWhereSpy = dbMocks.updateWhere
+    dbMocks.selectLimit.mockResolvedValue([{ executionData: {} }])
+
+    const session = new LoggingSession('wf-42', 'exec-42', 'api', 'req-1')
+    await session.markAsFailed('something went wrong')
+
+    expect(updateWhereSpy).toHaveBeenCalledTimes(1)
+  })
+
+  it('uses the provided errorMessage in the SQL set', async () => {
+    const sqlMock = dbMocks.sql
+    await LoggingSession.markExecutionAsFailed('exec-2', 'custom error', undefined, 'wf-2')
+
+    expect(sqlMock).toHaveBeenCalled()
+    const lastCall = sqlMock.mock.calls.at(-1)!
+    const [strings, ...values] = lastCall
+    const combined = String(Array.from(strings)).toLowerCase() + values.join(' ').toLowerCase()
+    expect(combined).toContain('force_failed')
+  })
+
+  it('clears Redis markers when marking failed (terminal boundary outside completeWorkflowExecution)', async () => {
+    await LoggingSession.markExecutionAsFailed('exec-3', 'boom', undefined, 'wf-3')
+    expect(clearProgressMarkersMock).toHaveBeenCalledWith('exec-3')
+  })
+
+  it('folds live Redis markers into the row before clearing on force-fail', async () => {
+    getProgressMarkersMock.mockResolvedValueOnce({
+      lastStartedBlock: { blockId: 'b1', blockName: 'Fetch', blockType: 'api', startedAt: 't1' },
+      lastCompletedBlock: {
+        blockId: 'b1',
+        blockName: 'Fetch',
+        blockType: 'api',
+        endedAt: 't2',
+        success: false,
+      },
+    })
+
+    await LoggingSession.markExecutionAsFailed('exec-9', 'boom', undefined, 'wf-9')
+
+    const folded = dbMocks.sql.mock.calls
+      .map((c) => String(Array.from(c[0] as TemplateStringsArray)))
+      .join(' ')
+    expect(folded).toContain('lastStartedBlock')
+    expect(folded).toContain('lastCompletedBlock')
+    expect(clearProgressMarkersMock).toHaveBeenCalledWith('exec-9')
+  })
+
+  it('does not clear markers when the Redis read fails (avoids wiping the only copy)', async () => {
+    getProgressMarkersMock.mockResolvedValueOnce(null)
+    await LoggingSession.markExecutionAsFailed('exec-readfail', 'boom', undefined, 'wf-x')
+    expect(clearProgressMarkersMock).not.toHaveBeenCalled()
+  })
+})
+
+describe('LoggingSession progress-marker write path', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    startWorkflowExecutionMock.mockResolvedValue({})
+    loadWorkflowStateForExecutionMock.mockResolvedValue({
+      blocks: {},
+      edges: [],
+      loops: {},
+      parallels: {},
+    })
+    dbMocks.execute.mockResolvedValue(undefined)
+  })
+
+  it('writes markers to Redis (not the row) when Redis accepts the write', async () => {
+    setLastStartedBlockMock.mockResolvedValue(true)
+    setLastCompletedBlockMock.mockResolvedValue(true)
+    const session = new LoggingSession('wf-1', 'exec-redis', 'manual', 'req-1')
+    await session.start({ workspaceId: 'ws-1' })
+
+    await session.onBlockStart('b1', 'Fetch', 'api', '2026-06-27T10:00:00.000Z')
+    await session.onBlockComplete('b1', 'Fetch', 'api', { endedAt: '2026-06-27T10:00:01.000Z' })
+
+    expect(setLastStartedBlockMock).toHaveBeenCalledWith(
+      'exec-redis',
+      expect.objectContaining({ blockId: 'b1', startedAt: '2026-06-27T10:00:00.000Z' })
+    )
+    expect(setLastCompletedBlockMock).toHaveBeenCalledWith(
+      'exec-redis',
+      expect.objectContaining({ blockId: 'b1', success: true })
+    )
+    expect(dbMocks.execute).not.toHaveBeenCalled()
+  })
+
+  it('falls back to the SQL UPDATE when the Redis write fails', async () => {
+    setLastStartedBlockMock.mockResolvedValue(false)
+    const session = new LoggingSession('wf-1', 'exec-redis-down', 'manual', 'req-1')
+    await session.start({ workspaceId: 'ws-1' })
+
+    await session.onBlockStart('b1', 'Fetch', 'api', '2026-06-27T10:00:00.000Z')
+
+    expect(setLastStartedBlockMock).toHaveBeenCalled()
+    expect(dbMocks.execute).toHaveBeenCalledTimes(1)
+  })
+})

apps/sim/lib/logs/execution/pii-large-values.test.ts

@@ -0,0 +1,180 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockMaterializeRef, mockCompact, mockMaterializeManifest, mockMaskBatch } = vi.hoisted(
+  () => ({
+    mockMaterializeRef: vi.fn(),
+    mockCompact: vi.fn(),
+    mockMaterializeManifest: vi.fn(),
+    mockMaskBatch: vi.fn(),
+  })
+)
+
+vi.mock('@/lib/execution/payloads/store', () => ({
+  materializeLargeValueRef: mockMaterializeRef,
+}))
+vi.mock('@/lib/execution/payloads/serializer', () => ({
+  compactExecutionPayload: mockCompact,
+}))
+vi.mock('@/lib/execution/payloads/large-array-manifest', () => ({
+  materializeLargeArrayManifest: mockMaterializeManifest,
+}))
+vi.mock('@/lib/execution/payloads/large-array-manifest-metadata', () => ({
+  isLargeArrayManifest: () => false,
+}))
+vi.mock('@/lib/guardrails/mask-client', () => ({
+  maskPIIBatchViaHttp: mockMaskBatch,
+}))
+
+import {
+  redactLargeValueRefs,
+  redactLargeValueRefsInValue,
+} from '@/lib/logs/execution/pii-large-values'
+import { PiiRedactionError } from '@/lib/logs/execution/pii-redaction'
+
+const REF = {
+  __simLargeValueRef: true,
+  version: 1,
+  id: 'lv_abcdef123456',
+  kind: 'object',
+  size: 9_000_000,
+} as const
+
+const STORE = { workspaceId: 'ws-1', workflowId: 'wf-1', executionId: 'ex-1', userId: 'u-1' }
+
+describe('redactLargeValueRefs', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMaskBatch.mockImplementation(async (texts: string[]) => texts.map((t) => `MASKED(${t})`))
+    // compact echoes its input so we can assert the masked content is what's re-stored.
+    mockCompact.mockImplementation(async (value: unknown) => value)
+  })
+
+  it('hydrates, masks, and re-stores a large-value ref (content preserved, PII masked)', async () => {
+    mockMaterializeRef.mockResolvedValue({ note: 'contact bob', id: 42 })
+
+    const result = await redactLargeValueRefs(
+      { finalOutput: REF },
+      { entityTypes: ['PERSON'], language: 'en', store: STORE }
+    )
+
+    expect(mockMaterializeRef).toHaveBeenCalledWith(
+      REF,
+      expect.objectContaining({ executionId: 'ex-1', trackReference: false })
+    )
+    expect(result.finalOutput).toEqual({ note: 'MASKED(contact bob)', id: 42 })
+    expect(mockCompact).toHaveBeenCalledTimes(1)
+  })
+
+  it('falls back to the marker when a ref cannot be materialized', async () => {
+    mockMaterializeRef.mockResolvedValue(undefined)
+    const result = await redactLargeValueRefs(
+      { finalOutput: REF },
+      { entityTypes: [], language: 'en', store: STORE }
+    )
+    expect(result.finalOutput).toBe('[REDACTION_FAILED]')
+    expect(mockCompact).not.toHaveBeenCalled()
+  })
+
+  it('falls back to the marker when re-store throws (never leaks)', async () => {
+    mockMaterializeRef.mockResolvedValue({ note: 'secret@x.com' })
+    mockCompact.mockRejectedValueOnce(new Error('s3 down'))
+    const result = await redactLargeValueRefs(
+      { finalOutput: REF },
+      { entityTypes: [], language: 'en', store: STORE }
+    )
+    expect(result.finalOutput).toBe('[REDACTION_FAILED]')
+  })
+
+  it('hydrates+masks refs across multiple payload keys (parallel, cross-key)', async () => {
+    const refA = { ...REF, id: 'lv_aaaaaaaaaaaa' }
+    const refB = { ...REF, id: 'lv_bbbbbbbbbbbb' }
+    mockMaterializeRef.mockImplementation(async (ref: { id: string }) =>
+      ref.id === 'lv_aaaaaaaaaaaa' ? { note: 'call bob' } : { note: 'email amy' }
+    )
+
+    const result = await redactLargeValueRefs(
+      { finalOutput: refA, traceSpans: [{ output: refB }] },
+      { entityTypes: [], language: 'en', store: STORE }
+    )
+
+    expect(result.finalOutput).toEqual({ note: 'MASKED(call bob)' })
+    expect((result.traceSpans as any[])[0].output).toEqual({ note: 'MASKED(email amy)' })
+    expect(mockMaterializeRef).toHaveBeenCalledTimes(2)
+    expect(mockCompact).toHaveBeenCalledTimes(2)
+  })
+
+  it('aborts (throws) when one of several refs fails in throw mode', async () => {
+    const refA = { ...REF, id: 'lv_aaaaaaaaaaaa' }
+    const refB = { ...REF, id: 'lv_bbbbbbbbbbbb' }
+    // refA materializes fine; refB can't — in throw mode the whole redaction must abort.
+    mockMaterializeRef.mockImplementation(async (ref: { id: string }) =>
+      ref.id === 'lv_aaaaaaaaaaaa' ? { note: 'ok' } : undefined
+    )
+
+    await expect(
+      redactLargeValueRefs(
+        { finalOutput: refA, traceSpans: [{ output: refB }] },
+        { entityTypes: [], language: 'en', store: STORE, onFailure: 'throw' }
+      )
+    ).rejects.toBeInstanceOf(PiiRedactionError)
+  })
+
+  it('leaves payloads without refs untouched', async () => {
+    const payload = { finalOutput: { answer: 'world', count: 5 } }
+    const result = await redactLargeValueRefs(payload, {
+      entityTypes: [],
+      language: 'en',
+      store: STORE,
+    })
+    expect(result).toEqual(payload)
+    expect(mockMaterializeRef).not.toHaveBeenCalled()
+  })
+})
+
+describe('redactLargeValueRefsInValue (arbitrary blockStates)', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMaskBatch.mockImplementation(async (texts: string[]) => texts.map((t) => `MASKED(${t})`))
+    mockCompact.mockImplementation(async (value: unknown) => value)
+  })
+
+  it('hydrates + re-stores a ref nested in a non-RedactablePayload shape', async () => {
+    mockMaterializeRef.mockResolvedValue({ note: 'contact bob' })
+    const blockStates = { 'block-1': { output: REF }, 'block-2': { output: { plain: 'hi' } } }
+
+    const result = await redactLargeValueRefsInValue(blockStates, {
+      entityTypes: ['PERSON'],
+      language: 'en',
+      store: STORE,
+    })
+
+    expect((result as any)['block-1'].output).toEqual({ note: 'MASKED(contact bob)' })
+    expect((result as any)['block-2'].output).toEqual({ plain: 'hi' })
+  })
+
+  it('throws PiiRedactionError on failure when onFailure is throw (aborts resume, no marker)', async () => {
+    mockMaterializeRef.mockResolvedValue(undefined)
+
+    await expect(
+      redactLargeValueRefsInValue(
+        { 'block-1': { output: REF } },
+        { entityTypes: [], language: 'en', store: STORE, onFailure: 'throw' }
+      )
+    ).rejects.toBeInstanceOf(PiiRedactionError)
+  })
+
+  it('rethrows a re-store failure as PiiRedactionError under throw mode', async () => {
+    mockMaterializeRef.mockResolvedValue({ note: 'secret@x.com' })
+    mockCompact.mockRejectedValueOnce(new Error('s3 down'))
+
+    await expect(
+      redactLargeValueRefsInValue(
+        { 'block-1': { output: REF } },
+        { entityTypes: [], language: 'en', store: STORE, onFailure: 'throw' }
+      )
+    ).rejects.toBeInstanceOf(PiiRedactionError)
+  })
+})

apps/sim/lib/logs/execution/pii-large-values.ts

@@ -0,0 +1,240 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { env } from '@/lib/core/config/env'
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import type { LargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest'
+import { materializeLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest'
+import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
+import { isLargeValueRef, type LargeValueRef } from '@/lib/execution/payloads/large-value-ref'
+import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
+import type { LargeValueStoreContext } from '@/lib/execution/payloads/store'
+import { materializeLargeValueRef } from '@/lib/execution/payloads/store'
+import {
+  PiiRedactionError,
+  type PiiRedactionFailureMode,
+  REDACTION_FAILED_MARKER,
+  type RedactablePayload,
+  redactObjectStrings,
+} from '@/lib/logs/execution/pii-redaction'
+
+const logger = createLogger('PiiLargeValues')
+
+export interface RedactLargeValueRefsOptions {
+  /** Presidio entity types to mask. Empty = redact all detected PII. */
+  entityTypes: string[]
+  language: string
+  /** Storage scope for materializing and re-storing the masked values. */
+  store: LargeValueStoreContext
+  /**
+   * How to handle a ref that can't be materialized/masked/re-stored. Defaults to
+   * `'scrub'` (marker) — safe for the logs path. The execution-altering restore
+   * path passes `'throw'` so an unmaskable restored value aborts the run rather
+   * than feeding a marker (or leaking raw bytes) downstream.
+   */
+  onFailure?: PiiRedactionFailureMode
+}
+
+/**
+ * Hydrate, mask, and re-store offloaded large values inside a log payload.
+ *
+ * The string redactor can't reach a value already offloaded to large-value
+ * storage (a >8MB ref) — its bytes live in object storage, not inline. This walks
+ * the payload and, for each ref / array manifest, materializes it, masks its
+ * content, and re-stores a fresh masked ref — so the log keeps the redacted
+ * content rather than losing the whole field. Any failure (materialization
+ * unavailable, missing storage scope, re-store error) falls back to
+ * {@link REDACTION_FAILED_MARKER} so raw PII is never left behind.
+ *
+ * Traversal is SYNCHRONOUS (collect refs, then substitute) so a large ref-free
+ * payload costs only a cheap walk — never a promise-per-node. Only the handful of
+ * actual refs incur async hydrate → mask → re-store work, and those run in
+ * parallel with bounded concurrency ({@link REF_CONCURRENCY}).
+ */
+export async function redactLargeValueRefs(
+  payload: RedactablePayload,
+  options: RedactLargeValueRefsOptions
+): Promise<RedactablePayload> {
+  // Collect refs across the WHOLE payload first (shared `seen`), so every ref is
+  // hydrated+masked+re-stored in one bounded-concurrency pass instead of one
+  // sequential pass per key. A ref shared across keys is walked/masked once.
+  const refs: object[] = []
+  const seen = new WeakSet<object>()
+  for (const key of Object.keys(payload) as (keyof RedactablePayload)[]) {
+    if (payload[key] !== undefined) collectRefs(payload[key], refs, seen)
+  }
+  if (refs.length === 0) return payload
+
+  const replacements = await resolveReplacements(refs, options)
+  const result: RedactablePayload = { ...payload }
+  for (const key of Object.keys(payload) as (keyof RedactablePayload)[]) {
+    if (payload[key] !== undefined) result[key] = substituteRefs(payload[key], replacements)
+  }
+  return result
+}
+
+/**
+ * Hydrate, mask, and re-store offloaded large values inside an arbitrary value
+ * (e.g. resumed snapshot `blockStates`) — the same walk as
+ * {@link redactLargeValueRefs}, but not bound to the {@link RedactablePayload}
+ * key set. Structure is preserved; only refs/manifests are replaced.
+ */
+export async function redactLargeValueRefsInValue<T>(
+  value: T,
+  options: RedactLargeValueRefsOptions
+): Promise<T> {
+  return (await redactValueRefs(value, options)) as T
+}
+
+/** Sync-collect every ref/manifest in `value`, then async-replace each, then sync-substitute. */
+async function redactValueRefs(
+  value: unknown,
+  options: RedactLargeValueRefsOptions
+): Promise<unknown> {
+  const refs: object[] = []
+  collectRefs(value, refs, new WeakSet())
+  if (refs.length === 0) return value
+
+  const replacements = await resolveReplacements(refs, options)
+  return substituteRefs(value, replacements)
+}
+
+/**
+ * Max large-value refs hydrated → masked → re-stored in parallel per payload.
+ * Multiplies with the mask-batch chunk concurrency for total in-flight Presidio
+ * load, which the load-balanced fleet behind the internal ALB absorbs.
+ */
+const REF_CONCURRENCY = env.PII_REF_CONCURRENCY ?? 4
+
+/**
+ * Dedupe the collected refs by identity, then replace each in parallel (bounded by
+ * {@link REF_CONCURRENCY}). `Map.set` is synchronous, so concurrent workers writing
+ * the shared map do not race.
+ *
+ * `mapWithConcurrency`'s `fn` must not reject (a rejection fails the pool
+ * non-deterministically), so the mapper is total: it catches per-ref errors and
+ * records the first one. In `onFailure: 'throw'` mode `replaceRef` throws, so after
+ * the pool drains we rethrow that first error — an unmaskable ref still aborts the
+ * run rather than passing through. In `'scrub'` mode `replaceRef` never throws.
+ */
+async function resolveReplacements(
+  refs: object[],
+  options: RedactLargeValueRefsOptions
+): Promise<Map<object, unknown>> {
+  const unique = [...new Set(refs)]
+  const replacements = new Map<object, unknown>()
+  let firstError: unknown
+  await mapWithConcurrency(unique, REF_CONCURRENCY, async (ref) => {
+    try {
+      replacements.set(ref, await replaceRef(ref, options))
+    } catch (error) {
+      if (firstError === undefined) firstError = error
+    }
+  })
+  if (firstError !== undefined) throw firstError
+  return replacements
+}
+
+/** Depth-first sync walk collecting ref/manifest nodes (not recursing into them). */
+function collectRefs(value: unknown, out: object[], seen: WeakSet<object>): void {
+  if (isLargeValueRef(value) || isLargeArrayManifest(value)) {
+    out.push(value as object)
+    return
+  }
+  if (value === null || typeof value !== 'object') return
+  if (seen.has(value)) return
+  seen.add(value)
+  if (Array.isArray(value)) {
+    for (const item of value) collectRefs(item, out, seen)
+    return
+  }
+  for (const v of Object.values(value)) collectRefs(v, out, seen)
+}
+
+/** Sync rebuild of `value` with each collected ref swapped for its replacement (by identity). */
+function substituteRefs(value: unknown, replacements: Map<object, unknown>): unknown {
+  if (isLargeValueRef(value) || isLargeArrayManifest(value)) {
+    return replacements.has(value as object) ? replacements.get(value as object) : value
+  }
+  if (value === null || typeof value !== 'object') return value
+  if (Array.isArray(value)) {
+    return value.map((item) => substituteRefs(item, replacements))
+  }
+  const out: Record<string, unknown> = {}
+  for (const [key, v] of Object.entries(value)) {
+    out[key] = substituteRefs(v, replacements)
+  }
+  return out
+}
+
+async function replaceRef(ref: object, options: RedactLargeValueRefsOptions): Promise<unknown> {
+  return isLargeValueRef(ref)
+    ? redactRef(ref, options)
+    : redactManifest(ref as LargeArrayManifest, options)
+}
+
+/**
+ * Mask a materialized large value and re-offload it: handle any nested refs
+ * first, then mask inline strings, then re-store. `redactObjectStrings` skips
+ * refs, so the nested re-stored refs are left intact while their siblings mask.
+ */
+async function maskAndReStore(
+  value: unknown,
+  options: RedactLargeValueRefsOptions
+): Promise<unknown> {
+  const nested = await redactValueRefs(value, options)
+  const masked = await redactObjectStrings(nested, {
+    entityTypes: options.entityTypes,
+    language: options.language,
+    onFailure: options.onFailure ?? 'scrub',
+  })
+  return compactExecutionPayload(masked, { ...options.store, requireDurable: true })
+}
+
+/** Rethrow (as {@link PiiRedactionError}) or scrub-to-marker, per `onFailure`. */
+function onRefFailure(
+  error: unknown,
+  options: RedactLargeValueRefsOptions,
+  context: string
+): never | string {
+  if ((options.onFailure ?? 'scrub') === 'throw') {
+    throw error instanceof PiiRedactionError
+      ? error
+      : new PiiRedactionError(`${context}: ${getErrorMessage(error)}`)
+  }
+  logger.error(`${context}; scrubbing`, { error: getErrorMessage(error) })
+  return REDACTION_FAILED_MARKER
+}
+
+async function redactRef(
+  ref: LargeValueRef,
+  options: RedactLargeValueRefsOptions
+): Promise<unknown> {
+  try {
+    const materialized = await materializeLargeValueRef(ref, {
+      ...options.store,
+      trackReference: false,
+    })
+    if (materialized === undefined) {
+      return onRefFailure(
+        new Error('large value could not be materialized'),
+        options,
+        'Failed to redact large value ref'
+      )
+    }
+    return await maskAndReStore(materialized, options)
+  } catch (error) {
+    return onRefFailure(error, options, 'Failed to redact large value ref')
+  }
+}
+
+async function redactManifest(
+  manifest: LargeArrayManifest,
+  options: RedactLargeValueRefsOptions
+): Promise<unknown> {
+  try {
+    const materialized = await materializeLargeArrayManifest(manifest, { ...options.store })
+    return await maskAndReStore(materialized, options)
+  } catch (error) {
+    return onRefFailure(error, options, 'Failed to redact large array manifest')
+  }
+}

apps/sim/lib/logs/execution/pii-redaction.test.ts

@@ -0,0 +1,194 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockMaskPIIBatch } = vi.hoisted(() => ({
+  mockMaskPIIBatch: vi.fn(),
+}))
+
+vi.mock('@/lib/guardrails/mask-client', () => ({
+  maskPIIBatchViaHttp: mockMaskPIIBatch,
+}))
+
+import {
+  PiiRedactionError,
+  REDACTION_FAILED_MARKER,
+  redactObjectStrings,
+  redactPIIFromExecution,
+} from '@/lib/logs/execution/pii-redaction'
+
+describe('redactPIIFromExecution', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    // Default: echo each input uppercased so we can assert substitution by position.
+    mockMaskPIIBatch.mockImplementation(async (texts: string[]) => texts.map((t) => `MASKED(${t})`))
+  })
+
+  it('collects and masks string leaves recursively, preserving structure', async () => {
+    const payload = {
+      traceSpans: [
+        {
+          blockId: 'b1',
+          status: 'success',
+          input: { email: 'a@b.com' },
+          output: { text: 'hello' },
+          children: [{ blockId: 'c1', output: { nested: 'deep' } }],
+        },
+      ],
+      finalOutput: { answer: 'world' },
+      workflowInput: 'start',
+    }
+
+    const result = await redactPIIFromExecution(payload, { entityTypes: ['EMAIL_ADDRESS'] })
+
+    const span = (result.traceSpans as any[])[0]
+    expect(span.blockId).toBe('b1')
+    expect(span.status).toBe('success')
+    expect(span.input.email).toBe('MASKED(a@b.com)')
+    expect(span.output.text).toBe('MASKED(hello)')
+    expect(span.children[0].output.nested).toBe('MASKED(deep)')
+    expect((result.finalOutput as any).answer).toBe('MASKED(world)')
+    expect(result.workflowInput).toBe('MASKED(start)')
+    expect(mockMaskPIIBatch).toHaveBeenCalledTimes(1)
+    expect(mockMaskPIIBatch.mock.calls[0][0]).toEqual([
+      'a@b.com',
+      'hello',
+      'deep',
+      'world',
+      'start',
+    ])
+  })
+
+  it('does not mutate the original payload', async () => {
+    const payload = { finalOutput: { answer: 'world' } }
+    await redactPIIFromExecution(payload, { entityTypes: [] })
+    expect(payload.finalOutput.answer).toBe('world')
+  })
+
+  it('scrubs all eligible strings when masking throws (no leak)', async () => {
+    mockMaskPIIBatch.mockRejectedValueOnce(new Error('presidio down'))
+    const payload = {
+      traceSpans: [{ output: { text: 'secret@x.com' } }],
+      finalOutput: 'another secret',
+    }
+
+    const result = await redactPIIFromExecution(payload, { entityTypes: [] })
+
+    expect((result.traceSpans as any[])[0].output.text).toBe(REDACTION_FAILED_MARKER)
+    expect(result.finalOutput).toBe(REDACTION_FAILED_MARKER)
+  })
+
+  it('masks large strings too (never left unredacted)', async () => {
+    const big = 'x'.repeat(200 * 1024)
+    const payload = { finalOutput: { big, small: 'pii' } }
+
+    const result = await redactPIIFromExecution(payload, { entityTypes: [] })
+
+    expect((result.finalOutput as any).big).toBe(`MASKED(${big})`)
+    expect((result.finalOutput as any).small).toBe('MASKED(pii)')
+    expect(mockMaskPIIBatch.mock.calls[0][0]).toEqual([big, 'pii'])
+  })
+
+  it('masks span error/errorMessage and top-level error, trigger, executionState, environment', async () => {
+    const payload = {
+      traceSpans: [{ blockId: 'b1', error: 'failed for bob@x.com', errorMessage: 'bad input z' }],
+      error: 'run failed: a@b.com',
+      completionFailure: 'cancelled by c@d.com',
+      trigger: { type: 'webhook', data: { from: 'caller@x.com' } },
+      executionState: { status: 'completed', note: 'state for e@f.com' },
+      environment: { variables: { CONTACT: 'admin@x.com' } },
+      correlation: { source: 'corr@x.com' },
+    }
+
+    const result = await redactPIIFromExecution(payload, { entityTypes: ['EMAIL_ADDRESS'] })
+
+    const span = (result.traceSpans as any[])[0]
+    expect(span.blockId).toBe('b1')
+    expect(span.error).toBe('MASKED(failed for bob@x.com)')
+    expect(span.errorMessage).toBe('MASKED(bad input z)')
+    expect(result.error).toBe('MASKED(run failed: a@b.com)')
+    expect(result.completionFailure).toBe('MASKED(cancelled by c@d.com)')
+    expect((result.trigger as any).type).toBe('MASKED(webhook)')
+    expect((result.trigger as any).data.from).toBe('MASKED(caller@x.com)')
+    expect((result.executionState as any).note).toBe('MASKED(state for e@f.com)')
+    expect((result.environment as any).variables.CONTACT).toBe('MASKED(admin@x.com)')
+    expect((result.correlation as any).source).toBe('MASKED(corr@x.com)')
+  })
+
+  it('returns payload unchanged when there is nothing to mask', async () => {
+    const payload = { traceSpans: [{ blockId: 'b1', count: 5 }] }
+    const result = await redactPIIFromExecution(payload, { entityTypes: [] })
+    expect(result).toBe(payload)
+    expect(mockMaskPIIBatch).not.toHaveBeenCalled()
+  })
+})
+
+describe('redactObjectStrings', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMaskPIIBatch.mockImplementation(async (texts: string[]) => texts.map((t) => `MASKED(${t})`))
+  })
+
+  it('masks every string leaf and preserves structure', async () => {
+    const value = { name: 'bob', nested: { email: 'a@b.com' }, list: ['x', 1, true] }
+    const result = await redactObjectStrings(value, { entityTypes: ['PERSON'] })
+    expect(result).toEqual({
+      name: 'MASKED(bob)',
+      nested: { email: 'MASKED(a@b.com)' },
+      list: ['MASKED(x)', 1, true],
+    })
+    expect(mockMaskPIIBatch).toHaveBeenCalledTimes(1)
+  })
+
+  it('leaves non-string and empty values untouched', async () => {
+    const value = { count: 5, flag: false, empty: '', nullish: null }
+    const result = await redactObjectStrings(value, { entityTypes: [] })
+    expect(result).toEqual(value)
+    expect(mockMaskPIIBatch).not.toHaveBeenCalled()
+  })
+
+  it('throws PiiRedactionError on masking failure when onFailure is throw', async () => {
+    mockMaskPIIBatch.mockRejectedValueOnce(new Error('presidio down'))
+    await expect(
+      redactObjectStrings({ text: 'a@b.com' }, { entityTypes: [], onFailure: 'throw' })
+    ).rejects.toBeInstanceOf(PiiRedactionError)
+  })
+
+  it('masks large payloads (no size ceiling) rather than scrubbing them', async () => {
+    const big = 'x'.repeat(17 * 1024 * 1024)
+    const result = (await redactObjectStrings({ big }, { entityTypes: [] })) as { big: string }
+    expect(result.big).toBe(`MASKED(${big})`)
+    expect(mockMaskPIIBatch).toHaveBeenCalledTimes(1)
+  })
+
+  it('scrubs (does not throw) by default on failure', async () => {
+    mockMaskPIIBatch.mockRejectedValueOnce(new Error('presidio down'))
+    const result = await redactObjectStrings({ text: 'a@b.com' }, { entityTypes: [] })
+    expect(result).toEqual({ text: REDACTION_FAILED_MARKER })
+  })
+})
+
+describe('transformStrings (via redactObjectStrings) leaves large-value refs intact', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMaskPIIBatch.mockImplementation(async (texts: string[]) => texts.map((t) => `MASKED(${t})`))
+  })
+
+  it('does not recurse into / corrupt a large-value ref while masking siblings', async () => {
+    const ref = {
+      __simLargeValueRef: true,
+      version: 1,
+      id: 'lv_abcdef123456',
+      kind: 'object',
+      size: 9_000_000,
+    }
+    const result = (await redactObjectStrings(
+      { name: 'bob', big: ref },
+      { entityTypes: ['PERSON'] }
+    )) as any
+    expect(result.name).toBe('MASKED(bob)')
+    // The ref is left byte-for-byte intact (its key/id are not masked).
+    expect(result.big).toEqual(ref)
+  })
+})

apps/sim/lib/logs/execution/pii-redaction.ts

@@ -0,0 +1,243 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
+import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
+import { maskPIIBatchViaHttp } from '@/lib/guardrails/mask-client'
+
+const logger = createLogger('PiiRedaction')
+
+/** Replaces text we could not safely mask, so PII is never persisted on failure. */
+export const REDACTION_FAILED_MARKER = '[REDACTION_FAILED]'
+
+/**
+ * How to handle a masking failure (Presidio error or over-ceiling payload):
+ * - `'scrub'` (default): replace eligible strings with {@link REDACTION_FAILED_MARKER}.
+ *   Safe for the log stage — execution already succeeded.
+ * - `'throw'`: throw {@link PiiRedactionError}. Used for the execution-altering
+ *   stages (input/block outputs) where a marker would corrupt computed data and
+ *   fail-open would leak — so the run aborts instead.
+ */
+export type PiiRedactionFailureMode = 'scrub' | 'throw'
+
+export interface PiiRedactionOptions {
+  /** Presidio entity types to mask. Empty = redact all detected PII. */
+  entityTypes: string[]
+  language?: string
+  /** Failure handling. Defaults to `'scrub'`. */
+  onFailure?: PiiRedactionFailureMode
+}
+
+/** Thrown when in-flight redaction (`onFailure: 'throw'`) cannot mask safely. */
+export class PiiRedactionError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'PiiRedactionError'
+  }
+}
+
+export interface RedactablePayload {
+  traceSpans?: unknown
+  finalOutput?: unknown
+  workflowInput?: unknown
+  error?: unknown
+  completionFailure?: unknown
+  trigger?: unknown
+  executionState?: unknown
+  environment?: unknown
+  correlation?: unknown
+}
+
+/** Keys of {@link RedactablePayload} processed by the redactor, in order. */
+const REDACTABLE_KEYS: (keyof RedactablePayload)[] = [
+  'traceSpans',
+  'finalOutput',
+  'workflowInput',
+  'error',
+  'completionFailure',
+  'trigger',
+  'executionState',
+  'environment',
+  'correlation',
+]
+
+/** Trace-span fields that carry runtime content (and therefore possible PII). */
+const SPAN_CONTENT_FIELDS = [
+  'input',
+  'output',
+  'thinking',
+  'modelToolCalls',
+  'toolCalls',
+  'error',
+  'errorMessage',
+] as const
+
+function isEligibleString(value: string): boolean {
+  return value.length > 0
+}
+
+/**
+ * Rebuild `value` replacing every eligible string leaf with `handle(leaf)`.
+ * Used for both collection (handle records and returns the input) and
+ * substitution (handle returns the masked value), so traversal order and
+ * eligibility are guaranteed identical across the two passes.
+ */
+function transformStrings(value: unknown, handle: (s: string) => string): unknown {
+  if (typeof value === 'string') {
+    return isEligibleString(value) ? handle(value) : value
+  }
+  // Treat offloaded large-value refs as opaque: masking their internal `key`/`id`
+  // strings would corrupt the reference. Their content is handled separately
+  // (hydrate → mask → re-store) before this runs.
+  if (isLargeValueRef(value) || isLargeArrayManifest(value)) {
+    return value
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => transformStrings(item, handle))
+  }
+  if (value !== null && typeof value === 'object') {
+    const out: Record<string, unknown> = {}
+    for (const [key, v] of Object.entries(value)) {
+      out[key] = transformStrings(v, handle)
+    }
+    return out
+  }
+  return value
+}
+
+/**
+ * Redact a trace span: only its content fields ({@link SPAN_CONTENT_FIELDS}) and
+ * nested `children` are walked, leaving structural metadata (blockId, name,
+ * status, timing) untouched so log correlation/display is preserved.
+ */
+function transformSpan(span: unknown, handle: (s: string) => string): unknown {
+  if (span === null || typeof span !== 'object' || Array.isArray(span)) {
+    return transformStrings(span, handle)
+  }
+  const source = span as Record<string, unknown>
+  const out: Record<string, unknown> = { ...source }
+  for (const field of SPAN_CONTENT_FIELDS) {
+    if (field in out) out[field] = transformStrings(out[field], handle)
+  }
+  if (Array.isArray(source.children)) {
+    out.children = source.children.map((child) => transformSpan(child, handle))
+  }
+  return out
+}
+
+function transformUnit(
+  key: keyof RedactablePayload,
+  value: unknown,
+  handle: (s: string) => string
+): unknown {
+  if (key === 'traceSpans' && Array.isArray(value)) {
+    return value.map((span) => transformSpan(span, handle))
+  }
+  return transformStrings(value, handle)
+}
+
+/**
+ * Mask a batch of collected strings via Presidio. On a hard failure, either scrub
+ * to {@link REDACTION_FAILED_MARKER} or throw {@link PiiRedactionError}, per
+ * `options.onFailure`. Returns masked values aligned 1:1 with `collected`.
+ *
+ * There is no total-size ceiling — the batching layer chunks the request and
+ * fans out with bounded concurrency, so payloads of any size are masked properly
+ * rather than scrubbed. The per-chunk request timeout is the real backstop.
+ */
+async function maskCollected(
+  collected: string[],
+  options: PiiRedactionOptions
+): Promise<{ masked: string[]; scrubbed: boolean }> {
+  const onFailure = options.onFailure ?? 'scrub'
+  const language = options.language ?? 'en'
+
+  try {
+    // Presidio runs only in the app container; the persist + execution paths also
+    // run in the trigger.dev runtime, so masking always goes over HTTP to the app.
+    const masked = await maskPIIBatchViaHttp(collected, options.entityTypes, language)
+    return { masked, scrubbed: false }
+  } catch (error) {
+    logger.error('PII masking failed', {
+      error: getErrorMessage(error),
+      stringCount: collected.length,
+      onFailure,
+    })
+    if (onFailure === 'throw') {
+      throw new PiiRedactionError(`PII redaction failed: ${getErrorMessage(error)}`)
+    }
+    return { masked: collected.map(() => REDACTION_FAILED_MARKER), scrubbed: true }
+  }
+}
+
+/**
+ * Mask every eligible string leaf of an arbitrary object in place-preserving
+ * fashion: collect all leaves in one deterministic pass, mask in a single batched
+ * Presidio call, then rebuild from the masked slice (the identical traversal
+ * order makes the two passes line up). Used for the execution-altering input and
+ * block-output stages, so it defaults callers toward `onFailure: 'throw'`.
+ */
+export async function redactObjectStrings<T>(value: T, options: PiiRedactionOptions): Promise<T> {
+  const collected: string[] = []
+  transformStrings(value, (s) => {
+    collected.push(s)
+    return s
+  })
+
+  if (collected.length === 0) return value
+
+  const { masked } = await maskCollected(collected, options)
+  let index = 0
+  return transformStrings(value, () => masked[index++]) as T
+}
+
+/**
+ * Mask PII across an execution's `traceSpans` / `finalOutput` / `workflowInput`.
+ *
+ * All eligible string leaves are collected in one deterministic pass and masked
+ * in a single batched (byte-chunked) Presidio call — so subprocess count scales
+ * with payload size, not block count. Each unit is then rebuilt independently
+ * from the masked slice, preserving the JSON structure (Presidio never sees the
+ * envelope). On a hard masking failure or when the payload exceeds the ceiling,
+ * eligible strings are replaced with {@link REDACTION_FAILED_MARKER} (the default
+ * `onFailure: 'scrub'`) rather than left unredacted — PII is never persisted on
+ * the failure path.
+ */
+export async function redactPIIFromExecution(
+  payload: RedactablePayload,
+  options: PiiRedactionOptions
+): Promise<RedactablePayload> {
+  const startedAt = performance.now()
+
+  const units = REDACTABLE_KEYS.filter((key) => payload[key] !== undefined).map((key) => ({
+    key,
+    value: payload[key],
+  }))
+
+  const collected: string[] = []
+  let totalBytes = 0
+  for (const unit of units) {
+    transformUnit(unit.key, unit.value, (s) => {
+      collected.push(s)
+      totalBytes += Buffer.byteLength(s, 'utf8')
+      return s
+    })
+  }
+
+  if (collected.length === 0) return payload
+
+  const { masked, scrubbed } = await maskCollected(collected, options)
+
+  let index = 0
+  const result: RedactablePayload = { ...payload }
+  for (const unit of units) {
+    result[unit.key] = transformUnit(unit.key, unit.value, () => masked[index++])
+  }
+
+  logger.info('PII redaction completed', {
+    stringCount: collected.length,
+    totalBytes,
+    durationMs: Math.round(performance.now() - startedAt),
+    scrubbed,
+  })
+  return result
+}

apps/sim/lib/logs/execution/progress-markers.test.ts

@@ -0,0 +1,203 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ExecutionLastCompletedBlock, ExecutionLastStartedBlock } from '@/lib/logs/types'
+
+const { mockGetRedisClient, mockRedis } = vi.hoisted(() => {
+  const mockRedis = {
+    eval: vi.fn(),
+    hgetall: vi.fn(),
+    del: vi.fn(),
+  }
+  return { mockGetRedisClient: vi.fn<[], typeof mockRedis | null>(() => mockRedis), mockRedis }
+})
+
+vi.mock('@/lib/core/config/redis', () => ({
+  getRedisClient: mockGetRedisClient,
+}))
+
+vi.mock('@/lib/core/execution-limits', () => ({
+  getExecutionReservationTtlMs: () => 5_460_000,
+}))
+
+import {
+  clearProgressMarkers,
+  getProgressMarkers,
+  pickLatestCompletedMarker,
+  pickLatestStartedMarker,
+  setLastCompletedBlock,
+  setLastStartedBlock,
+} from '@/lib/logs/execution/progress-markers'
+
+const EXECUTION_ID = 'exec-1'
+const KEY = `execution:progress:${EXECUTION_ID}`
+const EXPECTED_TTL_MS = '5460000' // getExecutionReservationTtlMs() mock value
+
+const startedMarker: ExecutionLastStartedBlock = {
+  blockId: 'b1',
+  blockName: 'Fetch',
+  blockType: 'api',
+  startedAt: '2026-06-27T10:00:00.000Z',
+}
+
+const completedMarker: ExecutionLastCompletedBlock = {
+  blockId: 'b1',
+  blockName: 'Fetch',
+  blockType: 'api',
+  endedAt: '2026-06-27T10:00:01.000Z',
+  success: true,
+}
+
+describe('progress-markers', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetRedisClient.mockReturnValue(mockRedis)
+    mockRedis.eval.mockResolvedValue(1)
+    mockRedis.hgetall.mockResolvedValue({})
+    mockRedis.del.mockResolvedValue(1)
+  })
+
+  describe('setLastStartedBlock', () => {
+    it('evals the monotonic-guard script with key, started field, timestamp, json, and TTL', async () => {
+      await setLastStartedBlock(EXECUTION_ID, startedMarker)
+
+      expect(mockRedis.eval).toHaveBeenCalledTimes(1)
+      const [, numKeys, key, field, timestampField, timestamp, json, ttl] =
+        mockRedis.eval.mock.calls[0]
+      expect(numKeys).toBe(1)
+      expect(key).toBe(KEY)
+      expect(field).toBe('started')
+      expect(timestampField).toBe('startedAt')
+      expect(timestamp).toBe(startedMarker.startedAt)
+      expect(JSON.parse(json as string)).toEqual(startedMarker)
+      expect(ttl).toBe(EXPECTED_TTL_MS)
+    })
+
+    it('returns true when the Redis write succeeds', async () => {
+      await expect(setLastStartedBlock(EXECUTION_ID, startedMarker)).resolves.toBe(true)
+    })
+
+    it('returns false (caller falls back to SQL) when the eval fails', async () => {
+      mockRedis.eval.mockRejectedValueOnce(new Error('redis down'))
+      await expect(setLastStartedBlock(EXECUTION_ID, startedMarker)).resolves.toBe(false)
+    })
+
+    it('returns false and no-ops when Redis is unavailable', async () => {
+      mockGetRedisClient.mockReturnValue(null)
+      await expect(setLastStartedBlock(EXECUTION_ID, startedMarker)).resolves.toBe(false)
+      expect(mockRedis.eval).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('setLastCompletedBlock', () => {
+    it('evals with the completed field and endedAt timestamp', async () => {
+      await setLastCompletedBlock(EXECUTION_ID, completedMarker)
+
+      const args = mockRedis.eval.mock.calls[0]
+      expect(args[3]).toBe('completed')
+      expect(args[4]).toBe('endedAt')
+      expect(args[5]).toBe(completedMarker.endedAt)
+      expect(JSON.parse(args[6] as string)).toEqual(completedMarker)
+    })
+  })
+
+  describe('getProgressMarkers', () => {
+    it('parses both markers from the hash', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({
+        started: JSON.stringify(startedMarker),
+        completed: JSON.stringify(completedMarker),
+      })
+
+      const result = await getProgressMarkers(EXECUTION_ID)
+      expect(mockRedis.hgetall).toHaveBeenCalledWith(KEY)
+      expect(result).toEqual({
+        lastStartedBlock: startedMarker,
+        lastCompletedBlock: completedMarker,
+      })
+    })
+
+    it('returns only the present field', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({ started: JSON.stringify(startedMarker) })
+      const result = await getProgressMarkers(EXECUTION_ID)
+      expect(result).toEqual({ lastStartedBlock: startedMarker })
+    })
+
+    it('returns {} for an empty / missing key', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({})
+      expect(await getProgressMarkers(EXECUTION_ID)).toEqual({})
+    })
+
+    it('returns {} and does not throw on malformed JSON', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({ started: '{not json' })
+      expect(await getProgressMarkers(EXECUTION_ID)).toEqual({})
+    })
+
+    it('drops wrong-shaped JSON so malformed markers never reach clients', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({
+        started: JSON.stringify('just a string'),
+        completed: JSON.stringify({ blockId: 123, blockName: 'x', blockType: 'api', endedAt: 'z' }),
+      })
+      expect(await getProgressMarkers(EXECUTION_ID)).toEqual({})
+    })
+
+    it('strips extra fields, returning only the validated marker shape', async () => {
+      mockRedis.hgetall.mockResolvedValueOnce({
+        started: JSON.stringify({ ...startedMarker, secret: 'leak', extra: 1 }),
+      })
+      expect(await getProgressMarkers(EXECUTION_ID)).toEqual({ lastStartedBlock: startedMarker })
+    })
+
+    it('returns {} when Redis is unavailable', async () => {
+      mockGetRedisClient.mockReturnValue(null)
+      expect(await getProgressMarkers(EXECUTION_ID)).toEqual({})
+      expect(mockRedis.hgetall).not.toHaveBeenCalled()
+    })
+
+    it('returns null when the Redis read fails so callers do not clear the only copy', async () => {
+      mockRedis.hgetall.mockRejectedValueOnce(new Error('redis down'))
+      expect(await getProgressMarkers(EXECUTION_ID)).toBeNull()
+    })
+  })
+
+  describe('clearProgressMarkers', () => {
+    it('deletes the key', async () => {
+      await clearProgressMarkers(EXECUTION_ID)
+      expect(mockRedis.del).toHaveBeenCalledWith(KEY)
+    })
+
+    it('swallows del errors', async () => {
+      mockRedis.del.mockRejectedValueOnce(new Error('redis down'))
+      await expect(clearProgressMarkers(EXECUTION_ID)).resolves.toBeUndefined()
+    })
+
+    it('no-ops when Redis is unavailable', async () => {
+      mockGetRedisClient.mockReturnValue(null)
+      await clearProgressMarkers(EXECUTION_ID)
+      expect(mockRedis.del).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('latest-wins pickers (stale-store safety)', () => {
+    const older = { ...startedMarker, blockId: 'old', startedAt: '2026-06-27T10:00:00.000Z' }
+    const newer = { ...startedMarker, blockId: 'new', startedAt: '2026-06-27T10:00:05.000Z' }
+
+    it('returns the defined side when the other is undefined', () => {
+      expect(pickLatestStartedMarker(older, undefined)).toBe(older)
+      expect(pickLatestStartedMarker(undefined, newer)).toBe(newer)
+      expect(pickLatestStartedMarker(undefined, undefined)).toBeUndefined()
+    })
+
+    it('picks the later startedAt regardless of argument order (row newer than Redis still wins)', () => {
+      expect(pickLatestStartedMarker(older, newer)).toBe(newer)
+      expect(pickLatestStartedMarker(newer, older)).toBe(newer)
+    })
+
+    it('picks the later endedAt for completed markers', () => {
+      const c1 = { ...completedMarker, endedAt: '2026-06-27T10:00:01.000Z' }
+      const c2 = { ...completedMarker, endedAt: '2026-06-27T10:00:09.000Z' }
+      expect(pickLatestCompletedMarker(c1, c2)).toBe(c2)
+      expect(pickLatestCompletedMarker(c2, c1)).toBe(c2)
+    })
+  })
+})

apps/sim/lib/logs/execution/progress-markers.ts

@@ -0,0 +1,247 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isRecordLike as isRecord } from '@sim/utils/object'
+import { getRedisClient } from '@/lib/core/config/redis'
+import { getExecutionReservationTtlMs } from '@/lib/core/execution-limits'
+import type { ExecutionLastCompletedBlock, ExecutionLastStartedBlock } from '@/lib/logs/types'
+
+const logger = createLogger('ExecutionProgressMarkers')
+
+/**
+ * Live per-block progress markers (`lastStartedBlock` / `lastCompletedBlock`)
+ * used to be written to `workflow_execution_logs.execution_data` via a
+ * `jsonb_set` UPDATE on every block start and complete — ~2N row UPDATEs per
+ * run and the single heaviest write query in the database. They carry no
+ * value beyond a breadcrumb folded into the final record (no client polls
+ * them; live progress comes from the executor over WebSocket), so they live
+ * in Redis during the run and are folded into the single terminal UPDATE at
+ * completion. See the logs-contention plan for the full rationale.
+ */
+
+/** Redis key namespace — matches the `execution:*` family (stream, cancel, budget). */
+const PROGRESS_KEY_PREFIX = 'execution:progress:'
+
+const STARTED_FIELD = 'started'
+const COMPLETED_FIELD = 'completed'
+
+/**
+ * Single source of the client used for progress markers. Indirection kept so a
+ * future split to a dedicated `LOGS_REDIS_URL` is a one-line change here.
+ */
+function getMarkerClient() {
+  return getRedisClient()
+}
+
+function markerKey(executionId: string): string {
+  return `${PROGRESS_KEY_PREFIX}${executionId}`
+}
+
+/**
+ * Atomic monotonic write: set the field only when the incoming marker's embedded
+ * timestamp is >= the stored one, then refresh the TTL — all in one script so
+ * concurrent block callbacks can't race a read-modify-write. Preserves the exact
+ * `<=` ordering the legacy SQL used (`COALESCE(stored, '') <= incoming`). ISO
+ * UTC timestamps compare correctly lexicographically.
+ */
+const SET_MARKER_SCRIPT = `
+local existing = redis.call('HGET', KEYS[1], ARGV[1])
+if existing then
+  local ok, decoded = pcall(cjson.decode, existing)
+  if ok and type(decoded) == 'table' and decoded[ARGV[2]] and tostring(decoded[ARGV[2]]) > ARGV[3] then
+    redis.call('PEXPIRE', KEYS[1], ARGV[5])
+    return 0
+  end
+end
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[4])
+redis.call('PEXPIRE', KEYS[1], ARGV[5])
+return 1
+`
+
+/**
+ * Write a marker field under the monotonic guard, refreshing the key TTL. The
+ * TTL is a backstop for executions that die without a terminal/pause boundary
+ * (deterministic cleanup is {@link clearProgressMarkers}); it mirrors the
+ * admission-reservation TTL so a crashed run's marker key and slot expire
+ * together. Returns `true` only when the marker was durably written to Redis,
+ * so callers can fall back to the SQL path on a missing client or a failure.
+ */
+async function setMarker(
+  executionId: string,
+  field: string,
+  timestampField: 'startedAt' | 'endedAt',
+  timestamp: string,
+  marker: ExecutionLastStartedBlock | ExecutionLastCompletedBlock
+): Promise<boolean> {
+  const redis = getMarkerClient()
+  if (!redis) return false
+
+  try {
+    await redis.eval(
+      SET_MARKER_SCRIPT,
+      1,
+      markerKey(executionId),
+      field,
+      timestampField,
+      timestamp,
+      JSON.stringify(marker),
+      getExecutionReservationTtlMs().toString()
+    )
+    return true
+  } catch (error) {
+    logger.error(`Failed to persist progress marker for execution ${executionId}`, {
+      field,
+      error: toError(error).message,
+    })
+    return false
+  }
+}
+
+/**
+ * Persist the last-started-block marker. Returns `false` (caller should fall
+ * back to the durable SQL path) when Redis is unavailable or the write fails.
+ */
+export async function setLastStartedBlock(
+  executionId: string,
+  marker: ExecutionLastStartedBlock
+): Promise<boolean> {
+  return setMarker(executionId, STARTED_FIELD, 'startedAt', marker.startedAt, marker)
+}
+
+/**
+ * Persist the last-completed-block marker. Returns `false` (caller should fall
+ * back to the durable SQL path) when Redis is unavailable or the write fails.
+ */
+export async function setLastCompletedBlock(
+  executionId: string,
+  marker: ExecutionLastCompletedBlock
+): Promise<boolean> {
+  return setMarker(executionId, COMPLETED_FIELD, 'endedAt', marker.endedAt, marker)
+}
+
+export interface ExecutionProgressMarkers {
+  lastStartedBlock?: ExecutionLastStartedBlock
+  lastCompletedBlock?: ExecutionLastCompletedBlock
+}
+
+/**
+ * Pick the later of two last-started markers by `startedAt`. Markers can split
+ * across stores — a failed Redis write falls back to the row, so an earlier
+ * successful Redis write may coexist with a newer row marker (or vice versa).
+ * Choosing by timestamp keeps the freshest breadcrumb regardless of which store
+ * holds it. ISO UTC timestamps compare correctly lexicographically.
+ */
+export function pickLatestStartedMarker(
+  a: ExecutionLastStartedBlock | undefined,
+  b: ExecutionLastStartedBlock | undefined
+): ExecutionLastStartedBlock | undefined {
+  if (!a) return b
+  if (!b) return a
+  return a.startedAt >= b.startedAt ? a : b
+}
+
+/** Pick the later of two last-completed markers by `endedAt`. See {@link pickLatestStartedMarker}. */
+export function pickLatestCompletedMarker(
+  a: ExecutionLastCompletedBlock | undefined,
+  b: ExecutionLastCompletedBlock | undefined
+): ExecutionLastCompletedBlock | undefined {
+  if (!a) return b
+  if (!b) return a
+  return a.endedAt >= b.endedAt ? a : b
+}
+
+function safeJsonParse(raw: string | undefined): unknown {
+  if (!raw) return undefined
+  try {
+    return JSON.parse(raw)
+  } catch {
+    return undefined
+  }
+}
+
+/**
+ * Parse a stored last-started marker, rebuilding it from validated fields so a
+ * stale or wrong-shaped Redis value can never reach API consumers.
+ */
+function parseStartedMarker(raw: string | undefined): ExecutionLastStartedBlock | undefined {
+  const v = safeJsonParse(raw)
+  if (!isRecord(v)) return undefined
+  const { blockId, blockName, blockType, startedAt } = v
+  if (
+    typeof blockId === 'string' &&
+    typeof blockName === 'string' &&
+    typeof blockType === 'string' &&
+    typeof startedAt === 'string'
+  ) {
+    return { blockId, blockName, blockType, startedAt }
+  }
+  return undefined
+}
+
+/**
+ * Parse a stored last-completed marker, rebuilding it from validated fields so a
+ * stale or wrong-shaped Redis value can never reach API consumers.
+ */
+function parseCompletedMarker(raw: string | undefined): ExecutionLastCompletedBlock | undefined {
+  const v = safeJsonParse(raw)
+  if (!isRecord(v)) return undefined
+  const { blockId, blockName, blockType, endedAt, success } = v
+  if (
+    typeof blockId === 'string' &&
+    typeof blockName === 'string' &&
+    typeof blockType === 'string' &&
+    typeof endedAt === 'string' &&
+    typeof success === 'boolean'
+  ) {
+    return { blockId, blockName, blockType, endedAt, success }
+  }
+  return undefined
+}
+
+/**
+ * Read both markers for an execution. Returns an empty object when Redis is
+ * unavailable (markers were never stored here) or the key holds nothing, and
+ * `null` when the Redis read itself failed — callers must treat `null` as
+ * "unknown" and skip {@link clearProgressMarkers}, so a transient read error
+ * never wipes the only copy of markers that are still in Redis.
+ */
+export async function getProgressMarkers(
+  executionId: string
+): Promise<ExecutionProgressMarkers | null> {
+  const redis = getMarkerClient()
+  if (!redis) return {}
+
+  try {
+    const fields = await redis.hgetall(markerKey(executionId))
+    if (!fields || Object.keys(fields).length === 0) return {}
+
+    const result: ExecutionProgressMarkers = {}
+    const started = parseStartedMarker(fields[STARTED_FIELD])
+    if (started) result.lastStartedBlock = started
+    const completed = parseCompletedMarker(fields[COMPLETED_FIELD])
+    if (completed) result.lastCompletedBlock = completed
+    return result
+  } catch (error) {
+    logger.error(`Failed to read progress markers for execution ${executionId}`, {
+      error: toError(error).message,
+    })
+    return null
+  }
+}
+
+/**
+ * Delete the markers for an execution. Called at every terminal/pause boundary
+ * after the durable record has been written, so paused executions (which can
+ * live indefinitely) hold no Redis keys. Fire-and-forget; no-op without Redis.
+ */
+export async function clearProgressMarkers(executionId: string): Promise<void> {
+  const redis = getMarkerClient()
+  if (!redis) return
+
+  try {
+    await redis.del(markerKey(executionId))
+  } catch (error) {
+    logger.error(`Failed to clear progress markers for execution ${executionId}`, {
+      error: toError(error).message,
+    })
+  }
+}

apps/sim/lib/logs/execution/snapshot/service.test.ts

@@ -0,0 +1,570 @@
+/**
+ * @vitest-environment node
+ */
+import { databaseMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn(() => 'generated-uuid-1'),
+  generateShortId: vi.fn(() => 'generated-short-1'),
+  isValidUuid: vi.fn((v: string) =>
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(v)
+  ),
+}))
+
+import { SnapshotService } from '@/lib/logs/execution/snapshot/service'
+import type { WorkflowState } from '@/lib/logs/types'
+
+const mockState: WorkflowState = {
+  blocks: {
+    block1: {
+      id: 'block1',
+      name: 'Test Agent',
+      type: 'agent',
+      position: { x: 100, y: 200 },
+      subBlocks: {},
+      outputs: {},
+      enabled: true,
+      horizontalHandles: true,
+      advancedMode: false,
+      height: 0,
+    },
+  },
+  edges: [{ id: 'edge1', source: 'block1', target: 'block2' }],
+  loops: {},
+  parallels: {},
+}
+
+describe('SnapshotService', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('computeStateHash', () => {
+    it.concurrent('should generate consistent hashes for identical states', () => {
+      const service = new SnapshotService()
+      const state: WorkflowState = {
+        blocks: {
+          block1: {
+            id: 'block1',
+            name: 'Test Agent',
+            type: 'agent',
+            position: { x: 100, y: 200 },
+
+            subBlocks: {},
+            outputs: {},
+            enabled: true,
+            horizontalHandles: true,
+            advancedMode: false,
+            height: 0,
+          },
+        },
+        edges: [{ id: 'edge1', source: 'block1', target: 'block2' }],
+        loops: {},
+        parallels: {},
+      }
+
+      const hash1 = service.computeStateHash(state)
+      const hash2 = service.computeStateHash(state)
+
+      expect(hash1).toBe(hash2)
+      expect(hash1).toHaveLength(64) // SHA-256 hex string
+    })
+
+    it.concurrent('should ignore position changes', () => {
+      const service = new SnapshotService()
+      const baseState: WorkflowState = {
+        blocks: {
+          block1: {
+            id: 'block1',
+            name: 'Test Agent',
+            type: 'agent',
+            position: { x: 100, y: 200 },
+
+            subBlocks: {},
+            outputs: {},
+            enabled: true,
+            horizontalHandles: true,
+            advancedMode: false,
+            height: 0,
+          },
+        },
+        edges: [],
+        loops: {},
+        parallels: {},
+      }
+
+      const stateWithDifferentPosition: WorkflowState = {
+        ...baseState,
+        blocks: {
+          block1: {
+            ...baseState.blocks.block1,
+            position: { x: 500, y: 600 },
+          },
+        },
+      }
+
+      const hash1 = service.computeStateHash(baseState)
+      const hash2 = service.computeStateHash(stateWithDifferentPosition)
+
+      expect(hash1).toBe(hash2)
+    })
+
+    it.concurrent('should detect meaningful changes', () => {
+      const service = new SnapshotService()
+      const baseState: WorkflowState = {
+        blocks: {
+          block1: {
+            id: 'block1',
+            name: 'Test Agent',
+            type: 'agent',
+            position: { x: 100, y: 200 },
+
+            subBlocks: {
+              prompt: {
+                id: 'prompt',
+                type: 'short-input',
+                value: 'Hello world',
+              },
+            },
+            outputs: {},
+            enabled: true,
+            horizontalHandles: true,
+            advancedMode: false,
+            height: 0,
+          },
+        },
+        edges: [],
+        loops: {},
+        parallels: {},
+      }
+
+      const stateWithDifferentPrompt: WorkflowState = {
+        ...baseState,
+        blocks: {
+          block1: {
+            ...baseState.blocks.block1,
+            // Different subBlock value - this is a meaningful change
+            subBlocks: {
+              prompt: {
+                id: 'prompt',
+                type: 'short-input',
+                value: 'Different prompt',
+              },
+            },
+          },
+        },
+      }
+
+      const hash1 = service.computeStateHash(baseState)
+      const hash2 = service.computeStateHash(stateWithDifferentPrompt)
+
+      expect(hash1).not.toBe(hash2)
+    })
+
+    it.concurrent('should handle edge order consistently', () => {
+      const service = new SnapshotService()
+      const state1: WorkflowState = {
+        blocks: {},
+        edges: [
+          { id: 'edge1', source: 'a', target: 'b' },
+          { id: 'edge2', source: 'b', target: 'c' },
+        ],
+        loops: {},
+        parallels: {},
+      }
+
+      const state2: WorkflowState = {
+        blocks: {},
+        edges: [
+          { id: 'edge2', source: 'b', target: 'c' },
+          { id: 'edge1', source: 'a', target: 'b' },
+        ],
+        loops: {},
+        parallels: {},
+      }
+
+      const hash1 = service.computeStateHash(state1)
+      const hash2 = service.computeStateHash(state2)
+
+      expect(hash1).toBe(hash2) // Should be same despite different order
+    })
+
+    it.concurrent('should handle empty states', () => {
+      const service = new SnapshotService()
+      const emptyState: WorkflowState = {
+        blocks: {},
+        edges: [],
+        loops: {},
+        parallels: {},
+      }
+
+      const hash = service.computeStateHash(emptyState)
+      expect(hash).toHaveLength(64)
+    })
+
+    it.concurrent('should handle complex nested structures', () => {
+      const service = new SnapshotService()
+      const complexState: WorkflowState = {
+        blocks: {
+          block1: {
+            id: 'block1',
+            name: 'Complex Agent',
+            type: 'agent',
+            position: { x: 100, y: 200 },
+
+            subBlocks: {
+              prompt: {
+                id: 'prompt',
+                type: 'short-input',
+                value: 'Test prompt',
+              },
+              model: {
+                id: 'model',
+                type: 'short-input',
+                value: 'gpt-4',
+              },
+            },
+            outputs: {
+              response: { type: 'string', description: 'Agent response' },
+            },
+            enabled: true,
+            horizontalHandles: true,
+            advancedMode: true,
+            height: 200,
+          },
+        },
+        edges: [{ id: 'edge1', source: 'block1', target: 'block2', sourceHandle: 'output' }],
+        loops: {
+          loop1: {
+            id: 'loop1',
+            nodes: ['block1'],
+            iterations: 10,
+            loopType: 'for',
+          },
+        },
+        parallels: {
+          parallel1: {
+            id: 'parallel1',
+            nodes: ['block1'],
+            count: 3,
+            parallelType: 'count',
+          },
+        },
+      }
+
+      const hash = service.computeStateHash(complexState)
+      expect(hash).toHaveLength(64)
+
+      const hash2 = service.computeStateHash(complexState)
+      expect(hash).toBe(hash2)
+    })
+
+    it.concurrent('should include variables in hash computation', () => {
+      const service = new SnapshotService()
+      const stateWithVariables: WorkflowState = {
+        blocks: {},
+        edges: [],
+        loops: {},
+        parallels: {},
+        variables: {
+          'var-1': {
+            id: 'var-1',
+            name: 'apiKey',
+            type: 'string',
+            value: 'secret123',
+          },
+        },
+      }
+
+      const stateWithoutVariables: WorkflowState = {
+        blocks: {},
+        edges: [],
+        loops: {},
+        parallels: {},
+      }
+
+      const hashWith = service.computeStateHash(stateWithVariables)
+      const hashWithout = service.computeStateHash(stateWithoutVariables)
+
+      expect(hashWith).not.toBe(hashWithout)
+    })
+
+    it.concurrent('should detect changes in variable values', () => {
+      const service = new SnapshotService()
+      const state1: WorkflowState = {
+        blocks: {},
+        edges: [],
+        loops: {},
+        parallels: {},
+        variables: {
+          'var-1': {
+            id: 'var-1',
+            name: 'myVar',
+            type: 'string',
+            value: 'value1',
+          },
+        },
+      }
+
+      const state2: WorkflowState = {
+        blocks: {},
+        edges: [],
+        loops: {},
+        parallels: {},
+        variables: {
+          'var-1': {
+            id: 'var-1',
+            name: 'myVar',
+            type: 'string',
+            value: 'value2', // Different value
+          },
+        },
+      }
+
+      const hash1 = service.computeStateHash(state1)
+      const hash2 = service.computeStateHash(state2)
+
+      expect(hash1).not.toBe(hash2)
+    })
+
+    it.concurrent('should generate consistent hashes for states with variables', () => {
+      const service = new SnapshotService()
+      const stateWithVariables: WorkflowState = {
+        blocks: {
+          block1: {
+            id: 'block1',
+            name: 'Test',
+            type: 'agent',
+            position: { x: 0, y: 0 },
+            subBlocks: {},
+            outputs: {},
+            enabled: true,
+            horizontalHandles: true,
+            advancedMode: false,
+            height: 0,
+          },
+        },
+        edges: [],
+        loops: {},
+        parallels: {},
+        variables: {
+          'var-1': {
+            id: 'var-1',
+            name: 'testVar',
+            type: 'plain',
+            value: 'testValue',
+          },
+          'var-2': {
+            id: 'var-2',
+            name: 'anotherVar',
+            type: 'number',
+            value: 42,
+          },
+        },
+      }
+
+      const hash1 = service.computeStateHash(stateWithVariables)
+      const hash2 = service.computeStateHash(stateWithVariables)
+
+      expect(hash1).toBe(hash2)
+      expect(hash1).toHaveLength(64)
+    })
+  })
+
+  describe('createSnapshotWithDeduplication', () => {
+    type SnapshotRow = {
+      id: string
+      workflowId: string
+      stateHash: string
+      stateData: WorkflowState
+      createdAt: Date
+    }
+
+    /** Mock the insert → values → onConflictDoUpdate → returning chain. */
+    function mockUpsertReturning(rows: SnapshotRow[]) {
+      let capturedConflictConfig: Record<string, unknown> | undefined
+      const onConflictDoUpdate = vi.fn().mockImplementation((config: Record<string, unknown>) => {
+        capturedConflictConfig = config
+        return { returning: vi.fn().mockResolvedValue(rows) }
+      })
+      const values = vi.fn().mockReturnValue({ onConflictDoUpdate })
+      databaseMock.db.insert = vi.fn().mockReturnValue({ values })
+      databaseMock.db.select = vi.fn()
+      return { values, onConflictDoUpdate, getConflictConfig: () => capturedConflictConfig }
+    }
+
+    it('inserts a new snapshot in a single atomic upsert', async () => {
+      const service = new SnapshotService()
+      const workflowId = 'wf-123'
+
+      const { values } = mockUpsertReturning([
+        {
+          id: 'generated-uuid-1',
+          workflowId,
+          stateHash: 'abc123',
+          stateData: mockState,
+          createdAt: new Date('2026-02-19T00:00:00Z'),
+        },
+      ])
+
+      const result = await service.createSnapshotWithDeduplication(workflowId, mockState)
+
+      expect(values).toHaveBeenCalledWith(
+        expect.objectContaining({ id: 'generated-uuid-1', workflowId, stateData: mockState })
+      )
+      expect(result.snapshot.id).toBe('generated-uuid-1')
+      expect(result.isNew).toBe(true)
+      // Single atomic statement — never a follow-up select (which would race with cleanup).
+      expect(databaseMock.db.select).not.toHaveBeenCalled()
+    })
+
+    it('reuses the existing snapshot atomically when the returned id differs', async () => {
+      const service = new SnapshotService()
+      const workflowId = 'wf-123'
+
+      mockUpsertReturning([
+        {
+          id: 'existing-snapshot-id',
+          workflowId,
+          stateHash: 'abc123',
+          stateData: mockState,
+          createdAt: new Date('2026-02-19T00:00:00Z'),
+        },
+      ])
+
+      const result = await service.createSnapshotWithDeduplication(workflowId, mockState)
+
+      expect(result.snapshot.id).toBe('existing-snapshot-id')
+      expect(result.isNew).toBe(false)
+      expect(databaseMock.db.select).not.toHaveBeenCalled()
+    })
+
+    it('SET targets only state_hash on conflict, never the large state_data', async () => {
+      const service = new SnapshotService()
+      const workflowId = 'wf-123'
+
+      const { onConflictDoUpdate, getConflictConfig } = mockUpsertReturning([
+        {
+          id: 'generated-uuid-1',
+          workflowId,
+          stateHash: 'abc123',
+          stateData: mockState,
+          createdAt: new Date('2026-02-19T00:00:00Z'),
+        },
+      ])
+
+      await service.createSnapshotWithDeduplication(workflowId, mockState)
+
+      expect(onConflictDoUpdate).toHaveBeenCalledTimes(1)
+      const config = getConflictConfig()
+      expect(config?.target).toBeDefined()
+      // The crux of this change: the SET touches state_hash only, so the unchanged
+      // TOASTed state_data jsonb is never rewritten.
+      expect(config?.set).toHaveProperty('stateHash')
+      expect(config?.set).not.toHaveProperty('stateData')
+    })
+
+    it('does not throw on concurrent inserts with the same hash', async () => {
+      const service = new SnapshotService()
+      const workflowId = 'wf-123'
+
+      const newRow: SnapshotRow = {
+        id: 'generated-uuid-1',
+        workflowId,
+        stateHash: 'abc123',
+        stateData: mockState,
+        createdAt: new Date('2026-02-19T00:00:00Z'),
+      }
+      const existingRow: SnapshotRow = { ...newRow, id: 'existing-snapshot-id' }
+
+      let upsertCall = 0
+      databaseMock.db.insert = vi.fn().mockImplementation(() => ({
+        values: vi.fn().mockReturnValue({
+          onConflictDoUpdate: vi.fn().mockReturnValue({
+            returning: vi.fn().mockResolvedValue(upsertCall++ === 0 ? [newRow] : [existingRow]),
+          }),
+        }),
+      }))
+
+      const [result1, result2] = await Promise.all([
+        service.createSnapshotWithDeduplication(workflowId, mockState),
+        service.createSnapshotWithDeduplication(workflowId, mockState),
+      ])
+
+      expect(result1.snapshot.id).toBe('generated-uuid-1')
+      expect(result1.isNew).toBe(true)
+      expect(result2.snapshot.id).toBe('existing-snapshot-id')
+      expect(result2.isNew).toBe(false)
+    })
+  })
+
+  describe('cleanupOrphanedSnapshots', () => {
+    function setupCleanupMocks(selectBatches: Array<Array<{ id: string }>>) {
+      const limitFn = vi.fn()
+      for (const batch of selectBatches) limitFn.mockResolvedValueOnce(batch)
+      limitFn.mockResolvedValue([])
+      const whereSelect = vi.fn().mockReturnValue({ limit: limitFn })
+      const fromFn = vi.fn().mockReturnValue({ where: whereSelect })
+      databaseMock.db.select = vi.fn().mockReturnValue({ from: fromFn })
+
+      const returningFn = vi.fn().mockImplementation(() => Promise.resolve([]))
+      const whereDelete = vi.fn().mockReturnValue({ returning: returningFn })
+      let batchIdx = 0
+      const deleteFn = vi.fn().mockImplementation(() => {
+        const batch = selectBatches[batchIdx] ?? []
+        batchIdx++
+        returningFn.mockImplementationOnce(() => Promise.resolve(batch.map((r) => ({ id: r.id }))))
+        return { where: whereDelete }
+      })
+      databaseMock.db.delete = deleteFn
+
+      return { deleteFn }
+    }
+
+    it('returns 0 and skips delete when nothing is orphaned', async () => {
+      const service = new SnapshotService()
+      const { deleteFn } = setupCleanupMocks([])
+
+      const count = await service.cleanupOrphanedSnapshots(7)
+
+      expect(count).toBe(0)
+      expect(deleteFn).not.toHaveBeenCalled()
+    })
+
+    it('stops after the first short batch', async () => {
+      const service = new SnapshotService()
+      const partial = Array.from({ length: 3 }, (_, i) => ({ id: `s${i}` }))
+      const { deleteFn } = setupCleanupMocks([partial])
+
+      const count = await service.cleanupOrphanedSnapshots(7)
+
+      expect(count).toBe(3)
+      expect(deleteFn).toHaveBeenCalledTimes(1)
+    })
+
+    it('loops through multiple full batches until exhausted', async () => {
+      const service = new SnapshotService()
+      const fullBatch = Array.from({ length: 1000 }, (_, i) => ({ id: `s${i}` }))
+      const tail = [{ id: 'tail-1' }]
+      const { deleteFn } = setupCleanupMocks([fullBatch, fullBatch, tail])
+
+      const count = await service.cleanupOrphanedSnapshots(7)
+
+      expect(count).toBe(2001)
+      expect(deleteFn).toHaveBeenCalledTimes(3)
+    })
+
+    it('caps at MAX_BATCHES (20 × 1000) even when more rows remain', async () => {
+      const service = new SnapshotService()
+      const fullBatch = Array.from({ length: 1000 }, (_, i) => ({ id: `s${i}` }))
+      const batches = Array.from({ length: 25 }, () => fullBatch)
+      const { deleteFn } = setupCleanupMocks(batches)
+
+      const count = await service.cleanupOrphanedSnapshots(7)
+
+      expect(count).toBe(20_000)
+      expect(deleteFn).toHaveBeenCalledTimes(20)
+    })
+  })
+})

apps/sim/lib/logs/execution/snapshot/service.ts

@@ -0,0 +1,163 @@
+import { db } from '@sim/db'
+import { workflowExecutionLogs, workflowExecutionSnapshots } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { sha256Hex } from '@sim/security/hash'
+import { generateId } from '@sim/utils/id'
+import { and, eq, inArray, lt, notExists, sql } from 'drizzle-orm'
+import type {
+  SnapshotService as ISnapshotService,
+  SnapshotCreationResult,
+  WorkflowExecutionSnapshot,
+  WorkflowExecutionSnapshotInsert,
+  WorkflowState,
+} from '@/lib/logs/types'
+import { normalizedStringify, normalizeWorkflowState } from '@/lib/workflows/comparison'
+
+const logger = createLogger('SnapshotService')
+
+export class SnapshotService implements ISnapshotService {
+  async createSnapshot(
+    workflowId: string,
+    state: WorkflowState
+  ): Promise<WorkflowExecutionSnapshot> {
+    const result = await this.createSnapshotWithDeduplication(workflowId, state)
+    return result.snapshot
+  }
+
+  async createSnapshotWithDeduplication(
+    workflowId: string,
+    state: WorkflowState
+  ): Promise<SnapshotCreationResult> {
+    const stateHash = this.computeStateHash(state)
+
+    const snapshotData: WorkflowExecutionSnapshotInsert = {
+      id: generateId(),
+      workflowId,
+      stateHash,
+      stateData: state,
+    }
+
+    /**
+     * Insert the snapshot, or — when an identical (workflowId, stateHash) row
+     * already exists — return it without rewriting the large stateData jsonb.
+     *
+     * The hash is a sha256 of the normalized state, so an existing row's stateData
+     * is byte-identical; there is nothing to update. The previous implementation
+     * SET state_data on conflict, which rewrote the full (tens-of-KB) jsonb every
+     * run. We keep a single atomic upsert — so RETURNING always yields the row and
+     * there is no race with snapshot cleanup (unlike DO NOTHING + a follow-up
+     * select) — but SET only the small state_hash column to itself. Under Postgres
+     * MVCC the unchanged, TOASTed stateData is not rewritten: its existing
+     * out-of-line storage is reused, so the per-execution write drops from the
+     * full blob to a tiny heap tuple.
+     */
+    const [upsertedSnapshot] = await db
+      .insert(workflowExecutionSnapshots)
+      .values(snapshotData)
+      .onConflictDoUpdate({
+        target: [workflowExecutionSnapshots.workflowId, workflowExecutionSnapshots.stateHash],
+        set: {
+          stateHash: sql`excluded.state_hash`,
+        },
+      })
+      .returning()
+
+    const isNew = upsertedSnapshot.id === snapshotData.id
+
+    logger.info(
+      isNew
+        ? `Created new snapshot for workflow ${workflowId} (hash: ${stateHash.slice(0, 12)}..., blocks: ${Object.keys(state.blocks || {}).length})`
+        : `Reusing existing snapshot for workflow ${workflowId} (hash: ${stateHash.slice(0, 12)}...)`
+    )
+
+    return {
+      snapshot: {
+        ...upsertedSnapshot,
+        stateData: upsertedSnapshot.stateData as WorkflowState,
+        createdAt: upsertedSnapshot.createdAt.toISOString(),
+      },
+      isNew,
+    }
+  }
+
+  async getSnapshot(id: string): Promise<WorkflowExecutionSnapshot | null> {
+    const [snapshot] = await db
+      .select()
+      .from(workflowExecutionSnapshots)
+      .where(eq(workflowExecutionSnapshots.id, id))
+      .limit(1)
+
+    if (!snapshot) return null
+
+    return {
+      ...snapshot,
+      stateData: snapshot.stateData as WorkflowState,
+      createdAt: snapshot.createdAt.toISOString(),
+    }
+  }
+
+  computeStateHash(state: WorkflowState): string {
+    const normalizedState = normalizeWorkflowState(state)
+    const stateString = normalizedStringify(normalizedState)
+    return sha256Hex(stateString)
+  }
+
+  async cleanupOrphanedSnapshots(olderThanDays: number): Promise<number> {
+    const cutoffDate = new Date()
+    cutoffDate.setDate(cutoffDate.getDate() - olderThanDays)
+
+    const BATCH_SIZE = 1000
+    const MAX_BATCHES = 20
+
+    let totalDeleted = 0
+    let stoppedEarly = false
+
+    for (let batch = 0; batch < MAX_BATCHES; batch++) {
+      const candidates = await db
+        .select({ id: workflowExecutionSnapshots.id })
+        .from(workflowExecutionSnapshots)
+        .where(
+          and(
+            lt(workflowExecutionSnapshots.createdAt, cutoffDate),
+            notExists(
+              db
+                .select({ one: sql`1` })
+                .from(workflowExecutionLogs)
+                .where(eq(workflowExecutionLogs.stateSnapshotId, workflowExecutionSnapshots.id))
+            )
+          )
+        )
+        .limit(BATCH_SIZE)
+
+      if (candidates.length === 0) break
+
+      const ids = candidates.map((c) => c.id)
+      const deleted = await db
+        .delete(workflowExecutionSnapshots)
+        .where(
+          and(
+            inArray(workflowExecutionSnapshots.id, ids),
+            notExists(
+              db
+                .select({ one: sql`1` })
+                .from(workflowExecutionLogs)
+                .where(eq(workflowExecutionLogs.stateSnapshotId, workflowExecutionSnapshots.id))
+            )
+          )
+        )
+        .returning({ id: workflowExecutionSnapshots.id })
+
+      totalDeleted += deleted.length
+
+      if (candidates.length < BATCH_SIZE) break
+      if (batch === MAX_BATCHES - 1) stoppedEarly = true
+    }
+
+    logger.info(
+      `Cleaned up ${totalDeleted} orphaned snapshots older than ${olderThanDays} days${stoppedEarly ? ' (batch cap reached, remainder deferred to next run)' : ''}`
+    )
+    return totalDeleted
+  }
+}
+
+export const snapshotService = new SnapshotService()

apps/sim/lib/logs/execution/trace-spans/iteration-grouping.ts

@@ -0,0 +1,324 @@
+import { createLogger } from '@sim/logger'
+import type { TraceSpan } from '@/lib/logs/types'
+import { stripCloneSuffixes } from '@/executor/utils/subflow-utils'
+
+const logger = createLogger('IterationGrouping')
+
+/** Counter state for generating sequential container names. */
+interface ContainerNameCounters {
+  loopNumbers: Map<string, number>
+  parallelNumbers: Map<string, number>
+  loopCounter: number
+  parallelCounter: number
+}
+
+/**
+ * Builds a container-level TraceSpan (iteration wrapper or top-level container)
+ * from its source spans and resolved children.
+ */
+function buildContainerSpan(opts: {
+  id: string
+  name: string
+  type: string
+  sourceSpans: TraceSpan[]
+  children: TraceSpan[]
+}): TraceSpan {
+  const startTimes = opts.sourceSpans.map((s) => new Date(s.startTime).getTime())
+  const endTimes = opts.sourceSpans.map((s) => new Date(s.endTime).getTime())
+
+  // Guard against empty sourceSpans — Math.min/max of empty array returns ±Infinity
+  // which produces NaN durations and invalid Dates downstream.
+  const nowMs = Date.now()
+  const earliestStart = startTimes.length > 0 ? Math.min(...startTimes) : nowMs
+  const latestEnd = endTimes.length > 0 ? Math.max(...endTimes) : nowMs
+
+  const hasErrors = opts.sourceSpans.some((s) => s.status === 'error')
+  const allErrorsHandled =
+    hasErrors && opts.children.every((s) => s.status !== 'error' || s.errorHandled)
+
+  return {
+    id: opts.id,
+    name: opts.name,
+    type: opts.type,
+    duration: Math.max(0, latestEnd - earliestStart),
+    startTime: new Date(earliestStart).toISOString(),
+    endTime: new Date(latestEnd).toISOString(),
+    status: hasErrors ? 'error' : 'success',
+    ...(allErrorsHandled && { errorHandled: true }),
+    children: opts.children,
+  }
+}
+
+/**
+ * Resolves a container name from normal (non-iteration) spans or assigns a sequential number.
+ * Strips clone suffixes so all clones of the same container share one name/number.
+ */
+function resolveContainerName(
+  containerId: string,
+  containerType: 'parallel' | 'loop',
+  normalSpans: TraceSpan[],
+  counters: ContainerNameCounters
+): string {
+  const originalId = stripCloneSuffixes(containerId)
+
+  const matchingBlock = normalSpans.find(
+    (s) => s.blockId === originalId && s.type === containerType
+  )
+  if (matchingBlock?.name) return matchingBlock.name
+
+  if (containerType === 'parallel') {
+    if (!counters.parallelNumbers.has(originalId)) {
+      counters.parallelNumbers.set(originalId, counters.parallelCounter++)
+    }
+    return `Parallel ${counters.parallelNumbers.get(originalId)}`
+  }
+  if (!counters.loopNumbers.has(originalId)) {
+    counters.loopNumbers.set(originalId, counters.loopCounter++)
+  }
+  return `Loop ${counters.loopNumbers.get(originalId)}`
+}
+
+/**
+ * Classifies a span's immediate container ID and type from its metadata.
+ * Returns undefined for non-iteration spans.
+ */
+function classifySpanContainer(
+  span: TraceSpan
+): { containerId: string; containerType: 'parallel' | 'loop' } | undefined {
+  if (span.parallelId) {
+    return { containerId: span.parallelId, containerType: 'parallel' }
+  }
+  if (span.loopId) {
+    return { containerId: span.loopId, containerType: 'loop' }
+  }
+  if (span.blockId?.includes('_parallel_')) {
+    const match = span.blockId.match(/_parallel_([^_]+)_iteration_/)
+    if (match) {
+      return { containerId: match[1], containerType: 'parallel' }
+    }
+  }
+  return undefined
+}
+
+/**
+ * Finds the outermost container for a span. For nested spans, this is parentIterations[0].
+ * For flat spans, this is the span's own immediate container.
+ */
+function getOutermostContainer(
+  span: TraceSpan
+): { containerId: string; containerType: 'parallel' | 'loop' } | undefined {
+  if (span.parentIterations && span.parentIterations.length > 0) {
+    const outermost = span.parentIterations[0]
+    return {
+      containerId: outermost.iterationContainerId,
+      containerType: outermost.iterationType as 'parallel' | 'loop',
+    }
+  }
+  return classifySpanContainer(span)
+}
+
+/**
+ * Builds the iteration-level hierarchy for a container, recursively nesting
+ * any deeper subflows. Works with both:
+ * - Direct spans (spans whose immediate container matches)
+ * - Nested spans (spans with parentIterations pointing through this container)
+ */
+function buildContainerChildren(
+  containerType: 'parallel' | 'loop',
+  containerId: string,
+  spans: TraceSpan[],
+  normalSpans: TraceSpan[],
+  counters: ContainerNameCounters
+): TraceSpan[] {
+  const iterationType = containerType === 'parallel' ? 'parallel-iteration' : 'loop-iteration'
+
+  const iterationGroups = new Map<number, TraceSpan[]>()
+
+  for (const span of spans) {
+    let iterIdx: number | undefined
+
+    if (
+      span.parentIterations &&
+      span.parentIterations.length > 0 &&
+      span.parentIterations[0].iterationContainerId === containerId
+    ) {
+      iterIdx = span.parentIterations[0].iterationCurrent
+    } else {
+      iterIdx = span.iterationIndex
+    }
+
+    if (iterIdx === undefined) {
+      logger.warn('Skipping iteration span without iterationIndex', {
+        spanId: span.id,
+        blockId: span.blockId,
+        containerId,
+      })
+      continue
+    }
+
+    if (!iterationGroups.has(iterIdx)) iterationGroups.set(iterIdx, [])
+    iterationGroups.get(iterIdx)!.push(span)
+  }
+
+  const iterationChildren: TraceSpan[] = []
+  const sortedIterations = Array.from(iterationGroups.entries()).sort(([a], [b]) => a - b)
+
+  for (const [iterationIndex, iterSpans] of sortedIterations) {
+    const directLeaves: TraceSpan[] = []
+    const deeperSpans: TraceSpan[] = []
+
+    for (const span of iterSpans) {
+      if (
+        span.parentIterations &&
+        span.parentIterations.length > 0 &&
+        span.parentIterations[0].iterationContainerId === containerId
+      ) {
+        deeperSpans.push({
+          ...span,
+          parentIterations: span.parentIterations.slice(1),
+        })
+      } else {
+        directLeaves.push({
+          ...span,
+          name: span.name.replace(/ \(iteration \d+\)$/, ''),
+        })
+      }
+    }
+
+    const nestedResult = groupIterationBlocksRecursive(
+      [...directLeaves, ...deeperSpans],
+      normalSpans,
+      counters
+    )
+
+    iterationChildren.push(
+      buildContainerSpan({
+        id: `${containerId}-iteration-${iterationIndex}`,
+        name: `Iteration ${iterationIndex}`,
+        type: iterationType,
+        sourceSpans: iterSpans,
+        children: nestedResult,
+      })
+    )
+  }
+
+  return iterationChildren
+}
+
+/**
+ * Core recursive algorithm for grouping iteration blocks.
+ *
+ * Handles two cases:
+ * 1. **Flat** (backward compat): spans have loopId/parallelId + iterationIndex but no
+ *    parentIterations. Grouped by immediate container -> iteration -> leaf.
+ * 2. **Nested** (new): spans have parentIterations chains. The outermost ancestor in the
+ *    chain determines the top-level container. Iteration spans are peeled one level at a
+ *    time and recursed.
+ */
+function groupIterationBlocksRecursive(
+  spans: TraceSpan[],
+  normalSpans: TraceSpan[],
+  counters: ContainerNameCounters
+): TraceSpan[] {
+  const result: TraceSpan[] = []
+  const iterationSpans: TraceSpan[] = []
+  const nonIterationSpans: TraceSpan[] = []
+
+  for (const span of spans) {
+    if (
+      (span.name.match(/^(.+) \(iteration (\d+)\)$/) &&
+        (span.loopId || span.parallelId || span.blockId?.includes('_parallel_'))) ||
+      (span.parentIterations && span.parentIterations.length > 0)
+    ) {
+      iterationSpans.push(span)
+    } else {
+      nonIterationSpans.push(span)
+    }
+  }
+
+  const containerIdsWithIterations = new Set<string>()
+  for (const span of iterationSpans) {
+    const outermost = getOutermostContainer(span)
+    if (outermost) containerIdsWithIterations.add(outermost.containerId)
+  }
+
+  const nonContainerSpans = nonIterationSpans.filter(
+    (span) =>
+      (span.type !== 'parallel' && span.type !== 'loop') ||
+      span.status === 'error' ||
+      (span.blockId && !containerIdsWithIterations.has(span.blockId))
+  )
+
+  if (iterationSpans.length === 0) {
+    result.push(...nonContainerSpans)
+    result.sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime())
+    return result
+  }
+
+  const containerGroups = new Map<
+    string,
+    { type: 'parallel' | 'loop'; containerId: string; containerName: string; spans: TraceSpan[] }
+  >()
+
+  for (const span of iterationSpans) {
+    const outermost = getOutermostContainer(span)
+    if (!outermost) continue
+
+    const { containerId, containerType } = outermost
+    const groupKey = `${containerType}_${containerId}`
+
+    if (!containerGroups.has(groupKey)) {
+      const containerName = resolveContainerName(containerId, containerType, normalSpans, counters)
+      containerGroups.set(groupKey, {
+        type: containerType,
+        containerId,
+        containerName,
+        spans: [],
+      })
+    }
+    containerGroups.get(groupKey)!.spans.push(span)
+  }
+
+  for (const [, group] of containerGroups) {
+    const { type, containerId, containerName, spans: containerSpans } = group
+
+    const iterationChildren = buildContainerChildren(
+      type,
+      containerId,
+      containerSpans,
+      normalSpans,
+      counters
+    )
+
+    result.push(
+      buildContainerSpan({
+        id: `${type === 'parallel' ? 'parallel' : 'loop'}-execution-${containerId}`,
+        name: containerName,
+        type,
+        sourceSpans: containerSpans,
+        children: iterationChildren,
+      })
+    )
+  }
+
+  result.push(...nonContainerSpans)
+  result.sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime())
+
+  return result
+}
+
+/**
+ * Groups iteration-based blocks (parallel and loop) by organizing their iteration spans
+ * into a hierarchical structure with proper parent-child relationships.
+ * Supports recursive nesting via parentIterations (e.g., parallel-in-parallel, loop-in-loop).
+ */
+export function groupIterationBlocks(spans: TraceSpan[]): TraceSpan[] {
+  const normalSpans = spans.filter((s) => !s.name.match(/^(.+) \(iteration (\d+)\)$/))
+  const counters: ContainerNameCounters = {
+    loopNumbers: new Map<string, number>(),
+    parallelNumbers: new Map<string, number>(),
+    loopCounter: 1,
+    parallelCounter: 1,
+  }
+  return groupIterationBlocksRecursive(spans, normalSpans, counters)
+}

apps/sim/lib/logs/execution/trace-spans/span-factory.ts

@@ -0,0 +1,390 @@
+import { createLogger } from '@sim/logger'
+import type { ProviderTiming, TraceSpan } from '@/lib/logs/types'
+import {
+  isConditionBlockType,
+  isWorkflowBlockType,
+  stripCustomToolPrefix,
+} from '@/executor/constants'
+import type {
+  BlockLog,
+  BlockToolCall,
+  NormalizedBlockOutput,
+  ProviderTimingSegment,
+} from '@/executor/types'
+
+const logger = createLogger('SpanFactory')
+
+/** A BlockLog that has already passed the id/type validity check. */
+type ValidBlockLog = BlockLog & { blockType: string }
+
+/**
+ * Creates a TraceSpan from a BlockLog. Returns null for invalid logs.
+ *
+ * Children are unified under `span.children` regardless of source:
+ *   - Provider `timeSegments` become model/tool child spans with tool I/O merged in
+ *   - `output.toolCalls` (no segments) become tool child spans
+ *   - Child workflow spans are flattened into children
+ */
+export function createSpanFromLog(log: BlockLog): TraceSpan | null {
+  if (!log.blockId || !log.blockType) return null
+  const validLog = log as ValidBlockLog
+
+  const span = createBaseSpan(validLog)
+
+  if (!isConditionBlockType(validLog.blockType)) {
+    enrichWithProviderMetadata(span, validLog)
+
+    if (!isWorkflowBlockType(validLog.blockType)) {
+      const segments = validLog.output?.providerTiming?.timeSegments
+      span.children = segments
+        ? buildChildrenFromTimeSegments(span, validLog, segments)
+        : buildChildrenFromToolCalls(span, validLog)
+    }
+  }
+
+  if (isWorkflowBlockType(validLog.blockType)) {
+    attachChildWorkflowSpans(span, validLog)
+  }
+
+  return span
+}
+
+/** Creates the base span with id, name, type, timing, status, and metadata. */
+function createBaseSpan(log: ValidBlockLog): TraceSpan {
+  const spanId = `${log.blockId}-${new Date(log.startedAt).getTime()}`
+  const output = extractDisplayOutput(log)
+  const childIds = extractChildWorkflowIds(log.output)
+
+  return {
+    id: spanId,
+    name: log.blockName ?? log.blockId,
+    type: log.blockType,
+    duration: log.durationMs,
+    startTime: log.startedAt,
+    endTime: log.endedAt,
+    status: log.error ? 'error' : 'success',
+    children: [],
+    blockId: log.blockId,
+    executionOrder: log.executionOrder,
+    input: log.input,
+    output,
+    ...(childIds ?? {}),
+    ...(log.errorHandled && { errorHandled: true }),
+    ...(log.loopId && { loopId: log.loopId }),
+    ...(log.parallelId && { parallelId: log.parallelId }),
+    ...(log.iterationIndex !== undefined && { iterationIndex: log.iterationIndex }),
+    ...(log.parentIterations?.length && { parentIterations: log.parentIterations }),
+  }
+}
+
+/**
+ * Strips internal fields from the block output for display and merges
+ * the block-level error into output so the UI renders it alongside data.
+ */
+function extractDisplayOutput(log: ValidBlockLog): Record<string, unknown> {
+  const { childWorkflowSnapshotId, childWorkflowId, ...rest } = log.output ?? {}
+  return log.error ? { ...rest, error: log.error } : rest
+}
+
+/** Pulls child-workflow identifiers off the output so they can live on the span. */
+function extractChildWorkflowIds(
+  output: NormalizedBlockOutput | undefined
+): { childWorkflowSnapshotId?: string; childWorkflowId?: string } | undefined {
+  if (!output) return undefined
+  const ids: { childWorkflowSnapshotId?: string; childWorkflowId?: string } = {}
+  if (typeof output.childWorkflowSnapshotId === 'string') {
+    ids.childWorkflowSnapshotId = output.childWorkflowSnapshotId
+  }
+  if (typeof output.childWorkflowId === 'string') {
+    ids.childWorkflowId = output.childWorkflowId
+  }
+  return ids.childWorkflowSnapshotId || ids.childWorkflowId ? ids : undefined
+}
+
+/** Enriches a span with provider timing, cost, tokens, and model from block output. */
+function enrichWithProviderMetadata(span: TraceSpan, log: ValidBlockLog): void {
+  const output = log.output
+  if (!output) return
+
+  if (output.providerTiming) {
+    const pt = output.providerTiming
+    const timing: ProviderTiming = {
+      duration: pt.duration,
+      startTime: pt.startTime,
+      endTime: pt.endTime,
+      segments: pt.timeSegments ?? [],
+    }
+    span.providerTiming = timing
+  }
+
+  if (output.cost) {
+    const { input, output: out, total, toolCost } = output.cost
+    span.cost = {
+      input,
+      output: out,
+      total,
+      ...(typeof toolCost === 'number' && toolCost > 0 ? { toolCost } : {}),
+    }
+  }
+
+  if (output.tokens) {
+    const t = output.tokens
+    const input =
+      typeof t.input === 'number' ? t.input : typeof t.prompt === 'number' ? t.prompt : undefined
+    const outputTokens =
+      typeof t.output === 'number'
+        ? t.output
+        : typeof t.completion === 'number'
+          ? t.completion
+          : undefined
+    const totalExplicit = typeof t.total === 'number' ? t.total : undefined
+    const total =
+      totalExplicit ??
+      (input !== undefined || outputTokens !== undefined
+        ? (input ?? 0) + (outputTokens ?? 0)
+        : undefined)
+    span.tokens = {
+      ...(input !== undefined && { input }),
+      ...(outputTokens !== undefined && { output: outputTokens }),
+      ...(total !== undefined && { total }),
+    }
+  }
+
+  if (typeof output.model === 'string') {
+    span.model = output.model
+  }
+}
+
+/**
+ * Builds child spans from provider `timeSegments`, matching tool segments to
+ * their corresponding tool call I/O by name in sequential order.
+ */
+function buildChildrenFromTimeSegments(
+  span: TraceSpan,
+  log: ValidBlockLog,
+  segments: ProviderTimingSegment[]
+): TraceSpan[] {
+  const toolCallsByName = groupToolCallsByName(resolveToolCallsList(log.output))
+  const toolCallIndices = new Map<string, number>()
+
+  return segments.map((segment, index) => {
+    const segmentStartTime = new Date(segment.startTime).toISOString()
+    let segmentEndTime = new Date(segment.endTime).toISOString()
+    let segmentDuration = segment.duration
+
+    // The final model segment sometimes closes before the block ends (e.g. when
+    // post-processing runs after the stream). Extend it to the block endTime so
+    // the Gantt bar fills to the edge rather than leaving a trailing gap.
+    if (index === segments.length - 1 && segment.type === 'model' && log.endedAt) {
+      const blockEndMs = new Date(log.endedAt).getTime()
+      const segmentEndMs = new Date(segment.endTime).getTime()
+      if (blockEndMs > segmentEndMs) {
+        segmentEndTime = log.endedAt
+        segmentDuration = blockEndMs - new Date(segment.startTime).getTime()
+      }
+    }
+
+    if (segment.type === 'tool') {
+      const normalizedName = stripCustomToolPrefix(segment.name ?? '')
+      const callsForName = toolCallsByName.get(normalizedName) ?? []
+      const currentIndex = toolCallIndices.get(normalizedName) ?? 0
+      const match = callsForName[currentIndex]
+      toolCallIndices.set(normalizedName, currentIndex + 1)
+
+      const toolChild: TraceSpan = {
+        id: `${span.id}-segment-${index}`,
+        name: normalizedName,
+        type: 'tool',
+        duration: segment.duration,
+        startTime: segmentStartTime,
+        endTime: segmentEndTime,
+        status: match?.error || segment.errorMessage ? 'error' : 'success',
+        input: match?.arguments ?? match?.input,
+        output: match?.error
+          ? { error: match.error, ...(match.result ?? match.output ?? {}) }
+          : (match?.result ?? match?.output),
+      }
+      if (segment.toolCallId) toolChild.toolCallId = segment.toolCallId
+      if (segment.errorType) toolChild.errorType = segment.errorType
+      if (segment.errorMessage) toolChild.errorMessage = segment.errorMessage
+      return toolChild
+    }
+
+    const modelChild: TraceSpan = {
+      id: `${span.id}-segment-${index}`,
+      name: segment.name ?? 'Model',
+      type: 'model',
+      duration: segmentDuration,
+      startTime: segmentStartTime,
+      endTime: segmentEndTime,
+      status: segment.errorMessage ? 'error' : 'success',
+    }
+
+    if (segment.assistantContent) {
+      modelChild.output = { content: segment.assistantContent }
+    }
+    if (segment.thinkingContent) {
+      modelChild.thinking = segment.thinkingContent
+    }
+    if (segment.toolCalls && segment.toolCalls.length > 0) {
+      modelChild.modelToolCalls = segment.toolCalls
+    }
+    if (segment.finishReason) {
+      modelChild.finishReason = segment.finishReason
+    }
+    if (segment.tokens) {
+      modelChild.tokens = segment.tokens
+    }
+    if (segment.cost) {
+      modelChild.cost = segment.cost
+    }
+    if (typeof segment.ttft === 'number' && segment.ttft >= 0) {
+      modelChild.ttft = segment.ttft
+    }
+    if (span.model) {
+      modelChild.model = span.model
+    }
+    if (segment.provider) {
+      modelChild.provider = segment.provider
+    }
+    if (segment.errorType) {
+      modelChild.errorType = segment.errorType
+    }
+    if (segment.errorMessage) {
+      modelChild.errorMessage = segment.errorMessage
+    }
+
+    return modelChild
+  })
+}
+
+/**
+ * Builds tool-call child spans when the provider did not emit `timeSegments`.
+ * Each tool call becomes a full TraceSpan of `type: 'tool'`.
+ */
+function buildChildrenFromToolCalls(span: TraceSpan, log: ValidBlockLog): TraceSpan[] {
+  const toolCalls = resolveToolCallsList(log.output)
+  if (toolCalls.length === 0) return []
+
+  return toolCalls.map((tc, index) => {
+    const startTime = tc.startTime ?? log.startedAt
+    const endTime = tc.endTime ?? log.endedAt
+    return {
+      id: `${span.id}-tool-${index}`,
+      name: stripCustomToolPrefix(tc.name ?? 'unnamed-tool'),
+      type: 'tool',
+      duration: tc.duration ?? 0,
+      startTime,
+      endTime,
+      status: tc.error ? 'error' : 'success',
+      input: tc.arguments ?? tc.input,
+      output: tc.error
+        ? { error: tc.error, ...(tc.result ?? tc.output ?? {}) }
+        : (tc.result ?? tc.output),
+    }
+  })
+}
+
+/** Groups tool calls by their stripped name for sequential matching against segments. */
+function groupToolCallsByName(toolCalls: BlockToolCall[]): Map<string, BlockToolCall[]> {
+  const byName = new Map<string, BlockToolCall[]>()
+  for (const tc of toolCalls) {
+    const name = stripCustomToolPrefix(tc.name ?? '')
+    const list = byName.get(name)
+    if (list) list.push(tc)
+    else byName.set(name, [tc])
+  }
+  return byName
+}
+
+/**
+ * Resolves the tool calls list from block output. Providers write a normalized
+ * `{list, count}` container; a legacy streaming path embeds calls under
+ * `executionData.output.toolCalls`. The `Array.isArray` branches guard against
+ * persisted logs from before the container shape was normalized, where
+ * `toolCalls` was stored as a plain array — still observed in older DB rows.
+ */
+function resolveToolCallsList(output: NormalizedBlockOutput | undefined): BlockToolCall[] {
+  if (!output) return []
+
+  const direct = output.toolCalls
+  if (direct) {
+    if (Array.isArray(direct)) return direct
+    if (direct.list) return direct.list
+    logger.warn('Unexpected toolCalls shape on block output — no list extracted', {
+      shape: typeof direct,
+    })
+    return []
+  }
+
+  const legacy = (output.executionData as { output?: { toolCalls?: unknown } } | undefined)?.output
+    ?.toolCalls
+  if (!legacy) return []
+  if (Array.isArray(legacy)) return legacy as BlockToolCall[]
+  if (typeof legacy === 'object' && legacy !== null && 'list' in legacy) {
+    return ((legacy as { list?: BlockToolCall[] }).list ?? []) as BlockToolCall[]
+  }
+  logger.warn('Unexpected legacy executionData.output.toolCalls shape — no list extracted', {
+    shape: typeof legacy,
+  })
+  return []
+}
+
+/** Extracts and flattens child workflow trace spans into the parent span's children. */
+function attachChildWorkflowSpans(span: TraceSpan, log: ValidBlockLog): void {
+  const childTraceSpans = log.childTraceSpans ?? log.output?.childTraceSpans
+  if (!childTraceSpans?.length) return
+
+  span.children = flattenWorkflowChildren(childTraceSpans)
+  span.output = stripChildTraceSpansFromOutput(span.output)
+}
+
+/** True when a span is a synthetic workflow wrapper (no blockId). */
+function isSyntheticWorkflowWrapper(span: TraceSpan): boolean {
+  return span.type === 'workflow' && !span.blockId
+}
+
+/** Reads nested `childTraceSpans` off a span's output, or `[]` if absent. */
+function extractOutputChildren(output: TraceSpan['output']): TraceSpan[] {
+  const nested = (output as { childTraceSpans?: TraceSpan[] } | undefined)?.childTraceSpans
+  return Array.isArray(nested) ? nested : []
+}
+
+/** Returns a copy of `output` with `childTraceSpans` removed, or undefined unchanged. */
+function stripChildTraceSpansFromOutput(
+  output: TraceSpan['output']
+): TraceSpan['output'] | undefined {
+  if (!output || !('childTraceSpans' in output)) return output
+  const { childTraceSpans: _, ...rest } = output as Record<string, unknown>
+  return rest
+}
+
+/** Recursively flattens synthetic workflow wrappers, preserving real block spans. */
+function flattenWorkflowChildren(spans: TraceSpan[]): TraceSpan[] {
+  const flattened: TraceSpan[] = []
+
+  for (const span of spans) {
+    if (isSyntheticWorkflowWrapper(span)) {
+      if (span.children?.length) {
+        flattened.push(...flattenWorkflowChildren(span.children))
+      }
+      continue
+    }
+
+    const directChildren = span.children ?? []
+    const outputChildren = extractOutputChildren(span.output)
+    const allChildren = [...directChildren, ...outputChildren]
+
+    const nextSpan: TraceSpan = { ...span }
+    if (allChildren.length > 0) {
+      nextSpan.children = flattenWorkflowChildren(allChildren)
+    }
+    if (outputChildren.length > 0) {
+      nextSpan.output = stripChildTraceSpansFromOutput(nextSpan.output)
+    }
+
+    flattened.push(nextSpan)
+  }
+
+  return flattened
+}

apps/sim/lib/logs/execution/trace-spans/trace-spans.ts

@@ -0,0 +1,143 @@
+import { groupIterationBlocks } from '@/lib/logs/execution/trace-spans/iteration-grouping'
+import { createSpanFromLog } from '@/lib/logs/execution/trace-spans/span-factory'
+import type { TraceSpan } from '@/lib/logs/types'
+import type { BlockLog, ExecutionResult } from '@/executor/types'
+
+/**
+ * Keys that should be recursively filtered from output display.
+ * These are internal fields used for execution tracking that shouldn't be shown to users.
+ */
+const HIDDEN_OUTPUT_KEYS = new Set(['childTraceSpans'])
+const SUCCESSFUL_CHILD_ERROR_BOUNDARY_BLOCK_TYPES = new Set(['mothership'])
+
+function setFilteredValue(output: Record<string, unknown>, key: string, value: unknown): void {
+  Object.defineProperty(output, key, {
+    value,
+    enumerable: true,
+    configurable: true,
+    writable: true,
+  })
+}
+
+/**
+ * Recursively filters hidden keys from nested objects for cleaner display.
+ * Used by both executor (for log output) and UI (for display).
+ */
+export function filterHiddenOutputKeys(value: unknown): unknown {
+  if (value === null || value === undefined) {
+    return value
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => filterHiddenOutputKeys(item))
+  }
+
+  if (typeof value === 'object') {
+    const filtered: Record<string, unknown> = {}
+    for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
+      if (HIDDEN_OUTPUT_KEYS.has(key)) {
+        continue
+      }
+      setFilteredValue(filtered, key, filterHiddenOutputKeys(val))
+    }
+    return filtered
+  }
+
+  return value
+}
+
+/**
+ * Builds a hierarchical trace span tree from execution logs.
+ *
+ * Pipeline:
+ *   1. Each BlockLog becomes a TraceSpan via `createSpanFromLog`.
+ *   2. Spans are sorted by start time to form a flat list of root spans.
+ *   3. Loop/parallel iterations are grouped into container spans via `groupIterationBlocks`.
+ *   4. A synthetic "Workflow Execution" root wraps the grouped spans and provides
+ *      relative timestamps + total duration derived from the earliest start / latest end.
+ */
+export function buildTraceSpans(result: ExecutionResult): {
+  traceSpans: TraceSpan[]
+  totalDuration: number
+} {
+  if (!result.logs?.length) {
+    return { traceSpans: [], totalDuration: 0 }
+  }
+
+  const spans = buildRootSpansFromLogs(result.logs)
+  const grouped = groupIterationBlocks(spans)
+
+  if (grouped.length === 0 || !result.metadata) {
+    const totalDuration = grouped.reduce((sum, span) => sum + span.duration, 0)
+    return { traceSpans: grouped, totalDuration }
+  }
+
+  return wrapInWorkflowRoot(grouped, spans)
+}
+
+/** Converts each BlockLog into a TraceSpan, sorted chronologically by start time. */
+function buildRootSpansFromLogs(logs: BlockLog[]): TraceSpan[] {
+  const spans: TraceSpan[] = []
+  for (const log of logs) {
+    const span = createSpanFromLog(log)
+    if (span) spans.push(span)
+  }
+  spans.sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime())
+  return spans
+}
+
+/**
+ * Wraps grouped spans in a synthetic workflow-execution root span using the
+ * true workflow bounds (earliest start / latest end across all leaf spans).
+ */
+function wrapInWorkflowRoot(
+  grouped: TraceSpan[],
+  leafSpans: TraceSpan[]
+): { traceSpans: TraceSpan[]; totalDuration: number } {
+  let earliestStart = Number.POSITIVE_INFINITY
+  let latestEnd = 0
+  for (const span of leafSpans) {
+    const startTime = new Date(span.startTime).getTime()
+    const endTime = new Date(span.endTime).getTime()
+    if (startTime < earliestStart) earliestStart = startTime
+    if (endTime > latestEnd) latestEnd = endTime
+  }
+
+  const actualWorkflowDuration = latestEnd - earliestStart
+  addRelativeTimestamps(grouped, earliestStart)
+
+  const totalCost = leafSpans.reduce((sum, s) => sum + (s.cost?.total ?? 0), 0)
+
+  const workflowSpan: TraceSpan = {
+    id: 'workflow-execution',
+    name: 'Workflow Execution',
+    type: 'workflow',
+    duration: actualWorkflowDuration,
+    startTime: new Date(earliestStart).toISOString(),
+    endTime: new Date(latestEnd).toISOString(),
+    status: grouped.some(hasUnhandledError) ? 'error' : 'success',
+    children: grouped,
+    ...(totalCost > 0 && { cost: { total: totalCost } }),
+  }
+
+  return { traceSpans: [workflowSpan], totalDuration: actualWorkflowDuration }
+}
+
+/** Recursively annotates spans with `relativeStartMs` (ms since workflow start). */
+function addRelativeTimestamps(spans: TraceSpan[], workflowStartMs: number): void {
+  for (const span of spans) {
+    span.relativeStartMs = new Date(span.startTime).getTime() - workflowStartMs
+    if (span.children?.length) {
+      addRelativeTimestamps(span.children, workflowStartMs)
+    }
+  }
+}
+
+/** True if this span (or any descendant) has an unhandled error. */
+function hasUnhandledError(span: TraceSpan): boolean {
+  if (span.status === 'error' && !span.errorHandled) return true
+  if (span.status === 'success' && SUCCESSFUL_CHILD_ERROR_BOUNDARY_BLOCK_TYPES.has(span.type)) {
+    return false
+  }
+  return span.children?.some(hasUnhandledError) ?? false
+}

apps/sim/lib/logs/execution/trace-store.ts

@@ -0,0 +1,179 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
+import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store'
+
+const logger = createLogger('TraceStore')
+
+/**
+ * Key under which the externalized-execution-data pointer (a `__simLargeValueRef`)
+ * is stored on the slim `execution_data` row.
+ */
+export const TRACE_STORE_REF_KEY = 'traceStoreRef'
+
+/**
+ * The only metadata kept inline on the slim row (everything else lives in the
+ * externalized object). These two describe trace presence/count and uniquely
+ * survive object expiry — so a reader can still report "trace data expired (N
+ * spans)" after retention without an object fetch. All other fields
+ * (environment, trigger, tokens, models, truncation flags, and of course the
+ * heavy payloads) are in the stored object and recovered on materialize, so
+ * keeping them inline too would just be duplication.
+ */
+const INLINE_MARKER_KEYS = ['hasTraceSpans', 'traceSpanCount'] as const
+
+/**
+ * Read-path context. Resolves an externalized payload by storage key, authorized
+ * via the (already-authorized) workspace — no owner needed.
+ */
+interface TraceStoreReadContext {
+  workspaceId: string | null
+  workflowId: string | null
+  executionId: string
+}
+
+/**
+ * Write-path context. Requires the execution owner's `userId`: the externalized
+ * object is tracked in `workspace_files`, whose `user_id` column is NOT NULL
+ * (FK -> user.id). Requiring it here makes "a write needs an owner" a
+ * compile-time invariant, so callers must resolve the owner before persisting.
+ */
+interface TraceStoreWriteContext extends TraceStoreReadContext {
+  userId: string
+}
+
+/**
+ * Recovers the workflowId embedded in a large-value storage key
+ * (`execution/{workspaceId}/{workflowId}/{executionId}/<file>`). Used when the
+ * log row's workflowId has been nulled by workflow deletion.
+ */
+function workflowIdFromStorageKey(key: string | undefined): string | undefined {
+  if (!key) return undefined
+  const parts = key.split('/')
+  return parts.length >= 5 && parts[0] === 'execution' ? parts[2] : undefined
+}
+
+/**
+ * Recursively removes `cost` from trace spans before persistence. Cost lives in
+ * exactly one place — the usage_log ledger — so persisted spans carry only
+ * structure, timing, and tokens (KTD7). Must run AFTER `calculateCostSummary`
+ * has consumed span costs in memory.
+ */
+export function stripSpanCosts(spans: unknown): void {
+  if (!Array.isArray(spans)) return
+  for (const span of spans) {
+    if (!span || typeof span !== 'object') continue
+    const record = span as { cost?: unknown; children?: unknown }
+    if ('cost' in record) record.cost = undefined
+    if (Array.isArray(record.children)) stripSpanCosts(record.children)
+  }
+}
+
+/**
+ * Externalizes heavy `execution_data` to object storage as a single large value
+ * (reusing the execution-context large-value store + its reference/dependency/GC
+ * machinery — KTD4/KTD8), returning a slim row payload that keeps inline markers
+ * plus the `__simLargeValueRef` pointer.
+ *
+ * On any failure (no scope, oversized, storage error) the original (already
+ * cost-stripped) execution data is returned unchanged so the log is never lost.
+ */
+export async function externalizeExecutionData(
+  executionData: Record<string, unknown>,
+  context: TraceStoreWriteContext
+): Promise<Record<string, unknown>> {
+  const { workspaceId, workflowId, executionId, userId } = context
+  // workspaceId/workflowId build the storage key and can be null for
+  // deleted-workflow rows. userId is type-guaranteed by TraceStoreWriteContext;
+  // the falsy check is a defensive guard against an empty string. If any are
+  // missing the durable write can't succeed, so keep the data inline.
+  if (!workspaceId || !workflowId || !userId) return executionData
+
+  try {
+    const json = JSON.stringify(executionData)
+    const size = Buffer.byteLength(json, 'utf8')
+
+    // storeLargeValue persists to the execution bucket with a conforming key and
+    // registers owner + dependency closure (trace -> nested span large values),
+    // so GC keeps nested children alive while this run's log row exists.
+    const ref = await storeLargeValue(executionData, json, size, {
+      workspaceId,
+      workflowId,
+      executionId,
+      userId,
+      requireDurable: true,
+    })
+
+    const { preview: _preview, ...slimRef } = ref
+
+    const slim: Record<string, unknown> = { [TRACE_STORE_REF_KEY]: slimRef }
+    for (const key of INLINE_MARKER_KEYS) {
+      if (key in executionData) slim[key] = executionData[key]
+    }
+    return slim
+  } catch (error) {
+    logger.warn('Failed to externalize execution data; keeping inline', {
+      executionId,
+      error: toError(error).message,
+    })
+    return executionData
+  }
+}
+
+/**
+ * Resolves an `execution_data` row into its full form for reads. When the row
+ * carries a trace-store pointer, the payload is materialized from storage and
+ * merged with the inline markers; otherwise the row is returned unchanged
+ * (inline / pre-externalization runs). One level only — nested span
+ * `__simLargeValueRef` stubs remain as previews, matching prior behavior.
+ *
+ * Returns metadata-only (the slim row minus the pointer) if the object is
+ * missing/unreadable (e.g. post-retention) so reads degrade rather than crash.
+ */
+export async function materializeExecutionData(
+  executionData: Record<string, unknown> | null | undefined,
+  context: TraceStoreReadContext
+): Promise<Record<string, unknown>> {
+  if (!executionData) return {}
+
+  const ref = executionData[TRACE_STORE_REF_KEY]
+  if (!isLargeValueRef(ref)) return executionData
+
+  const { [TRACE_STORE_REF_KEY]: _pointer, ...markers } = executionData
+
+  if (!context.workspaceId) return markers
+
+  // workflowId is `set null` on workflow delete, but the ref key embeds the
+  // original workflowId — recover it so deleted-workflow logs stay readable.
+  // Workspace authorization still comes from the (authorized) caller context.
+  const workflowId = context.workflowId ?? workflowIdFromStorageKey(ref.key)
+  if (!workflowId) return markers
+
+  try {
+    const materialized = await materializeLargeValueRef(ref, {
+      workspaceId: context.workspaceId,
+      workflowId,
+      executionId: context.executionId,
+      maxBytes: ref.size,
+      // Read-only: the value is already referenced by its own execution; don't
+      // re-register (or fail) on every view/export.
+      trackReference: false,
+    })
+
+    if (!materialized || typeof materialized !== 'object') {
+      logger.warn('Trace store object unavailable; returning metadata only', {
+        executionId: context.executionId,
+        key: ref.key,
+      })
+      return markers
+    }
+
+    return { ...(materialized as Record<string, unknown>), ...markers }
+  } catch (error) {
+    logger.warn('Failed to materialize execution data; returning metadata only', {
+      executionId: context.executionId,
+      error: toError(error).message,
+    })
+    return markers
+  }
+}