chore: import upstream snapshot with attribution

apps/sim/lib/table/__tests__/column-keys.test.ts

@@ -0,0 +1,162 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+
+const { mockGenerateId } = vi.hoisted(() => ({
+  mockGenerateId: vi.fn(),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: mockGenerateId,
+}))
+
+import {
+  buildIdByName,
+  buildNameById,
+  filterNamesToIds,
+  generateColumnId,
+  getColumnId,
+  remapGroupColumnRefs,
+  rowDataIdToName,
+  rowDataNameToId,
+  sortNamesToIds,
+  withGeneratedColumnIds,
+} from '@/lib/table/column-keys'
+import type { TableSchema, WorkflowGroup } from '@/lib/table/types'
+
+describe('getColumnId', () => {
+  it('returns the explicit id when present', () => {
+    expect(getColumnId({ id: 'col_abc', name: 'email' })).toBe('col_abc')
+  })
+  it('falls back to name for legacy id-less columns', () => {
+    expect(getColumnId({ name: 'email' })).toBe('email')
+  })
+})
+
+describe('generateColumnId', () => {
+  it('mints a col_-prefixed id with the uuid dashes stripped', () => {
+    mockGenerateId.mockReturnValue('11111111-2222-4333-8444-555566667777')
+    expect(generateColumnId()).toBe('col_11111111222243338444555566667777')
+  })
+
+  it('produces an id that satisfies NAME_PATTERN (valid JSONB key / filter field)', () => {
+    mockGenerateId.mockReturnValue('0a1b2c3d-4e5f-4607-8809-0a1b2c3d4e5f')
+    // Must start with a letter/underscore and contain only [a-z0-9_].
+    expect(generateColumnId()).toMatch(/^[a-z_][a-z0-9_]*$/i)
+  })
+})
+
+describe('name ↔ id maps', () => {
+  const schema: TableSchema = {
+    columns: [
+      { id: 'col_1', name: 'email', type: 'string' },
+      { name: 'age', type: 'number' }, // legacy: id == name
+    ],
+  }
+
+  it('buildIdByName maps display name → storage id', () => {
+    expect(Object.fromEntries(buildIdByName(schema))).toEqual({ email: 'col_1', age: 'age' })
+  })
+  it('buildNameById maps storage id → display name', () => {
+    expect(Object.fromEntries(buildNameById(schema))).toEqual({ col_1: 'email', age: 'age' })
+  })
+})
+
+describe('row data translation', () => {
+  const schema: TableSchema = {
+    columns: [
+      { id: 'col_1', name: 'email', type: 'string' },
+      { name: 'age', type: 'number' },
+    ],
+  }
+  const idByName = buildIdByName(schema)
+  const nameById = buildNameById(schema)
+
+  it('round-trips name → id → name', () => {
+    const wire = { email: 'a@b.c', age: 30 }
+    const stored = rowDataNameToId(wire, idByName)
+    expect(stored).toEqual({ col_1: 'a@b.c', age: 30 })
+    expect(rowDataIdToName(stored, nameById)).toEqual(wire)
+  })
+
+  it('drops keys with no matching column (orphans / unknowns)', () => {
+    expect(rowDataNameToId({ email: 'x', ghost: 1 }, idByName)).toEqual({ col_1: 'x' })
+    expect(rowDataIdToName({ col_1: 'x', col_gone: 9 }, nameById)).toEqual({ email: 'x' })
+  })
+})
+
+describe('filter / sort translation', () => {
+  const idByName = new Map([
+    ['email', 'col_1'],
+    ['age', 'col_2'],
+  ])
+
+  it('translates field names, recurses $or/$and, passes through unknown fields', () => {
+    const filter = {
+      email: 'a@b.c',
+      $or: [{ age: { $gt: 18 } }, { createdAt: { $gt: '2024' } }],
+    }
+    expect(filterNamesToIds(filter, idByName)).toEqual({
+      col_1: 'a@b.c',
+      $or: [{ col_2: { $gt: 18 } }, { createdAt: { $gt: '2024' } }],
+    })
+  })
+
+  it('translates sort field names, passes through unknown', () => {
+    expect(sortNamesToIds({ email: 'asc', createdAt: 'desc' }, idByName)).toEqual({
+      col_1: 'asc',
+      createdAt: 'desc',
+    })
+  })
+})
+
+describe('withGeneratedColumnIds', () => {
+  it('stamps ids on id-less columns and remaps group refs name → id', () => {
+    mockGenerateId.mockReturnValueOnce('a').mockReturnValueOnce('b')
+    const schema: TableSchema = {
+      columns: [
+        { name: 'email', type: 'string', workflowGroupId: 'g1' },
+        { name: 'score', type: 'number', workflowGroupId: 'g1' },
+      ],
+      workflowGroups: [
+        {
+          id: 'g1',
+          workflowId: 'wf',
+          outputs: [{ blockId: 'b', path: 'p', columnName: 'score' }],
+          dependencies: { columns: ['email'] },
+          inputMappings: [{ inputName: 'in', columnName: 'email' }],
+        },
+      ],
+    }
+    const out = withGeneratedColumnIds(schema)
+    expect(out.columns[0].id).toBe('col_a')
+    expect(out.columns[1].id).toBe('col_b')
+    const g = out.workflowGroups![0]
+    expect(g.outputs[0].columnName).toBe('col_b') // score
+    expect(g.dependencies!.columns).toEqual(['col_a']) // email
+    expect(g.inputMappings![0].columnName).toBe('col_a')
+  })
+
+  it('is idempotent for columns that already have an id', () => {
+    const schema: TableSchema = {
+      columns: [{ id: 'col_keep', name: 'email', type: 'string' }],
+    }
+    expect(withGeneratedColumnIds(schema).columns[0].id).toBe('col_keep')
+  })
+})
+
+describe('remapGroupColumnRefs', () => {
+  it('rewrites refs that are names, leaves refs that are already ids', () => {
+    const idByName = new Map([['email', 'col_1']])
+    const group: WorkflowGroup = {
+      id: 'g',
+      workflowId: 'wf',
+      outputs: [{ blockId: 'b', path: 'p', columnName: 'email' }],
+      dependencies: { columns: ['col_existing'] },
+    }
+    const out = remapGroupColumnRefs(group, idByName)
+    expect(out.outputs[0].columnName).toBe('col_1')
+    expect(out.dependencies!.columns).toEqual(['col_existing'])
+  })
+})

apps/sim/lib/table/__tests__/find-row-matches.test.ts

@@ -0,0 +1,115 @@
+/**
+ * @vitest-environment node
+ *
+ * Unit-tests the result mapping and truncation logic of `findRowMatches`. The
+ * SQL itself runs against a mocked `db.execute`, so these assertions cover the
+ * JS-side shaping (ordinal coercion, column rename, LIMIT+1 truncation), not
+ * the query semantics — those need a real Postgres.
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { sql } from 'drizzle-orm'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { ColumnDefinition, TableDefinition } from '@/lib/table/types'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+vi.mock('@/lib/table/sql', () => ({
+  buildFilterClause: vi.fn(() => sql`true`),
+  buildSortClause: vi.fn(() => sql`true`),
+  escapeLikePattern: vi.fn((s: string) => s),
+}))
+
+vi.mock('@/lib/table/trigger', () => ({ fireTableTrigger: vi.fn() }))
+vi.mock('@/lib/table/workflow-columns', () => ({
+  assertValidSchema: vi.fn(),
+  scheduleRunsForRows: vi.fn(),
+  scheduleRunsForTable: vi.fn(),
+  stripGroupDeps: vi.fn(),
+}))
+vi.mock('@/lib/table/validation', () => ({
+  validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
+  validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn(),
+  validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
+  validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  getUniqueColumns: vi.fn(() => []),
+  checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+  checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+}))
+
+import { findRowMatches } from '@/lib/table/rows/service'
+import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
+
+const COLUMNS: ColumnDefinition[] = [
+  { name: 'name', type: 'string' },
+  { name: 'email', type: 'string' },
+]
+
+const TABLE: TableDefinition = {
+  id: 'tbl-1',
+  name: 'People',
+  description: null,
+  schema: { columns: COLUMNS },
+  metadata: null,
+  rowCount: 0,
+  maxRows: 1000,
+  workspaceId: 'ws-1',
+  createdBy: 'user-1',
+  archivedAt: null,
+  createdAt: new Date('2024-01-01'),
+  updatedAt: new Date('2024-01-01'),
+}
+
+describe('findRowMatches', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('returns empty without querying when the table has no columns', async () => {
+    const result = await findRowMatches({ ...TABLE, schema: { columns: [] } }, { q: 'x' }, 'req')
+    expect(result).toEqual({ matches: [], truncated: false })
+    expect(dbChainMockFns.execute).not.toHaveBeenCalled()
+  })
+
+  it('maps rows to matches, coercing the bigint ordinal and renaming the column', async () => {
+    dbChainMockFns.execute.mockResolvedValue([
+      { ordinal: '2', id: 'r2', column_name: 'name' },
+      { ordinal: 5, id: 'r5', column_name: 'email' },
+    ])
+    const result = await findRowMatches(TABLE, { q: 'a' }, 'req')
+    expect(result.truncated).toBe(false)
+    expect(result.matches).toEqual([
+      { ordinal: 2, rowId: 'r2', column: 'name' },
+      { ordinal: 5, rowId: 'r5', column: 'email' },
+    ])
+  })
+
+  it('flags truncation and caps the result when the DB returns LIMIT+1 rows', async () => {
+    const over = Array.from({ length: 1001 }, (_, i) => ({
+      ordinal: i,
+      id: `r${i}`,
+      column_name: 'name',
+    }))
+    dbChainMockFns.execute.mockResolvedValue(over)
+    const result = await findRowMatches(TABLE, { q: 'a' }, 'req')
+    expect(result.truncated).toBe(true)
+    expect(result.matches).toHaveLength(1000)
+  })
+
+  it('threads filter and sort through the SQL builders', async () => {
+    dbChainMockFns.execute.mockResolvedValue([])
+    await findRowMatches(
+      TABLE,
+      { q: 'a', filter: { name: { $contains: 'a' } }, sort: { name: 'asc' } },
+      'req'
+    )
+    expect(buildFilterClause).toHaveBeenCalledWith(
+      { name: { $contains: 'a' } },
+      expect.any(String),
+      COLUMNS
+    )
+    expect(buildSortClause).toHaveBeenCalledWith({ name: 'asc' }, expect.any(String), COLUMNS)
+  })
+})

apps/sim/lib/table/__tests__/lock-order.test.ts

@@ -0,0 +1,107 @@
+/**
+ * @vitest-environment node
+ *
+ * Lock-order regression guard: a column-creating CSV import must acquire the
+ * per-table row-order advisory lock (`user_table_rows_pos`) BEFORE writing
+ * `user_table_definitions`, matching the rows_pos → definitions order that plain
+ * inserts take via the row-count trigger. The opposite order deadlocks
+ * concurrent inserts on the same table.
+ */
+import { userTableDefinitions } from '@sim/db/schema'
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { importAppendRows } from '@/lib/table/import-data'
+import type { TableDefinition } from '@/lib/table/types'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+vi.mock('@/lib/core/config/feature-flags', () => ({
+  isFeatureEnabled: vi.fn().mockResolvedValue(false),
+}))
+
+vi.mock('@/lib/table/validation', () => ({
+  validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
+  validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn((row) => row),
+  validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
+  validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  getUniqueColumns: vi.fn(() => []),
+  checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+  checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+}))
+
+const TABLE: TableDefinition = {
+  id: 'tbl-1',
+  name: 'People',
+  description: null,
+  schema: { columns: [{ name: 'name', type: 'string' }] },
+  metadata: null,
+  rowCount: 0,
+  maxRows: 1000,
+  workspaceId: 'ws-1',
+  createdBy: 'user-1',
+  archivedAt: null,
+  createdAt: new Date('2024-01-01'),
+  updatedAt: new Date('2024-01-01'),
+}
+
+/**
+ * invocationCallOrder of the first `tx.execute(...)` whose SQL contains
+ * `substring` — checks the template strings, the interpolated values (the
+ * advisory key is passed as a value), and `sql.raw` output.
+ */
+function executeOrderContaining(substring: string): number {
+  const { calls, invocationCallOrder } = dbChainMockFns.execute.mock
+  for (let i = 0; i < calls.length; i++) {
+    const arg = calls[i][0] as { strings?: unknown; values?: unknown; rawSql?: unknown } | undefined
+    const haystacks: string[] = []
+    if (Array.isArray(arg?.strings)) {
+      haystacks.push(
+        ...(arg.strings as unknown[]).filter((s): s is string => typeof s === 'string')
+      )
+    }
+    if (Array.isArray(arg?.values)) {
+      haystacks.push(...(arg.values as unknown[]).filter((v): v is string => typeof v === 'string'))
+    }
+    if (typeof arg?.rawSql === 'string') haystacks.push(arg.rawSql)
+    if (haystacks.some((s) => s.includes(substring))) return invocationCallOrder[i]
+  }
+  return -1
+}
+
+/** invocationCallOrder of the first `tx.update(table)` call. */
+function updateOrderForTable(table: unknown): number {
+  const { calls, invocationCallOrder } = dbChainMockFns.update.mock
+  for (let i = 0; i < calls.length; i++) {
+    if (calls[i][0] === table) return invocationCallOrder[i]
+  }
+  return -1
+}
+
+describe('table import lock ordering', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('acquires the rows_pos advisory before the user_table_definitions write when adding columns', async () => {
+    // The lock pre-acquire and the column-creating write both run at the top of
+    // the import, before the batch-insert loop. The loop's order-key aggregates
+    // aren't fully wired in this unit mock, so tolerate a downstream error after
+    // the locks under test have been recorded.
+    await importAppendRows(
+      TABLE,
+      [{ name: 'new_col', type: 'string' }],
+      [{ name: 'Alice', new_col: 'x' }],
+      { workspaceId: 'ws-1', userId: 'user-1', requestId: 'req-1' }
+    ).catch(() => {})
+
+    const rowsPosLockOrder = executeOrderContaining('user_table_rows_pos')
+    const definitionsWriteOrder = updateOrderForTable(userTableDefinitions)
+
+    expect(rowsPosLockOrder).toBeGreaterThan(0)
+    expect(definitionsWriteOrder).toBeGreaterThan(0)
+    expect(rowsPosLockOrder).toBeLessThan(definitionsWriteOrder)
+  })
+})

apps/sim/lib/table/__tests__/paging.test.ts

@@ -0,0 +1,33 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { trimRowsToByteBudget } from '@/lib/table/rows/paging'
+
+function row(id: string, bytes: number) {
+  // {"b":"aaa…"} serializes to bytes + 8 chars of envelope.
+  return { id, data: { b: 'a'.repeat(Math.max(0, bytes - 8)) } }
+}
+
+describe('trimRowsToByteBudget', () => {
+  it('returns all rows when they fit the budget', () => {
+    const rows = [row('r1', 100), row('r2', 100)]
+    expect(trimRowsToByteBudget(rows, 1000)).toBe(rows)
+  })
+
+  it('keeps the longest prefix within the budget', () => {
+    const rows = [row('r1', 400), row('r2', 400), row('r3', 400)]
+    const kept = trimRowsToByteBudget(rows, 900)
+    expect(kept.map((r) => r.id)).toEqual(['r1', 'r2'])
+  })
+
+  it('always keeps the first row even when it alone exceeds the budget', () => {
+    const rows = [row('r1', 5000), row('r2', 100)]
+    const kept = trimRowsToByteBudget(rows, 1000)
+    expect(kept.map((r) => r.id)).toEqual(['r1'])
+  })
+
+  it('returns empty input unchanged', () => {
+    expect(trimRowsToByteBudget([], 1000)).toEqual([])
+  })
+})

apps/sim/lib/table/__tests__/service-filter-threading.test.ts

@@ -0,0 +1,125 @@
+/**
+ * @vitest-environment node
+ *
+ * Integration test asserting that `table.schema.columns` is forwarded to
+ * `buildFilterClause` from each service function that filters rows. This
+ * guards the contract that type-aware JSONB casts (numeric for numbers,
+ * timestamp for dates) are always available at the SQL builder layer — the
+ * latent bug that PR #4657 was originally fixing.
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { sql } from 'drizzle-orm'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
+import type { ColumnDefinition, TableDefinition } from '@/lib/table/types'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+vi.mock('@/lib/table/sql', () => ({
+  buildFilterClause: vi.fn(() => sql`true`),
+  buildSortClause: vi.fn(() => sql`true`),
+}))
+
+vi.mock('@/lib/table/trigger', () => ({
+  fireTableTrigger: vi.fn(),
+}))
+
+vi.mock('@/lib/table/workflow-columns', () => ({
+  assertValidSchema: vi.fn(),
+  scheduleRunsForRows: vi.fn(),
+  scheduleRunsForTable: vi.fn(),
+  stripGroupDeps: vi.fn(),
+}))
+
+vi.mock('@/lib/table/validation', () => ({
+  validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
+  validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn(),
+  validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
+  validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  getUniqueColumns: vi.fn(() => []),
+  checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+  checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+}))
+
+import { deleteRowsByFilter, queryRows, updateRowsByFilter } from '@/lib/table/rows/service'
+
+const COLUMNS: ColumnDefinition[] = [
+  { name: 'name', type: 'string' },
+  { name: 'birthDate', type: 'date' },
+  { name: 'score', type: 'number' },
+]
+
+const TABLE: TableDefinition = {
+  id: 'tbl-1',
+  name: 'People',
+  description: null,
+  schema: { columns: COLUMNS },
+  metadata: null,
+  rowCount: 0,
+  maxRows: 1000,
+  workspaceId: 'ws-1',
+  createdBy: 'user-1',
+  archivedAt: null,
+  createdAt: new Date('2024-01-01'),
+  updatedAt: new Date('2024-01-01'),
+}
+
+describe('service filter threading', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('queryRows forwards table.schema.columns to buildFilterClause', async () => {
+    await queryRows(
+      TABLE,
+      { filter: { birthDate: { $gte: '2024-01-01' } }, includeTotal: false },
+      'req-1'
+    ).catch(() => {})
+
+    expect(buildFilterClause).toHaveBeenCalledTimes(1)
+    expect(buildFilterClause).toHaveBeenCalledWith(
+      { birthDate: { $gte: '2024-01-01' } },
+      expect.any(String),
+      COLUMNS
+    )
+  })
+
+  it('queryRows forwards columns to buildSortClause as well', async () => {
+    await queryRows(TABLE, { sort: { birthDate: 'asc' }, includeTotal: false }, 'req-1').catch(
+      () => {}
+    )
+
+    expect(buildSortClause).toHaveBeenCalledWith({ birthDate: 'asc' }, expect.any(String), COLUMNS)
+  })
+
+  it('updateRowsByFilter forwards table.schema.columns to buildFilterClause', async () => {
+    dbChainMockFns.where.mockResolvedValueOnce([])
+    await updateRowsByFilter(
+      TABLE,
+      { filter: { birthDate: { $lt: '2024-06-01' } }, data: { name: 'x' } },
+      'req-1'
+    )
+
+    expect(buildFilterClause).toHaveBeenCalledTimes(1)
+    expect(buildFilterClause).toHaveBeenCalledWith(
+      { birthDate: { $lt: '2024-06-01' } },
+      expect.any(String),
+      COLUMNS
+    )
+  })
+
+  it('deleteRowsByFilter forwards table.schema.columns to buildFilterClause', async () => {
+    dbChainMockFns.where.mockResolvedValueOnce([])
+    await deleteRowsByFilter(TABLE, { filter: { score: { $gt: 90 } } }, 'req-1')
+
+    expect(buildFilterClause).toHaveBeenCalledTimes(1)
+    expect(buildFilterClause).toHaveBeenCalledWith(
+      { score: { $gt: 90 } },
+      expect.any(String),
+      COLUMNS
+    )
+  })
+})

apps/sim/lib/table/__tests__/sql.test.ts

@@ -0,0 +1,462 @@
+/**
+ * @vitest-environment node
+ *
+ * SQL Builder Unit Tests
+ *
+ * Tests the table SQL query builder. Assertions inspect the generated SQL
+ * string so cast selection (numeric vs timestamptz) is verified end-to-end.
+ *
+ * Rendering: `drizzle-orm` is globally mocked in `vitest.setup.ts`. The mock
+ * represents tagged-template fragments as `{ strings, values }`, raw fragments
+ * as `{ rawSql }`, and joined fragments as `{ fragments, separator }`. The
+ * local `renderSql` helper walks that shape recursively so we can assert real
+ * substrings like `::timestamptz` against the generated SQL.
+ */
+import { describe, expect, it } from 'vitest'
+import { buildFilterClause, buildSortClause } from '@/lib/table/sql'
+import type { ColumnDefinition, Filter, Sort } from '@/lib/table/types'
+
+type SqlNode =
+  | { strings: ArrayLike<string>; values: unknown[] }
+  | { rawSql: string }
+  | { fragments: unknown[]; separator: unknown }
+  | string
+  | number
+  | boolean
+  | null
+  | undefined
+
+function isTemplateNode(n: unknown): n is { strings: ArrayLike<string>; values: unknown[] } {
+  return (
+    typeof n === 'object' &&
+    n !== null &&
+    'strings' in n &&
+    'values' in n &&
+    Array.isArray((n as { values: unknown[] }).values)
+  )
+}
+
+function isRawNode(n: unknown): n is { rawSql: string } {
+  return typeof n === 'object' && n !== null && 'rawSql' in n
+}
+
+function isJoinNode(n: unknown): n is { fragments: unknown[]; separator: unknown } {
+  return (
+    typeof n === 'object' &&
+    n !== null &&
+    'fragments' in n &&
+    Array.isArray((n as { fragments: unknown[] }).fragments)
+  )
+}
+
+/** Recursively render a mock SQL node into its generated SQL string. */
+function renderSql(node: SqlNode | unknown): string {
+  if (node == null) return String(node)
+  if (isRawNode(node)) return node.rawSql
+  if (isJoinNode(node)) {
+    const sep = isRawNode(node.separator) ? node.separator.rawSql : ', '
+    return node.fragments.map(renderSql).join(sep)
+  }
+  if (isTemplateNode(node)) {
+    const parts: string[] = []
+    for (let i = 0; i < node.strings.length; i++) {
+      parts.push(node.strings[i])
+      if (i < node.values.length) {
+        parts.push(renderSql(node.values[i]))
+      }
+    }
+    return parts.join('')
+  }
+  if (typeof node === 'string') return `'${node}'`
+  return String(node)
+}
+
+function render(node: unknown): string {
+  return renderSql(node)
+}
+
+const TABLE = 'user_table_rows'
+const NO_COLUMNS: ColumnDefinition[] = []
+
+describe('SQL Builder', () => {
+  describe('buildFilterClause', () => {
+    it('returns undefined for empty filter', () => {
+      expect(buildFilterClause({}, TABLE, NO_COLUMNS)).toBeUndefined()
+    })
+
+    it('handles simple equality via JSONB containment', () => {
+      const out = render(buildFilterClause({ name: 'John' }, TABLE, NO_COLUMNS))
+      expect(out).toContain('user_table_rows.data @>')
+      expect(out).toContain('"name":"John"')
+    })
+
+    it('emits ::numeric cast for $gt on a number column', () => {
+      const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
+      const out = render(buildFilterClause({ age: { $gt: 18 } }, TABLE, cols))
+      expect(out).toContain(`(${TABLE}.data->>'age')::numeric > `)
+      expect(out).not.toContain('::timestamp')
+    })
+
+    it('falls back to ::numeric when column type is unknown', () => {
+      const out = render(buildFilterClause({ score: { $gte: 5 } }, TABLE, NO_COLUMNS))
+      expect(out).toContain(`(${TABLE}.data->>'score')::numeric >= `)
+      expect(out).not.toContain('::timestamp')
+    })
+
+    it('handles $eq operator', () => {
+      const out = render(buildFilterClause({ status: { $eq: 'active' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('"status":"active"')
+    })
+
+    it('handles $ne operator', () => {
+      const out = render(buildFilterClause({ status: { $ne: 'deleted' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('NOT (')
+      expect(out).toContain('"status":"deleted"')
+    })
+
+    it('handles $in with multiple values via OR of containments', () => {
+      const out = render(
+        buildFilterClause({ status: { $in: ['active', 'pending'] } }, TABLE, NO_COLUMNS)
+      )
+      expect(out).toContain(' OR ')
+      expect(out).toContain('"status":"active"')
+      expect(out).toContain('"status":"pending"')
+    })
+
+    it('handles $nin', () => {
+      const out = render(
+        buildFilterClause({ status: { $nin: ['deleted', 'archived'] } }, TABLE, NO_COLUMNS)
+      )
+      expect(out).toContain('NOT (')
+      expect(out).toContain(' AND ')
+    })
+
+    it('handles $contains as ILIKE', () => {
+      const out = render(buildFilterClause({ name: { $contains: 'john' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain(`${TABLE}.data->>'name'`)
+      expect(out).toContain('ILIKE')
+      expect(out).toContain('%john%')
+    })
+
+    it('handles $ncontains as negated ILIKE that surfaces null cells', () => {
+      const out = render(buildFilterClause({ name: { $ncontains: 'john' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('IS NULL')
+      expect(out).toContain('NOT ILIKE')
+      expect(out).toContain('%john%')
+    })
+
+    it('handles $startsWith with a trailing wildcard only', () => {
+      const out = render(buildFilterClause({ name: { $startsWith: 'jo' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('ILIKE')
+      expect(out).toContain('jo%')
+      expect(out).not.toContain('%jo%')
+    })
+
+    it('handles $endsWith with a leading wildcard only', () => {
+      const out = render(buildFilterClause({ file: { $endsWith: '.pdf' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('ILIKE')
+      expect(out).toContain('%.pdf')
+    })
+
+    it('escapes ILIKE wildcards in pattern values', () => {
+      const out = render(buildFilterClause({ name: { $contains: '50%_off' } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('50\\%\\_off')
+    })
+
+    it('rejects an empty pattern value rather than matching every row', () => {
+      for (const op of ['$contains', '$ncontains', '$startsWith', '$endsWith'] as const) {
+        expect(() =>
+          buildFilterClause({ name: { [op]: '' } } as Filter, TABLE, NO_COLUMNS)
+        ).toThrow(/requires a non-empty value/)
+      }
+    })
+
+    it('handles $empty: true as null-or-empty-string check', () => {
+      const out = render(buildFilterClause({ phone: { $empty: true } }, TABLE, NO_COLUMNS))
+      expect(out).toContain(`${TABLE}.data->>'phone'`)
+      expect(out).toContain('IS NULL')
+      expect(out).toContain("= ''")
+      expect(out).toContain(' OR ')
+    })
+
+    it('handles $empty: false as present-and-non-empty check', () => {
+      const out = render(buildFilterClause({ phone: { $empty: false } }, TABLE, NO_COLUMNS))
+      expect(out).toContain('IS NOT NULL')
+      expect(out).toContain("<> ''")
+      expect(out).toContain(' AND ')
+    })
+
+    it('coerces string "true"/"false" $empty operands (lenient raw-API input)', () => {
+      const truthy = render(
+        buildFilterClause({ phone: { $empty: 'true' } } as Filter, TABLE, NO_COLUMNS)
+      )
+      expect(truthy).toContain('IS NULL')
+      const falsy = render(
+        buildFilterClause({ phone: { $empty: 'false' } } as Filter, TABLE, NO_COLUMNS)
+      )
+      expect(falsy).toContain('IS NOT NULL')
+    })
+
+    it('throws on a non-boolean $empty operand rather than silently inverting', () => {
+      expect(() =>
+        buildFilterClause({ phone: { $empty: 1 } } as unknown as Filter, TABLE, NO_COLUMNS)
+      ).toThrow(/\$empty on column "phone" requires a boolean/)
+    })
+
+    it('joins multiple top-level conditions with AND', () => {
+      const out = render(
+        buildFilterClause({ status: 'active', age: { $gt: 18 } }, TABLE, NO_COLUMNS)
+      )
+      expect(out).toContain(' AND ')
+    })
+
+    it('handles $or logical operator', () => {
+      const out = render(
+        buildFilterClause({ $or: [{ status: 'active' }, { status: 'pending' }] }, TABLE, NO_COLUMNS)
+      )
+      expect(out).toContain(' OR ')
+    })
+
+    it('handles $and logical operator', () => {
+      const out = render(
+        buildFilterClause({ $and: [{ status: 'active' }, { age: { $gt: 18 } }] }, TABLE, NO_COLUMNS)
+      )
+      expect(out).toContain(' AND ')
+    })
+
+    it('handles nested $or and $and', () => {
+      const out = render(
+        buildFilterClause(
+          { $or: [{ $and: [{ status: 'active' }, { verified: true }] }, { role: 'admin' }] },
+          TABLE,
+          NO_COLUMNS
+        )
+      )
+      expect(out).toContain(' OR ')
+      expect(out).toContain(' AND ')
+    })
+
+    it('skips undefined values', () => {
+      const result = buildFilterClause({ name: undefined, status: 'active' }, TABLE, NO_COLUMNS)
+      expect(result).toBeDefined()
+    })
+
+    it('handles boolean / null / numeric primitives', () => {
+      expect(render(buildFilterClause({ active: true }, TABLE, NO_COLUMNS))).toContain(
+        '"active":true'
+      )
+      expect(render(buildFilterClause({ deleted_at: null }, TABLE, NO_COLUMNS))).toContain(
+        '"deleted_at":null'
+      )
+      expect(render(buildFilterClause({ count: 42 }, TABLE, NO_COLUMNS))).toContain('"count":42')
+    })
+
+    it('throws on invalid field name', () => {
+      expect(() => buildFilterClause({ 'invalid-field': 'v' }, TABLE, NO_COLUMNS)).toThrow(
+        'Invalid field name'
+      )
+    })
+
+    it('throws on invalid operator', () => {
+      const f = { name: { $invalid: 'value' } } as unknown as Filter
+      expect(() => buildFilterClause(f, TABLE, NO_COLUMNS)).toThrow('Invalid operator')
+    })
+  })
+
+  describe('buildFilterClause > date column type', () => {
+    const dateCols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
+
+    it.each([
+      ['$gt', '>'],
+      ['$gte', '>='],
+      ['$lt', '<'],
+      ['$lte', '<='],
+    ] as const)('emits ::timestamptz on both sides for %s on a date column', (operator, sqlOp) => {
+      const filter = { birthDate: { [operator]: '2024-01-01' } } as Filter
+      const out = render(buildFilterClause(filter, TABLE, dateCols))
+      expect(out).toContain(`(${TABLE}.data->>'birthDate')::timestamptz ${sqlOp} `)
+      expect(out).toContain('::timestamptz')
+      expect(out).not.toContain('::numeric')
+      // RHS cast — without it Postgres would compare as text (lexicographic).
+      expect(out.match(/::timestamptz/g)?.length).toBe(2)
+    })
+
+    it('combined range ($gte + $lte) emits two ::timestamptz pairs', () => {
+      const out = render(
+        buildFilterClause(
+          { birthDate: { $gte: '2024-01-01', $lte: '2024-12-31' } },
+          TABLE,
+          dateCols
+        )
+      )
+      expect(out.match(/::timestamptz/g)?.length).toBe(4)
+      expect(out).not.toContain('::numeric')
+      expect(out).toContain(' AND ')
+    })
+
+    it('propagates date cast through nested $and', () => {
+      const out = render(
+        buildFilterClause(
+          { $and: [{ birthDate: { $gte: '2024-01-01' } }, { birthDate: { $lt: '2025-01-01' } }] },
+          TABLE,
+          dateCols
+        )
+      )
+      expect(out).toContain('::timestamptz')
+      expect(out).not.toContain('::numeric')
+    })
+
+    it('propagates date cast through nested $or', () => {
+      const out = render(
+        buildFilterClause(
+          { $or: [{ birthDate: { $lt: '2000-01-01' } }, { birthDate: { $gt: '2024-01-01' } }] },
+          TABLE,
+          dateCols
+        )
+      )
+      expect(out).toContain('::timestamptz')
+      expect(out).not.toContain('::numeric')
+      expect(out).toContain(' OR ')
+    })
+
+    it('a number column in the same query keeps ::numeric (no cross-contamination)', () => {
+      const cols: ColumnDefinition[] = [
+        { name: 'birthDate', type: 'date' },
+        { name: 'age', type: 'number' },
+      ]
+      const out = render(
+        buildFilterClause({ birthDate: { $gte: '2024-01-01' }, age: { $gt: 18 } }, TABLE, cols)
+      )
+      expect(out).toContain('::timestamptz')
+      expect(out).toContain('::numeric')
+    })
+  })
+
+  describe('buildFilterClause > range operator value type validation', () => {
+    it('throws when $gt on a number column receives a string', () => {
+      const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
+      expect(() => buildFilterClause({ age: { $gt: 'eighteen' } } as Filter, TABLE, cols)).toThrow(
+        /column "age" \(number\) requires a number, got string/
+      )
+    })
+
+    it('throws when $gte on a date column receives a number', () => {
+      const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
+      expect(() =>
+        buildFilterClause({ birthDate: { $gte: 1704067200000 } } as Filter, TABLE, cols)
+      ).toThrow(/column "birthDate" \(date\) requires a date string, got number/)
+    })
+
+    it('throws when $lt on an unknown column (numeric fallback) receives a string', () => {
+      expect(() =>
+        buildFilterClause({ score: { $lt: 'high' } } as Filter, TABLE, NO_COLUMNS)
+      ).toThrow(/column "score" \(number\) requires a number, got string/)
+    })
+
+    it('accepts valid number on number column', () => {
+      const cols: ColumnDefinition[] = [{ name: 'age', type: 'number' }]
+      expect(() => buildFilterClause({ age: { $gt: 18 } }, TABLE, cols)).not.toThrow()
+    })
+
+    it('accepts valid ISO string on date column', () => {
+      const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
+      expect(() =>
+        buildFilterClause({ birthDate: { $gte: '2024-01-01' } }, TABLE, cols)
+      ).not.toThrow()
+    })
+  })
+
+  describe('buildSortClause', () => {
+    it('returns undefined for empty sort', () => {
+      expect(buildSortClause({}, TABLE, NO_COLUMNS)).toBeUndefined()
+    })
+
+    it('sorts string columns as text (no cast)', () => {
+      const cols: ColumnDefinition[] = [{ name: 'name', type: 'string' }]
+      const out = render(buildSortClause({ name: 'asc' }, TABLE, cols))
+      expect(out).toBe(`${TABLE}.data->>'name' ASC`)
+      expect(out).not.toContain('::')
+    })
+
+    it('sorts number columns with ::numeric NULLS LAST', () => {
+      const cols: ColumnDefinition[] = [{ name: 'salary', type: 'number' }]
+      const out = render(buildSortClause({ salary: 'desc' }, TABLE, cols))
+      expect(out).toBe(`(${TABLE}.data->>'salary')::numeric DESC NULLS LAST`)
+    })
+
+    it('sorts date columns with ::timestamptz NULLS LAST', () => {
+      const cols: ColumnDefinition[] = [{ name: 'birthDate', type: 'date' }]
+      const out = render(buildSortClause({ birthDate: 'asc' }, TABLE, cols))
+      expect(out).toBe(`(${TABLE}.data->>'birthDate')::timestamptz ASC NULLS LAST`)
+    })
+
+    it('sorts createdAt / updatedAt as direct column refs', () => {
+      expect(render(buildSortClause({ createdAt: 'desc' }, TABLE, NO_COLUMNS))).toBe(
+        `${TABLE}.createdAt DESC`
+      )
+      expect(render(buildSortClause({ updatedAt: 'asc' }, TABLE, NO_COLUMNS))).toBe(
+        `${TABLE}.updatedAt ASC`
+      )
+    })
+
+    it('combines multiple sort fields with commas', () => {
+      const cols: ColumnDefinition[] = [
+        { name: 'name', type: 'string' },
+        { name: 'salary', type: 'number' },
+      ]
+      const out = render(buildSortClause({ name: 'asc', salary: 'desc' }, TABLE, cols))
+      expect(out).toBe(
+        `${TABLE}.data->>'name' ASC, (${TABLE}.data->>'salary')::numeric DESC NULLS LAST`
+      )
+    })
+
+    it('falls back to text sort for unknown column types', () => {
+      const sort: Sort = { unknownField: 'asc' }
+      const out = render(buildSortClause(sort, TABLE, NO_COLUMNS))
+      expect(out).toBe(`${TABLE}.data->>'unknownField' ASC`)
+    })
+
+    it('throws on invalid field name', () => {
+      const sort: Sort = { 'invalid-field': 'asc' }
+      expect(() => buildSortClause(sort, TABLE, NO_COLUMNS)).toThrow('Invalid field name')
+    })
+
+    it('throws on invalid direction', () => {
+      const sort = { name: 'invalid' as 'asc' | 'desc' }
+      expect(() => buildSortClause(sort, TABLE, NO_COLUMNS)).toThrow('Invalid sort direction')
+    })
+  })
+
+  describe('Field name validation', () => {
+    it('accepts valid identifiers', () => {
+      const valid = ['name', 'user_id', '_private', 'Count123', 'a']
+      for (const name of valid) {
+        expect(() => buildFilterClause({ [name]: 'v' }, TABLE, NO_COLUMNS)).not.toThrow()
+      }
+    })
+
+    it('rejects identifiers starting with a digit', () => {
+      expect(() => buildFilterClause({ '123name': 'v' }, TABLE, NO_COLUMNS)).toThrow(
+        'Invalid field name'
+      )
+    })
+
+    it('rejects identifiers with special characters', () => {
+      const invalid = ['field-name', 'field.name', 'field name', 'field@name']
+      for (const name of invalid) {
+        expect(() => buildFilterClause({ [name]: 'v' }, TABLE, NO_COLUMNS)).toThrow(
+          'Invalid field name'
+        )
+      }
+    })
+
+    it('rejects SQL injection attempts in field names', () => {
+      const attempts = ["'; DROP TABLE users; --", 'name OR 1=1', 'name; DELETE FROM']
+      for (const a of attempts) {
+        expect(() => buildFilterClause({ [a]: 'v' }, TABLE, NO_COLUMNS)).toThrow(
+          'Invalid field name'
+        )
+      }
+    })
+  })
+})

apps/sim/lib/table/__tests__/update-row.test.ts

@@ -0,0 +1,405 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { deleteColumn, renameColumn } from '@/lib/table/columns/service'
+import {
+  batchInsertRows,
+  insertRow,
+  replaceTableRows,
+  updateRow,
+  upsertRow,
+} from '@/lib/table/rows/service'
+import type { TableDefinition } from '@/lib/table/types'
+import { getUniqueColumns } from '@/lib/table/validation'
+
+vi.mock('@sim/db', () => dbChainMock)
+
+// Capacity is exercised in billing.test.ts; here it's a no-op so the timeout-scaling
+// suites can use large synthetic row counts without tripping the plan limit.
+vi.mock('@/lib/table/billing', () => ({
+  assertRowCapacity: vi.fn().mockResolvedValue(1_000_000),
+  notifyTableRowUsage: vi.fn(),
+  getMaxRowsPerTable: vi.fn().mockResolvedValue(1_000_000),
+  wouldExceedRowLimit: () => false,
+  TableRowLimitError: class TableRowLimitError extends Error {},
+}))
+
+vi.mock('@/lib/table/validation', () => ({
+  validateRowSize: vi.fn(() => ({ valid: true, errors: [] })),
+  validateRowAgainstSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowToSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  coerceRowValues: vi.fn(),
+  validateTableName: vi.fn(() => ({ valid: true, errors: [] })),
+  validateTableSchema: vi.fn(() => ({ valid: true, errors: [] })),
+  getUniqueColumns: vi.fn(() => []),
+  checkUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+  checkBatchUniqueConstraintsDb: vi.fn(async () => ({ valid: true, errors: [] })),
+}))
+
+/**
+ * Inspects the queued `trx.execute(...)` calls for SQL containing `substring`.
+ * Works with both `sql\`...\`` (produces `{ strings, values }`) and `sql.raw(...)`
+ * (produces `{ rawSql }`) from the global drizzle mock.
+ */
+function findExecutedSqlContaining(substring: string): boolean {
+  return dbChainMockFns.execute.mock.calls.some(([arg]) => {
+    if (!arg || typeof arg !== 'object') return false
+    const a = arg as Record<string, unknown>
+    if (Array.isArray(a.strings)) {
+      return (a.strings as string[]).some((s) => typeof s === 'string' && s.includes(substring))
+    }
+    if (typeof a.rawSql === 'string') {
+      return (a.rawSql as string).includes(substring)
+    }
+    return false
+  })
+}
+
+function findExecutedRawSql(substring: string): string | undefined {
+  for (const [arg] of dbChainMockFns.execute.mock.calls) {
+    if (!arg || typeof arg !== 'object') continue
+    const raw = (arg as { rawSql?: unknown }).rawSql
+    if (typeof raw === 'string' && raw.includes(substring)) return raw
+  }
+  return undefined
+}
+
+const EXISTING_ROW = {
+  id: 'row-1',
+  tableId: 'tbl-1',
+  workspaceId: 'ws-1',
+  data: { name: 'Alice', age: 30 },
+  position: 1,
+  createdAt: new Date('2024-01-01'),
+  updatedAt: new Date('2024-01-01'),
+}
+
+const TABLE: TableDefinition = {
+  id: 'tbl-1',
+  name: 'People',
+  description: null,
+  schema: {
+    columns: [
+      { name: 'name', type: 'string' },
+      { name: 'age', type: 'number' },
+    ],
+  },
+  metadata: null,
+  rowCount: 0,
+  maxRows: 1000,
+  workspaceId: 'ws-1',
+  createdBy: 'user-1',
+  archivedAt: null,
+  createdAt: new Date('2024-01-01'),
+  updatedAt: new Date('2024-01-01'),
+}
+
+describe('updateRow — partial merge', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    dbChainMockFns.limit.mockResolvedValue([EXISTING_ROW])
+  })
+
+  it('preserves columns not included in the partial update', async () => {
+    const result = await updateRow(
+      { tableId: 'tbl-1', rowId: 'row-1', data: { age: 31 }, workspaceId: 'ws-1' },
+      TABLE,
+      'req-1'
+    )
+
+    expect(result.data).toEqual({ name: 'Alice', age: 31 })
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({ data: { name: 'Alice', age: 31 } })
+    )
+  })
+
+  it('allows updating a single column without affecting others', async () => {
+    const result = await updateRow(
+      { tableId: 'tbl-1', rowId: 'row-1', data: { name: 'Bob' }, workspaceId: 'ws-1' },
+      TABLE,
+      'req-1'
+    )
+
+    expect(result.data).toEqual({ name: 'Bob', age: 30 })
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({ data: { name: 'Bob', age: 30 } })
+    )
+  })
+
+  it('allows explicitly nulling a field while preserving others', async () => {
+    const result = await updateRow(
+      { tableId: 'tbl-1', rowId: 'row-1', data: { age: null }, workspaceId: 'ws-1' },
+      TABLE,
+      'req-1'
+    )
+
+    expect(result.data).toEqual({ name: 'Alice', age: null })
+    expect(dbChainMockFns.set).toHaveBeenCalledWith(
+      expect.objectContaining({ data: { name: 'Alice', age: null } })
+    )
+  })
+
+  it('handles a full-row update correctly (idempotent merge)', async () => {
+    const result = await updateRow(
+      { tableId: 'tbl-1', rowId: 'row-1', data: { name: 'Bob', age: 25 }, workspaceId: 'ws-1' },
+      TABLE,
+      'req-1'
+    )
+
+    expect(result.data).toEqual({ name: 'Bob', age: 25 })
+  })
+
+  it('throws when the row does not exist', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+
+    await expect(
+      updateRow(
+        { tableId: 'tbl-1', rowId: 'row-missing', data: { age: 31 }, workspaceId: 'ws-1' },
+        TABLE,
+        'req-1'
+      )
+    ).rejects.toThrow('Row not found')
+  })
+})
+
+describe('insertRow — position race safety (migration 0198 + advisory lock)', () => {
+  beforeEach(() => {
+    vi.resetAllMocks()
+    resetDbChainMock()
+    vi.mocked(getUniqueColumns).mockReturnValue([])
+  })
+
+  it('auto-position inserts acquire the per-table advisory lock before reading max(position)', async () => {
+    await expect(
+      insertRow({ tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1' }, TABLE, 'req-1')
+    ).rejects.toBeDefined()
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+    expect(findExecutedSqlContaining('hashtextextended')).toBe(true)
+  })
+
+  it('explicit-position inserts also acquire the advisory lock to serialize order-key minting', async () => {
+    await expect(
+      insertRow(
+        { tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1', position: 5 },
+        TABLE,
+        'req-1'
+      )
+    ).rejects.toBeDefined()
+
+    // A position-based insert resolves its order_key from the neighbor at that
+    // rank; the lock serializes concurrent minting at the same slot.
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+  })
+
+  it('batchInsertRows acquires the advisory lock (always auto-positioned)', async () => {
+    await expect(
+      batchInsertRows(
+        { tableId: 'tbl-1', rows: [{ name: 'a' }, { name: 'b' }], workspaceId: 'ws-1' },
+        TABLE,
+        'req-1'
+      )
+    ).rejects.toBeDefined()
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+  })
+
+  it('upsertRow skips the advisory lock on the update path (match found)', async () => {
+    vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
+    dbChainMockFns.limit.mockResolvedValueOnce([
+      {
+        id: 'row-1',
+        tableId: 'tbl-1',
+        workspaceId: 'ws-1',
+        data: { name: 'Alice', age: 30 },
+        position: 0,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ])
+    dbChainMockFns.returning.mockResolvedValueOnce([
+      {
+        id: 'row-1',
+        tableId: 'tbl-1',
+        workspaceId: 'ws-1',
+        data: { name: 'Alice', age: 31 },
+        position: 0,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      },
+    ])
+
+    await upsertRow(
+      {
+        tableId: 'tbl-1',
+        workspaceId: 'ws-1',
+        data: { name: 'Alice', age: 31 },
+        conflictTarget: 'name',
+      },
+      TABLE,
+      'req-1'
+    )
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(false)
+  })
+
+  it('upsertRow acquires the advisory lock on the insert path (no match)', async () => {
+    vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
+    // Initial existing-row check + post-lock re-check both find no match.
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+
+    await expect(
+      upsertRow(
+        {
+          tableId: 'tbl-1',
+          workspaceId: 'ws-1',
+          data: { name: 'Bob', age: 25 },
+          conflictTarget: 'name',
+        },
+        TABLE,
+        'req-1'
+      )
+    ).rejects.toBeDefined()
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+  })
+
+  it('upsertRow re-checks after acquiring the lock and switches to UPDATE when a racing tx inserted the row', async () => {
+    vi.mocked(getUniqueColumns).mockReturnValue([{ name: 'name', type: 'string', unique: true }])
+    // Initial existing-row check: no match (another tx has not committed yet).
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+    // Post-lock re-check: a racing tx just inserted the row.
+    const racedRow = {
+      id: 'row-raced',
+      tableId: 'tbl-1',
+      workspaceId: 'ws-1',
+      data: { name: 'Bob', age: 25 },
+      position: 0,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    }
+    dbChainMockFns.limit.mockResolvedValueOnce([racedRow])
+    // UPDATE returning the patched row.
+    dbChainMockFns.returning.mockResolvedValueOnce([
+      { ...racedRow, data: { name: 'Bob', age: 26 } },
+    ])
+
+    const result = await upsertRow(
+      {
+        tableId: 'tbl-1',
+        workspaceId: 'ws-1',
+        data: { name: 'Bob', age: 26 },
+        conflictTarget: 'name',
+      },
+      TABLE,
+      'req-1'
+    )
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+    expect(result.operation).toBe('update')
+    expect(result.row.id).toBe('row-raced')
+    expect(dbChainMockFns.update).toHaveBeenCalled()
+    expect(dbChainMockFns.insert).not.toHaveBeenCalled()
+  })
+})
+
+describe('mutation paths — SET LOCAL timeouts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+  })
+
+  it('insertRow sets the default 10s/3s/5s timeouts', async () => {
+    await expect(
+      insertRow({ tableId: 'tbl-1', data: { name: 'a' }, workspaceId: 'ws-1' }, TABLE, 'req-1')
+    ).rejects.toBeDefined()
+
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '10000ms'")).toBeDefined()
+    expect(findExecutedRawSql("SET LOCAL lock_timeout = '3000ms'")).toBeDefined()
+    expect(
+      findExecutedRawSql("SET LOCAL idle_in_transaction_session_timeout = '5000ms'")
+    ).toBeDefined()
+  })
+
+  it('batchInsertRows raises statement_timeout to 60s', async () => {
+    await expect(
+      batchInsertRows(
+        { tableId: 'tbl-1', rows: [{ name: 'a' }], workspaceId: 'ws-1' },
+        TABLE,
+        'req-1'
+      )
+    ).rejects.toBeDefined()
+
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '60000ms'")).toBeDefined()
+  })
+
+  it('replaceTableRows scales statement_timeout with (existing + new) row count', async () => {
+    const bigTable: TableDefinition = { ...TABLE, rowCount: 100_000, maxRows: 1_000_000 }
+    const payload = Array.from({ length: 50_000 }, (_, i) => ({ name: `row-${i}` }))
+
+    await replaceTableRows(
+      { tableId: 'tbl-1', workspaceId: 'ws-1', rows: payload },
+      bigTable,
+      'req-1'
+    )
+
+    // (100_000 + 50_000) × 3ms/row = 450_000ms; above 120_000 floor, below 600_000 cap
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '450000ms'")).toBeDefined()
+  })
+
+  it('replaceTableRows caps scaled timeout at 10 minutes for very large tables', async () => {
+    const hugeTable: TableDefinition = { ...TABLE, rowCount: 10_000_000, maxRows: 20_000_000 }
+
+    await replaceTableRows({ tableId: 'tbl-1', workspaceId: 'ws-1', rows: [] }, hugeTable, 'req-1')
+
+    // 10M × 3ms = 30M ms, capped at 600_000ms (10 min)
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '600000ms'")).toBeDefined()
+  })
+
+  it('replaceTableRows uses the 120s floor on small tables', async () => {
+    const smallTable: TableDefinition = { ...TABLE, rowCount: 10 }
+
+    await replaceTableRows(
+      { tableId: 'tbl-1', workspaceId: 'ws-1', rows: [{ name: 'a' }, { name: 'b' }] },
+      smallTable,
+      'req-1'
+    )
+
+    // 12 × 3ms = 36ms → floored at 120_000ms
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '120000ms'")).toBeDefined()
+  })
+
+  it('renameColumn is metadata-only — no per-row JSONB rewrite regardless of row count', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ ...TABLE, rowCount: 500_000 }])
+
+    await renameColumn({ tableId: 'tbl-1', oldName: 'name', newName: 'full_name' }, 'req-1')
+
+    // Row data is keyed by the column's stable id (unchanged by a rename), so no
+    // `user_table_rows` key rewrite is executed — and thus no scaled timeout.
+    expect(findExecutedSqlContaining('jsonb_build_object')).toBe(false)
+  })
+
+  it('deleteColumn uses the 60s floor on small tables', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([{ ...TABLE, rowCount: 100 }])
+
+    await deleteColumn({ tableId: 'tbl-1', columnName: 'age' }, 'req-1')
+
+    // 100 × 2ms = 200ms → floored at 60_000ms
+    expect(findExecutedRawSql("SET LOCAL statement_timeout = '60000ms'")).toBeDefined()
+  })
+
+  it('replaceTableRows acquires the per-table advisory lock to serialize concurrent replaces', async () => {
+    await replaceTableRows(
+      { tableId: 'tbl-1', workspaceId: 'ws-1', rows: [{ name: 'a' }] },
+      { ...TABLE, rowCount: 5 },
+      'req-1'
+    )
+
+    expect(findExecutedSqlContaining('pg_advisory_xact_lock')).toBe(true)
+    expect(findExecutedSqlContaining('hashtextextended')).toBe(true)
+  })
+})

apps/sim/lib/table/__tests__/validation.test.ts

@@ -0,0 +1,496 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { TABLE_LIMITS } from '@/lib/table/constants'
+import {
+  type ColumnDefinition,
+  coerceRowToSchema,
+  coerceRowValues,
+  getUniqueColumns,
+  type TableSchema,
+  validateColumnDefinition,
+  validateRowAgainstSchema,
+  validateRowSize,
+  validateTableName,
+  validateTableSchema,
+  validateUniqueConstraints,
+} from '@/lib/table/validation'
+
+describe('Validation', () => {
+  describe('validateTableName', () => {
+    it('should accept valid table names', () => {
+      const validNames = ['users', 'user_data', '_private', 'Users123', 'a']
+
+      for (const name of validNames) {
+        const result = validateTableName(name)
+        expect(result.valid).toBe(true)
+        expect(result.errors).toHaveLength(0)
+      }
+    })
+
+    it('should reject empty name', () => {
+      const result = validateTableName('')
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Table name is required')
+    })
+
+    it('should reject null/undefined name', () => {
+      const result1 = validateTableName(null as unknown as string)
+      expect(result1.valid).toBe(false)
+
+      const result2 = validateTableName(undefined as unknown as string)
+      expect(result2.valid).toBe(false)
+    })
+
+    it('should reject names starting with number', () => {
+      const result = validateTableName('123table')
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must start with letter or underscore')
+    })
+
+    it('should reject names with special characters', () => {
+      const invalidNames = ['table-name', 'table.name', 'table name', 'table@name']
+
+      for (const name of invalidNames) {
+        const result = validateTableName(name)
+        expect(result.valid).toBe(false)
+      }
+    })
+
+    it('should reject names exceeding max length', () => {
+      const longName = 'a'.repeat(TABLE_LIMITS.MAX_TABLE_NAME_LENGTH + 1)
+      const result = validateTableName(longName)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('exceeds maximum length')
+    })
+  })
+
+  describe('validateColumnDefinition', () => {
+    it('should accept valid column definition', () => {
+      const column: ColumnDefinition = {
+        name: 'email',
+        type: 'string',
+        required: true,
+        unique: true,
+      }
+      const result = validateColumnDefinition(column)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should accept all valid column types', () => {
+      const types = ['string', 'number', 'boolean', 'date', 'json'] as const
+
+      for (const type of types) {
+        const result = validateColumnDefinition({ name: 'test', type })
+        expect(result.valid).toBe(true)
+      }
+    })
+
+    it('should reject empty column name', () => {
+      const result = validateColumnDefinition({ name: '', type: 'string' })
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Column name is required')
+    })
+
+    it('should reject invalid column type', () => {
+      const result = validateColumnDefinition({
+        name: 'test',
+        type: 'invalid' as any,
+      })
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('invalid type')
+    })
+
+    it('should reject column name exceeding max length', () => {
+      const longName = 'a'.repeat(TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH + 1)
+      const result = validateColumnDefinition({ name: longName, type: 'string' })
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('exceeds maximum length')
+    })
+  })
+
+  describe('validateTableSchema', () => {
+    it('should accept valid schema', () => {
+      const schema: TableSchema = {
+        columns: [
+          { name: 'id', type: 'string', required: true, unique: true },
+          { name: 'name', type: 'string', required: true },
+          { name: 'age', type: 'number' },
+        ],
+      }
+      const result = validateTableSchema(schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should reject empty columns array', () => {
+      const schema: TableSchema = { columns: [] }
+      const result = validateTableSchema(schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Schema must have at least one column')
+    })
+
+    it('should reject duplicate column names', () => {
+      const schema: TableSchema = {
+        columns: [
+          { name: 'id', type: 'string' },
+          { name: 'ID', type: 'number' },
+        ],
+      }
+      const result = validateTableSchema(schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Duplicate column names found')
+    })
+
+    it('should reject null schema', () => {
+      const result = validateTableSchema(null as unknown as TableSchema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Schema is required')
+    })
+
+    it('should reject schema without columns array', () => {
+      const result = validateTableSchema({} as TableSchema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Schema must have columns array')
+    })
+
+    it('should reject schema exceeding max columns', () => {
+      const columns = Array.from({ length: TABLE_LIMITS.MAX_COLUMNS_PER_TABLE + 1 }, (_, i) => ({
+        name: `col_${i}`,
+        type: 'string' as const,
+      }))
+      const result = validateTableSchema({ columns })
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('exceeds maximum columns')
+    })
+  })
+
+  describe('validateRowSize', () => {
+    it('should accept row within size limit', () => {
+      const data = { name: 'test', value: 123 }
+      const result = validateRowSize(data)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should reject row exceeding size limit', () => {
+      const largeString = 'a'.repeat(TABLE_LIMITS.MAX_ROW_SIZE_BYTES + 1)
+      const data = { content: largeString }
+      const result = validateRowSize(data)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('exceeds limit')
+    })
+
+    it('should measure UTF-8 bytes, not UTF-16 code units', () => {
+      // '工' is one UTF-16 code unit but three UTF-8 bytes — a char-count check
+      // would accept this row at ~1/3 of the real serialized size.
+      const chars = Math.ceil(TABLE_LIMITS.MAX_ROW_SIZE_BYTES / 3) + 1
+      const result = validateRowSize({ content: '工'.repeat(chars) })
+      expect(result.valid).toBe(false)
+    })
+  })
+
+  describe('validateRowAgainstSchema', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'age', type: 'number' },
+        { name: 'active', type: 'boolean' },
+        { name: 'created', type: 'date' },
+        { name: 'metadata', type: 'json' },
+      ],
+    }
+
+    it('should accept valid row data', () => {
+      const data = {
+        name: 'John',
+        age: 30,
+        active: true,
+        created: '2024-01-01',
+        metadata: { key: 'value' },
+      }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should reject missing required field', () => {
+      const data = { age: 30 }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Missing required field: name')
+    })
+
+    it('should reject wrong type for string field', () => {
+      const data = { name: 123 }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be string')
+    })
+
+    it('should reject wrong type for number field', () => {
+      const data = { name: 'John', age: 'thirty' }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be number')
+    })
+
+    it('should reject NaN for number field', () => {
+      const data = { name: 'John', age: Number.NaN }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be number')
+    })
+
+    it('should reject wrong type for boolean field', () => {
+      const data = { name: 'John', active: 'yes' }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be boolean')
+    })
+
+    it('should reject invalid date string', () => {
+      const data = { name: 'John', created: 'not-a-date' }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be valid date')
+    })
+
+    it('should accept valid ISO date string', () => {
+      const data = { name: 'John', created: '2024-01-15T10:30:00Z' }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should accept Date object', () => {
+      const data = { name: 'John', created: new Date() }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should allow null for optional fields', () => {
+      const data = { name: 'John', age: null }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should allow undefined for optional fields', () => {
+      const data = { name: 'John' }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should allow long strings — cell size is bounded by the row byte cap, not per-value', () => {
+      const longString = 'a'.repeat(100_000)
+      const data = { name: longString }
+      const result = validateRowAgainstSchema(data, schema)
+      expect(result.valid).toBe(true)
+    })
+  })
+
+  describe('coerceRowToSchema', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'age', type: 'number' },
+        { name: 'founded', type: 'number', required: true },
+        { name: 'active', type: 'boolean' },
+        { name: 'created', type: 'date' },
+        { name: 'metadata', type: 'json' },
+      ],
+    }
+
+    it('coerces a numeric string to a number in place', () => {
+      const data = { name: 'Acme', founded: '1999' }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.founded).toBe(1999)
+    })
+
+    it('nulls an un-coercible value for an optional number column', () => {
+      const data = { name: 'Acme', founded: 2000, age: 'unknown' }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.age).toBeNull()
+    })
+
+    it('rejects an un-coercible value for a required number column', () => {
+      const data = { name: 'Acme', founded: 'unknown' }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('founded must be number')
+      expect(data.founded).toBe('unknown')
+    })
+
+    it('coerces a number to a string for a string column', () => {
+      const data = { name: 12345, founded: 2000 }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.name).toBe('12345')
+    })
+
+    it('coerces "true"/"false" strings to booleans', () => {
+      const data = { name: 'Acme', founded: 2000, active: 'false' }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.active).toBe(false)
+    })
+
+    it('coerces an epoch number to an ISO date string', () => {
+      const epoch = Date.parse('2024-01-15T00:00:00Z')
+      const data = { name: 'Acme', founded: 2000, created: epoch }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.created).toBe(new Date(epoch).toISOString())
+    })
+
+    it('coerces a Date instance to an ISO date string', () => {
+      const date = new Date('2024-01-15T00:00:00Z')
+      const data = { name: 'Acme', founded: 2000, created: date }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.created).toBe(date.toISOString())
+    })
+
+    it('nulls an out-of-range epoch number for an optional date column without throwing', () => {
+      const data = { name: 'Acme', founded: 2000, created: 1e20 }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.created).toBeNull()
+    })
+
+    it('nulls an invalid Date instance for an optional date column without throwing', () => {
+      const data = { name: 'Acme', founded: 2000, created: new Date('not-a-date') }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data.created).toBeNull()
+    })
+
+    it('leaves already-correct values untouched and passes through json', () => {
+      const data = { name: 'Acme', founded: 2000, metadata: { k: 'v' } }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(true)
+      expect(data).toEqual({ name: 'Acme', founded: 2000, metadata: { k: 'v' } })
+    })
+
+    it('still rejects a missing required field', () => {
+      const data = { name: 'Acme' }
+      const result = coerceRowToSchema(data, schema)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toContain('Missing required field: founded')
+    })
+  })
+
+  describe('coerceRowValues', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'founded', type: 'number', required: true },
+        { name: 'age', type: 'number' },
+      ],
+    }
+
+    it('coerces a partial patch in place without flagging absent required fields', () => {
+      const patch = { age: '42' }
+      coerceRowValues(patch, schema)
+      expect(patch.age).toBe(42)
+    })
+
+    it('nulls an un-coercible optional value in a patch', () => {
+      const patch: { age: unknown } = { age: 'nope' }
+      coerceRowValues(patch as never, schema)
+      expect(patch.age).toBeNull()
+    })
+
+    it('leaves an un-coercible required value in place for downstream validation', () => {
+      const patch: { founded: unknown } = { founded: 'nope' }
+      coerceRowValues(patch as never, schema)
+      expect(patch.founded).toBe('nope')
+    })
+  })
+
+  describe('getUniqueColumns', () => {
+    it('should return only columns with unique=true', () => {
+      const schema: TableSchema = {
+        columns: [
+          { name: 'id', type: 'string', unique: true },
+          { name: 'email', type: 'string', unique: true },
+          { name: 'name', type: 'string' },
+          { name: 'count', type: 'number', unique: false },
+        ],
+      }
+      const result = getUniqueColumns(schema)
+      expect(result).toHaveLength(2)
+      expect(result.map((c) => c.name)).toEqual(['id', 'email'])
+    })
+
+    it('should return empty array when no unique columns', () => {
+      const schema: TableSchema = {
+        columns: [
+          { name: 'name', type: 'string' },
+          { name: 'value', type: 'number' },
+        ],
+      }
+      const result = getUniqueColumns(schema)
+      expect(result).toHaveLength(0)
+    })
+  })
+
+  describe('validateUniqueConstraints', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'id', type: 'string', unique: true },
+        { name: 'email', type: 'string', unique: true },
+        { name: 'name', type: 'string' },
+      ],
+    }
+
+    const existingRows = [
+      { id: 'row1', data: { id: 'abc123', email: 'john@example.com', name: 'John' } },
+      { id: 'row2', data: { id: 'def456', email: 'jane@example.com', name: 'Jane' } },
+    ]
+
+    it('should accept data with unique values', () => {
+      const data = { id: 'xyz789', email: 'new@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should reject duplicate unique value', () => {
+      const data = { id: 'abc123', email: 'new@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(false)
+      expect(result.errors[0]).toContain('must be unique')
+      expect(result.errors[0]).toContain('abc123')
+    })
+
+    it('should be case-insensitive for string comparisons', () => {
+      const data = { id: 'ABC123', email: 'new@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(false)
+    })
+
+    it('should exclude specified row from checks (for updates)', () => {
+      const data = { id: 'abc123', email: 'john@example.com', name: 'John Updated' }
+      const result = validateUniqueConstraints(data, schema, existingRows, 'row1')
+      expect(result.valid).toBe(true)
+    })
+
+    it('should allow null values for unique columns', () => {
+      const data = { id: null, email: 'new@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should allow undefined values for unique columns', () => {
+      const data = { email: 'new@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(true)
+    })
+
+    it('should report multiple violations', () => {
+      const data = { id: 'abc123', email: 'john@example.com', name: 'New User' }
+      const result = validateUniqueConstraints(data, schema, existingRows)
+      expect(result.valid).toBe(false)
+      expect(result.errors).toHaveLength(2)
+    })
+  })
+})

apps/sim/lib/table/backfill-runner.ts

@@ -0,0 +1,345 @@
+import { db } from '@sim/db'
+import { tableRowExecutions, userTableRows, workflowExecutionLogs } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, asc, count, eq, gt, inArray } from 'drizzle-orm'
+import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
+import { runDetached } from '@/lib/core/utils/background'
+import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
+import { appendTableEvent } from '@/lib/table/events'
+import {
+  markJobFailed,
+  markJobReady,
+  markTableJobRunning,
+  updateJobProgress,
+} from '@/lib/table/jobs/service'
+import { pluckByPath } from '@/lib/table/pluck'
+import { batchUpdateRows } from '@/lib/table/rows/service'
+import { getTableById } from '@/lib/table/service'
+import type {
+  RowData,
+  TableBackfillJobPayload,
+  TableDefinition,
+  WorkflowGroupOutput,
+} from '@/lib/table/types'
+
+const logger = createLogger('TableBackfillRunner')
+
+/** Completed-run count above which the backfill runs as a background job instead of inline. */
+const BACKFILL_ASYNC_THRESHOLD_ROWS = 500
+
+/** Completed sidecar rows fetched (and their logs materialized) per page. */
+const BACKFILL_PAGE_SIZE = 200
+
+/** Thrown when this worker loses the job (canceled / janitor-failed). */
+class JobSupersededError extends Error {}
+
+export interface TableBackfillPayload {
+  jobId: string
+  tableId: string
+  workspaceId: string
+  groupId: string
+  outputs: WorkflowGroupOutput[]
+  overwrite: boolean
+  /** User who triggered the schema change, for usage attribution on the row writes. */
+  actorUserId?: string | null
+}
+
+/** Minimal shape of a trace span we care about for backfill. */
+interface BackfillTraceSpan {
+  blockId?: string
+  output?: Record<string, unknown>
+  children?: BackfillTraceSpan[]
+}
+
+/** DFS the trace tree for the first span matching `blockId`. */
+function findSpanByBlockId(
+  spans: BackfillTraceSpan[] | undefined,
+  blockId: string
+): BackfillTraceSpan | undefined {
+  if (!spans) return undefined
+  for (const span of spans) {
+    if (span.blockId === blockId) return span
+    const child = findSpanByBlockId(span.children, blockId)
+    if (child) return child
+  }
+  return undefined
+}
+
+/** One keyset page of completed (rowId, executionId) pairs for the group, ordered by rowId. */
+async function selectCompletedExecPage(
+  tableId: string,
+  groupId: string,
+  afterRowId: string | undefined,
+  limit: number
+): Promise<Array<{ rowId: string; executionId: string | null }>> {
+  return db
+    .select({
+      rowId: tableRowExecutions.rowId,
+      executionId: tableRowExecutions.executionId,
+    })
+    .from(tableRowExecutions)
+    .where(
+      and(
+        eq(tableRowExecutions.tableId, tableId),
+        eq(tableRowExecutions.groupId, groupId),
+        eq(tableRowExecutions.status, 'completed'),
+        afterRowId ? gt(tableRowExecutions.rowId, afterRowId) : undefined
+      )
+    )
+    .orderBy(asc(tableRowExecutions.rowId))
+    .limit(limit)
+}
+
+/**
+ * Backfills one page of rows: pulls each target output's value out of the rows' saved trace
+ * spans (materialized from object storage with bounded concurrency) and writes it into row data.
+ * Returns the number of rows updated.
+ */
+async function processBackfillPage(opts: {
+  table: TableDefinition
+  outputs: WorkflowGroupOutput[]
+  overwrite: boolean
+  execs: Array<{ rowId: string; executionId: string | null }>
+  requestId: string
+  actorUserId?: string | null
+}): Promise<number> {
+  const { table, outputs, overwrite, execs, requestId, actorUserId } = opts
+
+  const executionIdsByRow = new Map<string, string>()
+  for (const e of execs) {
+    if (!e.executionId) continue
+    executionIdsByRow.set(e.rowId, e.executionId)
+  }
+  if (executionIdsByRow.size === 0) return 0
+
+  const rowRecords = await db
+    .select({ id: userTableRows.id, data: userTableRows.data })
+    .from(userTableRows)
+    .where(
+      and(
+        eq(userTableRows.tableId, table.id),
+        inArray(userTableRows.id, Array.from(executionIdsByRow.keys()))
+      )
+    )
+
+  const executionIds = Array.from(new Set(executionIdsByRow.values()))
+  const logs = await db
+    .select({
+      executionId: workflowExecutionLogs.executionId,
+      workflowId: workflowExecutionLogs.workflowId,
+      workspaceId: workflowExecutionLogs.workspaceId,
+      executionData: workflowExecutionLogs.executionData,
+    })
+    .from(workflowExecutionLogs)
+    .where(inArray(workflowExecutionLogs.executionId, executionIds))
+
+  const logByExecutionId = new Map<string, { traceSpans?: BackfillTraceSpan[] }>()
+  // Heavy execution data may live in object storage; resolve pointers (bounded concurrency).
+  await mapWithConcurrency(logs, MATERIALIZE_CONCURRENCY, async (log) => {
+    const executionData = await materializeExecutionData(
+      log.executionData as Record<string, unknown> | null,
+      { workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId }
+    )
+    logByExecutionId.set(
+      log.executionId,
+      (executionData as { traceSpans?: BackfillTraceSpan[] }) ?? {}
+    )
+  })
+
+  const updates: Array<{ rowId: string; data: RowData }> = []
+  for (const r of rowRecords) {
+    const execId = executionIdsByRow.get(r.id)
+    if (!execId) continue
+    const log = logByExecutionId.get(execId)
+    if (!log) continue
+
+    const dataPatch: RowData = {}
+    let mutated = false
+    for (const out of outputs) {
+      if (!overwrite && (r.data as RowData)[out.columnName] !== undefined) continue
+      const span = findSpanByBlockId(log.traceSpans, out.blockId)
+      if (!span?.output) continue
+      const picked = pluckByPath(span.output, out.path)
+      if (picked === undefined) continue
+      dataPatch[out.columnName] = picked as RowData[string]
+      mutated = true
+    }
+    if (!mutated) continue
+    updates.push({ rowId: r.id, data: dataPatch })
+  }
+
+  if (updates.length === 0) return 0
+
+  await batchUpdateRows(
+    { tableId: table.id, updates, workspaceId: table.workspaceId, actorUserId },
+    table,
+    requestId
+  )
+  return updates.length
+}
+
+/**
+ * Background worker for large output-column backfills. Pages the group's completed executions
+ * (keyset by rowId), materializing logs and writing values page by page. Ownership-gated per
+ * page; retry-safe (re-plucking the same spans writes the same values, and `overwrite: false`
+ * passes skip already-filled cells).
+ */
+export async function runTableBackfill(payload: TableBackfillPayload): Promise<void> {
+  const { jobId, tableId, groupId, outputs, overwrite, actorUserId } = payload
+  const requestId = generateId().slice(0, 8)
+
+  try {
+    const table = await getTableById(tableId, { includeArchived: true })
+    if (!table) throw new Error(`Backfill target table ${tableId} not found`)
+
+    let processed = 0
+    let updated = 0
+    let afterRowId: string | undefined
+
+    while (true) {
+      const owns = await updateJobProgress(tableId, processed, jobId)
+      if (!owns) throw new JobSupersededError()
+
+      const execs = await selectCompletedExecPage(tableId, groupId, afterRowId, BACKFILL_PAGE_SIZE)
+      if (execs.length === 0) break
+      afterRowId = execs[execs.length - 1].rowId
+
+      updated += await processBackfillPage({
+        table,
+        outputs,
+        overwrite,
+        execs,
+        requestId,
+        actorUserId,
+      })
+      processed += execs.length
+    }
+
+    await updateJobProgress(tableId, processed, jobId)
+    const becameReady = await markJobReady(tableId, jobId)
+    if (becameReady) {
+      void appendTableEvent({
+        kind: 'job',
+        type: 'backfill',
+        tableId,
+        jobId,
+        status: 'ready',
+        progress: updated,
+      })
+      logger.info(`[${requestId}] Backfill complete`, { tableId, groupId, processed, updated })
+    } else {
+      logger.info(`[${requestId}] Backfill finished but no longer owns the run`, { tableId, jobId })
+    }
+  } catch (err) {
+    if (err instanceof JobSupersededError) {
+      logger.info(`[${requestId}] Backfill superseded/canceled; stopping`, { tableId, jobId })
+    } else {
+      const message = getErrorMessage(err, 'Backfill failed')
+      logger.error(`[${requestId}] Backfill failed for table ${tableId}:`, err)
+      await markJobFailed(tableId, jobId, message).catch(() => {})
+      void appendTableEvent({
+        kind: 'job',
+        type: 'backfill',
+        tableId,
+        jobId,
+        status: 'failed',
+        error: message,
+      })
+    }
+  }
+}
+
+/**
+ * Hybrid entry the schema-change flows call after adding/remapping workflow outputs. Small
+ * tables (≤ {@link BACKFILL_ASYNC_THRESHOLD_ROWS} completed runs) backfill inline-awaited, so the
+ * response returns with row data already consistent — identical to the historical behavior. Above
+ * the threshold, the work runs as a `table_jobs`-tracked background job (trigger.dev when
+ * enabled). The job slot is shared with import/delete; if another job holds it, the backfill is
+ * skipped with a warning — mirroring the long-standing "a failed backfill never fails the schema
+ * change" posture (the data stays backfillable).
+ */
+export async function maybeBackfillGroupOutputs(opts: {
+  table: TableDefinition
+  groupId: string
+  outputs: WorkflowGroupOutput[]
+  overwrite: boolean
+  requestId: string
+  actorUserId?: string | null
+}): Promise<void> {
+  const { table, groupId, outputs, overwrite, requestId, actorUserId } = opts
+  if (outputs.length === 0) return
+
+  const [{ count: completedCount }] = await db
+    .select({ count: count() })
+    .from(tableRowExecutions)
+    .where(
+      and(
+        eq(tableRowExecutions.tableId, table.id),
+        eq(tableRowExecutions.groupId, groupId),
+        eq(tableRowExecutions.status, 'completed')
+      )
+    )
+  const total = Number(completedCount)
+  if (total === 0) return
+
+  if (total <= BACKFILL_ASYNC_THRESHOLD_ROWS) {
+    // Inline: page without job machinery so memory stays bounded but the caller can await
+    // full consistency.
+    let afterRowId: string | undefined
+    while (true) {
+      const execs = await selectCompletedExecPage(table.id, groupId, afterRowId, BACKFILL_PAGE_SIZE)
+      if (execs.length === 0) break
+      afterRowId = execs[execs.length - 1].rowId
+      await processBackfillPage({ table, outputs, overwrite, execs, requestId, actorUserId })
+    }
+    return
+  }
+
+  const jobId = generateId()
+  const jobPayload: TableBackfillJobPayload = { groupId, outputs, overwrite }
+  const claimed = await markTableJobRunning(table.id, jobId, 'backfill', jobPayload)
+  if (!claimed) {
+    logger.warn(
+      `[${requestId}] Skipping backfill for table ${table.id} group ${groupId}: another job is running`
+    )
+    return
+  }
+
+  const payload: TableBackfillPayload = {
+    jobId,
+    tableId: table.id,
+    workspaceId: table.workspaceId,
+    groupId,
+    outputs,
+    overwrite,
+    actorUserId,
+  }
+  if (isTriggerDevEnabled) {
+    try {
+      const [{ tableBackfillTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
+        import('@/background/table-backfill'),
+        import('@trigger.dev/sdk'),
+        import('@/lib/core/async-jobs/region'),
+      ])
+      await tasks.trigger<typeof tableBackfillTask>('table-backfill', payload, {
+        tags: [`tableId:${table.id}`, `jobId:${jobId}`],
+        region: await resolveTriggerRegion(),
+      })
+    } catch (error) {
+      // Release the claim so a ghost `running` job doesn't block imports/deletes.
+      // Swallowed (warn only): a failed backfill never fails the schema change —
+      // the data stays backfillable.
+      const { releaseJobClaim } = await import('@/lib/table/jobs/service')
+      await releaseJobClaim(table.id, jobId).catch(() => {})
+      logger.warn(
+        `[${requestId}] Backfill dispatch failed for table ${table.id} group ${groupId}; skipping`,
+        { error: getErrorMessage(error) }
+      )
+    }
+  } else {
+    runDetached('table-backfill', () => runTableBackfill(payload))
+  }
+}

apps/sim/lib/table/billing.test.ts

@@ -0,0 +1,190 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockGetWorkspaceBilledAccountUserId,
+  mockGetHighestPrioritySubscription,
+  mockGetPlanTypeForLimits,
+  mockGetTablePlanLimits,
+} = vi.hoisted(() => ({
+  mockGetWorkspaceBilledAccountUserId: vi.fn(),
+  mockGetHighestPrioritySubscription: vi.fn(),
+  mockGetPlanTypeForLimits: vi.fn(),
+  mockGetTablePlanLimits: vi.fn(),
+}))
+
+vi.mock('@/lib/workspaces/utils', () => ({
+  getWorkspaceBilledAccountUserId: mockGetWorkspaceBilledAccountUserId,
+}))
+vi.mock('@/lib/billing/core/subscription', () => ({
+  getHighestPrioritySubscription: mockGetHighestPrioritySubscription,
+}))
+vi.mock('@/lib/billing/plan-helpers', () => ({
+  getPlanTypeForLimits: mockGetPlanTypeForLimits,
+}))
+vi.mock('@/lib/table/constants', () => ({
+  getTablePlanLimits: mockGetTablePlanLimits,
+}))
+
+import {
+  assertRowCapacity,
+  getMaxRowsPerTable,
+  getWorkspaceTableLimits,
+  notifyTableRowUsage,
+  TableRowLimitError,
+  wouldExceedRowLimit,
+} from '@/lib/table/billing'
+
+const LIMITS = {
+  free: { maxTables: 3, maxRowsPerTable: 1000 },
+  pro: { maxTables: 25, maxRowsPerTable: 5000 },
+  team: { maxTables: 100, maxRowsPerTable: 10000 },
+  enterprise: { maxTables: 10000, maxRowsPerTable: 1000000 },
+}
+
+// The limits cache is module-level and keyed by workspaceId; a fresh id per test
+// keeps one test's cached value from leaking into the next.
+let wsCounter = 0
+const nextWorkspaceId = () => `ws-${++wsCounter}`
+
+beforeEach(() => {
+  vi.clearAllMocks()
+  mockGetTablePlanLimits.mockReturnValue(LIMITS)
+  mockGetWorkspaceBilledAccountUserId.mockResolvedValue('billed-user')
+  mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro' })
+  mockGetPlanTypeForLimits.mockReturnValue('pro')
+})
+
+describe('getWorkspaceTableLimits', () => {
+  it('returns the limits for the workspace subscription plan', async () => {
+    expect(await getWorkspaceTableLimits(nextWorkspaceId())).toEqual(LIMITS.pro)
+  })
+
+  it('caches the resolved limits within the TTL', async () => {
+    const ws = nextWorkspaceId()
+    await getWorkspaceTableLimits(ws)
+    await getWorkspaceTableLimits(ws)
+    expect(mockGetWorkspaceBilledAccountUserId).toHaveBeenCalledTimes(1)
+  })
+
+  it('returns free-tier limits when the workspace has no billed account', async () => {
+    mockGetWorkspaceBilledAccountUserId.mockResolvedValueOnce(null)
+    expect(await getWorkspaceTableLimits(nextWorkspaceId())).toEqual(LIMITS.free)
+  })
+
+  it('falls back to free tier without caching when the lookup throws', async () => {
+    const ws = nextWorkspaceId()
+    mockGetWorkspaceBilledAccountUserId.mockRejectedValueOnce(new Error('db down'))
+    expect(await getWorkspaceTableLimits(ws)).toEqual(LIMITS.free)
+    // The fallback is never cached, so the next call re-attempts and resolves the real plan.
+    expect(await getWorkspaceTableLimits(ws)).toEqual(LIMITS.pro)
+  })
+
+  it('stays bounded under a burst of distinct all-fresh workspaces', async () => {
+    // Far more distinct workspaces than the cap, all within one TTL window. The Map
+    // must not grow without limit; eviction keeps it at/under the ceiling.
+    for (let i = 0; i < 6_000; i++) {
+      await getWorkspaceTableLimits(`burst-${i}`)
+    }
+    // Re-resolving an early (evicted) workspace must re-hit the billing lookup.
+    mockGetWorkspaceBilledAccountUserId.mockClear()
+    await getWorkspaceTableLimits('burst-0')
+    expect(mockGetWorkspaceBilledAccountUserId).toHaveBeenCalledTimes(1)
+  })
+})
+
+describe('getMaxRowsPerTable', () => {
+  it('returns the plan maxRowsPerTable', async () => {
+    expect(await getMaxRowsPerTable(nextWorkspaceId())).toBe(5000)
+  })
+})
+
+describe('wouldExceedRowLimit', () => {
+  it('is false under the limit and at the limit exactly', () => {
+    expect(wouldExceedRowLimit(1000, 10, 5)).toBe(false)
+    expect(wouldExceedRowLimit(1000, 999, 1)).toBe(false)
+  })
+
+  it('is true when the sum crosses the limit', () => {
+    expect(wouldExceedRowLimit(1000, 1000, 1)).toBe(true)
+  })
+
+  it('treats a negative limit as unlimited', () => {
+    expect(wouldExceedRowLimit(-1, 10_000_000, 1)).toBe(false)
+  })
+
+  it('treats a zero limit as no rows allowed', () => {
+    expect(wouldExceedRowLimit(0, 0, 1)).toBe(true)
+  })
+})
+
+describe('assertRowCapacity', () => {
+  it('returns the resolved limit when the write stays under it', async () => {
+    await expect(
+      assertRowCapacity({ workspaceId: nextWorkspaceId(), currentRowCount: 10, addedRows: 5 })
+    ).resolves.toBe(5000)
+  })
+
+  it('allows reaching the limit exactly and returns it', async () => {
+    await expect(
+      assertRowCapacity({ workspaceId: nextWorkspaceId(), currentRowCount: 4999, addedRows: 1 })
+    ).resolves.toBe(5000)
+  })
+
+  it('throws TableRowLimitError when the write would exceed the limit', async () => {
+    await expect(
+      assertRowCapacity({ workspaceId: nextWorkspaceId(), currentRowCount: 5000, addedRows: 1 })
+    ).rejects.toBeInstanceOf(TableRowLimitError)
+  })
+
+  it('names the plan limit in the error message', async () => {
+    await expect(
+      assertRowCapacity({ workspaceId: nextWorkspaceId(), currentRowCount: 5000, addedRows: 1 })
+    ).rejects.toThrow(/row limit \(5,000 rows\)/)
+  })
+
+  it('skips the check when the plan is unlimited (-1)', async () => {
+    mockGetTablePlanLimits.mockReturnValue({
+      ...LIMITS,
+      pro: { maxTables: 25, maxRowsPerTable: -1 },
+    })
+    await expect(
+      assertRowCapacity({
+        workspaceId: nextWorkspaceId(),
+        currentRowCount: 10_000_000,
+        addedRows: 1,
+      })
+    ).resolves.toBe(-1)
+  })
+})
+
+describe('notifyTableRowUsage — edge-crossing gate', () => {
+  beforeEach(() => mockGetWorkspaceBilledAccountUserId.mockClear())
+
+  it('fires when an insert crosses UP into the warn band (limit 5000)', () => {
+    notifyTableRowUsage({ workspaceId: 'ws', currentRowCount: 3990, addedRows: 20, limit: 5000 })
+    expect(mockGetWorkspaceBilledAccountUserId).toHaveBeenCalledTimes(1)
+  })
+
+  it('fires when an insert crosses UP into the reached band', () => {
+    notifyTableRowUsage({ workspaceId: 'ws', currentRowCount: 4990, addedRows: 20, limit: 5000 })
+    expect(mockGetWorkspaceBilledAccountUserId).toHaveBeenCalledTimes(1)
+  })
+
+  it('does NOT fire when already above the band (no crossing)', () => {
+    notifyTableRowUsage({ workspaceId: 'ws', currentRowCount: 4200, addedRows: 100, limit: 5000 })
+    expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled()
+  })
+
+  it('does NOT fire well below the band', () => {
+    notifyTableRowUsage({ workspaceId: 'ws', currentRowCount: 100, addedRows: 10, limit: 5000 })
+    expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled()
+  })
+
+  it('does NOT fire for unlimited plans', () => {
+    notifyTableRowUsage({ workspaceId: 'ws', currentRowCount: 0, addedRows: 10_000, limit: -1 })
+    expect(mockGetWorkspaceBilledAccountUserId).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/table/billing.ts

@@ -0,0 +1,258 @@
+/**
+ * Billing helpers for table feature limits.
+ *
+ * Uses workspace billing account to determine plan-based limits.
+ */
+
+import { createLogger } from '@sim/logger'
+import { maybeNotifyLimit } from '@/lib/billing/core/limit-notifications'
+import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
+import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
+import { getTablePlanLimits, type PlanName, type TablePlanLimits } from '@/lib/table/constants'
+import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
+
+const logger = createLogger('TableBilling')
+
+/** Warn band; an insert that crosses up into it (or up into 100%) triggers a notify. */
+const TABLE_WARN_PERCENT = 80
+const TABLE_REACHED_PERCENT = 100
+
+/** Whether adding rows pushed the count UP across `threshold` (pre below, post at/above). */
+function crossedUp(prePct: number, postPct: number, threshold: number): boolean {
+  return prePct < threshold && postPct >= threshold
+}
+
+/**
+ * Best-effort table row-limit email after an accepted insert. Resolves the
+ * workspace's billed account, then delegates scope resolution + dedup + send to
+ * {@link maybeNotifyLimit}. Never throws.
+ */
+async function maybeNotifyTableRowLimit(
+  workspaceId: string,
+  projectedRowCount: number,
+  limit: number
+): Promise<void> {
+  try {
+    const billedUserId = await getWorkspaceBilledAccountUserId(workspaceId)
+    if (!billedUserId) return
+
+    await maybeNotifyLimit({
+      category: 'tables',
+      billedUserId,
+      workspaceId,
+      currentUsage: projectedRowCount,
+      limit,
+      usageLabel: `${projectedRowCount.toLocaleString('en-US')} rows`,
+      limitLabel: `${limit.toLocaleString('en-US')} rows`,
+    })
+  } catch (error) {
+    logger.error('Error evaluating table row-limit notification:', error)
+  }
+}
+
+/**
+ * Fire-and-forget the table row-limit threshold email for an accepted insert.
+ * Edge-triggered: it only does any billing work when the write CROSSES UP into
+ * the warn (80%) or reached (100%) band — not on every near-limit insert — so a
+ * table sitting at e.g. 90% being inserted into pays nothing until it crosses
+ * 100%. Shared by every insert path ({@link assertRowCapacity} and the
+ * transactional upsert/import branches that check capacity with
+ * {@link wouldExceedRowLimit} instead). Pass the pre-insert `currentRowCount` and
+ * `addedRows` so both the pre and post percentages are known.
+ *
+ * Tables warn once per threshold and do not re-arm: a table that hit a threshold
+ * then dropped (via deletes, which have no notify hook) won't re-warn on a later
+ * climb. Deliberate trade-off — re-arm is storage-only (its single decrement
+ * hook) to avoid per-delete billing-table reads. The atomic claim still dedups
+ * concurrent crossings (no race).
+ */
+export function notifyTableRowUsage(params: {
+  workspaceId: string
+  currentRowCount: number
+  addedRows: number
+  limit: number
+}): void {
+  if (params.limit <= 0) return
+  const prePct = (params.currentRowCount / params.limit) * 100
+  const postPct = ((params.currentRowCount + params.addedRows) / params.limit) * 100
+  if (
+    crossedUp(prePct, postPct, TABLE_WARN_PERCENT) ||
+    crossedUp(prePct, postPct, TABLE_REACHED_PERCENT)
+  ) {
+    void maybeNotifyTableRowLimit(
+      params.workspaceId,
+      params.currentRowCount + params.addedRows,
+      params.limit
+    )
+  }
+}
+
+/**
+ * Plan lookups hit billing + subscription tables (2-3 queries). Row-limit checks
+ * run on every insert, so a short TTL keeps the hot path off the DB. Plan changes
+ * are rare and enforcement is best-effort, so brief staleness is acceptable.
+ */
+const LIMITS_CACHE_TTL_MS = 30_000
+/** Hard ceiling on cached workspaces; a sweep drops expired entries before this is exceeded so the Map can't grow unbounded. */
+const LIMITS_CACHE_MAX_ENTRIES = 5_000
+const limitsCache = new Map<string, { limits: TablePlanLimits; expiresAt: number }>()
+
+/**
+ * Gets the table limits for a workspace based on its billing plan.
+ *
+ * Uses the workspace's billed account user to determine the subscription plan,
+ * then returns the corresponding table limits. Resolved limits are cached for
+ * {@link LIMITS_CACHE_TTL_MS}; the free-tier error fallback is never cached.
+ *
+ * @param workspaceId - The workspace ID to get limits for
+ * @returns Table limits based on the workspace's billing plan
+ */
+export async function getWorkspaceTableLimits(workspaceId: string): Promise<TablePlanLimits> {
+  const cached = limitsCache.get(workspaceId)
+  if (cached) {
+    if (cached.expiresAt > Date.now()) return cached.limits
+    limitsCache.delete(workspaceId)
+  }
+
+  const planLimits = getTablePlanLimits()
+
+  try {
+    const billedAccountUserId = await getWorkspaceBilledAccountUserId(workspaceId)
+
+    if (!billedAccountUserId) {
+      logger.warn('No billed account found for workspace, using free tier limits', { workspaceId })
+      cacheLimits(workspaceId, planLimits.free)
+      return planLimits.free
+    }
+
+    const subscription = await getHighestPrioritySubscription(billedAccountUserId)
+    const planName = getPlanTypeForLimits(subscription?.plan) as PlanName
+
+    const limits = planLimits[planName] ?? planLimits.free
+
+    logger.info('Retrieved workspace table limits', {
+      workspaceId,
+      billedAccountUserId,
+      planName,
+      limits,
+    })
+
+    cacheLimits(workspaceId, limits)
+    return limits
+  } catch (error) {
+    logger.error('Error getting workspace table limits, falling back to free tier', {
+      workspaceId,
+      error,
+    })
+    return planLimits.free
+  }
+}
+
+function cacheLimits(workspaceId: string, limits: TablePlanLimits): void {
+  // Keep the Map bounded for a new key: sweep expired entries, then (if a burst of
+  // all-fresh entries still sits at the cap) evict oldest-inserted ones. Map iteration
+  // is insertion order, so the first key is the oldest. Net: size never exceeds the cap.
+  if (limitsCache.size >= LIMITS_CACHE_MAX_ENTRIES && !limitsCache.has(workspaceId)) {
+    const now = Date.now()
+    for (const [key, entry] of limitsCache) {
+      if (entry.expiresAt <= now) limitsCache.delete(key)
+    }
+    while (limitsCache.size >= LIMITS_CACHE_MAX_ENTRIES) {
+      const oldest = limitsCache.keys().next().value
+      if (oldest === undefined) break
+      limitsCache.delete(oldest)
+    }
+  }
+  limitsCache.set(workspaceId, { limits, expiresAt: Date.now() + LIMITS_CACHE_TTL_MS })
+}
+
+/**
+ * Thrown by {@link assertRowCapacity} when a write would exceed the workspace's
+ * current plan row limit. The message includes the lowercase `row limit` token so
+ * `rowWriteErrorResponse` maps it to a 400 toast carrying the real reason.
+ */
+export class TableRowLimitError extends Error {
+  constructor(readonly limit: number) {
+    super(
+      `This table has reached its row limit (${limit.toLocaleString('en-US')} rows) on your current plan.`
+    )
+    this.name = 'TableRowLimitError'
+  }
+}
+
+/**
+ * Whether adding `addedRows` to `currentRowCount` would cross `limit`. A negative
+ * limit means unlimited. Single source of truth for the comparison so callers that
+ * fetch the limit themselves (e.g. inside a transaction, or to build a custom
+ * message) stay consistent with {@link assertRowCapacity}.
+ */
+export function wouldExceedRowLimit(
+  limit: number,
+  currentRowCount: number,
+  addedRows: number
+): boolean {
+  return limit >= 0 && currentRowCount + addedRows > limit
+}
+
+/**
+ * Best-effort capacity check against the workspace's CURRENT plan limit.
+ *
+ * Not transactional: reads the (trigger-maintained, possibly slightly stale) row
+ * count and the cached plan limit outside any lock, so concurrent writers may
+ * overshoot by a small amount. It rejects once the count is at/over the limit, so
+ * a table can't run away past its plan.
+ *
+ * Resolve the limit OUTSIDE any open transaction — `getMaxRowsPerTable` may hit the
+ * billing/subscription tables on the global pool, and doing that while holding a tx
+ * connection (and locks) risks pool starvation. Callers already inside a tx should
+ * fetch the limit up front and use {@link wouldExceedRowLimit} instead.
+ *
+ * Pure check (no side effects): returns the resolved limit so callers can fire
+ * {@link notifyTableRowUsage} AFTER their insert commits — a pre-commit notify
+ * would email (and burn the dedup claim) for a write that later rolls back.
+ *
+ * @returns the resolved plan row limit (-1 for unlimited)
+ * @throws {TableRowLimitError} if `currentRowCount + addedRows` exceeds the limit
+ */
+export async function assertRowCapacity(params: {
+  workspaceId: string
+  currentRowCount: number
+  addedRows: number
+}): Promise<number> {
+  const limit = await getMaxRowsPerTable(params.workspaceId)
+  if (wouldExceedRowLimit(limit, params.currentRowCount, params.addedRows)) {
+    throw new TableRowLimitError(limit)
+  }
+  return limit
+}
+
+/**
+ * Checks if a workspace can create more tables based on its plan limits.
+ *
+ * @param workspaceId - The workspace ID to check
+ * @param currentTableCount - The current number of tables in the workspace
+ * @returns Object with canCreate boolean and limit info
+ */
+async function canCreateTable(
+  workspaceId: string,
+  currentTableCount: number
+): Promise<{ canCreate: boolean; maxTables: number; currentCount: number }> {
+  const limits = await getWorkspaceTableLimits(workspaceId)
+
+  return {
+    canCreate: currentTableCount < limits.maxTables,
+    maxTables: limits.maxTables,
+    currentCount: currentTableCount,
+  }
+}
+
+/**
+ * Gets the maximum rows allowed per table for a workspace based on its plan.
+ *
+ * @param workspaceId - The workspace ID
+ * @returns Maximum rows per table (-1 for unlimited)
+ */
+export async function getMaxRowsPerTable(workspaceId: string): Promise<number> {
+  const limits = await getWorkspaceTableLimits(workspaceId)
+  return limits.maxRowsPerTable
+}

apps/sim/lib/table/cascade-lock.ts

@@ -0,0 +1,61 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { acquireLock, extendLock, releaseLock } from '@/lib/core/config/redis'
+
+const logger = createLogger('TableCascadeLock')
+
+/** Lock TTL. Crashed pods release within this many seconds. */
+const LOCK_TTL_SECONDS = 30
+/** Heartbeat cadence. ~3x within TTL — tolerates two missed beats. */
+const HEARTBEAT_INTERVAL_MS = 10_000
+
+/** Single source of truth for the cascade-lock key shape. The lock arbitrates
+ *  ownership of a row's full workflow-group cascade — only the owner advances
+ *  the row through its eligible groups. */
+export function cascadeLockKey(tableId: string, rowId: string): string {
+  return `table:cascade:${tableId}:${rowId}`
+}
+
+/**
+ * Run `fn` while holding the row's cascade lock, with a heartbeat extending
+ * the TTL every 10s so a crashed pod releases the lock in ≤30s. `ownerId`
+ * must be unique per holder (typically the cell-task's `executionId`) so
+ * `releaseLock` does compare-and-delete and can't accidentally drop another
+ * owner's lock.
+ *
+ * Returns `'acquired'` after `fn` resolves, or `'contended'` if another
+ * task already holds the lock — `fn` is NOT invoked in that case. The
+ * caller decides what to do on contention (cell-task bails; resume worker
+ * still writes the resumed-group's terminal state but skips the cascade).
+ *
+ * NOTE: when Redis is unavailable, `acquireLock` returns `true` as a
+ * single-replica fallback — concurrent cell-tasks would all "acquire" and
+ * run in parallel. The cell-write SQL guard mitigates double-writes but
+ * doesn't prevent duplicate workflow executions.
+ */
+export async function withCascadeLock<T>(
+  tableId: string,
+  rowId: string,
+  ownerId: string,
+  fn: () => Promise<T>
+): Promise<{ status: 'acquired'; result: T } | { status: 'contended' }> {
+  const key = cascadeLockKey(tableId, rowId)
+  const acquired = await acquireLock(key, ownerId, LOCK_TTL_SECONDS)
+  if (!acquired) return { status: 'contended' }
+
+  const heartbeat = setInterval(() => {
+    extendLock(key, ownerId, LOCK_TTL_SECONDS).catch((err) => {
+      logger.warn(`Heartbeat refresh failed for ${key}`, { error: toError(err).message })
+    })
+  }, HEARTBEAT_INTERVAL_MS)
+
+  try {
+    const result = await fn()
+    return { status: 'acquired', result }
+  } finally {
+    clearInterval(heartbeat)
+    await releaseLock(key, ownerId).catch((err) => {
+      logger.warn(`Lock release failed for ${key}`, { error: toError(err).message })
+    })
+  }
+}

apps/sim/lib/table/cell-write.ts

@@ -0,0 +1,207 @@
+/**
+ * Shared cell-write primitives for workflow-group execution paths.
+ *
+ * Both the scheduler (`runWorkflowGroupCell`) and the cell task body
+ * (`executeWorkflowGroupCellJob`) need to write `data` patches + `executions`
+ * patches together while honoring the `cancelled` state written by
+ * `cancelWorkflowGroupRuns` — without the guard, a stop click that lands
+ * mid-enqueue or mid-run would get clobbered by the in-flight code path's
+ * next write.
+ */
+
+import { createLogger } from '@sim/logger'
+import { isExecCancelled } from '@/lib/table/deps'
+import { appendTableEvent } from '@/lib/table/events'
+import type { RowData, RowExecutionMetadata, RowExecutions, WorkflowGroup } from '@/lib/table/types'
+
+const logger = createLogger('WorkflowCellWrite')
+
+export interface WriteWorkflowGroupContext {
+  tableId: string
+  rowId: string
+  workspaceId: string
+  groupId: string
+  executionId: string
+  /** Used as the `requestId` passed to `updateRow` for log correlation. */
+  requestId?: string
+}
+
+export interface WriteWorkflowGroupStatePayload {
+  /** Plain primitives to merge into `row.data`. Empty patch is fine. */
+  dataPatch?: RowData
+  /** New execution state for `executions[groupId]`. */
+  executionState: RowExecutionMetadata
+}
+
+/**
+ * Writes the row unless `cancelWorkflowGroupRuns` has already authoritatively
+ * written `cancelled` for this run. Returns `'skipped'` so the caller can
+ * short-circuit any follow-up writes / job dispatch.
+ */
+export async function writeWorkflowGroupState(
+  ctx: WriteWorkflowGroupContext,
+  payload: WriteWorkflowGroupStatePayload
+): Promise<'wrote' | 'skipped'> {
+  const { tableId, rowId, workspaceId, groupId, executionId } = ctx
+  const requestId = ctx.requestId ?? `wfgrp-${executionId}`
+  const { getTableById } = await import('@/lib/table/service')
+  const { getRowById, updateRow } = await import('@/lib/table/rows/service')
+
+  const table = await getTableById(tableId)
+  if (!table) {
+    logger.warn(`Table ${tableId} vanished before group state write`)
+    return 'wrote'
+  }
+  const row = await getRowById(tableId, rowId, workspaceId)
+  if (!row) {
+    logger.warn(`Row ${rowId} vanished before group state write`)
+    return 'wrote'
+  }
+  const current = row.executions?.[groupId] as RowExecutionMetadata | undefined
+  // Stale-worker guard: only blocks writes FROM an old worker (status =
+  // running / completed / error / pending). A `queued` stamp from the
+  // scheduler can claim the cell for a brand-new run — that's the new
+  // authority. Same for `cancelled` (always authoritative, written by stop).
+  const isCancelStamp = payload.executionState.status === 'cancelled'
+  const isQueuedStamp = payload.executionState.status === 'queued'
+  const isNewQueuedStamp = isQueuedStamp && current?.executionId !== executionId
+  const bypassStaleWorker = isNewQueuedStamp || isCancelStamp
+  if (!bypassStaleWorker && current && current.executionId && current.executionId !== executionId) {
+    logger.info(
+      `Skipping group write — stale worker (table=${tableId} row=${rowId} group=${groupId} mine=${executionId} active=${current.executionId})`
+    )
+    return 'skipped'
+  }
+  // A late `queued` stamp for the SAME run that's already moved past queued
+  // (worker called markWorkflowGroupPickedUp before our parallel stamp landed)
+  // must NOT overwrite the further-along state. Without this, a cell can show
+  // "queued" forever while the worker is actually running.
+  if (isQueuedStamp && current?.executionId === executionId && current.status !== 'pending') {
+    logger.info(
+      `Skipping queued stamp — same run already at status=${current.status} (table=${tableId} row=${rowId} group=${groupId} executionId=${executionId})`
+    )
+    return 'skipped'
+  }
+  // A `cancelled` cell rejects any worker write regardless of executionId — a
+  // stop click can only stamp the dispatcher pre-stamp's executionId (often
+  // null), so an executionId-matched guard would let the worker that later
+  // claims the cell with its real id resurrect it. `bypassStaleWorker` (a fresh
+  // `queued` claim from a new dispatch, or the authoritative cancel write
+  // itself) still passes; manual re-runs clear the tombstone before stamping.
+  if (!bypassStaleWorker && isExecCancelled(current)) {
+    logger.info(
+      `Skipping group write — cancelled (table=${tableId} row=${rowId} group=${groupId} executionId=${executionId})`
+    )
+    return 'skipped'
+  }
+  // Skip writing `cancelled` state with the guard — that's an authoritative
+  // write from `cancelWorkflowGroupRuns` and must always land. New `queued`
+  // stamps from the scheduler also bypass — they ARE the new authority. Cell-
+  // task writes (running/completed/error) get the SQL guard so an in-flight
+  // partial can't clobber a stop click or a newer run that already committed.
+  const cancellationGuard = bypassStaleWorker ? undefined : { groupId, executionId }
+  const result = await updateRow(
+    {
+      tableId,
+      rowId,
+      data: payload.dataPatch ?? {},
+      workspaceId,
+      executionsPatch: { [groupId]: payload.executionState },
+      cancellationGuard,
+    },
+    table,
+    requestId
+  )
+  if (result === null) {
+    logger.info(
+      `Skipping group write — SQL guard saw cancelled (table=${tableId} row=${rowId} group=${groupId} executionId=${executionId})`
+    )
+    return 'skipped'
+  }
+
+  const dataPatch = payload.dataPatch
+  const hasOutputs = dataPatch && Object.keys(dataPatch).length > 0
+  const runningBlockIds = payload.executionState.runningBlockIds
+  const blockErrors = payload.executionState.blockErrors
+  void appendTableEvent({
+    kind: 'cell',
+    tableId,
+    rowId,
+    groupId,
+    status: payload.executionState.status,
+    executionId: payload.executionState.executionId ?? null,
+    jobId: payload.executionState.jobId ?? null,
+    error: payload.executionState.error ?? null,
+    ...(hasOutputs ? { outputs: dataPatch } : {}),
+    ...(runningBlockIds && runningBlockIds.length > 0 ? { runningBlockIds } : {}),
+    ...(blockErrors && Object.keys(blockErrors).length > 0 ? { blockErrors } : {}),
+  })
+
+  return 'wrote'
+}
+
+/**
+ * Flips `queued` → `running` to signal the cell task body has actually been
+ * picked up by a worker. The renderer uses the `queued` vs `running` distinction
+ * to label cells "Queued" vs "Waiting" (worker started, this block hasn't run
+ * yet) — without this marker we couldn't tell if a row was sitting in the
+ * trigger.dev queue or actively executing.
+ */
+export async function markWorkflowGroupPickedUp(
+  ctx: WriteWorkflowGroupContext,
+  prev: Pick<RowExecutionMetadata, 'workflowId' | 'jobId'>
+): Promise<'wrote' | 'skipped'> {
+  return writeWorkflowGroupState(ctx, {
+    executionState: {
+      status: 'running',
+      executionId: ctx.executionId,
+      jobId: prev.jobId,
+      workflowId: prev.workflowId,
+      error: null,
+    },
+  })
+}
+
+/** Builds the canonical `cancelled` execution state used by every cancel path.
+ *  Preserves `blockErrors` from the prior state so errored cells keep
+ *  rendering Error after a stop click — only cells that hadn't yet produced
+ *  a value or an error should flip to "Cancelled". `cancelledAt` is the
+ *  tombstone the dispatcher reads to skip re-runs of cells the user killed
+ *  mid-cascade. */
+export function buildCancelledExecution(
+  prev: Pick<RowExecutionMetadata, 'executionId' | 'workflowId' | 'blockErrors'>
+): RowExecutionMetadata {
+  return {
+    status: 'cancelled',
+    executionId: prev.executionId ?? null,
+    jobId: null,
+    workflowId: prev.workflowId,
+    error: 'Cancelled',
+    cancelledAt: new Date().toISOString(),
+    ...(prev.blockErrors ? { blockErrors: prev.blockErrors } : {}),
+  }
+}
+
+/**
+ * Maps a group's `outputs[]` to a `blockId → Array<{path, columnName}>` map.
+ * The cell task uses this to fan a single block-complete event into N column
+ * writes.
+ */
+export function buildOutputsByBlockId(
+  group: WorkflowGroup
+): Map<string, Array<{ path: string; columnName: string }>> {
+  const map = new Map<string, Array<{ path: string; columnName: string }>>()
+  for (const out of group.outputs) {
+    const list = map.get(out.blockId) ?? []
+    list.push({ path: out.path, columnName: out.columnName })
+    map.set(out.blockId, list)
+  }
+  return map
+}
+
+/** Type-narrowing helper used by readers that can't assume `executions` is set. */
+export function readExecutions(
+  row: { executions?: RowExecutions } | null | undefined
+): RowExecutions {
+  return row?.executions ?? {}
+}

apps/sim/lib/table/column-keys.ts

@@ -0,0 +1,187 @@
+/**
+ * Column id ↔ name translation helpers.
+ *
+ * Stored row data (`user_table_rows.data`), table metadata, workflow-group
+ * refs, and filter/sort all key on a column's stable **id**. `name` is a
+ * display label that changes on rename. The two name-translating boundaries
+ * (public v1 API, mothership tool) and CSV convert between the two with the
+ * map builders here.
+ */
+
+import { generateId } from '@sim/utils/id'
+import type {
+  ColumnDefinition,
+  Filter,
+  RowData,
+  Sort,
+  TableSchema,
+  WorkflowGroup,
+} from '@/lib/table/types'
+
+/**
+ * Resolves a column's stable storage key. Falls back to `name` for legacy
+ * columns that predate the id backfill — those rows were written keyed by name,
+ * which is exactly the key the column still uses, so the fallback is correct.
+ */
+export function getColumnId(col: Pick<ColumnDefinition, 'id' | 'name'>): string {
+  return col.id ?? col.name
+}
+
+/**
+ * Mints a fresh column id. Generated ids are opaque (`col_<uuid>`) and
+ * deliberately distinct from display names so renames never disturb them. The
+ * `col_` prefix is required: the id is validated against `NAME_PATTERN` (it's a
+ * JSONB key and a filter/sort field) which must start with a letter/underscore,
+ * and a bare UUID can start with a digit. Dashes are stripped for the same
+ * reason. A v4 UUID's 122 random bits make a collision within a table's columns
+ * effectively impossible, so no uniqueness check is needed.
+ */
+export function generateColumnId(): string {
+  return `col_${generateId().replace(/-/g, '')}`
+}
+
+/**
+ * Matches a column against a reference that may be a stable id (first-party
+ * callers) or a display name (legacy / mothership / public API). Id match is
+ * exact; name match is case-insensitive (names are unique case-insensitively per
+ * schema validation). The single predicate behind every column-op resolver — use
+ * with `.find` / `.findIndex` so id-or-name resolution can't drift between sites.
+ */
+export function columnMatchesRef(col: ColumnDefinition, ref: string): boolean {
+  return getColumnId(col) === ref || col.name.toLowerCase() === ref.toLowerCase()
+}
+
+/**
+ * Returns a schema copy with a generated id stamped onto every column that
+ * lacks one, remapping any workflow-group refs that still hold a column **name**
+ * to the assigned id. Used at creation time (`createTable`) so a freshly created
+ * table is fully id-keyed from its first row write. Idempotent for columns that
+ * already carry an id.
+ */
+export function withGeneratedColumnIds(schema: TableSchema): TableSchema {
+  const idByName = new Map<string, string>()
+  const columns = schema.columns.map((col) => {
+    if (col.id) {
+      idByName.set(col.name, col.id)
+      return col
+    }
+    const id = generateColumnId()
+    idByName.set(col.name, id)
+    return { ...col, id }
+  })
+
+  const remap = (ref: string) => idByName.get(ref) ?? ref
+  const workflowGroups = schema.workflowGroups?.map((group) => ({
+    ...group,
+    outputs: group.outputs.map((o) => ({ ...o, columnName: remap(o.columnName) })),
+    ...(group.dependencies?.columns
+      ? { dependencies: { columns: group.dependencies.columns.map(remap) } }
+      : {}),
+    ...(group.inputMappings
+      ? {
+          inputMappings: group.inputMappings.map((m) => ({
+            ...m,
+            columnName: remap(m.columnName),
+          })),
+        }
+      : {}),
+  }))
+
+  return { ...schema, columns, ...(workflowGroups ? { workflowGroups } : {}) }
+}
+
+/**
+ * Rewrites a workflow group's column references (output `columnName`,
+ * `dependencies.columns`, `inputMapping.columnName`) from display name to stable
+ * id using `idByName`. A ref that is already an id (not a known column name) is
+ * left as-is, so this is safe whether the caller authored refs by name
+ * (mothership) or by id (first-party UI).
+ */
+export function remapGroupColumnRefs(
+  group: WorkflowGroup,
+  idByName: ReadonlyMap<string, string>
+): WorkflowGroup {
+  const remap = (ref: string) => idByName.get(ref) ?? ref
+  return {
+    ...group,
+    outputs: group.outputs.map((o) => ({ ...o, columnName: remap(o.columnName) })),
+    ...(group.dependencies?.columns
+      ? { dependencies: { columns: group.dependencies.columns.map(remap) } }
+      : {}),
+    ...(group.inputMappings
+      ? {
+          inputMappings: group.inputMappings.map((m) => ({
+            ...m,
+            columnName: remap(m.columnName),
+          })),
+        }
+      : {}),
+  }
+}
+
+/** `name → id` for translating inbound wire data (v1 / mothership / CSV import). */
+export function buildIdByName(schema: TableSchema): Map<string, string> {
+  const map = new Map<string, string>()
+  for (const col of schema.columns) map.set(col.name, getColumnId(col))
+  return map
+}
+
+/** `id → name` for translating outbound wire data (v1 / mothership / CSV export). */
+export function buildNameById(schema: TableSchema): Map<string, string> {
+  const map = new Map<string, string>()
+  for (const col of schema.columns) map.set(getColumnId(col), col.name)
+  return map
+}
+
+/**
+ * Remaps a wire row keyed by column **name** to the stored **id** keying. Used
+ * at the name-translating boundaries on the way in. Keys not matching a known
+ * column are dropped (validation has already run against the schema).
+ */
+export function rowDataNameToId(data: RowData, idByName: Map<string, string>): RowData {
+  const out: RowData = {}
+  for (const [name, value] of Object.entries(data)) {
+    const id = idByName.get(name)
+    if (id !== undefined) out[id] = value
+  }
+  return out
+}
+
+/**
+ * Translates a filter's field names → column ids (recursing into `$or`/`$and`).
+ * Fields with no matching column (e.g. `createdAt`) pass through unchanged. Used
+ * at the name-translating boundaries before handing a filter to the query layer.
+ */
+export function filterNamesToIds(filter: Filter, idByName: ReadonlyMap<string, string>): Filter {
+  const out: Filter = {}
+  for (const [key, value] of Object.entries(filter)) {
+    if ((key === '$or' || key === '$and') && Array.isArray(value)) {
+      out[key] = (value as Filter[]).map((f) => filterNamesToIds(f, idByName))
+    } else {
+      out[idByName.get(key) ?? key] = value
+    }
+  }
+  return out
+}
+
+/** Translates a sort's field names → column ids. Unknown fields pass through. */
+export function sortNamesToIds(sort: Sort, idByName: ReadonlyMap<string, string>): Sort {
+  const out: Sort = {}
+  for (const [field, dir] of Object.entries(sort)) out[idByName.get(field) ?? field] = dir
+  return out
+}
+
+/**
+ * Remaps a stored row keyed by column **id** back to **name** keying for the
+ * wire. Used at the name-translating boundaries on the way out. Ids with no
+ * current column (e.g. a column deleted by a not-yet-finished background strip)
+ * are dropped, so orphaned keys never surface.
+ */
+export function rowDataIdToName(data: RowData, nameById: Map<string, string>): RowData {
+  const out: RowData = {}
+  for (const [id, value] of Object.entries(data)) {
+    const name = nameById.get(id)
+    if (name !== undefined) out[name] = value
+  }
+  return out
+}

apps/sim/lib/table/column-naming.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared column-naming helpers used by every path that auto-derives a
+ * column name + type from a workflow block output: the table column-sidebar
+ * UI, and the Copilot/Mothership `add_workflow_group` op. Keeping one
+ * implementation means the AI's auto-named columns match what a user would
+ * get from the sidebar.
+ */
+
+import type { ColumnDefinition } from '@/lib/table/types'
+
+/**
+ * Slugifies a string into a `NAME_PATTERN`-safe column name. Lowercase,
+ * non-alphanum runs collapse to `_`, leading digits get a `c_` prefix, empty
+ * results fall back to `output`.
+ */
+export function slugifyColumnName(value: string): string {
+  let slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+  if (!slug) slug = 'output'
+  if (/^[0-9]/.test(slug)) slug = `c_${slug}`
+  return slug
+}
+
+/**
+ * Pick a non-colliding column name for a block-output `path`. Uses the bare
+ * path slug; on collision, appends `_0`, `_1`, …
+ */
+export function deriveOutputColumnName(path: string, taken: Set<string>): string {
+  const base = slugifyColumnName(path)
+  if (!taken.has(base)) return base
+  for (let i = 0; i < 1000; i++) {
+    const candidate = `${base}_${i}`
+    if (!taken.has(candidate)) return candidate
+  }
+  return `${base}_${Date.now()}`
+}
+
+/**
+ * Map a block-output leaf type onto a table column type. Block schemas use
+ * a superset (`array`, `object`, etc.); anything outside the column-type
+ * union falls back to `json`, the most permissive shape that still validates.
+ */
+export function columnTypeForLeaf(leafType: string | undefined): ColumnDefinition['type'] {
+  switch (leafType) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+    case 'date':
+    case 'json':
+      return leafType
+    default:
+      return 'json'
+  }
+}

apps/sim/lib/table/columns/service.ts

@@ -0,0 +1,668 @@
+/**
+ * Column and schema-management service for user tables.
+ *
+ * Standalone column-mutation operations (add, rename, delete, type change,
+ * constraint change) extracted from the table service. Each acquires the
+ * table's advisory lock via {@link withLockedTable} from `@/lib/table/service`.
+ *
+ * Use this for: workflow executor, background jobs, testing business logic.
+ * Use API routes for: HTTP requests, frontend clients.
+ */
+
+import { db } from '@sim/db'
+import { userTableDefinitions, userTableRows } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { and, count, eq, sql } from 'drizzle-orm'
+import { columnMatchesRef, generateColumnId, getColumnId } from '@/lib/table/column-keys'
+import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants'
+import { stripGroupExecutions } from '@/lib/table/rows/executions'
+import { withLockedTable } from '@/lib/table/service'
+import { scaledStatementTimeoutMs, setTableTxTimeouts } from '@/lib/table/tx'
+import type {
+  DeleteColumnData,
+  RenameColumnData,
+  RowData,
+  TableDefinition,
+  TableMetadata,
+  TableSchema,
+  UpdateColumnConstraintsData,
+  UpdateColumnTypeData,
+} from '@/lib/table/types'
+import { assertValidSchema, stripGroupDeps } from '@/lib/table/workflow-columns'
+
+const logger = createLogger('TableColumnService')
+
+/**
+ * Adds a column to an existing table's schema.
+ *
+ * @param tableId - Table ID to update
+ * @param column - Column definition to add
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if table not found or column name already exists
+ */
+export async function addTableColumn(
+  tableId: string,
+  column: {
+    id?: string
+    name: string
+    type: string
+    required?: boolean
+    unique?: boolean
+    position?: number
+  },
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(tableId, async (table, trx) => {
+    if (!NAME_PATTERN.test(column.name)) {
+      throw new Error(
+        `Invalid column name "${column.name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+      )
+    }
+
+    if (column.name.length > TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) {
+      throw new Error(
+        `Column name exceeds maximum length (${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters)`
+      )
+    }
+
+    if (!COLUMN_TYPES.includes(column.type as (typeof COLUMN_TYPES)[number])) {
+      throw new Error(
+        `Invalid column type "${column.type}". Must be one of: ${COLUMN_TYPES.join(', ')}`
+      )
+    }
+
+    const schema = table.schema
+    if (schema.columns.some((c) => c.name.toLowerCase() === column.name.toLowerCase())) {
+      throw new Error(`Column "${column.name}" already exists`)
+    }
+
+    if (schema.columns.length >= TABLE_LIMITS.MAX_COLUMNS_PER_TABLE) {
+      throw new Error(
+        `Table has reached maximum column limit (${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE})`
+      )
+    }
+
+    const newColumn: TableSchema['columns'][number] = {
+      // Honor a caller-provided id (undo of a delete reuses the original id);
+      // otherwise mint a fresh one.
+      id: column.id ?? generateColumnId(),
+      name: column.name,
+      type: column.type as TableSchema['columns'][number]['type'],
+      required: column.required ?? false,
+      unique: column.unique ?? false,
+    }
+    const newColumnId = getColumnId(newColumn)
+
+    const columns = [...schema.columns]
+    if (column.position !== undefined && column.position >= 0 && column.position < columns.length) {
+      columns.splice(column.position, 0, newColumn)
+    } else {
+      columns.push(newColumn)
+    }
+
+    const updatedSchema: TableSchema = { ...schema, columns }
+
+    // Keep `metadata.columnOrder` (a list of column ids) in sync: splicing the
+    // new column's id at the same index we used in `columns` keeps display
+    // ordering aligned with the user's intent for `position`-based inserts.
+    const existingOrder = table.metadata?.columnOrder
+    let updatedMetadata = table.metadata
+    if (existingOrder && existingOrder.length > 0 && !existingOrder.includes(newColumnId)) {
+      let insertIdx = existingOrder.length
+      if (column.position !== undefined && column.position >= 0) {
+        // Anchor on the column previously at `position` — that column shifted
+        // right by one in `columns`, so the new id slots in at its old spot.
+        const anchor = schema.columns[column.position]
+        if (anchor) {
+          const anchorIdx = existingOrder.indexOf(getColumnId(anchor))
+          if (anchorIdx !== -1) insertIdx = anchorIdx
+        }
+      }
+      const nextOrder = [...existingOrder]
+      nextOrder.splice(insertIdx, 0, newColumnId)
+      updatedMetadata = { ...table.metadata, columnOrder: nextOrder }
+    }
+
+    assertValidSchema(updatedSchema, updatedMetadata?.columnOrder)
+
+    const now = new Date()
+
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, tableId))
+
+    logger.info(`[${requestId}] Added column "${column.name}" to table ${tableId}`)
+
+    return {
+      ...table,
+      schema: updatedSchema,
+      metadata: updatedMetadata,
+      updatedAt: now,
+    }
+  })
+}
+
+/**
+ * Renames a column in a table's schema and updates all row data keys.
+ *
+ * @param data - Rename column data
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if table not found, column not found, or new name conflicts
+ */
+export async function renameColumn(
+  data: RenameColumnData,
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(data.tableId, async (table, trx) => {
+    if (!NAME_PATTERN.test(data.newName)) {
+      throw new Error(
+        `Invalid column name "${data.newName}". Column names must start with a letter or underscore, followed by alphanumeric characters or underscores.`
+      )
+    }
+
+    if (data.newName.length > TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) {
+      throw new Error(
+        `Column name exceeds maximum length (${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters)`
+      )
+    }
+
+    const schema = table.schema
+    const columnIndex = schema.columns.findIndex((c) => columnMatchesRef(c, data.oldName))
+    if (columnIndex === -1) {
+      throw new Error(`Column "${data.oldName}" not found`)
+    }
+
+    if (
+      schema.columns.some(
+        (c, i) => i !== columnIndex && c.name.toLowerCase() === data.newName.toLowerCase()
+      )
+    ) {
+      throw new Error(`Column "${data.newName}" already exists`)
+    }
+
+    const targetColumn = schema.columns[columnIndex]
+    const actualOldName = targetColumn.name
+
+    // Rename is metadata-only: stored rows, metadata, and workflow-group refs all
+    // key on the column's stable id, which a rename never changes — so this is a
+    // pure schema write, no per-row JSONB rewrite or group/metadata cascade.
+    // Stamp the current storage key as the id (for any not-yet-backfilled column)
+    // so existing rows stay reachable as the display name changes.
+    const columnId = targetColumn.id ?? actualOldName
+    const updatedColumns = schema.columns.map((c, i) =>
+      i === columnIndex ? { ...c, id: columnId, name: data.newName } : c
+    )
+    const updatedSchema: TableSchema = { ...schema, columns: updatedColumns }
+    assertValidSchema(updatedSchema, table.metadata?.columnOrder)
+
+    const now = new Date()
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    logger.info(
+      `[${requestId}] Renamed column "${actualOldName}" to "${data.newName}" in table ${data.tableId}`
+    )
+    return { ...table, schema: updatedSchema, updatedAt: now }
+  })
+}
+
+/** Removes the given column-id keys from a metadata blob (widths/order/pinned). */
+function stripColumnIdsFromMetadata(
+  metadata: TableMetadata | null,
+  ids: ReadonlySet<string>
+): TableMetadata | null {
+  if (!metadata) return metadata
+  let next = metadata
+  if (metadata.columnWidths) {
+    const widths = { ...metadata.columnWidths }
+    let changed = false
+    for (const id of ids)
+      if (id in widths) {
+        delete widths[id]
+        changed = true
+      }
+    if (changed) next = { ...next, columnWidths: widths }
+  }
+  if (metadata.columnOrder?.some((id) => ids.has(id))) {
+    next = { ...next, columnOrder: metadata.columnOrder.filter((id) => !ids.has(id)) }
+  }
+  if (metadata.pinnedColumns?.some((id) => ids.has(id))) {
+    next = { ...next, pinnedColumns: metadata.pinnedColumns.filter((id) => !ids.has(id)) }
+  }
+  return next
+}
+
+/**
+ * Fire-and-forget reclamation of a deleted column's row storage. The column is
+ * already gone from the schema, so reads never surface the orphaned id —
+ * dropping the JSONB key just frees space. Runs in its own transaction with a
+ * row-count-scaled timeout; failures are logged, not propagated.
+ */
+function stripColumnDataInBackground(
+  tableId: string,
+  columnIds: string[],
+  rowCount: number,
+  requestId: string
+): void {
+  if (columnIds.length === 0) return
+  void (async () => {
+    try {
+      await db.transaction(async (trx) => {
+        const statementMs = scaledStatementTimeoutMs(rowCount, {
+          baseMs: 60_000,
+          perRowMs: 2 * columnIds.length,
+        })
+        await setTableTxTimeouts(trx, { statementMs })
+        for (const id of columnIds) {
+          await trx.execute(
+            sql`UPDATE user_table_rows SET data = data - ${id}::text WHERE table_id = ${tableId} AND data ? ${id}::text`
+          )
+        }
+      })
+      logger.info(
+        `[${requestId}] Background-stripped deleted column data [${columnIds.join(', ')}] from table ${tableId}`
+      )
+    } catch (err) {
+      logger.error(
+        `[${requestId}] Background column-data strip failed for table ${tableId} [${columnIds.join(', ')}]:`,
+        err
+      )
+    }
+  })()
+}
+
+/**
+ * Deletes a column from a table's schema. When id-keyed, returns once the schema
+ * is updated and reclaims the column's row-data storage in the background
+ * (fire-and-forget); the legacy path strips the row key synchronously.
+ *
+ * @param data - Delete column data
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if table not found, column not found, or it's the last column
+ */
+export async function deleteColumn(
+  data: DeleteColumnData,
+  requestId: string
+): Promise<TableDefinition> {
+  const { def, stripKey } = await withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const columnIndex = schema.columns.findIndex((c) => columnMatchesRef(c, data.columnName))
+    if (columnIndex === -1) {
+      throw new Error(`Column "${data.columnName}" not found`)
+    }
+
+    if (schema.columns.length <= 1) {
+      throw new Error('Cannot delete the last column in a table')
+    }
+
+    const targetColumn = schema.columns[columnIndex]
+    const actualName = targetColumn.name
+    const columnId = getColumnId(targetColumn)
+    const ownerGroupId = targetColumn.workflowGroupId
+
+    // Drop this column's reference (by id) from every group's outputs and
+    // `columns` dependency. If the column is the last output of its parent
+    // group, the group itself is also removed (a group with zero outputs is
+    // invalid).
+    let groupRemovedId: string | null = null
+    const updatedGroups = (schema.workflowGroups ?? [])
+      .map((group) => {
+        let next = group
+        if (ownerGroupId && group.id === ownerGroupId) {
+          const remaining = group.outputs.filter((o) => o.columnName !== columnId)
+          if (remaining.length === 0) {
+            groupRemovedId = group.id
+          }
+          next = { ...next, outputs: remaining }
+        }
+        return stripGroupDeps(next, new Set([columnId]))
+      })
+      .filter((g) => g.id !== groupRemovedId)
+
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: schema.columns.filter((_, i) => i !== columnIndex),
+      ...(updatedGroups.length > 0 ? { workflowGroups: updatedGroups } : {}),
+    }
+    const updatedMetadata = stripColumnIdsFromMetadata(
+      table.metadata as TableMetadata | null,
+      new Set([columnId])
+    )
+    assertValidSchema(updatedSchema, updatedMetadata?.columnOrder)
+
+    const now = new Date()
+
+    // Schema/metadata update commits now; the column's row-data storage is
+    // reclaimed in the background (fire-and-forget) — reads never surface the
+    // orphaned id since the column is already gone from the schema.
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    if (groupRemovedId) await stripGroupExecutions(trx, data.tableId, [groupRemovedId])
+
+    logger.info(`[${requestId}] Deleted column "${actualName}" from table ${data.tableId}`)
+
+    return {
+      def: { ...table, schema: updatedSchema, metadata: updatedMetadata, updatedAt: now },
+      stripKey: columnId,
+    }
+  })
+
+  stripColumnDataInBackground(data.tableId, [stripKey], def.rowCount ?? 0, requestId)
+  return def
+}
+
+/**
+ * Deletes multiple columns from a table in a single transaction.
+ * Avoids the race condition of calling deleteColumn multiple times in parallel.
+ */
+export async function deleteColumns(
+  data: { tableId: string; columnNames: string[] },
+  requestId: string
+): Promise<TableDefinition> {
+  const { def, stripKeys } = await withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const namesToDelete = new Set<string>()
+    const idsToDelete = new Set<string>()
+    const notFound: string[] = []
+
+    for (const name of data.columnNames) {
+      const col = schema.columns.find((c) => columnMatchesRef(c, name))
+      if (!col) {
+        notFound.push(name)
+      } else {
+        namesToDelete.add(col.name)
+        idsToDelete.add(getColumnId(col))
+      }
+    }
+
+    if (notFound.length > 0) {
+      throw new Error(`Columns not found: ${notFound.join(', ')}`)
+    }
+
+    const remaining = schema.columns.filter((c) => !namesToDelete.has(c.name))
+    if (remaining.length === 0) {
+      throw new Error('Cannot delete all columns from a table')
+    }
+
+    // For each group, drop outputs whose column (by id) is being deleted. Groups
+    // that end up with zero outputs are removed entirely (they'd be invalid).
+    // Then any remaining group's dependencies referencing a removed column are
+    // cleaned up.
+    const removedGroupIds = new Set<string>()
+    let updatedGroups = (schema.workflowGroups ?? []).map((group) => {
+      const remainingOutputs = group.outputs.filter((o) => !idsToDelete.has(o.columnName))
+      if (remainingOutputs.length === 0) {
+        removedGroupIds.add(group.id)
+      }
+      return remainingOutputs.length === group.outputs.length
+        ? group
+        : { ...group, outputs: remainingOutputs }
+    })
+    updatedGroups = updatedGroups
+      .filter((g) => !removedGroupIds.has(g.id))
+      .map((group) => stripGroupDeps(group, idsToDelete))
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: remaining,
+      ...(updatedGroups.length > 0 ? { workflowGroups: updatedGroups } : {}),
+    }
+    const updatedMetadata = stripColumnIdsFromMetadata(
+      table.metadata as TableMetadata | null,
+      idsToDelete
+    )
+    assertValidSchema(updatedSchema, updatedMetadata?.columnOrder)
+
+    const now = new Date()
+
+    // Schema/metadata commit now; row storage for the deleted columns is
+    // reclaimed in the background (fire-and-forget).
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    await stripGroupExecutions(trx, data.tableId, removedGroupIds)
+
+    logger.info(
+      `[${requestId}] Deleted columns [${[...namesToDelete].join(', ')}] from table ${data.tableId}`
+    )
+
+    return {
+      def: { ...table, schema: updatedSchema, metadata: updatedMetadata, updatedAt: now },
+      stripKeys: Array.from(idsToDelete),
+    }
+  })
+
+  if (stripKeys.length > 0) {
+    stripColumnDataInBackground(data.tableId, stripKeys, def.rowCount ?? 0, requestId)
+  }
+  return def
+}
+
+/**
+ * Changes the type of a column. Validates that existing data is compatible.
+ *
+ * @param data - Update column type data
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if table not found, column not found, or existing data is incompatible
+ */
+export async function updateColumnType(
+  data: UpdateColumnTypeData,
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(data.tableId, async (table, trx) => {
+    // Scale both statement and idle timeouts to row count: the compatibility
+    // check below iterates every row in Node between the row SELECT and the
+    // schema UPDATE, leaving the transaction idle for that gap. The default 5s
+    // `idle_in_transaction_session_timeout` would abort a valid type change on
+    // a large table.
+    const timeoutMs = scaledStatementTimeoutMs(table.rowCount ?? 0, {
+      baseMs: 60_000,
+      perRowMs: 2,
+    })
+    await setTableTxTimeouts(trx, { statementMs: timeoutMs, idleMs: timeoutMs })
+
+    if (!(COLUMN_TYPES as readonly string[]).includes(data.newType)) {
+      throw new Error(
+        `Invalid column type "${data.newType}". Valid types: ${COLUMN_TYPES.join(', ')}`
+      )
+    }
+
+    const schema = table.schema
+    const columnIndex = schema.columns.findIndex((c) => columnMatchesRef(c, data.columnName))
+    if (columnIndex === -1) {
+      throw new Error(`Column "${data.columnName}" not found`)
+    }
+
+    const column = schema.columns[columnIndex]
+    if (column.type === data.newType) {
+      return table
+    }
+    const columnKey = getColumnId(column)
+
+    // Validate existing data is compatible with the new type
+    const rows = await trx
+      .select({ id: userTableRows.id, data: userTableRows.data })
+      .from(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.tableId, data.tableId),
+          sql`${userTableRows.data} ? ${columnKey}`,
+          sql`${userTableRows.data}->>${columnKey}::text IS NOT NULL`
+        )
+      )
+
+    let incompatibleCount = 0
+    for (const row of rows) {
+      const rowData = row.data as RowData
+      const value = rowData[columnKey]
+      if (value === null || value === undefined) continue
+
+      if (!isValueCompatibleWithType(value, data.newType)) {
+        incompatibleCount++
+      }
+    }
+
+    if (incompatibleCount > 0) {
+      throw new Error(
+        `Cannot change column "${column.name}" to type "${data.newType}": ${incompatibleCount} row(s) have incompatible values. Fix or remove the incompatible values first.`
+      )
+    }
+
+    const updatedColumns = schema.columns.map((c, i) =>
+      i === columnIndex ? { ...c, type: data.newType } : c
+    )
+    const updatedSchema: TableSchema = { ...schema, columns: updatedColumns }
+    const now = new Date()
+
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    logger.info(
+      `[${requestId}] Changed column "${column.name}" type from "${column.type}" to "${data.newType}" in table ${data.tableId}`
+    )
+
+    return { ...table, schema: updatedSchema, updatedAt: now }
+  })
+}
+
+/**
+ * Updates constraints (required, unique) on a column.
+ *
+ * @param data - Update column constraints data
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if table not found, column not found, or existing data violates the constraint
+ */
+export async function updateColumnConstraints(
+  data: UpdateColumnConstraintsData,
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(data.tableId, async (table, trx) => {
+    // Scale both statement and idle timeouts to row count: the required/unique
+    // validation runs between separate queries inside this transaction, leaving
+    // it briefly idle. Match `updateColumnType` so the default 5s
+    // `idle_in_transaction_session_timeout` can't abort a valid change on a
+    // large table.
+    const timeoutMs = scaledStatementTimeoutMs(table.rowCount ?? 0, {
+      baseMs: 60_000,
+      perRowMs: 2,
+    })
+    await setTableTxTimeouts(trx, { statementMs: timeoutMs, idleMs: timeoutMs })
+
+    const schema = table.schema
+    const columnIndex = schema.columns.findIndex((c) => columnMatchesRef(c, data.columnName))
+    if (columnIndex === -1) {
+      throw new Error(`Column "${data.columnName}" not found`)
+    }
+
+    const column = schema.columns[columnIndex]
+    const columnKey = getColumnId(column)
+    if (column.workflowGroupId) {
+      throw new Error(
+        `Cannot change constraints on workflow-output column "${column.name}". Constraints aren't applicable to columns whose values come from workflow execution.`
+      )
+    }
+    if (data.required === true && !column.required) {
+      const [result] = await trx
+        .select({ count: count() })
+        .from(userTableRows)
+        .where(
+          and(
+            eq(userTableRows.tableId, data.tableId),
+            sql`(NOT (${userTableRows.data} ? ${columnKey}) OR ${userTableRows.data}->>${columnKey}::text IS NULL)`
+          )
+        )
+
+      if (result.count > 0) {
+        throw new Error(
+          `Cannot set column "${column.name}" as required: ${result.count} row(s) have null or missing values`
+        )
+      }
+    }
+
+    if (data.unique === true && !column.unique) {
+      const duplicates = (await trx.execute(
+        sql`SELECT ${userTableRows.data}->>${columnKey}::text AS val, count(*) AS cnt FROM ${userTableRows} WHERE table_id = ${data.tableId} AND ${userTableRows.data} ? ${columnKey} AND ${userTableRows.data}->>${columnKey}::text IS NOT NULL GROUP BY val HAVING count(*) > 1 LIMIT 1`
+      )) as { val: string; cnt: number }[]
+
+      if (duplicates.length > 0) {
+        throw new Error(`Cannot set column "${column.name}" as unique: duplicate values exist`)
+      }
+    }
+
+    const updatedColumns = schema.columns.map((c, i) =>
+      i === columnIndex
+        ? {
+            ...c,
+            ...(data.required !== undefined ? { required: data.required } : {}),
+            ...(data.unique !== undefined ? { unique: data.unique } : {}),
+          }
+        : c
+    )
+    const updatedSchema: TableSchema = { ...schema, columns: updatedColumns }
+    const now = new Date()
+
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    logger.info(
+      `[${requestId}] Updated constraints for column "${column.name}" in table ${data.tableId}`
+    )
+
+    return { ...table, schema: updatedSchema, updatedAt: now }
+  })
+}
+
+/**
+ * Checks if a value is compatible with a target column type.
+ */
+function isValueCompatibleWithType(
+  value: unknown,
+  targetType: (typeof COLUMN_TYPES)[number]
+): boolean {
+  if (value === null || value === undefined) return true
+
+  switch (targetType) {
+    case 'string':
+      return true
+    case 'number': {
+      if (typeof value === 'number') return Number.isFinite(value)
+      if (typeof value === 'string') {
+        const num = Number(value)
+        return Number.isFinite(num) && value.trim() !== ''
+      }
+      return false
+    }
+    case 'boolean': {
+      if (typeof value === 'boolean') return true
+      if (typeof value === 'string')
+        return ['true', 'false', '1', '0'].includes(value.toLowerCase())
+      if (typeof value === 'number') return value === 0 || value === 1
+      return false
+    }
+    case 'date': {
+      if (value instanceof Date) return !Number.isNaN(value.getTime())
+      if (typeof value === 'string') return !Number.isNaN(Date.parse(value))
+      return false
+    }
+    case 'json':
+      return true
+    default:
+      return false
+  }
+}

apps/sim/lib/table/constants.ts

@@ -0,0 +1,365 @@
+/**
+ * Limits and constants for user-defined tables.
+ */
+
+import { randomInt, randomItem } from '@sim/utils/random'
+import { env, envNumber } from '@/lib/core/config/env'
+
+export const TABLE_LIMITS = {
+  MAX_TABLES_PER_WORKSPACE: 100,
+  MAX_ROWS_PER_TABLE: 10000,
+  MAX_ROW_SIZE_BYTES: 400 * 1024, // 400KB
+  MAX_COLUMNS_PER_TABLE: 50,
+  MAX_TABLE_NAME_LENGTH: 128,
+  MAX_COLUMN_NAME_LENGTH: 50,
+  MAX_DESCRIPTION_LENGTH: 500,
+  DEFAULT_QUERY_LIMIT: 100,
+  MAX_QUERY_LIMIT: 1000,
+  /** Batch size for bulk update operations */
+  UPDATE_BATCH_SIZE: 100,
+  /** Batch size for bulk delete operations */
+  DELETE_BATCH_SIZE: 1000,
+  /** Maximum rows per batch insert */
+  MAX_BATCH_INSERT_SIZE: 1000,
+  /** Maximum rows per bulk update/delete operation */
+  MAX_BULK_OPERATION_SIZE: 1000,
+  /** Maximum rows a single clipboard copy/cut serializes; beyond this the user is steered to Export. */
+  MAX_COPY_ROWS: 50000,
+  /** Rows selected + deleted per page in the async background delete-job loop. Each
+   *  DELETE_BATCH_SIZE chunk inside the page commits in its own transaction; the page is the
+   *  keyset-select and cancel/ownership-check granularity. */
+  DELETE_PAGE_SIZE: 10000,
+  /** Row count above which an export runs as a background job instead of a synchronous stream.
+   *  Tables at or under this stream instantly; larger ones fall back to an async export job. */
+  EXPORT_ASYNC_THRESHOLD_ROWS: 10000,
+  /** Cap on the exclusion set ("select all, minus these") sent to an async delete job. */
+  MAX_EXCLUDE_ROW_IDS: 10000,
+} as const
+
+/**
+ * Default plan-based table limits. Each value can be overridden via env vars
+ * (see `getTablePlanLimits`) so self-hosted deployments can raise the free-tier
+ * caps that apply when billing is disabled.
+ */
+export const DEFAULT_TABLE_PLAN_LIMITS = {
+  free: {
+    maxTables: 5,
+    maxRowsPerTable: 50000,
+  },
+  pro: {
+    maxTables: 100,
+    maxRowsPerTable: 100000,
+  },
+  team: {
+    maxTables: 1000,
+    maxRowsPerTable: 500000,
+  },
+  enterprise: {
+    maxTables: 10000,
+    maxRowsPerTable: 1000000,
+  },
+} as const
+
+/**
+ * Byte budget for one page of row reads, or null when disabled (the default).
+ * Dev-preview of the byte-bounded pagination follow-up: set `TABLE_MAX_PAGE_BYTES`
+ * to cut pages early once their serialized row data exceeds the budget. The
+ * production version moves the cut into SQL — see the pagination-hardening plan.
+ */
+export function getMaxPageBytes(): number | null {
+  const value = envNumber(env.TABLE_MAX_PAGE_BYTES, 0, { min: 0, integer: true })
+  return value > 0 ? value : null
+}
+
+/**
+ * Maximum serialized size in bytes of a single row. Defaults to
+ * `TABLE_LIMITS.MAX_ROW_SIZE_BYTES`; overridable via the
+ * `TABLE_MAX_ROW_SIZE_BYTES` env var (server-only, read at call time).
+ */
+export function getMaxRowSizeBytes(): number {
+  return envNumber(env.TABLE_MAX_ROW_SIZE_BYTES, TABLE_LIMITS.MAX_ROW_SIZE_BYTES, {
+    min: 1,
+    integer: true,
+  })
+}
+
+export type PlanName = keyof typeof DEFAULT_TABLE_PLAN_LIMITS
+
+export interface TablePlanLimits {
+  maxTables: number
+  maxRowsPerTable: number
+}
+
+export type TablePlanLimitsByPlan = Record<PlanName, TablePlanLimits>
+
+/**
+ * Returns plan-based table limits, applying env var overrides on top of the
+ * defaults. When no override is set the value falls back to the hosted-default
+ * constant so behavior is unchanged for the hosted product.
+ */
+export function getTablePlanLimits(): TablePlanLimitsByPlan {
+  return {
+    free: {
+      maxTables: envNumber(env.FREE_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.free.maxTables),
+      maxRowsPerTable: envNumber(
+        env.FREE_TABLE_ROWS_LIMIT,
+        DEFAULT_TABLE_PLAN_LIMITS.free.maxRowsPerTable
+      ),
+    },
+    pro: {
+      maxTables: envNumber(env.PRO_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.pro.maxTables),
+      maxRowsPerTable: envNumber(
+        env.PRO_TABLE_ROWS_LIMIT,
+        DEFAULT_TABLE_PLAN_LIMITS.pro.maxRowsPerTable
+      ),
+    },
+    team: {
+      maxTables: envNumber(env.TEAM_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.team.maxTables),
+      maxRowsPerTable: envNumber(
+        env.TEAM_TABLE_ROWS_LIMIT,
+        DEFAULT_TABLE_PLAN_LIMITS.team.maxRowsPerTable
+      ),
+    },
+    enterprise: {
+      maxTables: envNumber(
+        env.ENTERPRISE_TABLES_LIMIT,
+        DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxTables
+      ),
+      maxRowsPerTable: envNumber(
+        env.ENTERPRISE_TABLE_ROWS_LIMIT,
+        DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxRowsPerTable
+      ),
+    },
+  }
+}
+
+export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json'] as const
+
+export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i
+
+export const USER_TABLE_ROWS_SQL_NAME = 'user_table_rows'
+
+/**
+ * CSV/TSV uploads at or above this size import in the background (direct-to-storage
+ * upload + async worker) instead of being POSTed through the server. Kept safely under
+ * the Next.js proxy request-body cap (10MB) so a synchronous upload is never truncated.
+ */
+export const CSV_ASYNC_IMPORT_THRESHOLD_BYTES = 8 * 1024 * 1024
+
+const TABLE_NAME_ADJECTIVES = [
+  'Radiant',
+  'Luminous',
+  'Blazing',
+  'Glowing',
+  'Bright',
+  'Gleaming',
+  'Shining',
+  'Lustrous',
+  'Vivid',
+  'Dazzling',
+  'Stellar',
+  'Cosmic',
+  'Astral',
+  'Galactic',
+  'Nebular',
+  'Orbital',
+  'Lunar',
+  'Solar',
+  'Starlit',
+  'Celestial',
+  'Infinite',
+  'Vast',
+  'Boundless',
+  'Immense',
+  'Colossal',
+  'Titanic',
+  'Grand',
+  'Supreme',
+  'Eternal',
+  'Ancient',
+  'Timeless',
+  'Primal',
+  'Nascent',
+  'Elder',
+  'Swift',
+  'Drifting',
+  'Surging',
+  'Pulsing',
+  'Soaring',
+  'Rising',
+  'Spiraling',
+  'Crimson',
+  'Azure',
+  'Violet',
+  'Indigo',
+  'Amber',
+  'Sapphire',
+  'Obsidian',
+  'Silver',
+  'Golden',
+  'Scarlet',
+  'Cobalt',
+  'Emerald',
+  'Magnetic',
+  'Quantum',
+  'Photonic',
+  'Spectral',
+  'Charged',
+  'Atomic',
+  'Electric',
+  'Kinetic',
+  'Ethereal',
+  'Mystic',
+  'Phantom',
+  'Silent',
+  'Distant',
+  'Hidden',
+  'Arcane',
+  'Frozen',
+  'Burning',
+  'Molten',
+  'Volatile',
+  'Fiery',
+  'Searing',
+  'Frigid',
+  'Mighty',
+  'Fierce',
+  'Serene',
+  'Tranquil',
+  'Harmonic',
+  'Resonant',
+  'Bold',
+  'Noble',
+  'Pure',
+  'Rare',
+  'Pristine',
+  'Exotic',
+  'Divine',
+] as const
+
+const TABLE_NAME_NOUNS = [
+  'Star',
+  'Pulsar',
+  'Quasar',
+  'Magnetar',
+  'Nova',
+  'Supernova',
+  'Neutron',
+  'Protostar',
+  'Blazar',
+  'Cepheid',
+  'Galaxy',
+  'Nebula',
+  'Cluster',
+  'Void',
+  'Filament',
+  'Halo',
+  'Spiral',
+  'Remnant',
+  'Cloud',
+  'Planet',
+  'Moon',
+  'World',
+  'Exoplanet',
+  'Titan',
+  'Europa',
+  'Triton',
+  'Enceladus',
+  'Comet',
+  'Meteor',
+  'Asteroid',
+  'Fireball',
+  'Shard',
+  'Fragment',
+  'Orion',
+  'Andromeda',
+  'Perseus',
+  'Pegasus',
+  'Phoenix',
+  'Draco',
+  'Cygnus',
+  'Aquila',
+  'Lyra',
+  'Vega',
+  'Hydra',
+  'Sirius',
+  'Polaris',
+  'Altair',
+  'Eclipse',
+  'Aurora',
+  'Corona',
+  'Flare',
+  'Vortex',
+  'Pulse',
+  'Wave',
+  'Ripple',
+  'Shimmer',
+  'Spark',
+  'Horizon',
+  'Zenith',
+  'Apex',
+  'Meridian',
+  'Equinox',
+  'Solstice',
+  'Transit',
+  'Orbit',
+  'Cosmos',
+  'Dimension',
+  'Realm',
+  'Expanse',
+  'Infinity',
+  'Continuum',
+  'Abyss',
+  'Ether',
+  'Photon',
+  'Neutrino',
+  'Tachyon',
+  'Graviton',
+  'Sector',
+  'Quadrant',
+  'Belt',
+  'Ring',
+  'Field',
+  'Stream',
+  'Frontier',
+  'Beacon',
+  'Signal',
+  'Probe',
+  'Voyager',
+  'Pioneer',
+  'Sentinel',
+  'Gateway',
+  'Portal',
+  'Nexus',
+  'Conduit',
+  'Rift',
+  'Core',
+  'Matrix',
+  'Lattice',
+  'Array',
+  'Reactor',
+  'Engine',
+  'Forge',
+  'Crucible',
+] as const
+
+/**
+ * Generates a unique space-themed table name that doesn't collide with existing names.
+ * Uses lowercase with underscores to satisfy NAME_PATTERN validation.
+ */
+export function generateUniqueTableName(existingNames: string[]): string {
+  const taken = new Set(existingNames.map((n) => n.toLowerCase()))
+  const maxAttempts = 50
+
+  for (let i = 0; i < maxAttempts; i++) {
+    const adj = randomItem(TABLE_NAME_ADJECTIVES)
+    const noun = randomItem(TABLE_NAME_NOUNS)
+    const name = `${adj.toLowerCase()}_${noun.toLowerCase()}`
+    if (!taken.has(name)) return name
+  }
+
+  const adj = randomItem(TABLE_NAME_ADJECTIVES)
+  const noun = randomItem(TABLE_NAME_NOUNS)
+  const suffix = randomInt(100, 1000)
+  return `${adj.toLowerCase()}_${noun.toLowerCase()}_${suffix}`
+}

apps/sim/lib/table/dates.test.ts

@@ -0,0 +1,158 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  formatDateCellDisplay,
+  isCalendarDateString,
+  normalizeDateCellValue,
+  storedDateToEditable,
+} from '@/lib/table/dates'
+
+/** The runtime zone's offset suffix at a given local wall time, e.g. `-07:00`. */
+function localOffsetSuffix(local: Date): string {
+  const minutes = -local.getTimezoneOffset()
+  if (minutes === 0) return 'Z'
+  const sign = minutes > 0 ? '+' : '-'
+  const abs = Math.abs(minutes)
+  const pad = (n: number) => String(n).padStart(2, '0')
+  return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}`
+}
+
+describe('isCalendarDateString', () => {
+  it('accepts YYYY-MM-DD and rejects everything else', () => {
+    expect(isCalendarDateString('2026-07-06')).toBe(true)
+    expect(isCalendarDateString('2026-13-45')).toBe(false)
+    expect(isCalendarDateString('2026-07-06T00:00:00Z')).toBe(false)
+    expect(isCalendarDateString('07/06/2026')).toBe(false)
+  })
+})
+
+describe('normalizeDateCellValue', () => {
+  it('keeps calendar dates timezone-free', () => {
+    expect(normalizeDateCellValue('2026-07-06')).toBe('2026-07-06')
+    expect(normalizeDateCellValue(' 2026-07-06 ')).toBe('2026-07-06')
+  })
+
+  it('normalizes date-only inputs in other formats to calendar dates', () => {
+    expect(normalizeDateCellValue('07/06/2026')).toBe('2026-07-06')
+    expect(normalizeDateCellValue('7/6/2026')).toBe('2026-07-06')
+    expect(normalizeDateCellValue('July 6, 2026')).toBe('2026-07-06')
+  })
+
+  it('normalizes reduced-precision ISO forms via their UTC day', () => {
+    expect(normalizeDateCellValue('2026-07')).toBe('2026-07-01')
+    expect(normalizeDateCellValue('2026')).toBe('2026-01-01')
+  })
+
+  it('preserves the wall time and offset of explicit-offset inputs', () => {
+    expect(normalizeDateCellValue('2026-07-06T16:04:55-07:00')).toBe('2026-07-06T16:04:55-07:00')
+    expect(normalizeDateCellValue('2026-07-06 16:04:55 PDT')).toBe('2026-07-06T16:04:55-07:00')
+    expect(normalizeDateCellValue('2026-07-06T23:04:55.000Z')).toBe('2026-07-06T23:04:55Z')
+    expect(normalizeDateCellValue('2026-07-06 16:04:55+00')).toBe('2026-07-06T16:04:55Z')
+    expect(normalizeDateCellValue('2026-07-06 16:04:55 EST')).toBe('2026-07-06T16:04:55-05:00')
+  })
+
+  it('is idempotent on canonical instants', () => {
+    const canonical = '2026-07-06T16:04:55-07:00'
+    expect(normalizeDateCellValue(canonical)).toBe(canonical)
+    expect(normalizeDateCellValue(canonical, { timezone: 'Asia/Tokyo' })).toBe(canonical)
+  })
+
+  it('stamps naive datetimes with the runtime zone offset by default', () => {
+    const local = new Date(2026, 6, 6, 16, 4, 55)
+    expect(normalizeDateCellValue('2026-07-06 16:04:55')).toBe(
+      `2026-07-06T16:04:55${localOffsetSuffix(local)}`
+    )
+  })
+
+  it('stamps naive datetimes with the provided IANA zone offset', () => {
+    // July → America/New_York is EDT (UTC-4)
+    expect(normalizeDateCellValue('2026-07-06 16:04:55', { timezone: 'America/New_York' })).toBe(
+      '2026-07-06T16:04:55-04:00'
+    )
+    // January → EST (UTC-5); DST resolved per wall date, not per import date
+    expect(normalizeDateCellValue('2026-01-15 12:00', { timezone: 'America/New_York' })).toBe(
+      '2026-01-15T12:00:00-05:00'
+    )
+    expect(normalizeDateCellValue('7/6/2026 4:04 PM', { timezone: 'America/Los_Angeles' })).toBe(
+      '2026-07-06T16:04:00-07:00'
+    )
+  })
+
+  it('ignores the zone option when the input carries an explicit offset', () => {
+    expect(
+      normalizeDateCellValue('2026-07-06T23:04:55.000Z', { timezone: 'America/New_York' })
+    ).toBe('2026-07-06T23:04:55Z')
+    expect(
+      normalizeDateCellValue('2026-07-06 16:04:55 PDT', { timezone: 'America/New_York' })
+    ).toBe('2026-07-06T16:04:55-07:00')
+  })
+
+  it('leaves calendar dates untouched by the zone option', () => {
+    expect(normalizeDateCellValue('2026-07-06', { timezone: 'America/New_York' })).toBe(
+      '2026-07-06'
+    )
+  })
+
+  it('throws on an invalid IANA zone', () => {
+    expect(() => normalizeDateCellValue('2026-07-06 12:00', { timezone: 'Not/AZone' })).toThrow(
+      RangeError
+    )
+  })
+
+  it('returns null for unparseable input', () => {
+    expect(normalizeDateCellValue('not-a-date')).toBeNull()
+    expect(normalizeDateCellValue('')).toBeNull()
+    expect(normalizeDateCellValue('2026-13-45')).toBeNull()
+    expect(normalizeDateCellValue('13/06/2026')).toBeNull()
+  })
+})
+
+describe('formatDateCellDisplay', () => {
+  it('renders calendar dates as MM/DD/YYYY', () => {
+    expect(formatDateCellDisplay('2026-07-06')).toBe('07/06/2026')
+  })
+
+  it('renders legacy UTC-midnight instants as their UTC calendar day', () => {
+    expect(formatDateCellDisplay('2026-07-06T00:00:00.000Z')).toBe('07/06/2026')
+    expect(formatDateCellDisplay('2026-07-06T00:00:00Z')).toBe('07/06/2026')
+  })
+
+  it('renders the literal wall time — identical for every viewer', () => {
+    expect(formatDateCellDisplay('2026-07-06T16:04:55-07:00')).toBe('07/06/2026 4:04 PM')
+    expect(formatDateCellDisplay('2026-07-06T16:04:55-07:00', { seconds: true })).toBe(
+      '07/06/2026 4:04:55 PM'
+    )
+    // The offset never shifts the displayed wall time
+    expect(formatDateCellDisplay('2026-07-06T16:04:55+09:00')).toBe('07/06/2026 4:04 PM')
+    expect(formatDateCellDisplay('2026-07-06T23:04:55Z')).toBe('07/06/2026 11:04 PM')
+    expect(formatDateCellDisplay('2026-07-06T00:30:00-07:00')).toBe('07/06/2026 12:30 AM')
+  })
+
+  it('omits the seconds suffix when seconds are zero', () => {
+    expect(formatDateCellDisplay('2026-07-06T23:04:00Z', { seconds: true })).toBe(
+      '07/06/2026 11:04 PM'
+    )
+  })
+
+  it('returns unparseable legacy strings as-is', () => {
+    expect(formatDateCellDisplay('garbage')).toBe('garbage')
+  })
+})
+
+describe('storedDateToEditable', () => {
+  it('surfaces legacy UTC-midnight instants as their UTC calendar day', () => {
+    expect(storedDateToEditable('2026-07-06T00:00:00.000Z')).toBe('2026-07-06')
+  })
+
+  it('keeps calendar dates and canonicalizes instants', () => {
+    expect(storedDateToEditable('2026-07-06')).toBe('2026-07-06')
+    expect(storedDateToEditable('2026-07-06T16:04:55-07:00')).toBe('2026-07-06T16:04:55-07:00')
+    expect(storedDateToEditable('2026-07-06T23:04:55.000Z')).toBe('2026-07-06T23:04:55Z')
+  })
+
+  it('passes unparseable legacy strings through', () => {
+    expect(storedDateToEditable('garbage')).toBe('garbage')
+  })
+})

apps/sim/lib/table/dates.ts

@@ -0,0 +1,315 @@
+/**
+ * Canonical date-cell semantics for user tables.
+ *
+ * A `date` cell stores exactly one of two shapes:
+ *
+ * - **Calendar date** `YYYY-MM-DD` — a timezone-free day.
+ * - **Instant with preserved offset** — RFC 3339 `YYYY-MM-DDTHH:mm:ss±HH:MM`
+ *   (or `Z`). The wall-time part is what was written and is what every viewer
+ *   sees — display never converts across timezones. The offset suffix carries
+ *   the true instant for machine consumers (SQL `::timestamptz` casts,
+ *   workflows, agents, exports).
+ *
+ * The interpretation of an input is determined once, at write time: explicit
+ * offsets (`Z`, `-07:00`, `PDT`) are preserved as written; naive datetime
+ * strings are stamped with the offset of the writer's effective timezone
+ * (via {@link NormalizeDateCellOptions.timezone}), else the runtime's local
+ * zone — the browser for UI writes, the server (UTC in production) for raw
+ * API writes. After that the stored value is final: reads render its wall
+ * time verbatim, identically for everyone.
+ *
+ * This module is pure and shared by server coercion and client rendering.
+ * Client code must import it via this concrete path, never the `@/lib/table`
+ * barrel (the barrel is server-tainted).
+ */
+
+const CALENDAR_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/
+
+/**
+ * Canonical (or canonical-enough legacy) instant: a literal wall time with an
+ * optional fractional-seconds part and an optional offset suffix. The capture
+ * groups are the wall-time fields display renders verbatim.
+ */
+const WALL_INSTANT_PATTERN =
+  /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/
+
+/**
+ * Legacy shape: old CSV imports stored date-only columns as UTC-midnight
+ * instants. Treated as calendar dates so historical rows render as pure days
+ * rather than a spurious "12:00 AM".
+ */
+const UTC_MIDNIGHT_PATTERN = /^\d{4}-\d{2}-\d{2}T00:00:00(\.000)?Z$/
+
+/** A time-of-day component anywhere in the string (e.g. `16:04`). */
+const TIME_COMPONENT_PATTERN = /\d{1,2}:\d{2}/
+
+/**
+ * ISO reduced-precision date forms (`2026`, `2026-07`) parse as UTC per spec,
+ * unlike other date-only forms which V8 parses as local time.
+ */
+const ISO_REDUCED_DATE_PATTERN = /^\d{4}(-\d{2})?$/
+
+/**
+ * Fixed offsets (minutes east of UTC) for the RFC 2822 US timezone
+ * abbreviations — the only abbreviations `Date.parse` accepts, applied as
+ * literal offsets exactly as the engine does.
+ */
+const US_ABBREVIATION_OFFSET_MINUTES: Record<string, number> = {
+  EST: -300,
+  EDT: -240,
+  CST: -360,
+  CDT: -300,
+  MST: -420,
+  MDT: -360,
+  PST: -480,
+  PDT: -420,
+}
+
+/** True when `value` is a canonical timezone-free calendar date. */
+export function isCalendarDateString(value: string): boolean {
+  return CALENDAR_DATE_PATTERN.test(value) && !Number.isNaN(Date.parse(value))
+}
+
+/** A wall-clock reading of an instant in some timezone. */
+export interface WallClockParts {
+  year: number
+  /** 1-based month. */
+  month: number
+  day: number
+  hour: number
+  minute: number
+  second: number
+}
+
+/**
+ * The wall-clock reading of `date` in `timeZone` — or in the runtime's local
+ * zone when omitted. Throws a RangeError on an invalid IANA zone — callers
+ * validate at the boundary.
+ */
+export function getWallClockParts(date: Date, timeZone?: string): WallClockParts {
+  if (!timeZone) {
+    return {
+      year: date.getFullYear(),
+      month: date.getMonth() + 1,
+      day: date.getDate(),
+      hour: date.getHours(),
+      minute: date.getMinutes(),
+      second: date.getSeconds(),
+    }
+  }
+  const parts = new Intl.DateTimeFormat('en-US', {
+    timeZone,
+    hourCycle: 'h23',
+    year: 'numeric',
+    month: '2-digit',
+    day: '2-digit',
+    hour: '2-digit',
+    minute: '2-digit',
+    second: '2-digit',
+  }).formatToParts(date)
+  const get = (type: string) => Number(parts.find((p) => p.type === type)?.value)
+  return {
+    year: get('year'),
+    month: get('month'),
+    day: get('day'),
+    hour: get('hour'),
+    minute: get('minute'),
+    second: get('second'),
+  }
+}
+
+/** Offset of `timeZone` from UTC (ms east) at the moment `at`. */
+function zoneOffsetMs(timeZone: string, at: Date): number {
+  const wall = getWallClockParts(at, timeZone)
+  const asUtc = Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second)
+  return asUtc - at.getTime()
+}
+
+/**
+ * Converts a wall-clock reading in `timeZone` to the UTC instant it denotes.
+ * Two-pass so readings near a DST transition resolve with the offset in
+ * force at that wall time.
+ */
+function wallTimeInZoneToUtc(wall: Date, timeZone: string): Date {
+  const guess = Date.UTC(
+    wall.getFullYear(),
+    wall.getMonth(),
+    wall.getDate(),
+    wall.getHours(),
+    wall.getMinutes(),
+    wall.getSeconds(),
+    wall.getMilliseconds()
+  )
+  const adjusted = guess - zoneOffsetMs(timeZone, new Date(guess))
+  return new Date(guess - zoneOffsetMs(timeZone, new Date(adjusted)))
+}
+
+function pad(n: number): string {
+  return String(n).padStart(2, '0')
+}
+
+function toLocalCalendarDate(date: Date): string {
+  return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`
+}
+
+function toUtcCalendarDate(date: Date): string {
+  return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`
+}
+
+/** `Z` for zero, else `±HH:MM`. */
+function formatOffsetSuffix(offsetMinutes: number): string {
+  if (offsetMinutes === 0) return 'Z'
+  const sign = offsetMinutes > 0 ? '+' : '-'
+  const abs = Math.abs(offsetMinutes)
+  return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}`
+}
+
+/**
+ * Trailing offset (minutes east of UTC) of a datetime string, or null when
+ * naive. Recognizes exactly what `Date.parse` recognizes: numeric offsets,
+ * `Z`/`UT`/`UTC`/`GMT`, and the RFC 2822 US abbreviations. Deliberately does
+ * not match a trailing `AM`/`PM`.
+ */
+function extractExplicitOffsetMinutes(value: string): number | null {
+  const numeric = value.match(/([+-])(\d{1,2}):?(\d{2})?\s*$/)
+  if (numeric) {
+    const sign = numeric[1] === '-' ? -1 : 1
+    return sign * (Number(numeric[2]) * 60 + Number(numeric[3] ?? 0))
+  }
+  if (/(?:Z|UTC?|GMT)$/i.test(value)) return 0
+  const abbreviation = value.match(/([ECMP][SD]T)$/i)
+  if (abbreviation) return US_ABBREVIATION_OFFSET_MINUTES[abbreviation[1].toUpperCase()]
+  return null
+}
+
+/** Serializes UTC-read fields of `shifted` as a wall time with `offset`. */
+function formatUtcFieldsAsWall(shifted: Date, offsetMinutes: number): string {
+  return `${toUtcCalendarDate(shifted)}T${pad(shifted.getUTCHours())}:${pad(
+    shifted.getUTCMinutes()
+  )}:${pad(shifted.getUTCSeconds())}${formatOffsetSuffix(offsetMinutes)}`
+}
+
+/** Serializes local-read fields of `parsed` as a wall time with `offset`. */
+function formatLocalFieldsAsWall(parsed: Date, offsetMinutes: number): string {
+  return `${toLocalCalendarDate(parsed)}T${pad(parsed.getHours())}:${pad(
+    parsed.getMinutes()
+  )}:${pad(parsed.getSeconds())}${formatOffsetSuffix(offsetMinutes)}`
+}
+
+export interface NormalizeDateCellOptions {
+  /**
+   * IANA zone whose offset stamps naive datetime strings (no explicit
+   * offset), e.g. a CSV import applying the importing user's timezone.
+   * Defaults to the runtime's local zone — the author's wall clock in the
+   * browser, UTC on production servers. Throws a RangeError on an invalid
+   * zone.
+   */
+  timezone?: string
+}
+
+/**
+ * Normalizes a raw string to a canonical date-cell value, or `null` when it
+ * cannot be parsed. Date-only inputs become calendar dates; inputs carrying
+ * a time become offset-preserved instants: the wall time survives verbatim
+ * (explicit offsets kept as written, naive readings stamped per
+ * {@link NormalizeDateCellOptions.timezone}) — see module doc.
+ */
+export function normalizeDateCellValue(
+  raw: string,
+  options?: NormalizeDateCellOptions
+): string | null {
+  const trimmed = raw.trim()
+  if (!trimmed) return null
+  if (CALENDAR_DATE_PATTERN.test(trimmed)) {
+    return Number.isNaN(Date.parse(trimmed)) ? null : trimmed
+  }
+  const ms = Date.parse(trimmed)
+  if (Number.isNaN(ms)) return null
+  const parsed = new Date(ms)
+  if (!TIME_COMPONENT_PATTERN.test(trimmed)) {
+    return ISO_REDUCED_DATE_PATTERN.test(trimmed)
+      ? toUtcCalendarDate(parsed)
+      : toLocalCalendarDate(parsed)
+  }
+  const explicitOffset = extractExplicitOffsetMinutes(trimmed)
+  if (explicitOffset !== null) {
+    // The input's own wall time = the instant shifted east by its offset,
+    // read as UTC fields.
+    return formatUtcFieldsAsWall(new Date(ms + explicitOffset * 60_000), explicitOffset)
+  }
+  if (options?.timezone) {
+    // `parsed`'s local getters recover the wall-clock fields V8 read from the
+    // naive string; stamp them with the requested zone's offset at that time.
+    const instant = wallTimeInZoneToUtc(parsed, options.timezone)
+    const offsetMinutes = Math.round(zoneOffsetMs(options.timezone, instant) / 60_000)
+    return formatLocalFieldsAsWall(parsed, offsetMinutes)
+  }
+  return formatLocalFieldsAsWall(parsed, -parsed.getTimezoneOffset())
+}
+
+/**
+ * Canonical form a stored date cell should be edited (and re-saved) as.
+ * Legacy UTC-midnight instants surface as their UTC calendar day (old CSV
+ * imports stored date-only columns that way). Unparseable legacy strings
+ * pass through so the editor shows what is actually stored.
+ */
+export function storedDateToEditable(stored: string): string {
+  if (UTC_MIDNIGHT_PATTERN.test(stored)) return toUtcCalendarDate(new Date(stored))
+  return normalizeDateCellValue(stored) ?? stored
+}
+
+interface FormatDateCellDisplayOptions {
+  /** Include seconds on instants when non-zero (editor drafts round-trip precision). */
+  seconds?: boolean
+}
+
+function formatWallForDisplay(
+  month: string,
+  day: string,
+  year: string,
+  hour: number,
+  minute: string,
+  second: number,
+  withSeconds: boolean | undefined
+): string {
+  const hours12 = hour % 12 === 0 ? 12 : hour % 12
+  const meridiem = hour < 12 ? 'AM' : 'PM'
+  const secondsPart = withSeconds && second !== 0 ? `:${pad(second)}` : ''
+  return `${month}/${day}/${year} ${hours12}:${minute}${secondsPart} ${meridiem}`
+}
+
+/**
+ * Formats a stored date-cell value for display. Calendar dates (and legacy
+ * UTC-midnight instants) render as `MM/DD/YYYY`; instants render their
+ * **literal wall time** as `MM/DD/YYYY h:mm AM/PM` — identical for every
+ * viewer, no timezone conversion. Legacy strings that predate
+ * canonicalization render via a runtime-local normalization; unparseable
+ * ones are returned as-is.
+ */
+export function formatDateCellDisplay(
+  stored: string,
+  options?: FormatDateCellDisplayOptions
+): string {
+  const calendar = stored.match(/^(\d{4})-(\d{2})-(\d{2})$/)
+  if (calendar) return `${calendar[2]}/${calendar[3]}/${calendar[1]}`
+  if (UTC_MIDNIGHT_PATTERN.test(stored)) {
+    const date = new Date(stored)
+    return `${pad(date.getUTCMonth() + 1)}/${pad(date.getUTCDate())}/${date.getUTCFullYear()}`
+  }
+  const wall = stored.match(WALL_INSTANT_PATTERN)
+  if (wall) {
+    const [, year, month, day, hour, minute, second] = wall
+    return formatWallForDisplay(
+      month,
+      day,
+      year,
+      Number(hour),
+      minute,
+      Number(second ?? 0),
+      options?.seconds
+    )
+  }
+  const canonical = normalizeDateCellValue(stored)
+  if (!canonical) return stored
+  return formatDateCellDisplay(canonical, options)
+}

apps/sim/lib/table/delete-runner.test.ts

@@ -0,0 +1,234 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockGetTableById,
+  mockGetJobProgress,
+  mockSelectRowIdPage,
+  mockDeletePageByIds,
+  mockUpdateJobProgress,
+  mockMarkJobReady,
+  mockMarkJobFailed,
+  mockAppendTableEvent,
+  mockBuildFilterClause,
+} = vi.hoisted(() => ({
+  mockGetTableById: vi.fn(),
+  mockGetJobProgress: vi.fn(),
+  mockSelectRowIdPage: vi.fn(),
+  mockDeletePageByIds: vi.fn(),
+  mockUpdateJobProgress: vi.fn(),
+  mockMarkJobReady: vi.fn(),
+  mockMarkJobFailed: vi.fn(),
+  mockAppendTableEvent: vi.fn(),
+  mockBuildFilterClause: vi.fn(),
+}))
+
+vi.mock('@/lib/table/service', () => ({
+  getTableById: mockGetTableById,
+}))
+vi.mock('@/lib/table/jobs/service', () => ({
+  getJobProgress: mockGetJobProgress,
+  updateJobProgress: mockUpdateJobProgress,
+  markJobReady: mockMarkJobReady,
+  markJobFailed: mockMarkJobFailed,
+}))
+vi.mock('@/lib/table/rows/ordering', () => ({
+  selectRowIdPage: mockSelectRowIdPage,
+  deletePageByIds: mockDeletePageByIds,
+}))
+vi.mock('@/lib/table/events', () => ({ appendTableEvent: mockAppendTableEvent }))
+vi.mock('@/lib/table/sql', () => ({ buildFilterClause: mockBuildFilterClause }))
+vi.mock('@/lib/table/constants', () => ({
+  TABLE_LIMITS: { DELETE_PAGE_SIZE: 2 },
+  USER_TABLE_ROWS_SQL_NAME: 'user_table_rows',
+}))
+
+import { markTableDeleteFailed, runTableDelete } from '@/lib/table/delete-runner'
+
+const table = { id: 'tbl_1', workspaceId: 'ws_1', schema: { columns: [] } }
+const cutoff = new Date('2026-06-05T00:00:00Z')
+
+function basePayload(overrides = {}) {
+  return { jobId: 'job_1', tableId: 'tbl_1', workspaceId: 'ws_1', cutoff, ...overrides }
+}
+
+describe('runTableDelete', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetTableById.mockResolvedValue(table)
+    mockGetJobProgress.mockResolvedValue(0)
+    mockUpdateJobProgress.mockResolvedValue(true)
+    mockMarkJobReady.mockResolvedValue(true)
+    mockMarkJobFailed.mockResolvedValue(undefined)
+    mockDeletePageByIds.mockImplementation((_t, _w, ids: string[]) => Promise.resolve(ids.length))
+    mockBuildFilterClause.mockReturnValue({})
+  })
+
+  it('deletes every matching page then marks the job ready', async () => {
+    mockSelectRowIdPage
+      .mockResolvedValueOnce(['a', 'b'])
+      .mockResolvedValueOnce(['c'])
+      .mockResolvedValueOnce([])
+
+    await runTableDelete(basePayload({ filter: { status: 'old' } }))
+
+    expect(mockDeletePageByIds).toHaveBeenNthCalledWith(1, 'tbl_1', 'ws_1', ['a', 'b'])
+    expect(mockDeletePageByIds).toHaveBeenNthCalledWith(2, 'tbl_1', 'ws_1', ['c'])
+    expect(mockMarkJobReady).toHaveBeenCalledWith('tbl_1', 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'delete', status: 'ready', progress: 3 })
+    )
+  })
+
+  it('stops once maxRows is reached and caps the final page fetch to the remaining budget', async () => {
+    // budget 3 with page size 2: first page fills 2, the second is capped to the remaining 1.
+    mockSelectRowIdPage.mockResolvedValueOnce(['a', 'b']).mockResolvedValueOnce(['c'])
+
+    await runTableDelete(basePayload({ filter: { status: 'old' }, maxRows: 3 }))
+
+    expect(mockSelectRowIdPage).toHaveBeenCalledTimes(2)
+    expect(mockSelectRowIdPage.mock.calls[0][0]).toMatchObject({ limit: 2 })
+    expect(mockSelectRowIdPage.mock.calls[1][0]).toMatchObject({ limit: 1 })
+    expect(mockDeletePageByIds).toHaveBeenCalledTimes(2)
+    expect(mockMarkJobReady).toHaveBeenCalledWith('tbl_1', 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready', progress: 3 })
+    )
+  })
+
+  it('skips excluded rows but still advances the keyset cursor past them', async () => {
+    mockSelectRowIdPage.mockResolvedValueOnce(['keep', 'x']).mockResolvedValueOnce([])
+
+    await runTableDelete(basePayload({ excludeRowIds: ['keep'] }))
+
+    expect(mockDeletePageByIds).toHaveBeenCalledTimes(1)
+    expect(mockDeletePageByIds).toHaveBeenCalledWith('tbl_1', 'ws_1', ['x'])
+    // Second page is queried after the last id of the first page (cursor advanced past 'keep').
+    expect(mockSelectRowIdPage).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({ afterId: 'x' })
+    )
+    expect(mockMarkJobReady).toHaveBeenCalled()
+  })
+
+  it('stops without marking ready when the ownership gate is lost (cancel/supersede)', async () => {
+    mockSelectRowIdPage.mockResolvedValue(['a', 'b'])
+    mockUpdateJobProgress.mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+
+    await runTableDelete(basePayload())
+
+    expect(mockDeletePageByIds).toHaveBeenCalledTimes(1)
+    expect(mockMarkJobReady).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+    expect(mockAppendTableEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready' })
+    )
+  })
+
+  it('rethrows unexpected errors without failing the job (caller retries decide)', async () => {
+    mockSelectRowIdPage.mockRejectedValue(new Error('boom'))
+
+    await expect(runTableDelete(basePayload())).rejects.toThrow('boom')
+
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+    expect(mockAppendTableEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'failed' })
+    )
+  })
+
+  it('returns quietly when superseded mid-run without failing the job', async () => {
+    mockSelectRowIdPage.mockResolvedValue(['a', 'b'])
+    mockUpdateJobProgress.mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+
+    await expect(runTableDelete(basePayload())).resolves.toBeUndefined()
+
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('rethrows the root cause so the clean message survives serialization', async () => {
+    const cause = new Error('canceling statement due to statement timeout')
+    mockSelectRowIdPage.mockRejectedValue(new Error('Failed query: delete ...', { cause }))
+
+    await expect(runTableDelete(basePayload())).rejects.toThrow(
+      'canceling statement due to statement timeout'
+    )
+  })
+
+  it('resumes cumulative progress on retry instead of resetting to zero', async () => {
+    mockGetJobProgress.mockResolvedValue(7)
+    mockSelectRowIdPage.mockResolvedValueOnce(['a', 'b']).mockResolvedValueOnce([])
+
+    await runTableDelete(basePayload())
+
+    expect(mockUpdateJobProgress).toHaveBeenNthCalledWith(1, 'tbl_1', 7, 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready', progress: 9 })
+    )
+  })
+
+  it('stops at the seed read when the job is no longer owned', async () => {
+    mockGetJobProgress.mockResolvedValue(null)
+
+    await expect(runTableDelete(basePayload())).resolves.toBeUndefined()
+
+    expect(mockSelectRowIdPage).not.toHaveBeenCalled()
+    expect(mockDeletePageByIds).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('passes the cutoff and filter clause through to the page query', async () => {
+    mockSelectRowIdPage.mockResolvedValueOnce([])
+
+    await runTableDelete(basePayload({ filter: { status: 'old' } }))
+
+    expect(mockBuildFilterClause).toHaveBeenCalledWith(
+      { status: 'old' },
+      'user_table_rows',
+      table.schema.columns
+    )
+    expect(mockSelectRowIdPage).toHaveBeenCalledWith(
+      expect.objectContaining({ cutoff, filterClause: {}, limit: 2 })
+    )
+  })
+})
+
+describe('markTableDeleteFailed', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMarkJobFailed.mockResolvedValue(undefined)
+  })
+
+  it('marks the job failed and emits the failed event', async () => {
+    await markTableDeleteFailed('tbl_1', 'job_1', new Error('boom'))
+
+    expect(mockMarkJobFailed).toHaveBeenCalledWith('tbl_1', 'job_1', 'boom')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'delete', status: 'failed', error: 'boom' })
+    )
+  })
+
+  it('prefers the error cause over a verbose wrapper message', async () => {
+    const cause = new Error('canceling statement due to statement timeout')
+    const wrapper = new Error(`Failed query: delete from x where id in (${'$1,'.repeat(5000)})`, {
+      cause,
+    })
+
+    await markTableDeleteFailed('tbl_1', 'job_1', wrapper)
+
+    expect(mockMarkJobFailed).toHaveBeenCalledWith(
+      'tbl_1',
+      'job_1',
+      'canceling statement due to statement timeout'
+    )
+  })
+
+  it('truncates oversized messages', async () => {
+    await markTableDeleteFailed('tbl_1', 'job_1', new Error('x'.repeat(2000)))
+
+    const [, , message] = mockMarkJobFailed.mock.calls[0]
+    expect(message).toHaveLength(503)
+    expect(message.endsWith('...')).toBe(true)
+  })
+})

apps/sim/lib/table/delete-runner.ts

@@ -0,0 +1,183 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { truncate } from '@sim/utils/string'
+import type { Filter } from '@/lib/table'
+import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants'
+import { appendTableEvent } from '@/lib/table/events'
+import {
+  getJobProgress,
+  markJobFailed,
+  markJobReady,
+  updateJobProgress,
+} from '@/lib/table/jobs/service'
+import { deletePageByIds, selectRowIdPage } from '@/lib/table/rows/ordering'
+import { getTableById } from '@/lib/table/service'
+import { buildFilterClause } from '@/lib/table/sql'
+
+const logger = createLogger('TableDeleteRunner')
+
+/** Emit a progress event / heartbeat at most every this many rows. */
+const PROGRESS_INTERVAL_ROWS = 5000
+
+/**
+ * Thrown when this worker discovers it no longer owns the table's job (canceled, or the
+ * stale-job janitor marked it failed and a newer job took over). The worker stops deleting.
+ */
+class JobSupersededError extends Error {}
+
+export interface TableDeletePayload {
+  jobId: string
+  tableId: string
+  workspaceId: string
+  /** Optional filter narrowing which rows to delete; omitted = every row at/under the cutoff. */
+  filter?: Filter
+  /** Rows to spare ("select all, minus these"). Bounded by `MAX_EXCLUDE_ROW_IDS`. */
+  excludeRowIds?: string[]
+  /** Only rows created at/before this instant are deleted, so mid-job inserts survive. */
+  cutoff: Date
+  /**
+   * Stop after deleting this many rows (an explicit caller-supplied limit). Omitted = every match.
+   * Not combined with `excludeRowIds` (the UI's select-all path uses excludes and no cap; the
+   * copilot tool uses a cap and no excludes), so the per-page fetch can be bounded directly.
+   */
+  maxRows?: number
+}
+
+/**
+ * Background worker for large filtered row deletes (trigger.dev task, or detached on the web
+ * container when trigger.dev is disabled — see the delete-async kickoff route). Deletes in
+ * keyset-paginated pages — `created_at <= cutoff` spares rows inserted while the job runs, and
+ * `excludeRowIds` spares specific rows (the "select all then deselect a few" case).
+ * Ownership-gated per page so a cancel/supersede stops it within one page; committed batches are
+ * never rolled back. Progress and the terminal state are surfaced via the table-events SSE
+ * stream.
+ *
+ * Unexpected errors are rethrown so the caller's retry machinery sees them — the caller marks
+ * the job failed via `markTableDeleteFailed` once it gives up. A superseded run (cancel, or a
+ * newer job took the table) returns quietly.
+ */
+export async function runTableDelete(payload: TableDeletePayload): Promise<void> {
+  const { jobId, tableId, workspaceId, filter, excludeRowIds, cutoff, maxRows } = payload
+  const requestId = generateId().slice(0, 8)
+  const budget = maxRows ?? Number.POSITIVE_INFINITY
+
+  try {
+    const table = await getTableById(tableId, { includeArchived: true })
+    if (!table) throw new Error(`Delete target table ${tableId} not found`)
+
+    const filterClause = filter
+      ? buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, table.schema.columns)
+      : undefined
+    const excluded = new Set(excludeRowIds ?? [])
+
+    // Resume the persisted count: a retried attempt's earlier batches are already committed,
+    // so starting at zero would overwrite cumulative progress with this attempt's smaller
+    // number. Doubles as the initial ownership gate.
+    const resumed = await getJobProgress(tableId, jobId)
+    if (resumed === null) throw new JobSupersededError()
+
+    let processed = resumed
+    let lastReported = resumed
+    let afterId: string | undefined
+
+    while (processed < budget) {
+      // Ownership gate before every page: once this run loses the table (cancel/supersede),
+      // updateJobProgress returns false and we stop before deleting further.
+      const owns = await updateJobProgress(tableId, processed, jobId)
+      if (!owns) throw new JobSupersededError()
+
+      const page = await selectRowIdPage({
+        tableId,
+        workspaceId,
+        cutoff,
+        filterClause,
+        afterId,
+        limit: Math.min(TABLE_LIMITS.DELETE_PAGE_SIZE, budget - processed),
+      })
+      if (page.length === 0) break
+      // Advance the keyset cursor past the whole page — excluded ids are skipped (not deleted),
+      // so the cursor must move even when nothing in the page is deletable.
+      afterId = page[page.length - 1]
+
+      const toDelete = excluded.size > 0 ? page.filter((id) => !excluded.has(id)) : page
+      if (toDelete.length > 0) {
+        processed += await deletePageByIds(tableId, workspaceId, toDelete)
+      }
+
+      if (
+        processed - lastReported >= PROGRESS_INTERVAL_ROWS ||
+        (lastReported === 0 && processed > 0)
+      ) {
+        lastReported = processed
+        void appendTableEvent({
+          kind: 'job',
+          type: 'delete',
+          tableId,
+          jobId,
+          status: 'running',
+          progress: processed,
+        })
+      }
+    }
+
+    await updateJobProgress(tableId, processed, jobId)
+    // Only announce success if we still won the transition — a cancel/supersede at the very end
+    // makes this a no-op, and we must not emit a false `ready`.
+    const becameReady = await markJobReady(tableId, jobId)
+    if (becameReady) {
+      void appendTableEvent({
+        kind: 'job',
+        type: 'delete',
+        tableId,
+        jobId,
+        status: 'ready',
+        progress: processed,
+      })
+      logger.info(`[${requestId}] Delete complete`, { tableId, rows: processed })
+    } else {
+      logger.info(
+        `[${requestId}] Delete finished but no longer owns the run (canceled/superseded)`,
+        {
+          tableId,
+          jobId,
+        }
+      )
+    }
+  } catch (err) {
+    if (err instanceof JobSupersededError) {
+      logger.info(`[${requestId}] Delete superseded by a newer run; stopping`, { tableId, jobId })
+      return
+    }
+    // Rethrow the root cause, not the wrapper: drizzle query errors embed the full SQL + params
+    // list (tens of KB for a batch delete) in `message`, and `cause` does not survive
+    // trigger.dev's serialization between the failed `run` and `onFailure` — the clean message
+    // must already be the thrown error's own `message`.
+    const cause = toError(err).cause
+    const error = cause ? toError(cause) : toError(err)
+    logger.error(`[${requestId}] Delete failed for table ${tableId}:`, error)
+    throw error
+  }
+}
+
+/**
+ * Marks the delete job failed and emits the failed SSE event. Called once the caller gives up on
+ * the run: the trigger.dev task's `onFailure` (after retries are exhausted) or the detached
+ * web-container fallback (no retries). Scoped to jobId — a no-op if a newer job has taken over.
+ */
+export async function markTableDeleteFailed(
+  tableId: string,
+  jobId: string,
+  error: unknown
+): Promise<void> {
+  const message = truncate(getErrorMessage(toError(error).cause ?? error, 'Delete failed'), 500)
+  await markJobFailed(tableId, jobId, message).catch(() => {})
+  void appendTableEvent({
+    kind: 'job',
+    type: 'delete',
+    tableId,
+    jobId,
+    status: 'failed',
+    error: message,
+  })
+}

apps/sim/lib/table/deps.test.ts

@@ -0,0 +1,116 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  areGroupDepsSatisfied,
+  getUnmetGroupDeps,
+  isExecCancelled,
+  isExecCancelledAfter,
+  optimisticallyScheduleNewlyEligibleGroups,
+} from '@/lib/table/deps'
+import type { RowExecutionMetadata, TableRow, WorkflowGroup } from '@/lib/table/types'
+
+function makeGroup(overrides: Partial<WorkflowGroup> & { id: string }): WorkflowGroup {
+  return {
+    workflowId: `wf-${overrides.id}`,
+    outputs: [{ blockId: 'b1', path: 'out', columnName: `${overrides.id}_out` }],
+    ...overrides,
+  }
+}
+
+function makeRow(
+  data: Record<string, unknown> = {},
+  executions: Record<string, RowExecutionMetadata> = {}
+): TableRow {
+  return {
+    id: 'row1',
+    data: data as TableRow['data'],
+    executions,
+    position: 0,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  }
+}
+
+describe('areGroupDepsSatisfied — checkbox dependency', () => {
+  const group = makeGroup({ id: 'g1', dependencies: { columns: ['flag'] } })
+
+  it('treats a checked box (true) as satisfied', () => {
+    expect(areGroupDepsSatisfied(group, makeRow({ flag: true }))).toBe(true)
+  })
+
+  it('treats an unchecked box (false) as unmet', () => {
+    expect(areGroupDepsSatisfied(group, makeRow({ flag: false }))).toBe(false)
+  })
+
+  it('treats empty / null / undefined as unmet', () => {
+    expect(areGroupDepsSatisfied(group, makeRow({ flag: '' }))).toBe(false)
+    expect(areGroupDepsSatisfied(group, makeRow({ flag: null }))).toBe(false)
+    expect(areGroupDepsSatisfied(group, makeRow({}))).toBe(false)
+  })
+
+  it('reports an unchecked box in unmet deps', () => {
+    expect(getUnmetGroupDeps(group, makeRow({ flag: false })).columns).toEqual(['flag'])
+    expect(getUnmetGroupDeps(group, makeRow({ flag: true })).columns).toEqual([])
+  })
+})
+
+describe('optimisticallyScheduleNewlyEligibleGroups — checkbox toggle', () => {
+  const group = makeGroup({ id: 'g1', autoRun: true, dependencies: { columns: ['flag'] } })
+
+  it('flips the dependent to pending when checking (false → true)', () => {
+    const before = makeRow({ flag: false })
+    const next = optimisticallyScheduleNewlyEligibleGroups([group], before, { flag: true })
+    expect(next?.g1?.status).toBe('pending')
+  })
+
+  it('does NOT schedule anything when unchecking (true → false)', () => {
+    const before = makeRow({ flag: true }, { g1: completedExec('wf-g1') })
+    const next = optimisticallyScheduleNewlyEligibleGroups([group], before, { flag: false })
+    expect(next).toBeNull()
+  })
+})
+
+function completedExec(workflowId: string): RowExecutionMetadata {
+  return { status: 'completed', executionId: 'e1', jobId: null, workflowId, error: null }
+}
+
+describe('isExecCancelled', () => {
+  it('is true only for cancelled status', () => {
+    expect(isExecCancelled({ status: 'cancelled' } as RowExecutionMetadata)).toBe(true)
+    expect(isExecCancelled({ status: 'running' } as RowExecutionMetadata)).toBe(false)
+    expect(isExecCancelled(undefined)).toBe(false)
+  })
+
+  it('is true regardless of executionId — the resurrection-bug guard', () => {
+    // A stop click can only stamp the pre-stamp's (often null) executionId.
+    expect(
+      isExecCancelled({ status: 'cancelled', executionId: null } as RowExecutionMetadata)
+    ).toBe(true)
+  })
+})
+
+describe('isExecCancelledAfter — dispatcher tombstone', () => {
+  const since = new Date('2026-01-01T00:00:00Z')
+
+  it('is true when cancelled after the dispatch was requested', () => {
+    const exec = {
+      status: 'cancelled',
+      cancelledAt: '2026-01-01T00:00:05Z',
+    } as RowExecutionMetadata
+    expect(isExecCancelledAfter(exec, since)).toBe(true)
+  })
+
+  it('is false for a cancel that predates the dispatch (a prior, cleared run)', () => {
+    const exec = {
+      status: 'cancelled',
+      cancelledAt: '2025-12-31T23:59:59Z',
+    } as RowExecutionMetadata
+    expect(isExecCancelledAfter(exec, since)).toBe(false)
+  })
+
+  it('is false without a cancelledAt timestamp', () => {
+    expect(isExecCancelledAfter({ status: 'cancelled' } as RowExecutionMetadata, since)).toBe(false)
+  })
+})

apps/sim/lib/table/deps.ts

@@ -0,0 +1,186 @@
+/**
+ * Pure dep-satisfaction helpers shared by the server-side scheduler and the
+ * client UI. Lives in its own file (not `workflow-columns.ts`) so the client
+ * can import it without pulling in `@sim/db` and other server-only deps.
+ */
+
+import { createLogger } from '@sim/logger'
+import type {
+  RowData,
+  RowExecutionMetadata,
+  RowExecutions,
+  TableRow,
+  WorkflowGroup,
+} from '@/lib/table/types'
+
+const logger = createLogger('OptimisticCascade')
+
+/**
+ * True when the cell is `pending` / `queued` / `running`. Single source of
+ * truth for the "is this exec in flight" classification across the
+ * eligibility predicate, optimistic patches, status counters, and renderer.
+ * `pending` counts even without a jobId so the row-gutter Stop button is
+ * available the moment the user clicks Play — the cancel path writes
+ * `cancelled` authoritatively whether or not a real trigger.dev run exists
+ * yet, which is correct: cancel means "don't run this."
+ */
+export function isExecInFlight(exec: RowExecutionMetadata | undefined): boolean {
+  if (!exec) return false
+  const s = exec.status
+  return s === 'queued' || s === 'running' || s === 'pending'
+}
+
+/**
+ * A cell run the user/stop killed. The single source of truth for "do not run /
+ * do not write this cell" — used by the in-memory write guard, the worker's
+ * pre-execution check, and the resume worker. The SQL guard in
+ * `writeExecutionsPatch` mirrors this status test in its `WHERE`.
+ */
+export function isExecCancelled(exec: RowExecutionMetadata | undefined): boolean {
+  return exec?.status === 'cancelled'
+}
+
+/**
+ * Cancelled AND killed after `since`. The dispatcher's tombstone test: a cell
+ * cancelled after a dispatch was requested must be skipped by that dispatch's
+ * later windows, even though the dispatcher pre-stamped it before the stop.
+ */
+export function isExecCancelledAfter(exec: RowExecutionMetadata | undefined, since: Date): boolean {
+  if (!isExecCancelled(exec) || !exec?.cancelledAt) return false
+  const at = Date.parse(exec.cancelledAt)
+  return Number.isFinite(at) && at > since.getTime()
+}
+
+/**
+ * A dependency column counts as unmet when its value is empty OR explicitly
+ * `false`. An unchecked checkbox is treated as "dependency not satisfied", so
+ * only checking a box (false→true) makes dependents eligible — unchecking
+ * (true→false) never triggers a rerun.
+ */
+function isDepValueUnmet(value: unknown): boolean {
+  return value === null || value === undefined || value === '' || value === false
+}
+
+/**
+ * True when every output column the group writes still has a non-empty value
+ * on this row. The "completed" exec status is metadata, but the cells are the
+ * source of truth — if the user cleared an output cell, the row is effectively
+ * incomplete and should be re-run on dep-fill / manual incomplete-mode runs.
+ */
+export function areOutputsFilled(group: WorkflowGroup, row: TableRow): boolean {
+  if (group.outputs.length === 0) return true
+  for (const o of group.outputs) {
+    const v = row.data[o.columnName]
+    if (v === null || v === undefined || v === '') return false
+  }
+  return true
+}
+
+/**
+ * Returns true when every column this group depends on is non-empty on this
+ * row. Workflow output columns count the same as plain columns — the model
+ * is uniform.
+ */
+export function areGroupDepsSatisfied(group: WorkflowGroup, row: TableRow): boolean {
+  const cols = group.dependencies?.columns ?? []
+  for (const colName of cols) {
+    if (isDepValueUnmet(row.data[colName])) return false
+  }
+  return true
+}
+
+export interface UnmetDeps {
+  /** Column names whose value on this row is empty. */
+  columns: string[]
+}
+
+/**
+ * Like `areGroupDepsSatisfied` but returns *which* columns are unmet, so the
+ * UI can render "Waiting on column_a, column_b".
+ */
+export function getUnmetGroupDeps(group: WorkflowGroup, row: TableRow): UnmetDeps {
+  const cols = group.dependencies?.columns ?? []
+  const columns: string[] = []
+  for (const colName of cols) {
+    if (isDepValueUnmet(row.data[colName])) columns.push(colName)
+  }
+  return { columns }
+}
+
+/**
+ * Optimistic mirror of the server's row-update→scheduler cascade: for every
+ * workflow group whose deps were unmet *before* the patch and are satisfied
+ * *after*, OR whose dep column was touched by the patch (the server will
+ * cancel+re-run via `deriveExecClearsForDataPatch` + the in-flight cancel
+ * orchestration), return a new `executions` map with that group flipped to
+ * `pending`. The cell renderer treats `pending` as "Queued".
+ *
+ * Returns `null` when nothing changed, so callers can short-circuit.
+ */
+export function optimisticallyScheduleNewlyEligibleGroups(
+  groups: WorkflowGroup[],
+  beforeRow: TableRow,
+  patch: Partial<RowData>
+): RowExecutions | null {
+  if (groups.length === 0) return null
+
+  const afterRow: TableRow = {
+    ...beforeRow,
+    data: { ...beforeRow.data, ...patch } as RowData,
+  }
+  const patchedColumns = new Set(Object.keys(patch))
+
+  let next: RowExecutions | null = null
+  let flipped = 0
+  let skipped = 0
+  for (const group of groups) {
+    if (group.autoRun === false) {
+      skipped++
+      continue
+    }
+    if (!areGroupDepsSatisfied(group, afterRow)) {
+      skipped++
+      continue
+    }
+
+    const exec = beforeRow.executions?.[group.id]
+    if (exec?.status === 'pending' && exec.jobId) {
+      skipped++
+      continue
+    }
+
+    const isStaleCompleted = exec?.status === 'completed' && !areOutputsFilled(group, afterRow)
+    const wasSatisfied = areGroupDepsSatisfied(group, beforeRow)
+    const becameSatisfied = !wasSatisfied
+    const isRetryable = exec?.status === 'cancelled' || exec?.status === 'error'
+    // Dep-column touched: the server clears terminal entries + cancels in-
+    // flight downstream groups, so optimistically flip to `pending`
+    // regardless of current exec status (queued/running included — they're
+    // about to be cancelled and re-run).
+    const depTouched = (group.dependencies?.columns ?? []).some((d) => patchedColumns.has(d))
+
+    if (!depTouched && (exec?.status === 'queued' || exec?.status === 'running')) {
+      skipped++
+      continue
+    }
+    if (!becameSatisfied && !isStaleCompleted && !isRetryable && !depTouched && exec) {
+      skipped++
+      continue
+    }
+
+    flipped++
+    if (next === null) next = { ...(beforeRow.executions ?? {}) }
+    const pending: RowExecutionMetadata = {
+      status: 'pending',
+      executionId: exec?.executionId ?? null,
+      jobId: null,
+      workflowId: exec?.workflowId ?? group.workflowId,
+      error: null,
+    }
+    next[group.id] = pending
+  }
+  if (flipped > 0) {
+    logger.debug(`[OptimisticCascade] row=${beforeRow.id} flipped=${flipped} skipped=${skipped}`)
+  }
+  return next
+}

apps/sim/lib/table/dispatcher.ts

@@ -0,0 +1,802 @@
+import { db } from '@sim/db'
+import { tableRowExecutions, tableRunDispatches, userTableRows } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import {
+  and,
+  asc,
+  eq,
+  gt,
+  inArray,
+  isNotNull,
+  ne,
+  notInArray,
+  or,
+  type SQL,
+  sql,
+} from 'drizzle-orm'
+import { getJobQueue } from '@/lib/core/async-jobs/config'
+import { writeWorkflowGroupState } from '@/lib/table/cell-write'
+import { USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants'
+import { isExecCancelledAfter } from '@/lib/table/deps'
+import { appendTableEvent } from '@/lib/table/events'
+import { type DbExecutor, withSeqscanOff } from '@/lib/table/planner'
+import { buildFilterClause } from '@/lib/table/sql'
+import type { Filter, RowExecutionMetadata, RowExecutions, TableRow } from '@/lib/table/types'
+import {
+  buildEnqueueItems,
+  buildPendingRuns,
+  TABLE_CONCURRENCY_LIMIT,
+  toTableRow,
+  type WorkflowGroupCellPayload,
+} from '@/lib/table/workflow-columns'
+
+const logger = createLogger('TableRunDispatcher')
+
+/** Window size matches the cell-execution concurrency cap so one window
+ *  saturates the pool before the next is loaded — yields a row-major
+ *  scan-line crawl (rows 1-20 finish before 21-40 start). */
+const WINDOW_SIZE = TABLE_CONCURRENCY_LIMIT
+
+const ACTIVE_DISPATCH_STATUSES = ['pending', 'dispatching'] as const
+
+export type DispatchStatus = 'pending' | 'dispatching' | 'complete' | 'cancelled'
+export type DispatchMode = 'all' | 'incomplete' | 'new'
+
+export interface DispatchScope {
+  groupIds: string[]
+  rowIds?: string[]
+  /** "Select all matching a filter" — run every row matching this filter (mutually exclusive with
+   *  `rowIds`). Lets the action-bar Play/Refresh target a filtered view without materializing ids. */
+  filter?: Filter
+  /** Select-all scope only: deselected rows the walk skips (mirrors the delete job's exclusion set). */
+  excludeRowIds?: string[]
+}
+
+/**
+ * Optional cap on how much work a dispatch does before it completes. The
+ * discriminated `type` keeps it extensible: only `'rows'` exists today, but a
+ * future `'cells'` / `'cost'` / `'duration'` cap can be added by extending the
+ * union and teaching `dispatcherStep` how to count that unit — no schema or
+ * plumbing change. `max` is the hard ceiling in units of `type`.
+ */
+export interface DispatchLimit {
+  type: 'rows'
+  max: number
+}
+
+export interface DispatchRow {
+  id: string
+  tableId: string
+  workspaceId: string
+  requestId: string
+  mode: DispatchMode
+  scope: DispatchScope
+  status: DispatchStatus
+  cursor: number
+  /** Cap on work before completion; null = unbounded. */
+  limit: DispatchLimit | null
+  /** Units of `limit.type` already consumed (eligible rows dispatched). */
+  processedCount: number
+  isManualRun: boolean
+  /** User who triggered the run (for usage attribution); null for auto-fire. */
+  triggeredByUserId: string | null
+  requestedAt: Date
+}
+
+export type DispatcherStepResult = 'continue' | 'done'
+
+/** Eager bulk clear at click time so the user sees every targeted cell go
+ *  blank/Pending instantly — without it, only the rows the dispatcher has
+ *  reached visibly change, and the rest sit on stale data until the cursor
+ *  walks to them. For `mode: 'incomplete'` we skip rows whose outputs are
+ *  already filled, mirroring the eligibility predicate. */
+export async function bulkClearWorkflowGroupCells(input: {
+  tableId: string
+  groups: Array<{ id: string; outputs: Array<{ columnName: string }> }>
+  rowIds?: string[]
+  /** Select-all scope: deselected rows whose outputs must NOT be wiped. */
+  excludeRowIds?: string[]
+  mode: DispatchMode
+}): Promise<void> {
+  const { tableId, groups, rowIds, excludeRowIds, mode } = input
+  if (groups.length === 0) return
+  // `'new'` mode targets only rows with no prior attempt — nothing to clear.
+  // Pre-existing outputs on any other row must not be wiped by an auto-fire.
+  if (mode === 'new') return
+
+  const groupIds = groups.map((g) => g.id)
+  const rowScope = rowIds && rowIds.length > 0 ? rowIds : null
+  const excluded = !rowScope && excludeRowIds && excludeRowIds.length > 0 ? excludeRowIds : null
+
+  if (mode === 'all') {
+    // Run-all re-runs every targeted group: wipe all their output columns +
+    // executions for the rows in scope. (Prior in-flight runs were already
+    // cancelled by the caller.)
+    const outputCols = Array.from(
+      new Set(groups.flatMap((g) => g.outputs.map((o) => o.columnName)))
+    )
+    let dataExpr: SQL = sql`coalesce(${userTableRows.data}, '{}'::jsonb)`
+    for (const col of outputCols) dataExpr = sql`(${dataExpr}) - ${col}::text`
+    const filters: SQL[] = [eq(userTableRows.tableId, tableId)]
+    if (rowScope) filters.push(inArray(userTableRows.id, rowScope))
+    if (excluded) filters.push(notInArray(userTableRows.id, excluded))
+
+    await db.transaction(async (trx) => {
+      await trx
+        .update(userTableRows)
+        .set({ data: dataExpr, updatedAt: new Date() })
+        .where(and(...filters))
+      const execFilters: SQL[] = [
+        eq(tableRowExecutions.tableId, tableId),
+        inArray(tableRowExecutions.groupId, groupIds),
+      ]
+      if (rowScope) execFilters.push(inArray(tableRowExecutions.rowId, rowScope))
+      if (excluded) execFilters.push(notInArray(tableRowExecutions.rowId, excluded))
+      await trx.delete(tableRowExecutions).where(and(...execFilters))
+    })
+    return
+  }
+
+  // `incomplete`: clear per-group, not per-row. Only groups that are
+  // re-runnable (`error` / `cancelled`) get their output columns + exec wiped;
+  // `completed` and in-flight groups are left fully intact. A row-level "all
+  // filled" check would otherwise wipe a completed group's data + exec just
+  // because a *sibling* group on the same row is incomplete, re-running the
+  // completed one. (`never-run` groups have no exec/output to clear — the
+  // dispatcher runs them via eligibility.)
+  await db.transaction(async (trx) => {
+    for (const group of groups) {
+      const reRunnable = sql`EXISTS (
+        SELECT 1 FROM ${tableRowExecutions} re
+        WHERE re.row_id = ${userTableRows.id}
+          AND re.group_id = ${group.id}
+          AND re.status IN ('error', 'cancelled')
+      )`
+      const filters: SQL[] = [eq(userTableRows.tableId, tableId), reRunnable]
+      if (rowScope) filters.push(inArray(userTableRows.id, rowScope))
+      if (excluded) filters.push(notInArray(userTableRows.id, excluded))
+
+      let dataExpr: SQL = sql`coalesce(${userTableRows.data}, '{}'::jsonb)`
+      for (const out of group.outputs) dataExpr = sql`(${dataExpr}) - ${out.columnName}::text`
+      await trx
+        .update(userTableRows)
+        .set({ data: dataExpr, updatedAt: new Date() })
+        .where(and(...filters))
+
+      const execFilters: SQL[] = [
+        eq(tableRowExecutions.tableId, tableId),
+        eq(tableRowExecutions.groupId, group.id),
+        sql`${tableRowExecutions.status} IN ('error', 'cancelled')`,
+      ]
+      if (rowScope) execFilters.push(inArray(tableRowExecutions.rowId, rowScope))
+      if (excluded) execFilters.push(notInArray(tableRowExecutions.rowId, excluded))
+      await trx.delete(tableRowExecutions).where(and(...execFilters))
+    }
+  })
+}
+
+export async function insertDispatch(input: {
+  tableId: string
+  workspaceId: string
+  requestId: string
+  mode: DispatchMode
+  scope: DispatchScope
+  limit?: DispatchLimit | null
+  isManualRun: boolean
+  triggeredByUserId?: string | null
+}): Promise<string> {
+  const id = `tdsp_${generateId().replace(/-/g, '')}`
+  await db.insert(tableRunDispatches).values({
+    id,
+    tableId: input.tableId,
+    workspaceId: input.workspaceId,
+    requestId: input.requestId,
+    mode: input.mode,
+    scope: input.scope,
+    limit: input.limit ?? null,
+    status: 'pending',
+    // -1 = "haven't started." First window's filter `position > -1` matches
+    // position 0; subsequent iterations advance to `lastPosition` which then
+    // correctly excludes already-processed rows.
+    cursor: -1,
+    isManualRun: input.isManualRun,
+    triggeredByUserId: input.triggeredByUserId ?? null,
+  })
+  return id
+}
+
+/** Counts in-flight cells (queued / running / pending) per row across the
+ *  entire table — the authoritative source for the "X running" badge and the
+ *  per-row gutter Run/Stop button. All three statuses are user-cancellable, so
+ *  the gutter must surface Stop whenever any of them are present (else clicking
+ *  Play during the queued window would re-run an already-queued cell).
+ *
+ *  Excludes orphan pre-stamps — `pending` rows with no `executionId` — which
+ *  are dead placeholders left when a dispatcher loop wrote the stamp but no
+ *  cell-task ever picked it up (lock contention, queue failure, crash). The
+ *  cell already shows its prior value and `classifyEligibility` treats these as
+ *  claimable, so counting them stuck the "X running" badge above zero forever
+ *  even though nothing was running. Same `executionId == null` test used by
+ *  {@link classifyEligibility} / {@link pickNextEligibleGroupForRow}.
+ *
+ *  Hits the `(table_id, status)` partial index on table_row_executions. */
+export async function countRunningCells(
+  tableId: string,
+  opts?: { includeUnclaimedPreStamps?: boolean }
+): Promise<{ byRowId: Record<string, number>; hasRunning: boolean }> {
+  // `pending` + null-executionId rows are unclaimed pre-stamps. With an active
+  // dispatch they're real queued work (include); with none they're abandoned
+  // orphans that would pin the badge above zero forever (exclude).
+  const excludeOrphanPreStamps = !opts?.includeUnclaimedPreStamps
+  const rows = await db
+    .select({
+      rowId: tableRowExecutions.rowId,
+      runningCount: sql<number>`count(*)::int`,
+      // Cells actually claimed by a worker — drives the header's
+      // "Queueing" vs "N running" label table-wide (the client can only see
+      // claims on loaded rows; a long run's active window scrolls past them).
+      claimedCount: sql<number>`count(*) FILTER (WHERE ${tableRowExecutions.status} = 'running')::int`,
+    })
+    .from(tableRowExecutions)
+    .where(
+      and(
+        eq(tableRowExecutions.tableId, tableId),
+        inArray(tableRowExecutions.status, ['queued', 'running', 'pending']),
+        excludeOrphanPreStamps
+          ? or(ne(tableRowExecutions.status, 'pending'), isNotNull(tableRowExecutions.executionId))
+          : undefined
+      )
+    )
+    .groupBy(tableRowExecutions.rowId)
+  const byRowId: Record<string, number> = {}
+  let hasRunning = false
+  for (const r of rows) {
+    if (r.runningCount > 0) byRowId[r.rowId] = r.runningCount
+    if (r.claimedCount > 0) hasRunning = true
+  }
+  return { byRowId, hasRunning }
+}
+
+/** Read every dispatch on a table whose status is still `pending` or
+ *  `dispatching`. Drives the client-side "about to run" overlay: rows in an
+ *  active dispatch's scope ahead of its cursor are rendered as queued even
+ *  before the dispatcher has reached them, so refresh during a long Run-all
+ *  doesn't lose the queued indicators. */
+export async function listActiveDispatches(tableId: string): Promise<DispatchRow[]> {
+  const rows = await db
+    .select()
+    .from(tableRunDispatches)
+    .where(
+      and(
+        eq(tableRunDispatches.tableId, tableId),
+        inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES])
+      )
+    )
+  return rows.map((row) => ({
+    id: row.id,
+    tableId: row.tableId,
+    workspaceId: row.workspaceId,
+    requestId: row.requestId,
+    mode: row.mode as DispatchMode,
+    scope: row.scope as DispatchScope,
+    status: row.status as DispatchStatus,
+    cursor: row.cursor,
+    limit: (row.limit as DispatchLimit | null) ?? null,
+    processedCount: row.processedCount,
+    isManualRun: row.isManualRun,
+    triggeredByUserId: row.triggeredByUserId,
+    requestedAt: row.requestedAt,
+  }))
+}
+
+export async function readDispatch(dispatchId: string): Promise<DispatchRow | null> {
+  const [row] = await db
+    .select()
+    .from(tableRunDispatches)
+    .where(eq(tableRunDispatches.id, dispatchId))
+    .limit(1)
+  if (!row) return null
+  return {
+    id: row.id,
+    tableId: row.tableId,
+    workspaceId: row.workspaceId,
+    requestId: row.requestId,
+    mode: row.mode as DispatchMode,
+    scope: row.scope as DispatchScope,
+    status: row.status as DispatchStatus,
+    cursor: row.cursor,
+    limit: (row.limit as DispatchLimit | null) ?? null,
+    processedCount: row.processedCount,
+    isManualRun: row.isManualRun,
+    triggeredByUserId: row.triggeredByUserId,
+    requestedAt: row.requestedAt,
+  }
+}
+
+/** Drive `dispatcherStep` to completion. Shared between the trigger.dev task
+ *  wrapper (`tableRunDispatcherTask`) and the in-process inline path so both
+ *  runtimes use identical loop semantics + error logging. */
+export async function runDispatcherToCompletion(dispatchId: string): Promise<void> {
+  while ((await dispatcherStep(dispatchId)) === 'continue') {}
+}
+
+/** Run one window of the dispatcher state machine. Caller re-invokes (via the
+ *  trigger.dev task wrapper) until the returned status is `'done'`. */
+export async function dispatcherStep(dispatchId: string): Promise<DispatcherStepResult> {
+  const dispatch = await readDispatch(dispatchId)
+  if (!dispatch) {
+    logger.warn(`[${dispatchId}] dispatch row missing — aborting`)
+    return 'done'
+  }
+  if (dispatch.status === 'cancelled' || dispatch.status === 'complete') return 'done'
+
+  const { getTableById } = await import('@/lib/table/service')
+  const table = await getTableById(dispatch.tableId)
+  if (!table) {
+    logger.warn(`[${dispatchId}] table ${dispatch.tableId} missing — completing dispatch`)
+    await markDispatchComplete(dispatchId)
+    return 'done'
+  }
+
+  const allGroups = table.schema.workflowGroups ?? []
+  const targetGroups = allGroups.filter((g) => dispatch.scope.groupIds.includes(g.id))
+  if (targetGroups.length === 0) {
+    await markDispatchComplete(dispatchId)
+    return 'done'
+  }
+
+  // First iteration: just transition pending → dispatching. The bulk clear
+  // ran synchronously in `runWorkflowColumn` before this task fired, so the
+  // user already saw the column flip to empty/Pending before any cell
+  // started enqueueing.
+  if (dispatch.status === 'pending') {
+    await db
+      .update(tableRunDispatches)
+      .set({ status: 'dispatching' })
+      .where(eq(tableRunDispatches.id, dispatchId))
+    // Announce the dispatch the moment it starts — before the first window's
+    // cells finish. Without this, auto-fired and capped dispatches (no client-
+    // side optimistic seed) emit their first `dispatch` event only after window
+    // 1 completes, so the "X running" / Stop-all control stays hidden while a
+    // long first window runs. The client refetches the run-state count on this.
+    await appendTableEvent({
+      kind: 'dispatch',
+      tableId: dispatch.tableId,
+      dispatchId,
+      status: 'dispatching',
+      scope: dispatch.scope,
+      cursor: dispatch.cursor,
+      mode: dispatch.mode,
+      isManualRun: dispatch.isManualRun,
+      ...(dispatch.limit ? { limit: dispatch.limit } : {}),
+    })
+  }
+
+  const filters = [
+    eq(userTableRows.tableId, dispatch.tableId),
+    gt(userTableRows.position, dispatch.cursor),
+  ]
+  let hasJsonbFilter = false
+  if (dispatch.scope.rowIds && dispatch.scope.rowIds.length > 0) {
+    filters.push(inArray(userTableRows.id, dispatch.scope.rowIds))
+  } else if (dispatch.scope.filter) {
+    // "Select all under a filter": walk only the matching rows. Same cursor/window mechanism —
+    // non-matching rows are simply never selected, like mode eligibility.
+    const filterClause = buildFilterClause(
+      dispatch.scope.filter,
+      USER_TABLE_ROWS_SQL_NAME,
+      table.schema.columns
+    )
+    if (filterClause) {
+      filters.push(filterClause)
+      hasJsonbFilter = true
+    }
+  }
+  if (!dispatch.scope.rowIds?.length && dispatch.scope.excludeRowIds?.length) {
+    filters.push(notInArray(userTableRows.id, dispatch.scope.excludeRowIds))
+  }
+  // `'new'` mode targets only rows whose targeted groups haven't been
+  // attempted. Exclude a row only when EVERY targeted group already has a
+  // sidecar entry — if any one is missing, the row still has work to do
+  // and per-group JS filtering in `classifyEligibility` handles the rest.
+  if (dispatch.mode === 'new' && dispatch.scope.groupIds.length > 0) {
+    const gids = dispatch.scope.groupIds
+    filters.push(
+      sql`NOT EXISTS (
+        SELECT 1 FROM ${tableRowExecutions} re
+        WHERE re.row_id = ${userTableRows.id}
+          AND re.group_id = ANY(ARRAY[${sql.join(
+            gids.map((gid) => sql`${gid}`),
+            sql`, `
+          )}]::text[])
+        GROUP BY re.row_id
+        HAVING count(DISTINCT re.group_id) = ${gids.length}
+      )`
+    )
+  }
+
+  const windowQuery = (executor: DbExecutor) =>
+    executor
+      .select()
+      .from(userTableRows)
+      .where(and(...filters))
+      .orderBy(asc(userTableRows.position))
+      .limit(WINDOW_SIZE)
+  // Filtered scopes carry a jsonb predicate the planner can't estimate — left alone it
+  // seq-scans the whole shared relation per window; keep it on the tenant's position index.
+  const chunk = hasJsonbFilter
+    ? await withSeqscanOff(async (trx) => windowQuery(trx))
+    : await windowQuery(db)
+
+  if (chunk.length === 0) {
+    await markDispatchComplete(dispatchId)
+    await appendTableEvent({
+      kind: 'dispatch',
+      tableId: dispatch.tableId,
+      dispatchId,
+      status: 'complete',
+      scope: dispatch.scope,
+      cursor: dispatch.cursor,
+      mode: dispatch.mode,
+      isManualRun: dispatch.isManualRun,
+    })
+    return 'done'
+  }
+
+  // Pre-fetch executions for the chunk so per-row eligibility doesn't fan
+  // out into one query per row. Returns `Map<rowId, RowExecutions>`.
+  const chunkRowIds = chunk.map((r) => r.id)
+  const execRows = await db
+    .select()
+    .from(tableRowExecutions)
+    .where(inArray(tableRowExecutions.rowId, chunkRowIds))
+  const executionsByRow = new Map<string, RowExecutions>()
+  for (const r of execRows) {
+    const existing = executionsByRow.get(r.rowId) ?? {}
+    const meta: RowExecutionMetadata = {
+      status: r.status as RowExecutionMetadata['status'],
+      executionId: r.executionId ?? null,
+      jobId: r.jobId ?? null,
+      workflowId: r.workflowId,
+      error: r.error ?? null,
+      ...(r.runningBlockIds && r.runningBlockIds.length > 0
+        ? { runningBlockIds: r.runningBlockIds }
+        : {}),
+      ...(r.blockErrors && Object.keys(r.blockErrors as Record<string, string>).length > 0
+        ? { blockErrors: r.blockErrors as Record<string, string> }
+        : {}),
+      ...(r.cancelledAt ? { cancelledAt: r.cancelledAt.toISOString() } : {}),
+    }
+    existing[r.groupId] = meta
+    executionsByRow.set(r.rowId, existing)
+  }
+
+  // Strip rows the user cancelled mid-cascade (post-dispatch tombstones)
+  // before running the shared eligibility filter — `buildPendingRuns`
+  // doesn't know about the per-dispatch cancel tombstone.
+  const tombstoneFiltered: TableRow[] = []
+  for (const r of chunk) {
+    const tableRow = toTableRow(r, executionsByRow.get(r.id) ?? {})
+    const tombstoned = dispatch.scope.groupIds.some((gid) =>
+      isExecCancelledAfter(tableRow.executions?.[gid], dispatch.requestedAt)
+    )
+    if (!tombstoned) tombstoneFiltered.push(tableRow)
+  }
+
+  const pendingRuns = buildPendingRuns(table, tombstoneFiltered, {
+    isManualRun: dispatch.isManualRun,
+    groupIds: dispatch.scope.groupIds,
+    mode: dispatch.mode,
+  }).map((p) => ({ ...p, dispatchId, triggeredByUserId: dispatch.triggeredByUserId ?? undefined }))
+
+  // Cursor advances to the last position in this chunk regardless of
+  // eligibility — otherwise a window full of skipped cells loops forever.
+  const lastPosition = chunk[chunk.length - 1].position
+
+  // Apply the dispatch's row cap. With a `rows` limit, only the first
+  // `remaining` distinct eligible rows in this window are dispatched and the
+  // dispatch completes once the budget is spent. buildPendingRuns emits each
+  // row's groups consecutively in ascending position, so collecting distinct
+  // rowIds until the budget fills picks the lowest-position rows.
+  let windowRuns = pendingRuns
+  let dispatchedRows = 0
+  let budgetExhausted = false
+  if (dispatch.limit?.type === 'rows') {
+    const remaining = dispatch.limit.max - dispatch.processedCount
+    if (remaining <= 0) {
+      await completeDispatch(dispatch, lastPosition)
+      return 'done'
+    }
+    const allowedRowIds = new Set<string>()
+    for (const p of pendingRuns) {
+      if (allowedRowIds.has(p.rowId)) continue
+      if (allowedRowIds.size >= remaining) break
+      allowedRowIds.add(p.rowId)
+    }
+    windowRuns = pendingRuns.filter((p) => allowedRowIds.has(p.rowId))
+    dispatchedRows = allowedRowIds.size
+    budgetExhausted = dispatch.processedCount + dispatchedRows >= dispatch.limit.max
+  }
+
+  if (windowRuns.length > 0) {
+    await stampQueuedForBatch(windowRuns)
+
+    // Backend-agnostic batch dispatch: trigger.dev wraps `batchTriggerAndWait`
+    // (CRIU-checkpointed wait); database backend calls the cell-task runner
+    // directly via Promise.all (skips async_jobs since we're awaiting in-
+    // process anyway). Either way the parent dispatcher blocks until every
+    // cell in the window terminates — bounds queue depth at WINDOW_SIZE.
+    const items = await buildEnqueueItems(windowRuns)
+    const queue = await getJobQueue()
+    try {
+      await queue.batchEnqueueAndWait('workflow-group-cell', items)
+    } catch (err) {
+      logger.error(`[${dispatchId}] batch dispatch failed`, {
+        error: toError(err).message,
+      })
+      // These rows never actually ran, so they must not consume the row cap —
+      // otherwise a transient failure on the only window of a `max: N` run would
+      // exhaust the budget and complete the dispatch with zero rows started.
+      // The cursor still advances past the window (cells are flipped to a
+      // re-runnable `error` below), so later windows fulfill the remaining cap.
+      dispatchedRows = 0
+      budgetExhausted = false
+      // Cursor advances past this window, so flip the un-claimed pre-stamps to
+      // terminal `error` (+ SSE) — visible, not stuck pending, re-runnable.
+      const failedAt = new Date()
+      await Promise.allSettled(
+        windowRuns.map(async (p) => {
+          const updated = await db
+            .update(tableRowExecutions)
+            .set({ status: 'error', error: 'Failed to enqueue run', updatedAt: failedAt })
+            .where(
+              and(
+                eq(tableRowExecutions.rowId, p.rowId),
+                eq(tableRowExecutions.groupId, p.groupId),
+                eq(tableRowExecutions.status, 'pending'),
+                sql`${tableRowExecutions.executionId} IS NULL`
+              )
+            )
+            .returning({ rowId: tableRowExecutions.rowId })
+          if (updated.length === 0) return
+          await appendTableEvent({
+            kind: 'cell',
+            tableId: dispatch.tableId,
+            rowId: p.rowId,
+            groupId: p.groupId,
+            status: 'error',
+            executionId: null,
+            jobId: null,
+            error: 'Failed to enqueue run',
+          })
+        })
+      )
+    }
+  }
+
+  if (dispatchedRows > 0) await incrementProcessedCount(dispatchId, dispatchedRows)
+
+  // Budget spent → complete now rather than crawling the rest of the table.
+  if (budgetExhausted) {
+    await completeDispatch(dispatch, lastPosition)
+    return 'done'
+  }
+
+  // A cell may have halted the dispatch mid-window (e.g. usage limit calls
+  // completeDispatchIfActive). Re-read before emitting the per-window
+  // `dispatching` event — otherwise that stale event arrives after the client
+  // already dropped the dispatch and re-adds it, flickering "X running" back.
+  const current = await readDispatch(dispatchId)
+  if (!current || current.status === 'cancelled' || current.status === 'complete') return 'done'
+
+  await Promise.all([
+    advanceCursor(dispatchId, lastPosition),
+    appendTableEvent({
+      kind: 'dispatch',
+      tableId: dispatch.tableId,
+      dispatchId,
+      status: 'dispatching',
+      scope: dispatch.scope,
+      cursor: lastPosition,
+      mode: dispatch.mode,
+      isManualRun: dispatch.isManualRun,
+      ...(dispatch.limit ? { limit: dispatch.limit } : {}),
+    }),
+  ])
+
+  return 'continue'
+}
+
+/** Bump the processed-row counter so a row cap survives across the
+ *  checkpointed waits between windows. */
+async function incrementProcessedCount(dispatchId: string, delta: number): Promise<void> {
+  await db
+    .update(tableRunDispatches)
+    .set({ processedCount: sql`${tableRunDispatches.processedCount} + ${delta}` })
+    .where(eq(tableRunDispatches.id, dispatchId))
+}
+
+/** Mark a dispatch complete and emit the terminal SSE so the client overlay
+ *  clears. Shared by the row-cap exhaustion path. */
+async function completeDispatch(dispatch: DispatchRow, cursor: number): Promise<void> {
+  await markDispatchComplete(dispatch.id)
+  await appendTableEvent({
+    kind: 'dispatch',
+    tableId: dispatch.tableId,
+    dispatchId: dispatch.id,
+    status: 'complete',
+    scope: dispatch.scope,
+    cursor,
+    mode: dispatch.mode,
+    isManualRun: dispatch.isManualRun,
+    ...(dispatch.limit ? { limit: dispatch.limit } : {}),
+  })
+}
+
+/** Pre-batch stamp: write each targeted cell as `pending` (no executionId)
+ *  before firing the batch so the renderer shows the cell as in-flight
+ *  immediately. The cell-task overwrites with `running` (and its own
+ *  executionId) once it acquires the row's cascade lock — if another
+ *  cell-task already holds the lock, this task bails and the pending stamp
+ *  is later reconciled by whoever owns the cascade. */
+async function stampQueuedForBatch(pendingRuns: WorkflowGroupCellPayload[]): Promise<void> {
+  await Promise.allSettled(
+    pendingRuns.map((runOpts) =>
+      writeWorkflowGroupState(runOpts, {
+        executionState: {
+          status: 'pending',
+          executionId: null,
+          jobId: null,
+          workflowId: runOpts.workflowId,
+          error: null,
+        },
+      })
+    )
+  )
+}
+
+async function advanceCursor(dispatchId: string, newCursor: number): Promise<void> {
+  await db
+    .update(tableRunDispatches)
+    .set({ cursor: newCursor })
+    .where(eq(tableRunDispatches.id, dispatchId))
+}
+
+export async function markDispatchComplete(dispatchId: string): Promise<void> {
+  await db
+    .update(tableRunDispatches)
+    .set({ status: 'complete', completedAt: new Date() })
+    .where(eq(tableRunDispatches.id, dispatchId))
+}
+
+/** Cancel one dispatch by id (if still active) and emit the terminal SSE so
+ *  the client overlay clears. Used when `runWorkflowColumn` fails between
+ *  inserting its dispatch row and firing the dispatcher — without this the
+ *  orphaned `pending` row would pin the "about to run" overlay forever. */
+export async function cancelDispatchById(dispatchId: string): Promise<void> {
+  const [row] = await db
+    .update(tableRunDispatches)
+    .set({ status: 'cancelled', cancelledAt: new Date() })
+    .where(
+      and(
+        eq(tableRunDispatches.id, dispatchId),
+        inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES])
+      )
+    )
+    .returning()
+  if (!row) return
+  await appendTableEvent({
+    kind: 'dispatch',
+    tableId: row.tableId,
+    dispatchId: row.id,
+    status: 'cancelled',
+    scope: row.scope as DispatchScope,
+    cursor: row.cursor,
+    mode: row.mode as DispatchMode,
+    isManualRun: row.isManualRun,
+  })
+}
+
+/** Complete a dispatch only if it's still active, returning whether THIS call
+ *  performed the transition. Lets concurrent cells that all hit a hard stop
+ *  (e.g. usage limit) elect a single owner — only the winner emits the
+ *  user-facing event, instead of one toast per in-flight cell. */
+export async function completeDispatchIfActive(dispatchId: string): Promise<boolean> {
+  const transitioned = await db
+    .update(tableRunDispatches)
+    .set({ status: 'complete', completedAt: new Date() })
+    .where(
+      and(
+        eq(tableRunDispatches.id, dispatchId),
+        inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES])
+      )
+    )
+    .returning({ id: tableRunDispatches.id })
+  return transitioned.length > 0
+}
+
+export async function markDispatchCancelled(dispatchId: string): Promise<void> {
+  await db
+    .update(tableRunDispatches)
+    .set({ status: 'cancelled', cancelledAt: new Date() })
+    .where(
+      and(
+        eq(tableRunDispatches.id, dispatchId),
+        inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES])
+      )
+    )
+}
+
+/** Mark every active dispatch on this table as cancelled. Single atomic
+ *  UPDATE so the dispatcher's next iteration observes the cancel. Returns the
+ *  dispatches that were cancelled so the caller can emit per-dispatch SSE
+ *  events — without those the client's overlay would hang on "queued" until
+ *  the next refresh. Pass `scopeFilter` to cancel only dispatches whose scope
+ *  is that exact filter (a filtered "select all" Stop must not halt
+ *  whole-table or differently-filtered runs). Pass `spareExcludedRowIds`
+ *  (select-all-minus-deselections Stop) to spare row-scoped dispatches whose
+ *  rows are ALL deselected — that work wasn't in the stopped selection. Pass
+ *  `spareDispatchId` when the caller is a manual run cancelling *prior* work:
+ *  its own dispatch row is already inserted (so a concurrent Stop-all has
+ *  something to cancel) and must not cancel itself. */
+export async function markActiveDispatchesCancelled(
+  tableId: string,
+  opts?: { scopeFilter?: Filter; spareExcludedRowIds?: string[]; spareDispatchId?: string }
+): Promise<DispatchRow[]> {
+  const { scopeFilter, spareExcludedRowIds, spareDispatchId } = opts ?? {}
+  const cancelled = await db
+    .update(tableRunDispatches)
+    .set({ status: 'cancelled', cancelledAt: new Date() })
+    .where(
+      and(
+        eq(tableRunDispatches.tableId, tableId),
+        inArray(tableRunDispatches.status, [...ACTIVE_DISPATCH_STATUSES]),
+        spareDispatchId ? ne(tableRunDispatches.id, spareDispatchId) : undefined,
+        scopeFilter
+          ? sql`${tableRunDispatches.scope}->'filter' = ${JSON.stringify(scopeFilter)}::jsonb`
+          : undefined,
+        // coalesce(false): table-wide dispatches have no scope.rowIds (NULL <@ x
+        // is NULL) and must still cancel.
+        spareExcludedRowIds && spareExcludedRowIds.length > 0
+          ? sql`NOT coalesce(
+              ${tableRunDispatches.scope}->'rowIds' <@ ${JSON.stringify(spareExcludedRowIds)}::jsonb
+                AND jsonb_array_length(${tableRunDispatches.scope}->'rowIds') > 0,
+              false
+            )`
+          : undefined
+      )
+    )
+    .returning()
+  const dispatches = cancelled.map((row) => ({
+    id: row.id,
+    tableId: row.tableId,
+    workspaceId: row.workspaceId,
+    requestId: row.requestId,
+    mode: row.mode as DispatchMode,
+    scope: row.scope as DispatchScope,
+    status: 'cancelled' as DispatchStatus,
+    cursor: row.cursor,
+    limit: (row.limit as DispatchLimit | null) ?? null,
+    processedCount: row.processedCount,
+    isManualRun: row.isManualRun,
+    triggeredByUserId: row.triggeredByUserId,
+    requestedAt: row.requestedAt,
+  }))
+  await Promise.all(
+    dispatches.map((d) =>
+      appendTableEvent({
+        kind: 'dispatch',
+        tableId: d.tableId,
+        dispatchId: d.id,
+        status: 'cancelled',
+        scope: d.scope,
+        cursor: d.cursor,
+        mode: d.mode,
+        isManualRun: d.isManualRun,
+      })
+    )
+  )
+  return dispatches
+}

apps/sim/lib/table/events.test.ts

@@ -0,0 +1,78 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/redis', () => ({
+  getRedisClient: () => null,
+}))
+
+vi.mock('@/lib/core/config/env', () => ({
+  env: { REDIS_URL: undefined },
+}))
+
+import type { TableEvent } from '@/lib/table/events'
+import { appendTableEvent, getLatestTableEventId, readTableEventsSince } from '@/lib/table/events'
+
+/** Module-level memory buffer can't be reset without vi.resetModules — use a
+ *  unique tableId per test to avoid cross-test bleed. */
+let seq = 0
+function uniqueTableId(): string {
+  seq++
+  return `table-events-test-${seq}`
+}
+
+function cellEvent(tableId: string): TableEvent {
+  return {
+    kind: 'cell',
+    tableId,
+    rowId: 'row-1',
+    groupId: 'group-1',
+    status: 'running',
+  }
+}
+
+describe('getLatestTableEventId (memory buffer)', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('returns 0 for a table with no events, without allocating a stream', async () => {
+    const tableId = uniqueTableId()
+    expect(await getLatestTableEventId(tableId)).toBe(0)
+    // A pure read must not have created a buffer: appending afterwards still
+    // starts the sequence at 1.
+    const entry = await appendTableEvent(cellEvent(tableId))
+    expect(entry?.eventId).toBe(1)
+  })
+
+  it('returns the latest assigned eventId after appends', async () => {
+    const tableId = uniqueTableId()
+    await appendTableEvent(cellEvent(tableId))
+    const second = await appendTableEvent(cellEvent(tableId))
+    expect(second?.eventId).toBe(2)
+    expect(await getLatestTableEventId(tableId)).toBe(2)
+  })
+
+  it('tailing from the latest id yields no replayed events', async () => {
+    const tableId = uniqueTableId()
+    await appendTableEvent(cellEvent(tableId))
+    await appendTableEvent(cellEvent(tableId))
+    const latest = await getLatestTableEventId(tableId)
+    const result = await readTableEventsSince(tableId, latest)
+    expect(result).toEqual({ status: 'ok', events: [] })
+  })
+
+  it('a subsequent append is visible to a reader tailing from the prior latest', async () => {
+    const tableId = uniqueTableId()
+    await appendTableEvent(cellEvent(tableId))
+    const latest = await getLatestTableEventId(tableId)
+    await appendTableEvent(cellEvent(tableId))
+    const result = await readTableEventsSince(tableId, latest)
+    expect(result.status).toBe('ok')
+    if (result.status === 'ok') {
+      expect(result.events).toHaveLength(1)
+      expect(result.events[0].eventId).toBe(latest + 1)
+    }
+  })
+})

apps/sim/lib/table/events.ts

@@ -0,0 +1,367 @@
+/**
+ * Per-table event buffer for live cell-state updates.
+ *
+ * The grid subscribes to a per-table SSE stream and patches its React Query
+ * cache as events arrive. This buffer is the durable mid-tier between the
+ * cell-write paths (`writeWorkflowGroupState`, `cancelWorkflowGroupRuns`) and
+ * the SSE consumers — every status transition appends here with a monotonic
+ * eventId; SSE clients resume on reconnect via `?from=<lastEventId>` and the
+ * server replays from this buffer.
+ *
+ * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of
+ * complexity tables don't need: no per-execution lifecycle, no id reservation
+ * batching, no write-queue serialization. Tables are always-on; cell writes
+ * are sparse and independent.
+ */
+
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { env } from '@/lib/core/config/env'
+import { getRedisClient } from '@/lib/core/config/redis'
+
+const logger = createLogger('TableEventBuffer')
+
+const REDIS_PREFIX = 'table:stream:'
+export const TABLE_EVENT_TTL_SECONDS = 60 * 60 // 1 hour
+export const TABLE_EVENT_CAP = 5000
+/** Max events returned by a single read; the SSE route drains in chunks. */
+export const TABLE_EVENT_READ_CHUNK = 500
+
+/**
+ * Atomic append: INCR the seq counter to mint a new eventId, build the entry
+ * JSON inline, ZADD it, refresh TTL on events + seq + meta, trim to cap, then
+ * write the resulting earliestEventId to meta. Single round-trip per event.
+ * Without atomicity a slow reader could observe the trim before the meta
+ * update and miss the prune signal.
+ *
+ * KEYS: [events, seq, meta]
+ * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix]
+ *   The new eventId is spliced between prefix/suffix to form the entry JSON.
+ *   Returns the new eventId.
+ */
+const APPEND_EVENT_SCRIPT = `
+local eventId = redis.call('INCR', KEYS[2])
+local entry = ARGV[4] .. eventId .. ARGV[5]
+redis.call('ZADD', KEYS[1], eventId, entry)
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1]))
+redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1]))
+redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1)
+local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+if oldest[2] then
+  redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3])
+  redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1]))
+end
+return eventId
+`
+
+function getEventsKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:events`
+}
+
+function getSeqKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:seq`
+}
+
+function getMetaKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:meta`
+}
+
+export type TableCellStatus = 'pending' | 'queued' | 'running' | 'completed' | 'cancelled' | 'error'
+
+export type TableDispatchStatus = 'pending' | 'dispatching' | 'complete' | 'cancelled'
+
+export type TableEvent =
+  | {
+      kind: 'cell'
+      tableId: string
+      rowId: string
+      groupId: string
+      status: TableCellStatus
+      executionId: string | null
+      jobId: string | null
+      error: string | null
+      /**
+       * Present when this transition wrote new output values; absent on
+       * pure-status transitions (queued, running, cancelled). The publisher
+       * already has these in hand from the same updateRow call that wrote DB.
+       */
+      outputs?: Record<string, unknown>
+      /**
+       * Block-level metadata the renderer reads to distinguish "running" (some
+       * block actively executing) from "pending-upstream" (run started but this
+       * column's block hasn't fired yet). The worker fills these on partial
+       * writes; without them the cell stays on the amber Pending pill.
+       */
+      runningBlockIds?: string[]
+      blockErrors?: Record<string, string>
+    }
+  | {
+      /** Dispatcher status signal emitted by `dispatcherStep` and the cancel
+       *  path. Drives the client-side "about to run" overlay for rows the
+       *  dispatcher hasn't reached yet. `scope` + `cursor` + `mode` +
+       *  `isManualRun` are carried on every transition so the client can
+       *  upsert without refetching the dispatches list. */
+      kind: 'dispatch'
+      tableId: string
+      dispatchId: string
+      status: TableDispatchStatus
+      scope?: { groupIds: string[]; rowIds?: string[] }
+      cursor?: number
+      mode?: 'all' | 'incomplete' | 'new'
+      isManualRun?: boolean
+      /** Present when the run is capped — carried so the client overlay can
+       *  skip capped dispatches (see `resolveCellExec`). */
+      limit?: { type: 'rows'; max: number }
+    }
+  | {
+      /** Async background-job progress. Import and delete workers emit `running`
+       *  ticks as batches commit, then a terminal `ready`/`failed`/`canceled`.
+       *  `type` discriminates the work. The client reveals hidden import rows on
+       *  `ready`, and on a delete `failed`/`canceled` restores optimistically
+       *  hidden rows. See `import-runner.ts` / `delete-runner.ts`. */
+      kind: 'job'
+      tableId: string
+      jobId: string
+      type: 'import' | 'delete' | 'export' | 'backfill' | 'update'
+      status: 'running' | 'ready' | 'failed' | 'canceled'
+      /** Rows processed so far (running) or in total (ready). */
+      progress?: number
+      /** Byte-based completion percent (0–100) — exact and monotonic, for the determinate bar. */
+      percent?: number
+      error?: string
+    }
+  | {
+      /** A dispatch was stopped because the billed account is over its usage
+       *  limit. The client surfaces an upgrade prompt and redirects to billing.
+       *  The dispatch is halted via `markDispatchComplete` and the blocked
+       *  cells' pre-stamps are cleared so they revert to un-run. `dispatchId`
+       *  is absent for cascade/auto-fire payloads with no owning dispatch. */
+      kind: 'usageLimitReached'
+      tableId: string
+      dispatchId?: string
+      message: string
+    }
+
+export interface TableEventEntry {
+  eventId: number
+  tableId: string
+  event: TableEvent
+}
+
+export type TableEventsReadResult =
+  | { status: 'ok'; events: TableEventEntry[] }
+  | { status: 'pruned'; earliestEventId: number | undefined }
+  | { status: 'unavailable'; error: string }
+
+/** In-memory fallback for dev/tests when Redis isn't configured. */
+interface MemoryTableStream {
+  events: TableEventEntry[]
+  earliestEventId?: number
+  nextEventId: number
+  expiresAt: number
+}
+
+const memoryTableStreams = new Map<string, MemoryTableStream>()
+
+function canUseMemoryBuffer(): boolean {
+  return typeof window === 'undefined' && !env.REDIS_URL
+}
+
+function pruneExpiredMemoryStreams(now = Date.now()): void {
+  for (const [tableId, stream] of memoryTableStreams) {
+    if (stream.expiresAt <= now) {
+      memoryTableStreams.delete(tableId)
+    }
+  }
+}
+
+function getMemoryStream(tableId: string): MemoryTableStream {
+  pruneExpiredMemoryStreams()
+  let stream = memoryTableStreams.get(tableId)
+  if (!stream) {
+    stream = {
+      events: [],
+      nextEventId: 1,
+      expiresAt: Date.now() + TABLE_EVENT_TTL_SECONDS * 1000,
+    }
+    memoryTableStreams.set(tableId, stream)
+  }
+  return stream
+}
+
+function appendMemory(event: TableEvent): TableEventEntry {
+  const stream = getMemoryStream(event.tableId)
+  const entry: TableEventEntry = {
+    eventId: stream.nextEventId++,
+    tableId: event.tableId,
+    event,
+  }
+  stream.events.push(entry)
+  if (stream.events.length > TABLE_EVENT_CAP) {
+    stream.events = stream.events.slice(-TABLE_EVENT_CAP)
+    stream.earliestEventId = stream.events[0]?.eventId
+  }
+  stream.expiresAt = Date.now() + TABLE_EVENT_TTL_SECONDS * 1000
+  return entry
+}
+
+function readMemory(tableId: string, afterEventId: number): TableEventsReadResult {
+  pruneExpiredMemoryStreams()
+  const stream = memoryTableStreams.get(tableId)
+  if (!stream) {
+    // Mirror the Redis path: a non-zero afterEventId with no buffer at all
+    // means TTL expired or the stream never existed; either way the caller's
+    // cursor is stale.
+    if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined }
+    return { status: 'ok', events: [] }
+  }
+  if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) {
+    return { status: 'pruned', earliestEventId: stream.earliestEventId }
+  }
+  return {
+    status: 'ok',
+    events: stream.events
+      .filter((entry) => entry.eventId > afterEventId)
+      .slice(0, TABLE_EVENT_READ_CHUNK),
+  }
+}
+
+/**
+ * Append an event to the table's buffer. Fire-and-forget from the caller —
+ * this never throws, returns null on failure. A Redis blip must not fail a
+ * cell-write.
+ */
+export async function appendTableEvent(event: TableEvent): Promise<TableEventEntry | null> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      try {
+        return appendMemory(event)
+      } catch (error) {
+        logger.warn('appendTableEvent: memory append failed', {
+          tableId: event.tableId,
+          error: toError(error).message,
+        })
+        return null
+      }
+    }
+    return null
+  }
+  try {
+    // Build the entry JSON in two halves so Lua can splice the new eventId
+    // between them without us needing a round-trip just to mint the id first.
+    const tail = `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}`
+    const head = `{"eventId":`
+    const result = await redis.eval(
+      APPEND_EVENT_SCRIPT,
+      3,
+      getEventsKey(event.tableId),
+      getSeqKey(event.tableId),
+      getMetaKey(event.tableId),
+      TABLE_EVENT_TTL_SECONDS,
+      TABLE_EVENT_CAP,
+      new Date().toISOString(),
+      head,
+      tail
+    )
+    const eventId = typeof result === 'number' ? result : Number(result)
+    if (!Number.isFinite(eventId)) return null
+    return { eventId, tableId: event.tableId, event }
+  } catch (error) {
+    logger.warn('appendTableEvent: Redis append failed', {
+      tableId: event.tableId,
+      error: toError(error).message,
+    })
+    return null
+  }
+}
+
+/**
+ * The latest eventId assigned for a table, or 0 when the buffer is empty or
+ * expired. Used by the stream route to tail from "now" when a client connects
+ * without a replay cursor (fresh mount — its caches were just fetched from
+ * the DB, so replaying history would only rewind them).
+ *
+ * Redis errors propagate: silently falling back to 0 would replay the whole
+ * buffer over fresh state — the exact churn tail-from-latest exists to avoid.
+ * The stream route errors the stream instead and the client reconnects with
+ * backoff.
+ */
+export async function getLatestTableEventId(tableId: string): Promise<number> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      // Pure read — getMemoryStream() would allocate a stream as a side effect.
+      const stream = memoryTableStreams.get(tableId)
+      return stream ? stream.nextEventId - 1 : 0
+    }
+    return 0
+  }
+  const raw = await redis.get(getSeqKey(tableId))
+  if (!raw) return 0
+  const parsed = Number.parseInt(raw, 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 0
+}
+
+/**
+ * Read events for a table where eventId > afterEventId. Returns 'pruned' if
+ * the caller has fallen off the back of the buffer (TTL expired or cap rolled
+ * past their lastEventId). Caller should respond by full-refetching from DB
+ * and resuming streaming from the new earliestEventId.
+ */
+export async function readTableEventsSince(
+  tableId: string,
+  afterEventId: number
+): Promise<TableEventsReadResult> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      return readMemory(tableId, afterEventId)
+    }
+    return { status: 'unavailable', error: 'Redis client unavailable' }
+  }
+  try {
+    const meta = await redis.hgetall(getMetaKey(tableId))
+    const earliestEventId =
+      meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined
+    if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) {
+      return { status: 'pruned', earliestEventId }
+    }
+    // Read in capped chunks so a 5000-event backlog doesn't materialize as one
+    // multi-MB Redis reply + JSON parse + SSE flush. The route loop drains
+    // chunks across ticks.
+    const raw = await redis.zrangebyscore(
+      getEventsKey(tableId),
+      afterEventId + 1,
+      '+inf',
+      'LIMIT',
+      0,
+      TABLE_EVENT_READ_CHUNK
+    )
+    if (raw.length === 0 && afterEventId > 0) {
+      // Total TTL expiry: events + meta both gone. The seq counter has the
+      // same TTL — its absence means the buffer was wiped and the caller's
+      // `afterEventId` is stale. Signal pruned so the client refetches.
+      const seqExists = await redis.exists(getSeqKey(tableId))
+      if (seqExists === 0) {
+        return { status: 'pruned', earliestEventId: undefined }
+      }
+    }
+    return {
+      status: 'ok',
+      events: raw
+        .map((entry) => {
+          try {
+            return JSON.parse(entry) as TableEventEntry
+          } catch {
+            return null
+          }
+        })
+        .filter((entry): entry is TableEventEntry => Boolean(entry)),
+    }
+  } catch (error) {
+    const message = toError(error).message
+    logger.warn('readTableEventsSince failed', { tableId, error: message })
+    return { status: 'unavailable', error: message }
+  }
+}

apps/sim/lib/table/export-format.ts

@@ -0,0 +1,41 @@
+/**
+ * Shared serialization for table exports — used by both the synchronous streaming export route
+ * (small tables) and the background export job worker (large tables), so the two paths produce
+ * byte-identical files.
+ */
+
+export function sanitizeExportFilename(name: string): string {
+  const cleaned = name.replace(/[^a-zA-Z0-9_-]+/g, '_').replace(/^_+|_+$/g, '')
+  return cleaned || 'table'
+}
+
+/**
+ * Prefixes a single quote to values starting with a spreadsheet formula trigger
+ * (`=`, `+`, `-`, `@`, tab, CR), neutralizing CSV injection in Excel/Sheets.
+ */
+export function neutralizeCsvFormula(value: string): string {
+  return /^[=+\-@\t\r]/.test(value) ? `'${value}` : value
+}
+
+/**
+ * Serializes a cell for CSV. Only string cells are formula-neutralized; numbers,
+ * booleans, dates, and JSON objects can never form a trigger and pass through verbatim.
+ */
+export function formatCsvValue(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  if (value instanceof Date) return value.toISOString()
+  if (typeof value === 'object') return JSON.stringify(value)
+  if (typeof value === 'string') return neutralizeCsvFormula(value)
+  return String(value)
+}
+
+export function toCsvRow(values: string[]): string {
+  return values.map(escapeCsvField).join(',')
+}
+
+function escapeCsvField(field: string): string {
+  if (/[",\n\r]/.test(field)) {
+    return `"${field.replace(/"/g, '""')}"`
+  }
+  return field
+}

apps/sim/lib/table/export-runner.test.ts

@@ -0,0 +1,176 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockGetTableById,
+  mockSelectExportRowPage,
+  mockUpdateJobProgress,
+  mockMarkJobReady,
+  mockMarkJobFailed,
+  mockSetJobResultKey,
+  mockAppendTableEvent,
+  mockCreateMultipartUpload,
+  mockDeleteFile,
+} = vi.hoisted(() => ({
+  mockGetTableById: vi.fn(),
+  mockSelectExportRowPage: vi.fn(),
+  mockUpdateJobProgress: vi.fn(),
+  mockMarkJobReady: vi.fn(),
+  mockMarkJobFailed: vi.fn(),
+  mockSetJobResultKey: vi.fn(),
+  mockAppendTableEvent: vi.fn(),
+  mockCreateMultipartUpload: vi.fn(),
+  mockDeleteFile: vi.fn(),
+}))
+
+vi.mock('@/lib/table/service', () => ({
+  getTableById: mockGetTableById,
+}))
+vi.mock('@/lib/table/jobs/service', () => ({
+  selectExportRowPage: mockSelectExportRowPage,
+  updateJobProgress: mockUpdateJobProgress,
+  markJobReady: mockMarkJobReady,
+  markJobFailed: mockMarkJobFailed,
+  setJobResultKey: mockSetJobResultKey,
+}))
+vi.mock('@/lib/table/events', () => ({ appendTableEvent: mockAppendTableEvent }))
+vi.mock('@/lib/uploads/core/storage-service', () => ({
+  createMultipartUpload: mockCreateMultipartUpload,
+  deleteFile: mockDeleteFile,
+}))
+
+import { runTableExport } from '@/lib/table/export-runner'
+
+const table = {
+  id: 'tbl_1',
+  name: 'People',
+  workspaceId: 'ws_1',
+  schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] },
+}
+
+const payload = { jobId: 'job_1', tableId: 'tbl_1', workspaceId: 'ws_1', format: 'csv' as const }
+
+interface FakeHandle {
+  key: string
+  content: string
+  write: ReturnType<typeof vi.fn>
+  complete: ReturnType<typeof vi.fn>
+  abort: ReturnType<typeof vi.fn>
+}
+
+let lastHandle: FakeHandle | null
+
+describe('runTableExport', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    lastHandle = null
+    mockGetTableById.mockResolvedValue(table)
+    mockUpdateJobProgress.mockResolvedValue(true)
+    mockMarkJobReady.mockResolvedValue(true)
+    mockMarkJobFailed.mockResolvedValue(undefined)
+    mockSetJobResultKey.mockResolvedValue(undefined)
+    mockDeleteFile.mockResolvedValue(undefined)
+    // A handle that records every write so tests can assert the streamed bytes, and echoes the
+    // pinned key back from `complete` like the real uploader does.
+    mockCreateMultipartUpload.mockImplementation(({ key }: { key: string }) => {
+      const chunks: string[] = []
+      const handle: FakeHandle = {
+        key,
+        content: '',
+        write: vi.fn((chunk: Buffer | string) => {
+          chunks.push(typeof chunk === 'string' ? chunk : chunk.toString('utf8'))
+          return Promise.resolve()
+        }),
+        complete: vi.fn(() => {
+          handle.content = chunks.join('')
+          return Promise.resolve({ key, size: Buffer.byteLength(handle.content) })
+        }),
+        abort: vi.fn(() => Promise.resolve()),
+      }
+      lastHandle = handle
+      return Promise.resolve(handle)
+    })
+    mockSelectExportRowPage.mockResolvedValue([
+      { id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' },
+    ])
+  })
+
+  it('streams rows to the uploader, stamps the result key, and marks ready', async () => {
+    await runTableExport(payload)
+
+    expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1)
+    const init = mockCreateMultipartUpload.mock.calls[0][0]
+    expect(init.key).toBe('workspace/ws_1/exports/tbl_1/job_1/People.csv')
+    expect(init.context).toBe('workspace')
+    expect(init.contentType).toContain('text/csv')
+
+    expect(lastHandle?.content).toBe('name\nAda\n')
+    expect(lastHandle?.complete).toHaveBeenCalledTimes(1)
+    expect(lastHandle?.abort).not.toHaveBeenCalled()
+
+    expect(mockSetJobResultKey).toHaveBeenCalledWith('tbl_1', 'job_1', init.key)
+    expect(mockMarkJobReady).toHaveBeenCalledWith('tbl_1', 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'export', status: 'ready', progress: 1 })
+    )
+    expect(mockDeleteFile).not.toHaveBeenCalled()
+  })
+
+  it('serializes JSON exports with display-name keys', async () => {
+    await runTableExport({ ...payload, format: 'json' })
+    const init = mockCreateMultipartUpload.mock.calls[0][0]
+    expect(init.key.endsWith('/People.json')).toBe(true)
+    expect(JSON.parse(lastHandle?.content ?? '')).toEqual([{ name: 'Ada' }])
+  })
+
+  it('aborts the upload and never completes when ownership is lost (cancel)', async () => {
+    mockUpdateJobProgress.mockResolvedValue(false)
+
+    await runTableExport(payload)
+
+    expect(lastHandle?.complete).not.toHaveBeenCalled()
+    expect(lastHandle?.abort).toHaveBeenCalledTimes(1)
+    expect(mockMarkJobReady).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('aborts before completing when ownership is lost at the finalize gate', async () => {
+    mockUpdateJobProgress.mockResolvedValueOnce(true).mockResolvedValue(false)
+
+    await runTableExport(payload)
+
+    expect(mockSelectExportRowPage).toHaveBeenCalledTimes(1)
+    expect(lastHandle?.complete).not.toHaveBeenCalled()
+    expect(lastHandle?.abort).toHaveBeenCalledTimes(1)
+    expect(mockMarkJobReady).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('deletes the finalized object when the job was canceled at the wire', async () => {
+    mockMarkJobReady.mockResolvedValue(false)
+
+    await runTableExport(payload)
+
+    expect(lastHandle?.complete).toHaveBeenCalledTimes(1)
+    expect(mockDeleteFile).toHaveBeenCalledWith(
+      expect.objectContaining({ key: expect.stringContaining('exports/tbl_1/job_1') })
+    )
+    expect(mockAppendTableEvent).not.toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready' })
+    )
+  })
+
+  it('aborts the upload, marks the job failed, and emits a failed event on error', async () => {
+    mockSelectExportRowPage.mockRejectedValue(new Error('boom'))
+
+    await runTableExport(payload)
+
+    expect(lastHandle?.abort).toHaveBeenCalledTimes(1)
+    expect(mockMarkJobFailed).toHaveBeenCalledWith('tbl_1', 'job_1', 'boom')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'export', status: 'failed', error: 'boom' })
+    )
+  })
+})

apps/sim/lib/table/export-runner.ts

@@ -0,0 +1,162 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { buildNameById, getColumnId, rowDataIdToName } from '@/lib/table/column-keys'
+import { appendTableEvent } from '@/lib/table/events'
+import {
+  formatCsvValue,
+  neutralizeCsvFormula,
+  sanitizeExportFilename,
+  toCsvRow,
+} from '@/lib/table/export-format'
+import {
+  markJobFailed,
+  markJobReady,
+  selectExportRowPage,
+  setJobResultKey,
+  updateJobProgress,
+} from '@/lib/table/jobs/service'
+import { getTableById } from '@/lib/table/service'
+import {
+  createMultipartUpload,
+  deleteFile,
+  type MultipartUploadHandle,
+} from '@/lib/uploads/core/storage-service'
+
+const logger = createLogger('TableExportRunner')
+
+/** Rows per page while building the file. Internal caller — not bound by MAX_QUERY_LIMIT; rows
+ *  are fetched without executions, so even wide rows stay a few MB per batch. */
+const EXPORT_BATCH_SIZE = 5000
+
+/** Thrown when this worker loses the job (canceled / janitor-failed). */
+class JobSupersededError extends Error {}
+
+export interface TableExportPayload {
+  jobId: string
+  tableId: string
+  workspaceId: string
+  format: 'csv' | 'json'
+}
+
+/**
+ * Background worker for large table exports. Pages rows via `queryRows` (so the delete-job
+ * visibility mask applies — an export taken mid-delete excludes the doomed rows), accumulates the
+ * serialized file, uploads it to workspace storage, and stamps the storage key onto the job's
+ * payload (`resultKey`). The client downloads via a presigned URL from the download route; the
+ * janitor deletes the file when the terminal job is pruned. Ownership-gated per batch, so a
+ * cancel stops it within one page. Retry-safe: a retried attempt regenerates the file from
+ * scratch and overwrites nothing (fresh key per attempt; failures clean up their partial upload).
+ */
+export async function runTableExport(payload: TableExportPayload): Promise<void> {
+  const { jobId, tableId, workspaceId, format } = payload
+  const requestId = generateId().slice(0, 8)
+  let handle: MultipartUploadHandle | null = null
+  let uploadedKey: string | null = null
+
+  try {
+    const table = await getTableById(tableId, { includeArchived: true })
+    if (!table) throw new Error(`Export target table ${tableId} not found`)
+
+    const columns = table.schema.columns
+    // Stored row data is id-keyed; CSV headers and JSON keys are display names, so translate
+    // id → name on the way out (export is a name-friendly boundary).
+    const nameById = buildNameById(table.schema)
+
+    const fileName = `${sanitizeExportFilename(table.name)}.${format}`
+    // The key is pinned up front so the streaming upload writes exactly where the download
+    // route presigns; the *returned* key (from `complete`) is recorded as the source of truth.
+    const key = `workspace/${workspaceId}/exports/${tableId}/${jobId}/${fileName}`
+    const contentType = format === 'csv' ? 'text/csv; charset=utf-8' : 'application/json'
+
+    // Stream the serialized file straight into storage in bounded parts instead of buffering the
+    // whole thing in heap — a 1M-row export no longer holds hundreds of MB resident.
+    handle = await createMultipartUpload({ key, context: 'workspace', contentType })
+    await handle.write(
+      format === 'csv' ? `${toCsvRow(columns.map((c) => neutralizeCsvFormula(c.name)))}\n` : '['
+    )
+
+    let exported = 0
+    let firstJsonRow = true
+    let after: { orderKey: string; id: string } | null = null
+    while (true) {
+      // Ownership gate before every page: a canceled job stops within one batch.
+      const owns = await updateJobProgress(tableId, exported, jobId)
+      if (!owns) throw new JobSupersededError()
+
+      const page = await selectExportRowPage(table, after, EXPORT_BATCH_SIZE)
+      if (page.length === 0) break
+
+      const pageChunks: string[] = []
+      for (const row of page) {
+        if (format === 'csv') {
+          pageChunks.push(
+            `${toCsvRow(columns.map((c) => formatCsvValue(row.data[getColumnId(c)])))}\n`
+          )
+        } else {
+          const prefix = firstJsonRow ? '' : ','
+          firstJsonRow = false
+          pageChunks.push(prefix + JSON.stringify(rowDataIdToName(row.data, nameById)))
+        }
+      }
+      await handle.write(pageChunks.join(''))
+
+      exported += page.length
+      const last = page[page.length - 1]
+      after = { orderKey: last.orderKey, id: last.id }
+      if (page.length < EXPORT_BATCH_SIZE) break
+    }
+    if (format === 'json') await handle.write(']')
+
+    const ownsFinalize = await updateJobProgress(tableId, exported, jobId)
+    if (!ownsFinalize) throw new JobSupersededError()
+
+    const uploaded = await handle.complete()
+    uploadedKey = uploaded.key
+    await setJobResultKey(tableId, jobId, uploaded.key)
+
+    await updateJobProgress(tableId, exported, jobId)
+    // Only announce success if we still won the transition (not canceled at the wire).
+    const becameReady = await markJobReady(tableId, jobId)
+    if (becameReady) {
+      void appendTableEvent({
+        kind: 'job',
+        type: 'export',
+        tableId,
+        jobId,
+        status: 'ready',
+        progress: exported,
+      })
+      logger.info(`[${requestId}] Export complete`, { tableId, rows: exported, format })
+    } else {
+      // Canceled at the very end — the file is orphaned; remove it (janitor would otherwise
+      // only catch it via the pruned job's resultKey).
+      await deleteFile({ key: uploaded.key, context: 'workspace' }).catch(() => {})
+      logger.info(`[${requestId}] Export finished but no longer owns the run`, { tableId, jobId })
+    }
+  } catch (err) {
+    // A partial/orphaned upload from this attempt is useless — clean it up best-effort. An
+    // in-flight multipart upload (not yet completed) is aborted so no staged parts linger; a
+    // completed-but-unannounced upload is removed by key.
+    if (uploadedKey) {
+      await deleteFile({ key: uploadedKey, context: 'workspace' }).catch(() => {})
+    } else if (handle) {
+      await handle.abort().catch(() => {})
+    }
+    if (err instanceof JobSupersededError) {
+      logger.info(`[${requestId}] Export superseded/canceled; stopping`, { tableId, jobId })
+    } else {
+      const message = getErrorMessage(err, 'Export failed')
+      logger.error(`[${requestId}] Export failed for table ${tableId}:`, err)
+      await markJobFailed(tableId, jobId, message).catch(() => {})
+      void appendTableEvent({
+        kind: 'job',
+        type: 'export',
+        tableId,
+        jobId,
+        status: 'failed',
+        error: message,
+      })
+    }
+  }
+}

apps/sim/lib/table/hooks/index.ts

@@ -0,0 +1 @@
+export * from '@/lib/table/hooks/use-table-columns'

apps/sim/lib/table/hooks/use-table-columns.ts

@@ -0,0 +1,36 @@
+import { useMemo } from 'react'
+import type { ColumnOption } from '@/lib/table/types'
+import { useTable } from '@/hooks/queries/tables'
+import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
+
+interface UseTableColumnsOptions {
+  tableId: string | null | undefined
+  includeBuiltIn?: boolean
+}
+
+/** Fetches table schema columns as dropdown options. */
+export function useTableColumns({ tableId, includeBuiltIn = false }: UseTableColumnsOptions) {
+  const workspaceId = useWorkflowRegistry((state) => state.hydration.workspaceId)
+  const { data: tableData } = useTable(workspaceId ?? undefined, tableId ?? undefined)
+
+  const schemaColumns = useMemo<ColumnOption[]>(
+    () =>
+      (tableData?.schema?.columns || []).map((col) => ({
+        value: col.name,
+        label: col.name,
+      })),
+    [tableData]
+  )
+
+  return useMemo(() => {
+    if (includeBuiltIn) {
+      const builtInCols = [
+        { value: 'createdAt', label: 'createdAt' },
+        { value: 'updatedAt', label: 'updatedAt' },
+      ]
+      return [...schemaColumns, ...builtInCols]
+    }
+
+    return schemaColumns
+  }, [includeBuiltIn, schemaColumns])
+}

apps/sim/lib/table/import-data.ts

@@ -0,0 +1,253 @@
+/**
+ * Import-job table-data write operations — bulk insert, schema setup, and
+ * append/replace used by `import-runner.ts` and the import route. Distinct from
+ * `import.ts` (CSV parsing) and `import-runner.ts` (the job runner).
+ */
+
+import { db } from '@sim/db'
+import { userTableDefinitions, userTableRows } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
+import { eq } from 'drizzle-orm'
+import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing'
+import { CSV_MAX_BATCH_SIZE } from '@/lib/table/import'
+import { nKeysBetween } from '@/lib/table/order-key'
+import { acquireRowOrderLock } from '@/lib/table/rows/ordering'
+import { batchInsertRowsWithTx, replaceTableRowsWithTx } from '@/lib/table/rows/service'
+import { addTableColumnsWithTx, auditTableColumnsAdded } from '@/lib/table/service'
+import type {
+  ReplaceRowsResult,
+  RowData,
+  TableDefinition,
+  TableRow,
+  TableSchema,
+} from '@/lib/table/types'
+import {
+  checkBatchUniqueConstraintsDb,
+  coerceRowToSchema,
+  getUniqueColumns,
+  validateRowSize,
+} from '@/lib/table/validation'
+
+const logger = createLogger('TableImportData')
+
+/** One batch of rows for a background import (see {@link bulkInsertImportBatch}). */
+export interface BulkImportBatch {
+  tableId: string
+  workspaceId: string
+  userId?: string
+  rows: RowData[]
+  /** Position of the first row in this batch; rows get contiguous positions from here. */
+  startPosition: number
+  /** Previous batch's last `order_key` (the append anchor); null for the first batch / empty table. */
+  afterOrderKey?: string | null
+}
+
+/**
+ * Inserts one batch of rows for an async import in a single committed statement.
+ *
+ * Differs from {@link batchInsertRowsWithTx} for the bulk-load case: caller-supplied
+ * contiguous positions (no `acquireTablePositionLock` / `nextAutoPosition` scan — an
+ * import owns its hidden table as the sole writer), no `RETURNING`, and **no
+ * `fireTableTrigger` / `runWorkflowColumn`** (a 1M-row import must not dispatch a
+ * workflow run per row). `row_count` is maintained set-based by the statement-level
+ * trigger. There is no surrounding transaction and no rollback: each batch commits on
+ * its own, so committed batches persist even if a later batch fails.
+ *
+ * Throws on row-size/schema/unique violations or if the statement-level trigger rejects
+ * the batch for crossing `max_rows`; the caller marks the import failed.
+ */
+export async function bulkInsertImportBatch(
+  data: BulkImportBatch,
+  table: TableDefinition,
+  requestId: string
+): Promise<{ inserted: number; lastOrderKey: string | null }> {
+  for (let i = 0; i < data.rows.length; i++) {
+    const sizeValidation = validateRowSize(data.rows[i])
+    if (!sizeValidation.valid) {
+      throw new Error(`Row ${i + 1}: ${sizeValidation.errors.join(', ')}`)
+    }
+    const schemaValidation = coerceRowToSchema(data.rows[i], table.schema)
+    if (!schemaValidation.valid) {
+      throw new Error(`Row ${i + 1}: ${schemaValidation.errors.join(', ')}`)
+    }
+  }
+
+  const uniqueColumns = getUniqueColumns(table.schema)
+  if (uniqueColumns.length > 0) {
+    const uniqueResult = await checkBatchUniqueConstraintsDb(
+      data.tableId,
+      data.rows,
+      table.schema,
+      db
+    )
+    if (!uniqueResult.valid) {
+      throw new Error(
+        uniqueResult.errors.map((e) => `Row ${e.row + 1}: ${e.errors.join(', ')}`).join('; ')
+      )
+    }
+  }
+
+  const now = new Date()
+  // Import worker is the table's sole writer; append keys after the anchor the caller threads
+  // from the previous batch's last key — no per-batch max(order_key) scan over a growing table.
+  const orderKeys = nKeysBetween(data.afterOrderKey ?? null, null, data.rows.length)
+  const rowsToInsert = data.rows.map((rowData, i) => ({
+    id: `row_${generateId().replace(/-/g, '')}`,
+    tableId: data.tableId,
+    workspaceId: data.workspaceId,
+    data: rowData,
+    position: data.startPosition + i,
+    orderKey: orderKeys[i],
+    createdAt: now,
+    updatedAt: now,
+    ...(data.userId ? { createdBy: data.userId } : {}),
+  }))
+
+  await db.insert(userTableRows).values(rowsToInsert)
+  logger.info(`[${requestId}] Bulk-imported ${rowsToInsert.length} rows into table ${data.tableId}`)
+  return {
+    inserted: rowsToInsert.length,
+    lastOrderKey: orderKeys[orderKeys.length - 1] ?? data.afterOrderKey ?? null,
+  }
+}
+
+/** Deletes every row of a table (set-based; the statement-level trigger zeroes `row_count`). */
+export async function deleteAllTableRows(tableId: string): Promise<void> {
+  await db.delete(userTableRows).where(eq(userTableRows.tableId, tableId))
+}
+
+/**
+ * Adds columns to a table during an import (the `createColumns` flow), wrapping the
+ * tx-bound {@link addTableColumnsWithTx} in its own transaction. Returns the updated table.
+ */
+export async function addImportColumns(
+  table: TableDefinition,
+  additions: { name: string; type: string }[],
+  requestId: string,
+  actingUserId?: string
+): Promise<TableDefinition> {
+  const updated = await db.transaction((trx) =>
+    addTableColumnsWithTx(trx, table, additions, requestId)
+  )
+  auditTableColumnsAdded(
+    table,
+    additions.map((c) => c.name),
+    actingUserId
+  )
+  return updated
+}
+
+/** Overwrites a table's schema during an import (used when inferring columns from the file). */
+export async function setTableSchemaForImport(tableId: string, schema: TableSchema): Promise<void> {
+  await db
+    .update(userTableDefinitions)
+    .set({ schema, updatedAt: new Date() })
+    .where(eq(userTableDefinitions.id, tableId))
+}
+
+/**
+ * Owns the append-import transaction so the API route never holds a `trx`:
+ * optionally creates the new columns, then inserts every row in CSV-sized
+ * batches — all atomic. Caller fires {@link dispatchAfterBatchInsert} after this
+ * resolves (post-commit), mirroring the other batch-insert sites.
+ */
+export async function importAppendRows(
+  table: TableDefinition,
+  additions: { id?: string; name: string; type: string; required?: boolean; unique?: boolean }[],
+  rows: RowData[],
+  ctx: { workspaceId: string; userId?: string; requestId: string }
+): Promise<{ inserted: TableRow[]; table: TableDefinition }> {
+  // Gate capacity before opening the tx — the lookup is a separate pool read.
+  const rowLimit = await assertRowCapacity({
+    workspaceId: ctx.workspaceId,
+    currentRowCount: table.rowCount,
+    addedRows: rows.length,
+  })
+  const result = await db.transaction(async (trx) => {
+    let working = table
+    if (additions.length > 0) {
+      // Take the row-order lock before creating columns so this path uses the
+      // same rows_pos → user_table_definitions order as plain inserts. Creating
+      // columns first would lock the definition row before rows_pos, inverting
+      // the order and deadlocking concurrent inserts on this table. The lock is
+      // re-entrant, so the per-batch acquire below is a no-op.
+      await acquireRowOrderLock(trx, table.id)
+      working = await addTableColumnsWithTx(trx, table, additions, ctx.requestId)
+    }
+    const inserted: TableRow[] = []
+    for (let i = 0; i < rows.length; i += CSV_MAX_BATCH_SIZE) {
+      const batch = rows.slice(i, i + CSV_MAX_BATCH_SIZE)
+      const batchInserted = await batchInsertRowsWithTx(
+        trx,
+        { tableId: working.id, rows: batch, workspaceId: ctx.workspaceId, userId: ctx.userId },
+        working,
+        generateId().slice(0, 8)
+      )
+      inserted.push(...batchInserted)
+    }
+    return { inserted, table: working }
+  })
+  // Audit post-commit — a mid-import rollback means the columns weren't added.
+  if (additions.length > 0) {
+    auditTableColumnsAdded(
+      table,
+      additions.map((c) => c.name),
+      ctx.userId
+    )
+  }
+  notifyTableRowUsage({
+    workspaceId: ctx.workspaceId,
+    currentRowCount: table.rowCount,
+    addedRows: result.inserted.length,
+    limit: rowLimit,
+  })
+  return result
+}
+
+/**
+ * Owns the replace-import transaction: optionally creates the new columns, then
+ * replaces all rows — atomically. Keeps `trx` out of the API route.
+ */
+export async function importReplaceRows(
+  table: TableDefinition,
+  additions: { id?: string; name: string; type: string; required?: boolean; unique?: boolean }[],
+  data: { rows: RowData[]; workspaceId: string; userId?: string },
+  requestId: string
+): Promise<ReplaceRowsResult> {
+  // Replace deletes all existing rows, so the footprint is just the new set. Gate
+  // before opening the tx — the plan lookup is a separate pool read.
+  const rowLimit = await assertRowCapacity({
+    workspaceId: data.workspaceId,
+    currentRowCount: 0,
+    addedRows: data.rows.length,
+  })
+  const result = await db.transaction(async (trx) => {
+    let working = table
+    if (additions.length > 0) {
+      await acquireRowOrderLock(trx, table.id)
+      working = await addTableColumnsWithTx(trx, table, additions, requestId)
+    }
+    return replaceTableRowsWithTx(
+      trx,
+      { tableId: working.id, rows: data.rows, workspaceId: data.workspaceId, userId: data.userId },
+      working,
+      requestId
+    )
+  })
+  // Audit post-commit (see importAppendRows).
+  if (additions.length > 0) {
+    auditTableColumnsAdded(
+      table,
+      additions.map((c) => c.name),
+      data.userId
+    )
+  }
+  notifyTableRowUsage({
+    workspaceId: data.workspaceId,
+    currentRowCount: 0,
+    addedRows: result.insertedCount,
+    limit: rowLimit,
+  })
+  return result
+}

apps/sim/lib/table/import-runner.test.ts

@@ -0,0 +1,117 @@
+/**
+ * @vitest-environment node
+ */
+import { Readable } from 'node:stream'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockGetTableById,
+  mockBulkInsertImportBatch,
+  mockUpdateJobProgress,
+  mockMarkJobReady,
+  mockMarkJobFailed,
+  mockNextImportStartPosition,
+  mockNextImportStartOrderKey,
+  mockAppendTableEvent,
+  mockDeleteFile,
+  mockDownloadFileStream,
+  mockHeadObject,
+} = vi.hoisted(() => ({
+  mockGetTableById: vi.fn(),
+  mockBulkInsertImportBatch: vi.fn(),
+  mockUpdateJobProgress: vi.fn(),
+  mockMarkJobReady: vi.fn(),
+  mockMarkJobFailed: vi.fn(),
+  mockNextImportStartPosition: vi.fn(),
+  mockNextImportStartOrderKey: vi.fn(),
+  mockAppendTableEvent: vi.fn(),
+  mockDeleteFile: vi.fn(),
+  mockDownloadFileStream: vi.fn(),
+  mockHeadObject: vi.fn(),
+}))
+
+vi.mock('@/lib/table/service', () => ({
+  getTableById: mockGetTableById,
+}))
+vi.mock('@/lib/table/import-data', () => ({
+  addImportColumns: vi.fn(),
+  bulkInsertImportBatch: mockBulkInsertImportBatch,
+  deleteAllTableRows: vi.fn(),
+  setTableSchemaForImport: vi.fn(),
+}))
+vi.mock('@/lib/table/jobs/service', () => ({
+  markJobFailed: mockMarkJobFailed,
+  markJobReady: mockMarkJobReady,
+  updateJobProgress: mockUpdateJobProgress,
+}))
+vi.mock('@/lib/table/rows/ordering', () => ({
+  nextImportStartOrderKey: mockNextImportStartOrderKey,
+  nextImportStartPosition: mockNextImportStartPosition,
+}))
+vi.mock('@/lib/table/events', () => ({ appendTableEvent: mockAppendTableEvent }))
+vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() }))
+vi.mock('@/lib/uploads/core/storage-service', () => ({
+  deleteFile: mockDeleteFile,
+  downloadFileStream: mockDownloadFileStream,
+  headObject: mockHeadObject,
+}))
+vi.mock('@/app/api/table/utils', () => ({
+  normalizeColumn: (col: unknown) => col,
+}))
+
+import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner'
+
+const table = {
+  id: 'tbl_1',
+  name: 'People',
+  workspaceId: 'ws_1',
+  rowCount: 0,
+  maxRows: 1000,
+  schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] },
+}
+
+function buildPayload(overrides: Partial<TableImportPayload> = {}): TableImportPayload {
+  return {
+    importId: 'job_1',
+    tableId: 'tbl_1',
+    workspaceId: 'ws_1',
+    userId: 'user_1',
+    fileKey: 'workspace/ws_1/people.csv',
+    fileName: 'people.csv',
+    delimiter: ',',
+    mode: 'append',
+    ...overrides,
+  }
+}
+
+describe('runTableImport source-file cleanup', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetTableById.mockResolvedValue(table)
+    mockHeadObject.mockResolvedValue({ size: 20 })
+    mockDownloadFileStream.mockResolvedValue(Readable.from('name\nAlice\nBob\n'))
+    mockNextImportStartPosition.mockResolvedValue(0)
+    mockNextImportStartOrderKey.mockResolvedValue(null)
+    mockUpdateJobProgress.mockResolvedValue(true)
+    mockBulkInsertImportBatch.mockResolvedValue({ inserted: 2, lastOrderKey: 'a1' })
+    mockMarkJobReady.mockResolvedValue(true)
+    mockDeleteFile.mockResolvedValue(undefined)
+  })
+
+  it('deletes the single-use source object by default', async () => {
+    await runTableImport(buildPayload())
+
+    expect(mockMarkJobReady).toHaveBeenCalled()
+    expect(mockDeleteFile).toHaveBeenCalledWith({
+      key: 'workspace/ws_1/people.csv',
+      context: 'workspace',
+    })
+  })
+
+  it('keeps a persistent workspace file when deleteSourceFile is false', async () => {
+    await runTableImport(buildPayload({ deleteSourceFile: false }))
+
+    expect(mockMarkJobReady).toHaveBeenCalled()
+    expect(mockDeleteFile).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/table/import-runner.ts

@@ -0,0 +1,393 @@
+import { type Readable, Transform } from 'node:stream'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { truncate } from '@sim/utils/string'
+import { captureServerEvent } from '@/lib/posthog/server'
+import {
+  buildAutoMapping,
+  CSV_MAX_BATCH_SIZE,
+  CSV_SCHEMA_SAMPLE_SIZE,
+  type CsvHeaderMapping,
+  coerceRowsForTable,
+  createCsvParser,
+  inferColumnType,
+  inferSchemaFromCsv,
+  sanitizeName,
+  type TableSchema,
+  validateMapping,
+} from '@/lib/table'
+import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing'
+import { withGeneratedColumnIds } from '@/lib/table/column-keys'
+import { appendTableEvent } from '@/lib/table/events'
+import {
+  addImportColumns,
+  bulkInsertImportBatch,
+  deleteAllTableRows,
+  setTableSchemaForImport,
+} from '@/lib/table/import-data'
+import { markJobFailed, markJobReady, updateJobProgress } from '@/lib/table/jobs/service'
+import { nextImportStartOrderKey, nextImportStartPosition } from '@/lib/table/rows/ordering'
+import { getTableById } from '@/lib/table/service'
+import { deleteFile, downloadFileStream, headObject } from '@/lib/uploads/core/storage-service'
+import { normalizeColumn } from '@/app/api/table/utils'
+
+const logger = createLogger('TableImportRunner')
+
+/** Emit a progress event / DB update at most every this many rows. */
+const PROGRESS_INTERVAL_ROWS = 5000
+
+/**
+ * Thrown when this worker discovers it no longer owns the table's import (the stale-job janitor
+ * marked its run failed and a newer import took over). The worker stops inserting rather than
+ * writing into a table a second worker now owns.
+ */
+class ImportSupersededError extends Error {}
+
+/** `create` infers a schema for a new table; `append`/`replace` map onto an existing one. */
+export type TableImportMode = 'create' | 'append' | 'replace'
+
+export interface TableImportPayload {
+  importId: string
+  tableId: string
+  workspaceId: string
+  userId: string
+  /** Storage key of the already-uploaded CSV/TSV file. */
+  fileKey: string
+  fileName: string
+  delimiter: ',' | '\t'
+  mode: TableImportMode
+  /** (append/replace) Explicit CSV-header → column mapping; auto-mapped when omitted. */
+  mapping?: CsvHeaderMapping
+  /** (append/replace) CSV headers to auto-create as new columns (types inferred from the sample). */
+  createColumns?: string[]
+  /**
+   * Whether the source object is deleted once the import is terminal. Defaults
+   * to true (the UI routes upload a single-use temp object per import); pass
+   * false when importing a persistent workspace file (Mothership) that must
+   * survive the import.
+   */
+  deleteSourceFile?: boolean
+  /**
+   * IANA zone used to interpret naive datetime strings in the file. The
+   * kickoff routes resolve it (request → user setting → UTC) so the detached
+   * worker never needs a settings lookup.
+   */
+  timezone?: string
+}
+
+/**
+ * Background worker for large CSV/TSV imports. Runs detached on the web container
+ * (see the kickoff routes). Streams the stored file through `createCsvParser`, resolves
+ * the target schema + header→column mapping from the first sample (inferring a new schema
+ * for `create`, mapping onto the existing schema for `append`/`replace`), then bulk-inserts
+ * in committed batches — **no rollback**: committed batches persist even if a later batch
+ * fails. Progress and the terminal state are surfaced via the table-events SSE stream.
+ */
+export async function runTableImport(payload: TableImportPayload): Promise<void> {
+  const { importId, tableId, workspaceId, userId, fileKey, fileName, delimiter, mode } = payload
+  const requestId = generateId().slice(0, 8)
+  // Hoisted so `finally` can destroy it on any failure — otherwise the storage HTTP body leaks
+  // open until it times out.
+  let source: Readable | undefined
+
+  try {
+    const loaded = await getTableById(tableId, { includeArchived: true })
+    if (!loaded) throw new Error(`Import target table ${tableId} not found`)
+    const table = loaded
+
+    // Total byte size for the progress estimate — a cheap HEAD, no download. May be null on
+    // the local dev provider, in which case the bar stays indeterminate (rows still show).
+    const totalBytes = (await headObject(fileKey, 'workspace'))?.size ?? 0
+
+    // Stream the file rather than buffering it — a ~1M-row import must never be held in memory.
+    source = await downloadFileStream({ key: fileKey, context: 'workspace' })
+
+    // Append must continue after the existing rows; create/replace start empty. Read once up
+    // front (the import is the table's sole writer) and assign contiguous positions / threaded
+    // order keys from it.
+    const basePosition = mode === 'append' ? await nextImportStartPosition(tableId) : 0
+    let lastOrderKey = mode === 'append' ? await nextImportStartOrderKey(tableId) : null
+
+    // Append keeps the existing rows; create/replace start from empty (replace deletes
+    // existing rows in resolveSetup). Per-batch capacity is checked against this base + the
+    // running total, so a stream that crosses the plan limit fails within one batch.
+    const existingRowCount = mode === 'append' ? table.rowCount : 0
+
+    // Count bytes as they flow so the row total can be extrapolated from byte progress.
+    let bytesRead = 0
+    const byteCounter = new Transform({
+      transform(chunk: Buffer, _enc, cb) {
+        bytesRead += chunk.length
+        cb(null, chunk)
+      },
+    })
+
+    const parser = createCsvParser(delimiter)
+    // `.pipe` doesn't forward source errors; forward so the iterator throws.
+    source.on('error', (err) => parser.destroy(err))
+    byteCounter.on('error', (err) => parser.destroy(err))
+    source.pipe(byteCounter).pipe(parser)
+
+    let schema: TableSchema | null = null
+    let headerToColumn: Map<string, string> | null = null
+    let inserted = 0
+    let lastReported = 0
+    const sample: Record<string, unknown>[] = []
+    let batch: Record<string, unknown>[] = []
+
+    /**
+     * Resolve the schema + header→column mapping from the buffered sample (runs once).
+     * `create` infers a fresh schema and overwrites the placeholder; `append`/`replace`
+     * map onto the existing schema, optionally auto-creating `createColumns` first.
+     */
+    const resolveSetup = async () => {
+      const headers = Object.keys(sample[0])
+
+      if (mode === 'create') {
+        const inferred = inferSchemaFromCsv(headers, sample)
+        // Stamp ids so the imported table is id-native (rows coerce + persist by
+        // the same ids).
+        schema = withGeneratedColumnIds({ columns: inferred.columns.map(normalizeColumn) })
+        headerToColumn = inferred.headerToColumn
+        await setTableSchemaForImport(tableId, schema)
+        return
+      }
+
+      // append / replace into an existing table.
+      let targetSchema = table.schema
+      let effectiveMapping: CsvHeaderMapping =
+        payload.mapping ?? buildAutoMapping(headers, table.schema)
+
+      if (payload.createColumns && payload.createColumns.length > 0) {
+        const unknown = payload.createColumns.filter((h) => !headers.includes(h))
+        if (unknown.length > 0) {
+          throw new Error(`Columns to create are not in the CSV: ${unknown.join(', ')}`)
+        }
+        const usedNames = new Set(table.schema.columns.map((c) => c.name.toLowerCase()))
+        const additions: { name: string; type: string }[] = []
+        const updatedMapping: CsvHeaderMapping = { ...effectiveMapping }
+        for (const header of payload.createColumns) {
+          const base = sanitizeName(header)
+          let columnName = base
+          let suffix = 2
+          while (usedNames.has(columnName.toLowerCase())) {
+            columnName = `${base}_${suffix}`
+            suffix++
+          }
+          usedNames.add(columnName.toLowerCase())
+          additions.push({ name: columnName, type: inferColumnType(sample.map((r) => r[header])) })
+          updatedMapping[header] = columnName
+        }
+        const updated = await addImportColumns(table, additions, requestId, userId)
+        targetSchema = updated.schema
+        effectiveMapping = updatedMapping
+      }
+
+      const validation = validateMapping({
+        csvHeaders: headers,
+        mapping: effectiveMapping,
+        tableSchema: targetSchema,
+      })
+      schema = targetSchema
+      headerToColumn = validation.effectiveMap
+
+      // Replace deletes existing rows only after schema/mapping validation passes, so an
+      // invalid or empty file fails the import with the old rows still intact (a mid-stream
+      // insert failure after this point leaves a partial replace — replace is destructive).
+      if (mode === 'replace') await deleteAllTableRows(tableId)
+    }
+
+    const flush = async (rows: Record<string, unknown>[]) => {
+      if (rows.length === 0 || !schema || !headerToColumn) return
+      // Ownership gate before every insert: once this run loses the table (cancel/supersede),
+      // updateJobProgress returns false and we stop before writing into a table a newer import
+      // may own. Runs per batch (not just at the emit cadence) so we stop within one batch.
+      const owns = await updateJobProgress(tableId, inserted, importId)
+      if (!owns) throw new ImportSupersededError()
+      const coerced = coerceRowsForTable(rows, schema, headerToColumn, {
+        timezone: payload.timezone,
+      })
+      const rowLimit = await assertRowCapacity({
+        workspaceId,
+        currentRowCount: existingRowCount + inserted,
+        addedRows: coerced.length,
+      })
+      const result = await bulkInsertImportBatch(
+        {
+          tableId,
+          workspaceId,
+          userId,
+          rows: coerced,
+          startPosition: basePosition + inserted,
+          afterOrderKey: lastOrderKey,
+        },
+        { ...table, schema },
+        requestId
+      )
+      notifyTableRowUsage({
+        workspaceId,
+        currentRowCount: existingRowCount + inserted,
+        addedRows: result.inserted,
+        limit: rowLimit,
+      })
+      inserted += result.inserted
+      lastOrderKey = result.lastOrderKey
+      // Emit after the first batch, then every interval, so the bar appears early without flooding.
+      if (
+        inserted - lastReported >= PROGRESS_INTERVAL_ROWS ||
+        (lastReported === 0 && inserted > 0)
+      ) {
+        lastReported = inserted
+        // Exact, monotonic completion from bytes consumed — no wobbly row estimate.
+        const percent =
+          totalBytes > 0 ? Math.min(99, Math.round((bytesRead / totalBytes) * 100)) : undefined
+        void appendTableEvent({
+          kind: 'job',
+          type: 'import',
+          tableId,
+          jobId: importId,
+          status: 'running',
+          progress: inserted,
+          percent,
+        })
+      }
+    }
+
+    let ready = false
+    for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
+      if (!ready) {
+        sample.push(record)
+        if (sample.length >= CSV_SCHEMA_SAMPLE_SIZE) {
+          await resolveSetup()
+          await flush(sample)
+          ready = true
+        }
+        continue
+      }
+      batch.push(record)
+      if (batch.length >= CSV_MAX_BATCH_SIZE) {
+        await flush(batch)
+        batch = []
+      }
+    }
+
+    if (!ready) {
+      // Fewer than CSV_SCHEMA_SAMPLE_SIZE rows total (or zero).
+      if (sample.length === 0) {
+        // No data rows — fail rather than report a successful empty import (matches the sync route).
+        const message = 'CSV file has no data rows'
+        await markJobFailed(tableId, importId, message)
+        void appendTableEvent({
+          kind: 'job',
+          type: 'import',
+          tableId,
+          jobId: importId,
+          status: 'failed',
+          error: message,
+        })
+        captureServerEvent(
+          userId,
+          'table_import_completed',
+          {
+            table_id: tableId,
+            workspace_id: workspaceId,
+            import_id: importId,
+            status: 'failed',
+            row_count: null,
+            error_message: truncate(message, 200),
+          },
+          { groups: { workspace: workspaceId } }
+        )
+        logger.warn(`[${requestId}] Import has no data rows`, { tableId, fileName })
+        return
+      }
+      await resolveSetup()
+      await flush(sample)
+    } else {
+      await flush(batch)
+    }
+
+    await updateJobProgress(tableId, inserted, importId)
+    // Only announce success if we actually won the transition — a cancel/supersede that landed
+    // right at the end makes this a no-op, and we must not emit a false `ready`.
+    const becameReady = await markJobReady(tableId, importId)
+    if (becameReady) {
+      void appendTableEvent({
+        kind: 'job',
+        type: 'import',
+        tableId,
+        jobId: importId,
+        status: 'ready',
+        progress: inserted,
+        percent: 100,
+      })
+      captureServerEvent(
+        userId,
+        'table_import_completed',
+        {
+          table_id: tableId,
+          workspace_id: workspaceId,
+          import_id: importId,
+          status: 'completed',
+          row_count: inserted,
+        },
+        { groups: { workspace: workspaceId } }
+      )
+      logger.info(`[${requestId}] Import complete`, { tableId, fileName, mode, rows: inserted })
+    } else {
+      logger.info(
+        `[${requestId}] Import finished but no longer owns the run (canceled/superseded)`,
+        {
+          tableId,
+          importId,
+        }
+      )
+    }
+  } catch (err) {
+    if (err instanceof ImportSupersededError) {
+      // A newer import owns the table now — leave its status alone and just stop.
+      logger.info(`[${requestId}] Import superseded by a newer run; stopping`, {
+        tableId,
+        importId,
+      })
+    } else {
+      const message = getErrorMessage(err, 'Import failed')
+      logger.error(`[${requestId}] Import failed for table ${tableId}:`, err)
+      // Scoped to importId — a no-op if a newer import has taken over.
+      await markJobFailed(tableId, importId, message).catch(() => {})
+      void appendTableEvent({
+        kind: 'job',
+        type: 'import',
+        tableId,
+        jobId: importId,
+        status: 'failed',
+        error: message,
+      })
+      captureServerEvent(
+        userId,
+        'table_import_completed',
+        {
+          table_id: tableId,
+          workspace_id: workspaceId,
+          import_id: importId,
+          status: 'failed',
+          row_count: null,
+          error_message: truncate(message, 200),
+        },
+        { groups: { workspace: workspaceId } }
+      )
+    }
+  } finally {
+    // Release the storage stream so its HTTP connection doesn't leak on failure.
+    source?.destroy()
+    // The uploaded source file is single-use (a fresh upload per import) — delete it once the
+    // import is terminal so the workspace bucket doesn't accumulate. Best-effort. Skipped for
+    // persistent workspace files (deleteSourceFile: false).
+    if (payload.deleteSourceFile !== false) {
+      await deleteFile({ key: fileKey, context: 'workspace' }).catch((err) => {
+        logger.warn(`[${requestId}] Failed to delete imported file`, { fileKey, err })
+      })
+    }
+  }
+}

apps/sim/lib/table/import.test.ts

@@ -0,0 +1,323 @@
+/**
+ * @vitest-environment node
+ */
+import { Readable } from 'node:stream'
+import { describe, expect, it } from 'vitest'
+import {
+  buildAutoMapping,
+  CsvImportValidationError,
+  coerceRowsForTable,
+  coerceValue,
+  createCsvParser,
+  csvParseOptions,
+  inferColumnType,
+  inferSchemaFromCsv,
+  parseCsvBuffer,
+  sanitizeName,
+  validateMapping,
+} from '@/lib/table/import'
+import type { TableSchema } from '@/lib/table/types'
+
+describe('import', () => {
+  describe('parseCsvBuffer', () => {
+    it('parses a CSV string and extracts headers', async () => {
+      const { headers, rows } = await parseCsvBuffer('a,b\n1,2\n3,4')
+      expect(headers).toEqual(['a', 'b'])
+      expect(rows).toEqual([
+        { a: '1', b: '2' },
+        { a: '3', b: '4' },
+      ])
+    })
+
+    it('strips a UTF-8 BOM from the first header', async () => {
+      const text = `\uFEFFname,age\nAlice,30`
+      const { headers } = await parseCsvBuffer(text)
+      expect(headers).toEqual(['name', 'age'])
+    })
+
+    it('parses a Uint8Array input in browser-like environments', async () => {
+      const bytes = new TextEncoder().encode('a,b\n1,2')
+      const { headers, rows } = await parseCsvBuffer(bytes)
+      expect(headers).toEqual(['a', 'b'])
+      expect(rows).toHaveLength(1)
+    })
+
+    it('parses TSV when delimiter is tab', async () => {
+      const { headers, rows } = await parseCsvBuffer('a\tb\n1\t2', '\t')
+      expect(headers).toEqual(['a', 'b'])
+      expect(rows).toEqual([{ a: '1', b: '2' }])
+    })
+
+    it('throws when the file has no data rows', async () => {
+      await expect(parseCsvBuffer('a,b')).rejects.toThrow(/no data rows/i)
+    })
+  })
+
+  describe('inferColumnType', () => {
+    it('returns "string" for empty samples', () => {
+      expect(inferColumnType([])).toBe('string')
+      expect(inferColumnType([null, undefined, ''])).toBe('string')
+    })
+
+    it('detects numeric columns', () => {
+      expect(inferColumnType(['1', '2', '3.14'])).toBe('number')
+    })
+
+    it('detects boolean columns (case-insensitive)', () => {
+      expect(inferColumnType(['true', 'FALSE', 'True'])).toBe('boolean')
+    })
+
+    it('detects ISO date columns', () => {
+      expect(inferColumnType(['2024-01-01', '2024-02-01T12:00:00'])).toBe('date')
+    })
+
+    it('falls back to "string"', () => {
+      expect(inferColumnType(['abc', 'def'])).toBe('string')
+      expect(inferColumnType(['1', 'abc'])).toBe('string')
+    })
+  })
+
+  describe('sanitizeName', () => {
+    it('strips unsupported chars and collapses underscores', () => {
+      expect(sanitizeName('Hello World!')).toBe('Hello_World')
+      expect(sanitizeName('  foo-bar  ')).toBe('foo_bar')
+    })
+
+    it('prefixes names that start with a digit', () => {
+      expect(sanitizeName('123abc')).toBe('col_123abc')
+    })
+
+    it('fills in an empty name with the prefix', () => {
+      expect(sanitizeName('$$$')).toBe('col_')
+    })
+  })
+
+  describe('inferSchemaFromCsv', () => {
+    it('produces sanitized column names and inferred types', () => {
+      const { columns, headerToColumn } = inferSchemaFromCsv(
+        ['First Name', 'Age', 'Active'],
+        [
+          { 'First Name': 'Alice', Age: '30', Active: 'true' },
+          { 'First Name': 'Bob', Age: '40', Active: 'false' },
+        ]
+      )
+      expect(columns).toEqual([
+        { name: 'First_Name', type: 'string' },
+        { name: 'Age', type: 'number' },
+        { name: 'Active', type: 'boolean' },
+      ])
+      expect(headerToColumn.get('First Name')).toBe('First_Name')
+      expect(headerToColumn.get('Age')).toBe('Age')
+    })
+
+    it('disambiguates duplicate sanitized headers', () => {
+      const { columns } = inferSchemaFromCsv(
+        ['a b', 'a-b', 'a.b'],
+        [{ 'a b': '1', 'a-b': '2', 'a.b': '3' }]
+      )
+      expect(columns.map((c) => c.name)).toEqual(['a_b', 'a_b_2', 'a_b_3'])
+    })
+  })
+
+  describe('coerceValue', () => {
+    it('returns null for empty values', () => {
+      expect(coerceValue(null, 'string')).toBeNull()
+      expect(coerceValue(undefined, 'number')).toBeNull()
+      expect(coerceValue('', 'boolean')).toBeNull()
+    })
+
+    it('coerces numbers', () => {
+      expect(coerceValue('42', 'number')).toBe(42)
+      expect(coerceValue('not a number', 'number')).toBeNull()
+    })
+
+    it('coerces booleans strictly', () => {
+      expect(coerceValue('true', 'boolean')).toBe(true)
+      expect(coerceValue('FALSE', 'boolean')).toBe(false)
+      expect(coerceValue('yes', 'boolean')).toBeNull()
+    })
+
+    it('keeps date-only values as calendar dates, preserves datetime wall times with their offset, and falls back to the original string', () => {
+      expect(coerceValue('2024-01-01', 'date')).toBe('2024-01-01')
+      expect(coerceValue('2024-01-01T12:30:00-07:00', 'date')).toBe('2024-01-01T12:30:00-07:00')
+      expect(coerceValue('2024-01-01 12:30', 'date', { timezone: 'America/New_York' })).toBe(
+        '2024-01-01T12:30:00-05:00'
+      )
+      expect(coerceValue('not-a-date', 'date')).toBe('not-a-date')
+    })
+  })
+
+  describe('buildAutoMapping', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'First_Name', type: 'string' },
+        { name: 'age', type: 'number' },
+      ],
+    }
+
+    it('maps by exact sanitized name', () => {
+      const mapping = buildAutoMapping(['First_Name', 'age'], schema)
+      expect(mapping).toEqual({ First_Name: 'First_Name', age: 'age' })
+    })
+
+    it('falls back to a case/punctuation-insensitive match', () => {
+      const mapping = buildAutoMapping(['first name', 'AGE'], schema)
+      expect(mapping).toEqual({ 'first name': 'First_Name', AGE: 'age' })
+    })
+
+    it('returns null for headers without a match', () => {
+      const mapping = buildAutoMapping(['unmatched'], schema)
+      expect(mapping).toEqual({ unmatched: null })
+    })
+  })
+
+  describe('validateMapping', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string', required: true },
+        { name: 'age', type: 'number' },
+      ],
+    }
+
+    it('accepts a valid mapping and lists skipped/unmapped', () => {
+      const result = validateMapping({
+        csvHeaders: ['name', 'age', 'extra'],
+        mapping: { name: 'name', age: 'age', extra: null },
+        tableSchema: schema,
+      })
+      expect(result.mappedHeaders).toEqual(['name', 'age'])
+      expect(result.skippedHeaders).toEqual(['extra'])
+      expect(result.unmappedColumns).toEqual([])
+      expect(result.effectiveMap.get('name')).toBe('name')
+      expect(result.effectiveMap.has('extra')).toBe(false)
+    })
+
+    it('throws when a required column is missing', () => {
+      expect(() =>
+        validateMapping({
+          csvHeaders: ['age'],
+          mapping: { age: 'age' },
+          tableSchema: schema,
+        })
+      ).toThrow(CsvImportValidationError)
+    })
+
+    it('throws when a mapping targets a non-existent column', () => {
+      expect(() =>
+        validateMapping({
+          csvHeaders: ['name'],
+          mapping: { name: 'nonexistent' },
+          tableSchema: schema,
+        })
+      ).toThrow(/do not exist on the table/)
+    })
+
+    it('throws when multiple headers map to the same column', () => {
+      expect(() =>
+        validateMapping({
+          csvHeaders: ['a', 'b'],
+          mapping: { a: 'name', b: 'name' },
+          tableSchema: schema,
+        })
+      ).toThrow(/same column/)
+    })
+
+    it('throws when mapping references an unknown CSV header', () => {
+      expect(() =>
+        validateMapping({
+          csvHeaders: ['name'],
+          mapping: { name: 'name', bogus: 'age' },
+          tableSchema: schema,
+        })
+      ).toThrow(/unknown CSV headers/)
+    })
+
+    it('throws when a mapping value is neither a string nor null', () => {
+      expect(() =>
+        validateMapping({
+          csvHeaders: ['name'],
+          mapping: { name: 42 as unknown as string },
+          tableSchema: schema,
+        })
+      ).toThrow(/Mapping values must be/)
+    })
+  })
+
+  describe('coerceRowsForTable', () => {
+    const schema: TableSchema = {
+      columns: [
+        { name: 'name', type: 'string' },
+        { name: 'age', type: 'number' },
+        { name: 'active', type: 'boolean' },
+      ],
+    }
+
+    it('applies the table column type using the effective mapping', () => {
+      const rows = coerceRowsForTable(
+        [
+          { Name: 'Alice', Age: '30', Active: 'true' },
+          { Name: 'Bob', Age: 'oops', Active: 'false' },
+        ],
+        schema,
+        new Map([
+          ['Name', 'name'],
+          ['Age', 'age'],
+          ['Active', 'active'],
+        ])
+      )
+
+      expect(rows).toEqual([
+        { name: 'Alice', age: 30, active: true },
+        { name: 'Bob', age: null, active: false },
+      ])
+    })
+
+    it('drops CSV headers absent from the mapping', () => {
+      const rows = coerceRowsForTable(
+        [{ name: 'Alice', extra: 'keep me out' }],
+        schema,
+        new Map([['name', 'name']])
+      )
+      expect(rows).toEqual([{ name: 'Alice' }])
+    })
+  })
+
+  describe('createCsvParser', () => {
+    async function parseViaStream(csv: string, delimiter = ',') {
+      const parser = createCsvParser(delimiter)
+      Readable.from([csv]).pipe(parser)
+      const rows: Record<string, unknown>[] = []
+      for await (const record of parser as AsyncIterable<Record<string, unknown>>) {
+        rows.push(record)
+      }
+      return rows
+    }
+
+    it('streams records keyed by header, matching parseCsvBuffer', async () => {
+      const csv = 'name,age\nAlice,30\nBob,40\n'
+      const streamed = await parseViaStream(csv)
+      const { rows: buffered } = await parseCsvBuffer(csv)
+      expect(streamed).toEqual(buffered)
+      expect(streamed).toEqual([
+        { name: 'Alice', age: '30' },
+        { name: 'Bob', age: '40' },
+      ])
+    })
+
+    it('honors a TSV delimiter', async () => {
+      const rows = await parseViaStream('name\tage\nAlice\t30\n', '\t')
+      expect(rows).toEqual([{ name: 'Alice', age: '30' }])
+    })
+
+    it('strips a leading UTF-8 BOM', async () => {
+      const rows = await parseViaStream('name,age\nAlice,30\n')
+      expect(Object.keys(rows[0])).toEqual(['name', 'age'])
+    })
+  })
+
+  describe('csvParseOptions', () => {
+    it('sets columns, bom, and the delimiter', () => {
+      expect(csvParseOptions('\t')).toMatchObject({ columns: true, bom: true, delimiter: '\t' })
+    })
+  })
+})

apps/sim/lib/table/import.ts

@@ -0,0 +1,511 @@
+/**
+ * Shared CSV import helpers for user-defined tables.
+ *
+ * Used by:
+ * - `POST /api/table/import-csv` (create new table from CSV — streams via {@link createCsvParser})
+ * - `POST /api/table/[tableId]/import` (append/replace into existing table)
+ * - Copilot `user-table` tool (`create_from_file`, `import_file` — buffers via {@link parseCsvBuffer})
+ *
+ * Keeping a single implementation avoids drift between HTTP and agent code paths.
+ * Both the buffered ({@link parseCsvBuffer}) and streaming ({@link createCsvParser})
+ * parsers share {@link csvParseOptions} so their behavior can't drift.
+ */
+
+import { type Options as CsvParseOptions, type Parser, parse as parseCsvStream } from 'csv-parse'
+import { getColumnId } from '@/lib/table/column-keys'
+import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates'
+import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types'
+
+/**
+ * Single source of truth for the `csv-parse` options used by both the buffered
+ * sync parser and the streaming parser. `columns: true` emits each record as an
+ * object keyed by the (first-row) headers.
+ */
+export function csvParseOptions(delimiter = ','): CsvParseOptions {
+  return {
+    columns: true,
+    skip_empty_lines: true,
+    trim: true,
+    relax_column_count: true,
+    relax_quotes: true,
+    skip_records_with_error: true,
+    cast: false,
+    bom: true,
+    delimiter,
+  }
+}
+
+/**
+ * Returns a streaming `csv-parse` parser (a `Transform`/async-iterable). Pipe a
+ * file stream into it and iterate records with `for await`; backpressure flows
+ * back to the source while each record is processed. Use this for HTTP uploads
+ * so the file is never fully buffered in memory.
+ */
+export function createCsvParser(delimiter = ','): Parser {
+  return parseCsvStream(csvParseOptions(delimiter))
+}
+
+/** Narrower type than `COLUMN_TYPES` used internally for coercion. */
+export type CsvColumnType = 'string' | 'number' | 'boolean' | 'date' | 'json'
+
+/** Number of CSV rows sampled when inferring column types for a new table. */
+export const CSV_SCHEMA_SAMPLE_SIZE = 100
+
+/**
+ * Maximum rows inserted per import batch. Each batch is one `INSERT … VALUES` statement, and
+ * Postgres caps bind parameters at 65,535 — at 9 params per row that's a hard ceiling of ~7,200
+ * rows, so 5,000 keeps a margin while cutting per-batch overhead (validation, unique-constraint
+ * check, ownership heartbeat) 5× vs the old 1,000.
+ */
+export const CSV_MAX_BATCH_SIZE = 5000
+
+/** Maximum CSV/TSV file size accepted by import routes (25 MB). */
+export const CSV_MAX_FILE_SIZE_BYTES = 25 * 1024 * 1024
+
+/**
+ * Error thrown when the user-supplied mapping or CSV does not line up with the
+ * target table. Callers should translate this into a 400 response.
+ */
+export class CsvImportValidationError extends Error {
+  readonly code = 'CSV_IMPORT_VALIDATION' as const
+  readonly details: {
+    missingRequired?: string[]
+    duplicateTargets?: string[]
+    unknownColumns?: string[]
+    unknownHeaders?: string[]
+  }
+
+  constructor(
+    message: string,
+    details: {
+      missingRequired?: string[]
+      duplicateTargets?: string[]
+      unknownColumns?: string[]
+      unknownHeaders?: string[]
+    } = {}
+  ) {
+    super(message)
+    this.name = 'CsvImportValidationError'
+    this.details = details
+  }
+}
+
+/**
+ * Parses a CSV/TSV payload using `csv-parse/sync`. Accepts a Node `Buffer`,
+ * browser-friendly `Uint8Array`, or already-decoded string. A leading UTF-8 BOM
+ * is stripped by csv-parse (`bom: true` in {@link csvParseOptions}).
+ *
+ * For HTTP uploads prefer {@link createCsvParser} so the file isn't buffered.
+ */
+export async function parseCsvBuffer(
+  input: Buffer | Uint8Array | string,
+  delimiter = ','
+): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
+  const { parse } = await import('csv-parse/sync')
+
+  let text: string
+  if (typeof input === 'string') {
+    text = input
+  } else if (typeof Buffer !== 'undefined' && Buffer.isBuffer(input)) {
+    text = input.toString('utf-8')
+  } else {
+    text = new TextDecoder('utf-8').decode(input as Uint8Array)
+  }
+
+  // double-cast-allowed: shared csvParseOptions() loses the `columns: true` literal that drives
+  // csv-parse's record-vs-string[][] overload, but `columns: true` is always set so records are objects
+  const parsed = parse(text, csvParseOptions(delimiter)) as unknown as Record<string, unknown>[]
+
+  if (parsed.length === 0) {
+    throw new Error('CSV file has no data rows')
+  }
+
+  const headers = Object.keys(parsed[0])
+  if (headers.length === 0) {
+    throw new Error('CSV file has no headers')
+  }
+
+  return { headers, rows: parsed }
+}
+
+/**
+ * Infers a column type from a sample of non-empty values. Order matters: we
+ * prefer narrower types (number > boolean > ISO date) and fall back to string.
+ * JSON is never inferred automatically.
+ */
+export function inferColumnType(values: unknown[]): Exclude<CsvColumnType, 'json'> {
+  const nonEmpty = values.filter((v) => v !== null && v !== undefined && v !== '')
+  if (nonEmpty.length === 0) return 'string'
+
+  const allNumber = nonEmpty.every((v) => {
+    const n = Number(v)
+    return !Number.isNaN(n) && String(v).trim() !== ''
+  })
+  if (allNumber) return 'number'
+
+  const allBoolean = nonEmpty.every((v) => {
+    const s = String(v).toLowerCase()
+    return s === 'true' || s === 'false'
+  })
+  if (allBoolean) return 'boolean'
+
+  const isoDatePattern = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?)?/
+  const allDate = nonEmpty.every((v) => {
+    const s = String(v)
+    return isoDatePattern.test(s) && !Number.isNaN(Date.parse(s))
+  })
+  if (allDate) return 'date'
+
+  return 'string'
+}
+
+/**
+ * Sanitizes a raw header into a valid column/table name. Strips disallowed
+ * characters, collapses runs of underscores, and ensures the first character
+ * is a letter or underscore (prefixing with `fallbackPrefix` otherwise).
+ */
+export function sanitizeName(raw: string, fallbackPrefix = 'col'): string {
+  let name = raw
+    .trim()
+    .replace(/[^a-zA-Z0-9_]/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^_+|_+$/g, '')
+
+  if (!name || /^\d/.test(name)) {
+    name = `${fallbackPrefix}_${name}`
+  }
+
+  return name
+}
+
+/**
+ * Returns column definitions inferred from CSV headers + sample rows. Duplicate
+ * sanitized names are suffixed with `_2`, `_3`, etc. Also returns the header ->
+ * column-name mapping used when coercing row values.
+ */
+export function inferSchemaFromCsv(
+  headers: string[],
+  rows: Record<string, unknown>[]
+): { columns: ColumnDefinition[]; headerToColumn: Map<string, string> } {
+  const sample = rows.slice(0, CSV_SCHEMA_SAMPLE_SIZE)
+  const seen = new Set<string>()
+  const headerToColumn = new Map<string, string>()
+
+  const columns = headers.map((header) => {
+    const base = sanitizeName(header)
+    let colName = base
+    let suffix = 2
+    while (seen.has(colName.toLowerCase())) {
+      colName = `${base}_${suffix}`
+      suffix++
+    }
+    seen.add(colName.toLowerCase())
+    headerToColumn.set(header, colName)
+
+    return {
+      name: colName,
+      type: inferColumnType(sample.map((r) => r[header])),
+    } satisfies ColumnDefinition
+  })
+
+  return { columns, headerToColumn }
+}
+
+/**
+ * Coerces a single value to the requested column type. Returns `null` for
+ * empty inputs or values that cannot be parsed (numbers/booleans). Dates fall
+ * back to the original string when unparseable so that schema validation can
+ * reject it with context rather than silently inserting `null`.
+ */
+export function coerceValue(
+  value: unknown,
+  colType: CsvColumnType,
+  options?: NormalizeDateCellOptions
+): string | number | boolean | null | Record<string, unknown> | unknown[] {
+  if (value === null || value === undefined || value === '') return null
+  switch (colType) {
+    case 'number': {
+      const n = Number(value)
+      return Number.isNaN(n) ? null : n
+    }
+    case 'boolean': {
+      const s = String(value).toLowerCase()
+      if (s === 'true') return true
+      if (s === 'false') return false
+      return null
+    }
+    case 'date': {
+      return normalizeDateCellValue(String(value), options) ?? String(value)
+    }
+    case 'json': {
+      if (typeof value === 'object') return value as Record<string, unknown> | unknown[]
+      try {
+        return JSON.parse(String(value))
+      } catch {
+        return String(value)
+      }
+    }
+    default:
+      return String(value)
+  }
+}
+
+/**
+ * Mapping from raw CSV header to target column name, with `null` indicating
+ * "do not import this column".
+ */
+export type CsvHeaderMapping = Record<string, string | null>
+
+export interface CsvMappingValidationResult {
+  /** Columns present in the CSV that landed on a real table column. */
+  mappedHeaders: string[]
+  /** Columns in the CSV that the user/client chose to skip. */
+  skippedHeaders: string[]
+  /** Target column names that ended up unmapped (resolved from the mapping). */
+  unmappedColumns: string[]
+  /** Effective header -> column map (after dropping unknown / null targets). */
+  effectiveMap: Map<string, string>
+}
+
+/**
+ * Validates a user-supplied mapping against the target table schema. Rejects
+ * unknown target columns, duplicate targets, and required table columns that
+ * are not covered by the CSV. Returns the normalized header -> column map.
+ */
+export function validateMapping(params: {
+  csvHeaders: string[]
+  mapping: CsvHeaderMapping
+  tableSchema: TableSchema
+}): CsvMappingValidationResult {
+  const { csvHeaders, mapping, tableSchema } = params
+  const columnByName = new Map(tableSchema.columns.map((c) => [c.name, c]))
+
+  const unknownHeaders = Object.keys(mapping).filter((h) => !csvHeaders.includes(h))
+  if (unknownHeaders.length > 0) {
+    throw new CsvImportValidationError(
+      `Mapping references unknown CSV headers: ${unknownHeaders.join(', ')}`,
+      { unknownHeaders }
+    )
+  }
+
+  const invalidTargets = Object.entries(mapping).filter(
+    ([, target]) => target !== null && typeof target !== 'string'
+  )
+  if (invalidTargets.length > 0) {
+    throw new CsvImportValidationError(
+      `Mapping values must be a column name (string) or null, got: ${invalidTargets
+        .map(([header]) => header)
+        .join(', ')}`
+    )
+  }
+
+  const targetsSeen = new Map<string, string[]>()
+  const unknownColumns: string[] = []
+  const effectiveMap = new Map<string, string>()
+  const skippedHeaders: string[] = []
+
+  for (const header of csvHeaders) {
+    const target = header in mapping ? mapping[header] : undefined
+    if (target === null || target === undefined) {
+      skippedHeaders.push(header)
+      continue
+    }
+    if (!columnByName.has(target)) {
+      unknownColumns.push(target)
+      continue
+    }
+    const existing = targetsSeen.get(target) ?? []
+    existing.push(header)
+    targetsSeen.set(target, existing)
+    effectiveMap.set(header, target)
+  }
+
+  if (unknownColumns.length > 0) {
+    throw new CsvImportValidationError(
+      `Mapping references columns that do not exist on the table: ${unknownColumns.join(', ')}`,
+      { unknownColumns }
+    )
+  }
+
+  const duplicateTargets = [...targetsSeen.entries()]
+    .filter(([, headers]) => headers.length > 1)
+    .map(([col]) => col)
+  if (duplicateTargets.length > 0) {
+    throw new CsvImportValidationError(
+      `Multiple CSV headers map to the same column(s): ${duplicateTargets.join(', ')}`,
+      { duplicateTargets }
+    )
+  }
+
+  const mappedTargets = new Set(effectiveMap.values())
+  const unmappedColumns = tableSchema.columns
+    .filter((c) => !mappedTargets.has(c.name))
+    .map((c) => c.name)
+
+  const missingRequired = tableSchema.columns
+    .filter((c) => c.required && !mappedTargets.has(c.name))
+    .map((c) => c.name)
+  if (missingRequired.length > 0) {
+    throw new CsvImportValidationError(
+      `CSV is missing required columns: ${missingRequired.join(', ')}`,
+      { missingRequired }
+    )
+  }
+
+  return {
+    mappedHeaders: [...effectiveMap.keys()],
+    skippedHeaders,
+    unmappedColumns,
+    effectiveMap,
+  }
+}
+
+/**
+ * Builds an auto-mapping from CSV headers to table columns: prefers exact
+ * sanitized-name matches and falls back to a case- and punctuation-insensitive
+ * comparison. Unmapped headers are set to `null`.
+ */
+export function buildAutoMapping(csvHeaders: string[], tableSchema: TableSchema): CsvHeaderMapping {
+  const mapping: CsvHeaderMapping = {}
+  const columns = tableSchema.columns
+
+  const exactByName = new Map(columns.map((c) => [c.name, c.name]))
+  const loose = new Map<string, string>()
+  for (const col of columns) {
+    loose.set(col.name.toLowerCase().replace(/[^a-z0-9]/g, ''), col.name)
+  }
+
+  const usedTargets = new Set<string>()
+
+  for (const header of csvHeaders) {
+    const sanitized = sanitizeName(header)
+    const exact = exactByName.get(sanitized)
+    if (exact && !usedTargets.has(exact)) {
+      mapping[header] = exact
+      usedTargets.add(exact)
+      continue
+    }
+    const key = header.toLowerCase().replace(/[^a-z0-9]/g, '')
+    const fuzzy = loose.get(key)
+    if (fuzzy && !usedTargets.has(fuzzy)) {
+      mapping[header] = fuzzy
+      usedTargets.add(fuzzy)
+      continue
+    }
+    mapping[header] = null
+  }
+
+  return mapping
+}
+
+/**
+ * Coerces parsed CSV rows into `RowData` objects keyed by the target column's
+ * **stable id** (the row-data storage key), applying the column types declared in
+ * `tableSchema`. Headers not present in `headerToColumn` are dropped. Missing
+ * table columns remain unset (schema validation decides whether that's
+ * acceptable). Pass the schema returned by `createTable` so ids are resolved.
+ */
+export function coerceRowsForTable(
+  rows: Record<string, unknown>[],
+  tableSchema: TableSchema,
+  headerToColumn: Map<string, string>,
+  options?: NormalizeDateCellOptions
+): RowData[] {
+  const colByName = new Map(tableSchema.columns.map((c) => [c.name, c]))
+
+  return rows.map((row) => {
+    const coerced: RowData = {}
+    for (const [header, value] of Object.entries(row)) {
+      const colName = headerToColumn.get(header)
+      if (!colName) continue
+      const col = colByName.get(colName)
+      if (!col) continue
+      const colType = (col.type as CsvColumnType) ?? 'string'
+      coerced[getColumnId(col)] = coerceValue(value, colType, options) as RowData[string]
+    }
+    return coerced
+  })
+}
+
+/**
+ * Sanitizes raw JSON keys so they conform to the same column-name rules as CSV
+ * headers, letting `inferSchemaFromCsv` and `coerceRowsForTable` be reused for
+ * JSON imports. Collisions after sanitization are disambiguated with a trailing
+ * underscore. Returns the headers and rows untouched when no key needs renaming.
+ */
+export function sanitizeJsonHeaders(
+  headers: string[],
+  rows: Record<string, unknown>[]
+): { headers: string[]; rows: Record<string, unknown>[] } {
+  const renamed = new Map<string, string>()
+  const seen = new Set<string>()
+
+  for (const raw of headers) {
+    let safe = sanitizeName(raw)
+    while (seen.has(safe)) safe = `${safe}_`
+    seen.add(safe)
+    renamed.set(raw, safe)
+  }
+
+  const noChange = headers.every((h) => renamed.get(h) === h)
+  if (noChange) return { headers, rows }
+
+  return {
+    headers: headers.map((h) => renamed.get(h)!),
+    rows: rows.map((row) => {
+      const out: Record<string, unknown> = {}
+      for (const [raw, safe] of renamed) {
+        if (raw in row) out[safe] = row[raw]
+      }
+      return out
+    }),
+  }
+}
+
+/**
+ * Parses a JSON payload that must be an array of plain objects into the same
+ * `{ headers, rows }` shape produced by `parseCsvBuffer`. The header set is the
+ * union of all object keys, sanitized via {@link sanitizeJsonHeaders}.
+ */
+export function parseJsonRows(buffer: Buffer | string): {
+  headers: string[]
+  rows: Record<string, unknown>[]
+} {
+  const text = typeof buffer === 'string' ? buffer : buffer.toString('utf-8')
+  const parsed = JSON.parse(text)
+  if (!Array.isArray(parsed)) {
+    throw new Error('JSON file must contain an array of objects')
+  }
+  if (parsed.length === 0) {
+    throw new Error('JSON file contains an empty array')
+  }
+  const headerSet = new Set<string>()
+  for (const row of parsed) {
+    if (typeof row !== 'object' || row === null || Array.isArray(row)) {
+      throw new Error('Each element in the JSON array must be a plain object')
+    }
+    for (const key of Object.keys(row)) headerSet.add(key)
+  }
+  return sanitizeJsonHeaders([...headerSet], parsed)
+}
+
+/**
+ * Parses a tabular upload (CSV, TSV, or JSON array-of-objects) into a uniform
+ * `{ headers, rows }` shape, dispatching on file extension and falling back to
+ * the MIME content type. Throws on unsupported formats so callers fail fast.
+ */
+export async function parseFileRows(
+  buffer: Buffer,
+  fileName: string,
+  contentType?: string
+): Promise<{ headers: string[]; rows: Record<string, unknown>[] }> {
+  const ext = fileName.split('.').pop()?.toLowerCase()
+  if (ext === 'json' || contentType === 'application/json') {
+    return parseJsonRows(buffer)
+  }
+  if (ext === 'csv' || ext === 'tsv' || contentType === 'text/csv') {
+    const delimiter = ext === 'tsv' ? '\t' : ','
+    return parseCsvBuffer(buffer, delimiter)
+  }
+  throw new Error(`Unsupported file format: "${ext ?? fileName}". Supported: csv, tsv, json`)
+}

apps/sim/lib/table/index.ts

@@ -0,0 +1,23 @@
+/**
+ * Table utilities module.
+ *
+ * Hooks are not re-exported here to avoid pulling React into server code.
+ * Import hooks directly from '@/lib/table/hooks' in client components.
+ */
+
+export * from '@/lib/table/billing'
+export * from '@/lib/table/column-keys'
+export * from '@/lib/table/columns/service'
+export * from '@/lib/table/constants'
+export * from '@/lib/table/dates'
+export * from '@/lib/table/import'
+export * from '@/lib/table/import-data'
+export * from '@/lib/table/jobs/service'
+export * from '@/lib/table/llm'
+export * from '@/lib/table/query-builder'
+export * from '@/lib/table/rows/service'
+export * from '@/lib/table/service'
+export * from '@/lib/table/sql'
+export * from '@/lib/table/types'
+export * from '@/lib/table/validation'
+export * from '@/lib/table/workflow-groups/service'

apps/sim/lib/table/jobs/service.ts

@@ -0,0 +1,383 @@
+/**
+ * Table background-job service for user tables.
+ *
+ * The `table_jobs` state machine (claim / progress / terminal transitions), the
+ * latest-job reads that enrich a {@link TableDefinition}, and the export-job read
+ * paths — extracted from the table service. Operates purely on the `table_jobs`
+ * table (plus `selectExportRowPage`, which pages rows through the shared
+ * `pendingDeleteMask`), so it never imports the table-root service.
+ *
+ * Use this for: workflow executor, background jobs, testing business logic.
+ * Use API routes for: HTTP requests, frontend clients.
+ */
+
+import { db } from '@sim/db'
+import { tableJobs, userTableDefinitions, userTableRows } from '@sim/db/schema'
+import { and, asc, desc, eq, gt, inArray, ne, or, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { pendingDeleteMask } from '@/lib/table/rows/service'
+import type {
+  RowData,
+  TableDefinition,
+  TableDeleteJobPayload,
+  TableExportJobPayload,
+  TableJobType,
+} from '@/lib/table/types'
+
+/** Job fields projected onto a {@link TableDefinition}, derived from its latest `table_jobs` row. */
+interface DerivedJobFields {
+  jobStatus: TableDefinition['jobStatus']
+  jobId: string | null
+  jobType: TableDefinition['jobType']
+  jobError: string | null
+  jobRowsProcessed: number
+  /**
+   * Rows a running delete job still has to remove (its doomed estimate minus
+   * deletions so far). Internal to count adjustment — callers subtract it from
+   * the raw `row_count` so list/detail counts match the read path's delete
+   * mask (a mid-delete refresh must not resurrect the count). Not on the wire.
+   */
+  pendingDeleteRemaining: number
+}
+
+export const EMPTY_JOB_FIELDS: DerivedJobFields = {
+  jobStatus: null,
+  jobId: null,
+  jobType: null,
+  jobError: null,
+  jobRowsProcessed: 0,
+  pendingDeleteRemaining: 0,
+}
+
+function mapJobRow(
+  row:
+    | {
+        id: string
+        type: string
+        status: string
+        rowsProcessed: number
+        error: string | null
+        payload: unknown
+      }
+    | undefined
+): DerivedJobFields {
+  if (!row) return EMPTY_JOB_FIELDS
+  const doomedCount =
+    row.type === 'delete' && row.status === 'running'
+      ? ((row.payload as TableDeleteJobPayload | null)?.doomedCount ?? 0)
+      : 0
+  return {
+    jobStatus: row.status as TableDefinition['jobStatus'],
+    jobId: row.id,
+    jobType: row.type as TableDefinition['jobType'],
+    jobError: row.error,
+    jobRowsProcessed: row.rowsProcessed,
+    pendingDeleteRemaining: Math.max(0, doomedCount - row.rowsProcessed),
+  }
+}
+
+const JOB_PROJECTION = {
+  id: tableJobs.id,
+  type: tableJobs.type,
+  status: tableJobs.status,
+  rowsProcessed: tableJobs.rowsProcessed,
+  error: tableJobs.error,
+  payload: tableJobs.payload,
+} as const
+
+/**
+ * The latest job for one table (the running one if present, else the most recent terminal).
+ * Exports are excluded: they're read-only, run concurrently with other jobs, and have their own
+ * client surface — surfacing one here would clobber the import/delete/backfill status the tray
+ * and SSE consumer derive from these fields.
+ */
+export async function latestJobForTable(
+  tableId: string,
+  executor: DbOrTx = db
+): Promise<DerivedJobFields> {
+  const [row] = await executor
+    .select(JOB_PROJECTION)
+    .from(tableJobs)
+    .where(and(eq(tableJobs.tableId, tableId), ne(tableJobs.type, 'export')))
+    .orderBy(desc(tableJobs.startedAt))
+    .limit(1)
+  return mapJobRow(row)
+}
+
+/** Latest non-export job per table for a batch of ids, via `DISTINCT ON (table_id)`. */
+export async function latestJobsForTables(
+  tableIds: string[]
+): Promise<Map<string, DerivedJobFields>> {
+  const map = new Map<string, DerivedJobFields>()
+  if (tableIds.length === 0) return map
+  const rows = await db
+    .selectDistinctOn([tableJobs.tableId], { tableId: tableJobs.tableId, ...JOB_PROJECTION })
+    .from(tableJobs)
+    .where(and(inArray(tableJobs.tableId, tableIds), ne(tableJobs.type, 'export')))
+    .orderBy(tableJobs.tableId, desc(tableJobs.startedAt))
+  for (const row of rows) map.set(row.tableId, mapJobRow(row))
+  return map
+}
+
+/**
+ * Atomically claims a table's single background-job slot by inserting a `running` row into
+ * `table_jobs`. The partial-unique index on `table_id WHERE status = 'running'` is the
+ * concurrency gate: a second insert while a job runs hits `ON CONFLICT DO NOTHING` and returns no
+ * row, so import and delete (and two imports) are mutually exclusive for free. Returns whether it
+ * claimed the slot; the caller returns 409 when it didn't.
+ */
+export async function markTableJobRunning(
+  tableId: string,
+  jobId: string,
+  type: TableJobType,
+  /** Type-specific scope persisted to `table_jobs.payload` (e.g. {@link TableDeleteJobPayload})
+   *  so read paths can mask the job's effect while it runs. */
+  payload?: unknown
+): Promise<boolean> {
+  // workspace_id is immutable; the atomic gate is the INSERT's conflict, not this read.
+  const [def] = await db
+    .select({ workspaceId: userTableDefinitions.workspaceId })
+    .from(userTableDefinitions)
+    .where(eq(userTableDefinitions.id, tableId))
+    .limit(1)
+  if (!def) return false
+  const inserted = await db
+    .insert(tableJobs)
+    .values({
+      id: jobId,
+      tableId,
+      workspaceId: def.workspaceId,
+      type,
+      status: 'running',
+      payload: payload ?? null,
+    })
+    .onConflictDoNothing()
+    .returning({ id: tableJobs.id })
+  return inserted.length > 0
+}
+
+/**
+ * Releases a claim taken by {@link markTableJobRunning} for a synchronous job — deletes the
+ * transient claim row. Scoped to `jobId` + still-running so it only clears its own claim, never a
+ * newer run. A sync route claims, writes, then releases here in a `finally`.
+ */
+export async function releaseJobClaim(tableId: string, jobId: string): Promise<void> {
+  await db
+    .delete(tableJobs)
+    .where(
+      and(eq(tableJobs.id, jobId), eq(tableJobs.tableId, tableId), eq(tableJobs.status, 'running'))
+    )
+}
+
+/**
+ * Records job progress (rows processed so far) and bumps `updated_at` so the stale-job janitor
+ * (`cleanup-stale-executions`) sees a live heartbeat.
+ *
+ * Scoped to `jobId` AND `status = 'running'`: a stale/superseded worker no longer matches (its
+ * write is a no-op), and once the job is terminal (e.g. canceled) the match fails too — so this
+ * returning `false` is the worker's signal to stop. Returns whether this worker still owns an
+ * in-flight job.
+ */
+export async function updateJobProgress(
+  tableId: string,
+  rowsProcessed: number,
+  jobId: string
+): Promise<boolean> {
+  const updated = await db
+    .update(tableJobs)
+    .set({ rowsProcessed, updatedAt: new Date() })
+    .where(ownsActiveJob(tableId, jobId))
+    .returning({ id: tableJobs.id })
+  return updated.length > 0
+}
+
+/**
+ * Reads the persisted progress of an in-flight job this worker still owns (`null` when the job
+ * was canceled/superseded). A retried run seeds its counter from this so progress stays
+ * cumulative — earlier attempts' batches are already committed, and restarting from zero would
+ * clobber `rows_processed` (and every count derived from it) with the retry's smaller number.
+ */
+export async function getJobProgress(tableId: string, jobId: string): Promise<number | null> {
+  const [job] = await db
+    .select({ rowsProcessed: tableJobs.rowsProcessed })
+    .from(tableJobs)
+    .where(ownsActiveJob(tableId, jobId))
+    .limit(1)
+  return job ? job.rowsProcessed : null
+}
+
+/**
+ * One keyset page of rows for the export worker, ordered by `(order_key, id)` — the same
+ * authoritative visual order the grid (`queryRows`) uses, so exports and snapshots match what the
+ * user sees even after manual reorders. Keyset (not OFFSET) keeps each page O(page); `order_key` is
+ * present on every row (always assigned on insert, backfilled for legacy rows) with `id` as the
+ * tiebreaker, and the `(table_id, order_key, id)` index serves it. The delete-job visibility mask
+ * applies, like every user-facing read.
+ */
+export async function selectExportRowPage(
+  table: TableDefinition,
+  after: { orderKey: string; id: string } | null,
+  limit: number
+): Promise<Array<{ id: string; data: RowData; orderKey: string }>> {
+  const deleteMask = await pendingDeleteMask(table)
+  const rows = await db
+    .select({ id: userTableRows.id, data: userTableRows.data, orderKey: userTableRows.orderKey })
+    .from(userTableRows)
+    .where(
+      and(
+        eq(userTableRows.tableId, table.id),
+        eq(userTableRows.workspaceId, table.workspaceId),
+        deleteMask,
+        after
+          ? sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})`
+          : undefined
+      )
+    )
+    .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id))
+    .limit(limit)
+  return rows as Array<{ id: string; data: RowData; orderKey: string }>
+}
+
+/** How long a terminal export stays listable (and re-downloadable from the tray). */
+const EXPORT_JOB_VISIBILITY_MS = 10 * 60 * 1000
+
+export interface WorkspaceExportJob {
+  jobId: string
+  tableId: string
+  tableName: string
+  status: string
+  rowsProcessed: number
+  format: 'csv' | 'json'
+  hasResult: boolean
+  error: string | null
+}
+
+/**
+ * Export jobs the tray surfaces for a workspace: everything running, plus terminals from the last
+ * {@link EXPORT_JOB_VISIBILITY_MS} so a just-finished export stays re-downloadable. Exports live
+ * outside the table-level job derivation (which excludes them), so this is their read path.
+ */
+export async function listWorkspaceExportJobs(workspaceId: string): Promise<WorkspaceExportJob[]> {
+  const visibilityCutoff = new Date(Date.now() - EXPORT_JOB_VISIBILITY_MS)
+  const rows = await db
+    .select({
+      jobId: tableJobs.id,
+      tableId: tableJobs.tableId,
+      tableName: userTableDefinitions.name,
+      status: tableJobs.status,
+      rowsProcessed: tableJobs.rowsProcessed,
+      payload: tableJobs.payload,
+      error: tableJobs.error,
+    })
+    .from(tableJobs)
+    .innerJoin(userTableDefinitions, eq(userTableDefinitions.id, tableJobs.tableId))
+    .where(
+      and(
+        eq(tableJobs.workspaceId, workspaceId),
+        eq(tableJobs.type, 'export'),
+        or(eq(tableJobs.status, 'running'), gt(tableJobs.updatedAt, visibilityCutoff))
+      )
+    )
+    .orderBy(desc(tableJobs.startedAt))
+  return rows.map((r) => {
+    const payload = r.payload as TableExportJobPayload | null
+    return {
+      jobId: r.jobId,
+      tableId: r.tableId,
+      tableName: r.tableName,
+      status: r.status,
+      rowsProcessed: r.rowsProcessed,
+      format: payload?.format ?? 'csv',
+      hasResult: Boolean(payload?.resultKey),
+      error: r.error,
+    }
+  })
+}
+
+/** Reads one job row (type/status/payload) scoped to its table. Null when absent. */
+export async function getTableJob(
+  tableId: string,
+  jobId: string
+): Promise<{ id: string; type: string; status: string; payload: unknown } | null> {
+  const [job] = await db
+    .select({
+      id: tableJobs.id,
+      type: tableJobs.type,
+      status: tableJobs.status,
+      payload: tableJobs.payload,
+    })
+    .from(tableJobs)
+    .where(and(eq(tableJobs.id, jobId), eq(tableJobs.tableId, tableId)))
+    .limit(1)
+  return job ?? null
+}
+
+/**
+ * Stamps an export job's generated-file storage key onto its payload (`{ resultKey }` merge).
+ * Scoped to the still-running job so a superseded attempt can't clobber a newer run's result.
+ * The download route reads it; the janitor deletes the file when the terminal job is pruned.
+ */
+export async function setJobResultKey(
+  tableId: string,
+  jobId: string,
+  resultKey: string
+): Promise<void> {
+  await db
+    .update(tableJobs)
+    .set({
+      payload: sql`coalesce(${tableJobs.payload}, '{}'::jsonb) || jsonb_build_object('resultKey', ${resultKey}::text)`,
+      updatedAt: new Date(),
+    })
+    .where(ownsActiveJob(tableId, jobId))
+}
+
+/** Shared WHERE for terminal transitions: this job run, and still in-flight (write-once). */
+function ownsActiveJob(tableId: string, jobId: string) {
+  return and(
+    eq(tableJobs.id, jobId),
+    eq(tableJobs.tableId, tableId),
+    eq(tableJobs.status, 'running')
+  )
+}
+
+/**
+ * Marks a job complete. No-op unless it's still this in-flight run. Returns whether it
+ * transitioned, so the worker only emits the `ready` event when it actually won (and not after a
+ * cancel / supersede).
+ */
+export async function markJobReady(tableId: string, jobId: string): Promise<boolean> {
+  const now = new Date()
+  const updated = await db
+    .update(tableJobs)
+    .set({ status: 'ready', error: null, completedAt: now, updatedAt: now })
+    .where(ownsActiveJob(tableId, jobId))
+    .returning({ id: tableJobs.id })
+  return updated.length > 0
+}
+
+/**
+ * Marks a job failed, leaving any already-committed work in place. No-op unless it's still this
+ * in-flight run (so a stale worker can't clobber a newer job or a cancel).
+ */
+export async function markJobFailed(tableId: string, jobId: string, error: string): Promise<void> {
+  const now = new Date()
+  await db
+    .update(tableJobs)
+    .set({ status: 'failed', error: error.slice(0, 2000), completedAt: now, updatedAt: now })
+    .where(ownsActiveJob(tableId, jobId))
+}
+
+/**
+ * Marks an in-flight job canceled (user-initiated). No-op unless it's still running. The
+ * worker's next ownership check then returns `false` and it stops; committed work is left in
+ * place (no rollback). Returns whether a running job was actually canceled.
+ */
+export async function markJobCanceled(tableId: string, jobId: string): Promise<boolean> {
+  const now = new Date()
+  const updated = await db
+    .update(tableJobs)
+    .set({ status: 'canceled', completedAt: now, updatedAt: now })
+    .where(ownsActiveJob(tableId, jobId))
+    .returning({ id: tableJobs.id })
+  return updated.length > 0
+}

apps/sim/lib/table/llm/enrichment.ts

@@ -0,0 +1,201 @@
+/**
+ * LLM tool enrichment utilities for table operations.
+ *
+ * Provides functions to enrich tool descriptions and parameter schemas
+ * with table-specific information so LLMs can construct proper queries.
+ */
+
+import type { TableSummary } from '@/lib/table/types'
+
+/**
+ * Operations that use filters and need filter-specific enrichment.
+ */
+export const FILTER_OPERATIONS = new Set([
+  'table_query_rows',
+  'table_update_rows_by_filter',
+  'table_delete_rows_by_filter',
+])
+
+/**
+ * Operations that need column info for data construction.
+ */
+export const DATA_OPERATIONS = new Set([
+  'table_insert_row',
+  'table_batch_insert_rows',
+  'table_upsert_row',
+  'table_update_row',
+])
+
+/**
+ * Enriches a table tool description with table information based on the operation type.
+ */
+export function enrichTableToolDescription(
+  originalDescription: string,
+  table: TableSummary,
+  toolId: string
+): string {
+  if (!table.columns || table.columns.length === 0) {
+    return originalDescription
+  }
+
+  const columnList = table.columns.map((col) => `  - ${col.name} (${col.type})`).join('\n')
+
+  if (FILTER_OPERATIONS.has(toolId)) {
+    const stringCols = table.columns.filter((c) => c.type === 'string')
+    const numberCols = table.columns.filter((c) => c.type === 'number')
+
+    let filterExample = ''
+    if (stringCols.length > 0 && numberCols.length > 0) {
+      filterExample = `
+
+Example filter: {"${stringCols[0].name}": {"$eq": "value"}, "${numberCols[0].name}": {"$lt": 50}}`
+    } else if (stringCols.length > 0) {
+      filterExample = `
+
+Example filter: {"${stringCols[0].name}": {"$eq": "value"}}`
+    }
+
+    let sortExample = ''
+    if (toolId === 'table_query_rows' && numberCols.length > 0) {
+      sortExample = `
+Example sort: {"${numberCols[0].name}": "desc"} for highest first, {"${numberCols[0].name}": "asc"} for lowest first`
+    }
+
+    const queryInstructions =
+      toolId === 'table_query_rows'
+        ? `
+INSTRUCTIONS:
+1. ALWAYS include a filter based on the user's question - queries without filters will fail
+2. Construct the filter yourself from the user's question - do NOT ask for confirmation
+3. Use exact match ($eq) by default unless the user specifies otherwise
+4. For ranking queries (highest, lowest, Nth, top N):
+   - ALWAYS use sort with the relevant column (e.g., {"salary": "desc"} for highest salary)
+   - Use limit to get only the needed rows (e.g., limit=1 for highest, limit=2 for second highest)
+   - For "second highest X", use sort: {"X": "desc"} with limit: 2, then take the second result
+5. Only use limit=1000 when you need ALL matching rows`
+        : `
+INSTRUCTIONS:
+1. ALWAYS include a filter based on the user's question - queries without filters will fail
+2. Construct the filter yourself from the user's question - do NOT ask for confirmation
+3. Use exact match ($eq) by default unless the user specifies otherwise`
+
+    return `${originalDescription}
+${queryInstructions}
+
+Table "${table.name}" columns:
+${columnList}
+${filterExample}${sortExample}`
+  }
+
+  if (DATA_OPERATIONS.has(toolId)) {
+    const exampleCols = table.columns.slice(0, 3)
+    const dataExample = exampleCols.reduce(
+      (obj, col) => {
+        obj[col.name] = col.type === 'number' ? 123 : col.type === 'boolean' ? true : 'example'
+        return obj
+      },
+      {} as Record<string, unknown>
+    )
+
+    if (toolId === 'table_update_row') {
+      return `${originalDescription}
+
+Table "${table.name}" available columns:
+${columnList}
+
+For updates, only include the fields you want to change. Example: {"${exampleCols[0]?.name || 'field'}": "new_value"}`
+    }
+
+    return `${originalDescription}
+
+Table "${table.name}" available columns:
+${columnList}
+
+Pass the "data" parameter with an object like: ${JSON.stringify(dataExample)}`
+  }
+
+  return `${originalDescription}
+
+Table "${table.name}" columns:
+${columnList}`
+}
+
+/**
+ * Enriches LLM tool parameters with table-specific information.
+ */
+export function enrichTableToolParameters(
+  llmSchema: { properties?: Record<string, any>; required?: string[] },
+  table: TableSummary,
+  toolId: string
+): { properties: Record<string, any>; required: string[] } {
+  if (!table.columns || table.columns.length === 0) {
+    return {
+      properties: llmSchema.properties || {},
+      required: llmSchema.required || [],
+    }
+  }
+
+  const columnNames = table.columns.map((c) => c.name).join(', ')
+  const enrichedProperties = { ...llmSchema.properties }
+  const enrichedRequired = llmSchema.required ? [...llmSchema.required] : []
+
+  if (enrichedProperties.filter && FILTER_OPERATIONS.has(toolId)) {
+    enrichedProperties.filter = {
+      ...enrichedProperties.filter,
+      description: `REQUIRED - query will fail without a filter. Construct filter from user's question using columns: ${columnNames}. Syntax: {"column": {"$eq": "value"}}`,
+    }
+  }
+
+  if (FILTER_OPERATIONS.has(toolId) && !enrichedRequired.includes('filter')) {
+    enrichedRequired.push('filter')
+  }
+
+  if (enrichedProperties.sort && toolId === 'table_query_rows') {
+    enrichedProperties.sort = {
+      ...enrichedProperties.sort,
+      description: `Sort order as {field: "asc"|"desc"}. REQUIRED for ranking queries (highest, lowest, Nth). Example: {"salary": "desc"} for highest salary first.`,
+    }
+  }
+
+  if (enrichedProperties.limit && toolId === 'table_query_rows') {
+    enrichedProperties.limit = {
+      ...enrichedProperties.limit,
+      description: `Maximum rows to return (min: 1, max: 1000, default: 100). For ranking queries: use limit=1 for highest/lowest, limit=2 for second highest, etc.`,
+    }
+  }
+
+  if (enrichedProperties.data && DATA_OPERATIONS.has(toolId)) {
+    const exampleCols = table.columns.slice(0, 2)
+    const exampleData = exampleCols.reduce(
+      (obj: Record<string, unknown>, col: { name: string; type: string }) => {
+        obj[col.name] = col.type === 'number' ? 123 : col.type === 'boolean' ? true : 'value'
+        return obj
+      },
+      {} as Record<string, unknown>
+    )
+
+    if (toolId === 'table_update_row') {
+      enrichedProperties.data = {
+        ...enrichedProperties.data,
+        description: `Object containing fields to update. Only include fields you want to change. Available columns: ${columnNames}`,
+      }
+    } else {
+      enrichedProperties.data = {
+        ...enrichedProperties.data,
+        description: `REQUIRED object containing row values. Use columns: ${columnNames}. Example value: ${JSON.stringify(exampleData)}`,
+      }
+    }
+  }
+
+  if (enrichedProperties.rows && toolId === 'table_batch_insert_rows') {
+    enrichedProperties.rows = {
+      ...enrichedProperties.rows,
+      description: `REQUIRED. Array of row objects. Each object uses columns: ${columnNames}`,
+    }
+  }
+
+  return {
+    properties: enrichedProperties,
+    required: enrichedRequired,
+  }
+}

apps/sim/lib/table/llm/index.ts

@@ -0,0 +1 @@
+export * from '@/lib/table/llm/enrichment'

apps/sim/lib/table/llm/wand.ts

@@ -0,0 +1,64 @@
+/**
+ * Wand enricher for table schema context.
+ */
+
+import { db } from '@sim/db'
+import { userTableDefinitions } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { and, eq, isNull } from 'drizzle-orm'
+import type { TableSchema } from '@/lib/table/types'
+
+const logger = createLogger('TableWandEnricher')
+
+/**
+ * Wand enricher that provides table schema context.
+ * Used by the wand API to inject table column information into the system prompt.
+ */
+export async function enrichTableSchema(
+  workspaceId: string | null,
+  context: Record<string, unknown>
+): Promise<string | null> {
+  const tableId = context.tableId as string | undefined
+  if (!tableId || !workspaceId) {
+    return null
+  }
+
+  try {
+    const [table] = await db
+      .select({
+        name: userTableDefinitions.name,
+        schema: userTableDefinitions.schema,
+      })
+      .from(userTableDefinitions)
+      .where(
+        and(
+          eq(userTableDefinitions.id, tableId),
+          eq(userTableDefinitions.workspaceId, workspaceId),
+          isNull(userTableDefinitions.archivedAt)
+        )
+      )
+      .limit(1)
+
+    if (!table) {
+      return null
+    }
+
+    const schema = table.schema as TableSchema | null
+    if (!schema?.columns?.length) {
+      return null
+    }
+
+    const columnLines = schema.columns
+      .map((col) => {
+        const flags = [col.type, col.required && 'required', col.unique && 'unique'].filter(Boolean)
+        return `- ${col.name} (${flags.join(', ')})`
+      })
+      .join('\n')
+
+    const label = table.name ? `${table.name} (${tableId})` : tableId
+    return `Table schema for ${label}:\n${columnLines}\nBuilt-in columns: createdAt, updatedAt`
+  } catch (error) {
+    logger.warn('Failed to fetch table schema', { tableId, error })
+    return null
+  }
+}

apps/sim/lib/table/orchestration/index.ts

@@ -0,0 +1,64 @@
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateRequestId } from '@/lib/core/utils/request'
+import { getTableById, restoreTable, TableConflictError } from '@/lib/table/service'
+import type { TableDefinition } from '@/lib/table/types'
+
+const logger = createLogger('TableOrchestration')
+
+export type TableOrchestrationErrorCode = 'not_found' | 'validation' | 'conflict' | 'internal'
+
+export interface PerformRestoreTableParams {
+  tableId: string
+  userId: string
+  requestId?: string
+}
+
+export interface PerformRestoreTableResult {
+  success: boolean
+  error?: string
+  errorCode?: TableOrchestrationErrorCode
+  table?: TableDefinition
+}
+
+export async function performRestoreTable(
+  params: PerformRestoreTableParams
+): Promise<PerformRestoreTableResult> {
+  const { tableId, userId } = params
+  const requestId = params.requestId ?? generateRequestId()
+
+  const archivedTable = await getTableById(tableId, { includeArchived: true })
+  if (!archivedTable) {
+    return { success: false, error: 'Table not found', errorCode: 'not_found' }
+  }
+
+  try {
+    await restoreTable(tableId, requestId)
+    const table = (await getTableById(tableId)) ?? archivedTable
+
+    logger.info(`[${requestId}] Restored table ${tableId}`)
+
+    recordAudit({
+      workspaceId: archivedTable.workspaceId,
+      actorId: userId,
+      action: AuditAction.TABLE_RESTORED,
+      resourceType: AuditResourceType.TABLE,
+      resourceId: tableId,
+      resourceName: table.name,
+      description: `Restored table "${table.name}"`,
+      metadata: {
+        tableName: table.name,
+        workspaceId: table.workspaceId,
+      },
+    })
+
+    return { success: true, table }
+  } catch (error) {
+    logger.error(`[${requestId}] Failed to restore table ${tableId}`, { error })
+    if (error instanceof TableConflictError) {
+      return { success: false, error: error.message, errorCode: 'conflict' }
+    }
+    return { success: false, error: toError(error).message, errorCode: 'internal' }
+  }
+}

apps/sim/lib/table/order-key.ts

@@ -0,0 +1,34 @@
+/**
+ * Fractional order keys for table-row ordering.
+ *
+ * A row's order is a base-62 string key, not an integer position. Inserting
+ * between two rows mints a key strictly between their keys, so no other row's
+ * key changes — insert and delete become O(1) (no position reshift / recompact).
+ *
+ * Thin wrapper over the in-house fractional-indexing port (Figma/rocicorp
+ * algorithm) so the implementation is swappable. Keys never run out
+ * (variable-length strings); the only cost is gradual length growth under
+ * repeated same-spot inserts.
+ */
+
+import { generateKeyBetween, generateNKeysBetween } from '@sim/utils/fractional-indexing'
+
+/**
+ * Returns a key that sorts strictly between `a` and `b`. Pass `null` for an open
+ * end: `keyBetween(null, first)` prepends, `keyBetween(last, null)` appends,
+ * `keyBetween(null, null)` is the first key in an empty table.
+ *
+ * @throws if `a >= b` (callers must pass ordered, distinct bounds)
+ */
+export function keyBetween(a: string | null, b: string | null): string {
+  return generateKeyBetween(a, b)
+}
+
+/**
+ * Returns `n` keys evenly spaced strictly between `a` and `b` (same open-end
+ * semantics as {@link keyBetween}). Used for batch inserts and the backfill
+ * (`nKeysBetween(null, null, count)` mints an ordered run for an empty range).
+ */
+export function nKeysBetween(a: string | null, b: string | null, n: number): string[] {
+  return generateNKeysBetween(a, b, n)
+}

apps/sim/lib/table/planner.ts

@@ -0,0 +1,23 @@
+import { db } from '@sim/db'
+import { sql } from 'drizzle-orm'
+
+export type DbExecutor = typeof db | DbTransaction
+export type DbTransaction = Parameters<Parameters<typeof db.transaction>[0]>[0]
+
+/**
+ * Runs `fn` with seq scans penalized (`SET LOCAL`, so the flag dies with the
+ * transaction). JSONB predicates and sort keys (`->>` extraction, `@>`
+ * containment, lateral `jsonb_each_text`) are opaque to the planner — it
+ * estimates a handful of matching rows and picks a parallel seq scan over the
+ * entire shared `user_table_rows` relation (every tenant's rows) instead of the
+ * tenant's own index. Measured on a 1M-row table inside a 12M-row relation:
+ * filtered count 12.7s → 1.0s, sorted page 9.7s → 0.76s, filtered bulk select
+ * 14.4s → tenant-bounded. The flag only penalizes the plan shape: if no index
+ * plan exists, the seq scan still runs.
+ */
+export async function withSeqscanOff<T>(fn: (trx: DbTransaction) => Promise<T>): Promise<T> {
+  return db.transaction(async (trx) => {
+    await trx.execute(sql`SET LOCAL enable_seqscan = off`)
+    return fn(trx)
+  })
+}

apps/sim/lib/table/pluck.ts

@@ -0,0 +1,26 @@
+/**
+ * Pure utility for plucking a dot-and-bracket path from a value.
+ *
+ * Lives in its own leaf file (no server-only imports) so client components
+ * can import it without dragging in the rest of `lib/table` (which transitively
+ * pulls `@sim/db` and `next/headers`).
+ */
+
+/**
+ * Walk a dot-and-bracket path into a value (e.g. `a.b[0].c` or `result.items.0`).
+ * Returns undefined for any missing segment.
+ */
+export function pluckByPath(source: unknown, path: string): unknown {
+  if (source === null || source === undefined || !path) return source
+  const segments = path
+    .replace(/\[(\w+)\]/g, '.$1')
+    .split('.')
+    .filter(Boolean)
+  let cursor: unknown = source
+  for (const seg of segments) {
+    if (cursor === null || cursor === undefined) return undefined
+    if (typeof cursor !== 'object') return undefined
+    cursor = (cursor as Record<string, unknown>)[seg]
+  }
+  return cursor
+}

apps/sim/lib/table/query-builder/__tests__/converters.test.ts

@@ -0,0 +1,119 @@
+/**
+ * @vitest-environment node
+ *
+ * Converter unit tests for the table query builder. Cover the operator
+ * round-trips — UI rule → Filter object → UI rule — with attention to the
+ * valueless `$empty` operator that maps to two distinct UI operators.
+ */
+import { describe, expect, it } from 'vitest'
+import { filterRulesToFilter, filterToRules } from '@/lib/table/query-builder/converters'
+import type { FilterRule } from '@/lib/table/types'
+
+function rule(overrides: Partial<FilterRule>): FilterRule {
+  return {
+    id: 'rule-1',
+    logicalOperator: 'and',
+    column: 'name',
+    operator: 'eq',
+    value: '',
+    ...overrides,
+  }
+}
+
+describe('filterRulesToFilter', () => {
+  it('emits a bare value for eq (containment shorthand)', () => {
+    expect(filterRulesToFilter([rule({ operator: 'eq', value: 'John' })])).toEqual({ name: 'John' })
+  })
+
+  it('wraps non-eq operators in a $-prefixed operator object', () => {
+    expect(
+      filterRulesToFilter([rule({ column: 'email', operator: 'startsWith', value: 'a' })])
+    ).toEqual({ email: { $startsWith: 'a' } })
+    expect(
+      filterRulesToFilter([rule({ column: 'email', operator: 'ncontains', value: 'x' })])
+    ).toEqual({ email: { $ncontains: 'x' } })
+  })
+
+  it('parses comma-separated values into arrays for in / nin', () => {
+    expect(
+      filterRulesToFilter([rule({ column: 'status', operator: 'nin', value: 'a, b' })])
+    ).toEqual({ status: { $nin: ['a', 'b'] } })
+  })
+
+  it('serializes isEmpty / isNotEmpty to $empty without a value', () => {
+    expect(filterRulesToFilter([rule({ column: 'phone', operator: 'isEmpty' })])).toEqual({
+      phone: { $empty: true },
+    })
+    expect(filterRulesToFilter([rule({ column: 'phone', operator: 'isNotEmpty' })])).toEqual({
+      phone: { $empty: false },
+    })
+  })
+
+  it('merges two AND rules on the same column into one operator object', () => {
+    const filter = filterRulesToFilter([
+      rule({ id: 'a', column: 'age', operator: 'gt', value: '18' }),
+      rule({ id: 'b', column: 'age', operator: 'lt', value: '65' }),
+    ])
+    expect(filter).toEqual({ age: { $gt: 18, $lt: 65 } })
+  })
+
+  it('normalizes a bare-equality shorthand when merging with an operator', () => {
+    const filter = filterRulesToFilter([
+      rule({ id: 'a', column: 'name', operator: 'eq', value: 'John' }),
+      rule({ id: 'b', column: 'name', operator: 'contains', value: 'oh' }),
+    ])
+    expect(filter).toEqual({ name: { $eq: 'John', $contains: 'oh' } })
+  })
+
+  it('keeps same-column rules across an OR boundary in separate groups', () => {
+    const filter = filterRulesToFilter([
+      rule({ id: 'a', column: 'age', operator: 'gt', value: '18' }),
+      rule({ id: 'b', logicalOperator: 'or', column: 'age', operator: 'lt', value: '5' }),
+    ])
+    expect(filter).toEqual({ $or: [{ age: { $gt: 18 } }, { age: { $lt: 5 } }] })
+  })
+})
+
+describe('filterToRules', () => {
+  it('maps $empty: true back to isEmpty and $empty: false back to isNotEmpty', () => {
+    const empty = filterToRules({ phone: { $empty: true } })
+    expect(empty).toHaveLength(1)
+    expect(empty[0]).toMatchObject({ column: 'phone', operator: 'isEmpty', value: '' })
+
+    const notEmpty = filterToRules({ phone: { $empty: false } })
+    expect(notEmpty[0]).toMatchObject({ column: 'phone', operator: 'isNotEmpty', value: '' })
+  })
+
+  it("treats the string '$empty' operand the same as the boolean (no predicate flip)", () => {
+    const empty = filterToRules({ phone: { $empty: 'true' } } as unknown as Parameters<
+      typeof filterToRules
+    >[0])
+    expect(empty[0]).toMatchObject({ column: 'phone', operator: 'isEmpty', value: '' })
+
+    const notEmpty = filterToRules({ phone: { $empty: 'false' } } as unknown as Parameters<
+      typeof filterToRules
+    >[0])
+    expect(notEmpty[0]).toMatchObject({ column: 'phone', operator: 'isNotEmpty', value: '' })
+  })
+
+  it('round-trips string-pattern operators', () => {
+    for (const operator of ['contains', 'ncontains', 'startsWith', 'endsWith'] as const) {
+      const filter = filterRulesToFilter([rule({ column: 'name', operator, value: 'abc' })])
+      const back = filterToRules(filter)
+      expect(back[0]).toMatchObject({ column: 'name', operator, value: 'abc' })
+    }
+  })
+
+  it('round-trips isEmpty through filterRulesToFilter', () => {
+    const filter = filterRulesToFilter([rule({ column: 'name', operator: 'isEmpty' })])
+    const back = filterToRules(filter)
+    expect(back[0]).toMatchObject({ column: 'name', operator: 'isEmpty', value: '' })
+  })
+
+  it('round-trips a multi-operator column (Filter → rules → Filter) without loss', () => {
+    const original = { age: { $gte: 18, $lte: 65 } }
+    const rules = filterToRules(original)
+    expect(rules).toHaveLength(2)
+    expect(filterRulesToFilter(rules)).toEqual(original)
+  })
+})

apps/sim/lib/table/query-builder/constants.ts

@@ -0,0 +1,39 @@
+/**
+ * Constants for table query builder UI (filtering and sorting).
+ */
+
+export type { FilterRule, SortRule } from '@/lib/table/types'
+
+export const COMPARISON_OPERATORS = [
+  { value: 'eq', label: 'equals' },
+  { value: 'ne', label: 'not equals' },
+  { value: 'contains', label: 'contains' },
+  { value: 'ncontains', label: 'does not contain' },
+  { value: 'startsWith', label: 'starts with' },
+  { value: 'endsWith', label: 'ends with' },
+  { value: 'gt', label: 'greater than' },
+  { value: 'gte', label: 'greater or equal' },
+  { value: 'lt', label: 'less than' },
+  { value: 'lte', label: 'less or equal' },
+  { value: 'in', label: 'in array' },
+  { value: 'nin', label: 'not in array' },
+  { value: 'isEmpty', label: 'is empty' },
+  { value: 'isNotEmpty', label: 'is not empty' },
+] as const
+
+/**
+ * Operators that take no value — the filter is fully specified by column +
+ * operator alone. The UI hides the value input and skips the value-required
+ * check for these, and the converter serializes them to `{ $empty: bool }`.
+ */
+export const VALUELESS_OPERATORS = new Set<string>(['isEmpty', 'isNotEmpty'])
+
+export const LOGICAL_OPERATORS = [
+  { value: 'and', label: 'and' },
+  { value: 'or', label: 'or' },
+] as const
+
+export const SORT_DIRECTIONS = [
+  { value: 'asc', label: 'ascending' },
+  { value: 'desc', label: 'descending' },
+] as const

apps/sim/lib/table/query-builder/converters.ts

@@ -0,0 +1,229 @@
+/**
+ * Converters for transforming between UI builder state and API filter/sort objects.
+ */
+
+import { generateShortId } from '@sim/utils/id'
+import { isRecordLike } from '@sim/utils/object'
+import type {
+  Filter,
+  FilterRule,
+  JsonValue,
+  Sort,
+  SortDirection,
+  SortRule,
+} from '@/lib/table/types'
+
+/** Converts UI filter rules to a Filter object for API queries. */
+export function filterRulesToFilter(rules: FilterRule[]): Filter | null {
+  if (rules.length === 0) return null
+
+  const orGroups: Filter[] = []
+  let currentGroup: Filter = {}
+
+  for (const rule of rules) {
+    // Honor the OR boundary before skipping incomplete rows, so an incomplete
+    // `or` row between two valid conditions still starts a new group.
+    const isOr = rule.logicalOperator === 'or'
+    if (isOr && Object.keys(currentGroup).length > 0) {
+      orGroups.push({ ...currentGroup })
+      currentGroup = {}
+    }
+
+    // Skip incomplete rows (no column selected) so a blank builder row never
+    // serializes to a `{ '': ... }` predicate. The OR boundary above is still
+    // applied; the row just contributes no condition.
+    if (!rule.column) continue
+
+    const ruleValue = toRuleValue(rule.operator, rule.value)
+    const existing = currentGroup[rule.column]
+    currentGroup[rule.column] =
+      existing === undefined
+        ? (ruleValue as Filter[string])
+        : (mergeConditions(existing, ruleValue) as Filter[string])
+  }
+
+  if (Object.keys(currentGroup).length > 0) {
+    orGroups.push(currentGroup)
+  }
+
+  return orGroups.length > 1 ? { $or: orGroups } : orGroups[0] || null
+}
+
+/** Converts a Filter object back to UI filter rules. */
+export function filterToRules(filter: Filter | null): FilterRule[] {
+  if (!filter) return []
+
+  if (filter.$or && Array.isArray(filter.$or)) {
+    const groups = filter.$or
+      .map((orGroup) => parseFilterGroup(orGroup as Filter))
+      .filter((group) => group.length > 0)
+    return applyLogicalOperators(groups)
+  }
+
+  return parseFilterGroup(filter)
+}
+
+/** Converts a single UI sort rule to a Sort object for API queries. */
+export function sortRuleToSort(rule: SortRule | null): Sort | null {
+  if (!rule || !rule.column) return null
+  return { [rule.column]: rule.direction }
+}
+
+/** Converts multiple UI sort rules to a Sort object. */
+export function sortRulesToSort(rules: SortRule[]): Sort | null {
+  if (rules.length === 0) return null
+
+  const sort: Sort = {}
+  for (const rule of rules) {
+    if (rule.column) {
+      sort[rule.column] = rule.direction
+    }
+  }
+
+  return Object.keys(sort).length > 0 ? sort : null
+}
+
+/** Converts a Sort object back to UI sort rules. */
+export function sortToRules(sort: Sort | null): SortRule[] {
+  if (!sort) return []
+
+  return Object.entries(sort).map(([column, direction]) => ({
+    id: generateShortId(),
+    column,
+    direction: normalizeSortDirection(direction),
+  }))
+}
+
+function toRuleValue(operator: string, value: string): JsonValue {
+  if (operator === 'isEmpty') return { $empty: true }
+  if (operator === 'isNotEmpty') return { $empty: false }
+  const parsedValue = parseValue(value, operator)
+  return operator === 'eq' ? parsedValue : { [`$${operator}`]: parsedValue }
+}
+
+/**
+ * Merges two conditions targeting the same column within one AND group into a
+ * single operator object, so `age > 18 AND age < 65` becomes
+ * `{ age: { $gt: 18, $lt: 65 } }` instead of the second rule clobbering the
+ * first. Bare-equality shorthands are normalized to `{ $eq: value }` so they
+ * can coexist with operators. On a same-operator collision (e.g. two
+ * `$contains`) the later rule wins.
+ */
+function mergeConditions(existing: unknown, incoming: unknown): Record<string, JsonValue> {
+  return { ...toOperatorObject(existing), ...toOperatorObject(incoming) }
+}
+
+function toOperatorObject(value: unknown): Record<string, JsonValue> {
+  if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+    return { ...(value as Record<string, JsonValue>) }
+  }
+  return { $eq: value as JsonValue }
+}
+
+function applyLogicalOperators(groups: FilterRule[][]): FilterRule[] {
+  const rules: FilterRule[] = []
+
+  groups.forEach((group, groupIndex) => {
+    group.forEach((rule, ruleIndex) => {
+      rules.push({
+        ...rule,
+        logicalOperator:
+          groupIndex === 0 && ruleIndex === 0
+            ? 'and'
+            : groupIndex > 0 && ruleIndex === 0
+              ? 'or'
+              : 'and',
+      })
+    })
+  })
+
+  return rules
+}
+
+const ARRAY_OPERATORS = new Set(['in', 'nin'])
+const TEXT_MATCH_OPERATORS = new Set(['contains', 'ncontains', 'startsWith', 'endsWith'])
+
+function parseValue(value: string, operator: string): JsonValue {
+  if (ARRAY_OPERATORS.has(operator)) {
+    return value
+      .split(',')
+      .map((part) => part.trim())
+      .map((part) => parseScalar(part))
+  }
+
+  // Substring/prefix/suffix matches are textual — keep the raw string so a value
+  // like "123" isn't coerced to a number the SQL builder's ILIKE path can't use.
+  if (TEXT_MATCH_OPERATORS.has(operator)) {
+    return value
+  }
+
+  return parseScalar(value)
+}
+
+function parseScalar(value: string): JsonValue {
+  if (value === 'true') return true
+  if (value === 'false') return false
+  if (value === 'null') return null
+  if (!Number.isNaN(Number(value)) && value !== '') return Number(value)
+  return value
+}
+
+function parseFilterGroup(group: Filter): FilterRule[] {
+  if (!group || typeof group !== 'object' || Array.isArray(group)) return []
+
+  const rules: FilterRule[] = []
+
+  for (const [column, value] of Object.entries(group)) {
+    if (column === '$or' || column === '$and') continue
+
+    if (isRecordLike(value)) {
+      for (const [op, opValue] of Object.entries(value)) {
+        if (!op.startsWith('$')) continue
+        // `$empty` is a valueless boolean operator — map it back to the two
+        // distinct UI operators rather than exposing a raw `empty` operator.
+        // Accept the string forms `'true'`/`'false'` too, matching the lenient
+        // coercion in the SQL builder's `coerceEmptyFlag` so a filter authored
+        // via the raw API doesn't flip its predicate when re-opened in the UI.
+        if (op === '$empty') {
+          rules.push({
+            id: generateShortId(),
+            logicalOperator: 'and',
+            column,
+            operator: opValue === true || opValue === 'true' ? 'isEmpty' : 'isNotEmpty',
+            value: '',
+          })
+          continue
+        }
+        rules.push({
+          id: generateShortId(),
+          logicalOperator: 'and',
+          column,
+          operator: op.substring(1),
+          value: formatValueForBuilder(opValue as JsonValue),
+        })
+      }
+      continue
+    }
+
+    rules.push({
+      id: generateShortId(),
+      logicalOperator: 'and',
+      column,
+      operator: 'eq',
+      value: formatValueForBuilder(value as JsonValue),
+    })
+  }
+
+  return rules
+}
+
+function formatValueForBuilder(value: JsonValue): string {
+  if (value === null) return 'null'
+  if (typeof value === 'boolean') return String(value)
+  if (Array.isArray(value)) return value.map(formatValueForBuilder).join(', ')
+  return String(value)
+}
+
+function normalizeSortDirection(direction: string): SortDirection {
+  return direction === 'desc' ? 'desc' : 'asc'
+}

apps/sim/lib/table/query-builder/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Query builder UI utilities for filtering and sorting tables.
+ */
+
+export * from '@/lib/table/query-builder/constants'
+export * from '@/lib/table/query-builder/converters'
+export * from '@/lib/table/query-builder/use-query-builder'

apps/sim/lib/table/query-builder/use-query-builder.ts

@@ -0,0 +1,154 @@
+/**
+ * Hooks for query builder UI state management (filters and sorting).
+ */
+
+import { useCallback } from 'react'
+import { generateShortId } from '@sim/utils/id'
+import {
+  COMPARISON_OPERATORS,
+  type FilterRule,
+  LOGICAL_OPERATORS,
+  SORT_DIRECTIONS,
+  type SortRule,
+} from '@/lib/table/query-builder/constants'
+import type { ColumnOption } from '@/lib/table/types'
+
+const comparisonOptions: ColumnOption[] = COMPARISON_OPERATORS.map((op) => ({
+  value: op.value,
+  label: op.label,
+}))
+
+const logicalOptions: ColumnOption[] = LOGICAL_OPERATORS.map((op) => ({
+  value: op.value,
+  label: op.label,
+}))
+
+const sortDirectionOptions: ColumnOption[] = SORT_DIRECTIONS.map((d) => ({
+  value: d.value,
+  label: d.label,
+}))
+
+/** Manages filter rule state with add/remove/update operations. */
+export function useFilterBuilder({
+  columns,
+  rules,
+  setRules,
+  isReadOnly = false,
+}: UseFilterBuilderProps): UseFilterBuilderReturn {
+  const createDefaultRule = useCallback((): FilterRule => {
+    return {
+      id: generateShortId(),
+      logicalOperator: 'and',
+      column: columns[0]?.value || '',
+      operator: 'eq',
+      value: '',
+    }
+  }, [columns])
+
+  const addRule = useCallback(() => {
+    if (isReadOnly) return
+    setRules([...rules, createDefaultRule()])
+  }, [isReadOnly, rules, setRules, createDefaultRule])
+
+  const removeRule = useCallback(
+    (id: string) => {
+      if (isReadOnly) return
+      setRules(rules.filter((r) => r.id !== id))
+    },
+    [isReadOnly, rules, setRules]
+  )
+
+  const updateRule = useCallback(
+    (id: string, field: keyof FilterRule, value: string) => {
+      if (isReadOnly) return
+      setRules(rules.map((r) => (r.id === id ? { ...r, [field]: value } : r)))
+    },
+    [isReadOnly, rules, setRules]
+  )
+
+  return {
+    comparisonOptions,
+    logicalOptions,
+    sortDirectionOptions,
+    addRule,
+    removeRule,
+    updateRule,
+    createDefaultRule,
+  }
+}
+
+/** Manages sort rule state with add/remove/update operations. */
+export function useSortBuilder({
+  columns,
+  sortRule,
+  setSortRule,
+}: UseSortBuilderProps): UseSortBuilderReturn {
+  const addSort = useCallback(() => {
+    setSortRule({
+      id: generateShortId(),
+      column: columns[0]?.value || '',
+      direction: 'asc',
+    })
+  }, [columns, setSortRule])
+
+  const removeSort = useCallback(() => {
+    setSortRule(null)
+  }, [setSortRule])
+
+  const updateSortColumn = useCallback(
+    (column: string) => {
+      if (sortRule) {
+        setSortRule({ ...sortRule, column })
+      }
+    },
+    [sortRule, setSortRule]
+  )
+
+  const updateSortDirection = useCallback(
+    (direction: 'asc' | 'desc') => {
+      if (sortRule) {
+        setSortRule({ ...sortRule, direction })
+      }
+    },
+    [sortRule, setSortRule]
+  )
+
+  return {
+    sortDirectionOptions,
+    addSort,
+    removeSort,
+    updateSortColumn,
+    updateSortDirection,
+  }
+}
+
+export interface UseFilterBuilderProps {
+  columns: ColumnOption[]
+  rules: FilterRule[]
+  setRules: (rules: FilterRule[]) => void
+  isReadOnly?: boolean
+}
+
+export interface UseFilterBuilderReturn {
+  comparisonOptions: ColumnOption[]
+  logicalOptions: ColumnOption[]
+  sortDirectionOptions: ColumnOption[]
+  addRule: () => void
+  removeRule: (id: string) => void
+  updateRule: (id: string, field: keyof FilterRule, value: string) => void
+  createDefaultRule: () => FilterRule
+}
+
+interface UseSortBuilderProps {
+  columns: ColumnOption[]
+  sortRule: SortRule | null
+  setSortRule: (sort: SortRule | null) => void
+}
+
+interface UseSortBuilderReturn {
+  sortDirectionOptions: ColumnOption[]
+  addSort: () => void
+  removeSort: () => void
+  updateSortColumn: (column: string) => void
+  updateSortDirection: (direction: 'asc' | 'desc') => void
+}

apps/sim/lib/table/rows/executions.ts

@@ -0,0 +1,348 @@
+/**
+ * Row-executions (workflow-group results) internals for the table service layer.
+ *
+ * Internal module: not exposed via the `@/lib/table` barrel. Consumers import
+ * directly from `@/lib/table/rows/executions`.
+ */
+
+import { tableRowExecutions } from '@sim/db/schema'
+import { and, eq, inArray, type SQL, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { getColumnId } from '@/lib/table/column-keys'
+import { areGroupDepsSatisfied } from '@/lib/table/deps'
+import type {
+  EnrichmentRunDetail,
+  RowData,
+  RowExecutionMetadata,
+  RowExecutions,
+  TableRow,
+  TableSchema,
+} from '@/lib/table/types'
+
+/**
+ * Loads `tableRowExecutions` rows for the given row ids and groups them into a
+ * `Map<rowId, RowExecutions>` suitable for plugging into `TableRow.executions`.
+ */
+export async function loadExecutionsByRow(
+  trx: DbOrTx,
+  rowIds: Iterable<string>
+): Promise<Map<string, RowExecutions>> {
+  const ids = Array.from(new Set(rowIds))
+  const result = new Map<string, RowExecutions>()
+  if (ids.length === 0) return result
+  // Explicit column list, never `select()` — `enrichmentDetails` is large and
+  // must stay off the hot grid read path (fetched on demand via
+  // `loadEnrichmentDetail`).
+  const rows = await trx
+    .select({
+      rowId: tableRowExecutions.rowId,
+      groupId: tableRowExecutions.groupId,
+      status: tableRowExecutions.status,
+      executionId: tableRowExecutions.executionId,
+      jobId: tableRowExecutions.jobId,
+      workflowId: tableRowExecutions.workflowId,
+      error: tableRowExecutions.error,
+      runningBlockIds: tableRowExecutions.runningBlockIds,
+      blockErrors: tableRowExecutions.blockErrors,
+      cancelledAt: tableRowExecutions.cancelledAt,
+    })
+    .from(tableRowExecutions)
+    .where(inArray(tableRowExecutions.rowId, ids))
+  for (const r of rows) {
+    const existing = result.get(r.rowId) ?? {}
+    const meta: RowExecutionMetadata = {
+      status: r.status as RowExecutionMetadata['status'],
+      executionId: r.executionId ?? null,
+      jobId: r.jobId ?? null,
+      workflowId: r.workflowId,
+      error: r.error ?? null,
+      ...(r.runningBlockIds && r.runningBlockIds.length > 0
+        ? { runningBlockIds: r.runningBlockIds }
+        : {}),
+      ...(r.blockErrors && Object.keys(r.blockErrors as Record<string, string>).length > 0
+        ? { blockErrors: r.blockErrors as Record<string, string> }
+        : {}),
+      ...(r.cancelledAt ? { cancelledAt: r.cancelledAt.toISOString() } : {}),
+    }
+    existing[r.groupId] = meta
+    result.set(r.rowId, existing)
+  }
+  return result
+}
+
+/** Convenience: load executions for one row, returning `{}` when missing. */
+export async function loadExecutionsForRow(trx: DbOrTx, rowId: string): Promise<RowExecutions> {
+  const byRow = await loadExecutionsByRow(trx, [rowId])
+  return byRow.get(rowId) ?? {}
+}
+
+/**
+ * Loads the enrichment cascade breakdown for one `(tableId, rowId, groupId)`,
+ * or `null` when there is no exec row or it predates the feature. Read on demand
+ * by the enrichment details panel — kept off `loadExecutionsByRow`.
+ */
+export async function loadEnrichmentDetail(
+  trx: DbOrTx,
+  tableId: string,
+  rowId: string,
+  groupId: string
+): Promise<EnrichmentRunDetail | null> {
+  const [row] = await trx
+    .select({ enrichmentDetails: tableRowExecutions.enrichmentDetails })
+    .from(tableRowExecutions)
+    .where(
+      and(
+        eq(tableRowExecutions.tableId, tableId),
+        eq(tableRowExecutions.rowId, rowId),
+        eq(tableRowExecutions.groupId, groupId)
+      ) as SQL
+    )
+    .limit(1)
+  return (row?.enrichmentDetails as EnrichmentRunDetail | null | undefined) ?? null
+}
+
+/**
+ * Derive automatic clears + cancellation candidates from a row's data patch.
+ *
+ * Walks `schema.workflowGroups` left-to-right with a propagating `dirtied`
+ * column set. For each group whose deps overlap the dirty set, decide to
+ * clear (terminal exec) or cancel+rerun (in-flight exec), then add the
+ * group's outputs to the dirty set so later groups in the chain see them
+ * as dirty too. This models transitive dep chains as a single forward pass —
+ * editing column A propagates through group 1 (deps on A) to group 2 (deps
+ * on group 1's output) without explicit DAG traversal.
+ *
+ * Returns:
+ * - `executionsPatch`: caller's patch + nulls for cleared groups (or
+ *   undefined if nothing applied).
+ * - `inFlightDownstreamGroups`: groups whose dep was dirtied and that are
+ *   currently in-flight. Cancel-and-restart is the caller's job.
+ *
+ * Assumption: `workflowGroups[]` is in topological order — a group's deps
+ * may only reference columns to its left (enforced by `workflow-sidebar`'s
+ * "Run after" picker + the reorder scrub via `stripGroupDeps`). Violating
+ * this would silently miss the propagation.
+ */
+export function deriveExecClearsForDataPatch(
+  dataPatch: RowData,
+  schema: TableSchema,
+  existingExecutions: RowExecutions,
+  callerPatch: Record<string, RowExecutionMetadata | null> | undefined,
+  mergedData: RowData
+): {
+  executionsPatch: Record<string, RowExecutionMetadata | null> | undefined
+  inFlightDownstreamGroups: string[]
+} {
+  const dirtied = new Set(Object.keys(dataPatch))
+  const groupsToClear = new Set<string>()
+  const inFlightDownstreamGroups: string[] = []
+
+  // Own-output clears: when the user wipes a workflow output column, drop
+  // that group's exec entry so the auto-fire reactor re-arms the cell.
+  // Also flags the cleared output column as dirty so transitive downstream
+  // groups see it.
+  for (const [columnId, value] of Object.entries(dataPatch)) {
+    const cleared = value === null || value === undefined || value === ''
+    if (!cleared) continue
+    const col = schema.columns.find((c) => getColumnId(c) === columnId)
+    if (col?.workflowGroupId) groupsToClear.add(col.workflowGroupId)
+  }
+
+  // Left-to-right walk, propagating dirty columns forward.
+  const groups = schema.workflowGroups ?? []
+  const afterRow = { data: mergedData } as TableRow
+  for (const group of groups) {
+    const deps = group.dependencies?.columns ?? []
+    const depMatched = deps.some((d) => dirtied.has(d))
+    if (!depMatched) continue
+
+    // A dep column changed, but if the group's deps are no longer satisfied
+    // after the patch — a checkbox was unchecked or a text dep cleared — there's
+    // nothing to recompute. Leave the prior result alone instead of re-arming or
+    // cancelling it; only checking a box / filling a dep drives downstream work.
+    if (!areGroupDepsSatisfied(group, afterRow)) continue
+
+    const exec = existingExecutions[group.id]
+    if (exec) {
+      const status = exec.status
+      if (status === 'completed' || status === 'error' || status === 'cancelled') {
+        groupsToClear.add(group.id)
+      } else if (status === 'queued' || status === 'running' || status === 'pending') {
+        inFlightDownstreamGroups.push(group.id)
+      }
+    } else {
+      // No exec entry yet — `mode: 'new'` already covers this group. We
+      // still propagate the dirty signal forward so later groups in the
+      // chain see this group's outputs as dirty too.
+      groupsToClear.add(group.id)
+    }
+
+    // Propagate: this group is about to be re-computed, so groups whose
+    // deps reference its output columns are also dirty.
+    for (const out of group.outputs) dirtied.add(out.columnName)
+  }
+
+  if (groupsToClear.size === 0) {
+    return { executionsPatch: callerPatch, inFlightDownstreamGroups }
+  }
+  const merged: Record<string, RowExecutionMetadata | null> = { ...(callerPatch ?? {}) }
+  for (const gid of groupsToClear) {
+    if (!(gid in merged)) merged[gid] = null
+  }
+  return { executionsPatch: merged, inFlightDownstreamGroups }
+}
+
+/** Merges an `executionsPatch` into the row's existing executions blob. */
+export function applyExecutionsPatch(
+  existing: RowExecutions,
+  patch: Record<string, RowExecutionMetadata | null> | undefined
+): RowExecutions {
+  if (!patch) return existing
+  const next: RowExecutions = { ...existing }
+  for (const [gid, value] of Object.entries(patch)) {
+    if (value === null) {
+      delete next[gid]
+    } else {
+      next[gid] = value
+    }
+  }
+  return next
+}
+
+/**
+ * Writes a per-group execution patch for one row against the `tableRowExecutions`
+ * sidecar. Non-null values upsert into the table; nulls delete the entry. When
+ * `guard` is set, the upsert is gated to:
+ *  - reject if a `cancelled` row for the same execution already exists, and
+ *  - reject if the row exists but is owned by a different executionId
+ *    (with carve-outs for missing rows and null executionIds — the dispatcher's
+ *    pre-batch `pending` stamp leaves executionId unset so the first cell-task
+ *    can claim).
+ *
+ * Returns `'guard-rejected'` when the guarded group's upsert affected 0 rows
+ * (callers signal failure to the cell-task path). Returns `'wrote'` otherwise.
+ */
+export async function writeExecutionsPatch(
+  trx: DbOrTx,
+  tableId: string,
+  rowId: string,
+  patch: Record<string, RowExecutionMetadata | null> | undefined,
+  guard?: { groupId: string; executionId: string }
+): Promise<'wrote' | 'guard-rejected'> {
+  if (!patch) return 'wrote'
+  const entries = Object.entries(patch)
+  if (entries.length === 0) return 'wrote'
+
+  for (const [gid, value] of entries) {
+    if (value === null) {
+      await trx
+        .delete(tableRowExecutions)
+        .where(and(eq(tableRowExecutions.rowId, rowId), eq(tableRowExecutions.groupId, gid)) as SQL)
+      continue
+    }
+    const insertValues = {
+      tableId,
+      rowId,
+      groupId: gid,
+      status: value.status,
+      executionId: value.executionId,
+      jobId: value.jobId,
+      workflowId: value.workflowId,
+      error: value.error,
+      runningBlockIds: value.runningBlockIds ?? [],
+      blockErrors: value.blockErrors ?? {},
+      cancelledAt: value.cancelledAt ? new Date(value.cancelledAt) : null,
+      enrichmentDetails: value.enrichmentDetails ?? null,
+      updatedAt: new Date(),
+    } as const
+
+    const isGuarded = guard && guard.groupId === gid
+    if (isGuarded) {
+      // Gate by guard semantics. The original JSONB guard had two AND'd
+      // clauses; we collapse them onto the upsert's WHERE so a non-matching
+      // existing row leaves the table untouched and we observe 0 affected.
+      const guardExecutionId = guard.executionId
+      const updated = await trx
+        .insert(tableRowExecutions)
+        .values(insertValues)
+        .onConflictDoUpdate({
+          target: [tableRowExecutions.rowId, tableRowExecutions.groupId],
+          set: {
+            status: insertValues.status,
+            executionId: insertValues.executionId,
+            jobId: insertValues.jobId,
+            workflowId: insertValues.workflowId,
+            error: insertValues.error,
+            runningBlockIds: insertValues.runningBlockIds,
+            blockErrors: insertValues.blockErrors,
+            cancelledAt: insertValues.cancelledAt,
+            // Sticky: preserve a prior cascade breakdown when this write omits
+            // it (e.g. the running pickup stamp) so only an explicit detail
+            // overwrites it. Re-runs delete the row first, so this never serves
+            // stale detail across runs.
+            enrichmentDetails: sql`coalesce(excluded.enrichment_details, ${tableRowExecutions.enrichmentDetails})`,
+            updatedAt: insertValues.updatedAt,
+          },
+          where: and(
+            // Reject any guarded worker write when the cell is `cancelled` — a
+            // stop click wrote it authoritatively. SQL mirror of `isExecCancelled`
+            // (deps.ts). Status-only (not executionId-scoped): the cancel can
+            // only carry the pre-stamp's executionId (often null), so matching on
+            // id would let the worker's real-id claim resurrect a killed cell.
+            sql`${tableRowExecutions.status} <> 'cancelled'`,
+            // Stale-worker: the cell's active run has moved on. Carve-outs
+            // permit a fresh worker to take over when the row's executionId
+            // is unset (dispatcher's pre-batch `pending` stamp).
+            sql`(${tableRowExecutions.executionId} IS NULL OR ${tableRowExecutions.executionId} = ${guardExecutionId})`
+          ) as SQL,
+        })
+        .returning({ rowId: tableRowExecutions.rowId })
+      if (updated.length === 0) return 'guard-rejected'
+      continue
+    }
+
+    await trx
+      .insert(tableRowExecutions)
+      .values(insertValues)
+      .onConflictDoUpdate({
+        target: [tableRowExecutions.rowId, tableRowExecutions.groupId],
+        set: {
+          status: insertValues.status,
+          executionId: insertValues.executionId,
+          jobId: insertValues.jobId,
+          workflowId: insertValues.workflowId,
+          error: insertValues.error,
+          runningBlockIds: insertValues.runningBlockIds,
+          blockErrors: insertValues.blockErrors,
+          cancelledAt: insertValues.cancelledAt,
+          // Sticky: preserve a prior cascade breakdown when this write omits it
+          // (e.g. the running pickup stamp) so only an explicit detail overwrites
+          // it. Re-runs delete the row first, so this never serves stale detail.
+          enrichmentDetails: sql`coalesce(excluded.enrichment_details, ${tableRowExecutions.enrichmentDetails})`,
+          updatedAt: insertValues.updatedAt,
+        },
+      })
+  }
+
+  return 'wrote'
+}
+
+/**
+ * Strips the given workflow group ids from every row's executions on a table —
+ * used by the column / group delete paths so stale running/queued exec records
+ * don't linger and inflate counters after the group is gone. The caller wraps
+ * in their own transaction.
+ */
+export async function stripGroupExecutions(
+  trx: DbOrTx,
+  tableId: string,
+  groupIds: Iterable<string>
+): Promise<void> {
+  const ids = Array.from(new Set(groupIds))
+  if (ids.length === 0) return
+  await trx
+    .delete(tableRowExecutions)
+    .where(
+      and(eq(tableRowExecutions.tableId, tableId), inArray(tableRowExecutions.groupId, ids)) as SQL
+    )
+}

apps/sim/lib/table/rows/ordering.ts

@@ -0,0 +1,459 @@
+/**
+ * Row position / fractional-ordering internals for the table service layer.
+ *
+ * Internal module: only the import/delete-runner entry points are exposed via
+ * the `@/lib/table/rows/ordering` path. Not re-exported through the
+ * `@/lib/table` barrel.
+ */
+
+import { db } from '@sim/db'
+import { userTableRows } from '@sim/db/schema'
+import { and, asc, desc, eq, gt, inArray, lt, lte, type SQL, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { TABLE_LIMITS } from '@/lib/table/constants'
+import { keyBetween, nKeysBetween } from '@/lib/table/order-key'
+import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner'
+import { setTableTxTimeouts } from '@/lib/table/tx'
+import type { RowData } from '@/lib/table/types'
+
+/**
+ * Starting `position` for an append import — `max(position) + 1`, or 0 when empty. Read once,
+ * unlocked, before streaming: the import worker is the table's sole writer, so it can assign
+ * contiguous positions from this offset without per-batch position scans.
+ */
+export async function nextImportStartPosition(tableId: string): Promise<number> {
+  const [{ maxPos }] = await db
+    .select({
+      maxPos: sql<number>`coalesce(max(${userTableRows.position}), -1)`.mapWith(Number),
+    })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxPos + 1
+}
+
+/**
+ * Append anchor `order_key` for an import — `max(order_key)`, or null when empty. Read once,
+ * unlocked, before streaming (the import worker is the table's sole writer); each batch threads
+ * the previous batch's last key forward so no per-batch max scan is needed.
+ */
+export async function nextImportStartOrderKey(tableId: string): Promise<string | null> {
+  return maxOrderKey(db, tableId)
+}
+
+/**
+ * Serializes writers that assign `position` for the same table. The row-count
+ * trigger (migration 0198) serializes capacity via a row lock on
+ * `user_table_definitions`, but it fires AFTER INSERT, so two concurrent
+ * auto-positioned inserts could read the same snapshot and assign the same
+ * position (the `(table_id, position)` index is non-unique). This advisory lock
+ * restores per-table serialization. Released at COMMIT/ROLLBACK.
+ */
+export async function acquireRowOrderLock(trx: DbTransaction, tableId: string) {
+  await trx.execute(
+    sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_rows_pos:${tableId}`}, 0))`
+  )
+}
+
+/** Next append position for a table (max(position) + 1, or 0 if empty). */
+export async function nextRowPosition(trx: DbTransaction, tableId: string): Promise<number> {
+  const [{ maxPos }] = await trx
+    .select({
+      maxPos: sql<number>`coalesce(max(${userTableRows.position}), -1)`.mapWith(Number),
+    })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxPos + 1
+}
+
+/** Largest `order_key` for a table, or `null` when empty — the append anchor for new keys. */
+export async function maxOrderKey(executor: DbOrTx, tableId: string): Promise<string | null> {
+  const [{ maxKey }] = await executor
+    .select({ maxKey: sql<string | null>`max(${userTableRows.orderKey})` })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxKey ?? null
+}
+
+/**
+ * Computes the fractional `order_key` for a row inserted at the integer
+ * `requestedPosition` (or appended when omitted). Used by position-based callers
+ * (mothership tool, v1 API, undo position-fallback, transient old clients).
+ *
+ * The neighbor at slot `s` is the `s`-th row in `order_key, id` order (`OFFSET
+ * s`) — positions are gappy and non-authoritative, so `position = s` would miss;
+ * the visual ordinal is the key's ordinal. O(s), acceptable for these low-volume
+ * callers.
+ *
+ * Caller holds the row-order lock.
+ */
+export async function resolveInsertOrderKey(
+  trx: DbTransaction,
+  tableId: string,
+  requestedPosition?: number
+): Promise<string> {
+  const orderKeyAtSlot = async (slot: number): Promise<string | null> => {
+    if (slot < 0) return null
+    const [r] = await trx
+      .select({ orderKey: userTableRows.orderKey })
+      .from(userTableRows)
+      .where(eq(userTableRows.tableId, tableId))
+      .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id))
+      .limit(1)
+      .offset(slot)
+    return r?.orderKey ?? null
+  }
+  if (requestedPosition === undefined) {
+    return keyBetween(await maxOrderKey(trx, tableId), null)
+  }
+  const lo = await orderKeyAtSlot(requestedPosition - 1)
+  const hi = await orderKeyAtSlot(requestedPosition)
+  return keyBetween(lo, hi)
+}
+
+/**
+ * Resolves the `order_key` for an insert expressed by an anchor row id —
+ * `afterRowId` (place directly after) or `beforeRowId` (directly before). Finds
+ * the anchor and its adjacent key via the `(table_id, order_key, id)` index
+ * (O(1)) and mints a key between them. Caller holds the row-order lock.
+ */
+export async function resolveInsertByNeighbor(
+  trx: DbTransaction,
+  tableId: string,
+  afterRowId?: string,
+  beforeRowId?: string
+): Promise<string> {
+  const anchorId = afterRowId ?? beforeRowId!
+  const [anchor] = await trx
+    .select({ orderKey: userTableRows.orderKey })
+    .from(userTableRows)
+    .where(and(eq(userTableRows.tableId, tableId), eq(userTableRows.id, anchorId)))
+    .limit(1)
+  // The client targets a specific neighbor; a missing one (concurrent delete /
+  // stale view) is an error, not a silent insert at the front.
+  if (!anchor) throw new Error(`Row not found: ${anchorId}`)
+  const anchorKey = anchor.orderKey ?? null
+  // A null key on the anchor means the table isn't backfilled. order_key is
+  // authoritative, so the adjacent-key lookup below can't work — fail loudly
+  // rather than mint a wrong key.
+  if (anchorKey === null) {
+    throw new Error(`Row ${anchorId} has no order_key yet (table not backfilled)`)
+  }
+
+  if (afterRowId) {
+    // hi = the smallest key strictly GREATER than the anchor key. Comparing keys
+    // (not the `(order_key, id)` row tuple) skips past any sibling that shares the
+    // anchor's key, so `keyBetween` always gets strictly-ordered bounds and can't
+    // throw on a stray duplicate. Identical to the row tuple when keys are distinct.
+    const [next] = await trx
+      .select({ orderKey: userTableRows.orderKey })
+      .from(userTableRows)
+      .where(and(eq(userTableRows.tableId, tableId), gt(userTableRows.orderKey, anchorKey)))
+      .orderBy(asc(userTableRows.orderKey))
+      .limit(1)
+    return keyBetween(anchorKey, next?.orderKey ?? null)
+  }
+
+  // beforeRowId: lo = the largest key strictly LESS than the anchor key (distinct,
+  // same rationale as the afterRowId branch above).
+  const [prev] = await trx
+    .select({ orderKey: userTableRows.orderKey })
+    .from(userTableRows)
+    .where(and(eq(userTableRows.tableId, tableId), lt(userTableRows.orderKey, anchorKey)))
+    .orderBy(desc(userTableRows.orderKey))
+    .limit(1)
+  return keyBetween(prev?.orderKey ?? null, anchorKey)
+}
+
+/**
+ * Computes fractional `order_key`s for a batch insert by appending a contiguous
+ * run after the current max key. `order_key` is authoritative, so callers needing
+ * exact placement pass explicit `orderKeys` (handled before this function); here
+ * we just append a run. Caller holds the lock.
+ */
+export async function resolveBatchInsertOrderKeys(
+  trx: DbTransaction,
+  tableId: string,
+  count: number
+): Promise<string[]> {
+  return nKeysBetween(await maxOrderKey(trx, tableId), null, count)
+}
+
+/**
+ * Inserts a single row in its own transaction. Assigns a fractional `order_key`
+ * (authoritative) and a best-effort append `position` (no O(N) shift).
+ * Validation and side-effect dispatch stay with the caller; capacity is enforced
+ * by the `increment_user_table_row_count` trigger.
+ */
+export async function insertOrderedRow(params: {
+  tableId: string
+  workspaceId: string
+  data: RowData
+  rowId: string
+  position?: number
+  afterRowId?: string
+  beforeRowId?: string
+  createdBy?: string
+  now: Date
+}): Promise<{
+  id: string
+  data: RowData
+  position: number
+  orderKey: string | null
+  createdAt: Date
+  updatedAt: Date
+}> {
+  const { tableId, workspaceId, data, rowId, position, afterRowId, beforeRowId, createdBy, now } =
+    params
+  const [row] = await db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx)
+    await acquireRowOrderLock(trx, tableId)
+
+    // Resolve the authoritative order key from neighbor ids when given, else from
+    // the requested position.
+    const orderKey =
+      afterRowId || beforeRowId
+        ? await resolveInsertByNeighbor(trx, tableId, afterRowId, beforeRowId)
+        : await resolveInsertOrderKey(trx, tableId, position)
+
+    // order_key is authoritative — keep a best-effort, no-shift position.
+    const targetPosition = await nextRowPosition(trx, tableId)
+
+    return trx
+      .insert(userTableRows)
+      .values({
+        id: rowId,
+        tableId,
+        workspaceId,
+        data,
+        position: targetPosition,
+        orderKey,
+        createdAt: now,
+        updatedAt: now,
+        ...(createdBy ? { createdBy } : {}),
+      })
+      .returning()
+  })
+  return {
+    id: row.id,
+    data: row.data as RowData,
+    position: row.position,
+    orderKey: row.orderKey,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  }
+}
+
+/**
+ * Deletes a single row by id in its own transaction. Deleting a row never changes
+ * another row's `order_key`, so no positional reshift is needed. Returns `false`
+ * when no row matched.
+ */
+export async function deleteOrderedRow(params: {
+  tableId: string
+  rowId: string
+  workspaceId: string
+}): Promise<boolean> {
+  const { tableId, rowId, workspaceId } = params
+  return db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx)
+    const [deleted] = await trx
+      .delete(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.id, rowId),
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId)
+        )
+      )
+      .returning({ id: userTableRows.id })
+    return Boolean(deleted)
+  })
+}
+
+/**
+ * Deletes the given row ids in batches within one transaction. Deletes leave
+ * `order_key` untouched, so no positional recompaction is needed. Returns the
+ * deleted row ids. The caller resolves which ids to delete (used by both
+ * delete-by-ids and delete-by-filter).
+ */
+export async function deleteOrderedRowsByIds(params: {
+  tableId: string
+  workspaceId: string
+  rowIds: string[]
+}): Promise<{ id: string }[]> {
+  const { tableId, workspaceId, rowIds } = params
+  if (rowIds.length === 0) return []
+  return db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx, { statementMs: 60_000 })
+    const deleted: { id: string }[] = []
+    for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) {
+      const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE)
+      const rows = await trx
+        .delete(userTableRows)
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+      deleted.push(...rows)
+    }
+    return deleted
+  })
+}
+
+/**
+ * Selects one page of row ids to delete for the async delete-job worker: base scope plus a
+ * `created_at <= cutoff` floor (so rows inserted after the job started are never selected) and
+ * the caller's optional filter clause. Keyset paginated on `id` via `afterId` so excluded rows
+ * (which are skipped, not deleted) still advance the cursor — no OFFSET, no risk of looping on a
+ * fully-excluded page.
+ */
+export async function selectRowIdPage(params: {
+  tableId: string
+  workspaceId: string
+  cutoff: Date
+  filterClause?: SQL
+  afterId?: string
+  limit: number
+}): Promise<string[]> {
+  const { tableId, workspaceId, cutoff, filterClause, afterId, limit } = params
+  const selectPage = (executor: DbExecutor) =>
+    executor
+      .select({ id: userTableRows.id })
+      .from(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId),
+          lte(userTableRows.createdAt, cutoff),
+          afterId ? gt(userTableRows.id, afterId) : undefined,
+          filterClause
+        )
+      )
+      .orderBy(asc(userTableRows.id))
+      .limit(limit)
+  // A jsonb filter is unestimatable, so the planner would seq-scan the whole shared relation
+  // per page (12.6s measured) — keep it on the tenant's (table_id, id) index.
+  const rows = filterClause
+    ? await withSeqscanOff(async (trx) => selectPage(trx))
+    : await selectPage(db)
+  return rows.map((r) => r.id)
+}
+
+/**
+ * Like {@link selectRowIdPage} but returns each row's `data` too, for the bulk-update worker which
+ * must merge the patch into the existing row to validate the result. Same keyset walk on the
+ * `(table_id, id)` index, `created_at <= cutoff`, tenant-scoped, seqscan-off for jsonb filters.
+ *
+ * `excludeIfPatched` (a JSON patch string) skips rows that already contain the patch
+ * (`data @> patch`). The update worker passes it so a retried run doesn't re-walk and re-count
+ * rows an earlier attempt already updated — updated rows still exist (unlike deletes), and they
+ * still match the filter when the patch doesn't touch a filtered column, so without this a retry
+ * would double-count progress. It also skips no-op updates of rows that already hold those values.
+ */
+export async function selectRowDataPage(params: {
+  tableId: string
+  workspaceId: string
+  cutoff: Date
+  filterClause?: SQL
+  afterId?: string
+  limit: number
+  excludeIfPatched?: string
+}): Promise<Array<{ id: string; data: RowData }>> {
+  const { tableId, workspaceId, cutoff, filterClause, afterId, limit, excludeIfPatched } = params
+  const selectPage = (executor: DbExecutor) =>
+    executor
+      .select({ id: userTableRows.id, data: userTableRows.data })
+      .from(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId),
+          lte(userTableRows.createdAt, cutoff),
+          afterId ? gt(userTableRows.id, afterId) : undefined,
+          excludeIfPatched
+            ? sql`NOT (${userTableRows.data} @> ${excludeIfPatched}::jsonb)`
+            : undefined,
+          filterClause
+        )
+      )
+      .orderBy(asc(userTableRows.id))
+      .limit(limit)
+  const rows = filterClause
+    ? await withSeqscanOff(async (trx) => selectPage(trx))
+    : await selectPage(db)
+  return rows.map((r) => ({ id: r.id, data: r.data as RowData }))
+}
+
+/**
+ * Deletes one page of rows for the async delete-job worker, committing each `DELETE_BATCH_SIZE`
+ * chunk in its own short transaction. One statement per transaction bounds how long the
+ * statement-level row_count trigger's lock on the definition row is held (a page-wide transaction
+ * held it for the entire page, starving concurrent inserts and overrunning `statement_timeout`),
+ * and a mid-page failure loses at most one uncommitted batch — the keyset walker (or a task
+ * retry) re-walks whatever remains. Skips legacy position compaction: under fractional ordering
+ * it's unnecessary, and in the legacy path `position` gaps are harmless — rows still order by
+ * position. Returns the count deleted.
+ */
+export async function deletePageByIds(
+  tableId: string,
+  workspaceId: string,
+  rowIds: string[]
+): Promise<number> {
+  let deleted = 0
+  for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) {
+    const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE)
+    const rows = await db.transaction(async (trx) => {
+      await setTableTxTimeouts(trx, { statementMs: 60_000 })
+      return trx
+        .delete(userTableRows)
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+    })
+    deleted += rows.length
+  }
+  return deleted
+}
+
+/**
+ * Applies a JSONB-merge patch (`data || patchJson`) to a page of row ids, committed in
+ * UPDATE_BATCH_SIZE chunks (each its own transaction, 60s timeout) so a large background update
+ * makes incremental, resumable progress. Returns the number of rows updated.
+ */
+export async function updatePageByIds(
+  tableId: string,
+  workspaceId: string,
+  rowIds: string[],
+  patchJson: string
+): Promise<number> {
+  const now = new Date()
+  let updated = 0
+  for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.UPDATE_BATCH_SIZE) {
+    const batch = rowIds.slice(i, i + TABLE_LIMITS.UPDATE_BATCH_SIZE)
+    const rows = await db.transaction(async (trx) => {
+      await setTableTxTimeouts(trx, { statementMs: 60_000 })
+      return trx
+        .update(userTableRows)
+        .set({ data: sql`${userTableRows.data} || ${patchJson}::jsonb`, updatedAt: now })
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+    })
+    updated += rows.length
+  }
+  return updated
+}

apps/sim/lib/table/rows/paging.ts

@@ -0,0 +1,23 @@
+/**
+ * Cuts a fetched page to a byte budget: keeps the longest prefix of rows whose
+ * serialized `data` fits within `maxBytes`, always keeping at least one row so
+ * pagination makes forward progress even when a single row exceeds the budget.
+ *
+ * The budget counts `data` only — the per-row envelope (`id`, `position`,
+ * `orderKey`, timestamps, executions) is not measured, so actual response
+ * payloads run slightly over `maxBytes`. Callers must leave headroom; the
+ * production SQL-side cut should account for the same overhead.
+ */
+export function trimRowsToByteBudget<T extends { data: unknown }>(
+  rows: T[],
+  maxBytes: number
+): T[] {
+  let total = 0
+  let kept = 0
+  for (const row of rows) {
+    total += Buffer.byteLength(JSON.stringify(row.data))
+    if (kept > 0 && total > maxBytes) break
+    kept++
+  }
+  return kept === rows.length ? rows : rows.slice(0, kept)
+}

apps/sim/lib/table/service.ts

@@ -0,0 +1,749 @@
+/**
+ * Table service layer for internal programmatic access.
+ *
+ * Use this for: workflow executor, background jobs, testing business logic.
+ * Use API routes for: HTTP requests, frontend clients.
+ *
+ * Note: API routes have their own implementations for HTTP-specific concerns.
+ */
+
+import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit'
+import { db } from '@sim/db'
+import { tableJobs, userTableDefinitions, userTableRows } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getPostgresErrorCode } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, count, eq, isNull, sql } from 'drizzle-orm'
+import { generateRestoreName } from '@/lib/core/utils/restore-name'
+import type { DbOrTx } from '@/lib/db/types'
+import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing'
+import { generateColumnId, getColumnId, withGeneratedColumnIds } from '@/lib/table/column-keys'
+import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants'
+import { EMPTY_JOB_FIELDS, latestJobForTable, latestJobsForTables } from '@/lib/table/jobs/service'
+import { nKeysBetween } from '@/lib/table/order-key'
+import type { DbTransaction } from '@/lib/table/planner'
+import { setTableTxTimeouts } from '@/lib/table/tx'
+import type {
+  CreateTableData,
+  TableDefinition,
+  TableMetadata,
+  TableSchema,
+} from '@/lib/table/types'
+import { validateTableName, validateTableSchema } from '@/lib/table/validation'
+import { stripGroupDeps } from '@/lib/table/workflow-columns'
+
+const logger = createLogger('TableService')
+
+export class TableConflictError extends Error {
+  readonly code = 'TABLE_EXISTS' as const
+  constructor(name: string) {
+    super(`A table named "${name}" already exists in this workspace`)
+  }
+}
+
+export type TableScope = 'active' | 'archived' | 'all'
+
+/**
+ * Serializes schema/metadata read-modify-writes for a single table so
+ * concurrent mutators can't clobber each other's `schema` JSONB
+ * (last-writer-wins). Takes a transaction-scoped advisory lock keyed on
+ * `tableId`, then re-reads the table INSIDE the lock and hands the fresh
+ * definition + transaction to `mutate`. Each serialized writer therefore
+ * validates and computes against the prior writer's committed columns.
+ *
+ * Uses an advisory lock (not `SELECT ... FOR UPDATE` on the definition row) so
+ * it adds no edges to the row-lock graph — the row-count trigger (migration
+ * 0198) locks the definition row from `insertRow`/`deleteRow`, and a FOR UPDATE
+ * here would invert that order. Mirrors `acquireTablePositionLock`. The lock and
+ * the read both release at COMMIT/ROLLBACK; the wait is bounded by the
+ * `statement_timeout` set in `setTableTxTimeouts`.
+ */
+export async function withLockedTable<T>(
+  tableId: string,
+  mutate: (table: TableDefinition, trx: DbTransaction) => Promise<T>,
+  opts?: { includeArchived?: boolean }
+): Promise<T> {
+  return db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx)
+    await trx.execute(
+      sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_schema:${tableId}`}, 0))`
+    )
+    const table = await getTableById(tableId, { tx: trx, includeArchived: opts?.includeArchived })
+    if (!table) {
+      throw new Error('Table not found')
+    }
+    return mutate(table, trx)
+  })
+}
+
+/**
+ * Returns `schema` with `columns` sorted by `metadata.columnOrder` (the user-
+ * editable visible order). Columns missing from `columnOrder` are appended at
+ * the end in their original (schema-creation) order — covers tables created
+ * before `columnOrder` existed and any drift from out-of-band column adds.
+ *
+ * This makes `schema.columns` the single source of truth for column order on
+ * the wire. The client doesn't have to join the two arrays itself — every
+ * consumer (grid, sidebar, copilot, mothership) gets the same ordered list.
+ */
+function applyColumnOrderToSchema(
+  schema: TableSchema,
+  metadata: TableMetadata | null
+): TableSchema {
+  const order = metadata?.columnOrder
+  if (!order || order.length === 0) return schema
+  // `columnOrder` holds stable column ids (legacy entries equal the name == id).
+  const byId = new Map<string, TableSchema['columns'][number]>()
+  for (const c of schema.columns) byId.set(getColumnId(c), c)
+  const ordered: TableSchema['columns'] = []
+  for (const id of order) {
+    const c = byId.get(id)
+    if (c) {
+      ordered.push(c)
+      byId.delete(id)
+    }
+  }
+  for (const c of byId.values()) ordered.push(c)
+  return { ...schema, columns: ordered }
+}
+
+/**
+ * Gets a table by ID with full details.
+ *
+ * @param tableId - Table ID to fetch
+ * @returns Table definition or null if not found
+ */
+export async function getTableById(
+  tableId: string,
+  options?: { includeArchived?: boolean; tx?: DbOrTx }
+): Promise<TableDefinition | null> {
+  const { includeArchived = false, tx } = options ?? {}
+  const executor = tx ?? db
+  const results = await executor
+    .select({
+      id: userTableDefinitions.id,
+      name: userTableDefinitions.name,
+      description: userTableDefinitions.description,
+      schema: userTableDefinitions.schema,
+      metadata: userTableDefinitions.metadata,
+      maxRows: userTableDefinitions.maxRows,
+      workspaceId: userTableDefinitions.workspaceId,
+      createdBy: userTableDefinitions.createdBy,
+      archivedAt: userTableDefinitions.archivedAt,
+      createdAt: userTableDefinitions.createdAt,
+      updatedAt: userTableDefinitions.updatedAt,
+      rowCount: userTableDefinitions.rowCount,
+    })
+    .from(userTableDefinitions)
+    .where(
+      includeArchived
+        ? eq(userTableDefinitions.id, tableId)
+        : and(eq(userTableDefinitions.id, tableId), isNull(userTableDefinitions.archivedAt))
+    )
+    .limit(1)
+
+  if (results.length === 0) return null
+
+  const table = results[0]
+  const metadata = (table.metadata as TableMetadata) ?? null
+  const { pendingDeleteRemaining, ...jobFields } = await latestJobForTable(tableId, executor)
+  return {
+    id: table.id,
+    name: table.name,
+    description: table.description,
+    schema: applyColumnOrderToSchema(table.schema as TableSchema, metadata),
+    metadata,
+    rowCount: Math.max(0, table.rowCount - pendingDeleteRemaining),
+    maxRows: table.maxRows,
+    workspaceId: table.workspaceId,
+    createdBy: table.createdBy,
+    archivedAt: table.archivedAt,
+    createdAt: table.createdAt,
+    updatedAt: table.updatedAt,
+    ...jobFields,
+  }
+}
+
+/**
+ * Lists all tables in a workspace.
+ *
+ * @param workspaceId - Workspace ID to list tables for
+ * @returns Array of table definitions
+ */
+export async function listTables(
+  workspaceId: string,
+  options?: { scope?: TableScope }
+): Promise<TableDefinition[]> {
+  const { scope = 'active' } = options ?? {}
+  const tables = await db
+    .select({
+      id: userTableDefinitions.id,
+      name: userTableDefinitions.name,
+      description: userTableDefinitions.description,
+      schema: userTableDefinitions.schema,
+      metadata: userTableDefinitions.metadata,
+      maxRows: userTableDefinitions.maxRows,
+      workspaceId: userTableDefinitions.workspaceId,
+      createdBy: userTableDefinitions.createdBy,
+      archivedAt: userTableDefinitions.archivedAt,
+      createdAt: userTableDefinitions.createdAt,
+      updatedAt: userTableDefinitions.updatedAt,
+      rowCount: userTableDefinitions.rowCount,
+    })
+    .from(userTableDefinitions)
+    .where(
+      scope === 'all'
+        ? eq(userTableDefinitions.workspaceId, workspaceId)
+        : scope === 'archived'
+          ? and(
+              eq(userTableDefinitions.workspaceId, workspaceId),
+              sql`${userTableDefinitions.archivedAt} IS NOT NULL`
+            )
+          : and(
+              eq(userTableDefinitions.workspaceId, workspaceId),
+              isNull(userTableDefinitions.archivedAt)
+            )
+    )
+    .orderBy(userTableDefinitions.createdAt)
+
+  const jobsByTable = await latestJobsForTables(tables.map((t) => t.id))
+
+  return tables.map((t) => {
+    const metadata = (t.metadata as TableMetadata) ?? null
+    const { pendingDeleteRemaining, ...jobFields } = jobsByTable.get(t.id) ?? EMPTY_JOB_FIELDS
+    return {
+      id: t.id,
+      name: t.name,
+      description: t.description,
+      schema: applyColumnOrderToSchema(t.schema as TableSchema, metadata),
+      metadata,
+      rowCount: Math.max(0, t.rowCount - pendingDeleteRemaining),
+      maxRows: t.maxRows,
+      workspaceId: t.workspaceId,
+      createdBy: t.createdBy,
+      archivedAt: t.archivedAt,
+      createdAt: t.createdAt,
+      updatedAt: t.updatedAt,
+      ...jobFields,
+    }
+  })
+}
+
+/**
+ * Creates a new table.
+ *
+ * @param data - Table creation data
+ * @param requestId - Request ID for logging
+ * @returns Created table definition
+ * @throws Error if validation fails or limits exceeded
+ */
+export async function createTable(
+  data: CreateTableData,
+  requestId: string
+): Promise<TableDefinition> {
+  // Validate table name
+  const nameValidation = validateTableName(data.name)
+  if (!nameValidation.valid) {
+    throw new Error(`Invalid table name: ${nameValidation.errors.join(', ')}`)
+  }
+
+  // Validate schema
+  const schemaValidation = validateTableSchema(data.schema)
+  if (!schemaValidation.valid) {
+    throw new Error(`Invalid schema: ${schemaValidation.errors.join(', ')}`)
+  }
+
+  const tableId = `tbl_${generateId().replace(/-/g, '')}`
+  const now = new Date()
+
+  // Stamp stable ids so the table is id-keyed from its first row write.
+  const schema = withGeneratedColumnIds(data.schema)
+
+  // Row limits are enforced per-write against the current plan (see assertRowCapacity); the stored
+  // column is vestigial, so it just takes the caller's value (if any) or the default.
+  const maxRows = data.maxRows ?? TABLE_LIMITS.MAX_ROWS_PER_TABLE
+  const maxTables = data.maxTables ?? TABLE_LIMITS.MAX_TABLES_PER_WORKSPACE
+
+  const newTable = {
+    id: tableId,
+    name: data.name,
+    description: data.description ?? null,
+    schema,
+    workspaceId: data.workspaceId,
+    createdBy: data.userId,
+    maxRows,
+    archivedAt: null,
+    createdAt: now,
+    updatedAt: now,
+  }
+
+  // Create-mode CSV import is born with a running job so its rows stay hidden until ready.
+  const initialJob =
+    data.jobStatus === 'running' && data.jobId
+      ? { id: data.jobId, type: data.jobType ?? 'import', startedAt: now }
+      : null
+
+  // Starter rows count against the plan too. Checked before the tx (the lookup is a
+  // separate pool read) — a new table starts empty, so the footprint is just these.
+  const initialRowCount = data.initialRowCount ?? 0
+  let rowLimit: number | undefined
+  if (initialRowCount > 0) {
+    rowLimit = await assertRowCapacity({
+      workspaceId: data.workspaceId,
+      currentRowCount: 0,
+      addedRows: initialRowCount,
+    })
+  }
+
+  // Wrap count check, duplicate check, and insert in a transaction with FOR UPDATE
+  // to prevent TOCTOU race on the table count limit
+  try {
+    await db.transaction(async (trx) => {
+      await setTableTxTimeouts(trx)
+      await trx.execute(sql`SELECT 1 FROM workspace WHERE id = ${data.workspaceId} FOR UPDATE`)
+
+      const [{ count: existingCount }] = await trx
+        .select({ count: count() })
+        .from(userTableDefinitions)
+        .where(
+          and(
+            eq(userTableDefinitions.workspaceId, data.workspaceId),
+            isNull(userTableDefinitions.archivedAt)
+          )
+        )
+
+      if (Number(existingCount) >= maxTables) {
+        throw new Error(`Workspace has reached maximum table limit (${maxTables})`)
+      }
+
+      const duplicateName = await trx
+        .select({ id: userTableDefinitions.id })
+        .from(userTableDefinitions)
+        .where(
+          and(
+            eq(userTableDefinitions.workspaceId, data.workspaceId),
+            eq(userTableDefinitions.name, data.name),
+            isNull(userTableDefinitions.archivedAt)
+          )
+        )
+        .limit(1)
+
+      if (duplicateName.length > 0) {
+        throw new TableConflictError(data.name)
+      }
+
+      await trx.insert(userTableDefinitions).values(newTable)
+
+      if (initialJob) {
+        await trx.insert(tableJobs).values({
+          id: initialJob.id,
+          tableId,
+          workspaceId: data.workspaceId,
+          type: initialJob.type,
+          status: 'running',
+          startedAt: initialJob.startedAt,
+          updatedAt: initialJob.startedAt,
+        })
+      }
+
+      if (initialRowCount > 0) {
+        const orderKeys = nKeysBetween(null, null, initialRowCount)
+        const rowsToInsert = Array.from({ length: initialRowCount }, (_, i) => ({
+          id: `row_${generateId().replace(/-/g, '')}`,
+          tableId,
+          data: {},
+          position: i,
+          orderKey: orderKeys[i],
+          workspaceId: data.workspaceId,
+          createdAt: now,
+          updatedAt: now,
+        }))
+        await trx.insert(userTableRows).values(rowsToInsert)
+      }
+    })
+  } catch (error: unknown) {
+    if (error instanceof TableConflictError) {
+      throw error
+    }
+    if (getPostgresErrorCode(error) === '23505') {
+      throw new TableConflictError(data.name)
+    }
+    throw error
+  }
+
+  if (initialRowCount > 0 && rowLimit !== undefined) {
+    notifyTableRowUsage({
+      workspaceId: data.workspaceId,
+      currentRowCount: 0,
+      addedRows: initialRowCount,
+      limit: rowLimit,
+    })
+  }
+
+  logger.info(`[${requestId}] Created table ${tableId} in workspace ${data.workspaceId}`)
+
+  return {
+    id: newTable.id,
+    name: newTable.name,
+    description: newTable.description,
+    schema: newTable.schema as TableSchema,
+    metadata: null,
+    rowCount: data.initialRowCount ?? 0,
+    maxRows: newTable.maxRows,
+    workspaceId: newTable.workspaceId,
+    createdBy: newTable.createdBy,
+    archivedAt: newTable.archivedAt,
+    createdAt: newTable.createdAt,
+    updatedAt: newTable.updatedAt,
+    jobStatus: initialJob ? 'running' : null,
+    jobId: initialJob?.id ?? null,
+    jobType: initialJob?.type ?? null,
+    jobError: null,
+    jobRowsProcessed: 0,
+  }
+}
+
+/**
+ * Adds multiple columns to an existing table inside a caller-provided
+ * transaction. This is atomic with respect to the surrounding `trx`: either
+ * all columns are added or none are. Validates each column the same way
+ * `addTableColumn` does and rejects if any name collides with an existing
+ * column or another entry in `columns`.
+ *
+ * Use this when composing a column addition with other writes (e.g., row
+ * inserts) that must succeed or roll back together.
+ */
+export async function addTableColumnsWithTx(
+  trx: DbTransaction,
+  table: TableDefinition,
+  columns: { id?: string; name: string; type: string; required?: boolean; unique?: boolean }[],
+  requestId: string
+): Promise<TableDefinition> {
+  if (columns.length === 0) return table
+
+  const usedNames = new Set(table.schema.columns.map((c) => c.name.toLowerCase()))
+  const additions: TableSchema['columns'] = []
+
+  for (const column of columns) {
+    if (!NAME_PATTERN.test(column.name)) {
+      throw new Error(
+        `Invalid column name "${column.name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+      )
+    }
+    if (column.name.length > TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) {
+      throw new Error(
+        `Column name exceeds maximum length (${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters)`
+      )
+    }
+    if (!COLUMN_TYPES.includes(column.type as (typeof COLUMN_TYPES)[number])) {
+      throw new Error(
+        `Invalid column type "${column.type}". Must be one of: ${COLUMN_TYPES.join(', ')}`
+      )
+    }
+    const lower = column.name.toLowerCase()
+    if (usedNames.has(lower)) {
+      throw new Error(`Column "${column.name}" already exists`)
+    }
+    usedNames.add(lower)
+    // Honor a caller-assigned id (the CSV append path pre-assigns so coercion
+    // and persistence agree); otherwise mint one.
+    const id = column.id ?? generateColumnId()
+    additions.push({
+      id,
+      name: column.name,
+      type: column.type as TableSchema['columns'][number]['type'],
+      required: column.required ?? false,
+      unique: column.unique ?? false,
+    })
+  }
+
+  if (table.schema.columns.length + additions.length > TABLE_LIMITS.MAX_COLUMNS_PER_TABLE) {
+    throw new Error(
+      `Adding ${additions.length} column(s) would exceed maximum column limit (${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE})`
+    )
+  }
+
+  // Spread `table.schema` first so workflow groups (and any future top-level
+  // schema fields) survive a CSV import that only adds plain columns.
+  const updatedSchema: TableSchema = {
+    ...table.schema,
+    columns: [...table.schema.columns, ...additions],
+  }
+  const now = new Date()
+
+  await trx
+    .update(userTableDefinitions)
+    .set({ schema: updatedSchema, updatedAt: now })
+    .where(eq(userTableDefinitions.id, table.id))
+
+  logger.info(
+    `[${requestId}] Added ${additions.length} column(s) to table ${table.id}: ${additions.map((c) => c.name).join(', ')}`
+  )
+
+  return {
+    ...table,
+    schema: updatedSchema,
+    updatedAt: now,
+  }
+}
+
+/**
+ * Records the "columns added" audit for a table. The column add shares a
+ * transaction with the import's row inserts, so the caller MUST emit this only
+ * AFTER that transaction commits — auditing inside the tx would log a false
+ * success if a later row batch rolls it back. Skipped when no actor resolves.
+ */
+export function auditTableColumnsAdded(
+  table: TableDefinition,
+  columnNames: string[],
+  actingUserId?: string
+): void {
+  const actorId = actingUserId ?? table.createdBy
+  if (!actorId || columnNames.length === 0) return
+  recordAudit({
+    workspaceId: table.workspaceId ?? null,
+    actorId,
+    action: AuditAction.TABLE_UPDATED,
+    resourceType: AuditResourceType.TABLE,
+    resourceId: table.id,
+    resourceName: table.name,
+    description: `Added ${columnNames.length} column(s) to table "${table.name}"`,
+    metadata: { op: 'add_columns', columns: columnNames },
+  })
+}
+
+/**
+ * Renames a table.
+ *
+ * @param tableId - Table ID to rename
+ * @param newName - New table name
+ * @param requestId - Request ID for logging
+ * @returns Updated table definition
+ * @throws Error if name is invalid
+ */
+export async function renameTable(
+  tableId: string,
+  newName: string,
+  requestId: string,
+  actingUserId?: string
+): Promise<{ id: string; name: string }> {
+  const nameValidation = validateTableName(newName)
+  if (!nameValidation.valid) {
+    throw new Error(nameValidation.errors.join(', '))
+  }
+
+  const now = new Date()
+  try {
+    const result = await db
+      .update(userTableDefinitions)
+      .set({ name: newName, updatedAt: now })
+      .where(eq(userTableDefinitions.id, tableId))
+      .returning({
+        id: userTableDefinitions.id,
+        createdBy: userTableDefinitions.createdBy,
+        workspaceId: userTableDefinitions.workspaceId,
+      })
+
+    if (result.length === 0) {
+      throw new Error(`Table ${tableId} not found`)
+    }
+
+    const { createdBy, workspaceId } = result[0]
+    const renameActorId = actingUserId ?? createdBy
+    if (renameActorId) {
+      recordAudit({
+        workspaceId: workspaceId ?? null,
+        actorId: renameActorId,
+        action: AuditAction.TABLE_UPDATED,
+        resourceType: AuditResourceType.TABLE,
+        resourceId: tableId,
+        resourceName: newName,
+        description: `Renamed table to "${newName}"`,
+        metadata: { op: 'rename' },
+      })
+    }
+
+    logger.info(`[${requestId}] Renamed table ${tableId} to "${newName}"`)
+    return { id: tableId, name: newName }
+  } catch (error: unknown) {
+    if (getPostgresErrorCode(error) === '23505') {
+      throw new TableConflictError(newName)
+    }
+    throw error
+  }
+}
+
+/**
+ * Updates a table's metadata (UI state like column widths/order, plus behavioral
+ * settings like `workflowColumnBatchSize`). Merges into the existing metadata blob.
+ *
+ * @param tableId - Table ID to update
+ * @param metadata - Partial metadata object (merged with existing)
+ * @param existingMetadata - Existing metadata from a prior fetch (avoids redundant DB read)
+ * @returns Updated metadata
+ */
+export async function updateTableMetadata(
+  tableId: string,
+  metadata: TableMetadata,
+  existingMetadata?: TableMetadata | null
+): Promise<TableMetadata> {
+  const merged: TableMetadata = { ...(existingMetadata ?? {}), ...metadata }
+
+  // When `columnOrder` is in the patch, scrub any workflow-group dependency
+  // that now sits to the right of (or at the same index as) its group's
+  // leftmost column. Without this, reordering a column could leave a group
+  // depending on a column it can no longer reach in the dag — the group
+  // would never fire.
+  const newOrder = metadata.columnOrder
+  let nextSchema: TableSchema | null = null
+  if (Array.isArray(newOrder) && newOrder.length > 0) {
+    const [tableRow] = await db
+      .select({ schema: userTableDefinitions.schema })
+      .from(userTableDefinitions)
+      .where(eq(userTableDefinitions.id, tableId))
+      .limit(1)
+    if (tableRow) {
+      const schema = tableRow.schema as TableSchema
+      const groups = schema.workflowGroups ?? []
+      if (groups.length > 0) {
+        // `columnOrder` and group dep refs are both keyed by stable column id.
+        const positionOf = new Map<string, number>()
+        newOrder.forEach((id, i) => positionOf.set(id, i))
+        let mutated = false
+        const nextGroups = groups.map((group) => {
+          const ownCols = schema.columns.filter((c) => c.workflowGroupId === group.id)
+          let leftmost = Number.POSITIVE_INFINITY
+          for (const c of ownCols) {
+            const idx = positionOf.get(getColumnId(c)) ?? Number.POSITIVE_INFINITY
+            if (idx < leftmost) leftmost = idx
+          }
+          if (!Number.isFinite(leftmost)) return group
+          const deps = group.dependencies?.columns ?? []
+          const removed = new Set(deps.filter((dep) => (positionOf.get(dep) ?? -1) >= leftmost))
+          if (removed.size === 0) return group
+          const stripped = stripGroupDeps(group, removed)
+          if (stripped !== group) mutated = true
+          return stripped
+        })
+        if (mutated) nextSchema = { ...schema, workflowGroups: nextGroups }
+      }
+    }
+  }
+
+  await db
+    .update(userTableDefinitions)
+    .set(nextSchema ? { metadata: merged, schema: nextSchema } : { metadata: merged })
+    .where(eq(userTableDefinitions.id, tableId))
+
+  return merged
+}
+
+/**
+ * Archives a table.
+ *
+ * @param tableId - Table ID to delete
+ * @param requestId - Request ID for logging
+ */
+export async function deleteTable(
+  tableId: string,
+  requestId: string,
+  actingUserId?: string
+): Promise<void> {
+  const now = new Date()
+  const result = await db
+    .update(userTableDefinitions)
+    .set({ archivedAt: now, updatedAt: now })
+    .where(and(eq(userTableDefinitions.id, tableId), isNull(userTableDefinitions.archivedAt)))
+    .returning({
+      createdBy: userTableDefinitions.createdBy,
+      workspaceId: userTableDefinitions.workspaceId,
+      name: userTableDefinitions.name,
+    })
+
+  const deleted = result[0]
+  // Audit only genuine user deletes — rollback callers omit `actingUserId`. The
+  // caller emits the `table_deleted` PostHog event, so it is not duplicated here.
+  if (deleted && actingUserId) {
+    recordAudit({
+      workspaceId: deleted.workspaceId ?? null,
+      actorId: actingUserId,
+      action: AuditAction.TABLE_DELETED,
+      resourceType: AuditResourceType.TABLE,
+      resourceId: tableId,
+      resourceName: deleted.name,
+      description: `Archived table "${deleted.name}"`,
+    })
+  }
+
+  logger.info(`[${requestId}] Archived table ${tableId}`)
+}
+
+/**
+ * Restores an archived table.
+ */
+export async function restoreTable(tableId: string, requestId: string): Promise<void> {
+  const table = await getTableById(tableId, { includeArchived: true })
+  if (!table) {
+    throw new Error('Table not found')
+  }
+
+  if (!table.archivedAt) {
+    throw new Error('Table is not archived')
+  }
+
+  if (table.workspaceId) {
+    const { getWorkspaceWithOwner } = await import('@/lib/workspaces/permissions/utils')
+    const ws = await getWorkspaceWithOwner(table.workspaceId)
+    if (!ws || ws.archivedAt) {
+      throw new Error('Cannot restore table into an archived workspace')
+    }
+  }
+
+  /**
+   * A concurrent rename/create can claim the chosen name after `generateRestoreName`'s check (MVCC).
+   * Retries pick a new random suffix; 23505 maps to {@link TableConflictError} after exhaustion.
+   */
+  const maxUniqueViolationRetries = 8
+  let attemptedRestoreName = ''
+
+  for (let attempt = 0; attempt < maxUniqueViolationRetries; attempt++) {
+    attemptedRestoreName = ''
+    try {
+      await db.transaction(async (tx) => {
+        await setTableTxTimeouts(tx)
+        await tx.execute(sql`SELECT 1 FROM user_table_definitions WHERE id = ${tableId} FOR UPDATE`)
+
+        attemptedRestoreName = await generateRestoreName(table.name, async (candidate) => {
+          const [match] = await tx
+            .select({ id: userTableDefinitions.id })
+            .from(userTableDefinitions)
+            .where(
+              and(
+                eq(userTableDefinitions.workspaceId, table.workspaceId),
+                eq(userTableDefinitions.name, candidate),
+                isNull(userTableDefinitions.archivedAt)
+              )
+            )
+            .limit(1)
+          return !!match
+        })
+
+        const now = new Date()
+        await tx
+          .update(userTableDefinitions)
+          .set({ archivedAt: null, updatedAt: now, name: attemptedRestoreName })
+          .where(eq(userTableDefinitions.id, tableId))
+      })
+      break
+    } catch (error: unknown) {
+      if (getPostgresErrorCode(error) !== '23505') {
+        throw error
+      }
+      if (attempt === maxUniqueViolationRetries - 1) {
+        throw new TableConflictError(attemptedRestoreName || table.name)
+      }
+    }
+  }
+
+  logger.info(`[${requestId}] Restored table ${tableId} as "${attemptedRestoreName}"`)
+}

apps/sim/lib/table/snapshot-cache.test.ts

@@ -0,0 +1,188 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockDb,
+  mockSelectExportRowPage,
+  mockCreateMultipartUpload,
+  mockHeadObject,
+  mockDeleteFile,
+} = vi.hoisted(() => {
+  const limit = vi.fn()
+  return {
+    mockDb: { limit, select: () => ({ from: () => ({ where: () => ({ limit }) }) }) },
+    mockSelectExportRowPage: vi.fn(),
+    mockCreateMultipartUpload: vi.fn(),
+    mockHeadObject: vi.fn(),
+    mockDeleteFile: vi.fn(),
+  }
+})
+
+vi.mock('@sim/db', () => ({ db: mockDb }))
+vi.mock('@/lib/table/jobs/service', () => ({ selectExportRowPage: mockSelectExportRowPage }))
+vi.mock('@/lib/uploads/core/storage-service', () => ({
+  createMultipartUpload: mockCreateMultipartUpload,
+  headObject: mockHeadObject,
+  deleteFile: mockDeleteFile,
+}))
+
+import { getOrCreateTableSnapshot, TableSnapshotTooLargeError } from '@/lib/table/snapshot-cache'
+
+const table = {
+  id: 'tbl_1',
+  workspaceId: 'ws_1',
+  schema: { columns: [{ id: 'col_name', name: 'name', type: 'string' }] },
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+} as any
+
+let lastHandle: {
+  content: string
+  complete: ReturnType<typeof vi.fn>
+  abort: ReturnType<typeof vi.fn>
+} | null
+
+/** Queue the values successive `readRowsVersion` calls return. */
+function versions(...values: number[]) {
+  for (const v of values) mockDb.limit.mockResolvedValueOnce([{ rowsVersion: v }])
+}
+
+describe('getOrCreateTableSnapshot', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    lastHandle = null
+    mockDeleteFile.mockResolvedValue(undefined)
+    mockSelectExportRowPage.mockResolvedValueOnce([
+      { id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' },
+    ])
+    mockSelectExportRowPage.mockResolvedValue([])
+    mockCreateMultipartUpload.mockImplementation(({ key }: { key: string }) => {
+      const chunks: string[] = []
+      const handle = {
+        content: '',
+        write: vi.fn((chunk: Buffer | string) => {
+          chunks.push(typeof chunk === 'string' ? chunk : chunk.toString('utf8'))
+          return Promise.resolve()
+        }),
+        complete: vi.fn(() => {
+          handle.content = chunks.join('')
+          return Promise.resolve({ key, size: Buffer.byteLength(handle.content) })
+        }),
+        abort: vi.fn(() => Promise.resolve()),
+      }
+      lastHandle = handle
+      return Promise.resolve(handle)
+    })
+  })
+
+  it('returns the cached snapshot on a hit without reading rows', async () => {
+    versions(3)
+    mockHeadObject.mockResolvedValue({ size: 42 })
+
+    const ref = await getOrCreateTableSnapshot(table, 'req')
+
+    expect(ref).toEqual({
+      key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+      size: 42,
+      version: 3,
+    })
+    expect(mockCreateMultipartUpload).not.toHaveBeenCalled()
+    expect(mockSelectExportRowPage).not.toHaveBeenCalled()
+  })
+
+  it('materializes and stores on a miss, then cleans up the previous version', async () => {
+    versions(3, 3) // initial read, then unchanged recheck
+    mockHeadObject.mockResolvedValue(null)
+
+    const ref = await getOrCreateTableSnapshot(table, 'req')
+
+    expect(mockCreateMultipartUpload).toHaveBeenCalledWith(
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+        context: 'execution',
+      })
+    )
+    expect(lastHandle?.content).toBe('name\nAda\n')
+    expect(ref).toEqual({
+      key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+      size: Buffer.byteLength('name\nAda\n'),
+      version: 3,
+    })
+    // Best-effort prune of v2.
+    expect(mockDeleteFile).toHaveBeenCalledWith(
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v2-[0-9a-f]{12}\.csv$/),
+        context: 'execution',
+      })
+    )
+  })
+
+  it('keys the snapshot by tenant — the same table id in another workspace gets a different key', async () => {
+    versions(1)
+    mockHeadObject.mockResolvedValue({ size: 1 })
+    const ref = await getOrCreateTableSnapshot({ ...table, workspaceId: 'ws_2' }, 'req')
+    expect(ref.key).toMatch(/^table-snapshots\/ws_2\/tbl_1\/v1-[0-9a-f]{12}\.csv$/)
+  })
+
+  it('changes the key when the column shape changes (schema edits invalidate the cache)', async () => {
+    versions(7, 7)
+    mockHeadObject.mockResolvedValue({ size: 1 })
+
+    const a = await getOrCreateTableSnapshot(table, 'req')
+    const b = await getOrCreateTableSnapshot(
+      {
+        ...table,
+        schema: { columns: [{ id: 'col_name', name: 'renamed', type: 'string' }] },
+      } as never,
+      'req'
+    )
+
+    // Same workspace/table/row-version, but a renamed column flips the shape hash → different key.
+    expect(a.key).not.toBe(b.key)
+    expect(a.key).toMatch(/\/v7-[0-9a-f]{12}\.csv$/)
+    expect(b.key).toMatch(/\/v7-[0-9a-f]{12}\.csv$/)
+  })
+
+  it('re-keys and rebuilds when rows_version advances mid-scan', async () => {
+    versions(3, 4) // read v3, materialize, recheck sees v4
+    mockHeadObject.mockResolvedValueOnce(null) // v3 miss
+    mockHeadObject.mockResolvedValueOnce(null) // v4 miss → rebuild
+    // second materialize needs its own page sequence
+    mockSelectExportRowPage.mockReset()
+    mockSelectExportRowPage
+      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' }])
+      .mockResolvedValueOnce([])
+      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' }])
+      .mockResolvedValueOnce([])
+
+    const ref = await getOrCreateTableSnapshot(table, 'req')
+
+    expect(ref.version).toBe(4)
+    expect(ref.key).toMatch(/^table-snapshots\/ws_1\/tbl_1\/v4-[0-9a-f]{12}\.csv$/)
+    expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(2)
+    // the stale v3 object is dropped
+    expect(mockDeleteFile).toHaveBeenCalledWith(
+      expect.objectContaining({
+        key: expect.stringMatching(/^table-snapshots\/ws_1\/tbl_1\/v3-[0-9a-f]{12}\.csv$/),
+      })
+    )
+  })
+
+  it('aborts and throws when the CSV exceeds the size cap', async () => {
+    versions(1)
+    mockHeadObject.mockResolvedValue(null)
+    mockSelectExportRowPage.mockReset()
+    // A full batch of wide rows on every page → the materialize loop keeps paging until the running
+    // byte count crosses the cap, then aborts. Peak memory stays at one page (~MBs), not the cap.
+    const wideRow = { id: 'r', data: { col_name: 'x'.repeat(1000) }, orderKey: 'a0' }
+    const fullPage = Array.from({ length: 10000 }, () => wideRow)
+    mockSelectExportRowPage.mockResolvedValue(fullPage)
+
+    await expect(getOrCreateTableSnapshot(table, 'req')).rejects.toBeInstanceOf(
+      TableSnapshotTooLargeError
+    )
+    expect(lastHandle?.abort).toHaveBeenCalledTimes(1)
+    expect(lastHandle?.complete).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/table/snapshot-cache.ts

@@ -0,0 +1,188 @@
+/**
+ * Versioned CSV snapshot cache for table mounts.
+ *
+ * Materializes a table's CSV into object storage once per `rows_version` and reuses it across
+ * executions until the table mutates (the `bump_user_table_rows_version` trigger invalidates the
+ * key). This replaces draining the whole table into web-process heap on every mount.
+ *
+ * Tenant isolation: callers must pass a table they have already authorized (the
+ * `function-execute` mount path enforces `table.workspaceId === context.workspaceId`); the key is
+ * namespaced by `workspaceId` and the row reads are workspace-filtered, so a snapshot can only ever
+ * contain — and be addressed by — its owning tenant.
+ */
+
+import { createHash } from 'crypto'
+import { db } from '@sim/db'
+import { userTableDefinitions } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { eq } from 'drizzle-orm'
+import { getColumnId } from '@/lib/table/column-keys'
+import { formatCsvValue, neutralizeCsvFormula, toCsvRow } from '@/lib/table/export-format'
+import { selectExportRowPage } from '@/lib/table/jobs/service'
+import type { TableDefinition } from '@/lib/table/types'
+import { createMultipartUpload, deleteFile, headObject } from '@/lib/uploads/core/storage-service'
+
+const logger = createLogger('TableSnapshotCache')
+
+const SNAPSHOT_STORAGE_CONTEXT = 'execution' as const
+const SNAPSHOT_CONTENT_TYPE = 'text/csv; charset=utf-8'
+const SNAPSHOT_BATCH_SIZE = 5000
+
+/**
+ * Upper bound on a materialized snapshot. The sandbox now fetches snapshots by presigned URL (bytes
+ * never pass through web heap), so this is no longer a RAM limit — it bounds the worst-case inline
+ * materialization on a cache miss (a synchronous full-table scan in the copilot request). 500MB
+ * covers most large tables at ~tens of seconds; truly unbounded sizes want a background materializer.
+ */
+export const SNAPSHOT_MAX_BYTES = 500 * 1024 * 1024
+
+export interface TableSnapshotRef {
+  key: string
+  size: number
+  version: number
+}
+
+/** Thrown when a table's CSV would exceed {@link SNAPSHOT_MAX_BYTES}; surfaced as a mount error. */
+export class TableSnapshotTooLargeError extends Error {
+  constructor(tableId: string) {
+    super(
+      `Table ${tableId} is too large to mount (CSV exceeds ${SNAPSHOT_MAX_BYTES / 1024 / 1024}MB). Filter or split it before mounting.`
+    )
+    this.name = 'TableSnapshotTooLargeError'
+  }
+}
+
+/**
+ * Fingerprint of the table's column shape (id + display name + order). `rows_version` only advances
+ * on row mutations (the trigger fires on `user_table_rows`), so without this a schema edit — rename,
+ * add, remove, or reorder a column — would change the CSV header/columns but keep the same key and
+ * serve a stale snapshot. Folding it into the key invalidates the cache on any schema change. This
+ * is also the seam for a future column-subset / filtered projection (mix it into the same hash).
+ */
+function schemaFingerprint(table: TableDefinition): string {
+  const shape = table.schema.columns.map((c) => [getColumnId(c), c.name])
+  return createHash('sha1').update(JSON.stringify(shape)).digest('hex').slice(0, 12)
+}
+
+/** Storage key for a table's snapshot at a given row version + column shape. */
+function snapshotKey(
+  workspaceId: string,
+  tableId: string,
+  version: number,
+  shapeHash: string
+): string {
+  return `table-snapshots/${workspaceId}/${tableId}/v${version}-${shapeHash}.csv`
+}
+
+async function readRowsVersion(tableId: string): Promise<number> {
+  const [row] = await db
+    .select({ rowsVersion: userTableDefinitions.rowsVersion })
+    .from(userTableDefinitions)
+    .where(eq(userTableDefinitions.id, tableId))
+    .limit(1)
+  if (!row) throw new Error(`Table ${tableId} not found while reading rows_version`)
+  return row.rowsVersion
+}
+
+/**
+ * Streams the table CSV (keyset-paginated, like the export worker) into storage under `key`,
+ * aborting if it crosses {@link SNAPSHOT_MAX_BYTES}. Returns the stored byte size. Bytes match the
+ * canonical export format (id-keyed reads, display-name headers).
+ */
+async function materialize(table: TableDefinition, key: string): Promise<number> {
+  const columns = table.schema.columns
+  const handle = await createMultipartUpload({
+    key,
+    context: SNAPSHOT_STORAGE_CONTEXT,
+    contentType: SNAPSHOT_CONTENT_TYPE,
+  })
+
+  try {
+    let bytes = 0
+    const header = `${toCsvRow(columns.map((c) => neutralizeCsvFormula(c.name)))}\n`
+    bytes += Buffer.byteLength(header)
+    await handle.write(header)
+
+    let after: { orderKey: string; id: string } | null = null
+    while (true) {
+      const page = await selectExportRowPage(table, after, SNAPSHOT_BATCH_SIZE)
+      if (page.length === 0) break
+
+      const chunk = page
+        .map((row) => `${toCsvRow(columns.map((c) => formatCsvValue(row.data[getColumnId(c)])))}\n`)
+        .join('')
+      bytes += Buffer.byteLength(chunk)
+      if (bytes > SNAPSHOT_MAX_BYTES) throw new TableSnapshotTooLargeError(table.id)
+      await handle.write(chunk)
+
+      const last = page[page.length - 1]
+      after = { orderKey: last.orderKey, id: last.id }
+      if (page.length < SNAPSHOT_BATCH_SIZE) break
+    }
+
+    const { size } = await handle.complete()
+    return size
+  } catch (err) {
+    await handle.abort().catch(() => {})
+    throw err
+  }
+}
+
+/** Best-effort removal of the immediately-prior version (the common single-mutation case). */
+async function deletePreviousVersion(
+  table: TableDefinition,
+  version: number,
+  shapeHash: string
+): Promise<void> {
+  if (version <= 0) return
+  await deleteFile({
+    key: snapshotKey(table.workspaceId, table.id, version - 1, shapeHash),
+    context: SNAPSHOT_STORAGE_CONTEXT,
+  }).catch(() => {})
+}
+
+/**
+ * Returns the storage key + size of the table's snapshot at its current `rows_version`,
+ * materializing and storing it on a miss. The caller mounts by reference (head/download the key).
+ *
+ * Best-effort consistency: the version is read, the CSV materialized, then the version re-read. A
+ * mutation mid-scan (rare) re-keys to the new version and rebuilds once — no DB transaction is held
+ * across the upload. Concurrent misses write the same version-pinned key (idempotent).
+ */
+export async function getOrCreateTableSnapshot(
+  table: TableDefinition,
+  requestId: string
+): Promise<TableSnapshotRef> {
+  const shapeHash = schemaFingerprint(table)
+  const version = await readRowsVersion(table.id)
+  const key = snapshotKey(table.workspaceId, table.id, version, shapeHash)
+
+  const head = await headObject(key, SNAPSHOT_STORAGE_CONTEXT)
+  if (head) {
+    logger.info(`[${requestId}] Snapshot hit`, { tableId: table.id, version, size: head.size })
+    return { key, size: head.size, version }
+  }
+
+  logger.info(`[${requestId}] Snapshot miss; materializing`, { tableId: table.id, version })
+  const size = await materialize(table, key)
+
+  const after = await readRowsVersion(table.id)
+  if (after !== version) {
+    // The table mutated mid-scan: the bytes under `key` may be torn. Re-key to the new version and
+    // rebuild once (or reuse if a concurrent writer already stored it); drop the stale object.
+    logger.info(`[${requestId}] rows_version advanced during materialize; re-keying`, {
+      tableId: table.id,
+      from: version,
+      to: after,
+    })
+    const newKey = snapshotKey(table.workspaceId, table.id, after, shapeHash)
+    const newHead = await headObject(newKey, SNAPSHOT_STORAGE_CONTEXT)
+    const newSize = newHead ? newHead.size : await materialize(table, newKey)
+    await deleteFile({ key, context: SNAPSHOT_STORAGE_CONTEXT }).catch(() => {})
+    void deletePreviousVersion(table, after, shapeHash)
+    return { key: newKey, size: newSize, version: after }
+  }
+
+  void deletePreviousVersion(table, version, shapeHash)
+  return { key, size, version }
+}

apps/sim/lib/table/sql.ts

@@ -0,0 +1,604 @@
+/**
+ * SQL query builder utilities for user-defined tables.
+ *
+ * Uses JSONB containment operator (@>) for equality to leverage GIN index.
+ * Uses text extraction (->>) for comparisons and pattern matching.
+ */
+
+import { isRecordLike } from '@sim/utils/object'
+import type { SQL } from 'drizzle-orm'
+import { sql } from 'drizzle-orm'
+import { getColumnId } from '@/lib/table/column-keys'
+import { NAME_PATTERN } from '@/lib/table/constants'
+import type {
+  ColumnDefinition,
+  ConditionOperators,
+  Filter,
+  JsonValue,
+  Sort,
+} from '@/lib/table/types'
+
+/**
+ * Error thrown when caller-supplied filter or sort input is malformed.
+ * Routes should map this to HTTP 400 with the message preserved.
+ */
+export class TableQueryValidationError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'TableQueryValidationError'
+  }
+}
+
+type ColumnType = ColumnDefinition['type']
+type ColumnTypeMap = ReadonlyMap<string, ColumnType>
+
+/**
+ * Returns the Postgres cast needed to compare a JSONB text value of the given
+ * column type, or `null` when text comparison is correct. Single source of
+ * truth for both filter range operators and sort ordering — keeps the two
+ * paths from drifting apart.
+ */
+function jsonbCastForType(type: ColumnType | undefined): 'numeric' | 'timestamptz' | null {
+  switch (type) {
+    case 'number':
+      return 'numeric'
+    case 'date':
+      return 'timestamptz'
+    default:
+      return null
+  }
+}
+
+/**
+ * Maps a column's **stable id** (the JSONB storage key, via `getColumnId`) to
+ * its type. Filter/sort objects arrive keyed by column id, so the lookups in the
+ * clause builders use ids — not display names.
+ */
+function buildColumnTypeMap(columns: ColumnDefinition[]): ColumnTypeMap {
+  return new Map(columns.map((col) => [getColumnId(col), col.type]))
+}
+
+/**
+ * Whitelist of allowed operators for query filtering.
+ * Only these operators can be used in filter conditions.
+ */
+const ALLOWED_OPERATORS = new Set([
+  '$eq',
+  '$ne',
+  '$gt',
+  '$gte',
+  '$lt',
+  '$lte',
+  '$in',
+  '$nin',
+  '$contains',
+  '$ncontains',
+  '$startsWith',
+  '$endsWith',
+  '$empty',
+])
+
+/**
+ * Builds a WHERE clause from a filter object.
+ * Recursively processes logical operators ($or, $and) and field conditions.
+ *
+ * Index behavior: equality ($eq, $in) uses the JSONB containment operator (@>) and
+ * can leverage the GIN index on `user_table_rows.data` (jsonb_path_ops). Range
+ * operators ($gt, $gte, $lt, $lte), pattern matches ($contains, $ncontains,
+ * $startsWith, $endsWith), and emptiness checks ($empty) fall back to text
+ * extraction via `data->>'field'`, which defeats the GIN index and produces
+ * a sequential scan over the table's rows (bounded by a btree prefix on
+ * `table_id`). Prefer equality filters on hot paths; assume range filters are
+ * O(rows per table) until a per-column expression index is added.
+ *
+ * @param filter - Filter object with field conditions and logical operators
+ * @param tableName - Table name for the query (e.g., 'user_table_rows')
+ * @param columns - Column definitions; drives type-aware JSONB casts (numeric for numbers, timestamptz for dates)
+ * @returns SQL WHERE clause or undefined if no filter specified
+ * @throws {TableQueryValidationError} if field name is invalid or operator is not allowed
+ *
+ * @example
+ * // Simple equality
+ * buildFilterClause({ name: 'John' }, 'user_table_rows', [{ name: 'name', type: 'string' }])
+ *
+ * // Range on a date column — emits `::timestamptz` on both sides
+ * buildFilterClause(
+ *   { birthDate: { $gte: '2024-01-01' } },
+ *   'user_table_rows',
+ *   [{ name: 'birthDate', type: 'date' }],
+ * )
+ *
+ * // Logical operators
+ * buildFilterClause(
+ *   { $or: [{ status: 'active' }, { verified: true }] },
+ *   'user_table_rows',
+ *   [{ name: 'status', type: 'string' }, { name: 'verified', type: 'boolean' }],
+ * )
+ */
+export function buildFilterClause(
+  filter: Filter,
+  tableName: string,
+  columns: ColumnDefinition[]
+): SQL | undefined {
+  const columnTypeMap = buildColumnTypeMap(columns)
+  return buildFilterClauseInternal(filter, tableName, columnTypeMap)
+}
+
+function buildFilterClauseInternal(
+  filter: Filter,
+  tableName: string,
+  columnTypeMap: ColumnTypeMap
+): SQL | undefined {
+  const conditions: SQL[] = []
+
+  for (const [field, condition] of Object.entries(filter)) {
+    if (condition === undefined) {
+      continue
+    }
+
+    // This represents a case where the filter is a logical OR of multiple filters
+    // e.g. { $or: [{ status: 'active' }, { status: 'pending' }] }
+    if (field === '$or' && Array.isArray(condition)) {
+      const orClause = buildLogicalClause(condition as Filter[], tableName, 'OR', columnTypeMap)
+      if (orClause) {
+        conditions.push(orClause)
+      }
+      continue
+    }
+
+    // This represents a case where the filter is a logical AND of multiple filters
+    // e.g. { $and: [{ status: 'active' }, { status: 'pending' }] }
+    if (field === '$and' && Array.isArray(condition)) {
+      const andClause = buildLogicalClause(condition as Filter[], tableName, 'AND', columnTypeMap)
+      if (andClause) {
+        conditions.push(andClause)
+      }
+      continue
+    }
+
+    // Skip arrays for regular fields - arrays are only valid for $or and $and.
+    // If we encounter an array here, it's likely malformed input (e.g., { name: [filter1, filter2] })
+    // which doesn't have a clear semantic meaning, so we skip it.
+    if (Array.isArray(condition)) {
+      continue
+    }
+
+    // Build SQL conditions for this field. Returns array of SQL fragments for each operator.
+    const fieldConditions = buildFieldCondition(
+      tableName,
+      field,
+      condition as JsonValue | ConditionOperators,
+      columnTypeMap.get(field)
+    )
+    conditions.push(...fieldConditions)
+  }
+
+  if (conditions.length === 0) return undefined
+  if (conditions.length === 1) return conditions[0]
+
+  return sql.join(conditions, sql.raw(' AND '))
+}
+
+/**
+ * Builds an ORDER BY clause from a sort object.
+ *
+ * @param sort - Sort object with field names and directions
+ * @param tableName - Table name for the query (e.g., 'user_table_rows')
+ * @param columns - Column definitions; drives type-aware casts (numeric for numbers, timestamptz for dates)
+ * @returns SQL ORDER BY clause or undefined if no sort specified
+ * @throws {TableQueryValidationError} if field name or sort direction is invalid
+ *
+ * @example
+ * buildSortClause(
+ *   { name: 'asc' },
+ *   'user_table_rows',
+ *   [{ name: 'name', type: 'string' }],
+ * )
+ * // Returns: ORDER BY user_table_rows.data->>'name' ASC
+ *
+ * @example
+ * buildSortClause(
+ *   { salary: 'desc' },
+ *   'user_table_rows',
+ *   [{ name: 'salary', type: 'number' }],
+ * )
+ * // Returns: ORDER BY (user_table_rows.data->>'salary')::numeric DESC NULLS LAST
+ */
+export function buildSortClause(
+  sort: Sort,
+  tableName: string,
+  columns: ColumnDefinition[]
+): SQL | undefined {
+  const clauses: SQL[] = []
+  const columnTypeMap = buildColumnTypeMap(columns)
+
+  for (const [field, direction] of Object.entries(sort)) {
+    validateFieldName(field)
+
+    if (direction !== 'asc' && direction !== 'desc') {
+      throw new TableQueryValidationError(
+        `Invalid sort direction "${direction}". Must be "asc" or "desc".`
+      )
+    }
+
+    const columnType = columnTypeMap.get(field)
+    clauses.push(buildSortFieldClause(tableName, field, direction, columnType))
+  }
+
+  return clauses.length > 0 ? sql.join(clauses, sql.raw(', ')) : undefined
+}
+
+/**
+ * Validates a field name to prevent SQL injection.
+ * Field names must match the NAME_PATTERN (alphanumeric + underscore, starting with letter/underscore).
+ *
+ * @param field - The field name to validate
+ * @throws {TableQueryValidationError} if field name is invalid
+ */
+function validateFieldName(field: string): void {
+  if (!field || typeof field !== 'string') {
+    throw new TableQueryValidationError('Field name must be a non-empty string')
+  }
+
+  if (!NAME_PATTERN.test(field)) {
+    throw new TableQueryValidationError(
+      `Invalid field name "${field}". Field names must start with a letter or underscore, followed by alphanumeric characters or underscores.`
+    )
+  }
+}
+
+/**
+ * Validates an operator to ensure it's in the allowed list.
+ *
+ * @param operator - The operator to validate
+ * @throws {TableQueryValidationError} if operator is not allowed
+ */
+function validateOperator(operator: string): void {
+  if (!ALLOWED_OPERATORS.has(operator)) {
+    throw new TableQueryValidationError(
+      `Invalid operator "${operator}". Allowed operators: ${Array.from(ALLOWED_OPERATORS).join(', ')}`
+    )
+  }
+}
+
+/**
+ * Validates that a range-operator value matches its column's expected JS type
+ * before it reaches Postgres. Surfaces an actionable, column-named error at the
+ * SQL builder layer instead of a generic `invalid input syntax for type numeric`
+ * from the database.
+ */
+function validateComparisonValue(
+  field: string,
+  columnType: ColumnType | undefined,
+  cast: 'numeric' | 'timestamptz',
+  value: number | string
+): void {
+  if (cast === 'numeric' && typeof value !== 'number') {
+    const label = columnType ?? 'number'
+    throw new TableQueryValidationError(
+      `Range operator on column "${field}" (${label}) requires a number, got ${typeof value}`
+    )
+  }
+  if (cast === 'timestamptz' && typeof value !== 'string') {
+    throw new TableQueryValidationError(
+      `Range operator on column "${field}" (date) requires a date string, got ${typeof value}`
+    )
+  }
+}
+
+/**
+ * Builds SQL conditions for a single field based on the provided condition.
+ *
+ * Supports both simple equality checks (using JSONB containment) and complex
+ * operators like comparison, membership, and pattern matching. Field names are
+ * validated to prevent SQL injection, and operators are validated against an
+ * allowed whitelist.
+ *
+ * @param tableName - The name of the table to query (used for SQL table reference)
+ * @param field - The field name to filter on (must match NAME_PATTERN)
+ * @param condition - Either a simple value (for equality) or a ConditionOperators
+ *                    object with operators like $eq, $gt, $in, etc.
+ * @returns Array of SQL condition fragments. Multiple conditions are returned
+ *          when the condition object contains multiple operators.
+ * @throws {TableQueryValidationError} if field name is invalid or operator is not allowed
+ */
+function buildFieldCondition(
+  tableName: string,
+  field: string,
+  condition: JsonValue | ConditionOperators,
+  columnType: ColumnType | undefined
+): SQL[] {
+  validateFieldName(field)
+
+  const conditions: SQL[] = []
+
+  if (isRecordLike(condition)) {
+    for (const [op, value] of Object.entries(condition)) {
+      // Validate operator to ensure only allowed operators are used
+      validateOperator(op)
+
+      switch (op) {
+        case '$eq':
+          conditions.push(buildContainmentClause(tableName, field, value as JsonValue))
+          break
+
+        case '$ne':
+          conditions.push(
+            sql`NOT (${buildContainmentClause(tableName, field, value as JsonValue)})`
+          )
+          break
+
+        case '$gt':
+          conditions.push(
+            buildComparisonClause(tableName, field, '>', value as number | string, columnType)
+          )
+          break
+
+        case '$gte':
+          conditions.push(
+            buildComparisonClause(tableName, field, '>=', value as number | string, columnType)
+          )
+          break
+
+        case '$lt':
+          conditions.push(
+            buildComparisonClause(tableName, field, '<', value as number | string, columnType)
+          )
+          break
+
+        case '$lte':
+          conditions.push(
+            buildComparisonClause(tableName, field, '<=', value as number | string, columnType)
+          )
+          break
+
+        case '$in':
+          if (Array.isArray(value) && value.length > 0) {
+            if (value.length === 1) {
+              // Single value then use containment clause
+              conditions.push(buildContainmentClause(tableName, field, value[0]))
+            } else {
+              // Multiple values then use OR clause
+              const inConditions = value.map((v) => buildContainmentClause(tableName, field, v))
+              conditions.push(sql`(${sql.join(inConditions, sql.raw(' OR '))})`)
+            }
+          }
+          break
+
+        case '$nin':
+          if (Array.isArray(value) && value.length > 0) {
+            const ninConditions = value.map(
+              (v) => sql`NOT (${buildContainmentClause(tableName, field, v)})`
+            )
+            conditions.push(sql`(${sql.join(ninConditions, sql.raw(' AND '))})`)
+          }
+          break
+
+        case '$contains':
+          conditions.push(buildLikeClause(tableName, field, value as string, 'contains'))
+          break
+
+        case '$ncontains':
+          conditions.push(
+            buildLikeClause(tableName, field, value as string, 'contains', { negate: true })
+          )
+          break
+
+        case '$startsWith':
+          conditions.push(buildLikeClause(tableName, field, value as string, 'startsWith'))
+          break
+
+        case '$endsWith':
+          conditions.push(buildLikeClause(tableName, field, value as string, 'endsWith'))
+          break
+
+        case '$empty':
+          conditions.push(buildEmptyClause(tableName, field, coerceEmptyFlag(field, value)))
+          break
+
+        default:
+          // This should never happen due to validateOperator, but added for completeness.
+          // Throw a plain Error (→ 500) since reaching this default means the switch
+          // and ALLOWED_OPERATORS have drifted — that's a programmer error, not a caller error.
+          throw new Error(`Unsupported operator: ${op}`)
+      }
+    }
+  } else {
+    // Simple value (primitive or null) - shorthand for equality.
+    // Example: { name: 'John' } is equivalent to { name: { $eq: 'John' } }
+    // isRecordLike's negation can't structurally exclude ConditionOperators (no index
+    // signature), unlike the prior typeof-based narrowing, so the JsonValue-only shape
+    // of this branch is asserted rather than inferred.
+    conditions.push(buildContainmentClause(tableName, field, condition as JsonValue))
+  }
+
+  return conditions
+}
+
+/**
+ * Builds SQL clauses from nested filters and joins them with the specified operator.
+ *
+ * @example
+ * // OR operator
+ * buildLogicalClause(
+ *   [{ status: 'active' }, { status: 'pending' }],
+ *   'user_table_rows',
+ *   'OR'
+ * )
+ * // Returns: (data @> '{"status":"active"}'::jsonb OR data @> '{"status":"pending"}'::jsonb)
+ *
+ * @example
+ * // AND operator
+ * buildLogicalClause(
+ *   [{ age: { $gte: 18 } }, { verified: true }],
+ *   'user_table_rows',
+ *   'AND'
+ * )
+ * // Returns: ((data->>'age')::numeric >= 18 AND data @> '{"verified":true}'::jsonb)
+ */
+function buildLogicalClause(
+  subFilters: Filter[],
+  tableName: string,
+  operator: 'OR' | 'AND',
+  columnTypeMap: ColumnTypeMap
+): SQL | undefined {
+  const clauses: SQL[] = []
+  for (const subFilter of subFilters) {
+    const clause = buildFilterClauseInternal(subFilter, tableName, columnTypeMap)
+    if (clause) {
+      clauses.push(clause)
+    }
+  }
+
+  if (clauses.length === 0) return undefined
+  if (clauses.length === 1) return clauses[0]
+
+  return sql`(${sql.join(clauses, sql.raw(` ${operator} `))})`
+}
+
+/** Builds JSONB containment clause: `data @> '{"field": value}'::jsonb` (uses GIN index) */
+function buildContainmentClause(tableName: string, field: string, value: JsonValue): SQL {
+  const jsonObj = JSON.stringify({ [field]: value })
+  return sql`${sql.raw(`${tableName}.data`)} @> ${jsonObj}::jsonb`
+}
+
+/**
+ * Builds a typed range comparison against a JSONB cell.
+ *
+ * `number` columns cast both sides to `numeric`; `date` columns cast both sides
+ * to `timestamptz` so date strings compare chronologically and timezone offsets
+ * in ISO strings (e.g. `2024-01-01T00:00:00Z`) are preserved rather than
+ * silently stripped (which would make results depend on the server's TimeZone
+ * setting). Unknown/other types
+ * fall back to `numeric` (legacy default — preserves behavior for ad-hoc fields
+ * with no schema entry). The right-hand value is cast explicitly because
+ * drizzle parameterizes it as `text`; without the cast, Postgres would compare
+ * `text <op> text` and silently produce lexicographic results.
+ *
+ * Cannot use the GIN index — falls back to a sequential scan over the table's
+ * rows (bounded by the btree prefix on `table_id`).
+ */
+function buildComparisonClause(
+  tableName: string,
+  field: string,
+  operator: '>' | '>=' | '<' | '<=',
+  value: number | string,
+  columnType: ColumnType | undefined
+): SQL {
+  const escapedField = field.replace(/'/g, "''")
+  const cast = jsonbCastForType(columnType) ?? 'numeric'
+  validateComparisonValue(field, columnType, cast, value)
+  const cell = sql.raw(`(${tableName}.data->>'${escapedField}')::${cast}`)
+  return cast === 'timestamptz'
+    ? sql`${cell} ${sql.raw(operator)} ${value}::timestamptz`
+    : sql`${cell} ${sql.raw(operator)} ${value}`
+}
+
+/** Escapes LIKE/ILIKE wildcard characters so they match literally */
+export function escapeLikePattern(value: string): string {
+  return value.replace(/[\\%_]/g, '\\$&')
+}
+
+/**
+ * Builds a case-insensitive pattern match against a JSONB cell using ILIKE.
+ * `position` controls wildcard placement: `contains` → `%value%`, `startsWith`
+ * → `value%`, `endsWith` → `%value`. When `negate` is set the match is inverted
+ * and null cells are included — "does not contain X" should keep empty rows,
+ * mirroring `$ne` (which also surfaces nulls). Cannot use the GIN index; falls
+ * back to a sequential scan bounded by the `table_id` btree prefix.
+ */
+function buildLikeClause(
+  tableName: string,
+  field: string,
+  value: string,
+  position: 'contains' | 'startsWith' | 'endsWith',
+  options?: { negate?: boolean }
+): SQL {
+  const escapedField = field.replace(/'/g, "''")
+  // Coerce defensively: filters arriving via the raw v1 API / tools may carry a
+  // non-string value (e.g. `{ $contains: 123 }`), and ILIKE compares text anyway.
+  const text = String(value)
+  // An empty pattern collapses to `%`/`%%`, which matches every non-null row —
+  // a silent footgun for raw-API callers (the UI gates empty values out). Reject
+  // it, consistent with the range/`$empty` operand validation.
+  if (text.length === 0) {
+    const opName = position === 'contains' && options?.negate ? 'ncontains' : position
+    throw new TableQueryValidationError(
+      `$${opName} on column "${field}" requires a non-empty value`
+    )
+  }
+  const escaped = escapeLikePattern(text)
+  const pattern =
+    position === 'startsWith'
+      ? `${escaped}%`
+      : position === 'endsWith'
+        ? `%${escaped}`
+        : `%${escaped}%`
+  const cell = sql.raw(`${tableName}.data->>'${escapedField}'`)
+  return options?.negate
+    ? sql`(${cell} IS NULL OR ${cell} NOT ILIKE ${pattern})`
+    : sql`${cell} ILIKE ${pattern}`
+}
+
+/**
+ * Coerces a `$empty` operand to a boolean. Accepts a real boolean (the UI path)
+ * and the string forms `'true'` / `'false'` (lenient raw-API input). Anything
+ * else throws rather than silently inverting the check — a 400 with a clear
+ * message beats returning the opposite row set.
+ */
+function coerceEmptyFlag(field: string, value: unknown): boolean {
+  if (typeof value === 'boolean') return value
+  if (value === 'true') return true
+  if (value === 'false') return false
+  throw new TableQueryValidationError(
+    `$empty on column "${field}" requires a boolean, got ${typeof value}`
+  )
+}
+
+/**
+ * Builds an emptiness check against a JSONB cell. `isEmpty` matches null cells
+ * (absent key or JSON null, both surfaced as SQL NULL by `->>`) and empty
+ * strings; the negation requires the cell to be present and non-empty.
+ */
+function buildEmptyClause(tableName: string, field: string, isEmpty: boolean): SQL {
+  const escapedField = field.replace(/'/g, "''")
+  const cell = sql.raw(`${tableName}.data->>'${escapedField}'`)
+  return isEmpty
+    ? sql`(${cell} IS NULL OR ${cell} = '')`
+    : sql`(${cell} IS NOT NULL AND ${cell} <> '')`
+}
+
+/**
+ * Builds a single ORDER BY clause for a field.
+ * Timestamp fields use direct column access, others use JSONB text extraction.
+ * Numeric and date columns are cast to appropriate types for correct sorting.
+ *
+ * @param tableName - The table name
+ * @param field - The field name to sort by
+ * @param direction - Sort direction ('asc' or 'desc')
+ * @param columnType - Optional column type for type-aware sorting
+ */
+function buildSortFieldClause(
+  tableName: string,
+  field: string,
+  direction: 'asc' | 'desc',
+  columnType: ColumnType | undefined
+): SQL {
+  const escapedField = field.replace(/'/g, "''")
+  const directionSql = direction.toUpperCase()
+
+  if (field === 'createdAt' || field === 'updatedAt') {
+    return sql.raw(`${tableName}.${escapedField} ${directionSql}`)
+  }
+
+  const jsonbExtract = `${tableName}.data->>'${escapedField}'`
+  const cast = jsonbCastForType(columnType)
+
+  if (cast === null) {
+    // Sort as text (string, boolean, json, or unknown types)
+    return sql.raw(`${jsonbExtract} ${directionSql}`)
+  }
+
+  // NULLS LAST so rows with null/invalid values sort to the bottom regardless of direction
+  return sql.raw(`(${jsonbExtract})::${cast} ${directionSql} NULLS LAST`)
+}

apps/sim/lib/table/trigger.ts

@@ -0,0 +1,185 @@
+/**
+ * Direct trigger firing for table row events.
+ *
+ * When rows are inserted or updated in a table, this module looks up any
+ * active webhook triggers watching that table and fires workflow executions
+ * immediately - no polling or cron involved.
+ */
+
+import { createLogger } from '@sim/logger'
+import { generateShortId } from '@sim/utils/id'
+import { buildNameById, getColumnId, rowDataIdToName } from '@/lib/table/column-keys'
+import type { RowData, TableRow, TableSchema } from '@/lib/table/types'
+import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
+
+const logger = createLogger('TableTrigger')
+
+type EventType = 'insert' | 'update'
+
+interface TableTriggerPayload {
+  row: Record<string, unknown> | null
+  rawRow: Record<string, unknown>
+  previousRow: Record<string, unknown> | null
+  changedColumns: string[]
+  rowId: string
+  headers: string[]
+  tableId: string
+  tableName: string
+  timestamp: string
+}
+
+interface WebhookConfig {
+  tableId?: string
+  tableSelector?: string
+  manualTableId?: string
+  eventType?: string
+  watchColumns?: string | string[]
+  includeHeaders?: boolean
+}
+
+/**
+ * Fires workflow triggers for table row changes.
+ *
+ * This is fire-and-forget - errors are logged but never thrown.
+ * Call with `void fireTableTrigger(...)` to avoid blocking the caller.
+ *
+ * @param eventType - 'insert' for new rows, 'update' for changed rows
+ * @param oldRows - Map of row ID to previous data. Pass null for inserts.
+ */
+export async function fireTableTrigger(
+  tableId: string,
+  tableName: string,
+  eventType: EventType,
+  rows: TableRow[],
+  oldRows: Map<string, RowData> | null,
+  schema: TableSchema,
+  requestId: string
+): Promise<void> {
+  try {
+    // Lazy: the webhook utils/processor pull in the executor + blocks stack.
+    // Eager imports would force every `lib/table/service` consumer (e.g. the
+    // dispatcher) to pay that cold-start even when no trigger fires.
+    const { fetchActiveWebhooks } = await import('@/lib/webhooks/polling/utils')
+    const webhooks = await fetchActiveWebhooks('table')
+    if (webhooks.length === 0) return
+
+    const headers = schema.columns.map((c) => c.name)
+    // The webhook payload is name-keyed (the workflow author references columns
+    // by name); stored row data is id-keyed, so translate on the way out.
+    const nameById = buildNameById(schema)
+
+    // Filter to webhooks watching this table with a matching event type
+    const matching = webhooks.filter((entry) => {
+      const config = entry.webhook.providerConfig as WebhookConfig | null
+      // Canonical key `tableId` first; `tableSelector`/`manualTableId` are a transitional
+      // basic-first fallback for configs deployed before the canonical key was written.
+      const configTableId = readCanonicalTriggerValue(
+        config?.tableId,
+        config?.tableSelector,
+        config?.manualTableId
+      )
+      if (configTableId !== tableId) return false
+
+      const configEventType = config?.eventType ?? 'insert'
+      return configEventType === eventType
+    })
+
+    if (matching.length === 0) return
+
+    const { processPolledWebhookEvent } = await import('@/lib/webhooks/processor')
+
+    logger.info(
+      `[${requestId}] Firing ${matching.length} trigger(s) for ${rows.length} ${eventType} event(s) in table ${tableId}`
+    )
+
+    for (const { webhook: webhookData, workflow: workflowData } of matching) {
+      const config = webhookData.providerConfig as WebhookConfig | null
+      const watchColumns = parseWatchColumns(config?.watchColumns)
+      const includeHeaders = config?.includeHeaders !== false
+
+      for (const row of rows) {
+        const previousIdData = oldRows?.get(row.id) ?? null
+        // Translate id-keyed stored data → name-keyed for the external payload.
+        const rawRow = rowDataIdToName(row.data, nameById)
+        const previousRow = previousIdData ? rowDataIdToName(previousIdData, nameById) : null
+        const changedColumns = previousIdData
+          ? detectChangedColumns(previousIdData, row.data)
+              .map((id) => nameById.get(id))
+              .filter((name): name is string => name !== undefined)
+          : []
+
+        // For updates with watch columns, skip rows where no watched column changed
+        if (eventType === 'update' && watchColumns.length > 0) {
+          const hasWatchedChange = changedColumns.some((col) => watchColumns.includes(col))
+          if (!hasWatchedChange) continue
+        }
+
+        // Build mapped row if includeHeaders is enabled
+        let mappedRow: Record<string, unknown> | null = null
+        if (includeHeaders && headers.length > 0) {
+          mappedRow = {}
+          for (const col of schema.columns) {
+            mappedRow[col.name] = row.data[getColumnId(col)] ?? null
+          }
+        }
+
+        const payload: TableTriggerPayload = {
+          row: mappedRow,
+          rawRow,
+          previousRow,
+          changedColumns,
+          rowId: row.id,
+          headers,
+          tableId,
+          tableName,
+          timestamp: new Date().toISOString(),
+        }
+
+        const eventRequestId = generateShortId()
+
+        try {
+          const result = await processPolledWebhookEvent(
+            webhookData,
+            workflowData,
+            payload,
+            eventRequestId
+          )
+
+          if (!result.success) {
+            logger.error(
+              `[${eventRequestId}] Failed to fire table trigger for row ${row.id}:`,
+              result.statusCode,
+              result.error
+            )
+          }
+        } catch (error) {
+          logger.error(`[${eventRequestId}] Error firing table trigger for row ${row.id}:`, error)
+        }
+      }
+    }
+  } catch (error) {
+    logger.error(`[${requestId}] Error in fireTableTrigger:`, error)
+  }
+}
+
+function parseWatchColumns(watchColumns: string | string[] | undefined): string[] {
+  if (!watchColumns) return []
+  if (Array.isArray(watchColumns)) return watchColumns.filter(Boolean)
+  return watchColumns
+    .split(',')
+    .map((c) => c.trim())
+    .filter(Boolean)
+}
+
+function detectChangedColumns(oldData: RowData, newData: RowData): string[] {
+  const changed: string[] = []
+  const allKeys = new Set([...Object.keys(oldData), ...Object.keys(newData)])
+
+  for (const key of allKeys) {
+    if (JSON.stringify(oldData[key]) !== JSON.stringify(newData[key])) {
+      changed.push(key)
+    }
+  }
+
+  return changed
+}

apps/sim/lib/table/tx.ts

@@ -0,0 +1,50 @@
+/**
+ * Shared transaction / locking helpers for the table service layer.
+ *
+ * Internal module: not exposed via the `@/lib/table` barrel. Consumers import
+ * directly from `@/lib/table/tx`.
+ */
+
+import { sql } from 'drizzle-orm'
+import type { DbTransaction } from '@/lib/table/planner'
+
+const TIMEOUT_CAP_MS = 10 * 60_000
+
+/**
+ * Sets per-transaction Postgres timeouts via `SET LOCAL`.
+ *
+ * `lock_timeout` is the critical one: without it, a waiter inherits the full
+ * `statement_timeout` clock, so one stuck writer can drain the pool.
+ *
+ * Safe under pgBouncer transaction pooling — `SET LOCAL` is transaction-scoped
+ * and cleared at COMMIT/ROLLBACK before the session returns to the pool.
+ */
+export async function setTableTxTimeouts(
+  trx: DbTransaction,
+  opts?: { statementMs?: number; lockMs?: number; idleMs?: number }
+) {
+  const s = opts?.statementMs ?? 10_000
+  const l = opts?.lockMs ?? 3_000
+  const i = opts?.idleMs ?? 5_000
+  await trx.execute(sql.raw(`SET LOCAL statement_timeout = '${s}ms'`))
+  await trx.execute(sql.raw(`SET LOCAL lock_timeout = '${l}ms'`))
+  await trx.execute(sql.raw(`SET LOCAL idle_in_transaction_session_timeout = '${i}ms'`))
+}
+
+/**
+ * Scales `statement_timeout` to the expected row-count work.
+ *
+ * Bulk operations that rewrite JSONB or cascade row triggers (e.g.
+ * `replaceTableRows`, `deleteColumn`, `renameColumn`) scale roughly linearly
+ * with row count. A fixed cap would regress large-table users who never saw a
+ * timeout before `SET LOCAL` was introduced. This helper picks
+ * `max(baseMs, rowCount * perRowMs)`, capped at 10 minutes so a single
+ * runaway transaction cannot indefinitely pin a pool connection.
+ */
+export function scaledStatementTimeoutMs(
+  rowCount: number,
+  opts: { baseMs: number; perRowMs: number }
+): number {
+  const safeRowCount = Math.max(0, rowCount)
+  return Math.min(TIMEOUT_CAP_MS, Math.max(opts.baseMs, safeRowCount * opts.perRowMs))
+}

apps/sim/lib/table/types.ts

@@ -0,0 +1,700 @@
+/**
+ * Type definitions for user-defined tables.
+ */
+
+import type { COLUMN_TYPES } from '@/lib/table/constants'
+
+export type ColumnValue = string | number | boolean | null | Date
+export type JsonValue = ColumnValue | JsonValue[] | { [key: string]: JsonValue }
+
+/**
+ * Row data mapping **column id** → value at rest (in `user_table_rows.data`).
+ * The two name-translating boundaries (public v1 API, mothership tool) and CSV
+ * key by column name on the wire; everything else uses ids. Resolve a column's
+ * storage key with `getColumnId` from `./column-keys`.
+ */
+export type RowData = Record<string, JsonValue>
+
+export type SortDirection = 'asc' | 'desc'
+
+/** Sort specification mapping column names to direction. */
+export type Sort = Record<string, SortDirection>
+
+/** Option for dropdown/select components. */
+export interface ColumnOption {
+  value: string
+  label: string
+}
+
+export interface ColumnDefinition {
+  /**
+   * Stable storage key for this column. Row data, metadata, workflow-group
+   * refs, and filter/sort all key on this id; `name` is a pure display label
+   * that can change freely (rename is metadata-only). Absent only on legacy
+   * columns before the backfill — `getColumnId` falls back to `name`, which is
+   * the key those rows were already written under. New columns get a generated
+   * `col_…` from `generateColumnId`.
+   */
+  id?: string
+  name: string
+  type: (typeof COLUMN_TYPES)[number]
+  required?: boolean
+  unique?: boolean
+  /**
+   * When set, this column is one of a workflow group's outputs. The value in
+   * `row.data[getColumnId(col)]` is populated by the group's per-cell run.
+   */
+  workflowGroupId?: string
+}
+
+/** One group output → one plain column. */
+export interface WorkflowGroupOutput {
+  /** Source block id within the configured workflow. `''` for enrichment groups. */
+  blockId: string
+  /** Dot-path into that block's output. `''` for enrichment groups. */
+  path: string
+  /** Enrichment output id this column receives (enrichment groups only). */
+  outputId?: string
+  /**
+   * Stable **column id** (`getColumnId`) of the plain column in
+   * `schema.columns` that receives the produced value. Despite the field name,
+   * this holds the column id, not its display name — so a column rename never
+   * touches this ref. Legacy values equal the column name (== id pre-backfill).
+   */
+  columnName: string
+}
+
+export interface WorkflowGroupDependencies {
+  /**
+   * Stable **column ids** (`getColumnId`) that must be non-empty before this
+   * group runs. Workflow output columns count too — once an upstream group
+   * fills its output column, any downstream group depending on that column
+   * becomes eligible. The user model is uniform: deps are columns, not
+   * group-completion edges. Legacy values equal column names (== id pre-backfill).
+   */
+  columns?: string[]
+}
+
+/**
+ * How the group was created. `'manual'` groups are user-built workflow columns;
+ * `'enrichment'` groups are spawned from a shared enrichment template and hide
+ * launch / input-editing affordances in the config sidebar. Defaults to
+ * `'manual'` when absent (pre-feature groups).
+ */
+export type WorkflowGroupType = 'manual' | 'enrichment'
+
+/**
+ * Which workflow state a group's per-cell runs execute against: `'live'` runs
+ * the editable draft (current behavior); `'deployed'` runs the workflow's
+ * latest active deployment. Defaults to `'live'` when absent.
+ */
+export type WorkflowGroupDeploymentMode = 'live' | 'deployed'
+
+/** One workflow Start-block input field ← one table column. */
+export interface WorkflowGroupInputMapping {
+  /** `inputFormat` field name on the workflow's Start block. */
+  inputName: string
+  /**
+   * Stable **column id** (`getColumnId`) whose per-row value feeds that input.
+   * Despite the field name, this holds the column id, not its display name.
+   * Legacy values equal the column name (== id pre-backfill).
+   */
+  columnName: string
+}
+
+export interface WorkflowGroup {
+  id: string
+  /** Backing workflow id for `manual` groups. `''` for enrichment groups. */
+  workflowId: string
+  /** Registry enrichment id for `enrichment` groups. */
+  enrichmentId?: string
+  /** Display name; defaults to the workflow's / enrichment's name. */
+  name?: string
+  /** Provenance of the group. Defaults to `'manual'` when absent. */
+  type?: WorkflowGroupType
+  dependencies?: WorkflowGroupDependencies
+  outputs: WorkflowGroupOutput[]
+  /**
+   * Maps the workflow's Start-block input fields to the table columns that
+   * supply each per-row value. Absent / empty means no mapping configured yet.
+   */
+  inputMappings?: WorkflowGroupInputMapping[]
+  /**
+   * Which workflow state per-cell runs execute against. Defaults to `'live'`
+   * (editable draft) when absent. `'deployed'` runs the workflow's latest
+   * active deployment. Only meaningful for `manual` groups.
+   */
+  deploymentMode?: WorkflowGroupDeploymentMode
+  /**
+   * When `false`, the group never auto-fires from the scheduler — it can only
+   * be triggered manually via the "Run" actions. Defaults to `true` so
+   * existing groups keep firing on dep satisfaction. Persisted alongside the
+   * group definition; the scheduler reads it in `isGroupEligible`.
+   */
+  autoRun?: boolean
+}
+
+/**
+ * State of one provider in an enrichment cascade run. `matched`/`no_match`/
+ * `error` actually called the tool; `skipped` had insufficient inputs; `not_run`
+ * was never reached because an earlier provider matched.
+ */
+export type EnrichmentProviderStatus = 'matched' | 'no_match' | 'skipped' | 'error' | 'not_run'
+
+/**
+ * Outcome of one provider attempt in an enrichment cascade, for the enrichment
+ * details panel. The full configured cascade is recorded: `skipped` providers
+ * had insufficient inputs, `not_run` providers sit after the match.
+ */
+export interface EnrichmentProviderOutcome {
+  /** Provider id, e.g. `'hunter'`. */
+  id: string
+  /** Human label, e.g. `'Hunter'`. */
+  label: string
+  /** Tool id the provider runs, e.g. `'hunter_find_email'` — resolves the block
+   *  icon for the details panel. */
+  toolId: string
+  status: EnrichmentProviderStatus
+  /** Hosted-key cost (USD) this provider incurred; `0` for skip / no_match / error / BYOK. */
+  cost: number
+  /** Wall-clock ms this provider's tool call took; `0` for skipped. */
+  durationMs: number
+  /** Error message when `status === 'error'`, else `null`. */
+  error: string | null
+}
+
+/**
+ * Per-(row, group) cascade breakdown for an enrichment run, surfaced in the
+ * enrichment details panel. Persisted on the `tableRowExecutions` sidecar but
+ * deliberately kept out of the hot grid read path (fetched on demand) — it can
+ * carry a dozen provider outcomes per cell.
+ */
+export interface EnrichmentRunDetail {
+  /** ISO timestamp when the cascade started. */
+  startedAt: string
+  /** ISO timestamp when the cascade finished. */
+  completedAt: string
+  /** Wall-clock ms across the whole cascade. */
+  durationMs: number
+  /** Sum of per-provider hosted-key cost (USD). */
+  totalCost: number
+  /** Provider id that produced the match, or `null` on no match. */
+  matchedProvider: string | null
+  /** True when the run was cancelled (stop / signal abort) — drives a
+   *  "Cancelled" result rather than inferring no-match/not-run from the cascade. */
+  aborted: boolean
+  /** Every configured provider, in cascade order (including `not_run` ones). */
+  providers: EnrichmentProviderOutcome[]
+}
+
+/**
+ * Per-row execution state for one workflow group, persisted as a row in the
+ * `tableRowExecutions` sidecar keyed by `(rowId, groupId)`. Holds run
+ * metadata only — picked output values land in `row.data` directly.
+ */
+export interface RowExecutionMetadata {
+  status: 'pending' | 'queued' | 'running' | 'completed' | 'error' | 'cancelled'
+  executionId: string | null
+  /**
+   * Async-job id (e.g. trigger.dev run id) for the in-flight execution.
+   * Persisted on `running` / `pending` rows so the cancel API can call
+   * `backend.cancelJob(jobId)` from any pod regardless of which one
+   * initiated the run. Null for terminal states.
+   */
+  jobId: string | null
+  workflowId: string
+  error: string | null
+  /** Block ids currently mid-execution. Empty / absent on terminal states. */
+  runningBlockIds?: string[]
+  /**
+   * Per-block error messages keyed by `blockId`. Errors are a normal Sim
+   * concept (error-port edges) — only the column sourced from the failing
+   * block should render `Error`, not every output column.
+   */
+  blockErrors?: Record<string, string>
+  /** ISO timestamp set when a cell is cancelled. The dispatcher skips
+   *  re-runs whose `cancelledAt > dispatch.requestedAt` — a user cancel
+   *  mid-dispatch must not be overridden by `isManualRun`. */
+  cancelledAt?: string
+  /**
+   * Enrichment cascade breakdown for `enrichment`-type groups, written on the
+   * terminal cell write. Persisted on `tableRowExecutions` but NOT hydrated by
+   * `loadExecutionsByRow` (kept off the hot grid read) — read it on demand via
+   * `loadEnrichmentDetail` for the details panel.
+   */
+  enrichmentDetails?: EnrichmentRunDetail | null
+}
+
+/** Map of `WorkflowGroup.id` → execution state. Stored on every row. */
+export type RowExecutions = Record<string, RowExecutionMetadata>
+
+export interface TableSchema {
+  columns: ColumnDefinition[]
+  /**
+   * Workflow groups keyed by id. Each group has N output columns (each
+   * referenced by `outputs[].columnName` in this same schema).
+   */
+  workflowGroups?: WorkflowGroup[]
+}
+
+/**
+ * Table-level metadata stored alongside the table definition. UI state only
+ * (column widths, column order, pinned columns) — workflow-group concurrency
+ * is enforced at the trigger.dev queue layer, not via metadata.
+ */
+export interface TableMetadata {
+  /** Pixel widths keyed by **column id** (`getColumnId`). */
+  columnWidths?: Record<string, number>
+  /** Visible left-to-right order as **column ids** (`getColumnId`). */
+  columnOrder?: string[]
+  /** **Column ids** pinned to the left while scrolling horizontally. */
+  pinnedColumns?: string[]
+}
+
+/** Async background-job lifecycle state for a table. NULL/undefined = idle (no job). */
+export type TableJobStatus = 'running' | 'ready' | 'failed' | 'canceled'
+
+/**
+ * Which kind of background job a `table_jobs` row tracks. `import`, `delete`, and `backfill`
+ * mutate row data and share the single-running-job gate; `export` is read-only and bypasses it
+ * (the partial-unique index excludes it), so an export can run alongside any other job.
+ */
+export type TableJobType = 'import' | 'delete' | 'export' | 'backfill' | 'update'
+
+/**
+ * Persisted scope of a running delete job (`table_jobs.payload`). Defines the doomed row set —
+ * `matches(filter) AND created_at <= cutoff AND id NOT IN excludeRowIds` — so the rows read-path
+ * can mask those rows out while the job runs, making mid-job reads (refresh, other clients)
+ * consistent with the eventual result.
+ */
+export interface TableDeleteJobPayload {
+  filter?: Filter
+  excludeRowIds?: string[]
+  /** ISO timestamp; rows created after it are spared. */
+  cutoff: string
+  /** Doomed-row estimate captured at kickoff — display-only: list/detail counts subtract the
+   *  not-yet-deleted remainder (doomedCount - rows_processed) while the job runs. Set only for an
+   *  unbounded delete (the masked "delete everything matching" path); omitted when `maxRows` is set. */
+  doomedCount?: number
+  /**
+   * Stop after deleting this many rows (an explicit caller-supplied limit above the inline cap).
+   * Omitted = delete every match. When set, reads are NOT masked: the delete is eventually
+   * consistent (rows disappear as they're deleted) like a bounded update, because the filter-based
+   * mask would over-hide the rows beyond the cap that this job never deletes.
+   */
+  maxRows?: number
+}
+
+/**
+ * Persisted scope of a running bulk-update job (`table_jobs.payload`): the same `data` patch is
+ * merged into every row matching `filter` with `created_at <= cutoff` (so mid-job inserts are
+ * spared, matching the delete job's snapshot semantics). `affectedCount` is the kickoff estimate,
+ * display-only. Unlike delete, reads are not masked — updated rows still exist, so a background
+ * update is eventually consistent (readers may see a mix of patched/unpatched rows mid-job).
+ */
+export interface TableUpdateJobPayload {
+  filter: Filter
+  /** Column-id-keyed partial patch applied to every matched row (JSONB merge). */
+  data: RowData
+  /** ISO timestamp; rows created after it are not patched. */
+  cutoff: string
+  affectedCount?: number
+  /** Stop after updating this many rows (an explicit caller-supplied limit). Omitted = every match. */
+  maxRows?: number
+}
+
+/**
+ * Persisted scope of an export job (`table_jobs.payload`). `resultKey` is merged in by the worker
+ * on completion — the storage key of the generated file, served to the client via a presigned URL
+ * and deleted by the janitor when the terminal job is pruned.
+ */
+export interface TableExportJobPayload {
+  format: 'csv' | 'json'
+  resultKey?: string
+}
+
+/**
+ * Keyset cursor for paginating a table's default row order, `(order_key, id)`. The grid's
+ * infinite scroll threads this instead of an OFFSET — offset paging re-scans every prior row per
+ * page (O(N²) to drain a table); the cursor makes each page an index seek on
+ * `(table_id, order_key, id)`. Only valid for the default order: sorted views fall back to offset.
+ */
+export interface TableRowsCursor {
+  orderKey: string
+  id: string
+}
+
+/** Persisted scope of an output-column backfill job (`table_jobs.payload`). */
+export interface TableBackfillJobPayload {
+  groupId: string
+  outputs: WorkflowGroupOutput[]
+  /** Remaps overwrite existing cell values; added columns never clobber hand-edits. */
+  overwrite: boolean
+}
+
+export interface TableDefinition {
+  id: string
+  name: string
+  description?: string | null
+  schema: TableSchema
+  metadata?: TableMetadata | null
+  rowCount: number
+  maxRows: number
+  workspaceId: string
+  createdBy: string
+  archivedAt?: Date | string | null
+  createdAt: Date | string
+  updatedAt: Date | string
+  /**
+   * Async background-job state, derived from the table's latest `table_jobs` row (running if any,
+   * else the most recent terminal). See `import-runner.ts` / `delete-runner.ts`.
+   */
+  jobStatus?: TableJobStatus | null
+  jobId?: string | null
+  jobType?: TableJobType | null
+  jobError?: string | null
+  jobRowsProcessed?: number
+}
+
+/** Minimal table info for UI components. */
+export type TableInfo = Pick<TableDefinition, 'id' | 'name' | 'schema'>
+
+/** Simplified table summary for LLM enrichment and display contexts. */
+export interface TableSummary {
+  name: string
+  columns: Array<Pick<ColumnDefinition, 'name' | 'type'>>
+}
+
+export interface TableRow {
+  id: string
+  data: RowData
+  /** Per-group execution state for this row. Empty `{}` if nothing has run. */
+  executions: RowExecutions
+  position: number
+  /**
+   * Fractional order key — the authoritative row order. Absent only for rows not
+   * yet backfilled (clients fall back to `position`).
+   */
+  orderKey?: string
+  createdAt: Date | string
+  updatedAt: Date | string
+}
+
+/**
+ * MongoDB-style query operators for field comparisons.
+ *
+ * @example
+ * { $eq: 'John' }
+ * { $gte: 18, $lt: 65 }
+ * { $in: ['active', 'pending'] }
+ */
+export interface ConditionOperators {
+  $eq?: ColumnValue
+  $ne?: ColumnValue
+  $gt?: number | string
+  $gte?: number | string
+  $lt?: number | string
+  $lte?: number | string
+  $in?: ColumnValue[]
+  $nin?: ColumnValue[]
+  $contains?: string
+  /** Case-insensitive negated substring match. Null/empty cells match. */
+  $ncontains?: string
+  /** Case-insensitive prefix match. */
+  $startsWith?: string
+  /** Case-insensitive suffix match. */
+  $endsWith?: string
+  /** `true` → cell is null or empty string; `false` → cell is present and non-empty. */
+  $empty?: boolean
+}
+
+/**
+ * Filter object for querying table rows. Supports direct equality shorthand,
+ * operator objects, and logical $or/$and combinators.
+ *
+ * @example
+ * { name: 'John' }
+ * { age: { $gte: 18 } }
+ * { $or: [{ status: 'active' }, { status: 'pending' }] }
+ */
+export interface Filter {
+  $or?: Filter[]
+  $and?: Filter[]
+  [key: string]: ColumnValue | ConditionOperators | Filter[] | undefined
+}
+
+export interface ValidationResult {
+  valid: boolean
+  errors: string[]
+}
+
+/**
+ * UI builder state for a single filter rule.
+ * Includes an `id` field for React keys and string values for form inputs.
+ */
+export interface FilterRule {
+  id: string
+  logicalOperator: 'and' | 'or'
+  column: string
+  operator: string
+  value: string
+  collapsed?: boolean
+}
+
+/**
+ * UI builder state for a single sort rule.
+ * Includes an `id` field for React keys.
+ */
+export interface SortRule {
+  id: string
+  column: string
+  direction: SortDirection
+  collapsed?: boolean
+}
+
+export interface QueryOptions {
+  filter?: Filter
+  sort?: Sort
+  limit?: number
+  offset?: number
+  /** Keyset cursor for the default `(order_key, id)` order — see {@link TableRowsCursor}.
+   *  Mutually exclusive with `sort` and `offset`; takes precedence over `offset` when set. */
+  after?: TableRowsCursor
+  /**
+   * When true (default), runs a `COUNT(*)` and returns `totalCount` as a number.
+   * Pass `false` to skip the count query (grid UI doesn't need it); `totalCount`
+   * is returned as `null` to signal it was not computed.
+   */
+  includeTotal?: boolean
+  /**
+   * When true (default), each returned row's `executions` is populated from the
+   * `tableRowExecutions` sidecar. Pass `false` to skip the join and return `{}`
+   * (the public v1 route does not expose executions).
+   */
+  withExecutions?: boolean
+}
+
+export interface QueryResult {
+  rows: TableRow[]
+  rowCount: number
+  totalCount: number | null
+  limit: number
+  offset: number
+}
+
+export interface BulkOperationResult {
+  affectedCount: number
+  affectedRowIds: string[]
+}
+
+export interface CreateTableData {
+  name: string
+  description?: string
+  schema: TableSchema
+  workspaceId: string
+  userId: string
+  /** Optional stored row cap. Vestigial under plan-based enforcement (the column is no longer
+   *  consulted on insert), but retained so callers that still set it type-check. */
+  maxRows?: number
+  /** Optional max tables override based on billing plan. Defaults to TABLE_LIMITS.MAX_TABLES_PER_WORKSPACE. */
+  maxTables?: number
+  /** Number of empty rows to create with the table. Defaults to 0. */
+  initialRowCount?: number
+  /** When set, the table is created with this job already running (rows hidden until ready). */
+  jobStatus?: TableJobStatus
+  /** Job kind, paired with `jobStatus` (create-mode import sets `'import'`). */
+  jobType?: TableJobType
+  /** Async job id stamped on the table when `jobStatus` is set. */
+  jobId?: string
+}
+
+export interface InsertRowData {
+  tableId: string
+  data: RowData
+  workspaceId: string
+  userId?: string
+  /** Optional explicit position. When omitted, the row is appended after the last position. */
+  position?: number
+  /** Insert directly after this row (fractional ordering). Takes precedence over `position`. */
+  afterRowId?: string
+  /** Insert directly before this row (fractional ordering). Takes precedence over `position`. */
+  beforeRowId?: string
+}
+
+export interface BatchInsertData {
+  tableId: string
+  rows: RowData[]
+  workspaceId: string
+  userId?: string
+  /**
+   * Optional per-row exact order keys (undo restore re-inserts at the saved key).
+   * Length must equal `rows.length`.
+   */
+  orderKeys?: string[]
+}
+
+export interface UpsertRowData {
+  tableId: string
+  workspaceId: string
+  data: RowData
+  userId?: string
+  /** Which unique column to match on. Required when multiple unique columns exist. */
+  conflictTarget?: string
+}
+
+export interface UpsertResult {
+  row: TableRow
+  operation: 'insert' | 'update'
+  previousData?: RowData
+}
+
+export interface UpdateRowData {
+  tableId: string
+  rowId: string
+  data: RowData
+  workspaceId: string
+  /**
+   * Optional partial patch to apply to the row's `tableRowExecutions`
+   * entries. Top-level keys are `WorkflowGroup.id`; pass `null` for a key
+   * to delete that group's execution row. Used by the cell task and cancel
+   * paths.
+   */
+  executionsPatch?: Record<string, RowExecutionMetadata | null>
+  /**
+   * Optional SQL-level guard: the update is a no-op if the row's
+   * `executions[groupId]` already shows `cancelled` for the same
+   * `executionId`. The cell task passes this so a `running` partial-write
+   * landing after a stop click can't clobber the authoritative `cancelled`
+   * state. `updateRow` returns `null` when the guard rejects the write.
+   */
+  cancellationGuard?: { groupId: string; executionId: string }
+  /**
+   * The member who performed this write. Billed and usage-gated for any
+   * enrichment the write triggers (auto-fire or dependency-cascade re-run), so
+   * costs land on the editor's per-member meter rather than the workspace billed
+   * account. Omitted only for internal `executionsPatch`-only writes.
+   */
+  actorUserId?: string | null
+}
+
+export interface BulkUpdateData {
+  filter: Filter
+  data: RowData
+  limit?: number
+  /** The member who performed this write — billed/gated for triggered enrichment. */
+  actorUserId?: string | null
+}
+
+export interface BatchUpdateByIdData {
+  tableId: string
+  updates: Array<{
+    rowId: string
+    data: RowData
+    executionsPatch?: Record<string, RowExecutionMetadata | null>
+  }>
+  workspaceId: string
+  /** The member who performed this write — billed/gated for triggered enrichment. */
+  actorUserId?: string | null
+}
+
+export interface BulkDeleteData {
+  filter: Filter
+  limit?: number
+}
+
+export interface BulkDeleteByIdsData {
+  tableId: string
+  rowIds: string[]
+  workspaceId: string
+}
+
+export interface BulkDeleteByIdsResult {
+  deletedCount: number
+  deletedRowIds: string[]
+  requestedCount: number
+  missingRowIds: string[]
+}
+
+export interface ReplaceRowsData {
+  tableId: string
+  rows: RowData[]
+  workspaceId: string
+  userId?: string
+}
+
+export interface ReplaceRowsResult {
+  deletedCount: number
+  insertedCount: number
+}
+
+export interface RenameColumnData {
+  tableId: string
+  oldName: string
+  newName: string
+}
+
+export interface UpdateColumnTypeData {
+  tableId: string
+  columnName: string
+  newType: (typeof COLUMN_TYPES)[number]
+}
+
+export interface UpdateColumnConstraintsData {
+  tableId: string
+  columnName: string
+  required?: boolean
+  unique?: boolean
+}
+
+export interface DeleteColumnData {
+  tableId: string
+  columnName: string
+}
+
+/** Payload for `addWorkflowGroup` — atomic insert of a group + its outputs. */
+export interface AddWorkflowGroupData {
+  tableId: string
+  group: WorkflowGroup
+  outputColumns: ColumnDefinition[]
+  /** When `false`, the post-add row-scheduling pass is skipped. Defaults to
+   *  `true` (UI behavior). Mothership passes `false` so groups can be staged
+   *  without firing every dep-satisfied row. */
+  autoRun?: boolean
+  /** The member adding the group — billed/gated for the auto-run enrichment pass. */
+  actorUserId?: string | null
+}
+
+/** Payload for `updateWorkflowGroup` — diffs outputs and writes columns. */
+export interface UpdateWorkflowGroupData {
+  tableId: string
+  groupId: string
+  workflowId?: string
+  name?: string
+  dependencies?: WorkflowGroupDependencies
+  /** Full replacement set; service computes adds/removes vs current state. */
+  outputs?: WorkflowGroupOutput[]
+  /** Column definitions for any newly-added outputs. */
+  newOutputColumns?: ColumnDefinition[]
+  /**
+   * Per-column mapping swaps: keep the existing column, repoint it at a new
+   * `(blockId, path)`. Applied before the `outputs` diff and clears the
+   * affected columns' row data so the next run repopulates from the new
+   * source.
+   */
+  mappingUpdates?: Array<{ columnName: string; blockId: string; path: string }>
+  /** Replace the group's input mappings. Omit to leave them unchanged. */
+  inputMappings?: WorkflowGroupInputMapping[]
+  /** Change which workflow state the group runs against. Omit to leave unchanged. */
+  deploymentMode?: WorkflowGroupDeploymentMode
+  /** Update the group's provenance. Omit to leave it unchanged. */
+  type?: WorkflowGroupType
+  /** Toggle the group's auto-run flag. Omit to leave it unchanged. */
+  autoRun?: boolean
+  /** The member updating the group — billed/gated for any triggered re-run. */
+  actorUserId?: string | null
+}
+
+export interface DeleteWorkflowGroupData {
+  tableId: string
+  groupId: string
+}

apps/sim/lib/table/update-runner.test.ts

@@ -0,0 +1,219 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const {
+  mockGetTableById,
+  mockGetJobProgress,
+  mockSelectRowDataPage,
+  mockUpdatePageByIds,
+  mockUpdateJobProgress,
+  mockMarkJobReady,
+  mockMarkJobFailed,
+  mockAppendTableEvent,
+  mockBuildFilterClause,
+  mockValidateRowSize,
+  mockCoerceRowToSchema,
+  mockCoerceRowValues,
+} = vi.hoisted(() => ({
+  mockGetTableById: vi.fn(),
+  mockGetJobProgress: vi.fn(),
+  mockSelectRowDataPage: vi.fn(),
+  mockUpdatePageByIds: vi.fn(),
+  mockUpdateJobProgress: vi.fn(),
+  mockMarkJobReady: vi.fn(),
+  mockMarkJobFailed: vi.fn(),
+  mockAppendTableEvent: vi.fn(),
+  mockBuildFilterClause: vi.fn(),
+  mockValidateRowSize: vi.fn(),
+  mockCoerceRowToSchema: vi.fn(),
+  mockCoerceRowValues: vi.fn(),
+}))
+
+vi.mock('@/lib/table/service', () => ({ getTableById: mockGetTableById }))
+vi.mock('@/lib/table/jobs/service', () => ({
+  getJobProgress: mockGetJobProgress,
+  updateJobProgress: mockUpdateJobProgress,
+  markJobReady: mockMarkJobReady,
+  markJobFailed: mockMarkJobFailed,
+}))
+vi.mock('@/lib/table/rows/ordering', () => ({
+  selectRowDataPage: mockSelectRowDataPage,
+  updatePageByIds: mockUpdatePageByIds,
+}))
+vi.mock('@/lib/table/events', () => ({ appendTableEvent: mockAppendTableEvent }))
+vi.mock('@/lib/table/sql', () => ({ buildFilterClause: mockBuildFilterClause }))
+vi.mock('@/lib/table/validation', () => ({
+  validateRowSize: mockValidateRowSize,
+  coerceRowToSchema: mockCoerceRowToSchema,
+  coerceRowValues: mockCoerceRowValues,
+}))
+vi.mock('@/lib/table/constants', () => ({
+  TABLE_LIMITS: { DELETE_PAGE_SIZE: 2, UPDATE_BATCH_SIZE: 100 },
+  USER_TABLE_ROWS_SQL_NAME: 'user_table_rows',
+}))
+
+import { markTableUpdateFailed, runTableUpdate } from '@/lib/table/update-runner'
+
+const table = { id: 'tbl_1', workspaceId: 'ws_1', schema: { columns: [] } }
+const cutoff = new Date('2026-06-05T00:00:00Z')
+
+function basePayload(overrides = {}) {
+  return {
+    jobId: 'job_1',
+    tableId: 'tbl_1',
+    workspaceId: 'ws_1',
+    filter: { status: 'old' },
+    data: { flag: true },
+    cutoff,
+    ...overrides,
+  }
+}
+const row = (id: string) => ({ id, data: {} })
+
+describe('runTableUpdate', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockGetTableById.mockResolvedValue(table)
+    mockGetJobProgress.mockResolvedValue(0)
+    mockUpdateJobProgress.mockResolvedValue(true)
+    mockMarkJobReady.mockResolvedValue(true)
+    mockMarkJobFailed.mockResolvedValue(undefined)
+    mockUpdatePageByIds.mockImplementation((_t, _w, ids: string[]) => Promise.resolve(ids.length))
+    mockBuildFilterClause.mockReturnValue({})
+    mockValidateRowSize.mockReturnValue({ valid: true, errors: [] })
+    mockCoerceRowToSchema.mockReturnValue({ valid: true, errors: [] })
+  })
+
+  it('updates every matching page then marks the job ready', async () => {
+    mockSelectRowDataPage
+      .mockResolvedValueOnce([row('a'), row('b')])
+      .mockResolvedValueOnce([row('c')])
+      .mockResolvedValueOnce([])
+
+    await runTableUpdate(basePayload())
+
+    expect(mockUpdatePageByIds).toHaveBeenNthCalledWith(
+      1,
+      'tbl_1',
+      'ws_1',
+      ['a', 'b'],
+      expect.any(String)
+    )
+    expect(mockUpdatePageByIds).toHaveBeenNthCalledWith(
+      2,
+      'tbl_1',
+      'ws_1',
+      ['c'],
+      expect.any(String)
+    )
+    expect(mockMarkJobReady).toHaveBeenCalledWith('tbl_1', 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'update', status: 'ready', progress: 3 })
+    )
+  })
+
+  it('fails (rethrows) when a merged row is invalid, without writing that page', async () => {
+    mockSelectRowDataPage.mockResolvedValueOnce([row('a')])
+    mockValidateRowSize.mockReturnValueOnce({ valid: false, errors: ['row too large'] })
+
+    await expect(runTableUpdate(basePayload())).rejects.toThrow(/Row a: row too large/)
+    expect(mockUpdatePageByIds).not.toHaveBeenCalled()
+    expect(mockMarkJobFailed).not.toHaveBeenCalled() // caller decides via markTableUpdateFailed
+  })
+
+  it('stops without marking ready when the ownership gate is lost', async () => {
+    mockSelectRowDataPage.mockResolvedValue([row('a'), row('b')])
+    mockUpdateJobProgress.mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+
+    await runTableUpdate(basePayload())
+
+    expect(mockUpdatePageByIds).toHaveBeenCalledTimes(1)
+    expect(mockMarkJobReady).not.toHaveBeenCalled()
+  })
+
+  it('rethrows the root cause so the clean message survives serialization', async () => {
+    const cause = new Error('canceling statement due to statement timeout')
+    mockSelectRowDataPage.mockRejectedValue(new Error('Failed query: update ...', { cause }))
+
+    await expect(runTableUpdate(basePayload())).rejects.toThrow(
+      'canceling statement due to statement timeout'
+    )
+    expect(mockMarkJobFailed).not.toHaveBeenCalled()
+  })
+
+  it('resumes cumulative progress on retry instead of resetting to zero', async () => {
+    mockGetJobProgress.mockResolvedValue(7)
+    mockSelectRowDataPage.mockResolvedValueOnce([row('a'), row('b')]).mockResolvedValueOnce([])
+
+    await runTableUpdate(basePayload())
+
+    expect(mockUpdateJobProgress).toHaveBeenNthCalledWith(1, 'tbl_1', 7, 'job_1')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready', progress: 9 })
+    )
+  })
+
+  it('stops at the seed read when the job is no longer owned', async () => {
+    mockGetJobProgress.mockResolvedValue(null)
+
+    await expect(runTableUpdate(basePayload())).resolves.toBeUndefined()
+    expect(mockSelectRowDataPage).not.toHaveBeenCalled()
+    expect(mockUpdatePageByIds).not.toHaveBeenCalled()
+  })
+
+  it('stops once maxRows is reached and never over-fetches a page', async () => {
+    // budget 3 with page size 2: first page fills 2, second page is capped to the remaining 1.
+    mockSelectRowDataPage
+      .mockResolvedValueOnce([row('a'), row('b')])
+      .mockResolvedValueOnce([row('c')])
+
+    await runTableUpdate(basePayload({ maxRows: 3 }))
+
+    expect(mockSelectRowDataPage).toHaveBeenCalledTimes(2)
+    expect(mockSelectRowDataPage.mock.calls[0][0]).toMatchObject({ limit: 2 })
+    expect(mockSelectRowDataPage.mock.calls[1][0]).toMatchObject({ limit: 1 })
+    expect(mockUpdatePageByIds).toHaveBeenCalledTimes(2)
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'ready', progress: 3 })
+    )
+  })
+
+  it('passes the cutoff and filter clause through to the page query', async () => {
+    mockSelectRowDataPage.mockResolvedValueOnce([])
+
+    await runTableUpdate(basePayload())
+
+    expect(mockBuildFilterClause).toHaveBeenCalledWith(
+      { status: 'old' },
+      'user_table_rows',
+      table.schema.columns
+    )
+    expect(mockSelectRowDataPage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        cutoff,
+        filterClause: {},
+        limit: 2,
+        // Already-patched rows are excluded so a retry doesn't re-walk/double-count.
+        excludeIfPatched: JSON.stringify({ flag: true }),
+      })
+    )
+  })
+})
+
+describe('markTableUpdateFailed', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockMarkJobFailed.mockResolvedValue(undefined)
+  })
+
+  it('marks the job failed and emits the failed event', async () => {
+    await markTableUpdateFailed('tbl_1', 'job_1', new Error('boom'))
+
+    expect(mockMarkJobFailed).toHaveBeenCalledWith('tbl_1', 'job_1', 'boom')
+    expect(mockAppendTableEvent).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: 'job', type: 'update', status: 'failed', error: 'boom' })
+    )
+  })
+})

apps/sim/lib/table/update-runner.ts

@@ -0,0 +1,196 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { truncate } from '@sim/utils/string'
+import type { Filter, RowData } from '@/lib/table'
+import { TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from '@/lib/table/constants'
+import { appendTableEvent } from '@/lib/table/events'
+import {
+  getJobProgress,
+  markJobFailed,
+  markJobReady,
+  updateJobProgress,
+} from '@/lib/table/jobs/service'
+import { selectRowDataPage, updatePageByIds } from '@/lib/table/rows/ordering'
+import { getTableById } from '@/lib/table/service'
+import { buildFilterClause } from '@/lib/table/sql'
+import { coerceRowToSchema, coerceRowValues, validateRowSize } from '@/lib/table/validation'
+
+const logger = createLogger('TableUpdateRunner')
+
+/** Emit a progress event / heartbeat at most every this many rows. */
+const PROGRESS_INTERVAL_ROWS = 5000
+
+/**
+ * Thrown when this worker discovers it no longer owns the table's job (canceled, or the
+ * stale-job janitor marked it failed and a newer job took over). The worker stops updating.
+ */
+class JobSupersededError extends Error {}
+
+export interface TableUpdatePayload {
+  jobId: string
+  tableId: string
+  workspaceId: string
+  /** Rows matching this filter get the patch. */
+  filter: Filter
+  /** Column-id-keyed partial patch merged into every matched row. */
+  data: RowData
+  /** Only rows created at/before this instant are patched, so mid-job inserts are spared. */
+  cutoff: Date
+  /** Stop after updating this many rows (an explicit caller-supplied limit). Omitted = every match. */
+  maxRows?: number
+}
+
+/**
+ * Background worker for large filtered row updates (trigger.dev task, or detached on the web
+ * container when trigger.dev is disabled — see the update dispatch in the user_table tool).
+ * Applies the same `data` patch (JSONB merge) to every row matching `filter` with
+ * `created_at <= cutoff`, in keyset-paginated pages. Each page validates the merged result per
+ * row, then commits in batches — **best-effort, not atomic**: committed pages persist even if a
+ * later page fails validation (unlike the inline `updateRowsByFilter`, which pre-validates all
+ * rows in one transaction). Reads are not masked: updated rows still exist, so mid-job reads are
+ * eventually consistent. Ownership-gated per page so a cancel/supersede stops within one page.
+ *
+ * Unlike the inline path, the worker does NOT fire per-row table triggers or auto-recompute
+ * workflow/enrichment columns — that would be a runaway cascade across thousands of rows. Run
+ * the affected columns explicitly afterward if downstream recompute is needed.
+ *
+ * Unexpected errors are rethrown for the caller's retry machinery; the caller marks the job
+ * failed via `markTableUpdateFailed`. A superseded run returns quietly.
+ */
+export async function runTableUpdate(payload: TableUpdatePayload): Promise<void> {
+  const { jobId, tableId, workspaceId, filter, data, cutoff, maxRows } = payload
+  const requestId = generateId().slice(0, 8)
+  const budget = maxRows ?? Number.POSITIVE_INFINITY
+
+  try {
+    const table = await getTableById(tableId, { includeArchived: true })
+    if (!table) throw new Error(`Update target table ${tableId} not found`)
+
+    const filterClause = buildFilterClause(filter, USER_TABLE_ROWS_SQL_NAME, table.schema.columns)
+    if (!filterClause) throw new Error('Filter is required for bulk update')
+
+    // Coerce the patch once to the schema's types — the merged validation below and the persisted
+    // JSONB merge both use this normalized copy.
+    coerceRowValues(data, table.schema)
+    const patchJson = JSON.stringify(data)
+
+    // Resume the persisted count: a retried attempt's earlier pages are already committed, so
+    // starting at zero would overwrite cumulative progress. Doubles as the initial ownership gate.
+    const resumed = await getJobProgress(tableId, jobId)
+    if (resumed === null) throw new JobSupersededError()
+
+    let processed = resumed
+    let lastReported = resumed
+    let afterId: string | undefined
+
+    while (processed < budget) {
+      const owns = await updateJobProgress(tableId, processed, jobId)
+      if (!owns) throw new JobSupersededError()
+
+      const page = await selectRowDataPage({
+        tableId,
+        workspaceId,
+        cutoff,
+        filterClause,
+        afterId,
+        limit: Math.min(TABLE_LIMITS.DELETE_PAGE_SIZE, budget - processed),
+        // Skip rows already carrying the patch so a retried run resumes without re-walking /
+        // double-counting the rows an earlier attempt updated (updated rows still exist and may
+        // still match the filter, unlike deletes).
+        excludeIfPatched: patchJson,
+      })
+      if (page.length === 0) break
+      afterId = page[page.length - 1].id
+
+      // Validate each merged result before writing the page — a row that would overflow the size
+      // cap or violate the schema fails the job (earlier pages stay applied; best-effort).
+      for (const row of page) {
+        const merged = { ...row.data, ...data }
+        const sizeValidation = validateRowSize(merged)
+        if (!sizeValidation.valid) {
+          throw new Error(`Row ${row.id}: ${sizeValidation.errors.join(', ')}`)
+        }
+        const schemaValidation = coerceRowToSchema(merged, table.schema)
+        if (!schemaValidation.valid) {
+          throw new Error(`Row ${row.id}: ${schemaValidation.errors.join(', ')}`)
+        }
+      }
+
+      processed += await updatePageByIds(
+        tableId,
+        workspaceId,
+        page.map((r) => r.id),
+        patchJson
+      )
+
+      if (
+        processed - lastReported >= PROGRESS_INTERVAL_ROWS ||
+        (lastReported === 0 && processed > 0)
+      ) {
+        lastReported = processed
+        void appendTableEvent({
+          kind: 'job',
+          type: 'update',
+          tableId,
+          jobId,
+          status: 'running',
+          progress: processed,
+        })
+      }
+    }
+
+    await updateJobProgress(tableId, processed, jobId)
+    const becameReady = await markJobReady(tableId, jobId)
+    if (becameReady) {
+      void appendTableEvent({
+        kind: 'job',
+        type: 'update',
+        tableId,
+        jobId,
+        status: 'ready',
+        progress: processed,
+      })
+      logger.info(`[${requestId}] Update complete`, { tableId, rows: processed })
+    } else {
+      logger.info(
+        `[${requestId}] Update finished but no longer owns the run (canceled/superseded)`,
+        {
+          tableId,
+          jobId,
+        }
+      )
+    }
+  } catch (err) {
+    if (err instanceof JobSupersededError) {
+      logger.info(`[${requestId}] Update superseded by a newer run; stopping`, { tableId, jobId })
+      return
+    }
+    const cause = toError(err).cause
+    const error = cause ? toError(cause) : toError(err)
+    logger.error(`[${requestId}] Update failed for table ${tableId}:`, error)
+    throw error
+  }
+}
+
+/**
+ * Marks the update job failed and emits the failed SSE event. Called once the caller gives up on
+ * the run (trigger.dev `onFailure` after retries, or the detached fallback). Scoped to jobId — a
+ * no-op if a newer job has taken over.
+ */
+export async function markTableUpdateFailed(
+  tableId: string,
+  jobId: string,
+  error: unknown
+): Promise<void> {
+  const message = truncate(getErrorMessage(toError(error).cause ?? error, 'Update failed'), 500)
+  await markJobFailed(tableId, jobId, message).catch(() => {})
+  void appendTableEvent({
+    kind: 'job',
+    type: 'update',
+    tableId,
+    jobId,
+    status: 'failed',
+    error: message,
+  })
+}

apps/sim/lib/table/validation.ts

@@ -0,0 +1,693 @@
+/**
+ * Validation utilities for table schemas and row data.
+ */
+
+import { db } from '@sim/db'
+import { userTableRows } from '@sim/db/schema'
+import { and, eq, or, type SQL, sql } from 'drizzle-orm'
+import { NextResponse } from 'next/server'
+import { getColumnId } from '@/lib/table/column-keys'
+import { COLUMN_TYPES, getMaxRowSizeBytes, NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants'
+import { normalizeDateCellValue } from '@/lib/table/dates'
+import { withSeqscanOff } from '@/lib/table/planner'
+import type {
+  ColumnDefinition,
+  JsonValue,
+  RowData,
+  TableSchema,
+  ValidationResult,
+} from '@/lib/table/types'
+
+export type { ColumnDefinition, TableSchema, ValidationResult }
+
+type ValidationSuccess = { valid: true }
+type ValidationFailure = { valid: false; response: NextResponse }
+
+/** Options for validating a single row. */
+export interface ValidateRowOptions {
+  rowData: RowData
+  schema: TableSchema
+  tableId: string
+  excludeRowId?: string
+  checkUnique?: boolean
+}
+
+/** Error information for a single row in batch validation. */
+interface BatchRowError {
+  row: number
+  errors: string[]
+}
+
+/** Options for validating multiple rows in batch. */
+export interface ValidateBatchRowsOptions {
+  rows: RowData[]
+  schema: TableSchema
+  tableId: string
+  checkUnique?: boolean
+}
+
+/**
+ * Validates a single row (size, schema, unique constraints) and returns a formatted response on failure.
+ * Uses optimized database queries for unique constraint checks to avoid loading all rows into memory.
+ */
+export async function validateRowData(
+  options: ValidateRowOptions
+): Promise<ValidationSuccess | ValidationFailure> {
+  const { rowData, schema, tableId, excludeRowId, checkUnique = true } = options
+
+  const sizeValidation = validateRowSize(rowData)
+  if (!sizeValidation.valid) {
+    return {
+      valid: false,
+      response: NextResponse.json(
+        { error: 'Invalid row data', details: sizeValidation.errors },
+        { status: 400 }
+      ),
+    }
+  }
+
+  const schemaValidation = coerceRowToSchema(rowData, schema)
+  if (!schemaValidation.valid) {
+    return {
+      valid: false,
+      response: NextResponse.json(
+        { error: 'Row data does not match schema', details: schemaValidation.errors },
+        { status: 400 }
+      ),
+    }
+  }
+
+  if (checkUnique) {
+    // Use optimized database query instead of loading all rows
+    const uniqueValidation = await checkUniqueConstraintsDb(tableId, rowData, schema, excludeRowId)
+
+    if (!uniqueValidation.valid) {
+      return {
+        valid: false,
+        response: NextResponse.json(
+          { error: 'Unique constraint violation', details: uniqueValidation.errors },
+          { status: 400 }
+        ),
+      }
+    }
+  }
+
+  return { valid: true }
+}
+
+/**
+ * Validates multiple rows for batch insert (size, schema, unique constraints including within batch).
+ * Uses optimized database queries for unique constraint checks to avoid loading all rows into memory.
+ */
+export async function validateBatchRows(
+  options: ValidateBatchRowsOptions
+): Promise<ValidationSuccess | ValidationFailure> {
+  const { rows, schema, tableId, checkUnique = true } = options
+  const errors: BatchRowError[] = []
+
+  for (let i = 0; i < rows.length; i++) {
+    const rowData = rows[i]
+
+    const sizeValidation = validateRowSize(rowData)
+    if (!sizeValidation.valid) {
+      errors.push({ row: i, errors: sizeValidation.errors })
+      continue
+    }
+
+    const schemaValidation = coerceRowToSchema(rowData, schema)
+    if (!schemaValidation.valid) {
+      errors.push({ row: i, errors: schemaValidation.errors })
+    }
+  }
+
+  if (errors.length > 0) {
+    return {
+      valid: false,
+      response: NextResponse.json(
+        { error: 'Validation failed for some rows', details: errors },
+        { status: 400 }
+      ),
+    }
+  }
+
+  if (checkUnique) {
+    const uniqueColumns = getUniqueColumns(schema)
+    if (uniqueColumns.length > 0) {
+      // Use optimized batch unique constraint check
+      const uniqueResult = await checkBatchUniqueConstraintsDb(tableId, rows, schema)
+
+      if (!uniqueResult.valid) {
+        return {
+          valid: false,
+          response: NextResponse.json(
+            { error: 'Unique constraint violations in batch', details: uniqueResult.errors },
+            { status: 400 }
+          ),
+        }
+      }
+    }
+  }
+
+  return { valid: true }
+}
+
+/** Validates table name format and length. */
+export function validateTableName(name: string): ValidationResult {
+  const errors: string[] = []
+
+  if (!name || typeof name !== 'string') {
+    errors.push('Table name is required')
+    return { valid: false, errors }
+  }
+
+  if (name.length > TABLE_LIMITS.MAX_TABLE_NAME_LENGTH) {
+    errors.push(
+      `Table name exceeds maximum length (${TABLE_LIMITS.MAX_TABLE_NAME_LENGTH} characters)`
+    )
+  }
+
+  if (!NAME_PATTERN.test(name)) {
+    errors.push(
+      'Table name must start with letter or underscore, followed by alphanumeric or underscore'
+    )
+  }
+
+  return { valid: errors.length === 0, errors }
+}
+
+/** Validates table schema structure and column definitions. */
+export function validateTableSchema(schema: TableSchema): ValidationResult {
+  const errors: string[] = []
+
+  if (!schema || typeof schema !== 'object') {
+    errors.push('Schema is required')
+    return { valid: false, errors }
+  }
+
+  if (!Array.isArray(schema.columns)) {
+    errors.push('Schema must have columns array')
+    return { valid: false, errors }
+  }
+
+  if (schema.columns.length === 0) {
+    errors.push('Schema must have at least one column')
+  }
+
+  if (schema.columns.length > TABLE_LIMITS.MAX_COLUMNS_PER_TABLE) {
+    errors.push(`Schema exceeds maximum columns (${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE})`)
+  }
+
+  for (const column of schema.columns) {
+    const columnResult = validateColumnDefinition(column)
+    errors.push(...columnResult.errors)
+  }
+
+  const columnNames = schema.columns.map((c) => c.name.toLowerCase())
+  const uniqueNames = new Set(columnNames)
+  if (uniqueNames.size !== columnNames.length) {
+    errors.push('Duplicate column names found')
+  }
+
+  return { valid: errors.length === 0, errors }
+}
+
+/** Validates row data matches schema column types and required fields. */
+export function validateRowAgainstSchema(data: RowData, schema: TableSchema): ValidationResult {
+  const errors: string[] = []
+
+  for (const column of schema.columns) {
+    const value = data[getColumnId(column)]
+
+    if (column.required && (value === undefined || value === null)) {
+      errors.push(`Missing required field: ${column.name}`)
+      continue
+    }
+
+    if (value === null || value === undefined) continue
+
+    switch (column.type) {
+      case 'string':
+        if (typeof value !== 'string') {
+          errors.push(`${column.name} must be string, got ${typeof value}`)
+        }
+        break
+      case 'number':
+        if (typeof value !== 'number' || Number.isNaN(value)) {
+          errors.push(`${column.name} must be number`)
+        }
+        break
+      case 'boolean':
+        if (typeof value !== 'boolean') {
+          errors.push(`${column.name} must be boolean`)
+        }
+        break
+      case 'date':
+        if (
+          !(value instanceof Date) &&
+          (typeof value !== 'string' || Number.isNaN(Date.parse(value)))
+        ) {
+          errors.push(`${column.name} must be valid date`)
+        }
+        break
+      case 'json':
+        try {
+          JSON.stringify(value)
+        } catch {
+          errors.push(`${column.name} must be valid JSON`)
+        }
+        break
+    }
+  }
+
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Attempts to coerce a non-null value to a column's declared type. Returns the
+ * coerced value when the value already matches or can be converted without
+ * ambiguity (e.g. the string `"1999"` to the number `1999`), and `ok: false`
+ * when no safe conversion exists.
+ */
+function coerceValueToColumnType(
+  value: JsonValue,
+  type: ColumnDefinition['type']
+): { ok: true; value: JsonValue } | { ok: false } {
+  switch (type) {
+    case 'string':
+      if (typeof value === 'string') return { ok: true, value }
+      if (typeof value === 'number' || typeof value === 'boolean') {
+        return { ok: true, value: String(value) }
+      }
+      return { ok: false }
+    case 'number':
+      if (typeof value === 'number') {
+        return Number.isFinite(value) ? { ok: true, value } : { ok: false }
+      }
+      if (typeof value === 'string' && value.trim() !== '') {
+        const parsed = Number(value)
+        return Number.isFinite(parsed) ? { ok: true, value: parsed } : { ok: false }
+      }
+      return { ok: false }
+    case 'boolean':
+      if (typeof value === 'boolean') return { ok: true, value }
+      if (typeof value === 'string') {
+        const normalized = value.trim().toLowerCase()
+        if (normalized === 'true') return { ok: true, value: true }
+        if (normalized === 'false') return { ok: true, value: false }
+      }
+      return { ok: false }
+    case 'date': {
+      if (typeof value === 'string') {
+        const normalized = normalizeDateCellValue(value)
+        return normalized === null ? { ok: false } : { ok: true, value: normalized }
+      }
+      // Date instances and epoch numbers may still be out of the representable
+      // range (>±8.64e15ms) — guard `toISOString()`, which throws RangeError on
+      // an Invalid Date, so an over-range value degrades to `{ ok: false }`
+      // rather than crashing the write.
+      const date =
+        value instanceof Date ? value : typeof value === 'number' ? new Date(value) : null
+      if (date && !Number.isNaN(date.getTime())) return { ok: true, value: date.toISOString() }
+      return { ok: false }
+    }
+    default:
+      return { ok: true, value }
+  }
+}
+
+/**
+ * Coerces each present value in `data` toward its column's declared type **in
+ * place**. Values that already match are untouched; unambiguous conversions
+ * (e.g. `"1999"` → `1999`) are applied; values that cannot be coerced are set to
+ * `null` when the column is optional, or left in place when required (so a
+ * subsequent {@link validateRowAgainstSchema} reports them).
+ *
+ * Operates per-present-column, so it is safe on a partial patch (columns absent
+ * from `data` are skipped — it never invents a missing-required-field error).
+ */
+export function coerceRowValues(data: RowData, schema: TableSchema): void {
+  for (const column of schema.columns) {
+    const key = getColumnId(column)
+    const value = data[key]
+    if (value === null || value === undefined) continue
+
+    const coerced = coerceValueToColumnType(value, column.type)
+    if (coerced.ok) {
+      data[key] = coerced.value
+    } else if (!column.required) {
+      data[key] = null
+    }
+  }
+}
+
+/**
+ * Coerces a full row toward its schema **in place** (see {@link coerceRowValues})
+ * then validates the result.
+ *
+ * This is the write-path entry point — callers that persist a complete row use
+ * it instead of {@link validateRowAgainstSchema} so a single off-type field (a
+ * tool returning `"unknown"` for a numeric column, say) nulls that one cell
+ * rather than failing the entire row write. Callers persisting only a partial
+ * patch should use {@link coerceRowValues} on the patch and validate the merged
+ * row separately.
+ */
+export function coerceRowToSchema(data: RowData, schema: TableSchema): ValidationResult {
+  coerceRowValues(data, schema)
+  return validateRowAgainstSchema(data, schema)
+}
+
+/** Validates row data size (UTF-8 bytes of the serialized row) is within limits. */
+export function validateRowSize(data: RowData): ValidationResult {
+  const maxRowSizeBytes = getMaxRowSizeBytes()
+  const size = Buffer.byteLength(JSON.stringify(data))
+  if (size > maxRowSizeBytes) {
+    return {
+      valid: false,
+      errors: [`Row size exceeds limit (${size} bytes > ${maxRowSizeBytes} bytes)`],
+    }
+  }
+  return { valid: true, errors: [] }
+}
+
+/** Returns columns with unique constraint. */
+export function getUniqueColumns(schema: TableSchema): ColumnDefinition[] {
+  return schema.columns.filter((col) => col.unique === true)
+}
+
+/** Validates unique constraints against existing rows (in-memory version for batch validation within a batch). */
+export function validateUniqueConstraints(
+  data: RowData,
+  schema: TableSchema,
+  existingRows: { id: string; data: RowData; position?: number }[],
+  excludeRowId?: string
+): ValidationResult {
+  const errors: string[] = []
+  const uniqueColumns = getUniqueColumns(schema)
+
+  for (const column of uniqueColumns) {
+    const key = getColumnId(column)
+    const value = data[key]
+    if (value === null || value === undefined) continue
+
+    const duplicate = existingRows.find((row) => {
+      if (excludeRowId && row.id === excludeRowId) return false
+
+      const existingValue = row.data[key]
+      if (typeof value === 'string' && typeof existingValue === 'string') {
+        return value.toLowerCase() === existingValue.toLowerCase()
+      }
+      return value === existingValue
+    })
+
+    if (duplicate) {
+      const rowLabel =
+        typeof duplicate.position === 'number' ? `row ${duplicate.position + 1}` : duplicate.id
+      errors.push(
+        `Column "${column.name}" must be unique. Value "${value}" already exists in ${rowLabel}`
+      )
+    }
+  }
+
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Checks unique constraints using targeted database queries.
+ * Only queries for specific conflicting values instead of loading all rows.
+ * This reduces memory usage from O(n) to O(1) where n is the number of rows.
+ *
+ * Pass a transaction as `executor` when running inside an open tx so the
+ * lookup runs on the transaction's connection and observes its uncommitted
+ * writes; otherwise the default `db` connection only observes committed state.
+ */
+export async function checkUniqueConstraintsDb(
+  tableId: string,
+  data: RowData,
+  schema: TableSchema,
+  excludeRowId?: string,
+  executor: UniqueCheckExecutor = db
+): Promise<ValidationResult> {
+  const errors: string[] = []
+  const uniqueColumns = getUniqueColumns(schema)
+
+  if (uniqueColumns.length === 0) {
+    return { valid: true, errors: [] }
+  }
+
+  // Build conditions for each unique column value
+  const conditions: Array<{ column: ColumnDefinition; value: unknown; sql: SQL }> = []
+
+  for (const column of uniqueColumns) {
+    const key = getColumnId(column)
+    if (!NAME_PATTERN.test(key)) {
+      throw new Error(`Invalid column id: ${key}`)
+    }
+
+    const value = data[key]
+    if (value === null || value === undefined) continue
+
+    if (typeof value === 'string') {
+      conditions.push({
+        column,
+        value,
+        sql: sql`lower(${userTableRows.data}->>${sql.raw(`'${key}'`)}) = ${value.toLowerCase()}`,
+      })
+    } else {
+      // For other types, use direct JSONB comparison
+      conditions.push({
+        column,
+        value,
+        sql: sql`(${userTableRows.data}->${sql.raw(`'${key}'`)})::jsonb = ${JSON.stringify(value)}::jsonb`,
+      })
+    }
+  }
+
+  if (conditions.length === 0) {
+    return { valid: true, errors: [] }
+  }
+
+  // Query for each unique column separately to provide specific error messages.
+  // Tenant-bounded: `lower(data->>'col') = ...` is unestimatable, so the planner
+  // otherwise seq-scans the whole shared relation per check — 3.5s on every
+  // insert/edit when the value is unique (no early exit). With an external
+  // transaction the flag is set on it directly — opening our own transaction
+  // inside the caller's would be the nested pool checkout the migration-
+  // hardening work eliminated (self-deadlock under pool exhaustion).
+  const checkConditions = async (ex: UniqueCheckExecutor) => {
+    for (const condition of conditions) {
+      const baseCondition = and(eq(userTableRows.tableId, tableId), condition.sql)
+
+      const whereClause = excludeRowId
+        ? and(baseCondition, sql`${userTableRows.id} != ${excludeRowId}`)
+        : baseCondition
+
+      const conflictingRow = await ex
+        .select({ id: userTableRows.id, position: userTableRows.position })
+        .from(userTableRows)
+        .where(whereClause)
+        .limit(1)
+
+      if (conflictingRow.length > 0) {
+        errors.push(
+          `Column "${condition.column.name}" must be unique. Value "${condition.value}" already exists in row ${conflictingRow[0].position + 1}`
+        )
+      }
+    }
+  }
+
+  if (executor === db) {
+    await withSeqscanOff(async (trx) => checkConditions(trx))
+  } else {
+    await executor.execute(sql`SET LOCAL enable_seqscan = off`)
+    await checkConditions(executor)
+  }
+
+  return { valid: errors.length === 0, errors }
+}
+
+/**
+ * Minimal executor surface needed by unique-constraint checks. Both `db` and a
+ * drizzle transaction (`trx`) satisfy this, letting callers run the lookup
+ * inside an open transaction so it observes uncommitted prior-batch inserts.
+ */
+type UniqueCheckExecutor = Pick<typeof db, 'select' | 'execute'>
+
+/**
+ * Checks unique constraints for a batch of rows using targeted database queries.
+ * Validates both against existing database rows and within the batch itself.
+ *
+ * Pass a transaction as `executor` when running inside an open tx so the lookup
+ * sees rows inserted by earlier batches in the same transaction; otherwise the
+ * default `db` connection only observes committed state.
+ */
+export async function checkBatchUniqueConstraintsDb(
+  tableId: string,
+  rows: RowData[],
+  schema: TableSchema,
+  executor: UniqueCheckExecutor = db
+): Promise<{ valid: boolean; errors: Array<{ row: number; errors: string[] }> }> {
+  const uniqueColumns = getUniqueColumns(schema)
+  const rowErrors: Array<{ row: number; errors: string[] }> = []
+
+  if (uniqueColumns.length === 0) {
+    return { valid: true, errors: [] }
+  }
+
+  // Build a set of all unique values for each column to check against DB.
+  // Keyed by the stable column id (the row-data storage key).
+  const valuesByColumn = new Map<string, { values: Set<string>; column: ColumnDefinition }>()
+
+  for (const column of uniqueColumns) {
+    valuesByColumn.set(getColumnId(column), { values: new Set(), column })
+  }
+
+  // Collect all unique values from the batch and check for duplicates within the batch
+  const batchValueMap = new Map<string, Map<string, number>>() // columnId -> (normalizedValue -> firstRowIndex)
+
+  for (const column of uniqueColumns) {
+    batchValueMap.set(getColumnId(column), new Map())
+  }
+
+  for (let i = 0; i < rows.length; i++) {
+    const rowData = rows[i]
+    const currentRowErrors: string[] = []
+
+    for (const column of uniqueColumns) {
+      const key = getColumnId(column)
+      const value = rowData[key]
+      if (value === null || value === undefined) continue
+
+      const normalizedValue =
+        typeof value === 'string' ? value.toLowerCase() : JSON.stringify(value)
+
+      // Check for duplicate within batch
+      const columnValueMap = batchValueMap.get(key)!
+      if (columnValueMap.has(normalizedValue)) {
+        const firstRowIndex = columnValueMap.get(normalizedValue)!
+        currentRowErrors.push(
+          `Column "${column.name}" must be unique. Value "${value}" duplicates row ${firstRowIndex + 1} in batch`
+        )
+      } else {
+        columnValueMap.set(normalizedValue, i)
+        valuesByColumn.get(key)!.values.add(normalizedValue)
+      }
+    }
+
+    if (currentRowErrors.length > 0) {
+      rowErrors.push({ row: i, errors: currentRowErrors })
+    }
+  }
+
+  // Now check against database for all unique values at once. Tenant-bounded
+  // for the same reason as checkUniqueConstraintsDb: the lower(data->>...)
+  // predicates are unestimatable and otherwise trigger whole-relation seq
+  // scans. With an external transaction the flag is set on it directly (SET
+  // LOCAL dies at its commit; it only penalizes plan shape, and the statements
+  // that follow in those transactions are tenant-scoped writes).
+  const checkColumns = async (ex: UniqueCheckExecutor) => {
+    for (const [columnId, { values, column }] of valuesByColumn) {
+      if (values.size === 0) continue
+
+      if (!NAME_PATTERN.test(columnId)) {
+        throw new Error(`Invalid column id: ${columnId}`)
+      }
+
+      const valueArray = Array.from(values)
+      const valueConditions = valueArray.map((normalizedValue) => {
+        // Check if the original values are strings (normalized values for strings are lowercase)
+        // We need to determine the type from the column definition or the first row that has this value
+        const isStringColumn = column.type === 'string'
+
+        if (isStringColumn) {
+          return sql`lower(${userTableRows.data}->>${sql.raw(`'${columnId}'`)}) = ${normalizedValue}`
+        }
+        return sql`(${userTableRows.data}->${sql.raw(`'${columnId}'`)})::jsonb = ${normalizedValue}::jsonb`
+      })
+
+      const conflictingRows = await ex
+        .select({
+          id: userTableRows.id,
+          data: userTableRows.data,
+          position: userTableRows.position,
+        })
+        .from(userTableRows)
+        .where(and(eq(userTableRows.tableId, tableId), or(...valueConditions)))
+        .limit(valueArray.length) // We only need up to one conflict per value
+
+      // Map conflicts back to batch rows
+      for (const conflict of conflictingRows) {
+        const conflictData = conflict.data as RowData
+        const conflictValue = conflictData[columnId]
+        const normalizedConflictValue =
+          typeof conflictValue === 'string'
+            ? conflictValue.toLowerCase()
+            : JSON.stringify(conflictValue)
+
+        // Find which batch rows have this conflicting value
+        for (let i = 0; i < rows.length; i++) {
+          const rowValue = rows[i][columnId]
+          if (rowValue === null || rowValue === undefined) continue
+
+          const normalizedRowValue =
+            typeof rowValue === 'string' ? rowValue.toLowerCase() : JSON.stringify(rowValue)
+
+          if (normalizedRowValue === normalizedConflictValue) {
+            // Check if this row already has errors for this column
+            let rowError = rowErrors.find((e) => e.row === i)
+            if (!rowError) {
+              rowError = { row: i, errors: [] }
+              rowErrors.push(rowError)
+            }
+
+            const errorMsg = `Column "${column.name}" must be unique. Value "${rowValue}" already exists in row ${conflict.position + 1}`
+            if (!rowError.errors.includes(errorMsg)) {
+              rowError.errors.push(errorMsg)
+            }
+          }
+        }
+      }
+    }
+  }
+
+  if (executor === db) {
+    await withSeqscanOff(async (trx) => checkColumns(trx))
+  } else {
+    await executor.execute(sql`SET LOCAL enable_seqscan = off`)
+    await checkColumns(executor)
+  }
+
+  // Sort errors by row index
+  rowErrors.sort((a, b) => a.row - b.row)
+
+  return { valid: rowErrors.length === 0, errors: rowErrors }
+}
+
+/** Validates column definition format and type. */
+export function validateColumnDefinition(column: ColumnDefinition): ValidationResult {
+  const errors: string[] = []
+
+  if (!column.name || typeof column.name !== 'string') {
+    errors.push('Column name is required')
+    return { valid: false, errors }
+  }
+
+  if (column.name.length > TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) {
+    errors.push(
+      `Column name "${column.name}" exceeds maximum length (${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters)`
+    )
+  }
+
+  if (!NAME_PATTERN.test(column.name)) {
+    errors.push(
+      `Column name "${column.name}" must start with letter or underscore, followed by alphanumeric or underscore`
+    )
+  }
+
+  if (!COLUMN_TYPES.includes(column.type)) {
+    errors.push(
+      `Column "${column.name}" has invalid type "${column.type}". Valid types: ${COLUMN_TYPES.join(', ')}`
+    )
+  }
+
+  return { valid: errors.length === 0, errors }
+}

apps/sim/lib/table/workflow-columns.test.ts

@@ -0,0 +1,95 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import type {
+  RowExecutionMetadata,
+  TableDefinition,
+  TableRow,
+  WorkflowGroup,
+} from '@/lib/table/types'
+import { pickNextEligibleGroupForRow } from '@/lib/table/workflow-columns'
+
+function makeGroup(overrides: Partial<WorkflowGroup> & { id: string }): WorkflowGroup {
+  return {
+    workflowId: `wf-${overrides.id}`,
+    outputs: [{ blockId: 'b1', path: 'out', columnName: `${overrides.id}_out` }],
+    ...overrides,
+  }
+}
+
+function makeTable(groups: WorkflowGroup[]): TableDefinition {
+  return {
+    id: 'tbl1',
+    name: 'T',
+    schema: { columns: [], workflowGroups: groups },
+    rowCount: 1,
+    maxRows: 1000,
+    workspaceId: 'ws1',
+    createdBy: 'u1',
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  }
+}
+
+function makeRow(
+  executions: Record<string, RowExecutionMetadata>,
+  data: Record<string, unknown> = {}
+): TableRow {
+  return {
+    id: 'row1',
+    data: data as TableRow['data'],
+    executions,
+    position: 0,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  }
+}
+
+/** The dispatcher's "queued marker" pre-stamp: pending with no executionId. */
+function queuedMarker(workflowId: string): RowExecutionMetadata {
+  return { status: 'pending', executionId: null, jobId: null, workflowId, error: null }
+}
+
+describe('pickNextEligibleGroupForRow — queued-marker handoff', () => {
+  it('runs an autoRun:false group that carries a queued marker (explicit request)', () => {
+    const group = makeGroup({ id: 'g1', autoRun: false })
+    const table = makeTable([group])
+    const row = makeRow({ g1: queuedMarker('wf-g1') })
+
+    expect(pickNextEligibleGroupForRow(table, row)?.id).toBe('g1')
+  })
+
+  it('does NOT run an autoRun:false group with no marker (auto-cascade respects autoRun)', () => {
+    const group = makeGroup({ id: 'g1', autoRun: false })
+    const table = makeTable([group])
+    const row = makeRow({})
+
+    expect(pickNextEligibleGroupForRow(table, row)).toBeNull()
+  })
+
+  it('does NOT run an autoRun:true marker whose deps are unmet (no spin)', () => {
+    const group = makeGroup({ id: 'g1', autoRun: true, dependencies: { columns: ['need'] } })
+    const table = makeTable([group])
+    // marker present, but the dep column is empty → deps-unmet
+    const row = makeRow({ g1: queuedMarker('wf-g1') }, { need: '' })
+
+    expect(pickNextEligibleGroupForRow(table, row)).toBeNull()
+  })
+
+  it('still runs a normal autoRun:true group whose deps are satisfied (no marker)', () => {
+    const group = makeGroup({ id: 'g1', autoRun: true })
+    const table = makeTable([group])
+    const row = makeRow({})
+
+    expect(pickNextEligibleGroupForRow(table, row)?.id).toBe('g1')
+  })
+
+  it('skips excludeGroupId so the just-finished group does not self-retrigger', () => {
+    const group = makeGroup({ id: 'g1', autoRun: true })
+    const table = makeTable([group])
+    const row = makeRow({})
+
+    expect(pickNextEligibleGroupForRow(table, row, 'g1')).toBeNull()
+  })
+})

apps/sim/lib/table/workflow-groups/service.ts

@@ -0,0 +1,964 @@
+/**
+ * Workflow-group operations on user tables.
+ *
+ * Extracted from the table service: add/update/delete workflow groups and their
+ * output columns, plus stale-output pruning after a workflow deploy. These ops
+ * mutate `schema.workflowGroups` (and the bound output columns + row data) under
+ * the per-table advisory lock from `withLockedTable`.
+ */
+
+import { db } from '@sim/db'
+import { userTableDefinitions } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { and, eq, isNull, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import {
+  columnMatchesRef,
+  generateColumnId,
+  getColumnId,
+  remapGroupColumnRefs,
+} from '@/lib/table/column-keys'
+import { NAME_PATTERN, TABLE_LIMITS } from '@/lib/table/constants'
+import { stripGroupExecutions } from '@/lib/table/rows/executions'
+import { getTableById, withLockedTable } from '@/lib/table/service'
+import { setTableTxTimeouts } from '@/lib/table/tx'
+import type {
+  AddWorkflowGroupData,
+  ColumnDefinition,
+  DeleteWorkflowGroupData,
+  TableDefinition,
+  TableMetadata,
+  TableSchema,
+  UpdateWorkflowGroupData,
+  WorkflowGroup,
+  WorkflowGroupOutput,
+} from '@/lib/table/types'
+import { assertValidSchema, runWorkflowColumn, stripGroupDeps } from '@/lib/table/workflow-columns'
+
+const logger = createLogger('TableWorkflowGroupsService')
+/**
+ * Drops references to deleted blocks from every workflow group on every table
+ * that targets the just-deployed workflow. Called from the workflow deploy
+ * orchestrator after the new deployment commits, so the table UI never holds
+ * stale `{blockId, path}` entries for blocks the user removed.
+ *
+ * - Filters `outputs[]` per group. If every output would be filtered out, the
+ *   group is left untouched and a warning is logged — the user must
+ *   reconfigure it manually.
+ * - Scoped to the workflow's workspace.
+ * - Idempotent: running twice with the same `validBlockIds` is a no-op on the
+ *   second pass. Existing row data is left alone.
+ */
+export async function pruneStaleWorkflowGroupOutputs({
+  workflowId,
+  workspaceId,
+  validBlockIds,
+  requestId,
+  tx,
+}: {
+  workflowId: string
+  workspaceId: string
+  validBlockIds: Set<string>
+  requestId: string
+  tx?: DbOrTx
+}): Promise<void> {
+  const executor = tx ?? db
+  const tables = await executor
+    .select({
+      id: userTableDefinitions.id,
+      schema: userTableDefinitions.schema,
+    })
+    .from(userTableDefinitions)
+    .where(
+      and(
+        eq(userTableDefinitions.workspaceId, workspaceId),
+        isNull(userTableDefinitions.archivedAt)
+      )
+    )
+
+  for (const t of tables) {
+    const schema = t.schema as TableSchema
+    const groups = schema.workflowGroups ?? []
+    if (groups.length === 0) continue
+
+    let mutated = false
+    const nextGroups = groups.map((group) => {
+      if (group.workflowId !== workflowId) return group
+      const filtered = group.outputs.filter((o) => validBlockIds.has(o.blockId))
+      if (filtered.length === group.outputs.length) return group
+      if (filtered.length === 0) {
+        logger.warn(
+          `[${requestId}] All outputs for workflow group "${group.name ?? group.id}" in table ${t.id} reference deleted blocks; leaving group intact for user reconfiguration.`
+        )
+        return group
+      }
+      mutated = true
+      return { ...group, outputs: filtered }
+    })
+
+    if (!mutated) continue
+
+    await executor
+      .update(userTableDefinitions)
+      .set({
+        schema: { ...schema, workflowGroups: nextGroups },
+        updatedAt: new Date(),
+      })
+      .where(eq(userTableDefinitions.id, t.id))
+
+    logger.info(`[${requestId}] Pruned stale workflow=${workflowId} block refs from table ${t.id}`)
+  }
+}
+
+/**
+ * Atomically inserts a workflow group plus its output columns into a table's
+ * schema. Both arrays update in one DB write so the schema is never observed
+ * mid-mutation (e.g. columns referencing a group that doesn't yet exist).
+ */
+export async function addWorkflowGroup(
+  data: AddWorkflowGroupData,
+  requestId: string
+): Promise<TableDefinition> {
+  const updatedTable = await withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const groups = schema.workflowGroups ?? []
+    if (groups.some((g) => g.id === data.group.id)) {
+      throw new Error(`Workflow group "${data.group.id}" already exists`)
+    }
+
+    const existingNames = new Set(schema.columns.map((c) => c.name.toLowerCase()))
+    for (const col of data.outputColumns) {
+      if (!NAME_PATTERN.test(col.name)) {
+        throw new Error(
+          `Invalid output column name "${col.name}". Must satisfy ${NAME_PATTERN.source}.`
+        )
+      }
+      if (existingNames.has(col.name.toLowerCase())) {
+        throw new Error(`Column "${col.name}" already exists`)
+      }
+    }
+
+    if (schema.columns.length + data.outputColumns.length > TABLE_LIMITS.MAX_COLUMNS_PER_TABLE) {
+      throw new Error(
+        `Adding ${data.outputColumns.length} columns would exceed the maximum (${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE}).`
+      )
+    }
+
+    // Assign stable ids to the new output columns, then rewrite the group's
+    // column refs from name → id so outputs/deps/inputMappings key on ids —
+    // matching the row-data storage key and surviving future renames.
+    const outputColumns = data.outputColumns.map((col) =>
+      col.id ? col : { ...col, id: generateColumnId() }
+    )
+    const updatedColumns = [...schema.columns, ...outputColumns]
+    const idByName = new Map(updatedColumns.map((c) => [c.name, getColumnId(c)]))
+    const group = remapGroupColumnRefs(data.group, idByName)
+
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: updatedColumns,
+      workflowGroups: [...groups, group],
+    }
+
+    // Keep `metadata.columnOrder` (column ids) in sync — see `addTableColumn`.
+    // New output columns get appended in the order the caller supplied.
+    const existingOrder = table.metadata?.columnOrder
+    let updatedMetadata = table.metadata
+    if (existingOrder && existingOrder.length > 0) {
+      const known = new Set(existingOrder)
+      const append = outputColumns.map(getColumnId).filter((id) => !known.has(id))
+      if (append.length > 0) {
+        updatedMetadata = { ...table.metadata, columnOrder: [...existingOrder, ...append] }
+      }
+    }
+
+    assertValidSchema(updatedSchema, updatedMetadata?.columnOrder)
+
+    const now = new Date()
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    logger.info(
+      `[${requestId}] Added workflow group "${data.group.id}" with ${data.outputColumns.length} output column(s) to table ${data.tableId}`
+    )
+
+    return {
+      ...table,
+      schema: updatedSchema,
+      metadata: updatedMetadata,
+      updatedAt: now,
+    }
+  })
+
+  // Auto-fire existing rows whose deps are already met for the new group.
+  // Fire-and-forget — the dispatcher bounds queue depth (window of 20) and
+  // walks the table in the background. HTTP returns instantly; cells fill
+  // in over the next minutes as the dispatcher walks. Mothership opts out
+  // by setting `autoRun: false`.
+  if (data.autoRun !== false) {
+    void runWorkflowColumn({
+      tableId: updatedTable.id,
+      workspaceId: updatedTable.workspaceId,
+      mode: 'new',
+      isManualRun: false,
+      groupIds: [data.group.id],
+      requestId,
+      triggeredByUserId: data.actorUserId,
+    }).catch((err) => logger.error(`[${requestId}] auto-dispatch (addWorkflowGroup) failed:`, err))
+  }
+
+  return updatedTable
+}
+
+/**
+ * Updates a workflow group: any combination of workflowId, name, dependencies,
+ * outputs[]. Computes added/removed outputs vs current state and inserts /
+ * removes columns transactionally. Removed outputs also clear their key from
+ * every row's `data`.
+ */
+export async function updateWorkflowGroup(
+  data: UpdateWorkflowGroupData,
+  requestId: string
+): Promise<TableDefinition> {
+  const mappingUpdates = data.mappingUpdates ?? []
+
+  // Phase 1 (no lock): when there are mapping updates, load the workflow once to
+  // resolve each remap's new leaf type. Kept OFF the advisory-lock critical
+  // section so concurrent group edits on the same table don't time out waiting
+  // on this DB load. Best-effort — a resolution failure leaves column types
+  // unchanged (workflow deleted, block removed). The result is applied against
+  // the fresh schema under the lock in phase 2.
+  const remapLeafTypeByColumn = new Map<string, ColumnDefinition['type']>()
+  // The workflow id the leaf types above were resolved against. Phase 2 only
+  // applies the resolved types if the group still points at this workflow under
+  // the lock — a concurrent `workflowId` change would make them stale.
+  let resolvedForWorkflowId: string | undefined
+  if (mappingUpdates.length > 0) {
+    try {
+      const preTable = await getTableById(data.tableId)
+      const preGroup = preTable?.schema.workflowGroups?.find((g) => g.id === data.groupId)
+      const targetWorkflowId = data.workflowId ?? preGroup?.workflowId
+      if (targetWorkflowId) {
+        resolvedForWorkflowId = targetWorkflowId
+        const [
+          { loadWorkflowFromNormalizedTables },
+          { flattenWorkflowOutputs },
+          { columnTypeForLeaf },
+        ] = await Promise.all([
+          import('@/lib/workflows/persistence/utils'),
+          import('@/lib/workflows/blocks/flatten-outputs'),
+          import('@/lib/table/column-naming'),
+        ])
+        const normalized = await loadWorkflowFromNormalizedTables(targetWorkflowId)
+        if (normalized) {
+          const blocks = Object.values(normalized.blocks ?? {}).map((b) => ({
+            id: b.id,
+            type: b.type,
+            name: b.name,
+            triggerMode: (b as { triggerMode?: boolean }).triggerMode,
+            subBlocks: b.subBlocks as Record<string, unknown> | undefined,
+          }))
+          const flattened = flattenWorkflowOutputs(blocks, normalized.edges ?? [])
+          const flatByKey = new Map(flattened.map((f) => [`${f.blockId}::${f.path}`, f]))
+          for (const u of mappingUpdates) {
+            const match = flatByKey.get(`${u.blockId}::${u.path}`)
+            if (!match) continue
+            const newType = columnTypeForLeaf(match.leafType)
+            if (newType) remapLeafTypeByColumn.set(u.columnName, newType)
+          }
+        }
+      }
+    } catch (err) {
+      logger.warn(
+        `[${requestId}] Could not resolve new leaf types for remap on group ${data.groupId}; leaving column types unchanged:`,
+        err
+      )
+    }
+  }
+
+  const { updatedTable, added, remappedColumnIds, newOutputs, previousAutoRun } =
+    await withLockedTable(data.tableId, async (table, trx) => {
+      await setTableTxTimeouts(trx, { statementMs: 60_000 })
+
+      const schema = table.schema
+      const groups = schema.workflowGroups ?? []
+      const groupIndex = groups.findIndex((g) => g.id === data.groupId)
+      if (groupIndex === -1) {
+        throw new Error(`Workflow group "${data.groupId}" not found`)
+      }
+      const group = groups[groupIndex]
+
+      // Normalize every caller-supplied column reference to its stable id, so
+      // the diff/splice/clear logic below operates uniformly in id-space (the
+      // row-data storage key). New output columns get ids first; then output
+      // `columnName`, deps, input mappings, and mapping-update targets are
+      // remapped name → id. Callers that already pass ids are unaffected.
+      const newColDefs = (data.newOutputColumns ?? []).map((col) =>
+        col.id ? col : { ...col, id: generateColumnId() }
+      )
+      const idByName = new Map(
+        [...schema.columns, ...newColDefs].map((c) => [c.name, getColumnId(c)])
+      )
+      const remapRef = (ref: string) => idByName.get(ref) ?? ref
+      const outputsInput = data.outputs?.map((o) => ({ ...o, columnName: remapRef(o.columnName) }))
+      const dependenciesInput = data.dependencies
+        ? { columns: data.dependencies.columns?.map(remapRef) }
+        : undefined
+      const inputMappingsInput = data.inputMappings?.map((m) => ({
+        ...m,
+        columnName: remapRef(m.columnName),
+      }))
+      const mappingUpdatesNorm = mappingUpdates.map((u) => ({
+        ...u,
+        columnName: remapRef(u.columnName),
+      }))
+      // Re-key the out-of-lock leaf-type resolution to ids to match.
+      const remapLeafTypeById = new Map<string, ColumnDefinition['type']>()
+      for (const [name, type] of remapLeafTypeByColumn) remapLeafTypeById.set(remapRef(name), type)
+
+      // Apply `mappingUpdates` first: each entry repoints an existing output's
+      // `(blockId, path)` while preserving the column. We patch the **old** view
+      // of outputs so the downstream `(blockId, path)`-keyed diff doesn't see the
+      // swap as a remove+add. The corresponding row data is cleared after the
+      // schema write so stale values from the old source don't linger.
+      const remappedColumnIds = new Set<string>()
+      // Per-column type override (keyed by id) resolved (out-of-lock) from the
+      // new mapping's leaf type. Only populated when a remap actually changes
+      // the column's type against the fresh schema.
+      const remappedColumnTypes = new Map<string, ColumnDefinition['type']>()
+      let oldOutputs = group.outputs
+      if (mappingUpdatesNorm.length > 0) {
+        const updateById = new Map(mappingUpdatesNorm.map((u) => [u.columnName, u]))
+        for (const u of mappingUpdatesNorm) {
+          const exists = oldOutputs.some((o) => o.columnName === u.columnName)
+          if (!exists) {
+            throw new Error(
+              `Mapping update for unknown column "${u.columnName}" (group ${data.groupId}).`
+            )
+          }
+        }
+        oldOutputs = oldOutputs.map((o) => {
+          const u = updateById.get(o.columnName)
+          if (!u) return o
+          remappedColumnIds.add(o.columnName)
+          return { ...o, blockId: u.blockId, path: u.path }
+        })
+
+        // Only apply the out-of-lock leaf-type resolution if the group still
+        // points at the workflow we resolved against. If a concurrent writer
+        // changed `workflowId` between phase 1 and now, those types are stale —
+        // leave column types unchanged (best-effort, same as a resolution
+        // failure) rather than stamping types from the old workflow.
+        const finalWorkflowId = data.workflowId ?? group.workflowId
+        if (remapLeafTypeById.size > 0 && resolvedForWorkflowId !== finalWorkflowId) {
+          logger.warn(
+            `[${requestId}] Workflow group "${data.groupId}" workflowId changed between leaf-type resolution and apply; leaving remapped column types unchanged.`
+          )
+        } else {
+          const colById = new Map(schema.columns.map((c) => [getColumnId(c), c]))
+          for (const u of mappingUpdatesNorm) {
+            const newType = remapLeafTypeById.get(u.columnName)
+            if (!newType) continue
+            const oldType = colById.get(u.columnName)?.type
+            if (newType !== oldType) {
+              remappedColumnTypes.set(u.columnName, newType)
+            }
+          }
+        }
+      }
+
+      // If the caller passed `outputs`, that's the new full set. If only
+      // `mappingUpdates` was sent, the new set is the remapped old set.
+      const newOutputs = outputsInput ?? oldOutputs
+      // Enrichment outputs all share empty `blockId`/`path`, so keying on those
+      // alone collapses every sibling to one entry (dropping columns on diff). Key
+      // on the registry `outputId` when present; fall back to `blockId::path` for
+      // workflow outputs.
+      const oldKey = (o: WorkflowGroupOutput) =>
+        o.outputId ? `out::${o.outputId}` : `${o.blockId}::${o.path}`
+      const oldByKey = new Map(oldOutputs.map((o) => [oldKey(o), o]))
+      const newByKey = new Map(newOutputs.map((o) => [oldKey(o), o]))
+
+      const removed = oldOutputs.filter((o) => !newByKey.has(oldKey(o)))
+      const added = newOutputs.filter((o) => !oldByKey.has(oldKey(o)))
+      const newColById = new Map(newColDefs.map((c) => [getColumnId(c), c]))
+
+      for (const out of added) {
+        if (!newColById.has(out.columnName)) {
+          throw new Error(
+            `Missing column definition for new output "${out.columnName}" (group ${data.groupId}).`
+          )
+        }
+      }
+
+      const removedColumnIds = new Set(removed.map((o) => o.columnName))
+      let nextColumns = schema.columns
+        .filter((c) => !removedColumnIds.has(getColumnId(c)))
+        .map((c) => {
+          const newType = remappedColumnTypes.get(getColumnId(c))
+          return newType ? { ...c, type: newType } : c
+        })
+      if (newColDefs.length > 0) {
+        // Splice the new column defs into the group's contiguous run rather than
+        // appending at the end. The desired in-group order is `newOutputs` (the
+        // sidebar's BFS-of-the-workflow ordering); we walk it, anchor at the first
+        // surviving sibling's index in `nextColumns`, and emit each output's
+        // column def in turn.
+        const groupColIds = new Set(newOutputs.map((o) => o.columnName))
+        const firstGroupIdx = nextColumns.findIndex((c) => groupColIds.has(getColumnId(c)))
+        const anchorIdx = firstGroupIdx === -1 ? nextColumns.length : firstGroupIdx
+        const orderedGroupCols: ColumnDefinition[] = []
+        for (const out of newOutputs) {
+          const fresh = newColById.get(out.columnName)
+          if (fresh) {
+            orderedGroupCols.push(fresh)
+          } else {
+            const existing = nextColumns.find((c) => getColumnId(c) === out.columnName)
+            if (existing) orderedGroupCols.push(existing)
+          }
+        }
+        const remaining = nextColumns.filter((c) => !groupColIds.has(getColumnId(c)))
+        nextColumns = [
+          ...remaining.slice(0, anchorIdx),
+          ...orderedGroupCols,
+          ...remaining.slice(anchorIdx),
+        ]
+      }
+
+      const updatedGroup: WorkflowGroup = {
+        ...group,
+        workflowId: data.workflowId ?? group.workflowId,
+        name: data.name ?? group.name,
+        dependencies: dependenciesInput ?? group.dependencies,
+        outputs: newOutputs,
+        ...(inputMappingsInput !== undefined ? { inputMappings: inputMappingsInput } : {}),
+        ...(data.deploymentMode !== undefined ? { deploymentMode: data.deploymentMode } : {}),
+        ...(data.type !== undefined ? { type: data.type } : {}),
+        ...(data.autoRun !== undefined ? { autoRun: data.autoRun } : {}),
+      }
+      // Removed outputs may be referenced as deps by sibling groups; strip those
+      // refs so we don't leave dangling-column deps that fail schema validation.
+      const nextGroups = groups
+        .map((g, i) => (i === groupIndex ? updatedGroup : g))
+        .map((g) => (g.id === updatedGroup.id ? g : stripGroupDeps(g, removedColumnIds)))
+      const updatedSchema: TableSchema = {
+        ...schema,
+        columns: nextColumns,
+        workflowGroups: nextGroups,
+      }
+
+      // `columnOrder` (column ids) mirrors the schema layout. Drop removed
+      // columns, then splice the new ones in at the same anchor as `nextColumns`
+      // so the table renders them inside the group's contiguous run.
+      let updatedColumnOrder = table.metadata?.columnOrder?.filter(
+        (id) => !removedColumnIds.has(id)
+      )
+      if (updatedColumnOrder && newColDefs.length > 0) {
+        const newColIds = new Set(newColDefs.map(getColumnId))
+        const orderWithoutNew = updatedColumnOrder.filter((id) => !newColIds.has(id))
+        const groupColIds = new Set(newOutputs.map((o) => o.columnName))
+        const orderedGroupIds = newOutputs.map((o) => o.columnName)
+        const firstGroupOrderIdx = orderWithoutNew.findIndex((id) => groupColIds.has(id))
+        const anchorOrderIdx =
+          firstGroupOrderIdx === -1 ? orderWithoutNew.length : firstGroupOrderIdx
+        const remainingOrder = orderWithoutNew.filter((id) => !groupColIds.has(id))
+        updatedColumnOrder = [
+          ...remainingOrder.slice(0, anchorOrderIdx),
+          ...orderedGroupIds,
+          ...remainingOrder.slice(anchorOrderIdx),
+        ]
+      }
+      assertValidSchema(updatedSchema, updatedColumnOrder)
+
+      const updatedMetadata: TableMetadata | null =
+        updatedColumnOrder && table.metadata
+          ? { ...table.metadata, columnOrder: updatedColumnOrder }
+          : table.metadata
+            ? { ...table.metadata }
+            : null
+
+      const now = new Date()
+      await trx
+        .update(userTableDefinitions)
+        .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+        .where(eq(userTableDefinitions.id, data.tableId))
+      for (const id of removedColumnIds) {
+        await trx.execute(
+          sql`UPDATE user_table_rows SET data = data - ${id}::text WHERE table_id = ${data.tableId} AND data ? ${id}::text`
+        )
+      }
+      // Remapped columns: clear stale values in-tx so rows the backfill can't
+      // repopulate (no log, no matching span output) end up empty rather than
+      // retaining the previous mapping's value. The backfill below then writes
+      // the new mapping's value into rows where it can find one.
+      for (const id of remappedColumnIds) {
+        if (removedColumnIds.has(id)) continue
+        await trx.execute(
+          sql`UPDATE user_table_rows SET data = data - ${id}::text WHERE table_id = ${data.tableId} AND data ? ${id}::text`
+        )
+      }
+
+      logger.info(
+        `[${requestId}] Updated workflow group "${data.groupId}" in table ${data.tableId} (added=${added.length}, removed=${removed.length}, remapped=${remappedColumnIds.size})`
+      )
+
+      const updatedTable: TableDefinition = {
+        ...table,
+        schema: updatedSchema,
+        metadata: updatedMetadata,
+        updatedAt: now,
+      }
+      return {
+        updatedTable,
+        added,
+        remappedColumnIds,
+        newOutputs,
+        previousAutoRun: group.autoRun,
+      }
+    })
+
+  // Backfill from saved execution logs so already-completed group runs surface
+  // the schema changes without re-running the workflow. Two passes:
+  //   - added outputs (new columns): never overwrite hand-edited values.
+  //   - remapped outputs (existing column re-pointed): overwrite, since the
+  //     new mapping is the source of truth and the user expects the cell to
+  //     refresh to the new output's value.
+  // Small tables backfill inline-awaited (response returns with consistent
+  // data); large ones run as a background job. A failed backfill is logged
+  // but doesn't fail the request — the schema change has already committed.
+  // Lazy import: backfill-runner closes a cycle back to this module.
+  const { maybeBackfillGroupOutputs } = await import('@/lib/table/backfill-runner')
+  if (added.length > 0) {
+    try {
+      await maybeBackfillGroupOutputs({
+        table: updatedTable,
+        groupId: data.groupId,
+        outputs: added,
+        overwrite: false,
+        requestId,
+        actorUserId: data.actorUserId,
+      })
+    } catch (err) {
+      logger.warn(
+        `[${requestId}] Backfill from execution logs failed for ${data.tableId} group ${data.groupId}:`,
+        err
+      )
+    }
+  }
+  if (remappedColumnIds.size > 0) {
+    const remappedOutputs = newOutputs.filter((o) => remappedColumnIds.has(o.columnName))
+    try {
+      await maybeBackfillGroupOutputs({
+        table: updatedTable,
+        groupId: data.groupId,
+        outputs: remappedOutputs,
+        overwrite: true,
+        requestId,
+        actorUserId: data.actorUserId,
+      })
+    } catch (err) {
+      logger.warn(
+        `[${requestId}] Remap backfill from execution logs failed for ${data.tableId} group ${data.groupId}:`,
+        err
+      )
+    }
+  }
+
+  // autoRun toggled false → true: fire deps-satisfied rows now via the
+  // dispatcher. Mirrors the post-add path so re-enabling auto-fire doesn't
+  // require manual run clicks for rows that are already eligible.
+  if (previousAutoRun === false && data.autoRun === true) {
+    void runWorkflowColumn({
+      tableId: updatedTable.id,
+      workspaceId: updatedTable.workspaceId,
+      mode: 'new',
+      isManualRun: false,
+      groupIds: [data.groupId],
+      requestId,
+      triggeredByUserId: data.actorUserId,
+    }).catch((err) =>
+      logger.error(`[${requestId}] auto-dispatch (updateWorkflowGroup autoRun=true) failed:`, err)
+    )
+  }
+
+  return updatedTable
+}
+
+/**
+ * Adds a single output to an existing workflow group. Mirrors `addTableColumn`
+ * for plain columns: one canonical op, one column created, type inferred from
+ * the workflow's flattened outputs (`leafType` for `(blockId, path)`). The
+ * column is spliced into the group's contiguous run so the table renders the
+ * new output next to its siblings.
+ */
+export async function addWorkflowGroupOutput(
+  data: {
+    tableId: string
+    groupId: string
+    blockId: string
+    path: string
+    /** Optional override; defaults to a slug derived from `path`. */
+    columnName?: string
+    /** The member adding the output — billed/gated for any backfill-triggered re-run. */
+    actorUserId?: string | null
+  },
+  requestId: string
+): Promise<TableDefinition> {
+  // Phase 1 (no lock): load the workflow and resolve the pickable output plus
+  // its execution-order index. This depends only on the workflow graph (which
+  // is stable), so it runs OFF the advisory-lock critical section — holding the
+  // lock during this DB load would make concurrent adders on the same table
+  // time out waiting (the Mothership fan-out this fix targets). Phase 2
+  // re-validates that the group still maps to the same workflow under the lock.
+  const preTable = await getTableById(data.tableId)
+  if (!preTable) throw new Error('Table not found')
+  const preGroup = (preTable.schema.workflowGroups ?? []).find((g) => g.id === data.groupId)
+  if (!preGroup) {
+    throw new Error(`Workflow group "${data.groupId}" not found`)
+  }
+  const workflowId = preGroup.workflowId
+
+  const [
+    { loadWorkflowFromNormalizedTables },
+    { flattenWorkflowOutputs, getBlockExecutionOrder },
+    { columnTypeForLeaf, deriveOutputColumnName },
+  ] = await Promise.all([
+    import('@/lib/workflows/persistence/utils'),
+    import('@/lib/workflows/blocks/flatten-outputs'),
+    import('@/lib/table/column-naming'),
+  ])
+  const normalized = await loadWorkflowFromNormalizedTables(workflowId)
+  if (!normalized) {
+    throw new Error(`Workflow ${workflowId} not found`)
+  }
+  const blocks = Object.values(normalized.blocks ?? {}).map((b) => ({
+    id: b.id,
+    type: b.type,
+    name: b.name,
+    triggerMode: (b as { triggerMode?: boolean }).triggerMode,
+    subBlocks: b.subBlocks as Record<string, unknown> | undefined,
+  }))
+  const flattened = flattenWorkflowOutputs(blocks, normalized.edges ?? [])
+  const match = flattened.find((f) => f.blockId === data.blockId && f.path === data.path)
+  if (!match) {
+    throw new Error(
+      `Output ${data.blockId}::${data.path} is not a valid pickable output on workflow ${workflowId}`
+    )
+  }
+  const newColumnType = columnTypeForLeaf(match.leafType)
+  const distances = getBlockExecutionOrder(blocks, normalized.edges ?? [])
+  const flatIndex = new Map(flattened.map((f, i) => [`${f.blockId}::${f.path}`, i]))
+
+  // Phase 2 (locked): re-read fresh, validate against the current schema, and
+  // write. The critical section holds no I/O — just the in-memory splice + the
+  // schema UPDATE — so concurrent adders queue behind it quickly.
+  const { updatedTable, newOutput } = await withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const groups = schema.workflowGroups ?? []
+    const groupIndex = groups.findIndex((g) => g.id === data.groupId)
+    if (groupIndex === -1) {
+      throw new Error(`Workflow group "${data.groupId}" not found`)
+    }
+    const group = groups[groupIndex]
+    if (group.workflowId !== workflowId) {
+      throw new Error(
+        `Workflow group "${data.groupId}" was remapped to a different workflow concurrently; retry the add.`
+      )
+    }
+
+    if (group.outputs.some((o) => o.blockId === data.blockId && o.path === data.path)) {
+      throw new Error(
+        `Workflow group "${data.groupId}" already has an output at ${data.blockId}::${data.path}`
+      )
+    }
+
+    const taken = new Set(schema.columns.map((c) => c.name))
+    const columnName = data.columnName ?? deriveOutputColumnName(data.path, taken)
+    if (!NAME_PATTERN.test(columnName)) {
+      throw new Error(`Invalid column name "${columnName}". Must satisfy ${NAME_PATTERN.source}.`)
+    }
+    if (taken.has(columnName)) {
+      throw new Error(`Column "${columnName}" already exists`)
+    }
+    if (schema.columns.length + 1 > TABLE_LIMITS.MAX_COLUMNS_PER_TABLE) {
+      throw new Error(
+        `Adding a column would exceed the maximum (${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE}).`
+      )
+    }
+
+    const newColDef: ColumnDefinition = {
+      id: generateColumnId(),
+      name: columnName,
+      type: newColumnType,
+      required: false,
+      unique: false,
+      workflowGroupId: data.groupId,
+    }
+    const newColumnId = getColumnId(newColDef)
+    const newOutput: WorkflowGroupOutput = {
+      blockId: data.blockId,
+      path: data.path,
+      columnName: newColumnId,
+    }
+
+    // Sort all of the group's outputs (existing + new) in workflow execution
+    // order: BFS distance from the start block ASC, with discovery order as
+    // tiebreak. This matches what the column-sidebar does at create time, so
+    // columns from the same workflow always read in the order their blocks run
+    // — regardless of whether they were added at create time or one-by-one.
+    const groupColIdsBefore = new Set(group.outputs.map((o) => o.columnName))
+    const orderKey = (o: { blockId: string; path: string }) => {
+      const d = distances[o.blockId]
+      const dist = d === undefined || d < 0 ? Number.POSITIVE_INFINITY : d
+      const idx = flatIndex.get(`${o.blockId}::${o.path}`) ?? Number.POSITIVE_INFINITY
+      return [dist, idx] as const
+    }
+    const allGroupOutputs = [...group.outputs, newOutput].sort((a, b) => {
+      const [da, ia] = orderKey(a)
+      const [db, ib] = orderKey(b)
+      return da !== db ? da - db : ia - ib
+    })
+    const orderedGroupColIds = allGroupOutputs.map((o) => o.columnName)
+    const updatedGroup: WorkflowGroup = {
+      ...group,
+      outputs: allGroupOutputs,
+    }
+    const nextGroups = groups.map((g, i) => (i === groupIndex ? updatedGroup : g))
+
+    // Splice the new column run into nextColumns: keep the columns outside the
+    // group where they were, replace the group's contiguous run with the
+    // BFS-ordered list. Anchor at the position of the first existing sibling
+    // (or append if the group was empty).
+    const colById = new Map(schema.columns.map((c) => [getColumnId(c), c]))
+    const orderedGroupCols: ColumnDefinition[] = orderedGroupColIds.map((id) => {
+      if (id === newColumnId) return newColDef
+      const existing = colById.get(id)
+      if (!existing) {
+        throw new Error(`Internal: column "${id}" missing while splicing group outputs`)
+      }
+      return existing
+    })
+    const remainingCols = schema.columns.filter((c) => !groupColIdsBefore.has(getColumnId(c)))
+    const firstGroupIdx = schema.columns.findIndex((c) => groupColIdsBefore.has(getColumnId(c)))
+    const colAnchor = firstGroupIdx === -1 ? remainingCols.length : firstGroupIdx
+    const nextColumns = [
+      ...remainingCols.slice(0, colAnchor),
+      ...orderedGroupCols,
+      ...remainingCols.slice(colAnchor),
+    ]
+
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: nextColumns,
+      workflowGroups: nextGroups,
+    }
+
+    const updatedColumnOrder = table.metadata?.columnOrder
+      ? (() => {
+          const orderWithoutGroup = table.metadata!.columnOrder!.filter(
+            (id) => !groupColIdsBefore.has(id)
+          )
+          const firstGroupOrderIdx = table.metadata!.columnOrder!.findIndex((id) =>
+            groupColIdsBefore.has(id)
+          )
+          const orderAnchor =
+            firstGroupOrderIdx === -1 ? orderWithoutGroup.length : firstGroupOrderIdx
+          return [
+            ...orderWithoutGroup.slice(0, orderAnchor),
+            ...orderedGroupColIds,
+            ...orderWithoutGroup.slice(orderAnchor),
+          ]
+        })()
+      : undefined
+
+    assertValidSchema(updatedSchema, updatedColumnOrder)
+
+    const updatedMetadata: TableMetadata | null =
+      updatedColumnOrder && table.metadata
+        ? { ...table.metadata, columnOrder: updatedColumnOrder }
+        : table.metadata
+          ? { ...table.metadata }
+          : null
+
+    const now = new Date()
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+
+    logger.info(
+      `[${requestId}] Added output "${columnName}" (${newColDef.type}) to workflow group "${data.groupId}" in table ${data.tableId}`
+    )
+
+    const updatedTable: TableDefinition = {
+      ...table,
+      schema: updatedSchema,
+      metadata: updatedMetadata,
+      updatedAt: now,
+    }
+    return { updatedTable, newOutput }
+  })
+
+  // Backfill from saved execution logs — same flow `updateWorkflowGroup`
+  // uses for added outputs. Reads each row's saved trace spans for the
+  // group's executionId and writes the new output's value back. Existing
+  // rows that have hand-edited values are left alone (overwrite: false).
+  // Cheap compared to re-running the workflow on every row, which is what
+  // an earlier version of this code did — that mistakenly fanned out N
+  // workflow-group-cell jobs and burned compute the user didn't ask for.
+  // Small tables backfill inline; large ones run as a background job.
+  // Lazy import: backfill-runner closes a cycle back to this module.
+  try {
+    const { maybeBackfillGroupOutputs } = await import('@/lib/table/backfill-runner')
+    await maybeBackfillGroupOutputs({
+      table: updatedTable,
+      groupId: data.groupId,
+      outputs: [newOutput],
+      overwrite: false,
+      requestId,
+      actorUserId: data.actorUserId,
+    })
+  } catch (err) {
+    logger.warn(
+      `[${requestId}] Backfill from execution logs failed for ${data.tableId} group ${data.groupId} after adding output "${newOutput.columnName}":`,
+      err
+    )
+  }
+
+  return updatedTable
+}
+
+/**
+ * Removes a single output from a workflow group. Drops the bound column and
+ * strips the value from every row's `data` JSONB. If the output is the
+ * group's last, the empty group is left in place — drop it explicitly with
+ * `deleteWorkflowGroup` if needed.
+ */
+export async function deleteWorkflowGroupOutput(
+  data: { tableId: string; groupId: string; columnName: string },
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const groups = schema.workflowGroups ?? []
+    const groupIndex = groups.findIndex((g) => g.id === data.groupId)
+    if (groupIndex === -1) {
+      throw new Error(`Workflow group "${data.groupId}" not found`)
+    }
+    const group = groups[groupIndex]
+    // `data.columnName` may be a column id (first-party) or display name
+    // (mothership/legacy); resolve to the stable id used everywhere below.
+    const targetColumn = schema.columns.find((c) => columnMatchesRef(c, data.columnName))
+    const columnId = targetColumn ? getColumnId(targetColumn) : data.columnName
+    if (!group.outputs.some((o) => o.columnName === columnId)) {
+      throw new Error(
+        `Workflow group "${data.groupId}" has no output bound to column "${data.columnName}"`
+      )
+    }
+
+    const updatedGroup: WorkflowGroup = {
+      ...group,
+      outputs: group.outputs.filter((o) => o.columnName !== columnId),
+    }
+    const nextGroups = groups.map((g, i) => (i === groupIndex ? updatedGroup : g))
+    const nextColumns = schema.columns.filter((c) => getColumnId(c) !== columnId)
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: nextColumns,
+      workflowGroups: nextGroups,
+    }
+
+    const updatedColumnOrder = table.metadata?.columnOrder?.filter((id) => id !== columnId)
+    assertValidSchema(updatedSchema, updatedColumnOrder)
+
+    const updatedMetadata: TableMetadata | null =
+      updatedColumnOrder && table.metadata
+        ? { ...table.metadata, columnOrder: updatedColumnOrder }
+        : table.metadata
+          ? { ...table.metadata }
+          : null
+
+    const now = new Date()
+    await setTableTxTimeouts(trx, { statementMs: 60_000 })
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+    await trx.execute(
+      sql`UPDATE user_table_rows SET data = data - ${columnId}::text WHERE table_id = ${data.tableId} AND data ? ${columnId}::text`
+    )
+
+    logger.info(
+      `[${requestId}] Removed output "${data.columnName}" from workflow group "${data.groupId}" in table ${data.tableId}`
+    )
+
+    return { ...table, schema: updatedSchema, metadata: updatedMetadata, updatedAt: now }
+  })
+}
+
+/**
+ * Removes a workflow group plus all its output columns. Also strips the
+ * group's `executions[groupId]` entry from every row.
+ */
+export async function deleteWorkflowGroup(
+  data: DeleteWorkflowGroupData,
+  requestId: string
+): Promise<TableDefinition> {
+  return withLockedTable(data.tableId, async (table, trx) => {
+    const schema = table.schema
+    const groups = schema.workflowGroups ?? []
+    const group = groups.find((g) => g.id === data.groupId)
+    if (!group) {
+      throw new Error(`Workflow group "${data.groupId}" not found`)
+    }
+
+    const removedColumnIds = new Set(group.outputs.map((o) => o.columnName))
+    // Removed group's output columns may be referenced as deps by sibling groups.
+    // Strip those refs so we don't leave dangling-column deps behind.
+    const nextGroups = groups
+      .filter((g) => g.id !== data.groupId)
+      .map((g) => stripGroupDeps(g, removedColumnIds))
+    const updatedSchema: TableSchema = {
+      ...schema,
+      columns: schema.columns.filter((c) => !removedColumnIds.has(getColumnId(c))),
+      workflowGroups: nextGroups,
+    }
+    const updatedColumnOrder = table.metadata?.columnOrder?.filter(
+      (id) => !removedColumnIds.has(id)
+    )
+    assertValidSchema(updatedSchema, updatedColumnOrder)
+
+    const updatedMetadata: TableMetadata | null =
+      updatedColumnOrder && table.metadata
+        ? { ...table.metadata, columnOrder: updatedColumnOrder }
+        : table.metadata
+          ? { ...table.metadata }
+          : null
+
+    const now = new Date()
+    await setTableTxTimeouts(trx, { statementMs: 60_000 })
+    await trx
+      .update(userTableDefinitions)
+      .set({ schema: updatedSchema, metadata: updatedMetadata, updatedAt: now })
+      .where(eq(userTableDefinitions.id, data.tableId))
+    for (const id of removedColumnIds) {
+      await trx.execute(
+        sql`UPDATE user_table_rows SET data = data - ${id}::text WHERE table_id = ${data.tableId} AND data ? ${id}::text`
+      )
+    }
+    await stripGroupExecutions(trx, data.tableId, [data.groupId])
+
+    logger.info(
+      `[${requestId}] Deleted workflow group "${data.groupId}" from table ${data.tableId}`
+    )
+
+    return {
+      ...table,
+      schema: updatedSchema,
+      metadata: updatedMetadata,
+      updatedAt: now,
+    }
+  })
+}