chore: import upstream snapshot with attribution

apps/sim/lib/table/rows/executions.ts

@@ -0,0 +1,348 @@
+/**
+ * Row-executions (workflow-group results) internals for the table service layer.
+ *
+ * Internal module: not exposed via the `@/lib/table` barrel. Consumers import
+ * directly from `@/lib/table/rows/executions`.
+ */
+
+import { tableRowExecutions } from '@sim/db/schema'
+import { and, eq, inArray, type SQL, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { getColumnId } from '@/lib/table/column-keys'
+import { areGroupDepsSatisfied } from '@/lib/table/deps'
+import type {
+  EnrichmentRunDetail,
+  RowData,
+  RowExecutionMetadata,
+  RowExecutions,
+  TableRow,
+  TableSchema,
+} from '@/lib/table/types'
+
+/**
+ * Loads `tableRowExecutions` rows for the given row ids and groups them into a
+ * `Map<rowId, RowExecutions>` suitable for plugging into `TableRow.executions`.
+ */
+export async function loadExecutionsByRow(
+  trx: DbOrTx,
+  rowIds: Iterable<string>
+): Promise<Map<string, RowExecutions>> {
+  const ids = Array.from(new Set(rowIds))
+  const result = new Map<string, RowExecutions>()
+  if (ids.length === 0) return result
+  // Explicit column list, never `select()` — `enrichmentDetails` is large and
+  // must stay off the hot grid read path (fetched on demand via
+  // `loadEnrichmentDetail`).
+  const rows = await trx
+    .select({
+      rowId: tableRowExecutions.rowId,
+      groupId: tableRowExecutions.groupId,
+      status: tableRowExecutions.status,
+      executionId: tableRowExecutions.executionId,
+      jobId: tableRowExecutions.jobId,
+      workflowId: tableRowExecutions.workflowId,
+      error: tableRowExecutions.error,
+      runningBlockIds: tableRowExecutions.runningBlockIds,
+      blockErrors: tableRowExecutions.blockErrors,
+      cancelledAt: tableRowExecutions.cancelledAt,
+    })
+    .from(tableRowExecutions)
+    .where(inArray(tableRowExecutions.rowId, ids))
+  for (const r of rows) {
+    const existing = result.get(r.rowId) ?? {}
+    const meta: RowExecutionMetadata = {
+      status: r.status as RowExecutionMetadata['status'],
+      executionId: r.executionId ?? null,
+      jobId: r.jobId ?? null,
+      workflowId: r.workflowId,
+      error: r.error ?? null,
+      ...(r.runningBlockIds && r.runningBlockIds.length > 0
+        ? { runningBlockIds: r.runningBlockIds }
+        : {}),
+      ...(r.blockErrors && Object.keys(r.blockErrors as Record<string, string>).length > 0
+        ? { blockErrors: r.blockErrors as Record<string, string> }
+        : {}),
+      ...(r.cancelledAt ? { cancelledAt: r.cancelledAt.toISOString() } : {}),
+    }
+    existing[r.groupId] = meta
+    result.set(r.rowId, existing)
+  }
+  return result
+}
+
+/** Convenience: load executions for one row, returning `{}` when missing. */
+export async function loadExecutionsForRow(trx: DbOrTx, rowId: string): Promise<RowExecutions> {
+  const byRow = await loadExecutionsByRow(trx, [rowId])
+  return byRow.get(rowId) ?? {}
+}
+
+/**
+ * Loads the enrichment cascade breakdown for one `(tableId, rowId, groupId)`,
+ * or `null` when there is no exec row or it predates the feature. Read on demand
+ * by the enrichment details panel — kept off `loadExecutionsByRow`.
+ */
+export async function loadEnrichmentDetail(
+  trx: DbOrTx,
+  tableId: string,
+  rowId: string,
+  groupId: string
+): Promise<EnrichmentRunDetail | null> {
+  const [row] = await trx
+    .select({ enrichmentDetails: tableRowExecutions.enrichmentDetails })
+    .from(tableRowExecutions)
+    .where(
+      and(
+        eq(tableRowExecutions.tableId, tableId),
+        eq(tableRowExecutions.rowId, rowId),
+        eq(tableRowExecutions.groupId, groupId)
+      ) as SQL
+    )
+    .limit(1)
+  return (row?.enrichmentDetails as EnrichmentRunDetail | null | undefined) ?? null
+}
+
+/**
+ * Derive automatic clears + cancellation candidates from a row's data patch.
+ *
+ * Walks `schema.workflowGroups` left-to-right with a propagating `dirtied`
+ * column set. For each group whose deps overlap the dirty set, decide to
+ * clear (terminal exec) or cancel+rerun (in-flight exec), then add the
+ * group's outputs to the dirty set so later groups in the chain see them
+ * as dirty too. This models transitive dep chains as a single forward pass —
+ * editing column A propagates through group 1 (deps on A) to group 2 (deps
+ * on group 1's output) without explicit DAG traversal.
+ *
+ * Returns:
+ * - `executionsPatch`: caller's patch + nulls for cleared groups (or
+ *   undefined if nothing applied).
+ * - `inFlightDownstreamGroups`: groups whose dep was dirtied and that are
+ *   currently in-flight. Cancel-and-restart is the caller's job.
+ *
+ * Assumption: `workflowGroups[]` is in topological order — a group's deps
+ * may only reference columns to its left (enforced by `workflow-sidebar`'s
+ * "Run after" picker + the reorder scrub via `stripGroupDeps`). Violating
+ * this would silently miss the propagation.
+ */
+export function deriveExecClearsForDataPatch(
+  dataPatch: RowData,
+  schema: TableSchema,
+  existingExecutions: RowExecutions,
+  callerPatch: Record<string, RowExecutionMetadata | null> | undefined,
+  mergedData: RowData
+): {
+  executionsPatch: Record<string, RowExecutionMetadata | null> | undefined
+  inFlightDownstreamGroups: string[]
+} {
+  const dirtied = new Set(Object.keys(dataPatch))
+  const groupsToClear = new Set<string>()
+  const inFlightDownstreamGroups: string[] = []
+
+  // Own-output clears: when the user wipes a workflow output column, drop
+  // that group's exec entry so the auto-fire reactor re-arms the cell.
+  // Also flags the cleared output column as dirty so transitive downstream
+  // groups see it.
+  for (const [columnId, value] of Object.entries(dataPatch)) {
+    const cleared = value === null || value === undefined || value === ''
+    if (!cleared) continue
+    const col = schema.columns.find((c) => getColumnId(c) === columnId)
+    if (col?.workflowGroupId) groupsToClear.add(col.workflowGroupId)
+  }
+
+  // Left-to-right walk, propagating dirty columns forward.
+  const groups = schema.workflowGroups ?? []
+  const afterRow = { data: mergedData } as TableRow
+  for (const group of groups) {
+    const deps = group.dependencies?.columns ?? []
+    const depMatched = deps.some((d) => dirtied.has(d))
+    if (!depMatched) continue
+
+    // A dep column changed, but if the group's deps are no longer satisfied
+    // after the patch — a checkbox was unchecked or a text dep cleared — there's
+    // nothing to recompute. Leave the prior result alone instead of re-arming or
+    // cancelling it; only checking a box / filling a dep drives downstream work.
+    if (!areGroupDepsSatisfied(group, afterRow)) continue
+
+    const exec = existingExecutions[group.id]
+    if (exec) {
+      const status = exec.status
+      if (status === 'completed' || status === 'error' || status === 'cancelled') {
+        groupsToClear.add(group.id)
+      } else if (status === 'queued' || status === 'running' || status === 'pending') {
+        inFlightDownstreamGroups.push(group.id)
+      }
+    } else {
+      // No exec entry yet — `mode: 'new'` already covers this group. We
+      // still propagate the dirty signal forward so later groups in the
+      // chain see this group's outputs as dirty too.
+      groupsToClear.add(group.id)
+    }
+
+    // Propagate: this group is about to be re-computed, so groups whose
+    // deps reference its output columns are also dirty.
+    for (const out of group.outputs) dirtied.add(out.columnName)
+  }
+
+  if (groupsToClear.size === 0) {
+    return { executionsPatch: callerPatch, inFlightDownstreamGroups }
+  }
+  const merged: Record<string, RowExecutionMetadata | null> = { ...(callerPatch ?? {}) }
+  for (const gid of groupsToClear) {
+    if (!(gid in merged)) merged[gid] = null
+  }
+  return { executionsPatch: merged, inFlightDownstreamGroups }
+}
+
+/** Merges an `executionsPatch` into the row's existing executions blob. */
+export function applyExecutionsPatch(
+  existing: RowExecutions,
+  patch: Record<string, RowExecutionMetadata | null> | undefined
+): RowExecutions {
+  if (!patch) return existing
+  const next: RowExecutions = { ...existing }
+  for (const [gid, value] of Object.entries(patch)) {
+    if (value === null) {
+      delete next[gid]
+    } else {
+      next[gid] = value
+    }
+  }
+  return next
+}
+
+/**
+ * Writes a per-group execution patch for one row against the `tableRowExecutions`
+ * sidecar. Non-null values upsert into the table; nulls delete the entry. When
+ * `guard` is set, the upsert is gated to:
+ *  - reject if a `cancelled` row for the same execution already exists, and
+ *  - reject if the row exists but is owned by a different executionId
+ *    (with carve-outs for missing rows and null executionIds — the dispatcher's
+ *    pre-batch `pending` stamp leaves executionId unset so the first cell-task
+ *    can claim).
+ *
+ * Returns `'guard-rejected'` when the guarded group's upsert affected 0 rows
+ * (callers signal failure to the cell-task path). Returns `'wrote'` otherwise.
+ */
+export async function writeExecutionsPatch(
+  trx: DbOrTx,
+  tableId: string,
+  rowId: string,
+  patch: Record<string, RowExecutionMetadata | null> | undefined,
+  guard?: { groupId: string; executionId: string }
+): Promise<'wrote' | 'guard-rejected'> {
+  if (!patch) return 'wrote'
+  const entries = Object.entries(patch)
+  if (entries.length === 0) return 'wrote'
+
+  for (const [gid, value] of entries) {
+    if (value === null) {
+      await trx
+        .delete(tableRowExecutions)
+        .where(and(eq(tableRowExecutions.rowId, rowId), eq(tableRowExecutions.groupId, gid)) as SQL)
+      continue
+    }
+    const insertValues = {
+      tableId,
+      rowId,
+      groupId: gid,
+      status: value.status,
+      executionId: value.executionId,
+      jobId: value.jobId,
+      workflowId: value.workflowId,
+      error: value.error,
+      runningBlockIds: value.runningBlockIds ?? [],
+      blockErrors: value.blockErrors ?? {},
+      cancelledAt: value.cancelledAt ? new Date(value.cancelledAt) : null,
+      enrichmentDetails: value.enrichmentDetails ?? null,
+      updatedAt: new Date(),
+    } as const
+
+    const isGuarded = guard && guard.groupId === gid
+    if (isGuarded) {
+      // Gate by guard semantics. The original JSONB guard had two AND'd
+      // clauses; we collapse them onto the upsert's WHERE so a non-matching
+      // existing row leaves the table untouched and we observe 0 affected.
+      const guardExecutionId = guard.executionId
+      const updated = await trx
+        .insert(tableRowExecutions)
+        .values(insertValues)
+        .onConflictDoUpdate({
+          target: [tableRowExecutions.rowId, tableRowExecutions.groupId],
+          set: {
+            status: insertValues.status,
+            executionId: insertValues.executionId,
+            jobId: insertValues.jobId,
+            workflowId: insertValues.workflowId,
+            error: insertValues.error,
+            runningBlockIds: insertValues.runningBlockIds,
+            blockErrors: insertValues.blockErrors,
+            cancelledAt: insertValues.cancelledAt,
+            // Sticky: preserve a prior cascade breakdown when this write omits
+            // it (e.g. the running pickup stamp) so only an explicit detail
+            // overwrites it. Re-runs delete the row first, so this never serves
+            // stale detail across runs.
+            enrichmentDetails: sql`coalesce(excluded.enrichment_details, ${tableRowExecutions.enrichmentDetails})`,
+            updatedAt: insertValues.updatedAt,
+          },
+          where: and(
+            // Reject any guarded worker write when the cell is `cancelled` — a
+            // stop click wrote it authoritatively. SQL mirror of `isExecCancelled`
+            // (deps.ts). Status-only (not executionId-scoped): the cancel can
+            // only carry the pre-stamp's executionId (often null), so matching on
+            // id would let the worker's real-id claim resurrect a killed cell.
+            sql`${tableRowExecutions.status} <> 'cancelled'`,
+            // Stale-worker: the cell's active run has moved on. Carve-outs
+            // permit a fresh worker to take over when the row's executionId
+            // is unset (dispatcher's pre-batch `pending` stamp).
+            sql`(${tableRowExecutions.executionId} IS NULL OR ${tableRowExecutions.executionId} = ${guardExecutionId})`
+          ) as SQL,
+        })
+        .returning({ rowId: tableRowExecutions.rowId })
+      if (updated.length === 0) return 'guard-rejected'
+      continue
+    }
+
+    await trx
+      .insert(tableRowExecutions)
+      .values(insertValues)
+      .onConflictDoUpdate({
+        target: [tableRowExecutions.rowId, tableRowExecutions.groupId],
+        set: {
+          status: insertValues.status,
+          executionId: insertValues.executionId,
+          jobId: insertValues.jobId,
+          workflowId: insertValues.workflowId,
+          error: insertValues.error,
+          runningBlockIds: insertValues.runningBlockIds,
+          blockErrors: insertValues.blockErrors,
+          cancelledAt: insertValues.cancelledAt,
+          // Sticky: preserve a prior cascade breakdown when this write omits it
+          // (e.g. the running pickup stamp) so only an explicit detail overwrites
+          // it. Re-runs delete the row first, so this never serves stale detail.
+          enrichmentDetails: sql`coalesce(excluded.enrichment_details, ${tableRowExecutions.enrichmentDetails})`,
+          updatedAt: insertValues.updatedAt,
+        },
+      })
+  }
+
+  return 'wrote'
+}
+
+/**
+ * Strips the given workflow group ids from every row's executions on a table —
+ * used by the column / group delete paths so stale running/queued exec records
+ * don't linger and inflate counters after the group is gone. The caller wraps
+ * in their own transaction.
+ */
+export async function stripGroupExecutions(
+  trx: DbOrTx,
+  tableId: string,
+  groupIds: Iterable<string>
+): Promise<void> {
+  const ids = Array.from(new Set(groupIds))
+  if (ids.length === 0) return
+  await trx
+    .delete(tableRowExecutions)
+    .where(
+      and(eq(tableRowExecutions.tableId, tableId), inArray(tableRowExecutions.groupId, ids)) as SQL
+    )
+}

apps/sim/lib/table/rows/ordering.ts

@@ -0,0 +1,459 @@
+/**
+ * Row position / fractional-ordering internals for the table service layer.
+ *
+ * Internal module: only the import/delete-runner entry points are exposed via
+ * the `@/lib/table/rows/ordering` path. Not re-exported through the
+ * `@/lib/table` barrel.
+ */
+
+import { db } from '@sim/db'
+import { userTableRows } from '@sim/db/schema'
+import { and, asc, desc, eq, gt, inArray, lt, lte, type SQL, sql } from 'drizzle-orm'
+import type { DbOrTx } from '@/lib/db/types'
+import { TABLE_LIMITS } from '@/lib/table/constants'
+import { keyBetween, nKeysBetween } from '@/lib/table/order-key'
+import { type DbExecutor, type DbTransaction, withSeqscanOff } from '@/lib/table/planner'
+import { setTableTxTimeouts } from '@/lib/table/tx'
+import type { RowData } from '@/lib/table/types'
+
+/**
+ * Starting `position` for an append import — `max(position) + 1`, or 0 when empty. Read once,
+ * unlocked, before streaming: the import worker is the table's sole writer, so it can assign
+ * contiguous positions from this offset without per-batch position scans.
+ */
+export async function nextImportStartPosition(tableId: string): Promise<number> {
+  const [{ maxPos }] = await db
+    .select({
+      maxPos: sql<number>`coalesce(max(${userTableRows.position}), -1)`.mapWith(Number),
+    })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxPos + 1
+}
+
+/**
+ * Append anchor `order_key` for an import — `max(order_key)`, or null when empty. Read once,
+ * unlocked, before streaming (the import worker is the table's sole writer); each batch threads
+ * the previous batch's last key forward so no per-batch max scan is needed.
+ */
+export async function nextImportStartOrderKey(tableId: string): Promise<string | null> {
+  return maxOrderKey(db, tableId)
+}
+
+/**
+ * Serializes writers that assign `position` for the same table. The row-count
+ * trigger (migration 0198) serializes capacity via a row lock on
+ * `user_table_definitions`, but it fires AFTER INSERT, so two concurrent
+ * auto-positioned inserts could read the same snapshot and assign the same
+ * position (the `(table_id, position)` index is non-unique). This advisory lock
+ * restores per-table serialization. Released at COMMIT/ROLLBACK.
+ */
+export async function acquireRowOrderLock(trx: DbTransaction, tableId: string) {
+  await trx.execute(
+    sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_rows_pos:${tableId}`}, 0))`
+  )
+}
+
+/** Next append position for a table (max(position) + 1, or 0 if empty). */
+export async function nextRowPosition(trx: DbTransaction, tableId: string): Promise<number> {
+  const [{ maxPos }] = await trx
+    .select({
+      maxPos: sql<number>`coalesce(max(${userTableRows.position}), -1)`.mapWith(Number),
+    })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxPos + 1
+}
+
+/** Largest `order_key` for a table, or `null` when empty — the append anchor for new keys. */
+export async function maxOrderKey(executor: DbOrTx, tableId: string): Promise<string | null> {
+  const [{ maxKey }] = await executor
+    .select({ maxKey: sql<string | null>`max(${userTableRows.orderKey})` })
+    .from(userTableRows)
+    .where(eq(userTableRows.tableId, tableId))
+  return maxKey ?? null
+}
+
+/**
+ * Computes the fractional `order_key` for a row inserted at the integer
+ * `requestedPosition` (or appended when omitted). Used by position-based callers
+ * (mothership tool, v1 API, undo position-fallback, transient old clients).
+ *
+ * The neighbor at slot `s` is the `s`-th row in `order_key, id` order (`OFFSET
+ * s`) — positions are gappy and non-authoritative, so `position = s` would miss;
+ * the visual ordinal is the key's ordinal. O(s), acceptable for these low-volume
+ * callers.
+ *
+ * Caller holds the row-order lock.
+ */
+export async function resolveInsertOrderKey(
+  trx: DbTransaction,
+  tableId: string,
+  requestedPosition?: number
+): Promise<string> {
+  const orderKeyAtSlot = async (slot: number): Promise<string | null> => {
+    if (slot < 0) return null
+    const [r] = await trx
+      .select({ orderKey: userTableRows.orderKey })
+      .from(userTableRows)
+      .where(eq(userTableRows.tableId, tableId))
+      .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id))
+      .limit(1)
+      .offset(slot)
+    return r?.orderKey ?? null
+  }
+  if (requestedPosition === undefined) {
+    return keyBetween(await maxOrderKey(trx, tableId), null)
+  }
+  const lo = await orderKeyAtSlot(requestedPosition - 1)
+  const hi = await orderKeyAtSlot(requestedPosition)
+  return keyBetween(lo, hi)
+}
+
+/**
+ * Resolves the `order_key` for an insert expressed by an anchor row id —
+ * `afterRowId` (place directly after) or `beforeRowId` (directly before). Finds
+ * the anchor and its adjacent key via the `(table_id, order_key, id)` index
+ * (O(1)) and mints a key between them. Caller holds the row-order lock.
+ */
+export async function resolveInsertByNeighbor(
+  trx: DbTransaction,
+  tableId: string,
+  afterRowId?: string,
+  beforeRowId?: string
+): Promise<string> {
+  const anchorId = afterRowId ?? beforeRowId!
+  const [anchor] = await trx
+    .select({ orderKey: userTableRows.orderKey })
+    .from(userTableRows)
+    .where(and(eq(userTableRows.tableId, tableId), eq(userTableRows.id, anchorId)))
+    .limit(1)
+  // The client targets a specific neighbor; a missing one (concurrent delete /
+  // stale view) is an error, not a silent insert at the front.
+  if (!anchor) throw new Error(`Row not found: ${anchorId}`)
+  const anchorKey = anchor.orderKey ?? null
+  // A null key on the anchor means the table isn't backfilled. order_key is
+  // authoritative, so the adjacent-key lookup below can't work — fail loudly
+  // rather than mint a wrong key.
+  if (anchorKey === null) {
+    throw new Error(`Row ${anchorId} has no order_key yet (table not backfilled)`)
+  }
+
+  if (afterRowId) {
+    // hi = the smallest key strictly GREATER than the anchor key. Comparing keys
+    // (not the `(order_key, id)` row tuple) skips past any sibling that shares the
+    // anchor's key, so `keyBetween` always gets strictly-ordered bounds and can't
+    // throw on a stray duplicate. Identical to the row tuple when keys are distinct.
+    const [next] = await trx
+      .select({ orderKey: userTableRows.orderKey })
+      .from(userTableRows)
+      .where(and(eq(userTableRows.tableId, tableId), gt(userTableRows.orderKey, anchorKey)))
+      .orderBy(asc(userTableRows.orderKey))
+      .limit(1)
+    return keyBetween(anchorKey, next?.orderKey ?? null)
+  }
+
+  // beforeRowId: lo = the largest key strictly LESS than the anchor key (distinct,
+  // same rationale as the afterRowId branch above).
+  const [prev] = await trx
+    .select({ orderKey: userTableRows.orderKey })
+    .from(userTableRows)
+    .where(and(eq(userTableRows.tableId, tableId), lt(userTableRows.orderKey, anchorKey)))
+    .orderBy(desc(userTableRows.orderKey))
+    .limit(1)
+  return keyBetween(prev?.orderKey ?? null, anchorKey)
+}
+
+/**
+ * Computes fractional `order_key`s for a batch insert by appending a contiguous
+ * run after the current max key. `order_key` is authoritative, so callers needing
+ * exact placement pass explicit `orderKeys` (handled before this function); here
+ * we just append a run. Caller holds the lock.
+ */
+export async function resolveBatchInsertOrderKeys(
+  trx: DbTransaction,
+  tableId: string,
+  count: number
+): Promise<string[]> {
+  return nKeysBetween(await maxOrderKey(trx, tableId), null, count)
+}
+
+/**
+ * Inserts a single row in its own transaction. Assigns a fractional `order_key`
+ * (authoritative) and a best-effort append `position` (no O(N) shift).
+ * Validation and side-effect dispatch stay with the caller; capacity is enforced
+ * by the `increment_user_table_row_count` trigger.
+ */
+export async function insertOrderedRow(params: {
+  tableId: string
+  workspaceId: string
+  data: RowData
+  rowId: string
+  position?: number
+  afterRowId?: string
+  beforeRowId?: string
+  createdBy?: string
+  now: Date
+}): Promise<{
+  id: string
+  data: RowData
+  position: number
+  orderKey: string | null
+  createdAt: Date
+  updatedAt: Date
+}> {
+  const { tableId, workspaceId, data, rowId, position, afterRowId, beforeRowId, createdBy, now } =
+    params
+  const [row] = await db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx)
+    await acquireRowOrderLock(trx, tableId)
+
+    // Resolve the authoritative order key from neighbor ids when given, else from
+    // the requested position.
+    const orderKey =
+      afterRowId || beforeRowId
+        ? await resolveInsertByNeighbor(trx, tableId, afterRowId, beforeRowId)
+        : await resolveInsertOrderKey(trx, tableId, position)
+
+    // order_key is authoritative — keep a best-effort, no-shift position.
+    const targetPosition = await nextRowPosition(trx, tableId)
+
+    return trx
+      .insert(userTableRows)
+      .values({
+        id: rowId,
+        tableId,
+        workspaceId,
+        data,
+        position: targetPosition,
+        orderKey,
+        createdAt: now,
+        updatedAt: now,
+        ...(createdBy ? { createdBy } : {}),
+      })
+      .returning()
+  })
+  return {
+    id: row.id,
+    data: row.data as RowData,
+    position: row.position,
+    orderKey: row.orderKey,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  }
+}
+
+/**
+ * Deletes a single row by id in its own transaction. Deleting a row never changes
+ * another row's `order_key`, so no positional reshift is needed. Returns `false`
+ * when no row matched.
+ */
+export async function deleteOrderedRow(params: {
+  tableId: string
+  rowId: string
+  workspaceId: string
+}): Promise<boolean> {
+  const { tableId, rowId, workspaceId } = params
+  return db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx)
+    const [deleted] = await trx
+      .delete(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.id, rowId),
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId)
+        )
+      )
+      .returning({ id: userTableRows.id })
+    return Boolean(deleted)
+  })
+}
+
+/**
+ * Deletes the given row ids in batches within one transaction. Deletes leave
+ * `order_key` untouched, so no positional recompaction is needed. Returns the
+ * deleted row ids. The caller resolves which ids to delete (used by both
+ * delete-by-ids and delete-by-filter).
+ */
+export async function deleteOrderedRowsByIds(params: {
+  tableId: string
+  workspaceId: string
+  rowIds: string[]
+}): Promise<{ id: string }[]> {
+  const { tableId, workspaceId, rowIds } = params
+  if (rowIds.length === 0) return []
+  return db.transaction(async (trx) => {
+    await setTableTxTimeouts(trx, { statementMs: 60_000 })
+    const deleted: { id: string }[] = []
+    for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) {
+      const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE)
+      const rows = await trx
+        .delete(userTableRows)
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+      deleted.push(...rows)
+    }
+    return deleted
+  })
+}
+
+/**
+ * Selects one page of row ids to delete for the async delete-job worker: base scope plus a
+ * `created_at <= cutoff` floor (so rows inserted after the job started are never selected) and
+ * the caller's optional filter clause. Keyset paginated on `id` via `afterId` so excluded rows
+ * (which are skipped, not deleted) still advance the cursor — no OFFSET, no risk of looping on a
+ * fully-excluded page.
+ */
+export async function selectRowIdPage(params: {
+  tableId: string
+  workspaceId: string
+  cutoff: Date
+  filterClause?: SQL
+  afterId?: string
+  limit: number
+}): Promise<string[]> {
+  const { tableId, workspaceId, cutoff, filterClause, afterId, limit } = params
+  const selectPage = (executor: DbExecutor) =>
+    executor
+      .select({ id: userTableRows.id })
+      .from(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId),
+          lte(userTableRows.createdAt, cutoff),
+          afterId ? gt(userTableRows.id, afterId) : undefined,
+          filterClause
+        )
+      )
+      .orderBy(asc(userTableRows.id))
+      .limit(limit)
+  // A jsonb filter is unestimatable, so the planner would seq-scan the whole shared relation
+  // per page (12.6s measured) — keep it on the tenant's (table_id, id) index.
+  const rows = filterClause
+    ? await withSeqscanOff(async (trx) => selectPage(trx))
+    : await selectPage(db)
+  return rows.map((r) => r.id)
+}
+
+/**
+ * Like {@link selectRowIdPage} but returns each row's `data` too, for the bulk-update worker which
+ * must merge the patch into the existing row to validate the result. Same keyset walk on the
+ * `(table_id, id)` index, `created_at <= cutoff`, tenant-scoped, seqscan-off for jsonb filters.
+ *
+ * `excludeIfPatched` (a JSON patch string) skips rows that already contain the patch
+ * (`data @> patch`). The update worker passes it so a retried run doesn't re-walk and re-count
+ * rows an earlier attempt already updated — updated rows still exist (unlike deletes), and they
+ * still match the filter when the patch doesn't touch a filtered column, so without this a retry
+ * would double-count progress. It also skips no-op updates of rows that already hold those values.
+ */
+export async function selectRowDataPage(params: {
+  tableId: string
+  workspaceId: string
+  cutoff: Date
+  filterClause?: SQL
+  afterId?: string
+  limit: number
+  excludeIfPatched?: string
+}): Promise<Array<{ id: string; data: RowData }>> {
+  const { tableId, workspaceId, cutoff, filterClause, afterId, limit, excludeIfPatched } = params
+  const selectPage = (executor: DbExecutor) =>
+    executor
+      .select({ id: userTableRows.id, data: userTableRows.data })
+      .from(userTableRows)
+      .where(
+        and(
+          eq(userTableRows.tableId, tableId),
+          eq(userTableRows.workspaceId, workspaceId),
+          lte(userTableRows.createdAt, cutoff),
+          afterId ? gt(userTableRows.id, afterId) : undefined,
+          excludeIfPatched
+            ? sql`NOT (${userTableRows.data} @> ${excludeIfPatched}::jsonb)`
+            : undefined,
+          filterClause
+        )
+      )
+      .orderBy(asc(userTableRows.id))
+      .limit(limit)
+  const rows = filterClause
+    ? await withSeqscanOff(async (trx) => selectPage(trx))
+    : await selectPage(db)
+  return rows.map((r) => ({ id: r.id, data: r.data as RowData }))
+}
+
+/**
+ * Deletes one page of rows for the async delete-job worker, committing each `DELETE_BATCH_SIZE`
+ * chunk in its own short transaction. One statement per transaction bounds how long the
+ * statement-level row_count trigger's lock on the definition row is held (a page-wide transaction
+ * held it for the entire page, starving concurrent inserts and overrunning `statement_timeout`),
+ * and a mid-page failure loses at most one uncommitted batch — the keyset walker (or a task
+ * retry) re-walks whatever remains. Skips legacy position compaction: under fractional ordering
+ * it's unnecessary, and in the legacy path `position` gaps are harmless — rows still order by
+ * position. Returns the count deleted.
+ */
+export async function deletePageByIds(
+  tableId: string,
+  workspaceId: string,
+  rowIds: string[]
+): Promise<number> {
+  let deleted = 0
+  for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.DELETE_BATCH_SIZE) {
+    const batch = rowIds.slice(i, i + TABLE_LIMITS.DELETE_BATCH_SIZE)
+    const rows = await db.transaction(async (trx) => {
+      await setTableTxTimeouts(trx, { statementMs: 60_000 })
+      return trx
+        .delete(userTableRows)
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+    })
+    deleted += rows.length
+  }
+  return deleted
+}
+
+/**
+ * Applies a JSONB-merge patch (`data || patchJson`) to a page of row ids, committed in
+ * UPDATE_BATCH_SIZE chunks (each its own transaction, 60s timeout) so a large background update
+ * makes incremental, resumable progress. Returns the number of rows updated.
+ */
+export async function updatePageByIds(
+  tableId: string,
+  workspaceId: string,
+  rowIds: string[],
+  patchJson: string
+): Promise<number> {
+  const now = new Date()
+  let updated = 0
+  for (let i = 0; i < rowIds.length; i += TABLE_LIMITS.UPDATE_BATCH_SIZE) {
+    const batch = rowIds.slice(i, i + TABLE_LIMITS.UPDATE_BATCH_SIZE)
+    const rows = await db.transaction(async (trx) => {
+      await setTableTxTimeouts(trx, { statementMs: 60_000 })
+      return trx
+        .update(userTableRows)
+        .set({ data: sql`${userTableRows.data} || ${patchJson}::jsonb`, updatedAt: now })
+        .where(
+          and(
+            eq(userTableRows.tableId, tableId),
+            eq(userTableRows.workspaceId, workspaceId),
+            inArray(userTableRows.id, batch)
+          )
+        )
+        .returning({ id: userTableRows.id })
+    })
+    updated += rows.length
+  }
+  return updated
+}

apps/sim/lib/table/rows/paging.ts

@@ -0,0 +1,23 @@
+/**
+ * Cuts a fetched page to a byte budget: keeps the longest prefix of rows whose
+ * serialized `data` fits within `maxBytes`, always keeping at least one row so
+ * pagination makes forward progress even when a single row exceeds the budget.
+ *
+ * The budget counts `data` only — the per-row envelope (`id`, `position`,
+ * `orderKey`, timestamps, executions) is not measured, so actual response
+ * payloads run slightly over `maxBytes`. Callers must leave headroom; the
+ * production SQL-side cut should account for the same overhead.
+ */
+export function trimRowsToByteBudget<T extends { data: unknown }>(
+  rows: T[],
+  maxBytes: number
+): T[] {
+  let total = 0
+  let kept = 0
+  for (const row of rows) {
+    total += Buffer.byteLength(JSON.stringify(row.data))
+    if (kept > 0 && total > maxBytes) break
+    kept++
+  }
+  return kept === rows.length ? rows : rows.slice(0, kept)
+}