chore: import upstream snapshot with attribution

apps/sim/lib/core/outbox/service.test.ts

@@ -0,0 +1,389 @@
+/**
+ * @vitest-environment node
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+type OutboxRow = {
+  id: string
+  eventType: string
+  payload: unknown
+  status: 'pending' | 'processing' | 'completed' | 'dead_letter'
+  attempts: number
+  maxAttempts: number
+  availableAt: Date
+  lockedAt: Date | null
+  lastError: string | null
+  createdAt: Date
+  processedAt: Date | null
+}
+
+// Hoisted mock state — all tests manipulate these directly.
+const { state, mockDb } = vi.hoisted(() => {
+  const state = {
+    // Rows returned from the FOR UPDATE SKIP LOCKED select in claimBatch.
+    claimedRows: [] as OutboxRow[],
+    // Whether the terminal update (lease CAS) should report a match.
+    leaseHeld: true,
+    // IDs the reaper's UPDATE should return (simulates stuck `processing` rows).
+    reapedRowIds: [] as string[],
+    // Everything written (for assertions).
+    inserts: [] as Array<{ values: unknown }>,
+    updates: [] as Array<{ set: Record<string, unknown>; where?: unknown }>,
+  }
+
+  const makeUpdateChain = () => {
+    const row: { set: Record<string, unknown>; where?: unknown } = { set: {} }
+    const chain: Record<string, unknown> = {}
+    chain.set = vi.fn((s: Record<string, unknown>) => {
+      row.set = s
+      return chain
+    })
+    chain.where = vi.fn((w: unknown) => {
+      row.where = w
+      state.updates.push(row)
+      return chain
+    })
+    chain.returning = vi.fn(async () => {
+      // Terminal UPDATE (lease CAS): has `attempts` + `availableAt`
+      // on retry, or explicit completed/dead_letter. Reaper path sets
+      // status='pending' without attempts/availableAt.
+      const isReaperUpdate =
+        row.set.status === 'pending' && !('attempts' in row.set) && !('availableAt' in row.set)
+
+      if (isReaperUpdate) {
+        return state.reapedRowIds.map((id) => ({ id }))
+      }
+
+      if (
+        row.set.status === 'completed' ||
+        row.set.status === 'dead_letter' ||
+        (row.set.status === 'pending' && 'attempts' in row.set && 'availableAt' in row.set) ||
+        (!('status' in row.set) && 'attempts' in row.set && 'lockedAt' in row.set)
+      ) {
+        return state.leaseHeld ? [{ id: 'evt-1' }] : []
+      }
+
+      return []
+    })
+    return chain
+  }
+
+  const makeSelectChain = () => {
+    const chain: Record<string, unknown> = {}
+    const self = () => chain
+    chain.from = vi.fn(self)
+    chain.where = vi.fn(self)
+    chain.orderBy = vi.fn(self)
+    chain.limit = vi.fn(self)
+    chain.for = vi.fn(async () => state.claimedRows.splice(0, 1))
+    return chain
+  }
+
+  const mockDb = {
+    insert: vi.fn(() => {
+      const chain: Record<string, unknown> = {}
+      chain.values = vi.fn(async (v: unknown) => {
+        state.inserts.push({ values: v })
+      })
+      return chain
+    }),
+    update: vi.fn(() => makeUpdateChain()),
+    select: vi.fn(() => makeSelectChain()),
+    transaction: vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn(mockDb)),
+  }
+
+  return { state, mockDb }
+})
+
+vi.mock('@sim/db', () => ({ db: mockDb }))
+
+vi.mock('@sim/db/schema', () => ({
+  outboxEvent: {
+    id: 'outbox_event.id',
+    eventType: 'outbox_event.event_type',
+    payload: 'outbox_event.payload',
+    status: 'outbox_event.status',
+    attempts: 'outbox_event.attempts',
+    maxAttempts: 'outbox_event.max_attempts',
+    availableAt: 'outbox_event.available_at',
+    lockedAt: 'outbox_event.locked_at',
+    lastError: 'outbox_event.last_error',
+    createdAt: 'outbox_event.created_at',
+    processedAt: 'outbox_event.processed_at',
+    $inferSelect: {} as OutboxRow,
+  },
+}))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }),
+}))
+
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn((...args) => ({ _op: 'and', args })),
+  asc: vi.fn((col) => ({ _op: 'asc', col })),
+  eq: vi.fn((col, val) => ({ _op: 'eq', col, val })),
+  inArray: vi.fn((col, vals) => ({ _op: 'inArray', col, vals })),
+  lte: vi.fn((col, val) => ({ _op: 'lte', col, val })),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn(() => 'test-event-id'),
+}))
+
+import { enqueueOutboxEvent, processOutboxEvents } from './service'
+
+function makePendingRow(overrides: Partial<OutboxRow> = {}): OutboxRow {
+  return {
+    id: 'evt-1',
+    eventType: 'test.event',
+    payload: { foo: 'bar' },
+    status: 'pending',
+    attempts: 0,
+    maxAttempts: 10,
+    availableAt: new Date(Date.now() - 1000),
+    lockedAt: null,
+    lastError: null,
+    createdAt: new Date(Date.now() - 5000),
+    processedAt: null,
+    ...overrides,
+  }
+}
+
+function resetState() {
+  state.claimedRows = []
+  state.leaseHeld = true
+  state.reapedRowIds = []
+  state.inserts.length = 0
+  state.updates.length = 0
+}
+
+describe('enqueueOutboxEvent', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('inserts a row with the given event type and payload', async () => {
+    const id = await enqueueOutboxEvent(mockDb, 'test.event', { foo: 'bar' })
+    expect(id).toBe('test-event-id')
+    expect(state.inserts[0].values).toMatchObject({
+      id: 'test-event-id',
+      eventType: 'test.event',
+      payload: { foo: 'bar' },
+      maxAttempts: 10,
+    })
+  })
+
+  it('respects maxAttempts override', async () => {
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { maxAttempts: 3 })
+    expect(state.inserts[0].values).toMatchObject({ maxAttempts: 3 })
+  })
+
+  it('respects availableAt override for delayed processing', async () => {
+    const future = new Date(Date.now() + 60_000)
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { availableAt: future })
+    expect((state.inserts[0].values as { availableAt: Date }).availableAt).toBe(future)
+  })
+})
+
+describe('processOutboxEvents — empty / no handler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('returns zero counts when no events are due', async () => {
+    const result = await processOutboxEvents({})
+    expect(result).toEqual({
+      processed: 0,
+      retried: 0,
+      deadLettered: 0,
+      leaseLost: 0,
+      reaped: 0,
+    })
+  })
+
+  it('dead-letters events with no registered handler', async () => {
+    state.claimedRows = [makePendingRow({ eventType: 'unknown.event' })]
+
+    const result = await processOutboxEvents({})
+
+    expect(result.deadLettered).toBe(1)
+    const terminal = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(terminal).toBeDefined()
+    expect(terminal?.set.lastError).toMatch(/No handler registered/)
+  })
+})
+
+describe('processOutboxEvents — handler success and retry', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('transitions to completed on handler success and passes context to handler', async () => {
+    const handlerCalls: Array<{ payload: unknown; eventId: string; attempts: number }> = []
+    const handler = vi.fn(async (payload: unknown, ctx: { eventId: string; attempts: number }) => {
+      handlerCalls.push({ payload, eventId: ctx.eventId, attempts: ctx.attempts })
+    })
+
+    state.claimedRows = [makePendingRow()]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.processed).toBe(1)
+    expect(handlerCalls).toEqual([{ payload: { foo: 'bar' }, eventId: 'evt-1', attempts: 0 }])
+    const completeUpdate = state.updates.find((u) => u.set.status === 'completed')
+    expect(completeUpdate).toBeDefined()
+  })
+
+  it('schedules retry with exponential backoff on handler failure below maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+
+    const before = Date.now()
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.retried).toBe(1)
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    expect(retryUpdate?.set.attempts).toBe(3)
+    expect(retryUpdate?.set.lastError).toBe('transient failure')
+    // Backoff after nextAttempts=3: 1000 * 2^3 = 8000ms
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 7500)
+    expect(scheduledAt.getTime()).toBeLessThan(before + 10_000)
+  })
+
+  it('dead-letters on failure when attempts reaches maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('permanent failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 9, maxAttempts: 10 })]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.deadLettered).toBe(1)
+    const deadUpdate = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(deadUpdate).toBeDefined()
+    expect(deadUpdate?.set.attempts).toBe(10)
+    expect(deadUpdate?.set.lastError).toBe('permanent failure')
+  })
+
+  it('caps exponential backoff at 1 hour', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 20, maxAttempts: 100 })]
+
+    const before = Date.now()
+    await processOutboxEvents({ 'test.event': handler })
+
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    // 1hr = 3,600,000ms
+    expect(scheduledAt.getTime()).toBeLessThan(before + 3_600_000 + 1000)
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 3_599_000)
+  })
+})
+
+describe('processOutboxEvents — lease CAS / reaper race', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reports leaseLost when completion UPDATE affects zero rows', async () => {
+    const handler = vi.fn(async () => {
+      // "succeeds" but terminal write will fail the lease CAS
+    })
+
+    state.claimedRows = [makePendingRow()]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.processed).toBe(0)
+  })
+
+  it('reports leaseLost on retry-schedule UPDATE when row was reclaimed', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.retried).toBe(0)
+  })
+})
+
+describe('processOutboxEvents — handler timeout', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('times out a stuck handler without releasing it for overlapping retry', async () => {
+    const neverResolves = vi.fn(() => new Promise<void>(() => {}))
+
+    state.claimedRows = [makePendingRow({ attempts: 0 })]
+
+    const promise = processOutboxEvents({ 'test.event': neverResolves })
+    // Must exceed DEFAULT_HANDLER_TIMEOUT_MS (90s).
+    await vi.advanceTimersByTimeAsync(90 * 1000 + 1)
+    const result = await promise
+
+    expect(result.leaseLost).toBe(1)
+    const timeoutUpdate = state.updates.find(
+      (u) => !('status' in u.set) && 'attempts' in u.set && 'lockedAt' in u.set
+    )
+    expect(timeoutUpdate?.set.attempts).toBe(1)
+    expect(timeoutUpdate?.set.lastError).toMatch(/timed out/)
+  })
+})
+
+describe('processOutboxEvents — reaper recovery', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reaps stuck processing rows back to pending and reports count', async () => {
+    state.reapedRowIds = ['stuck-1', 'stuck-2', 'stuck-3']
+
+    const result = await processOutboxEvents({})
+
+    expect(result.reaped).toBe(3)
+    expect(result.processed).toBe(0)
+
+    // The reaper's UPDATE sets status='pending' with NO attempts / availableAt
+    // fields — that's how runHandler's retry update is distinguished from it.
+    const reaperUpdate = state.updates.find(
+      (u) => u.set.status === 'pending' && !('attempts' in u.set) && !('availableAt' in u.set)
+    )
+    expect(reaperUpdate).toBeDefined()
+    expect(reaperUpdate?.set.lockedAt).toBeNull()
+  })
+
+  it('returns zero reaped when no rows are stuck', async () => {
+    const result = await processOutboxEvents({})
+    expect(result.reaped).toBe(0)
+  })
+})

apps/sim/lib/core/outbox/service.ts

@@ -0,0 +1,580 @@
+import { db } from '@sim/db'
+import { outboxEvent } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, asc, eq, inArray, lte, sql } from 'drizzle-orm'
+
+const logger = createLogger('OutboxService')
+
+const DEFAULT_MAX_ATTEMPTS = 10
+const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes
+const MAX_BACKOFF_MS = 60 * 60 * 1000 // 1 hour
+const BASE_BACKOFF_MS = 1000 // 1 second, doubled per attempt
+// Kept below the serverless route `maxDuration` (120s) so our in-process
+// timeout fires before the platform kills the invocation and leaves the
+// row stranded in `processing` for the 10-minute reaper window. Also well
+// under `STUCK_PROCESSING_THRESHOLD_MS` so the reaper cannot steal a row
+// a worker is still actively processing.
+const DEFAULT_HANDLER_TIMEOUT_MS = 90 * 1000 // 90 seconds
+
+class OutboxHandlerTimeoutError extends Error {
+  constructor(timeoutMs: number) {
+    super(`Outbox handler timed out after ${timeoutMs}ms`)
+    this.name = 'OutboxHandlerTimeoutError'
+  }
+}
+
+/**
+ * Context passed to every outbox handler. Use `eventId` as the Stripe
+ * (or any external service) idempotency key so that handler retries
+ * collapse on the external side: a second execution of the same event
+ * lands on the same Stripe invoice id / charge id rather than creating
+ * a duplicate. The outbox lease CAS handles our DB side.
+ */
+interface OutboxEventContext {
+  eventId: string
+  eventType: string
+  /** How many times this event has been attempted (zero on first run). */
+  attempts: number
+}
+
+/**
+ * A handler invoked by the outbox worker for events of a given type.
+ * Throwing bumps `attempts` and schedules a retry via exponential
+ * backoff; a successful return transitions the event to `completed`.
+ */
+export type OutboxHandler<T = unknown> = (payload: T, context: OutboxEventContext) => Promise<void>
+
+/**
+ * Map of `eventType` → handler. Register all handlers in one place
+ * and pass them to `processOutboxEvents`.
+ */
+export type OutboxHandlerRegistry = Record<string, OutboxHandler>
+
+export interface EnqueueOptions {
+  /** Total attempts before the event moves to `dead_letter`. Default 10. */
+  maxAttempts?: number
+  /** Earliest time a worker may pick up this event. Default now. */
+  availableAt?: Date
+}
+
+export interface ProcessOutboxResult {
+  processed: number
+  retried: number
+  deadLettered: number
+  leaseLost: number
+  reaped: number
+}
+
+export type ProcessSingleOutboxResult =
+  | 'completed'
+  | 'pending'
+  | 'dead_letter'
+  | 'lease_lost'
+  | 'not_found'
+  | 'processing'
+
+/**
+ * Transactional outbox for reliable "DB write + external system" flows.
+ *
+ * Callers enqueue an event *inside* a `db.transaction` alongside the
+ * primary write; the event row commits or rolls back with the business
+ * data. A polling worker (invoked via the cron endpoint) claims pending
+ * rows with `SELECT ... FOR UPDATE SKIP LOCKED`, marks them as
+ * `processing`, runs the registered handler outside the transaction,
+ * and transitions the event to `completed` / `pending` (retry) /
+ * `dead_letter` (max attempts exceeded).
+ *
+ * Two-phase claim-then-process keeps external API calls out of DB
+ * transactions. A reaper at the top of each run reclaims `processing`
+ * rows whose worker died mid-operation (stale `lockedAt`).
+ *
+ * Enqueue must be called with a `tx` from `db.transaction` so atomicity
+ * with the primary write is preserved. `db` itself is also accepted but
+ * then the caller must guarantee the enqueue and the primary write share
+ * a transaction some other way (or none at all).
+ */
+export async function enqueueOutboxEvent<T>(
+  executor: Pick<typeof db, 'insert'>,
+  eventType: string,
+  payload: T,
+  options: EnqueueOptions = {}
+): Promise<string> {
+  const id = generateId()
+  await executor.insert(outboxEvent).values({
+    id,
+    eventType,
+    payload: payload as never,
+    maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
+    availableAt: options.availableAt ?? new Date(),
+  })
+  logger.info('Enqueued outbox event', { id, eventType })
+  return id
+}
+
+/** Cap on how many dead-lettered rows a single reconciler scan materializes. */
+const DEAD_LETTER_SCAN_LIMIT = 100
+
+/**
+ * Return events currently in `dead_letter` for the given event types (capped at
+ * `DEAD_LETTER_SCAN_LIMIT`). Used by periodic reconcilers to surface stuck work
+ * that exhausted its retries and now needs operator attention — the cap keeps a
+ * runaway backlog from materializing unboundedly into memory each run.
+ */
+export async function findDeadLetteredEvents(
+  eventTypes: string[]
+): Promise<(typeof outboxEvent.$inferSelect)[]> {
+  if (eventTypes.length === 0) return []
+  return db
+    .select()
+    .from(outboxEvent)
+    .where(and(eq(outboxEvent.status, 'dead_letter'), inArray(outboxEvent.eventType, eventTypes)))
+    .limit(DEAD_LETTER_SCAN_LIMIT)
+}
+
+/**
+ * True when an event of the given type whose JSON payload has
+ * `payload->>payloadKey === payloadValue` is still `pending` or `processing`.
+ * Lets a competing writer detect that a DB→external sync is already in flight
+ * for a subject and avoid clobbering the not-yet-pushed DB value.
+ */
+export async function hasInflightOutboxEvent(
+  eventType: string,
+  payloadKey: string,
+  payloadValue: string
+): Promise<boolean> {
+  const [row] = await db
+    .select({ id: outboxEvent.id })
+    .from(outboxEvent)
+    .where(
+      and(
+        eq(outboxEvent.eventType, eventType),
+        inArray(outboxEvent.status, ['pending', 'processing']),
+        sql`${outboxEvent.payload} ->> ${payloadKey} = ${payloadValue}`
+      )
+    )
+    .limit(1)
+  return Boolean(row)
+}
+
+/**
+ * Process one batch of outbox events. Safe to call concurrently from
+ * multiple workers — `SELECT FOR UPDATE SKIP LOCKED` serializes claims.
+ */
+export async function processOutboxEvents(
+  handlers: OutboxHandlerRegistry,
+  options: { batchSize?: number; maxRuntimeMs?: number; minRemainingMs?: number } = {}
+): Promise<ProcessOutboxResult> {
+  const batchSize = options.batchSize ?? 10
+  const deadline = options.maxRuntimeMs ? Date.now() + options.maxRuntimeMs : undefined
+  const minRemainingMs = options.minRemainingMs ?? DEFAULT_HANDLER_TIMEOUT_MS + 5000
+
+  const reaped = await reapStuckProcessingRows()
+
+  let processed = 0
+  let retried = 0
+  let deadLettered = 0
+  let leaseLost = 0
+
+  for (let i = 0; i < batchSize; i++) {
+    if (deadline && Date.now() + minRemainingMs > deadline) break
+
+    const [event] = await claimBatch(1)
+    if (!event) break
+
+    const result = await runHandler(event, handlers)
+    if (result === 'completed') processed++
+    else if (result === 'dead_letter') deadLettered++
+    else if (result === 'lease_lost') leaseLost++
+    else retried++
+  }
+
+  return { processed, retried, deadLettered, leaseLost, reaped }
+}
+
+/**
+ * Process a specific outbox event immediately after its surrounding
+ * transaction commits. Safe to race with the cron worker: the claim uses
+ * `FOR UPDATE SKIP LOCKED`, and non-pending rows are left alone.
+ */
+export async function processOutboxEventById(
+  eventId: string,
+  handlers: OutboxHandlerRegistry
+): Promise<ProcessSingleOutboxResult> {
+  const now = new Date()
+  const event = await db.transaction(async (tx) => {
+    const [row] = await tx
+      .select()
+      .from(outboxEvent)
+      .where(eq(outboxEvent.id, eventId))
+      .limit(1)
+      .for('update', { skipLocked: true })
+
+    if (!row) return null
+    if (row.status !== 'pending') return row.status as ProcessSingleOutboxResult
+    if (row.availableAt > now) return 'pending' as const
+
+    await tx
+      .update(outboxEvent)
+      .set({ status: 'processing', lockedAt: now })
+      .where(eq(outboxEvent.id, eventId))
+
+    return {
+      ...row,
+      status: 'processing' as const,
+      lockedAt: now,
+    }
+  })
+
+  if (!event) {
+    const [current] = await db
+      .select({ status: outboxEvent.status })
+      .from(outboxEvent)
+      .where(eq(outboxEvent.id, eventId))
+      .limit(1)
+    return current ? (current.status as ProcessSingleOutboxResult) : 'not_found'
+  }
+  if (typeof event === 'string') return event
+  return runHandler(event, handlers)
+}
+
+/**
+ * Reaper: move `processing` rows whose worker died (stale `lockedAt`)
+ * back to `pending` so another worker can pick them up. Without this,
+ * a SIGKILL between claim and result-write would permanently strand
+ * the row in `processing`.
+ */
+async function reapStuckProcessingRows(): Promise<number> {
+  const stuckBefore = new Date(Date.now() - STUCK_PROCESSING_THRESHOLD_MS)
+  const result = await db
+    .update(outboxEvent)
+    .set({ status: 'pending', lockedAt: null })
+    .where(and(eq(outboxEvent.status, 'processing'), lte(outboxEvent.lockedAt, stuckBefore)))
+    .returning({ id: outboxEvent.id })
+
+  if (result.length > 0) {
+    logger.warn('Reaped stuck outbox processing rows', {
+      count: result.length,
+      thresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
+    })
+  }
+  return result.length
+}
+
+/**
+ * Phase 1: claim a batch of due pending events.
+ *
+ * `SELECT ... FOR UPDATE SKIP LOCKED` atomically picks rows that no
+ * other worker is currently looking at. We then flip those rows to
+ * `processing` inside the same tx so the claim survives the lock
+ * release — the status change becomes the out-of-band mutual exclusion.
+ */
+async function claimBatch(batchSize: number): Promise<(typeof outboxEvent.$inferSelect)[]> {
+  const now = new Date()
+  return db.transaction(async (tx) => {
+    const rows = await tx
+      .select()
+      .from(outboxEvent)
+      .where(and(eq(outboxEvent.status, 'pending'), lte(outboxEvent.availableAt, now)))
+      .orderBy(asc(outboxEvent.createdAt))
+      .limit(batchSize)
+      .for('update', { skipLocked: true })
+
+    if (rows.length === 0) return []
+
+    await tx
+      .update(outboxEvent)
+      .set({ status: 'processing', lockedAt: now })
+      .where(
+        inArray(
+          outboxEvent.id,
+          rows.map((r) => r.id)
+        )
+      )
+
+    // Return rows with the claim state we just committed. `lockedAt`
+    // on this object is the authoritative lease timestamp used by the
+    // terminal-update lease CAS (see `runHandler`).
+    return rows.map((row) => ({
+      ...row,
+      status: 'processing' as const,
+      lockedAt: now,
+    }))
+  })
+}
+
+/**
+ * Phase 2: invoke the handler for a claimed event, outside any DB
+ * transaction, then transition the row to its terminal or retry state.
+ *
+ * Every terminal UPDATE is guarded by a lease CAS (`WHERE status =
+ * 'processing' AND locked_at = event.lockedAt`). This defends against
+ * the "slow handler + reaper" race: if our handler takes longer than
+ * `STUCK_PROCESSING_THRESHOLD_MS`, the reaper will have reset the row
+ * to `pending` and another worker may have reclaimed it with a fresh
+ * `locked_at`. Our stale terminal write's WHERE clause won't match —
+ * rowCount is 0 — and we log+skip instead of clobbering the new lease.
+ */
+async function runHandler(
+  event: typeof outboxEvent.$inferSelect,
+  handlers: OutboxHandlerRegistry
+): Promise<'completed' | 'pending' | 'dead_letter' | 'lease_lost'> {
+  const handler = handlers[event.eventType]
+
+  if (!handler) {
+    logger.error('No handler registered for outbox event type', {
+      eventId: event.id,
+      eventType: event.eventType,
+    })
+    await updateIfLeaseHeld(event, {
+      status: 'dead_letter',
+      lastError: `No handler registered for event type '${event.eventType}'`,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    return 'dead_letter'
+  }
+
+  try {
+    await runHandlerWithTimeout(handler, event)
+    const updated = await updateIfLeaseHeld(event, {
+      status: 'completed',
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) {
+      logger.warn('Outbox event completion skipped — lease lost (reaped + reclaimed)', {
+        eventId: event.id,
+        eventType: event.eventType,
+      })
+      return 'lease_lost'
+    }
+    logger.info('Outbox event processed', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: event.attempts + 1,
+    })
+    return 'completed'
+  } catch (error) {
+    if (error instanceof OutboxHandlerTimeoutError) {
+      return recordTimedOutAttempt(event, error.message)
+    }
+
+    const nextAttempts = event.attempts + 1
+    const isDead = nextAttempts >= event.maxAttempts
+    const errMsg = toError(error).message
+
+    if (isDead) {
+      const updated = await updateIfLeaseHeld(event, {
+        attempts: nextAttempts,
+        status: 'dead_letter',
+        lastError: errMsg,
+        processedAt: new Date(),
+        lockedAt: null,
+      })
+      if (!updated) {
+        logger.warn('Outbox event dead-letter skipped — lease lost', {
+          eventId: event.id,
+          eventType: event.eventType,
+        })
+        return 'lease_lost'
+      }
+      logger.error('Outbox event dead-lettered after max attempts', {
+        eventId: event.id,
+        eventType: event.eventType,
+        attempts: nextAttempts,
+        error: errMsg,
+      })
+      return 'dead_letter'
+    }
+
+    return scheduleRetry(event, errMsg)
+  }
+}
+
+async function recordTimedOutAttempt(
+  event: typeof outboxEvent.$inferSelect,
+  errMsg: string
+): Promise<'dead_letter' | 'lease_lost'> {
+  const nextAttempts = event.attempts + 1
+  const isDead = nextAttempts >= event.maxAttempts
+
+  if (isDead) {
+    const updated = await updateIfLeaseHeld(event, {
+      attempts: nextAttempts,
+      status: 'dead_letter',
+      lastError: errMsg,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) return 'lease_lost'
+    logger.error('Outbox event dead-lettered after handler timeout max attempts', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: nextAttempts,
+      error: errMsg,
+    })
+    return 'dead_letter'
+  }
+
+  const updated = await updateProcessingIfLeaseHeld(event, {
+    attempts: nextAttempts,
+    lastError: errMsg,
+    lockedAt: new Date(),
+  })
+  if (!updated) return 'lease_lost'
+
+  logger.warn('Outbox event handler timed out; leaving lease for stuck-row reaper', {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: nextAttempts,
+    reaperThresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
+    error: errMsg,
+  })
+  return 'lease_lost'
+}
+
+async function scheduleRetry(
+  event: typeof outboxEvent.$inferSelect,
+  errMsg: string,
+  minimumBackoffMs = 0
+): Promise<'pending' | 'dead_letter' | 'lease_lost'> {
+  const nextAttempts = event.attempts + 1
+  const isDead = nextAttempts >= event.maxAttempts
+
+  if (isDead) {
+    const updated = await updateIfLeaseHeld(event, {
+      attempts: nextAttempts,
+      status: 'dead_letter',
+      lastError: errMsg,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) {
+      logger.warn('Outbox event dead-letter skipped — lease lost', {
+        eventId: event.id,
+        eventType: event.eventType,
+      })
+      return 'lease_lost'
+    }
+    logger.error('Outbox event dead-lettered after max attempts', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: nextAttempts,
+      error: errMsg,
+    })
+    return 'dead_letter'
+  }
+
+  const backoffMs = Math.max(
+    minimumBackoffMs,
+    Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** nextAttempts)
+  )
+  const nextAvailableAt = new Date(Date.now() + backoffMs)
+  const updated = await updateIfLeaseHeld(event, {
+    attempts: nextAttempts,
+    status: 'pending',
+    lastError: errMsg,
+    availableAt: nextAvailableAt,
+    lockedAt: null,
+  })
+  if (!updated) {
+    logger.warn('Outbox event retry-schedule skipped — lease lost', {
+      eventId: event.id,
+      eventType: event.eventType,
+    })
+    return 'lease_lost'
+  }
+  logger.warn('Outbox event failed, scheduled retry', {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: nextAttempts,
+    backoffMs,
+    nextAvailableAt: nextAvailableAt.toISOString(),
+    error: errMsg,
+  })
+  return 'pending'
+}
+
+async function updateProcessingIfLeaseHeld(
+  event: typeof outboxEvent.$inferSelect,
+  patch: {
+    attempts: number
+    lastError: string
+    lockedAt: Date
+  }
+): Promise<boolean> {
+  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
+  if (event.lockedAt) {
+    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
+  }
+
+  const result = await db
+    .update(outboxEvent)
+    .set(patch)
+    .where(and(...whereClauses))
+    .returning({ id: outboxEvent.id })
+
+  return result.length > 0
+}
+
+function runHandlerWithTimeout(
+  handler: OutboxHandler,
+  event: typeof outboxEvent.$inferSelect,
+  timeoutMs: number = DEFAULT_HANDLER_TIMEOUT_MS
+): Promise<void> {
+  const context: OutboxEventContext = {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: event.attempts,
+  }
+
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new OutboxHandlerTimeoutError(timeoutMs))
+    }, timeoutMs)
+
+    handler(event.payload, context)
+      .then((value) => {
+        clearTimeout(timeout)
+        resolve(value)
+      })
+      .catch((err) => {
+        clearTimeout(timeout)
+        reject(err)
+      })
+  })
+}
+
+/**
+ * Conditional terminal update scoped to the lease acquired at claim
+ * time. Returns true if the UPDATE affected a row, false if the row's
+ * lease was revoked (reaped, reclaimed by another worker). Callers
+ * treat `false` as a "lease lost" signal and skip without retrying —
+ * the newer owner is responsible for the row now.
+ */
+async function updateIfLeaseHeld(
+  event: typeof outboxEvent.$inferSelect,
+  patch: {
+    status: 'completed' | 'pending' | 'dead_letter'
+    attempts?: number
+    lastError?: string | null
+    availableAt?: Date
+    lockedAt: Date | null
+    processedAt?: Date | null
+  }
+): Promise<boolean> {
+  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
+  if (event.lockedAt) {
+    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
+  }
+
+  const result = await db
+    .update(outboxEvent)
+    .set(patch)
+    .where(and(...whereClauses))
+    .returning({ id: outboxEvent.id })
+
+  return result.length > 0
+}