chore: import upstream snapshot with attribution

apps/sim/lib/core/outbox/service.test.ts

@@ -0,0 +1,389 @@
+/**
+ * @vitest-environment node
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+type OutboxRow = {
+  id: string
+  eventType: string
+  payload: unknown
+  status: 'pending' | 'processing' | 'completed' | 'dead_letter'
+  attempts: number
+  maxAttempts: number
+  availableAt: Date
+  lockedAt: Date | null
+  lastError: string | null
+  createdAt: Date
+  processedAt: Date | null
+}
+
+// Hoisted mock state — all tests manipulate these directly.
+const { state, mockDb } = vi.hoisted(() => {
+  const state = {
+    // Rows returned from the FOR UPDATE SKIP LOCKED select in claimBatch.
+    claimedRows: [] as OutboxRow[],
+    // Whether the terminal update (lease CAS) should report a match.
+    leaseHeld: true,
+    // IDs the reaper's UPDATE should return (simulates stuck `processing` rows).
+    reapedRowIds: [] as string[],
+    // Everything written (for assertions).
+    inserts: [] as Array<{ values: unknown }>,
+    updates: [] as Array<{ set: Record<string, unknown>; where?: unknown }>,
+  }
+
+  const makeUpdateChain = () => {
+    const row: { set: Record<string, unknown>; where?: unknown } = { set: {} }
+    const chain: Record<string, unknown> = {}
+    chain.set = vi.fn((s: Record<string, unknown>) => {
+      row.set = s
+      return chain
+    })
+    chain.where = vi.fn((w: unknown) => {
+      row.where = w
+      state.updates.push(row)
+      return chain
+    })
+    chain.returning = vi.fn(async () => {
+      // Terminal UPDATE (lease CAS): has `attempts` + `availableAt`
+      // on retry, or explicit completed/dead_letter. Reaper path sets
+      // status='pending' without attempts/availableAt.
+      const isReaperUpdate =
+        row.set.status === 'pending' && !('attempts' in row.set) && !('availableAt' in row.set)
+
+      if (isReaperUpdate) {
+        return state.reapedRowIds.map((id) => ({ id }))
+      }
+
+      if (
+        row.set.status === 'completed' ||
+        row.set.status === 'dead_letter' ||
+        (row.set.status === 'pending' && 'attempts' in row.set && 'availableAt' in row.set) ||
+        (!('status' in row.set) && 'attempts' in row.set && 'lockedAt' in row.set)
+      ) {
+        return state.leaseHeld ? [{ id: 'evt-1' }] : []
+      }
+
+      return []
+    })
+    return chain
+  }
+
+  const makeSelectChain = () => {
+    const chain: Record<string, unknown> = {}
+    const self = () => chain
+    chain.from = vi.fn(self)
+    chain.where = vi.fn(self)
+    chain.orderBy = vi.fn(self)
+    chain.limit = vi.fn(self)
+    chain.for = vi.fn(async () => state.claimedRows.splice(0, 1))
+    return chain
+  }
+
+  const mockDb = {
+    insert: vi.fn(() => {
+      const chain: Record<string, unknown> = {}
+      chain.values = vi.fn(async (v: unknown) => {
+        state.inserts.push({ values: v })
+      })
+      return chain
+    }),
+    update: vi.fn(() => makeUpdateChain()),
+    select: vi.fn(() => makeSelectChain()),
+    transaction: vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn(mockDb)),
+  }
+
+  return { state, mockDb }
+})
+
+vi.mock('@sim/db', () => ({ db: mockDb }))
+
+vi.mock('@sim/db/schema', () => ({
+  outboxEvent: {
+    id: 'outbox_event.id',
+    eventType: 'outbox_event.event_type',
+    payload: 'outbox_event.payload',
+    status: 'outbox_event.status',
+    attempts: 'outbox_event.attempts',
+    maxAttempts: 'outbox_event.max_attempts',
+    availableAt: 'outbox_event.available_at',
+    lockedAt: 'outbox_event.locked_at',
+    lastError: 'outbox_event.last_error',
+    createdAt: 'outbox_event.created_at',
+    processedAt: 'outbox_event.processed_at',
+    $inferSelect: {} as OutboxRow,
+  },
+}))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }),
+}))
+
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn((...args) => ({ _op: 'and', args })),
+  asc: vi.fn((col) => ({ _op: 'asc', col })),
+  eq: vi.fn((col, val) => ({ _op: 'eq', col, val })),
+  inArray: vi.fn((col, vals) => ({ _op: 'inArray', col, vals })),
+  lte: vi.fn((col, val) => ({ _op: 'lte', col, val })),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn(() => 'test-event-id'),
+}))
+
+import { enqueueOutboxEvent, processOutboxEvents } from './service'
+
+function makePendingRow(overrides: Partial<OutboxRow> = {}): OutboxRow {
+  return {
+    id: 'evt-1',
+    eventType: 'test.event',
+    payload: { foo: 'bar' },
+    status: 'pending',
+    attempts: 0,
+    maxAttempts: 10,
+    availableAt: new Date(Date.now() - 1000),
+    lockedAt: null,
+    lastError: null,
+    createdAt: new Date(Date.now() - 5000),
+    processedAt: null,
+    ...overrides,
+  }
+}
+
+function resetState() {
+  state.claimedRows = []
+  state.leaseHeld = true
+  state.reapedRowIds = []
+  state.inserts.length = 0
+  state.updates.length = 0
+}
+
+describe('enqueueOutboxEvent', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('inserts a row with the given event type and payload', async () => {
+    const id = await enqueueOutboxEvent(mockDb, 'test.event', { foo: 'bar' })
+    expect(id).toBe('test-event-id')
+    expect(state.inserts[0].values).toMatchObject({
+      id: 'test-event-id',
+      eventType: 'test.event',
+      payload: { foo: 'bar' },
+      maxAttempts: 10,
+    })
+  })
+
+  it('respects maxAttempts override', async () => {
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { maxAttempts: 3 })
+    expect(state.inserts[0].values).toMatchObject({ maxAttempts: 3 })
+  })
+
+  it('respects availableAt override for delayed processing', async () => {
+    const future = new Date(Date.now() + 60_000)
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { availableAt: future })
+    expect((state.inserts[0].values as { availableAt: Date }).availableAt).toBe(future)
+  })
+})
+
+describe('processOutboxEvents — empty / no handler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('returns zero counts when no events are due', async () => {
+    const result = await processOutboxEvents({})
+    expect(result).toEqual({
+      processed: 0,
+      retried: 0,
+      deadLettered: 0,
+      leaseLost: 0,
+      reaped: 0,
+    })
+  })
+
+  it('dead-letters events with no registered handler', async () => {
+    state.claimedRows = [makePendingRow({ eventType: 'unknown.event' })]
+
+    const result = await processOutboxEvents({})
+
+    expect(result.deadLettered).toBe(1)
+    const terminal = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(terminal).toBeDefined()
+    expect(terminal?.set.lastError).toMatch(/No handler registered/)
+  })
+})
+
+describe('processOutboxEvents — handler success and retry', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('transitions to completed on handler success and passes context to handler', async () => {
+    const handlerCalls: Array<{ payload: unknown; eventId: string; attempts: number }> = []
+    const handler = vi.fn(async (payload: unknown, ctx: { eventId: string; attempts: number }) => {
+      handlerCalls.push({ payload, eventId: ctx.eventId, attempts: ctx.attempts })
+    })
+
+    state.claimedRows = [makePendingRow()]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.processed).toBe(1)
+    expect(handlerCalls).toEqual([{ payload: { foo: 'bar' }, eventId: 'evt-1', attempts: 0 }])
+    const completeUpdate = state.updates.find((u) => u.set.status === 'completed')
+    expect(completeUpdate).toBeDefined()
+  })
+
+  it('schedules retry with exponential backoff on handler failure below maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+
+    const before = Date.now()
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.retried).toBe(1)
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    expect(retryUpdate?.set.attempts).toBe(3)
+    expect(retryUpdate?.set.lastError).toBe('transient failure')
+    // Backoff after nextAttempts=3: 1000 * 2^3 = 8000ms
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 7500)
+    expect(scheduledAt.getTime()).toBeLessThan(before + 10_000)
+  })
+
+  it('dead-letters on failure when attempts reaches maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('permanent failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 9, maxAttempts: 10 })]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.deadLettered).toBe(1)
+    const deadUpdate = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(deadUpdate).toBeDefined()
+    expect(deadUpdate?.set.attempts).toBe(10)
+    expect(deadUpdate?.set.lastError).toBe('permanent failure')
+  })
+
+  it('caps exponential backoff at 1 hour', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 20, maxAttempts: 100 })]
+
+    const before = Date.now()
+    await processOutboxEvents({ 'test.event': handler })
+
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    // 1hr = 3,600,000ms
+    expect(scheduledAt.getTime()).toBeLessThan(before + 3_600_000 + 1000)
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 3_599_000)
+  })
+})
+
+describe('processOutboxEvents — lease CAS / reaper race', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reports leaseLost when completion UPDATE affects zero rows', async () => {
+    const handler = vi.fn(async () => {
+      // "succeeds" but terminal write will fail the lease CAS
+    })
+
+    state.claimedRows = [makePendingRow()]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.processed).toBe(0)
+  })
+
+  it('reports leaseLost on retry-schedule UPDATE when row was reclaimed', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.retried).toBe(0)
+  })
+})
+
+describe('processOutboxEvents — handler timeout', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('times out a stuck handler without releasing it for overlapping retry', async () => {
+    const neverResolves = vi.fn(() => new Promise<void>(() => {}))
+
+    state.claimedRows = [makePendingRow({ attempts: 0 })]
+
+    const promise = processOutboxEvents({ 'test.event': neverResolves })
+    // Must exceed DEFAULT_HANDLER_TIMEOUT_MS (90s).
+    await vi.advanceTimersByTimeAsync(90 * 1000 + 1)
+    const result = await promise
+
+    expect(result.leaseLost).toBe(1)
+    const timeoutUpdate = state.updates.find(
+      (u) => !('status' in u.set) && 'attempts' in u.set && 'lockedAt' in u.set
+    )
+    expect(timeoutUpdate?.set.attempts).toBe(1)
+    expect(timeoutUpdate?.set.lastError).toMatch(/timed out/)
+  })
+})
+
+describe('processOutboxEvents — reaper recovery', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reaps stuck processing rows back to pending and reports count', async () => {
+    state.reapedRowIds = ['stuck-1', 'stuck-2', 'stuck-3']
+
+    const result = await processOutboxEvents({})
+
+    expect(result.reaped).toBe(3)
+    expect(result.processed).toBe(0)
+
+    // The reaper's UPDATE sets status='pending' with NO attempts / availableAt
+    // fields — that's how runHandler's retry update is distinguished from it.
+    const reaperUpdate = state.updates.find(
+      (u) => u.set.status === 'pending' && !('attempts' in u.set) && !('availableAt' in u.set)
+    )
+    expect(reaperUpdate).toBeDefined()
+    expect(reaperUpdate?.set.lockedAt).toBeNull()
+  })
+
+  it('returns zero reaped when no rows are stuck', async () => {
+    const result = await processOutboxEvents({})
+    expect(result.reaped).toBe(0)
+  })
+})