simstudioai--sim/apps/sim/lib/core/outbox/service.ts

import { db } from '@sim/db'
import { outboxEvent } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { and, asc, eq, inArray, lte, sql } from 'drizzle-orm'

const logger = createLogger('OutboxService')

const DEFAULT_MAX_ATTEMPTS = 10
const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes
const MAX_BACKOFF_MS = 60 * 60 * 1000 // 1 hour
const BASE_BACKOFF_MS = 1000 // 1 second, doubled per attempt
// Kept below the serverless route `maxDuration` (120s) so our in-process
// timeout fires before the platform kills the invocation and leaves the
// row stranded in `processing` for the 10-minute reaper window. Also well
// under `STUCK_PROCESSING_THRESHOLD_MS` so the reaper cannot steal a row
// a worker is still actively processing.
const DEFAULT_HANDLER_TIMEOUT_MS = 90 * 1000 // 90 seconds

class OutboxHandlerTimeoutError extends Error {
  constructor(timeoutMs: number) {
    super(`Outbox handler timed out after ${timeoutMs}ms`)
    this.name = 'OutboxHandlerTimeoutError'
  }
}

/**
 * Context passed to every outbox handler. Use `eventId` as the Stripe
 * (or any external service) idempotency key so that handler retries
 * collapse on the external side: a second execution of the same event
 * lands on the same Stripe invoice id / charge id rather than creating
 * a duplicate. The outbox lease CAS handles our DB side.
 */
interface OutboxEventContext {
  eventId: string
  eventType: string
  /** How many times this event has been attempted (zero on first run). */
  attempts: number
}

/**
 * A handler invoked by the outbox worker for events of a given type.
 * Throwing bumps `attempts` and schedules a retry via exponential
 * backoff; a successful return transitions the event to `completed`.
 */
export type OutboxHandler<T = unknown> = (payload: T, context: OutboxEventContext) => Promise<void>

/**
 * Map of `eventType` → handler. Register all handlers in one place
 * and pass them to `processOutboxEvents`.
 */
export type OutboxHandlerRegistry = Record<string, OutboxHandler>

export interface EnqueueOptions {
  /** Total attempts before the event moves to `dead_letter`. Default 10. */
  maxAttempts?: number
  /** Earliest time a worker may pick up this event. Default now. */
  availableAt?: Date
}

export interface ProcessOutboxResult {
  processed: number
  retried: number
  deadLettered: number
  leaseLost: number
  reaped: number
}

export type ProcessSingleOutboxResult =
  | 'completed'
  | 'pending'
  | 'dead_letter'
  | 'lease_lost'
  | 'not_found'
  | 'processing'

/**
 * Transactional outbox for reliable "DB write + external system" flows.
 *
 * Callers enqueue an event *inside* a `db.transaction` alongside the
 * primary write; the event row commits or rolls back with the business
 * data. A polling worker (invoked via the cron endpoint) claims pending
 * rows with `SELECT ... FOR UPDATE SKIP LOCKED`, marks them as
 * `processing`, runs the registered handler outside the transaction,
 * and transitions the event to `completed` / `pending` (retry) /
 * `dead_letter` (max attempts exceeded).
 *
 * Two-phase claim-then-process keeps external API calls out of DB
 * transactions. A reaper at the top of each run reclaims `processing`
 * rows whose worker died mid-operation (stale `lockedAt`).
 *
 * Enqueue must be called with a `tx` from `db.transaction` so atomicity
 * with the primary write is preserved. `db` itself is also accepted but
 * then the caller must guarantee the enqueue and the primary write share
 * a transaction some other way (or none at all).
 */
export async function enqueueOutboxEvent<T>(
  executor: Pick<typeof db, 'insert'>,
  eventType: string,
  payload: T,
  options: EnqueueOptions = {}
): Promise<string> {
  const id = generateId()
  await executor.insert(outboxEvent).values({
    id,
    eventType,
    payload: payload as never,
    maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
    availableAt: options.availableAt ?? new Date(),
  })
  logger.info('Enqueued outbox event', { id, eventType })
  return id
}

/** Cap on how many dead-lettered rows a single reconciler scan materializes. */
const DEAD_LETTER_SCAN_LIMIT = 100

/**
 * Return events currently in `dead_letter` for the given event types (capped at
 * `DEAD_LETTER_SCAN_LIMIT`). Used by periodic reconcilers to surface stuck work
 * that exhausted its retries and now needs operator attention — the cap keeps a
 * runaway backlog from materializing unboundedly into memory each run.
 */
export async function findDeadLetteredEvents(
  eventTypes: string[]
): Promise<(typeof outboxEvent.$inferSelect)[]> {
  if (eventTypes.length === 0) return []
  return db
    .select()
    .from(outboxEvent)
    .where(and(eq(outboxEvent.status, 'dead_letter'), inArray(outboxEvent.eventType, eventTypes)))
    .limit(DEAD_LETTER_SCAN_LIMIT)
}

/**
 * True when an event of the given type whose JSON payload has
 * `payload->>payloadKey === payloadValue` is still `pending` or `processing`.
 * Lets a competing writer detect that a DB→external sync is already in flight
 * for a subject and avoid clobbering the not-yet-pushed DB value.
 */
export async function hasInflightOutboxEvent(
  eventType: string,
  payloadKey: string,
  payloadValue: string
): Promise<boolean> {
  const [row] = await db
    .select({ id: outboxEvent.id })
    .from(outboxEvent)
    .where(
      and(
        eq(outboxEvent.eventType, eventType),
        inArray(outboxEvent.status, ['pending', 'processing']),
        sql`${outboxEvent.payload} ->> ${payloadKey} = ${payloadValue}`
      )
    )
    .limit(1)
  return Boolean(row)
}

/**
 * Process one batch of outbox events. Safe to call concurrently from
 * multiple workers — `SELECT FOR UPDATE SKIP LOCKED` serializes claims.
 */
export async function processOutboxEvents(
  handlers: OutboxHandlerRegistry,
  options: { batchSize?: number; maxRuntimeMs?: number; minRemainingMs?: number } = {}
): Promise<ProcessOutboxResult> {
  const batchSize = options.batchSize ?? 10
  const deadline = options.maxRuntimeMs ? Date.now() + options.maxRuntimeMs : undefined
  const minRemainingMs = options.minRemainingMs ?? DEFAULT_HANDLER_TIMEOUT_MS + 5000

  const reaped = await reapStuckProcessingRows()

  let processed = 0
  let retried = 0
  let deadLettered = 0
  let leaseLost = 0

  for (let i = 0; i < batchSize; i++) {
    if (deadline && Date.now() + minRemainingMs > deadline) break

    const [event] = await claimBatch(1)
    if (!event) break

    const result = await runHandler(event, handlers)
    if (result === 'completed') processed++
    else if (result === 'dead_letter') deadLettered++
    else if (result === 'lease_lost') leaseLost++
    else retried++
  }

  return { processed, retried, deadLettered, leaseLost, reaped }
}

/**
 * Process a specific outbox event immediately after its surrounding
 * transaction commits. Safe to race with the cron worker: the claim uses
 * `FOR UPDATE SKIP LOCKED`, and non-pending rows are left alone.
 */
export async function processOutboxEventById(
  eventId: string,
  handlers: OutboxHandlerRegistry
): Promise<ProcessSingleOutboxResult> {
  const now = new Date()
  const event = await db.transaction(async (tx) => {
    const [row] = await tx
      .select()
      .from(outboxEvent)
      .where(eq(outboxEvent.id, eventId))
      .limit(1)
      .for('update', { skipLocked: true })

    if (!row) return null
    if (row.status !== 'pending') return row.status as ProcessSingleOutboxResult
    if (row.availableAt > now) return 'pending' as const

    await tx
      .update(outboxEvent)
      .set({ status: 'processing', lockedAt: now })
      .where(eq(outboxEvent.id, eventId))

    return {
      ...row,
      status: 'processing' as const,
      lockedAt: now,
    }
  })

  if (!event) {
    const [current] = await db
      .select({ status: outboxEvent.status })
      .from(outboxEvent)
      .where(eq(outboxEvent.id, eventId))
      .limit(1)
    return current ? (current.status as ProcessSingleOutboxResult) : 'not_found'
  }
  if (typeof event === 'string') return event
  return runHandler(event, handlers)
}

/**
 * Reaper: move `processing` rows whose worker died (stale `lockedAt`)
 * back to `pending` so another worker can pick them up. Without this,
 * a SIGKILL between claim and result-write would permanently strand
 * the row in `processing`.
 */
async function reapStuckProcessingRows(): Promise<number> {
  const stuckBefore = new Date(Date.now() - STUCK_PROCESSING_THRESHOLD_MS)
  const result = await db
    .update(outboxEvent)
    .set({ status: 'pending', lockedAt: null })
    .where(and(eq(outboxEvent.status, 'processing'), lte(outboxEvent.lockedAt, stuckBefore)))
    .returning({ id: outboxEvent.id })

  if (result.length > 0) {
    logger.warn('Reaped stuck outbox processing rows', {
      count: result.length,
      thresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
    })
  }
  return result.length
}

/**
 * Phase 1: claim a batch of due pending events.
 *
 * `SELECT ... FOR UPDATE SKIP LOCKED` atomically picks rows that no
 * other worker is currently looking at. We then flip those rows to
 * `processing` inside the same tx so the claim survives the lock
 * release — the status change becomes the out-of-band mutual exclusion.
 */
async function claimBatch(batchSize: number): Promise<(typeof outboxEvent.$inferSelect)[]> {
  const now = new Date()
  return db.transaction(async (tx) => {
    const rows = await tx
      .select()
      .from(outboxEvent)
      .where(and(eq(outboxEvent.status, 'pending'), lte(outboxEvent.availableAt, now)))
      .orderBy(asc(outboxEvent.createdAt))
      .limit(batchSize)
      .for('update', { skipLocked: true })

    if (rows.length === 0) return []

    await tx
      .update(outboxEvent)
      .set({ status: 'processing', lockedAt: now })
      .where(
        inArray(
          outboxEvent.id,
          rows.map((r) => r.id)
        )
      )

    // Return rows with the claim state we just committed. `lockedAt`
    // on this object is the authoritative lease timestamp used by the
    // terminal-update lease CAS (see `runHandler`).
    return rows.map((row) => ({
      ...row,
      status: 'processing' as const,
      lockedAt: now,
    }))
  })
}

/**
 * Phase 2: invoke the handler for a claimed event, outside any DB
 * transaction, then transition the row to its terminal or retry state.
 *
 * Every terminal UPDATE is guarded by a lease CAS (`WHERE status =
 * 'processing' AND locked_at = event.lockedAt`). This defends against
 * the "slow handler + reaper" race: if our handler takes longer than
 * `STUCK_PROCESSING_THRESHOLD_MS`, the reaper will have reset the row
 * to `pending` and another worker may have reclaimed it with a fresh
 * `locked_at`. Our stale terminal write's WHERE clause won't match —
 * rowCount is 0 — and we log+skip instead of clobbering the new lease.
 */
async function runHandler(
  event: typeof outboxEvent.$inferSelect,
  handlers: OutboxHandlerRegistry
): Promise<'completed' | 'pending' | 'dead_letter' | 'lease_lost'> {
  const handler = handlers[event.eventType]

  if (!handler) {
    logger.error('No handler registered for outbox event type', {
      eventId: event.id,
      eventType: event.eventType,
    })
    await updateIfLeaseHeld(event, {
      status: 'dead_letter',
      lastError: `No handler registered for event type '${event.eventType}'`,
      processedAt: new Date(),
      lockedAt: null,
    })
    return 'dead_letter'
  }

  try {
    await runHandlerWithTimeout(handler, event)
    const updated = await updateIfLeaseHeld(event, {
      status: 'completed',
      processedAt: new Date(),
      lockedAt: null,
    })
    if (!updated) {
      logger.warn('Outbox event completion skipped — lease lost (reaped + reclaimed)', {
        eventId: event.id,
        eventType: event.eventType,
      })
      return 'lease_lost'
    }
    logger.info('Outbox event processed', {
      eventId: event.id,
      eventType: event.eventType,
      attempts: event.attempts + 1,
    })
    return 'completed'
  } catch (error) {
    if (error instanceof OutboxHandlerTimeoutError) {
      return recordTimedOutAttempt(event, error.message)
    }

    const nextAttempts = event.attempts + 1
    const isDead = nextAttempts >= event.maxAttempts
    const errMsg = toError(error).message

    if (isDead) {
      const updated = await updateIfLeaseHeld(event, {
        attempts: nextAttempts,
        status: 'dead_letter',
        lastError: errMsg,
        processedAt: new Date(),
        lockedAt: null,
      })
      if (!updated) {
        logger.warn('Outbox event dead-letter skipped — lease lost', {
          eventId: event.id,
          eventType: event.eventType,
        })
        return 'lease_lost'
      }
      logger.error('Outbox event dead-lettered after max attempts', {
        eventId: event.id,
        eventType: event.eventType,
        attempts: nextAttempts,
        error: errMsg,
      })
      return 'dead_letter'
    }

    return scheduleRetry(event, errMsg)
  }
}

async function recordTimedOutAttempt(
  event: typeof outboxEvent.$inferSelect,
  errMsg: string
): Promise<'dead_letter' | 'lease_lost'> {
  const nextAttempts = event.attempts + 1
  const isDead = nextAttempts >= event.maxAttempts

  if (isDead) {
    const updated = await updateIfLeaseHeld(event, {
      attempts: nextAttempts,
      status: 'dead_letter',
      lastError: errMsg,
      processedAt: new Date(),
      lockedAt: null,
    })
    if (!updated) return 'lease_lost'
    logger.error('Outbox event dead-lettered after handler timeout max attempts', {
      eventId: event.id,
      eventType: event.eventType,
      attempts: nextAttempts,
      error: errMsg,
    })
    return 'dead_letter'
  }

  const updated = await updateProcessingIfLeaseHeld(event, {
    attempts: nextAttempts,
    lastError: errMsg,
    lockedAt: new Date(),
  })
  if (!updated) return 'lease_lost'

  logger.warn('Outbox event handler timed out; leaving lease for stuck-row reaper', {
    eventId: event.id,
    eventType: event.eventType,
    attempts: nextAttempts,
    reaperThresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
    error: errMsg,
  })
  return 'lease_lost'
}

async function scheduleRetry(
  event: typeof outboxEvent.$inferSelect,
  errMsg: string,
  minimumBackoffMs = 0
): Promise<'pending' | 'dead_letter' | 'lease_lost'> {
  const nextAttempts = event.attempts + 1
  const isDead = nextAttempts >= event.maxAttempts

  if (isDead) {
    const updated = await updateIfLeaseHeld(event, {
      attempts: nextAttempts,
      status: 'dead_letter',
      lastError: errMsg,
      processedAt: new Date(),
      lockedAt: null,
    })
    if (!updated) {
      logger.warn('Outbox event dead-letter skipped — lease lost', {
        eventId: event.id,
        eventType: event.eventType,
      })
      return 'lease_lost'
    }
    logger.error('Outbox event dead-lettered after max attempts', {
      eventId: event.id,
      eventType: event.eventType,
      attempts: nextAttempts,
      error: errMsg,
    })
    return 'dead_letter'
  }

  const backoffMs = Math.max(
    minimumBackoffMs,
    Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** nextAttempts)
  )
  const nextAvailableAt = new Date(Date.now() + backoffMs)
  const updated = await updateIfLeaseHeld(event, {
    attempts: nextAttempts,
    status: 'pending',
    lastError: errMsg,
    availableAt: nextAvailableAt,
    lockedAt: null,
  })
  if (!updated) {
    logger.warn('Outbox event retry-schedule skipped — lease lost', {
      eventId: event.id,
      eventType: event.eventType,
    })
    return 'lease_lost'
  }
  logger.warn('Outbox event failed, scheduled retry', {
    eventId: event.id,
    eventType: event.eventType,
    attempts: nextAttempts,
    backoffMs,
    nextAvailableAt: nextAvailableAt.toISOString(),
    error: errMsg,
  })
  return 'pending'
}

async function updateProcessingIfLeaseHeld(
  event: typeof outboxEvent.$inferSelect,
  patch: {
    attempts: number
    lastError: string
    lockedAt: Date
  }
): Promise<boolean> {
  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
  if (event.lockedAt) {
    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
  }

  const result = await db
    .update(outboxEvent)
    .set(patch)
    .where(and(...whereClauses))
    .returning({ id: outboxEvent.id })

  return result.length > 0
}

function runHandlerWithTimeout(
  handler: OutboxHandler,
  event: typeof outboxEvent.$inferSelect,
  timeoutMs: number = DEFAULT_HANDLER_TIMEOUT_MS
): Promise<void> {
  const context: OutboxEventContext = {
    eventId: event.id,
    eventType: event.eventType,
    attempts: event.attempts,
  }

  return new Promise((resolve, reject) => {
    const timeout = setTimeout(() => {
      reject(new OutboxHandlerTimeoutError(timeoutMs))
    }, timeoutMs)

    handler(event.payload, context)
      .then((value) => {
        clearTimeout(timeout)
        resolve(value)
      })
      .catch((err) => {
        clearTimeout(timeout)
        reject(err)
      })
  })
}

/**
 * Conditional terminal update scoped to the lease acquired at claim
 * time. Returns true if the UPDATE affected a row, false if the row's
 * lease was revoked (reaped, reclaimed by another worker). Callers
 * treat `false` as a "lease lost" signal and skip without retrying —
 * the newer owner is responsible for the row now.
 */
async function updateIfLeaseHeld(
  event: typeof outboxEvent.$inferSelect,
  patch: {
    status: 'completed' | 'pending' | 'dead_letter'
    attempts?: number
    lastError?: string | null
    availableAt?: Date
    lockedAt: Date | null
    processedAt?: Date | null
  }
): Promise<boolean> {
  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
  if (event.lockedAt) {
    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
  }

  const result = await db
    .update(outboxEvent)
    .set(patch)
    .where(and(...whereClauses))
    .returning({ id: outboxEvent.id })

  return result.length > 0
}