chore: import upstream snapshot with attribution

apps/sim/lib/table/events.ts

@@ -0,0 +1,367 @@
+/**
+ * Per-table event buffer for live cell-state updates.
+ *
+ * The grid subscribes to a per-table SSE stream and patches its React Query
+ * cache as events arrive. This buffer is the durable mid-tier between the
+ * cell-write paths (`writeWorkflowGroupState`, `cancelWorkflowGroupRuns`) and
+ * the SSE consumers — every status transition appends here with a monotonic
+ * eventId; SSE clients resume on reconnect via `?from=<lastEventId>` and the
+ * server replays from this buffer.
+ *
+ * Modeled after `apps/sim/lib/execution/event-buffer.ts` but stripped of
+ * complexity tables don't need: no per-execution lifecycle, no id reservation
+ * batching, no write-queue serialization. Tables are always-on; cell writes
+ * are sparse and independent.
+ */
+
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { env } from '@/lib/core/config/env'
+import { getRedisClient } from '@/lib/core/config/redis'
+
+const logger = createLogger('TableEventBuffer')
+
+const REDIS_PREFIX = 'table:stream:'
+export const TABLE_EVENT_TTL_SECONDS = 60 * 60 // 1 hour
+export const TABLE_EVENT_CAP = 5000
+/** Max events returned by a single read; the SSE route drains in chunks. */
+export const TABLE_EVENT_READ_CHUNK = 500
+
+/**
+ * Atomic append: INCR the seq counter to mint a new eventId, build the entry
+ * JSON inline, ZADD it, refresh TTL on events + seq + meta, trim to cap, then
+ * write the resulting earliestEventId to meta. Single round-trip per event.
+ * Without atomicity a slow reader could observe the trim before the meta
+ * update and miss the prune signal.
+ *
+ * KEYS: [events, seq, meta]
+ * ARGV: [ttlSec, cap, updatedAtIso, entryPrefix, entrySuffix]
+ *   The new eventId is spliced between prefix/suffix to form the entry JSON.
+ *   Returns the new eventId.
+ */
+const APPEND_EVENT_SCRIPT = `
+local eventId = redis.call('INCR', KEYS[2])
+local entry = ARGV[4] .. eventId .. ARGV[5]
+redis.call('ZADD', KEYS[1], eventId, entry)
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1]))
+redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1]))
+redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1)
+local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+if oldest[2] then
+  redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3])
+  redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1]))
+end
+return eventId
+`
+
+function getEventsKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:events`
+}
+
+function getSeqKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:seq`
+}
+
+function getMetaKey(tableId: string) {
+  return `${REDIS_PREFIX}${tableId}:meta`
+}
+
+export type TableCellStatus = 'pending' | 'queued' | 'running' | 'completed' | 'cancelled' | 'error'
+
+export type TableDispatchStatus = 'pending' | 'dispatching' | 'complete' | 'cancelled'
+
+export type TableEvent =
+  | {
+      kind: 'cell'
+      tableId: string
+      rowId: string
+      groupId: string
+      status: TableCellStatus
+      executionId: string | null
+      jobId: string | null
+      error: string | null
+      /**
+       * Present when this transition wrote new output values; absent on
+       * pure-status transitions (queued, running, cancelled). The publisher
+       * already has these in hand from the same updateRow call that wrote DB.
+       */
+      outputs?: Record<string, unknown>
+      /**
+       * Block-level metadata the renderer reads to distinguish "running" (some
+       * block actively executing) from "pending-upstream" (run started but this
+       * column's block hasn't fired yet). The worker fills these on partial
+       * writes; without them the cell stays on the amber Pending pill.
+       */
+      runningBlockIds?: string[]
+      blockErrors?: Record<string, string>
+    }
+  | {
+      /** Dispatcher status signal emitted by `dispatcherStep` and the cancel
+       *  path. Drives the client-side "about to run" overlay for rows the
+       *  dispatcher hasn't reached yet. `scope` + `cursor` + `mode` +
+       *  `isManualRun` are carried on every transition so the client can
+       *  upsert without refetching the dispatches list. */
+      kind: 'dispatch'
+      tableId: string
+      dispatchId: string
+      status: TableDispatchStatus
+      scope?: { groupIds: string[]; rowIds?: string[] }
+      cursor?: number
+      mode?: 'all' | 'incomplete' | 'new'
+      isManualRun?: boolean
+      /** Present when the run is capped — carried so the client overlay can
+       *  skip capped dispatches (see `resolveCellExec`). */
+      limit?: { type: 'rows'; max: number }
+    }
+  | {
+      /** Async background-job progress. Import and delete workers emit `running`
+       *  ticks as batches commit, then a terminal `ready`/`failed`/`canceled`.
+       *  `type` discriminates the work. The client reveals hidden import rows on
+       *  `ready`, and on a delete `failed`/`canceled` restores optimistically
+       *  hidden rows. See `import-runner.ts` / `delete-runner.ts`. */
+      kind: 'job'
+      tableId: string
+      jobId: string
+      type: 'import' | 'delete' | 'export' | 'backfill' | 'update'
+      status: 'running' | 'ready' | 'failed' | 'canceled'
+      /** Rows processed so far (running) or in total (ready). */
+      progress?: number
+      /** Byte-based completion percent (0–100) — exact and monotonic, for the determinate bar. */
+      percent?: number
+      error?: string
+    }
+  | {
+      /** A dispatch was stopped because the billed account is over its usage
+       *  limit. The client surfaces an upgrade prompt and redirects to billing.
+       *  The dispatch is halted via `markDispatchComplete` and the blocked
+       *  cells' pre-stamps are cleared so they revert to un-run. `dispatchId`
+       *  is absent for cascade/auto-fire payloads with no owning dispatch. */
+      kind: 'usageLimitReached'
+      tableId: string
+      dispatchId?: string
+      message: string
+    }
+
+export interface TableEventEntry {
+  eventId: number
+  tableId: string
+  event: TableEvent
+}
+
+export type TableEventsReadResult =
+  | { status: 'ok'; events: TableEventEntry[] }
+  | { status: 'pruned'; earliestEventId: number | undefined }
+  | { status: 'unavailable'; error: string }
+
+/** In-memory fallback for dev/tests when Redis isn't configured. */
+interface MemoryTableStream {
+  events: TableEventEntry[]
+  earliestEventId?: number
+  nextEventId: number
+  expiresAt: number
+}
+
+const memoryTableStreams = new Map<string, MemoryTableStream>()
+
+function canUseMemoryBuffer(): boolean {
+  return typeof window === 'undefined' && !env.REDIS_URL
+}
+
+function pruneExpiredMemoryStreams(now = Date.now()): void {
+  for (const [tableId, stream] of memoryTableStreams) {
+    if (stream.expiresAt <= now) {
+      memoryTableStreams.delete(tableId)
+    }
+  }
+}
+
+function getMemoryStream(tableId: string): MemoryTableStream {
+  pruneExpiredMemoryStreams()
+  let stream = memoryTableStreams.get(tableId)
+  if (!stream) {
+    stream = {
+      events: [],
+      nextEventId: 1,
+      expiresAt: Date.now() + TABLE_EVENT_TTL_SECONDS * 1000,
+    }
+    memoryTableStreams.set(tableId, stream)
+  }
+  return stream
+}
+
+function appendMemory(event: TableEvent): TableEventEntry {
+  const stream = getMemoryStream(event.tableId)
+  const entry: TableEventEntry = {
+    eventId: stream.nextEventId++,
+    tableId: event.tableId,
+    event,
+  }
+  stream.events.push(entry)
+  if (stream.events.length > TABLE_EVENT_CAP) {
+    stream.events = stream.events.slice(-TABLE_EVENT_CAP)
+    stream.earliestEventId = stream.events[0]?.eventId
+  }
+  stream.expiresAt = Date.now() + TABLE_EVENT_TTL_SECONDS * 1000
+  return entry
+}
+
+function readMemory(tableId: string, afterEventId: number): TableEventsReadResult {
+  pruneExpiredMemoryStreams()
+  const stream = memoryTableStreams.get(tableId)
+  if (!stream) {
+    // Mirror the Redis path: a non-zero afterEventId with no buffer at all
+    // means TTL expired or the stream never existed; either way the caller's
+    // cursor is stale.
+    if (afterEventId > 0) return { status: 'pruned', earliestEventId: undefined }
+    return { status: 'ok', events: [] }
+  }
+  if (stream.earliestEventId !== undefined && afterEventId + 1 < stream.earliestEventId) {
+    return { status: 'pruned', earliestEventId: stream.earliestEventId }
+  }
+  return {
+    status: 'ok',
+    events: stream.events
+      .filter((entry) => entry.eventId > afterEventId)
+      .slice(0, TABLE_EVENT_READ_CHUNK),
+  }
+}
+
+/**
+ * Append an event to the table's buffer. Fire-and-forget from the caller —
+ * this never throws, returns null on failure. A Redis blip must not fail a
+ * cell-write.
+ */
+export async function appendTableEvent(event: TableEvent): Promise<TableEventEntry | null> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      try {
+        return appendMemory(event)
+      } catch (error) {
+        logger.warn('appendTableEvent: memory append failed', {
+          tableId: event.tableId,
+          error: toError(error).message,
+        })
+        return null
+      }
+    }
+    return null
+  }
+  try {
+    // Build the entry JSON in two halves so Lua can splice the new eventId
+    // between them without us needing a round-trip just to mint the id first.
+    const tail = `,"tableId":${JSON.stringify(event.tableId)},"event":${JSON.stringify(event)}}`
+    const head = `{"eventId":`
+    const result = await redis.eval(
+      APPEND_EVENT_SCRIPT,
+      3,
+      getEventsKey(event.tableId),
+      getSeqKey(event.tableId),
+      getMetaKey(event.tableId),
+      TABLE_EVENT_TTL_SECONDS,
+      TABLE_EVENT_CAP,
+      new Date().toISOString(),
+      head,
+      tail
+    )
+    const eventId = typeof result === 'number' ? result : Number(result)
+    if (!Number.isFinite(eventId)) return null
+    return { eventId, tableId: event.tableId, event }
+  } catch (error) {
+    logger.warn('appendTableEvent: Redis append failed', {
+      tableId: event.tableId,
+      error: toError(error).message,
+    })
+    return null
+  }
+}
+
+/**
+ * The latest eventId assigned for a table, or 0 when the buffer is empty or
+ * expired. Used by the stream route to tail from "now" when a client connects
+ * without a replay cursor (fresh mount — its caches were just fetched from
+ * the DB, so replaying history would only rewind them).
+ *
+ * Redis errors propagate: silently falling back to 0 would replay the whole
+ * buffer over fresh state — the exact churn tail-from-latest exists to avoid.
+ * The stream route errors the stream instead and the client reconnects with
+ * backoff.
+ */
+export async function getLatestTableEventId(tableId: string): Promise<number> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      // Pure read — getMemoryStream() would allocate a stream as a side effect.
+      const stream = memoryTableStreams.get(tableId)
+      return stream ? stream.nextEventId - 1 : 0
+    }
+    return 0
+  }
+  const raw = await redis.get(getSeqKey(tableId))
+  if (!raw) return 0
+  const parsed = Number.parseInt(raw, 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 0
+}
+
+/**
+ * Read events for a table where eventId > afterEventId. Returns 'pruned' if
+ * the caller has fallen off the back of the buffer (TTL expired or cap rolled
+ * past their lastEventId). Caller should respond by full-refetching from DB
+ * and resuming streaming from the new earliestEventId.
+ */
+export async function readTableEventsSince(
+  tableId: string,
+  afterEventId: number
+): Promise<TableEventsReadResult> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryBuffer()) {
+      return readMemory(tableId, afterEventId)
+    }
+    return { status: 'unavailable', error: 'Redis client unavailable' }
+  }
+  try {
+    const meta = await redis.hgetall(getMetaKey(tableId))
+    const earliestEventId =
+      meta?.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined
+    if (earliestEventId !== undefined && afterEventId + 1 < earliestEventId) {
+      return { status: 'pruned', earliestEventId }
+    }
+    // Read in capped chunks so a 5000-event backlog doesn't materialize as one
+    // multi-MB Redis reply + JSON parse + SSE flush. The route loop drains
+    // chunks across ticks.
+    const raw = await redis.zrangebyscore(
+      getEventsKey(tableId),
+      afterEventId + 1,
+      '+inf',
+      'LIMIT',
+      0,
+      TABLE_EVENT_READ_CHUNK
+    )
+    if (raw.length === 0 && afterEventId > 0) {
+      // Total TTL expiry: events + meta both gone. The seq counter has the
+      // same TTL — its absence means the buffer was wiped and the caller's
+      // `afterEventId` is stale. Signal pruned so the client refetches.
+      const seqExists = await redis.exists(getSeqKey(tableId))
+      if (seqExists === 0) {
+        return { status: 'pruned', earliestEventId: undefined }
+      }
+    }
+    return {
+      status: 'ok',
+      events: raw
+        .map((entry) => {
+          try {
+            return JSON.parse(entry) as TableEventEntry
+          } catch {
+            return null
+          }
+        })
+        .filter((entry): entry is TableEventEntry => Boolean(entry)),
+    }
+  } catch (error) {
+    const message = toError(error).message
+    logger.warn('readTableEventsSince failed', { tableId, error: message })
+    return { status: 'unavailable', error: message }
+  }
+}