chore: import upstream snapshot with attribution

apps/sim/lib/core/admission/gate.ts

@@ -0,0 +1,62 @@
+import { createLogger } from '@sim/logger'
+import { NextResponse } from 'next/server'
+import { env } from '@/lib/core/config/env'
+
+const logger = createLogger('AdmissionGate')
+
+const MAX_INFLIGHT = Number.parseInt(env.ADMISSION_GATE_MAX_INFLIGHT ?? '') || 500
+
+let inflight = 0
+
+export interface AdmissionTicket {
+  release: () => void
+}
+
+/**
+ * Attempts to admit a request through the in-process gate.
+ * Returns a ticket with a release() handle on success, or null if at capacity.
+ * Zero external calls — purely in-process atomic counter. Each pod maintains its
+ * own counter, so the effective aggregate limit across N pods is N × MAX_INFLIGHT.
+ * Configure ADMISSION_GATE_MAX_INFLIGHT per pod based on what each pod can sustain.
+ */
+export function tryAdmit(): AdmissionTicket | null {
+  if (inflight >= MAX_INFLIGHT) {
+    return null
+  }
+
+  inflight++
+  let released = false
+
+  return {
+    release() {
+      if (released) return
+      released = true
+      inflight--
+    },
+  }
+}
+
+/**
+ * Returns a 429 response for requests rejected by the admission gate.
+ */
+export function admissionRejectedResponse(): NextResponse {
+  logger.warn('Admission gate rejecting request', { inflight, maxInflight: MAX_INFLIGHT })
+  return NextResponse.json(
+    {
+      error: 'Too many requests',
+      message: 'Server is at capacity. Please retry shortly.',
+      retryAfterSeconds: 5,
+    },
+    {
+      status: 429,
+      headers: { 'Retry-After': '5' },
+    }
+  )
+}
+
+/**
+ * Returns the current gate metrics for observability.
+ */
+export function getAdmissionGateStatus(): { inflight: number; maxInflight: number } {
+  return { inflight, maxInflight: MAX_INFLIGHT }
+}

apps/sim/lib/core/async-jobs/backends/database.ts

@@ -0,0 +1,327 @@
+import { asyncJobs, db } from '@sim/db'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateShortId } from '@sim/utils/id'
+import { eq, sql } from 'drizzle-orm'
+import {
+  type EnqueueOptions,
+  JOB_STATUS,
+  type Job,
+  type JobMetadata,
+  type JobQueueBackend,
+  type JobStatus,
+  type JobType,
+} from '@/lib/core/async-jobs/types'
+
+const logger = createLogger('DatabaseJobQueue')
+
+type AsyncJobRow = typeof asyncJobs.$inferSelect
+type Runner = NonNullable<EnqueueOptions['runner']>
+
+function rowToJob(row: AsyncJobRow): Job {
+  return {
+    id: row.id,
+    type: row.type as JobType,
+    payload: row.payload,
+    status: row.status as JobStatus,
+    createdAt: row.createdAt,
+    startedAt: row.startedAt ?? undefined,
+    completedAt: row.completedAt ?? undefined,
+    attempts: row.attempts,
+    maxAttempts: row.maxAttempts,
+    error: row.error ?? undefined,
+    output: row.output as unknown,
+    metadata: (row.metadata ?? {}) as JobMetadata,
+  }
+}
+
+const inlineAbortControllers = new Map<string, AbortController>()
+
+/**
+ * Per-cancel-key abort controllers for the `batchEnqueueAndWait` direct-call
+ * path. Distinct from `inlineAbortControllers` (which keys by jobId) — this
+ * map keys by the domain `cancelKey` callers pass in, since the await-blocking
+ * path skips `async_jobs` entirely and has no jobId to cancel by.
+ */
+const inlineCancelKeyControllers = new Map<string, AbortController>()
+
+interface Semaphore {
+  limit: number
+  available: number
+  waiters: Array<() => void>
+}
+const semaphores = new Map<string, Semaphore>()
+
+async function acquireSlot(key: string, limit: number): Promise<void> {
+  let s = semaphores.get(key)
+  if (!s) {
+    s = { limit, available: limit, waiters: [] }
+    semaphores.set(key, s)
+  }
+  if (s.available > 0) {
+    s.available -= 1
+    return
+  }
+  await new Promise<void>((resolve) => s.waiters.push(resolve))
+}
+
+function releaseSlot(key: string): void {
+  const s = semaphores.get(key)
+  if (!s) return
+  const next = s.waiters.shift()
+  if (next) {
+    next()
+    return
+  }
+  s.available += 1
+  if (s.available === s.limit) {
+    semaphores.delete(key)
+  }
+}
+
+export class DatabaseJobQueue implements JobQueueBackend {
+  async enqueue<TPayload>(
+    type: JobType,
+    payload: TPayload,
+    options?: EnqueueOptions
+  ): Promise<string> {
+    const jobId = options?.jobId ?? `run_${generateShortId(20)}`
+    const now = new Date()
+
+    await db
+      .insert(asyncJobs)
+      .values({
+        id: jobId,
+        type,
+        payload: payload as Record<string, unknown>,
+        status: JOB_STATUS.PENDING,
+        createdAt: now,
+        runAt:
+          options?.delayMs && options.delayMs > 0 ? new Date(now.getTime() + options.delayMs) : now,
+        attempts: 0,
+        maxAttempts: options?.maxAttempts ?? 3,
+        metadata: (options?.metadata ?? {}) as Record<string, unknown>,
+        updatedAt: now,
+      })
+      .onConflictDoNothing()
+
+    logger.debug('Enqueued job', { jobId, type })
+    if (options?.runner) {
+      this.runInline(
+        type,
+        jobId,
+        payload,
+        options.runner,
+        options.concurrencyKey,
+        options.concurrencyLimit
+      )
+    }
+    return jobId
+  }
+
+  async batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    const now = new Date()
+    const rows = items.map(({ payload, options }) => ({
+      id: `run_${generateShortId(20)}`,
+      type,
+      payload: payload as Record<string, unknown>,
+      status: JOB_STATUS.PENDING,
+      createdAt: now,
+      attempts: 0,
+      maxAttempts: options?.maxAttempts ?? 3,
+      metadata: (options?.metadata ?? {}) as Record<string, unknown>,
+      updatedAt: now,
+    }))
+
+    await db.insert(asyncJobs).values(rows)
+
+    logger.debug('Batch-enqueued jobs', { count: rows.length, type })
+
+    for (let i = 0; i < items.length; i++) {
+      const { payload, options } = items[i]
+      if (options?.runner) {
+        this.runInline(
+          type,
+          rows[i].id,
+          payload,
+          options.runner,
+          options.concurrencyKey,
+          options.concurrencyLimit
+        )
+      }
+    }
+
+    return rows.map((r) => r.id)
+  }
+
+  /** Skips `async_jobs` entirely — ids are returned empty since callers can't
+   *  look up rows that don't exist. Cancel goes through `cancelByKey`. */
+  async batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    const tracked: Array<{ key: string; controller: AbortController }> = []
+    const runs = items.map((item) => {
+      const runner = item.options?.runner
+      if (!runner) return Promise.resolve()
+      const controller = new AbortController()
+      const cancelKey = item.options?.cancelKey
+      if (cancelKey) {
+        inlineCancelKeyControllers.set(cancelKey, controller)
+        tracked.push({ key: cancelKey, controller })
+      }
+      return runner(item.payload, controller.signal).catch((err) => {
+        logger.error(`[${type}] Inline run failed`, {
+          cancelKey,
+          error: toError(err).message,
+        })
+      })
+    })
+    try {
+      await Promise.all(runs)
+    } finally {
+      // Compare-and-delete guards against a re-enqueue under the same key
+      // racing with our cleanup.
+      for (const t of tracked) {
+        if (inlineCancelKeyControllers.get(t.key) === t.controller) {
+          inlineCancelKeyControllers.delete(t.key)
+        }
+      }
+    }
+    return items.map(() => '')
+  }
+
+  async getJob(jobId: string): Promise<Job | null> {
+    const [row] = await db.select().from(asyncJobs).where(eq(asyncJobs.id, jobId)).limit(1)
+
+    return row ? rowToJob(row) : null
+  }
+
+  async startJob(jobId: string): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.PROCESSING,
+        startedAt: now,
+        attempts: sql`${asyncJobs.attempts} + 1`,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Started job', { jobId })
+  }
+
+  async completeJob(jobId: string, output: unknown): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.COMPLETED,
+        completedAt: now,
+        output: output as Record<string, unknown>,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Completed job', { jobId })
+  }
+
+  async markJobFailed(jobId: string, error: string): Promise<void> {
+    const now = new Date()
+
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.FAILED,
+        completedAt: now,
+        error,
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Marked job as failed', { jobId })
+  }
+
+  async cancelJob(jobId: string): Promise<void> {
+    // Abort any in-process inline execution first so the running workflow
+    // observes the signal and stops mid-flight. Then mark the row failed so
+    // any future poller skips it.
+    const controller = inlineAbortControllers.get(jobId)
+    let aborted = false
+    if (controller) {
+      controller.abort('Cancelled')
+      inlineAbortControllers.delete(jobId)
+      aborted = true
+    }
+
+    const now = new Date()
+    await db
+      .update(asyncJobs)
+      .set({
+        status: JOB_STATUS.FAILED,
+        completedAt: now,
+        error: 'Cancelled',
+        updatedAt: now,
+      })
+      .where(eq(asyncJobs.id, jobId))
+
+    logger.debug('Marked job as cancelled (DB queue)', { jobId, abortedInline: aborted })
+  }
+
+  cancelByKey(cancelKey: string): boolean {
+    const controller = inlineCancelKeyControllers.get(cancelKey)
+    if (!controller) return false
+    controller.abort('Cancelled')
+    inlineCancelKeyControllers.delete(cancelKey)
+    return true
+  }
+
+  /**
+   * Fire-and-forget IIFE that owns the lifecycle for an inline job: registers
+   * the abort controller (so `cancelJob` can interrupt mid-flight), acquires
+   * a concurrency slot if `concurrencyKey` is set, drives
+   * `startJob → runner → completeJob | markJobFailed`.
+   */
+  private runInline<TPayload>(
+    type: JobType,
+    jobId: string,
+    payload: TPayload,
+    runner: Runner,
+    concurrencyKey?: string,
+    concurrencyLimit?: number
+  ): void {
+    const abortController = new AbortController()
+    inlineAbortControllers.set(jobId, abortController)
+    void (async () => {
+      if (concurrencyKey && concurrencyLimit && concurrencyLimit > 0) {
+        await acquireSlot(concurrencyKey, concurrencyLimit)
+      }
+      try {
+        await this.startJob(jobId)
+        await runner(payload, abortController.signal)
+        await this.completeJob(jobId, null)
+      } catch (err) {
+        const message = toError(err).message
+        logger.error(`[${type}] Inline job ${jobId} failed`, { error: message })
+        try {
+          await this.markJobFailed(jobId, message)
+        } catch (markErr) {
+          logger.error(`[${type}] Failed to mark job ${jobId} as failed`, { markErr })
+        }
+      } finally {
+        inlineAbortControllers.delete(jobId)
+        if (concurrencyKey && concurrencyLimit && concurrencyLimit > 0) {
+          releaseSlot(concurrencyKey)
+        }
+      }
+    })()
+  }
+}

apps/sim/lib/core/async-jobs/backends/trigger-dev.ts

@@ -0,0 +1,260 @@
+import { createLogger } from '@sim/logger'
+import { taskContext } from '@trigger.dev/core/v3'
+import { runs, type TriggerOptions, tasks } from '@trigger.dev/sdk'
+import { resolveTriggerRegion } from '@/lib/core/async-jobs/region'
+import {
+  type EnqueueOptions,
+  JOB_STATUS,
+  type Job,
+  type JobMetadata,
+  type JobQueueBackend,
+  type JobStatus,
+  type JobType,
+} from '@/lib/core/async-jobs/types'
+
+const logger = createLogger('TriggerDevJobQueue')
+
+/**
+ * Maps trigger.dev task IDs to our JobType
+ */
+const JOB_TYPE_TO_TASK_ID: Record<JobType, string> = {
+  'workflow-execution': 'workflow-execution',
+  'schedule-execution': 'schedule-execution',
+  'webhook-execution': 'webhook-execution',
+  'tiktok-webhook-ingress': 'tiktok-webhook-ingress',
+  'resume-execution': 'resume-execution',
+  'workflow-group-cell': 'workflow-group-cell',
+  'cleanup-logs': 'cleanup-logs',
+  'cleanup-soft-deletes': 'cleanup-soft-deletes',
+  'cleanup-tasks': 'cleanup-tasks',
+  'run-data-drain': 'run-data-drain',
+}
+
+/**
+ * Maps trigger.dev run status to our JobStatus
+ */
+function mapTriggerDevStatus(status: string): JobStatus {
+  switch (status) {
+    case 'QUEUED':
+    case 'WAITING_FOR_DEPLOY':
+      return JOB_STATUS.PENDING
+    case 'EXECUTING':
+    case 'RESCHEDULED':
+    case 'FROZEN':
+      return JOB_STATUS.PROCESSING
+    case 'COMPLETED':
+      return JOB_STATUS.COMPLETED
+    case 'CANCELED':
+    case 'FAILED':
+    case 'CRASHED':
+    case 'INTERRUPTED':
+    case 'SYSTEM_FAILURE':
+    case 'EXPIRED':
+      return JOB_STATUS.FAILED
+    default:
+      return JOB_STATUS.PENDING
+  }
+}
+
+/**
+ * Adapter that wraps the trigger.dev SDK to conform to JobQueueBackend interface.
+ */
+export class TriggerDevJobQueue implements JobQueueBackend {
+  async enqueue<TPayload>(
+    type: JobType,
+    payload: TPayload,
+    options?: EnqueueOptions
+  ): Promise<string> {
+    const taskId = JOB_TYPE_TO_TASK_ID[type]
+    if (!taskId) {
+      throw new Error(`Unknown job type: ${type}`)
+    }
+
+    const enrichedPayload =
+      options?.metadata && typeof payload === 'object' && payload !== null
+        ? { ...payload, ...options.metadata }
+        : payload
+
+    const tags = buildTags(options)
+    const triggerOptions: TriggerOptions = {}
+    if (tags.length > 0) triggerOptions.tags = tags
+    if (options?.concurrencyKey) triggerOptions.concurrencyKey = options.concurrencyKey
+    if (options?.jobId) {
+      triggerOptions.idempotencyKey = options.jobId
+      triggerOptions.idempotencyKeyTTL = '14d'
+    }
+    if (options?.delayMs && options.delayMs > 0) {
+      triggerOptions.delay = new Date(Date.now() + options.delayMs)
+    }
+    triggerOptions.region = await resolveTriggerRegion()
+    const handle = await tasks.trigger(taskId, enrichedPayload, triggerOptions)
+
+    logger.debug('Enqueued job via trigger.dev', { jobId: handle.id, type, taskId, tags })
+    return handle.id
+  }
+
+  async batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    // tasks.batchTrigger returns only a batchId, not per-item run IDs, so we
+    // can't use it when callers need to track individual runs (e.g. table cell
+    // tasks need per-row jobIds for cancellation). Sequential `tasks.trigger`
+    // gives us per-item IDs and naturally preserves input order in the queue.
+    const ids: string[] = []
+    for (const { payload, options } of items) {
+      const id = await this.enqueue(type, payload, options)
+      ids.push(id)
+    }
+    return ids
+  }
+
+  async batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]> {
+    if (items.length === 0) return []
+    // The SDK's checkpoint-and-resume requires task runtime context. The only
+    // caller (`dispatcherStep` invoked by `tableRunDispatcherTask.run`) is
+    // always inside a task; check defensively so misuse fails at the boundary
+    // instead of as a confusing SDK internal error.
+    if (!taskContext.isInsideTask) {
+      throw new Error(
+        'batchEnqueueAndWait requires trigger.dev task runtime context — call from within a registered task'
+      )
+    }
+
+    const taskId = JOB_TYPE_TO_TASK_ID[type]
+    if (!taskId) throw new Error(`Unknown job type: ${type}`)
+
+    const region = await resolveTriggerRegion()
+    const batchItems = items.map(({ payload, options }) => {
+      const enrichedPayload =
+        options?.metadata && typeof payload === 'object' && payload !== null
+          ? { ...payload, ...options.metadata }
+          : payload
+      const tags = buildTags(options)
+      const batchItem: {
+        payload: unknown
+        options?: { concurrencyKey?: string; tags?: string[]; region?: string }
+      } = { payload: enrichedPayload }
+      const batchOpts: { concurrencyKey?: string; tags?: string[]; region?: string } = { region }
+      if (options?.concurrencyKey) batchOpts.concurrencyKey = options.concurrencyKey
+      if (tags.length > 0) batchOpts.tags = tags
+      batchItem.options = batchOpts
+      return batchItem
+    })
+
+    const result = await tasks.batchTriggerAndWait(taskId, batchItems)
+    logger.debug('batchTriggerAndWait completed', {
+      type,
+      taskId,
+      runCount: result.runs.length,
+    })
+    return result.runs.map((r) => r.id)
+  }
+
+  async getJob(jobId: string): Promise<Job | null> {
+    try {
+      const run = await runs.retrieve(jobId)
+
+      const payload = run.payload as Record<string, unknown>
+      const metadata: JobMetadata = {
+        workflowId: payload?.workflowId as string | undefined,
+        userId: payload?.userId as string | undefined,
+        correlation:
+          payload?.correlation && typeof payload.correlation === 'object'
+            ? (payload.correlation as JobMetadata['correlation'])
+            : undefined,
+      }
+
+      return {
+        id: jobId,
+        type: run.taskIdentifier as JobType,
+        payload: run.payload,
+        status: mapTriggerDevStatus(run.status),
+        createdAt: run.createdAt ? new Date(run.createdAt) : new Date(),
+        startedAt: run.startedAt ? new Date(run.startedAt) : undefined,
+        completedAt: run.finishedAt ? new Date(run.finishedAt) : undefined,
+        attempts: run.attemptCount ?? 1,
+        maxAttempts: 3,
+        error: run.error?.message,
+        output: run.output as unknown,
+        metadata,
+      }
+    } catch (error) {
+      const isNotFound =
+        (error instanceof Error && error.message.toLowerCase().includes('not found')) ||
+        (error && typeof error === 'object' && 'status' in error && error.status === 404)
+
+      if (isNotFound) {
+        logger.debug('Job not found in trigger.dev', { jobId })
+        return null
+      }
+
+      logger.error('Failed to get job from trigger.dev', { jobId, error })
+      throw error
+    }
+  }
+
+  async startJob(_jobId: string): Promise<void> {}
+
+  async completeJob(_jobId: string, _output: unknown): Promise<void> {}
+
+  async markJobFailed(_jobId: string, _error: string): Promise<void> {}
+
+  async cancelJob(jobId: string): Promise<void> {
+    try {
+      await runs.cancel(jobId)
+      logger.debug('Cancelled trigger.dev run', { jobId })
+    } catch (error) {
+      const isNotFound =
+        (error instanceof Error && error.message.toLowerCase().includes('not found')) ||
+        (error && typeof error === 'object' && 'status' in error && error.status === 404)
+      if (isNotFound) {
+        logger.debug('Cancel target not found in trigger.dev (already finished?)', { jobId })
+        return
+      }
+      logger.error('Failed to cancel trigger.dev run', { jobId, error })
+      throw error
+    }
+  }
+
+  cancelByKey(_cancelKey: string): boolean {
+    // No in-process AbortControllers to abort — trigger.dev runs are cancelled
+    // by jobId or via tag sweep (see `cancelCellRunsByTags`). Callers that
+    // need both surfaces should fan out themselves.
+    return false
+  }
+}
+
+/**
+ * Derives trigger.dev tags from job type, metadata, and explicit tags.
+ * Tags follow the `namespace:value` convention for consistent filtering.
+ * Max 10 tags per run, each max 128 chars.
+ */
+function buildTags(options?: EnqueueOptions): string[] {
+  const tags: string[] = []
+  const meta = options?.metadata
+
+  if (meta?.workspaceId) tags.push(`workspaceId:${meta.workspaceId}`)
+  if (meta?.workflowId) tags.push(`workflowId:${meta.workflowId}`)
+  if (meta?.userId) tags.push(`userId:${meta.userId}`)
+
+  if (meta?.correlation) {
+    const c = meta.correlation
+    tags.push(`source:${c.source}`)
+    if (c.webhookId) tags.push(`webhookId:${c.webhookId}`)
+    if (c.scheduleId) tags.push(`scheduleId:${c.scheduleId}`)
+    if (c.provider) tags.push(`provider:${c.provider}`)
+  }
+
+  if (options?.tags) {
+    for (const tag of options.tags) {
+      if (!tags.includes(tag)) tags.push(tag)
+    }
+  }
+
+  return tags.slice(0, 10)
+}

apps/sim/lib/core/async-jobs/config.ts

@@ -0,0 +1,101 @@
+import { createLogger } from '@sim/logger'
+import { taskContext } from '@trigger.dev/core/v3'
+import type { AsyncBackendType, JobQueueBackend } from '@/lib/core/async-jobs/types'
+import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
+
+const logger = createLogger('AsyncJobsConfig')
+
+let cachedBackend: JobQueueBackend | null = null
+let cachedBackendType: AsyncBackendType | null = null
+let cachedInlineBackend: JobQueueBackend | null = null
+
+/**
+ * Determines which async backend to use based on environment configuration.
+ * Falls back to the database backend when trigger.dev isn't enabled — except
+ * when this process IS a trigger.dev worker (`taskContext.isInsideTask`), in
+ * which case the SDK runtime is available regardless of env vars and we
+ * always want to enqueue back through trigger.dev. Without this carve-out, a
+ * worker pod missing `TRIGGER_DEV_ENABLED=true` silently routes cell jobs to
+ * the database backend that nothing's draining.
+ */
+export function getAsyncBackendType(): AsyncBackendType {
+  if (isTriggerDevEnabled || taskContext.isInsideTask) {
+    return 'trigger-dev'
+  }
+
+  return 'database'
+}
+
+/**
+ * Gets the job queue backend singleton.
+ * Creates the appropriate backend based on environment configuration.
+ */
+export async function getJobQueue(): Promise<JobQueueBackend> {
+  if (cachedBackend) {
+    return cachedBackend
+  }
+
+  const type = getAsyncBackendType()
+
+  switch (type) {
+    case 'trigger-dev': {
+      const { TriggerDevJobQueue } = await import('@/lib/core/async-jobs/backends/trigger-dev')
+      cachedBackend = new TriggerDevJobQueue()
+      break
+    }
+    case 'database': {
+      const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
+      cachedBackend = new DatabaseJobQueue()
+      break
+    }
+  }
+
+  cachedBackendType = type
+  logger.info(`Async job backend initialized: ${type}`)
+
+  if (!cachedBackend) {
+    throw new Error(`Failed to initialize async backend: ${type}`)
+  }
+
+  return cachedBackend
+}
+
+/**
+ * Gets the current backend type (for logging/debugging)
+ */
+export function getCurrentBackendType(): AsyncBackendType | null {
+  return cachedBackendType
+}
+
+/**
+ * Gets a job queue backend that bypasses Trigger.dev (Database only).
+ * Used for execution paths that must avoid Trigger.dev cold starts.
+ */
+export async function getInlineJobQueue(): Promise<JobQueueBackend> {
+  if (cachedInlineBackend) {
+    return cachedInlineBackend
+  }
+
+  const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
+  cachedInlineBackend = new DatabaseJobQueue()
+
+  logger.info('Inline job backend initialized: database')
+  return cachedInlineBackend
+}
+
+/**
+ * Checks if jobs should be executed inline in-process.
+ * Database fallback is the only mode that still relies on inline execution.
+ */
+export function shouldExecuteInline(): boolean {
+  return getAsyncBackendType() === 'database'
+}
+
+/**
+ * Resets the cached backend (useful for testing)
+ */
+export function resetJobQueueCache(): void {
+  cachedBackend = null
+  cachedBackendType = null
+  cachedInlineBackend = null
+}

apps/sim/lib/core/async-jobs/index.ts

@@ -0,0 +1,14 @@
+export {
+  getAsyncBackendType,
+  getCurrentBackendType,
+  getInlineJobQueue,
+  getJobQueue,
+  resetJobQueueCache,
+  shouldExecuteInline,
+} from './config'
+export {
+  JOB_MAX_LIFETIME_SECONDS,
+  JOB_RETENTION_HOURS,
+  JOB_RETENTION_SECONDS,
+  JOB_STATUS,
+} from './types'

apps/sim/lib/core/async-jobs/region.test.ts

@@ -0,0 +1,42 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockIsFeatureEnabled } = vi.hoisted(() => ({
+  mockIsFeatureEnabled: vi.fn(),
+}))
+
+vi.mock('@/lib/core/config/feature-flags', () => ({
+  isFeatureEnabled: mockIsFeatureEnabled,
+}))
+
+import {
+  resolveTriggerRegion,
+  TRIGGER_REGION_EU_CENTRAL,
+  TRIGGER_REGION_US_EAST,
+} from '@/lib/core/async-jobs/region'
+
+describe('resolveTriggerRegion', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('returns eu-central-1 when the flag is enabled', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(true)
+    expect(await resolveTriggerRegion()).toBe(TRIGGER_REGION_EU_CENTRAL)
+    expect(mockIsFeatureEnabled).toHaveBeenCalledWith('trigger-eu-region')
+  })
+
+  it('returns us-east-1 when the flag is disabled', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(false)
+    expect(await resolveTriggerRegion()).toBe(TRIGGER_REGION_US_EAST)
+  })
+
+  it('evaluates globally, passing no gating context', async () => {
+    mockIsFeatureEnabled.mockResolvedValue(false)
+    await resolveTriggerRegion()
+    expect(mockIsFeatureEnabled).toHaveBeenCalledTimes(1)
+    expect(mockIsFeatureEnabled.mock.calls[0]).toEqual(['trigger-eu-region'])
+  })
+})

apps/sim/lib/core/async-jobs/region.ts

@@ -0,0 +1,21 @@
+import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
+
+/** Default Trigger.dev region — the project default when the eu-central flag is off. */
+export const TRIGGER_REGION_US_EAST = 'us-east-1'
+
+/** Target region when the `trigger-eu-region` flag is enabled. */
+export const TRIGGER_REGION_EU_CENTRAL = 'eu-central-1'
+
+/**
+ * Resolve which Trigger.dev region a run should execute in. Gated globally by the
+ * `trigger-eu-region` feature flag (all-or-nothing — no per-user/org targeting):
+ * `eu-central-1` when enabled, otherwise `us-east-1`.
+ *
+ * The result is passed as the `region` option to `tasks.trigger` / `batchTrigger`,
+ * overriding the project's dashboard default per run.
+ */
+export async function resolveTriggerRegion(): Promise<string> {
+  return (await isFeatureEnabled('trigger-eu-region'))
+    ? TRIGGER_REGION_EU_CENTRAL
+    : TRIGGER_REGION_US_EAST
+}

apps/sim/lib/core/async-jobs/types.test.ts

@@ -0,0 +1,32 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { JOB_MAX_LIFETIME_SECONDS, JOB_RETENTION_HOURS, JOB_RETENTION_SECONDS } from './types'
+
+describe('Job retention constants', () => {
+  it.concurrent('JOB_RETENTION_HOURS should be 24', async () => {
+    expect(JOB_RETENTION_HOURS).toBe(24)
+  })
+
+  it.concurrent('JOB_RETENTION_SECONDS should be derived from JOB_RETENTION_HOURS', async () => {
+    expect(JOB_RETENTION_SECONDS).toBe(JOB_RETENTION_HOURS * 60 * 60)
+  })
+
+  it.concurrent('JOB_RETENTION_SECONDS should equal 86400 (24 hours)', async () => {
+    expect(JOB_RETENTION_SECONDS).toBe(86400)
+  })
+
+  it.concurrent('constants should be consistent with each other', async () => {
+    const hoursToSeconds = JOB_RETENTION_HOURS * 60 * 60
+    expect(JOB_RETENTION_SECONDS).toBe(hoursToSeconds)
+  })
+
+  it.concurrent(
+    'JOB_MAX_LIFETIME_SECONDS should be greater than JOB_RETENTION_SECONDS',
+    async () => {
+      expect(JOB_MAX_LIFETIME_SECONDS).toBeGreaterThan(JOB_RETENTION_SECONDS)
+      expect(JOB_MAX_LIFETIME_SECONDS).toBe(48 * 60 * 60)
+    }
+  )
+})

apps/sim/lib/core/async-jobs/types.ts

@@ -0,0 +1,189 @@
+/**
+ * Types and constants for the async job queue system
+ */
+
+/** Retention period for completed/failed jobs (in hours) */
+export const JOB_RETENTION_HOURS = 24
+
+/** Retention period for completed/failed jobs (in seconds, for Redis TTL) */
+export const JOB_RETENTION_SECONDS = JOB_RETENTION_HOURS * 60 * 60
+
+/** Max lifetime for jobs in Redis (in seconds) - cleanup for stuck pending/processing jobs */
+export const JOB_MAX_LIFETIME_SECONDS = 48 * 60 * 60
+
+export const JOB_STATUS = {
+  PENDING: 'pending',
+  PROCESSING: 'processing',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+} as const
+
+export type JobStatus = (typeof JOB_STATUS)[keyof typeof JOB_STATUS]
+
+export type JobType =
+  | 'workflow-execution'
+  | 'schedule-execution'
+  | 'webhook-execution'
+  | 'tiktok-webhook-ingress'
+  | 'resume-execution'
+  | 'workflow-group-cell'
+  | 'cleanup-logs'
+  | 'cleanup-soft-deletes'
+  | 'cleanup-tasks'
+  | 'run-data-drain'
+
+export type AsyncExecutionCorrelationSource = 'workflow' | 'schedule' | 'webhook'
+
+export interface AsyncExecutionCorrelation {
+  executionId: string
+  requestId: string
+  source: AsyncExecutionCorrelationSource
+  workflowId: string
+  triggerType?: string
+  webhookId?: string
+  scheduleId?: string
+  path?: string
+  provider?: string
+  scheduledFor?: string
+}
+
+export interface Job<TPayload = unknown, TOutput = unknown> {
+  id: string
+  type: JobType
+  payload: TPayload
+  status: JobStatus
+  createdAt: Date
+  startedAt?: Date
+  completedAt?: Date
+  attempts: number
+  maxAttempts: number
+  error?: string
+  output?: TOutput
+  metadata: JobMetadata
+}
+
+export interface JobMetadata {
+  workflowId?: string
+  workspaceId?: string
+  userId?: string
+  correlation?: AsyncExecutionCorrelation
+  [key: string]: unknown
+}
+
+export interface EnqueueOptions {
+  maxAttempts?: number
+  metadata?: JobMetadata
+  jobId?: string
+  priority?: number
+  name?: string
+  delayMs?: number
+  tags?: string[]
+  /**
+   * Combined with the task's `queue.concurrencyLimit`, caps parallel runs
+   * sharing this key. Trigger.dev enforces server-side; the database backend
+   * enforces in-process via a FIFO semaphore.
+   */
+  concurrencyKey?: string
+  /**
+   * Per-key concurrency cap. Database backend only — trigger.dev reads this
+   * from the task definition (`queue.concurrencyLimit`).
+   */
+  concurrencyLimit?: number
+  /**
+   * Job body invoked when the queue backend lacks an external worker.
+   * Trigger.dev ignores this (its workers execute the task definition);
+   * the database backend kicks it off as a fire-and-forget IIFE so the
+   * row drives through `processing → completed | failed`. Receives the
+   * payload and an `AbortSignal` driven by `cancelJob`.
+   */
+  runner?: <TPayload>(payload: TPayload, signal: AbortSignal) => Promise<void>
+  /**
+   * Stable identity for cancellation lookups on the database backend's
+   * `batchEnqueueAndWait` path (which skips `async_jobs` entirely, so there
+   * is no jobId to cancel by). Lets callers map a domain identity (e.g.
+   * `tableId:rowId:groupId`) to the in-flight `AbortController`. Ignored
+   * by trigger.dev — runs there are cancelled by tag or jobId.
+   */
+  cancelKey?: string
+}
+
+/**
+ * Backend interface for job queue implementations.
+ * All backends must implement this interface.
+ */
+export interface JobQueueBackend {
+  /**
+   * Add a job to the queue
+   */
+  enqueue<TPayload>(type: JobType, payload: TPayload, options?: EnqueueOptions): Promise<string>
+
+  /**
+   * Enqueue multiple jobs as a single batch. Returns one jobId per item, in
+   * input order. Backends preserve input order in queue dispatch (trigger.dev
+   * via tasks.batchTrigger, database via a single multi-row INSERT).
+   */
+  batchEnqueue<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]>
+
+  /**
+   * Enqueue a batch and block until every job has reached a terminal state
+   * (completed, failed, or cancelled). The caller — typically a dispatcher
+   * walking work in windows — uses this to gate window N+1 on window N's
+   * completion.
+   *
+   * Backend implementations:
+   * - Trigger.dev: wraps `tasks.batchTriggerAndWait`. MUST be called from
+   *   inside a registered trigger.dev task (the SDK's checkpoint-and-resume
+   *   requires task runtime context). Backends guard with
+   *   `taskContext.isInsideTask` and throw a clear error otherwise.
+   * - Database (in-process): bypasses `async_jobs` entirely. Since the
+   *   caller is awaiting in-process, the row would serve no live purpose
+   *   (no cross-process recovery, no by-id lookup, no semaphore needed —
+   *   window size IS the concurrency cap). Calls the runner directly via
+   *   `Promise.all` and resolves on the runner's exit.
+   */
+  batchEnqueueAndWait<TPayload>(
+    type: JobType,
+    items: Array<{ payload: TPayload; options?: EnqueueOptions }>
+  ): Promise<string[]>
+
+  /**
+   * Get a job by ID
+   */
+  getJob(jobId: string): Promise<Job | null>
+
+  /**
+   * Mark a job as started/processing
+   */
+  startJob(jobId: string): Promise<void>
+
+  /**
+   * Mark a job as completed with output
+   */
+  completeJob(jobId: string, output: unknown): Promise<void>
+
+  /**
+   * Mark a job as failed with error message
+   */
+  markJobFailed(jobId: string, error: string): Promise<void>
+
+  /**
+   * Request cancellation of a queued or running job. Best-effort: backends should
+   * fail loudly if the underlying provider rejects, but a missing/unknown jobId
+   * should resolve quietly so callers can drive cancel from possibly-stale state.
+   */
+  cancelJob(jobId: string): Promise<void>
+
+  /**
+   * Cancel an in-flight job by its `cancelKey` (the domain identity callers
+   * stamped on enqueue via `EnqueueOptions.cancelKey`). Used by
+   * `batchEnqueueAndWait` paths that skip per-job ids; the trigger.dev
+   * backend has no in-process AbortControllers to abort and returns `false`.
+   * Returns `true` if a matching controller was found and aborted.
+   */
+  cancelByKey(cancelKey: string): boolean
+}
+
+export type AsyncBackendType = 'trigger-dev' | 'database'

apps/sim/lib/core/config/api-keys.ts

@@ -0,0 +1,61 @@
+import { env } from '@/lib/core/config/env'
+
+/**
+ * Rotates through available API keys for a provider
+ * @param provider - The provider to get a key for (e.g., 'openai')
+ * @returns The selected API key
+ * @throws Error if no API keys are configured for rotation
+ */
+export function getRotatingApiKey(provider: string): string {
+  if (
+    provider !== 'openai' &&
+    provider !== 'anthropic' &&
+    provider !== 'gemini' &&
+    provider !== 'cohere' &&
+    provider !== 'zai' &&
+    provider !== 'xai'
+  ) {
+    throw new Error(`No rotation implemented for provider: ${provider}`)
+  }
+
+  const keys = []
+
+  if (provider === 'openai') {
+    if (env.OPENAI_API_KEY_1) keys.push(env.OPENAI_API_KEY_1)
+    if (env.OPENAI_API_KEY_2) keys.push(env.OPENAI_API_KEY_2)
+    if (env.OPENAI_API_KEY_3) keys.push(env.OPENAI_API_KEY_3)
+  } else if (provider === 'anthropic') {
+    if (env.ANTHROPIC_API_KEY_1) keys.push(env.ANTHROPIC_API_KEY_1)
+    if (env.ANTHROPIC_API_KEY_2) keys.push(env.ANTHROPIC_API_KEY_2)
+    if (env.ANTHROPIC_API_KEY_3) keys.push(env.ANTHROPIC_API_KEY_3)
+  } else if (provider === 'gemini') {
+    if (env.GEMINI_API_KEY_1) keys.push(env.GEMINI_API_KEY_1)
+    if (env.GEMINI_API_KEY_2) keys.push(env.GEMINI_API_KEY_2)
+    if (env.GEMINI_API_KEY_3) keys.push(env.GEMINI_API_KEY_3)
+  } else if (provider === 'cohere') {
+    if (env.COHERE_API_KEY_1) keys.push(env.COHERE_API_KEY_1)
+    if (env.COHERE_API_KEY_2) keys.push(env.COHERE_API_KEY_2)
+    if (env.COHERE_API_KEY_3) keys.push(env.COHERE_API_KEY_3)
+  } else if (provider === 'zai') {
+    if (env.ZAI_API_KEY_1) keys.push(env.ZAI_API_KEY_1)
+    if (env.ZAI_API_KEY_2) keys.push(env.ZAI_API_KEY_2)
+    if (env.ZAI_API_KEY_3) keys.push(env.ZAI_API_KEY_3)
+  } else if (provider === 'xai') {
+    if (env.XAI_API_KEY_1) keys.push(env.XAI_API_KEY_1)
+    if (env.XAI_API_KEY_2) keys.push(env.XAI_API_KEY_2)
+    if (env.XAI_API_KEY_3) keys.push(env.XAI_API_KEY_3)
+  }
+
+  if (keys.length === 0) {
+    throw new Error(
+      `No API keys configured for rotation. Please configure ${provider.toUpperCase()}_API_KEY_1, ${provider.toUpperCase()}_API_KEY_2, or ${provider.toUpperCase()}_API_KEY_3.`
+    )
+  }
+
+  // Simple round-robin rotation based on current minute
+  // This distributes load across keys and is stateless
+  const currentMinute = new Date().getMinutes()
+  const keyIndex = currentMinute % keys.length
+
+  return keys[keyIndex]
+}

apps/sim/lib/core/config/appconfig-rules.test.ts

@@ -0,0 +1,69 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { matchesRule, normalizeRule, parseGateConfig } from '@/lib/core/config/appconfig-rules'
+
+describe('normalizeRule', () => {
+  it('returns null for non-object values', () => {
+    expect(normalizeRule('nope')).toBeNull()
+    expect(normalizeRule(null)).toBeNull()
+    expect(normalizeRule(42)).toBeNull()
+  })
+
+  it('keeps only boolean enabled/adminEnabled', () => {
+    expect(normalizeRule({ enabled: 'true', adminEnabled: 1 })).toEqual({})
+    expect(normalizeRule({ enabled: true, adminEnabled: false })).toEqual({
+      enabled: true,
+      adminEnabled: false,
+    })
+  })
+
+  it('trims, dedupes, and drops empty ids', () => {
+    expect(normalizeRule({ orgIds: ['Org_1', ' org_1 ', '', 'org_2'], userIds: 'nope' })).toEqual({
+      orgIds: ['Org_1', 'org_1', 'org_2'],
+    })
+  })
+})
+
+describe('parseGateConfig', () => {
+  it('drops malformed entries and coerces the rest', () => {
+    const rules = parseGateConfig({
+      a: { enabled: true },
+      b: 'not-an-object',
+      c: { userIds: ['u1'] },
+    })
+    expect(rules.a).toEqual({ enabled: true })
+    expect(rules.b).toBeUndefined()
+    expect(rules.c).toEqual({ userIds: ['u1'] })
+  })
+
+  it('degrades to an empty map on a malformed document', () => {
+    expect(parseGateConfig('not-an-object')).toEqual({})
+    expect(parseGateConfig(null)).toEqual({})
+  })
+})
+
+describe('matchesRule', () => {
+  it('returns false for a missing rule', () => {
+    expect(matchesRule(undefined, { userId: 'u1' }, true)).toBe(false)
+  })
+
+  it('matches the global enabled clause', () => {
+    expect(matchesRule({ enabled: true }, {}, false)).toBe(true)
+    expect(matchesRule({ enabled: false }, {}, false)).toBe(false)
+  })
+
+  it('matches the userId and orgId allowlists', () => {
+    expect(matchesRule({ userIds: ['u1'] }, { userId: 'u1' }, false)).toBe(true)
+    expect(matchesRule({ userIds: ['u1'] }, { userId: 'u2' }, false)).toBe(false)
+    expect(matchesRule({ orgIds: ['o1'] }, { orgId: 'o1' }, false)).toBe(true)
+    expect(matchesRule({ orgIds: ['o1'] }, {}, false)).toBe(false)
+  })
+
+  it('matches the admin clause only with the supplied isAdmin', () => {
+    expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, true)).toBe(true)
+    expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, false)).toBe(false)
+    expect(matchesRule({ enabled: false }, { userId: 'u1' }, true)).toBe(false)
+  })
+})

apps/sim/lib/core/config/appconfig-rules.ts

@@ -0,0 +1,80 @@
+/**
+ * Shared parsing and clause evaluation for AppConfig gating documents
+ * (`feature-flags`, `block-visibility`). Both documents are maps of key →
+ * gate rule with identical rule shapes; this module is the single copy of the
+ * security-sensitive normalization that prevents a malformed document from
+ * granting access. Admin-resolution *scheduling* deliberately stays with the
+ * callers (feature-flags resolves lazily per rule; block-visibility resolves
+ * once per document), so {@link matchesRule} takes an explicit `isAdmin`.
+ */
+
+/**
+ * A single gating rule. A gate is open for a context when ANY clause matches:
+ * the global `enabled` default, the org/user allowlists, or `adminEnabled` for
+ * platform admins. An absent clause never matches.
+ */
+export interface AppConfigGateRule {
+  enabled?: boolean
+  orgIds?: string[]
+  userIds?: string[]
+  adminEnabled?: boolean
+}
+
+/**
+ * Per-request evaluation context. Pass only the ids you have — a missing id
+ * skips its clause. `isAdmin` is a fast-path override for callers that already
+ * resolved platform-admin status.
+ */
+export interface AppConfigGateContext {
+  userId?: string | null
+  orgId?: string | null
+  isAdmin?: boolean
+}
+
+function normalizeIds(values: unknown): string[] | undefined {
+  if (!Array.isArray(values)) return undefined
+  const ids = Array.from(new Set(values.map((v) => String(v).trim()).filter(Boolean)))
+  return ids.length > 0 ? ids : undefined
+}
+
+/** Coerce a single arbitrary JSON value into a rule, or `null` when malformed. */
+export function normalizeRule(value: unknown): AppConfigGateRule | null {
+  if (!value || typeof value !== 'object') return null
+  const obj = value as Record<string, unknown>
+  const rule: AppConfigGateRule = {}
+  if (typeof obj.enabled === 'boolean') rule.enabled = obj.enabled
+  if (typeof obj.adminEnabled === 'boolean') rule.adminEnabled = obj.adminEnabled
+  const orgIds = normalizeIds(obj.orgIds)
+  if (orgIds) rule.orgIds = orgIds
+  const userIds = normalizeIds(obj.userIds)
+  if (userIds) rule.userIds = userIds
+  return rule
+}
+
+/** Coerce an arbitrary AppConfig/JSON document into a rule map, dropping malformed entries. */
+export function parseGateConfig(json: unknown): Record<string, AppConfigGateRule> {
+  const obj = (json && typeof json === 'object' ? json : {}) as Record<string, unknown>
+  const rules: Record<string, AppConfigGateRule> = {}
+  for (const [key, value] of Object.entries(obj)) {
+    const rule = normalizeRule(value)
+    if (rule) rules[key] = rule
+  }
+  return rules
+}
+
+/**
+ * Pure OR-of-clauses check. The caller supplies `isAdmin` — pass `false` to
+ * evaluate only the non-admin clauses (for lazy admin resolution).
+ */
+export function matchesRule(
+  rule: AppConfigGateRule | undefined,
+  ctx: AppConfigGateContext,
+  isAdmin: boolean
+): boolean {
+  if (!rule) return false
+  if (rule.enabled) return true
+  if (ctx.userId && rule.userIds?.includes(ctx.userId)) return true
+  if (ctx.orgId && rule.orgIds?.includes(ctx.orgId)) return true
+  if (rule.adminEnabled && isAdmin) return true
+  return false
+}

apps/sim/lib/core/config/appconfig.test.ts

@@ -0,0 +1,136 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockSend } = vi.hoisted(() => ({
+  mockSend: vi.fn(),
+}))
+
+vi.mock('@aws-sdk/client-appconfigdata', () => ({
+  AppConfigDataClient: class {
+    send = mockSend
+  },
+  StartConfigurationSessionCommand: class {
+    __type = 'start'
+    constructor(public input: unknown) {}
+  },
+  GetLatestConfigurationCommand: class {
+    __type = 'get'
+    constructor(public input: unknown) {}
+  },
+}))
+
+import { fetchAppConfigProfile } from '@/lib/core/config/appconfig'
+
+const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value))
+
+let counter = 0
+/** Unique identifiers per test so the module-level cache never bleeds across tests. */
+function uniqueIds() {
+  counter += 1
+  return { application: `app-${counter}`, environment: `env-${counter}`, profile: 'access-control' }
+}
+
+describe('fetchAppConfigProfile', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('starts a session then returns the parsed configuration', async () => {
+    mockSend.mockImplementation((command: { __type: string }) => {
+      if (command.__type === 'start') return Promise.resolve({ InitialConfigurationToken: 'tok-1' })
+      return Promise.resolve({
+        Configuration: encode({ blockedSignupDomains: ['spam.example'] }),
+        NextPollConfigurationToken: 'tok-2',
+      })
+    })
+
+    const result = await fetchAppConfigProfile(
+      uniqueIds(),
+      (json) => json as Record<string, unknown>
+    )
+    expect(result).toEqual({ blockedSignupDomains: ['spam.example'] })
+
+    const sentTypes = mockSend.mock.calls.map(([c]) => c.__type)
+    expect(sentTypes).toEqual(['start', 'get'])
+  })
+
+  it('returns null when the cold fetch fails (never throws)', async () => {
+    mockSend.mockRejectedValue(new Error('appconfig down'))
+    const result = await fetchAppConfigProfile(uniqueIds(), (json) => json)
+    expect(result).toBeNull()
+  })
+
+  it('applies the parse function to the decoded JSON', async () => {
+    mockSend.mockImplementation((command: { __type: string }) => {
+      if (command.__type === 'start') return Promise.resolve({ InitialConfigurationToken: 'tok-1' })
+      return Promise.resolve({
+        Configuration: encode({ count: 2 }),
+        NextPollConfigurationToken: 'tok-2',
+      })
+    })
+
+    const result = await fetchAppConfigProfile(
+      uniqueIds(),
+      (json) => (json as { count: number }).count * 10
+    )
+    expect(result).toBe(20)
+  })
+
+  it('warms the cache on an empty payload and does not re-poll (unseeded profile)', async () => {
+    mockSend.mockImplementation((command: { __type: string }) => {
+      if (command.__type === 'start') return Promise.resolve({ InitialConfigurationToken: 'tok-1' })
+      return Promise.resolve({
+        Configuration: new Uint8Array(),
+        NextPollConfigurationToken: 'tok-2',
+        NextPollIntervalInSeconds: 60,
+      })
+    })
+
+    const ids = uniqueIds()
+    expect(await fetchAppConfigProfile(ids, (json) => json)).toBeNull()
+    const callsAfterFirst = mockSend.mock.calls.length
+
+    expect(await fetchAppConfigProfile(ids, (json) => json)).toBeNull()
+    expect(mockSend.mock.calls.length).toBe(callsAfterFirst)
+  })
+
+  it('keeps the session on a parse error (no re-StartConfigurationSession, no throw)', async () => {
+    mockSend.mockImplementation((command: { __type: string }) => {
+      if (command.__type === 'start') return Promise.resolve({ InitialConfigurationToken: 'tok-1' })
+      return Promise.resolve({
+        Configuration: new TextEncoder().encode('not json{'),
+        NextPollConfigurationToken: 'tok-2',
+        NextPollIntervalInSeconds: 60,
+      })
+    })
+
+    const ids = uniqueIds()
+    expect(await fetchAppConfigProfile(ids, (json) => json)).toBeNull()
+
+    // Network round trip succeeded, so exactly one session was started despite the
+    // parse failure — the rotated token was preserved, not discarded.
+    expect(mockSend.mock.calls.filter(([c]) => c.__type === 'start')).toHaveLength(1)
+  })
+
+  it('dedupes concurrent cold fetches into a single poll', async () => {
+    mockSend.mockImplementation((command: { __type: string }) => {
+      if (command.__type === 'start') return Promise.resolve({ InitialConfigurationToken: 'tok-1' })
+      return Promise.resolve({
+        Configuration: encode({ x: 1 }),
+        NextPollConfigurationToken: 'tok-2',
+      })
+    })
+
+    const ids = uniqueIds()
+    const [a, b] = await Promise.all([
+      fetchAppConfigProfile(ids, (json) => json),
+      fetchAppConfigProfile(ids, (json) => json),
+    ])
+
+    expect(a).toEqual({ x: 1 })
+    expect(b).toEqual({ x: 1 })
+    expect(mockSend.mock.calls.map(([c]) => c.__type)).toEqual(['start', 'get'])
+  })
+})

apps/sim/lib/core/config/appconfig.ts

@@ -0,0 +1,167 @@
+import {
+  AppConfigDataClient,
+  GetLatestConfigurationCommand,
+  type GetLatestConfigurationCommandOutput,
+  StartConfigurationSessionCommand,
+} from '@aws-sdk/client-appconfigdata'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { getAwsCredentialsFromEnv } from '@/lib/core/config/aws'
+import { env } from '@/lib/core/config/env'
+
+const logger = createLogger('AppConfig')
+
+const DEFAULT_TTL_MS = 30_000
+
+export interface AppConfigProfileIdentifiers {
+  application: string
+  environment: string
+  profile: string
+}
+
+interface CacheEntry<T> {
+  /** Last successfully parsed value, or `null` if the config is empty/unseeded. */
+  value: T | null
+  /** True once any poll has completed (success, empty payload, or error). */
+  loaded: boolean
+  /** Token for the next `GetLatestConfiguration` poll, rotated on each call. */
+  nextToken: string | undefined
+  expiresAt: number
+  /** In-flight poll, shared so concurrent callers don't each hit AppConfig. */
+  inflight: Promise<T | null> | null
+}
+
+const cache = new Map<string, CacheEntry<unknown>>()
+
+let client: AppConfigDataClient | null = null
+
+/**
+ * Lazily construct the AppConfig data-plane client. Never instantiated unless a
+ * caller actually fetches a profile, so deployments without AppConfig configured
+ * never reach for AWS credentials.
+ */
+function getClient(): AppConfigDataClient {
+  if (!client) {
+    client = new AppConfigDataClient({
+      region: env.AWS_REGION,
+      credentials: getAwsCredentialsFromEnv(),
+    })
+  }
+  return client
+}
+
+function cacheKey(ids: AppConfigProfileIdentifiers): string {
+  return `${ids.application}/${ids.environment}/${ids.profile}`
+}
+
+/**
+ * Run one AppConfig poll for `entry`: starts a session if no token is held, then
+ * calls `GetLatestConfiguration`. An empty payload means "unchanged" (or an
+ * unseeded profile) and the previous value is kept. Any error is logged and the
+ * last good value is retained. Marks the entry `loaded` on any outcome so callers
+ * never re-block on the cold path, and honors AppConfig's `NextPollInterval` so we
+ * don't poll faster than the server allows (which would throttle).
+ */
+async function poll<T>(
+  ids: AppConfigProfileIdentifiers,
+  parse: (json: unknown) => T,
+  entry: CacheEntry<T>
+): Promise<T | null> {
+  let response: GetLatestConfigurationCommandOutput
+  try {
+    const dataClient = getClient()
+
+    if (!entry.nextToken) {
+      const session = await dataClient.send(
+        new StartConfigurationSessionCommand({
+          ApplicationIdentifier: ids.application,
+          EnvironmentIdentifier: ids.environment,
+          ConfigurationProfileIdentifier: ids.profile,
+        })
+      )
+      entry.nextToken = session.InitialConfigurationToken
+    }
+
+    response = await dataClient.send(
+      new GetLatestConfigurationCommand({ ConfigurationToken: entry.nextToken })
+    )
+    entry.nextToken = response.NextPollConfigurationToken ?? entry.nextToken
+  } catch (error) {
+    // Network/session failure: drop the token so the next attempt starts a fresh
+    // session (handles expired or invalid tokens). Mark loaded + back off so we
+    // serve the fallback and retry in the background rather than blocking every
+    // request during an outage.
+    entry.nextToken = undefined
+    entry.expiresAt = Date.now() + DEFAULT_TTL_MS
+    entry.loaded = true
+    logger.error('AppConfig fetch failed; serving last known value', {
+      profile: cacheKey(ids),
+      error: getErrorMessage(error),
+    })
+    return entry.value
+  }
+
+  // Parse outside the network try: a decode/parse error must NOT discard the
+  // already-rotated session token — the round trip succeeded, so the next poll
+  // can reuse it instead of opening a new session. Keep the last good value.
+  try {
+    if (response.Configuration && response.Configuration.length > 0) {
+      const text = new TextDecoder().decode(response.Configuration)
+      entry.value = parse(JSON.parse(text))
+    }
+  } catch (error) {
+    logger.error('AppConfig response parse failed; serving last known value', {
+      profile: cacheKey(ids),
+      error: getErrorMessage(error),
+    })
+  }
+
+  const intervalMs = (response.NextPollIntervalInSeconds ?? 60) * 1000
+  entry.expiresAt = Date.now() + Math.max(DEFAULT_TTL_MS, intervalMs)
+  entry.loaded = true
+  return entry.value
+}
+
+/**
+ * Fetch and cache a single AppConfig configuration profile as JSON.
+ *
+ * Profile-agnostic: pass the `application`/`environment` (from env) and a
+ * `profile` constant owned by the calling feature. Uses an in-process TTL cache
+ * with stale-while-revalidate — a warm cache returns immediately and refreshes
+ * in the background once the TTL lapses, so no request blocks on the AppConfig
+ * round trip after the first (cold) fetch. Concurrent callers share one in-flight
+ * poll (avoids racing the rotating session token). Returns `null` when the config
+ * is empty/unseeded or the first fetch fails.
+ */
+export async function fetchAppConfigProfile<T>(
+  ids: AppConfigProfileIdentifiers,
+  parse: (json: unknown) => T
+): Promise<T | null> {
+  const key = cacheKey(ids)
+  const entry = (cache.get(key) as CacheEntry<T> | undefined) ?? {
+    value: null,
+    loaded: false,
+    nextToken: undefined,
+    expiresAt: 0,
+    inflight: null,
+  }
+  cache.set(key, entry)
+
+  // Cold: never polled — await a single shared poll so concurrent callers don't
+  // each hit AppConfig (and don't race the rotating session token).
+  if (!entry.loaded) {
+    entry.inflight ??= poll(ids, parse, entry).finally(() => {
+      entry.inflight = null
+    })
+    return entry.inflight
+  }
+
+  // Warm but stale: serve cached value, refresh once in the background.
+  if (Date.now() >= entry.expiresAt && !entry.inflight) {
+    entry.inflight = poll(ids, parse, entry).finally(() => {
+      entry.inflight = null
+    })
+  }
+
+  return entry.value
+}

apps/sim/lib/core/config/aws.ts

@@ -0,0 +1,24 @@
+import { env } from '@/lib/core/config/env'
+
+interface AwsCredentials {
+  accessKeyId: string
+  secretAccessKey: string
+}
+
+/**
+ * Explicit AWS credentials from the environment, or `undefined` to defer to the
+ * default AWS provider chain (the ECS task role in our deployments).
+ *
+ * Shared by every AWS SDK client (S3, AppConfig, …) so credential resolution is
+ * identical everywhere: explicit keys when both `AWS_ACCESS_KEY_ID` and
+ * `AWS_SECRET_ACCESS_KEY` are set (self-hosted, trigger.dev workers), otherwise
+ * the instance/task role.
+ */
+export function getAwsCredentialsFromEnv(): AwsCredentials | undefined {
+  return env.AWS_ACCESS_KEY_ID && env.AWS_SECRET_ACCESS_KEY
+    ? {
+        accessKeyId: env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
+      }
+    : undefined
+}

apps/sim/lib/core/config/block-visibility.test.ts

@@ -0,0 +1,157 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockFetch, mockIsPlatformAdmin, envRef, flagRef } = vi.hoisted(() => ({
+  mockFetch: vi.fn(),
+  mockIsPlatformAdmin: vi.fn(),
+  envRef: {
+    APPCONFIG_APPLICATION: 'sim-staging' as string | undefined,
+    APPCONFIG_ENVIRONMENT: 'staging' as string | undefined,
+  },
+  flagRef: { isAppConfigEnabled: false, previewBlocks: [] as string[] },
+}))
+
+vi.mock('@/lib/core/config/appconfig', () => ({
+  fetchAppConfigProfile: mockFetch,
+}))
+
+vi.mock('@/lib/core/config/env', () => ({
+  get env() {
+    return envRef
+  },
+}))
+
+vi.mock('@/lib/core/config/env-flags', () => ({
+  get isAppConfigEnabled() {
+    return flagRef.isAppConfigEnabled
+  },
+  getPreviewBlocksFromEnv: () => flagRef.previewBlocks,
+}))
+
+vi.mock('@/lib/permissions/super-user', () => ({
+  isPlatformAdmin: mockIsPlatformAdmin,
+}))
+
+import { getBlockVisibility } from '@/lib/core/config/block-visibility'
+
+/** Make `getBlockVisibility` resolve `doc` via the AppConfig path (also exercises parsing). */
+function withAppConfig(doc: unknown) {
+  flagRef.isAppConfigEnabled = true
+  mockFetch.mockImplementation((_ids, parse) => Promise.resolve(parse(doc)))
+}
+
+describe('getBlockVisibility', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    flagRef.isAppConfigEnabled = false
+    flagRef.previewBlocks = []
+  })
+
+  describe('off-AppConfig (env fallback)', () => {
+    it('reveals and preview-tags the PREVIEW_BLOCKS types without fetching', async () => {
+      flagRef.previewBlocks = ['gmail_v2', 'notion_v3']
+      const vis = await getBlockVisibility({ userId: 'u1' })
+      expect(vis.revealed).toEqual(new Set(['gmail_v2', 'notion_v3']))
+      expect(vis.previewTagged).toEqual(new Set(['gmail_v2', 'notion_v3']))
+      expect(vis.disabled.size).toBe(0)
+      expect(mockFetch).not.toHaveBeenCalled()
+    })
+
+    it('returns empty state when PREVIEW_BLOCKS is unset', async () => {
+      const vis = await getBlockVisibility()
+      expect(vis.revealed.size).toBe(0)
+      expect(vis.disabled.size).toBe(0)
+      expect(vis.previewTagged.size).toBe(0)
+    })
+  })
+
+  it('fetches the block-visibility profile', async () => {
+    withAppConfig({})
+    await getBlockVisibility()
+    expect(mockFetch).toHaveBeenCalledWith(
+      { application: 'sim-staging', environment: 'staging', profile: 'block-visibility' },
+      expect.any(Function)
+    )
+  })
+
+  it('GA rule (enabled: true) reveals without a preview tag', async () => {
+    withAppConfig({ gmail_v2: { enabled: true } })
+    const vis = await getBlockVisibility({ userId: 'u1' })
+    expect(vis.revealed.has('gmail_v2')).toBe(true)
+    expect(vis.previewTagged.has('gmail_v2')).toBe(false)
+    expect(vis.disabled.has('gmail_v2')).toBe(false)
+  })
+
+  it('allowlist rule reveals with a preview tag; non-matching viewers get disabled', async () => {
+    withAppConfig({ gmail_v2: { enabled: false, orgIds: ['o1'], userIds: ['u9'] } })
+
+    const allowedOrg = await getBlockVisibility({ orgId: 'o1' })
+    expect(allowedOrg.revealed.has('gmail_v2')).toBe(true)
+    expect(allowedOrg.previewTagged.has('gmail_v2')).toBe(true)
+
+    const allowedUser = await getBlockVisibility({ userId: 'u9' })
+    expect(allowedUser.revealed.has('gmail_v2')).toBe(true)
+
+    const denied = await getBlockVisibility({ userId: 'u1', orgId: 'o2' })
+    expect(denied.revealed.has('gmail_v2')).toBe(false)
+    expect(denied.disabled.has('gmail_v2')).toBe(true)
+  })
+
+  it('kill switch (enabled: false, no allowlists) disables for everyone', async () => {
+    withAppConfig({ slack: { enabled: false } })
+    const vis = await getBlockVisibility({ userId: 'u1', orgId: 'o1' })
+    expect(vis.disabled.has('slack')).toBe(true)
+    expect(vis.revealed.has('slack')).toBe(false)
+  })
+
+  it('drops custom_block_* keys so custom blocks can never be gated', async () => {
+    withAppConfig({ custom_block_abc123: { enabled: false }, gmail_v2: { enabled: true } })
+    const vis = await getBlockVisibility({ userId: 'u1' })
+    expect(vis.disabled.has('custom_block_abc123')).toBe(false)
+    expect(vis.revealed.has('custom_block_abc123')).toBe(false)
+    expect(vis.revealed.has('gmail_v2')).toBe(true)
+  })
+
+  it('drops malformed entries', async () => {
+    withAppConfig({ a: 'nope', b: { enabled: false, orgIds: [' o1 ', ''] } })
+    const vis = await getBlockVisibility({ orgId: 'o1' })
+    expect(vis.disabled.has('a')).toBe(false)
+    expect(vis.revealed.has('b')).toBe(true)
+  })
+
+  describe('admin resolution (once per call)', () => {
+    it('resolves admin exactly once for a document with multiple adminEnabled rules', async () => {
+      withAppConfig({
+        a: { enabled: false, adminEnabled: true },
+        b: { enabled: false, adminEnabled: true },
+        c: { enabled: false },
+      })
+      mockIsPlatformAdmin.mockResolvedValue(true)
+      const vis = await getBlockVisibility({ userId: 'u1' })
+      expect(mockIsPlatformAdmin).toHaveBeenCalledTimes(1)
+      expect(vis.revealed).toEqual(new Set(['a', 'b']))
+      expect(vis.previewTagged).toEqual(new Set(['a', 'b']))
+      expect(vis.disabled).toEqual(new Set(['c']))
+    })
+
+    it('uses the isAdmin fast-path without querying', async () => {
+      withAppConfig({ a: { enabled: false, adminEnabled: true } })
+      const vis = await getBlockVisibility({ userId: 'u1', isAdmin: true })
+      expect(vis.revealed.has('a')).toBe(true)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+
+    it('does not query when no rule has adminEnabled or when userId is absent', async () => {
+      withAppConfig({ a: { enabled: false, orgIds: ['o1'] } })
+      await getBlockVisibility({ userId: 'u1' })
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+
+      withAppConfig({ a: { enabled: false, adminEnabled: true } })
+      const vis = await getBlockVisibility({ orgId: 'o1' })
+      expect(vis.disabled.has('a')).toBe(true)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+  })
+})

apps/sim/lib/core/config/block-visibility.ts

@@ -0,0 +1,110 @@
+import { fetchAppConfigProfile } from '@/lib/core/config/appconfig'
+import type { AppConfigGateContext, AppConfigGateRule } from '@/lib/core/config/appconfig-rules'
+import { matchesRule, parseGateConfig } from '@/lib/core/config/appconfig-rules'
+import { env } from '@/lib/core/config/env'
+import { getPreviewBlocksFromEnv, isAppConfigEnabled } from '@/lib/core/config/env-flags'
+
+/**
+ * Name of the AppConfig configuration profile holding per-block visibility rules.
+ * Cross-repo contract: must match the `CfnConfigurationProfile` name created by
+ * the infra stack (`BLOCK_VISIBILITY_PROFILE_NAME`).
+ */
+const BLOCK_VISIBILITY_PROFILE = 'block-visibility'
+
+/**
+ * Custom (deploy-as-block) block types are org-scoped and managed by their own
+ * enabled/disabled lifecycle — the visibility document must never gate them.
+ * Literal mirrors `CUSTOM_BLOCK_TYPE_PREFIX` in `@/blocks/custom/build-config`,
+ * not imported to keep the blocks graph out of this config module.
+ */
+const CUSTOM_BLOCK_KEY_PREFIX = 'custom_block_'
+
+/** Per-request evaluation context; same shape as the feature-flag context. */
+export type BlockVisibilityContext = AppConfigGateContext
+
+/**
+ * The evaluated per-viewer visibility projection.
+ *
+ * - `revealed` — preview block types this viewer may see.
+ * - `disabled` — types whose rule exists but matched no clause; hides
+ *   non-preview (shipped) blocks from discovery surfaces (the kill switch).
+ * - `previewTagged` — revealed types not globally GA (`enabled !== true`);
+ *   the registry appends " (Preview)" to their names.
+ *
+ * All three are needed: `revealed \ previewTagged` is the "GA'd via config while
+ * `preview: true` is still in code" window, and `disabled` targets a disjoint
+ * (non-preview) population.
+ */
+export interface BlockVisibilityState {
+  revealed: Set<string>
+  disabled: Set<string>
+  previewTagged: Set<string>
+}
+
+function parseVisibilityConfig(json: unknown): Record<string, AppConfigGateRule> {
+  const rules = parseGateConfig(json)
+  for (const key of Object.keys(rules)) {
+    if (key.startsWith(CUSTOM_BLOCK_KEY_PREFIX)) delete rules[key]
+  }
+  return rules
+}
+
+/**
+ * Resolve platform-admin status lazily. Dynamically imported so the DB-backed
+ * helper (and `@sim/db`) stay out of this config module's load graph for callers
+ * that never reach an admin-gated rule.
+ */
+async function resolveAdmin(userId: string): Promise<boolean> {
+  const { isPlatformAdmin } = await import('@/lib/permissions/super-user')
+  return isPlatformAdmin(userId)
+}
+
+/**
+ * Evaluate the block-visibility document for a viewer.
+ *
+ * On hosted deployments the rules come from the AppConfig profile (cached,
+ * ~30s TTL); off-AppConfig the `PREVIEW_BLOCKS` env allowlist is the only
+ * reveal path and nothing is disabled.
+ *
+ * Unlike feature-flags (one rule per call, admin resolved lazily per rule),
+ * this evaluates the whole document, so platform-admin status is resolved at
+ * most ONCE per call — and only when some rule actually has `adminEnabled` and
+ * the caller didn't already supply `ctx.isAdmin`.
+ */
+export async function getBlockVisibility(
+  ctx: BlockVisibilityContext = {}
+): Promise<BlockVisibilityState> {
+  if (!isAppConfigEnabled) {
+    const revealed = new Set(getPreviewBlocksFromEnv())
+    return { revealed, disabled: new Set(), previewTagged: new Set(revealed) }
+  }
+
+  const rules =
+    (await fetchAppConfigProfile(
+      {
+        application: env.APPCONFIG_APPLICATION as string,
+        environment: env.APPCONFIG_ENVIRONMENT as string,
+        profile: BLOCK_VISIBILITY_PROFILE,
+      },
+      parseVisibilityConfig
+    )) ?? {}
+
+  const needsAdmin =
+    ctx.isAdmin === undefined &&
+    Boolean(ctx.userId) &&
+    Object.values(rules).some((rule) => rule.adminEnabled)
+  const isAdmin = ctx.isAdmin ?? (needsAdmin ? await resolveAdmin(ctx.userId as string) : false)
+
+  const revealed = new Set<string>()
+  const disabled = new Set<string>()
+  const previewTagged = new Set<string>()
+  for (const [type, rule] of Object.entries(rules)) {
+    if (matchesRule(rule, ctx, isAdmin)) {
+      revealed.add(type)
+      if (rule.enabled !== true) previewTagged.add(type)
+    } else {
+      disabled.add(type)
+    }
+  }
+  return { revealed, disabled, previewTagged }
+}

apps/sim/lib/core/config/env-flags.ts

@@ -0,0 +1,354 @@
+/**
+ * Environment utility functions for consistent environment detection across the application
+ */
+import { env, getEnv, isFalsy, isTruthy } from './env'
+
+/**
+ * Is the application running in production mode
+ */
+export const isProd = env.NODE_ENV === 'production'
+
+/**
+ * Is the application running in development mode
+ */
+export const isDev = env.NODE_ENV === 'development'
+
+/**
+ * Is the application running in test mode
+ */
+export const isTest = env.NODE_ENV === 'test'
+
+/**
+ * Is this the hosted version of the application.
+ * True for sim.ai and any subdomain of sim.ai (e.g. staging.sim.ai, dev.sim.ai).
+ */
+const appUrl = getEnv('NEXT_PUBLIC_APP_URL')
+let appHostname = ''
+try {
+  appHostname = appUrl ? new URL(appUrl).hostname : ''
+} catch {
+  // invalid URL — isHosted stays false
+}
+export const isHosted = appHostname === 'sim.ai' || appHostname.endsWith('.sim.ai')
+
+/**
+ * Is billing enforcement enabled
+ */
+export const isBillingEnabled = isTruthy(env.BILLING_ENABLED)
+
+/**
+ * Block free-plan accounts from programmatic workflow execution (API key, public
+ * API, MCP server, generic webhooks, cross-origin chat embeds).
+ * Gated behind {@link isBillingEnabled}; off by default so the paywall can ship
+ * dark and be enabled per-deployment once verified.
+ */
+export const isFreeApiDeploymentGateEnabled = isTruthy(env.FREE_API_DEPLOYMENT_GATE_ENABLED)
+
+/**
+ * Is email verification enabled
+ */
+export const isEmailVerificationEnabled = isTruthy(env.EMAIL_VERIFICATION_ENABLED)
+
+/**
+ * Is authentication disabled (for self-hosted deployments behind private networks)
+ * This flag is blocked when isHosted is true.
+ */
+export const isAuthDisabled = isTruthy(env.DISABLE_AUTH) && !isHosted
+
+if (isTruthy(env.DISABLE_AUTH)) {
+  import('@sim/logger')
+    .then(({ createLogger }) => {
+      const logger = createLogger('EnvFlags')
+      if (isHosted) {
+        logger.error(
+          'DISABLE_AUTH is set but ignored on hosted environment. Authentication remains enabled for security.'
+        )
+      } else {
+        logger.warn(
+          'DISABLE_AUTH is enabled. Authentication is bypassed and all requests use an anonymous session. Only use this in trusted private networks.'
+        )
+      }
+    })
+    .catch(() => {
+      // Fallback during config compilation when logger is unavailable
+    })
+}
+
+/**
+ * Whether database/connector tools may connect to private, reserved, or loopback
+ * hosts (e.g. Docker/K8s service names, localhost). Off by default: the SSRF guard
+ * in {@link validateDatabaseHost} blocks these so an untrusted user cannot pivot
+ * into the deployment's internal network. Self-hosted operators can opt in when
+ * their database lives on the same private network. Blocked on the hosted platform
+ * regardless of the env var, mirroring {@link isAuthDisabled}.
+ */
+export const isPrivateDatabaseHostsAllowed = isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS) && !isHosted
+
+if (isTruthy(env.ALLOW_PRIVATE_DATABASE_HOSTS)) {
+  import('@sim/logger')
+    .then(({ createLogger }) => {
+      const logger = createLogger('EnvFlags')
+      if (isHosted) {
+        logger.error(
+          'ALLOW_PRIVATE_DATABASE_HOSTS is set but ignored on hosted environment. Private/reserved database hosts remain blocked for security.'
+        )
+      } else {
+        logger.warn(
+          'ALLOW_PRIVATE_DATABASE_HOSTS is enabled. Database/connector tools may reach private, reserved, and loopback hosts. Only use this in trusted private networks.'
+        )
+      }
+    })
+    .catch(() => {
+      // Fallback during config compilation when logger is unavailable
+    })
+}
+
+/**
+ * Is user registration disabled
+ */
+export const isRegistrationDisabled = isTruthy(env.DISABLE_REGISTRATION)
+
+/**
+ * Is email/password authentication enabled (defaults to true)
+ */
+export const isEmailPasswordEnabled = !isFalsy(env.EMAIL_PASSWORD_SIGNUP_ENABLED)
+
+/**
+ * Is MX-based signup validation enabled (blocks no-MX domains and denylisted shared spam
+ * mail backends). Opt-in to avoid adding a DNS dependency or blocking legitimate signups on
+ * self-hosted deployments with non-standard mail setups; enable on abuse-targeted deployments.
+ */
+export const isSignupMxValidationEnabled = isTruthy(env.SIGNUP_MX_VALIDATION_ENABLED)
+
+/**
+ * Is AWS AppConfig the source of truth for the signup/login gating lists.
+ * Hosted-only and requires both AppConfig identifiers (injected by the infra
+ * stack). Self-hosted/OSS deployments always use the env-var fallback, so the
+ * AppConfig client is never reached off-hosted.
+ */
+export const isAppConfigEnabled =
+  isHosted && Boolean(env.APPCONFIG_APPLICATION && env.APPCONFIG_ENVIRONMENT)
+
+/**
+ * Is Trigger.dev enabled for async job processing
+ */
+export const isTriggerDevEnabled = isTruthy(env.TRIGGER_DEV_ENABLED)
+
+/**
+ * Is SSO enabled for enterprise authentication
+ */
+export const isSsoEnabled = isTruthy(env.SSO_ENABLED)
+
+/**
+ * Is access control (permission groups) enabled via env var override
+ * This bypasses plan requirements for self-hosted deployments
+ */
+export const isAccessControlEnabled = isTruthy(env.ACCESS_CONTROL_ENABLED)
+
+/**
+ * Is organizations enabled
+ * True if billing is enabled (orgs come with billing), OR explicitly enabled via env var,
+ * OR if access control is enabled (access control requires organizations)
+ */
+export const isOrganizationsEnabled =
+  isBillingEnabled || isTruthy(env.ORGANIZATIONS_ENABLED) || isAccessControlEnabled
+
+/**
+ * Is inbox (Sim Mailer) enabled via env var override
+ * This bypasses hosted requirements for self-hosted deployments
+ */
+export const isInboxEnabled = isTruthy(env.INBOX_ENABLED)
+
+/**
+ * Is whitelabeling enabled via env var override
+ * This bypasses hosted requirements for self-hosted deployments
+ */
+export const isWhitelabelingEnabled = isTruthy(env.WHITELABELING_ENABLED)
+
+/**
+ * Is audit logs enabled via env var override
+ * This bypasses hosted requirements for self-hosted deployments
+ */
+export const isAuditLogsEnabled = isTruthy(env.AUDIT_LOGS_ENABLED)
+
+/**
+ * Is data retention enabled via env var override
+ * This bypasses hosted requirements for self-hosted deployments
+ */
+export const isDataRetentionEnabled = isTruthy(env.DATA_RETENTION_ENABLED)
+
+/**
+ * Is data drains enabled via env var override
+ * This bypasses hosted requirements for self-hosted deployments
+ */
+export const isDataDrainsEnabled = isTruthy(env.DATA_DRAINS_ENABLED)
+
+/**
+ * Is workspace forking enabled via env var override
+ * This bypasses hosted (Enterprise) requirements for self-hosted deployments
+ */
+export const isForkingEnabled = isTruthy(env.FORKING_ENABLED)
+
+/**
+ * Is E2B enabled for remote code execution
+ */
+export const isE2bEnabled = isTruthy(env.E2B_ENABLED)
+
+/**
+ * Whether the E2B document-generation sandbox is enabled.
+ *
+ * Requires E2B (with an API key) AND a dedicated doc-generation template id.
+ * When true, ALL four formats compile in the E2B doc sandbox: pptx/docx via Node
+ * (pptxgenjs/docx + react-icons/sharp icons), pdf/xlsx via Python
+ * (reportlab/openpyxl). When false, compilation stays on the JavaScript
+ * (isolated-vm) path, byte-identical to its prior behavior (and xlsx is
+ * unavailable). Drives both the Sim compile backend and the `docCompiler` flag
+ * sent to the copilot file subagent so the agent's output and compiler agree.
+ */
+export const isE2BDocEnabled =
+  isE2bEnabled && Boolean(env.E2B_API_KEY) && Boolean(env.MOTHERSHIP_E2B_DOC_TEMPLATE_ID)
+
+/**
+ * Whether Ollama is configured (OLLAMA_URL is set).
+ * When true, models that are not in the static cloud model list and have no
+ * slash-prefixed provider namespace are assumed to be Ollama models
+ * and do not require an API key.
+ */
+export const isOllamaConfigured = Boolean(env.OLLAMA_URL)
+
+/**
+ * Whether Azure OpenAI / Azure Anthropic credentials are pre-configured at the server level
+ * (via AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_API_KEY, AZURE_ANTHROPIC_ENDPOINT, etc.).
+ * When true, the endpoint, API key, and API version fields are hidden in the Agent block UI.
+ * Set NEXT_PUBLIC_AZURE_CONFIGURED=true in self-hosted deployments on Azure.
+ */
+export const isAzureConfigured = isTruthy(getEnv('NEXT_PUBLIC_AZURE_CONFIGURED'))
+
+/**
+ * Whether a Cohere API key is pre-configured server-side for the Knowledge block reranker
+ * (`COHERE_API_KEY` or `COHERE_API_KEY_1/2/3`). When true, the Cohere API Key field is hidden
+ * in the Knowledge block UI.
+ * Set NEXT_PUBLIC_COHERE_CONFIGURED=true in self-hosted deployments that ship a Cohere key.
+ */
+export const isCohereConfigured = isTruthy(getEnv('NEXT_PUBLIC_COHERE_CONFIGURED'))
+
+/**
+ * Are invitations disabled globally
+ * When true, workspace invitations are disabled for all users
+ */
+export const isInvitationsDisabled = isTruthy(env.DISABLE_INVITATIONS)
+
+/**
+ * Is public API access disabled globally
+ * When true, the public API toggle is hidden and public API access is blocked
+ */
+export const isPublicApiDisabled = isTruthy(env.DISABLE_PUBLIC_API)
+
+/**
+ * Is Google OAuth login disabled
+ * When true, the Google OAuth login button is hidden even when credentials are configured
+ */
+export const isGoogleAuthDisabled = isTruthy(env.DISABLE_GOOGLE_AUTH)
+
+/**
+ * Is GitHub OAuth login disabled
+ * When true, the GitHub OAuth login button is hidden even when credentials are configured
+ */
+export const isGithubAuthDisabled = isTruthy(env.DISABLE_GITHUB_AUTH)
+
+/**
+ * Is Microsoft OAuth login disabled
+ * When true, the Microsoft OAuth login button is hidden even when credentials are configured
+ */
+export const isMicrosoftAuthDisabled = isTruthy(env.DISABLE_MICROSOFT_AUTH)
+
+/**
+ * Is email/password signup disabled
+ * When true, new registrations via email/password are blocked at the server level.
+ * Existing users can still sign in with email/password.
+ */
+export const isEmailSignupDisabled = isTruthy(env.DISABLE_EMAIL_SIGNUP)
+
+/**
+ * Is React Grab enabled for UI element debugging
+ * When true and in development mode, enables React Grab for copying UI element context to clipboard
+ */
+export const isReactGrabEnabled = isDev && isTruthy(env.REACT_GRAB_ENABLED)
+
+/**
+ * Is React Scan enabled for performance debugging
+ * When true and in development mode, enables React Scan for detecting render performance issues
+ */
+export const isReactScanEnabled = isDev && isTruthy(env.REACT_SCAN_ENABLED)
+
+/**
+ * Returns the parsed allowlist of integration block types from the environment variable.
+ * If not set or empty, returns null (meaning all integrations are allowed).
+ */
+export function getAllowedIntegrationsFromEnv(): string[] | null {
+  if (!env.ALLOWED_INTEGRATIONS) return null
+  const parsed = env.ALLOWED_INTEGRATIONS.split(',')
+    .map((i) => i.trim().toLowerCase())
+    .filter(Boolean)
+  return parsed.length > 0 ? parsed : null
+}
+
+/**
+ * Returns the preview block types revealed via the environment variable — the
+ * off-AppConfig reveal path for self-hosters and local dev. If not set or empty,
+ * returns an empty array (all `preview: true` blocks stay hidden). Block types
+ * are already lowercase snake_case, so entries are trimmed but not lowercased.
+ */
+export function getPreviewBlocksFromEnv(): string[] {
+  if (!env.PREVIEW_BLOCKS) return []
+  return env.PREVIEW_BLOCKS.split(',')
+    .map((t) => t.trim())
+    .filter(Boolean)
+}
+
+/**
+ * Returns the list of blacklisted provider IDs from the environment variable.
+ * If not set or empty, returns an empty array (meaning no providers are blacklisted).
+ */
+export function getBlacklistedProvidersFromEnv(): string[] {
+  if (!env.BLACKLISTED_PROVIDERS) return []
+  return env.BLACKLISTED_PROVIDERS.split(',')
+    .map((p) => p.trim().toLowerCase())
+    .filter(Boolean)
+}
+
+/**
+ * Normalizes a domain entry from the ALLOWED_MCP_DOMAINS env var.
+ * Accepts bare hostnames (e.g., "mcp.company.com") or full URLs (e.g., "https://mcp.company.com").
+ * Extracts the hostname in either case.
+ */
+function normalizeDomainEntry(entry: string): string {
+  const trimmed = entry.trim().toLowerCase()
+  if (!trimmed) return ''
+  if (trimmed.includes('://')) {
+    try {
+      return new URL(trimmed).hostname
+    } catch {
+      return trimmed
+    }
+  }
+  return trimmed
+}
+
+/**
+ * Get allowed MCP server domains from the ALLOWED_MCP_DOMAINS env var.
+ * Returns null if not set (all domains allowed), or parsed array of lowercase hostnames.
+ * Accepts both bare hostnames and full URLs in the env var value.
+ */
+export function getAllowedMcpDomainsFromEnv(): string[] | null {
+  if (!env.ALLOWED_MCP_DOMAINS) return null
+  const parsed = env.ALLOWED_MCP_DOMAINS.split(',').map(normalizeDomainEntry).filter(Boolean)
+  return parsed.length > 0 ? parsed : null
+}
+
+/**
+ * Get cost multiplier based on environment
+ */
+export function getCostMultiplier(): number {
+  return isProd ? (env.COST_MULTIPLIER ?? 1) : 1
+}

apps/sim/lib/core/config/env.test.ts

@@ -0,0 +1,13 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { envNumber } from '@/lib/core/config/env'
+
+describe('envNumber', () => {
+  it('can require integer env values for count-like settings', () => {
+    expect(envNumber('5', 1, { min: 1, integer: true })).toBe(5)
+    expect(envNumber('5.5', 1, { min: 1, integer: true })).toBe(1)
+    expect(envNumber(5.5, 1, { min: 1, integer: true })).toBe(1)
+  })
+})

apps/sim/lib/core/config/env.ts

@@ -0,0 +1,648 @@
+import { createEnv } from '@t3-oss/env-nextjs'
+import { z } from 'zod'
+
+/**
+ * Reads NEXT_PUBLIC_* env vars in both client and server contexts.
+ * Client reads `window.__ENV` (populated by `<PublicEnvScript>`); server reads `process.env`.
+ * We do not use next-runtime-env's `env()` helper because it calls `unstable_noStore()`,
+ * which Next 16.2+ rejects outside a request scope.
+ */
+const getEnv = (variable: string): string | undefined => {
+  if (typeof window === 'undefined') return process.env[variable]
+  return window.__ENV?.[variable] ?? process.env[variable]
+}
+
+// biome-ignore format: keep alignment for readability
+export const env = createEnv({
+  skipValidation: true,
+
+  server: {
+    // Core Database & Authentication
+    DATABASE_URL:                          z.string().url(),                       // Primary database connection string
+    DATABASE_REPLICA_URL:                  z.string().url().optional(),            // Read-replica connection string; opt-in reads fall back to the primary when unset
+    DB_APP_NAME:                           z.string().optional(),                  // Postgres application_name for query attribution (sim-app/sim-trigger/sim-realtime)
+    SIM_DB_ROLE:                           z.enum(['web', 'trigger', 'realtime']).optional(), // Per-process pool profile selector (read directly by @sim/db)
+    DATABASE_URL_WEB:                      z.string().url().optional(),            // Per-role primary URL override; @sim/db falls back to DATABASE_URL
+    DATABASE_URL_TRIGGER:                  z.string().url().optional(),            // Per-role primary URL override (trigger)
+    DATABASE_URL_REALTIME:                 z.string().url().optional(),            // Per-role primary URL override (realtime)
+    DATABASE_REPLICA_URL_WEB:              z.string().url().optional(),            // Per-role replica URL override; falls back to DATABASE_REPLICA_URL
+    DATABASE_REPLICA_URL_TRIGGER:          z.string().url().optional(),            // Per-role replica URL override (trigger)
+    DATABASE_REPLICA_URL_REALTIME:         z.string().url().optional(),            // Per-role replica URL override (realtime)
+    BETTER_AUTH_URL:                       z.string().url(),                       // Base URL for Better Auth service
+    BETTER_AUTH_SECRET:                    z.string().min(32),                     // Secret key for Better Auth JWT signing
+    DISABLE_REGISTRATION:                  z.boolean().optional(),                 // Flag to disable new user registration
+    EMAIL_PASSWORD_SIGNUP_ENABLED:         z.boolean().optional().default(true),   // Enable email/password authentication (server-side enforcement)
+    DISABLE_AUTH:                          z.boolean().optional(),                 // Bypass authentication entirely (self-hosted only, creates anonymous session)
+    ALLOW_PRIVATE_DATABASE_HOSTS:          z.boolean().optional(),                 // Opt-in (self-hosted only): let database/connector tools reach private/reserved/loopback hosts (e.g. Docker/K8s service names). Loosens the SSRF boundary; ignored on the hosted platform.
+    ALLOWED_LOGIN_EMAILS:                  z.string().optional(),                  // Comma-separated list of allowed email addresses for login
+    ALLOWED_LOGIN_DOMAINS:                 z.string().optional(),                  // Comma-separated list of allowed email domains for login
+    BLOCKED_SIGNUP_DOMAINS:                z.string().optional(),                  // Comma-separated list of email domains blocked from signing up (e.g., "gmail.com,yahoo.com")
+    BLOCKED_EMAILS:                        z.string().optional(),                  // Comma-separated list of specific email addresses banned from the platform (signup, sign-in, executions)
+    SIGNUP_MX_VALIDATION_ENABLED:          z.boolean().optional(),                 // Opt-in: validate the email's MX backend at signup (blocks no-MX domains and denylisted shared spam backends). Off by default; enable on hosted/abuse-targeted deployments.
+    BLOCKED_EMAIL_MX_HOSTS:                z.string().optional(),                  // Comma-separated MX-host substrings blocked from signing up; matched against the domain's resolved MX backend to catch throwaway domains that share a mail backend. No defaults — operators supply their own list. Only used when SIGNUP_MX_VALIDATION_ENABLED is set.
+    TRUSTED_ORIGINS:                       z.string().optional(),                  // Comma-separated additional origins to trust for auth (e.g., "https://app.example.com,https://www.example.com"). Merged into Better Auth trustedOrigins.
+    TURNSTILE_SECRET_KEY:                  z.string().min(1).optional(),           // Cloudflare Turnstile secret key for captcha verification
+    ENCRYPTION_KEY:                        z.string().min(32),                     // Key for encrypting sensitive data
+    API_ENCRYPTION_KEY:                    z.string().min(32).optional(),          // Dedicated key for encrypting API keys (optional for OSS)
+    INTERNAL_API_SECRET:                   z.string().min(32),                     // Secret for internal API authentication
+    INTERNAL_JWT_SECRET:                   z.string().min(32).optional(),          // Dedicated signing key for internal JWTs (falls back to INTERNAL_API_SECRET); separating limits blast radius if one leaks
+
+    // Copilot
+    COPILOT_API_KEY:                       z.string().min(1).optional(),           // Secret for internal sim agent API authentication
+    SIM_AGENT_API_URL:                     z.string().url().optional(),            // URL for internal sim agent API
+    COPILOT_SOURCE_ENV:                    z.enum(['dev', 'staging', 'prod']).optional(), // Source Sim environment sent to mothership for callbacks
+    COPILOT_DEV_URL:                       z.string().url().optional(),            // Sim agent API URL for the dev mothership environment
+    COPILOT_STAGING_URL:                   z.string().url().optional(),            // Sim agent API URL for the staging mothership environment
+    COPILOT_PROD_URL:                      z.string().url().optional(),            // Sim agent API URL for the production mothership environment
+    AGENT_INDEXER_URL:                     z.string().url().optional(),            // URL for agent training data indexer
+    AGENT_INDEXER_API_KEY:                 z.string().min(1).optional(),           // API key for agent indexer authentication
+    COPILOT_STREAM_TTL_SECONDS:            z.number().optional(),                  // Redis TTL for copilot SSE buffer
+    COPILOT_STREAM_EVENT_LIMIT:            z.number().optional(),                  // Max events retained per stream
+
+    // Database & Storage
+    REDIS_URL:                             z.string().url().optional(),            // Redis connection string for caching/sessions
+    REDIS_TLS_SERVERNAME:                  z.string().min(1).optional(),           // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN
+
+    // Payment & Billing
+    STRIPE_SECRET_KEY:                     z.string().min(1).optional(),           // Stripe secret key for payment processing
+    STRIPE_WEBHOOK_SECRET:                 z.string().min(1).optional(),           // General Stripe webhook secret
+    STRIPE_FREE_PRICE_ID:                  z.string().min(1).optional(),           // Stripe price ID for free tier
+    FREE_TIER_COST_LIMIT:                  z.number().optional(),                  // Cost limit for free tier users
+    FREE_STORAGE_LIMIT_GB:                 z.number().optional().default(5),       // Storage limit in GB for free tier users
+    STRIPE_PRO_PRICE_ID:                   z.string().min(1).optional(),           // Stripe price ID for pro tier
+    PRO_TIER_COST_LIMIT:                   z.number().optional(),                  // Cost limit for pro tier users
+    PRO_STORAGE_LIMIT_GB:                  z.number().optional().default(50),      // Storage limit in GB for pro tier users
+    STRIPE_TEAM_PRICE_ID:                  z.string().min(1).optional(),           // Stripe price ID for team tier
+    TEAM_TIER_COST_LIMIT:                  z.number().optional(),                  // Cost limit for team tier users
+    TEAM_STORAGE_LIMIT_GB:                 z.number().optional().default(500),     // Storage limit in GB for team tier organizations (pooled)
+    STRIPE_ENTERPRISE_PRICE_ID:            z.string().min(1).optional(),           // Stripe price ID for enterprise tier
+    ENTERPRISE_TIER_COST_LIMIT:            z.number().optional(),                  // Cost limit for enterprise tier users
+    ENTERPRISE_STORAGE_LIMIT_GB:           z.number().optional().default(500),     // Default storage limit in GB for enterprise tier (can be overridden per org)
+    BILLING_ENABLED:                       z.boolean().optional(),                 // Enable billing enforcement and usage tracking
+    FREE_API_DEPLOYMENT_GATE_ENABLED:      z.boolean().optional(),                 // Block free-plan accounts from programmatic execution (API/MCP/A2A/generic webhooks/chat embeds). Requires BILLING_ENABLED. Off by default for dark rollout
+    TABLE_SNAPSHOT_CACHE:                  z.boolean().optional(),                 // Mount tables into sandboxes by reference via a version-keyed CSV snapshot in object storage instead of draining the whole table into web-process heap
+    PII_REDACTION:                         z.boolean().optional(),                 // Redact PII from workflow logs via configurable Data Retention rules (Presidio at the logger persist choke point) and expose the Data Retention config UI
+    PII_GRANULAR_REDACTION:                z.boolean().optional(),                 // Expose the execution-altering PII redaction stages (redact workflow input + block outputs in-flight) in the Data Retention config; layered on top of PII_REDACTION
+    TRIGGER_EU_REGION:                     z.boolean().optional(),                 // Route Trigger.dev runs to eu-central-1 instead of the default us-east-1 (fallback for the trigger-eu-region flag when AppConfig is not the source of truth)
+
+    // Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans.
+    FREE_TABLES_LIMIT:                     z.number().optional(),                  // Max user tables per workspace on free tier (default: 5)
+    FREE_TABLE_ROWS_LIMIT:                 z.number().optional(),                  // Max rows per table on free tier (default: 50000)
+    PRO_TABLES_LIMIT:                      z.number().optional(),                  // Max user tables per workspace on pro tier (default: 100)
+    PRO_TABLE_ROWS_LIMIT:                  z.number().optional(),                  // Max rows per table on pro tier (default: 100000)
+    TEAM_TABLES_LIMIT:                     z.number().optional(),                  // Max user tables per workspace on team tier (default: 1000)
+    TEAM_TABLE_ROWS_LIMIT:                 z.number().optional(),                  // Max rows per table on team tier (default: 500000)
+    ENTERPRISE_TABLES_LIMIT:               z.number().optional(),                  // Max user tables per workspace on enterprise tier (default: 10000)
+    ENTERPRISE_TABLE_ROWS_LIMIT:           z.number().optional(),                  // Max rows per table on enterprise tier (default: 1000000)
+    TABLE_MAX_ROW_SIZE_BYTES:              z.number().optional(),                  // Max serialized size in bytes of a single user-table row (default: 409600)
+    TABLE_MAX_PAGE_BYTES:                  z.number().optional(),                  // Dev-preview: byte budget per row-page read; pages cut early past it (unset = disabled)
+
+    // Credit-tier Stripe prices (monthly)
+    STRIPE_PRICE_TIER_25_MO:               z.string().min(1).optional(),           // Pro: $25/mo (6,000 credits)
+    STRIPE_PRICE_TIER_100_MO:              z.string().min(1).optional(),           // Max: $100/mo (25,000 credits)
+
+    // Credit-tier Stripe prices (annual, 15% discount)
+    STRIPE_PRICE_TIER_25_YR:               z.string().min(1).optional(),           // Pro: $255/yr (15% off $300)
+    STRIPE_PRICE_TIER_100_YR:              z.string().min(1).optional(),           // Max: $1,020/yr (15% off $1,200)
+
+    // Team-specific Stripe prices (separate products for Billing Portal compat)
+    STRIPE_PRICE_TEAM_25_MO:               z.string().min(1).optional(),           // Team Pro: $25/seat/mo
+    STRIPE_PRICE_TEAM_25_YR:               z.string().min(1).optional(),           // Team Pro: $255/seat/yr
+    STRIPE_PRICE_TEAM_100_MO:              z.string().min(1).optional(),           // Team Max: $100/seat/mo
+    STRIPE_PRICE_TEAM_100_YR:              z.string().min(1).optional(),           // Team Max: $1,020/seat/yr
+    OVERAGE_THRESHOLD_DOLLARS:             z.number().optional().default(100),     // Dollar threshold for incremental overage billing (default: $100)
+
+    // Email & Communication
+    EMAIL_VERIFICATION_ENABLED:            z.boolean().optional(),                 // Enable email verification for user registration and login (defaults to false)
+    RESEND_API_KEY:                        z.string().min(1).optional(),           // Resend API key for transactional emails
+    FROM_EMAIL_ADDRESS:                    z.string().min(1).optional(),           // Complete from address (e.g., "Sim <noreply@domain.com>" or "noreply@domain.com")
+    PERSONAL_EMAIL_FROM:                   z.string().min(1).optional(),           // From address for personalized emails
+    EMAIL_DOMAIN:                          z.string().min(1).optional(),           // Domain for sending emails (fallback when FROM_EMAIL_ADDRESS not set)
+    AZURE_ACS_CONNECTION_STRING:           z.string().optional(),                  // Azure Communication Services connection string
+    AWS_SES_REGION:                        z.string().min(1).optional(),           // AWS region for SES (credentials resolved via default SDK provider chain)
+    SMTP_HOST:                             z.string().min(1).optional(),           // SMTP server hostname
+    SMTP_PORT:                             z.coerce.number().int().min(1).max(65535).optional(),
+    SMTP_USER:                             z.string().min(1).optional(),           // SMTP username
+    SMTP_PASS:                             z.string().min(1).optional(),           // SMTP password
+    SMTP_SECURE:                           z.boolean().optional(),                 // Force TLS on connect (defaults to true on port 465); read via envBoolean to handle string values from process.env
+
+    // SMS & Messaging
+    TWILIO_ACCOUNT_SID:                    z.string().min(1).optional(),           // Twilio Account SID for SMS sending
+    TWILIO_AUTH_TOKEN:                     z.string().min(1).optional(),           // Twilio Auth Token for API authentication
+    TWILIO_PHONE_NUMBER:                   z.string().min(1).optional(),           // Twilio phone number for sending SMS
+
+    // AI/LLM Provider API Keys
+    OPENAI_API_KEY:                        z.string().min(1).optional(),           // Primary OpenAI API key
+    OPENAI_API_KEY_1:                      z.string().min(1).optional(),           // Additional OpenAI API key for load balancing
+    OPENAI_API_KEY_2:                      z.string().min(1).optional(),           // Additional OpenAI API key for load balancing
+    OPENAI_API_KEY_3:                      z.string().min(1).optional(),           // Additional OpenAI API key for load balancing
+    MISTRAL_API_KEY:                       z.string().min(1).optional(),           // Mistral AI API key
+    ANTHROPIC_API_KEY_1:                   z.string().min(1).optional(),           // Primary Anthropic Claude API key
+    ANTHROPIC_API_KEY_2:                   z.string().min(1).optional(),           // Additional Anthropic API key for load balancing
+    ANTHROPIC_API_KEY_3:                   z.string().min(1).optional(),           // Additional Anthropic API key for load balancing
+    GEMINI_API_KEY:                        z.string().min(1).optional(),           // Singular Gemini API key (used as fallback when rotation keys are unset)
+    GEMINI_API_KEY_1:                      z.string().min(1).optional(),           // Primary Gemini API key
+    GEMINI_API_KEY_2:                      z.string().min(1).optional(),           // Additional Gemini API key for load balancing
+    GEMINI_API_KEY_3:                      z.string().min(1).optional(),           // Additional Gemini API key for load balancing
+    ZAI_API_KEY_1:                         z.string().min(1).optional(),           // Primary Z.ai API key for load balancing
+    ZAI_API_KEY_2:                         z.string().min(1).optional(),           // Additional Z.ai API key for load balancing
+    ZAI_API_KEY_3:                         z.string().min(1).optional(),           // Additional Z.ai API key for load balancing
+    XAI_API_KEY_1:                         z.string().min(1).optional(),           // Primary xAI API key for load balancing
+    XAI_API_KEY_2:                         z.string().min(1).optional(),           // Additional xAI API key for load balancing
+    XAI_API_KEY_3:                         z.string().min(1).optional(),           // Additional xAI API key for load balancing
+    OLLAMA_URL:                            z.string().url().optional(),            // Ollama local LLM server URL
+    VLLM_BASE_URL:                         z.string().url().optional(),            // vLLM self-hosted base URL (OpenAI-compatible)
+    VLLM_API_KEY:                          z.string().optional(),                  // Optional bearer token for vLLM
+    LITELLM_BASE_URL:                      z.string().url().optional(),            // LiteLLM proxy base URL (OpenAI-compatible)
+    LITELLM_API_KEY:                       z.string().optional(),                  // Optional bearer token for LiteLLM
+    FIREWORKS_API_KEY:                     z.string().optional(),                  // Optional Fireworks AI API key for model listing
+    TOGETHER_API_KEY:                      z.string().optional(),                  // Optional Together AI API key for model listing and inference
+    BASETEN_API_KEY:                       z.string().optional(),                  // Optional Baseten API key for model listing and inference
+    COHERE_API_KEY:                        z.string().min(1).optional(),           // Cohere API key for reranker (rerank-v4.0-pro, rerank-v4.0-fast, rerank-v3.5)
+    COHERE_API_KEY_1:                      z.string().min(1).optional(),           // Primary Cohere API key for rotation
+    COHERE_API_KEY_2:                      z.string().min(1).optional(),           // Additional Cohere API key for load balancing
+    COHERE_API_KEY_3:                      z.string().min(1).optional(),           // Additional Cohere API key for load balancing
+    ELEVENLABS_API_KEY:                    z.string().min(1).optional(),           // ElevenLabs API key for text-to-speech in deployed chat
+    SERPER_API_KEY:                        z.string().min(1).optional(),           // Serper API key for online search
+    EXA_API_KEY:                           z.string().min(1).optional(),           // Exa AI API key for enhanced online search
+    BLACKLISTED_PROVIDERS:                 z.string().optional(),                  // Comma-separated provider IDs to hide (e.g., "openai,anthropic")
+    BLACKLISTED_MODELS:                    z.string().optional(),                  // Comma-separated model names/prefixes to hide (e.g., "gpt-4,claude-*")
+    ALLOWED_MCP_DOMAINS:                   z.string().optional(),                  // Comma-separated domains for MCP servers (e.g., "internal.company.com,mcp.example.org"). Empty = all allowed.
+    ALLOWED_INTEGRATIONS:                  z.string().optional(),                  // Comma-separated block types to allow (e.g., "slack,github,agent"). Empty = all allowed.
+    PREVIEW_BLOCKS:                        z.string().optional(),                  // Comma-separated preview block types to reveal off-AppConfig (e.g., "gmail_v2,notion_v3"). Empty = all preview blocks hidden.
+
+    // Azure Configuration - Shared credentials with feature-specific models
+    AZURE_OPENAI_ENDPOINT:                 z.string().url().optional(),            // Shared Azure OpenAI service endpoint
+    AZURE_OPENAI_API_VERSION:              z.string().optional(),                  // Shared Azure OpenAI API version
+    AZURE_OPENAI_API_KEY:                  z.string().min(1).optional(),           // Shared Azure OpenAI API key
+    AZURE_ANTHROPIC_ENDPOINT:              z.string().url().optional(),            // Azure Anthropic service endpoint
+    AZURE_ANTHROPIC_API_KEY:               z.string().min(1).optional(),           // Azure Anthropic API key
+    AZURE_ANTHROPIC_API_VERSION:           z.string().min(1).optional(),           // Azure Anthropic API version (e.g. 2023-06-01)
+    KB_OPENAI_MODEL_NAME:                  z.string().optional(),                  // Azure deployment name serving the configured KB embedding model (used only when AZURE_OPENAI_* credentials are set).
+    KB_EMBEDDING_MODEL:                    z.string().optional(),                  // Embedding model used for all new knowledge bases. Must be one of the supported model ids; defaults to text-embedding-3-small.
+    WAND_OPENAI_MODEL_NAME:                z.string().optional(),                  // Wand generation OpenAI model name (works with both regular OpenAI and Azure OpenAI)
+    OCR_AZURE_ENDPOINT:                    z.string().url().optional(),            // Azure Mistral OCR service endpoint
+    OCR_AZURE_MODEL_NAME:                  z.string().optional(),                  // Azure Mistral OCR model name for document processing
+    OCR_AZURE_API_KEY:                     z.string().min(1).optional(),           // Azure Mistral OCR API key
+
+    // Vertex AI Configuration
+    VERTEX_PROJECT:                        z.string().optional(),                  // Google Cloud project ID for Vertex AI
+    VERTEX_LOCATION:                       z.string().optional(),                  // Google Cloud location/region for Vertex AI (defaults to us-central1)
+
+    // Monitoring & Analytics
+    TELEMETRY_ENDPOINT:                    z.string().url().optional(),            // Custom telemetry/analytics endpoint
+    COST_MULTIPLIER:                       z.number().optional(),                  // Multiplier for cost calculations
+    LOG_LEVEL:                             z.enum(['DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), // Minimum log level to display (defaults to ERROR in production, DEBUG in development)
+    PROFOUND_API_KEY:                      z.string().min(1).optional(),           // Profound analytics API key
+    PROFOUND_ENDPOINT:                     z.string().url().optional(),            // Profound analytics endpoint
+    GRAFANA_OTLP_ENDPOINT:                 z.string().url().optional(),            // Grafana Cloud OTLP HTTP gateway base URL (e.g., https://otlp-gateway-prod-us-east-0.grafana.net/otlp). Trigger.dev exporters append /v1/traces, /v1/logs, /v1/metrics.
+    GRAFANA_OTLP_HEADERS:                  z.string().min(1).optional(),           // Comma-separated key=value headers for OTLP requests (e.g., "Authorization=Basic <base64(instanceId:token)>"). Same format as the OTEL_EXPORTER_OTLP_HEADERS spec.
+    GRAFANA_DEPLOYMENT_ENVIRONMENT:        z.string().min(1).optional(),           // Deployment tier label (e.g., "production", "staging", "development"). Emitted as the stable `deployment.environment.name` resource attribute on Trigger.dev telemetry to match the rest of the Sim OTEL stack.
+
+    // External Services
+    BROWSERBASE_API_KEY:                   z.string().min(1).optional(),           // Browserbase API key for browser automation
+    BROWSERBASE_PROJECT_ID:                z.string().min(1).optional(),           // Browserbase project ID
+    GITHUB_TOKEN:                          z.string().optional(),                  // GitHub personal access token for API access
+
+    // Admin API
+    ADMIN_API_KEY:                         z.string().min(32).optional(),          // Admin API key for self-hosted GitOps access (generate with: openssl rand -hex 32)
+
+    // Mothership Admin
+    MOTHERSHIP_API_ADMIN_KEY:              z.string().min(1).optional(),           // Admin API key for mothership/copilot admin endpoints
+    MOTHERSHIP_DEV_URL:                    z.string().url().optional(),            // Mothership dev environment URL
+    MOTHERSHIP_STAGING_URL:                z.string().url().optional(),            // Mothership staging environment URL
+    MOTHERSHIP_PROD_URL:                   z.string().url().optional(),            // Mothership production environment URL
+
+    // Infrastructure & Deployment
+    NEXT_RUNTIME:                          z.string().optional(),                  // Next.js runtime environment
+    DOCKER_BUILD:                          z.boolean().optional(),                 // Flag indicating Docker build environment
+
+    // Background Jobs & Scheduling
+    TRIGGER_PROJECT_ID:                    z.string().optional(),                  // Trigger.dev project ID
+    TRIGGER_SECRET_KEY:                    z.string().min(1).optional(),           // Trigger.dev secret key for background jobs
+    TRIGGER_DEV_ENABLED:                   z.boolean().optional(),                 // Toggle to enable/disable Trigger.dev for async jobs
+    CRON_SECRET:                           z.string().optional(),                  // Secret for authenticating cron job requests
+    JOB_RETENTION_DAYS:                    z.string().optional().default('1'),     // Days to retain job logs/data
+    SCHEDULE_EXECUTION_CONCURRENCY_LIMIT:  z.string().optional().default('30'),
+    WORKFLOW_EXECUTION_CONCURRENCY_LIMIT:  z.string().optional().default('75'),
+    WEBHOOK_EXECUTION_CONCURRENCY_LIMIT:   z.string().optional().default('75'),
+    RESUME_EXECUTION_CONCURRENCY_LIMIT:    z.string().optional().default('50'),
+    SCHEDULE_ENQUEUE_BUDGET_MULTIPLIER:    z.string().optional().default('2'),
+    SCHEDULE_JITTER_MAX_MS:                z.string().optional().default('30000'),
+    SCHEDULE_INFRA_RETRY_BASE_MS:          z.string().optional().default('60000'),
+    SCHEDULE_INFRA_RETRY_MAX_MS:           z.string().optional().default('300000'),
+    SCHEDULE_INFRA_RETRY_MAX_ATTEMPTS:     z.string().optional().default('10'),
+
+    // Cloud Storage - AWS S3
+    AWS_REGION:                            z.string().optional(),                  // AWS region for S3 buckets
+    AWS_ACCESS_KEY_ID:                     z.string().optional(),                  // AWS access key ID
+    AWS_SECRET_ACCESS_KEY:                 z.string().optional(),                  // AWS secret access key
+    S3_BUCKET_NAME:                        z.string().optional(),                  // S3 bucket for general file storage
+    S3_LOGS_BUCKET_NAME:                   z.string().optional(),                  // S3 bucket for storing logs
+    S3_KB_BUCKET_NAME:                     z.string().optional(),                  // S3 bucket for knowledge base files
+    S3_EXECUTION_FILES_BUCKET_NAME:        z.string().optional(),                  // S3 bucket for workflow execution files
+    S3_CHAT_BUCKET_NAME:                   z.string().optional(),                  // S3 bucket for chat logos
+    S3_COPILOT_BUCKET_NAME:                z.string().optional(),                  // S3 bucket for copilot files
+    S3_PROFILE_PICTURES_BUCKET_NAME:       z.string().optional(),                  // S3 bucket for profile pictures
+    S3_OG_IMAGES_BUCKET_NAME:              z.string().optional(),                  // S3 bucket for OpenGraph images
+    S3_WORKSPACE_LOGOS_BUCKET_NAME:        z.string().optional(),                  // S3 bucket for workspace logos
+    S3_ENDPOINT:                           z.string().optional(),                  // Custom endpoint for S3-compatible storage (Cloudflare R2, MinIO, Backblaze B2). Leave unset for AWS S3
+    S3_FORCE_PATH_STYLE:                   z.string().optional(),                  // Force path-style addressing (MinIO/Ceph RGW). Defaults to false (AWS S3, R2). Coerced via envBoolean at the consumption site
+
+    // Dynamic config - AWS AppConfig (hosted source of truth for signup/login gating lists; unset => env-var fallback)
+    APPCONFIG_APPLICATION:                 z.string().optional(),                  // AppConfig application id/name. On hosted deployments, when set with APPCONFIG_ENVIRONMENT, gating lists come from AppConfig instead of env vars
+    APPCONFIG_ENVIRONMENT:                 z.string().optional(),                  // AppConfig environment id/name. Profile name is an app-side constant ('access-control'), not an env var
+
+    // Cloud Storage - Azure Blob
+    AZURE_ACCOUNT_NAME:                    z.string().optional(),                  // Azure storage account name
+    AZURE_ACCOUNT_KEY:                     z.string().optional(),                  // Azure storage account key
+    AZURE_CONNECTION_STRING:               z.string().optional(),                  // Azure storage connection string
+    AZURE_STORAGE_CONTAINER_NAME:          z.string().optional(),                  // Azure container for general files
+    AZURE_STORAGE_KB_CONTAINER_NAME:       z.string().optional(),                  // Azure container for knowledge base files
+    AZURE_STORAGE_EXECUTION_FILES_CONTAINER_NAME: z.string().optional(),          // Azure container for workflow execution files
+    AZURE_STORAGE_CHAT_CONTAINER_NAME:     z.string().optional(),                  // Azure container for chat logos
+    AZURE_STORAGE_COPILOT_CONTAINER_NAME:  z.string().optional(),                  // Azure container for copilot files
+    AZURE_STORAGE_PROFILE_PICTURES_CONTAINER_NAME: z.string().optional(),          // Azure container for profile pictures
+    AZURE_STORAGE_OG_IMAGES_CONTAINER_NAME: z.string().optional(),                 // Azure container for OpenGraph images
+    AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME: z.string().optional(),            // Azure container for workspace logos
+
+
+    // Admission & Burst Protection
+    ADMISSION_GATE_MAX_INFLIGHT:           z.string().optional().default('500'),   // Max concurrent in-flight execution requests per pod
+    API_MAX_JSON_BODY_BYTES:               z.string().optional().default('52428800'),// Default max JSON request body size for contract routes (50 MB)
+    CHAT_MAX_REQUEST_BYTES:                z.string().optional().default('230686720'),// Max request body size for the public deployed-chat endpoint (220 MB; covers 15 base64 file attachments)
+    WEBHOOK_MAX_REQUEST_BYTES:             z.string().optional().default('10485760'),// Max request body size for public webhook receiver endpoints (10 MB; provider payloads rarely exceed a few MB)
+
+    // Rate Limiting Configuration
+    RATE_LIMIT_WINDOW_MS:                  z.string().optional().default('60000'), // Rate limit window duration in milliseconds (default: 1 minute)
+    MANUAL_EXECUTION_LIMIT:                z.string().optional().default('999999'),// Manual execution bypass value (effectively unlimited)
+    RATE_LIMIT_FREE_SYNC:                  z.string().optional().default('50'),    // Free tier sync API executions per minute
+    RATE_LIMIT_FREE_ASYNC:                 z.string().optional().default('200'),   // Free tier async API executions per minute
+    RATE_LIMIT_PRO_SYNC:                   z.string().optional().default('150'),   // Pro tier sync API executions per minute
+    RATE_LIMIT_PRO_ASYNC:                  z.string().optional().default('1000'),  // Pro tier async API executions per minute
+    RATE_LIMIT_TEAM_SYNC:                  z.string().optional().default('300'),   // Team tier sync API executions per minute
+    RATE_LIMIT_TEAM_ASYNC:                 z.string().optional().default('2500'),  // Team tier async API executions per minute
+    RATE_LIMIT_ENTERPRISE_SYNC:            z.string().optional().default('600'),   // Enterprise tier sync API executions per minute
+    RATE_LIMIT_ENTERPRISE_ASYNC:           z.string().optional().default('5000'),  // Enterprise tier async API executions per minute
+    // Timeout Configuration
+    EXECUTION_TIMEOUT_FREE:                z.string().optional().default('300'),   // 5 minutes
+    EXECUTION_TIMEOUT_PRO:                 z.string().optional().default('3000'),  // 50 minutes
+    EXECUTION_TIMEOUT_TEAM:                z.string().optional().default('3000'),  // 50 minutes
+    EXECUTION_TIMEOUT_ENTERPRISE:          z.string().optional().default('3000'),  // 50 minutes
+    EXECUTION_TIMEOUT_ASYNC_FREE:          z.string().optional().default('5400'),  // 90 minutes
+    EXECUTION_TIMEOUT_ASYNC_PRO:           z.string().optional().default('5400'),  // 90 minutes
+    EXECUTION_TIMEOUT_ASYNC_TEAM:          z.string().optional().default('5400'),  // 90 minutes
+    EXECUTION_TIMEOUT_ASYNC_ENTERPRISE:    z.string().optional().default('5400'),  // 90 minutes
+
+    // Isolated-VM Worker Pool Configuration
+    IVM_POOL_SIZE:                         z.string().optional().default('4'),      // Max worker processes in pool
+    IVM_MAX_CONCURRENT:                    z.string().optional().default('10000'),  // Max concurrent executions globally
+    IVM_MAX_PER_WORKER:                    z.string().optional().default('2500'),   // Max concurrent executions per worker
+    IVM_WORKER_IDLE_TIMEOUT_MS:            z.string().optional().default('60000'),  // Worker idle cleanup timeout (ms)
+    IVM_MAX_QUEUE_SIZE:                    z.string().optional().default('10000'),  // Max pending queued executions in memory
+    IVM_MAX_FETCH_RESPONSE_BYTES:          z.string().optional().default('8388608'),// Max bytes read from sandbox fetch responses
+    IVM_MAX_FETCH_RESPONSE_CHARS:          z.string().optional().default('4000000'),// Max chars returned to sandbox from fetch body
+    IVM_MAX_FETCH_OPTIONS_JSON_CHARS:      z.string().optional().default('262144'), // Max JSON payload size for sandbox fetch options
+    IVM_MAX_FETCH_URL_LENGTH:              z.string().optional().default('8192'),   // Max URL length accepted by sandbox fetch
+    IVM_MAX_STDOUT_CHARS:                  z.string().optional().default('200000'), // Max captured stdout characters per execution
+    IVM_MAX_ACTIVE_PER_OWNER:              z.string().optional().default('200'),    // Max active executions per owner (per process)
+    IVM_MAX_QUEUED_PER_OWNER:              z.string().optional().default('2000'),   // Max queued executions per owner (per process)
+    IVM_MAX_OWNER_WEIGHT:                  z.string().optional().default('5'),      // Max accepted weight for weighted owner scheduling
+    IVM_DISTRIBUTED_MAX_INFLIGHT_PER_OWNER:z.string().optional().default('2200'),   // Max owner in-flight leases across replicas
+    IVM_DISTRIBUTED_LEASE_MIN_TTL_MS:      z.string().optional().default('120000'), // Min TTL for distributed in-flight leases (ms)
+    IVM_QUEUE_TIMEOUT_MS:                  z.string().optional().default('300000'), // Max queue wait before rejection (ms)
+    IVM_MAX_EXECUTIONS_PER_WORKER:         z.string().optional().default('200'),    // Max lifetime executions before worker is recycled
+    IVM_MAX_BROKER_ARGS_JSON_CHARS:        z.string().optional().default('262144'),  // Max JSON payload size for sandbox task broker args (isolate→host)
+    IVM_MAX_BROKER_RESULT_JSON_CHARS:      z.string().optional().default('16777216'),// Max JSON payload size for sandbox task broker results (host→isolate)
+    IVM_MAX_BROKERS_PER_EXECUTION:         z.string().optional().default('1000'),    // Max broker calls per sandbox task execution
+
+    // Knowledge Base Processing Configuration - Shared across all processing methods
+    KB_CONFIG_MAX_DURATION:                z.number().optional().default(600),     // Max processing duration in seconds (10 minutes)
+    KB_CONFIG_MAX_ATTEMPTS:                z.number().optional().default(3),       // Max retry attempts
+    KB_CONFIG_RETRY_FACTOR:                z.number().optional().default(2),       // Retry backoff factor
+    KB_CONFIG_MIN_TIMEOUT:                 z.number().optional().default(1000),    // Min timeout in ms
+    KB_CONFIG_MAX_TIMEOUT:                 z.number().optional().default(10000),   // Max timeout in ms
+    KB_CONFIG_CONCURRENCY_LIMIT:           z.number().optional().default(50),      // Concurrent embedding API calls
+    KB_CONFIG_BATCH_SIZE:                  z.number().optional().default(2000),    // Chunks to process per embedding batch
+    KB_CONFIG_DELAY_BETWEEN_BATCHES:       z.number().optional().default(0),       // Delay between batches in ms (0 for max speed)
+    KB_CONFIG_DELAY_BETWEEN_DOCUMENTS:     z.number().optional().default(50),      // Delay between documents in ms
+    KB_CONFIG_CHUNK_CONCURRENCY:           z.number().optional().default(10),      // Concurrent PDF chunk OCR processing
+
+    // Real-time Communication
+    SOCKET_SERVER_URL:                     z.string().url().optional(),            // WebSocket server URL for real-time features
+    PORT:                                  z.number().optional(),                  // Main application port
+    INTERNAL_API_BASE_URL:                 z.string().optional(),                  // Optional internal base URL for server-side self-calls; must include protocol if set (e.g., http://sim-app.namespace.svc.cluster.local:3000)
+    ALLOWED_ORIGINS:                       z.string().optional(),                  // CORS allowed origins
+    PII_URL:                               z.string().optional(),                  // Presidio PII service base URL serving /analyze + /anonymize (standalone ECS service; default http://localhost:5001 for local dev)
+    PII_MASK_CHUNK_CONCURRENCY:            z.coerce.number().int().positive().optional(), // Max in-flight mask-batch requests per redaction (default 64); tune to the Presidio fleet size behind the internal ALB, lower to 1 for a single instance
+    PII_REF_CONCURRENCY:                   z.coerce.number().int().positive().optional(), // Max large-value refs hydrated+masked+re-stored in parallel per payload (default 4); multiplies with PII_MASK_CHUNK_CONCURRENCY for total in-flight Presidio load
+    PII_SERVICE_CHUNK_CONCURRENCY:         z.coerce.number().int().positive().optional(), // Max Presidio requests in flight from a single mask-batch call (route -> Presidio fan-out, default 4); inner to PII_MASK_CHUNK_CONCURRENCY
+
+    // OAuth Integration Credentials - All optional, enables third-party integrations
+    GOOGLE_CLIENT_ID:                      z.string().optional(),                  // Google OAuth client ID for Google services
+    GOOGLE_CLIENT_SECRET:                  z.string().optional(),                  // Google OAuth client secret
+    GITHUB_CLIENT_ID:                      z.string().optional(),                  // GitHub OAuth client ID for GitHub integration
+    GITHUB_CLIENT_SECRET:                  z.string().optional(),                  // GitHub OAuth client secret
+    DISABLE_GOOGLE_AUTH:                   z.boolean().optional(),                 // Disable Google OAuth login even when credentials are configured
+    DISABLE_GITHUB_AUTH:                   z.boolean().optional(),                 // Disable GitHub OAuth login even when credentials are configured
+    DISABLE_MICROSOFT_AUTH:               z.boolean().optional(),                 // Disable Microsoft OAuth login even when credentials are configured
+    DISABLE_EMAIL_SIGNUP:                  z.boolean().optional(),                 // Block new email/password registrations while keeping email login working
+
+    X_CLIENT_ID:                           z.string().optional(),                  // X (Twitter) OAuth client ID
+    X_CLIENT_SECRET:                       z.string().optional(),                  // X (Twitter) OAuth client secret
+    TIKTOK_CLIENT_ID:                      z.string().optional(),                  // TikTok OAuth client key (TikTok calls this "client_key")
+    TIKTOK_CLIENT_SECRET:                  z.string().optional(),                  // TikTok OAuth client secret
+    CONFLUENCE_CLIENT_ID:                  z.string().optional(),                  // Atlassian Confluence OAuth client ID
+    CONFLUENCE_CLIENT_SECRET:              z.string().optional(),                  // Atlassian Confluence OAuth client secret
+    JIRA_CLIENT_ID:                        z.string().optional(),                  // Atlassian Jira OAuth client ID
+    JIRA_CLIENT_SECRET:                    z.string().optional(),                  // Atlassian Jira OAuth client secret
+    ASANA_CLIENT_ID:                       z.string().optional(),                  // Asana OAuth client ID
+    ASANA_CLIENT_SECRET:                   z.string().optional(),                  // Asana OAuth client secret
+    AIRTABLE_CLIENT_ID:                    z.string().optional(),                  // Airtable OAuth client ID
+    AIRTABLE_CLIENT_SECRET:                z.string().optional(),                  // Airtable OAuth client secret
+    APOLLO_API_KEY:                        z.string().optional(),                  // Apollo API key (optional system-wide config)
+    SUPABASE_CLIENT_ID:                    z.string().optional(),                  // Supabase OAuth client ID
+    SUPABASE_CLIENT_SECRET:                z.string().optional(),                  // Supabase OAuth client secret
+    NOTION_CLIENT_ID:                      z.string().optional(),                  // Notion OAuth client ID
+    NOTION_CLIENT_SECRET:                  z.string().optional(),                  // Notion OAuth client secret
+    MONDAY_CLIENT_ID:                      z.string().optional(),                  // Monday.com OAuth client ID
+    MONDAY_CLIENT_SECRET:                  z.string().optional(),                  // Monday.com OAuth client secret
+    DISCORD_CLIENT_ID:                     z.string().optional(),                  // Discord OAuth client ID
+    DISCORD_CLIENT_SECRET:                 z.string().optional(),                  // Discord OAuth client secret
+    DOCUSIGN_CLIENT_ID:                    z.string().optional(),                  // DocuSign OAuth client ID
+    DOCUSIGN_CLIENT_SECRET:                z.string().optional(),                  // DocuSign OAuth client secret
+    MICROSOFT_CLIENT_ID:                   z.string().optional(),                  // Microsoft OAuth client ID for Office 365/Teams
+    MICROSOFT_CLIENT_SECRET:               z.string().optional(),                  // Microsoft OAuth client secret
+    HUBSPOT_CLIENT_ID:                     z.string().optional(),                  // HubSpot OAuth client ID
+    HUBSPOT_CLIENT_SECRET:                 z.string().optional(),                  // HubSpot OAuth client secret
+    SALESFORCE_CLIENT_ID:                  z.string().optional(),                  // Salesforce OAuth client ID
+    SALESFORCE_CLIENT_SECRET:              z.string().optional(),                  // Salesforce OAuth client secret
+    WEALTHBOX_CLIENT_ID:                   z.string().optional(),                  // WealthBox OAuth client ID
+    WEALTHBOX_CLIENT_SECRET:               z.string().optional(),                  // WealthBox OAuth client secret
+    PIPEDRIVE_CLIENT_ID:                   z.string().optional(),                  // Pipedrive OAuth client ID
+    PIPEDRIVE_CLIENT_SECRET:               z.string().optional(),                  // Pipedrive OAuth client secret
+    LINEAR_CLIENT_ID:                      z.string().optional(),                  // Linear OAuth client ID
+    LINEAR_CLIENT_SECRET:                  z.string().optional(),                  // Linear OAuth client secret
+    BOX_CLIENT_ID:                         z.string().optional(),                  // Box OAuth client ID
+    BOX_CLIENT_SECRET:                     z.string().optional(),                  // Box OAuth client secret
+    DROPBOX_CLIENT_ID:                     z.string().optional(),                  // Dropbox OAuth client ID
+    DROPBOX_CLIENT_SECRET:                 z.string().optional(),                  // Dropbox OAuth client secret
+    SLACK_CLIENT_ID:                       z.string().optional(),                  // Slack OAuth client ID
+    SLACK_CLIENT_SECRET:                   z.string().optional(),                  // Slack OAuth client secret
+    SLACK_SIGNING_SECRET:                  z.string().optional(),                  // Official Sim Slack app signing secret (verifies inbound events for the native OAuth trigger)
+    REDDIT_CLIENT_ID:                      z.string().optional(),                  // Reddit OAuth client ID
+    REDDIT_CLIENT_SECRET:                  z.string().optional(),                  // Reddit OAuth client secret
+    WEBFLOW_CLIENT_ID:                     z.string().optional(),                  // Webflow OAuth client ID
+    WEBFLOW_CLIENT_SECRET:                 z.string().optional(),                  // Webflow OAuth client secret
+    TRELLO_API_KEY:                        z.string().optional(),                  // Trello API Key
+    LINKEDIN_CLIENT_ID:                    z.string().optional(),                  // LinkedIn OAuth client ID
+    LINKEDIN_CLIENT_SECRET:                z.string().optional(),                  // LinkedIn OAuth client secret
+    SHOPIFY_CLIENT_ID:                     z.string().optional(),                  // Shopify OAuth client ID
+    SHOPIFY_CLIENT_SECRET:                 z.string().optional(),                  // Shopify OAuth client secret
+    ZOOM_CLIENT_ID:                        z.string().optional(),                  // Zoom OAuth client ID
+    ZOOM_CLIENT_SECRET:                    z.string().optional(),                  // Zoom OAuth client secret
+    WORDPRESS_CLIENT_ID:                   z.string().optional(),                  // WordPress.com OAuth client ID
+    WORDPRESS_CLIENT_SECRET:               z.string().optional(),                  // WordPress.com OAuth client secret
+    SPOTIFY_CLIENT_ID:                     z.string().optional(),                  // Spotify OAuth client ID
+    SPOTIFY_CLIENT_SECRET:                 z.string().optional(),                  // Spotify OAuth client secret
+    CALCOM_CLIENT_ID:                      z.string().optional(),                  // Cal.com OAuth client ID
+    ATTIO_CLIENT_ID:                       z.string().optional(),                  // Attio OAuth client ID
+    ATTIO_CLIENT_SECRET:                   z.string().optional(),                  // Attio OAuth client secret
+
+    // AgentMail - Mothership Email Inbox
+    AGENTMAIL_API_KEY:                     z.string().min(1).optional(),           // AgentMail API key for mothership email inbox
+    AGENTMAIL_DOMAIN:                      z.string().optional(),                  // Custom domain for AgentMail inboxes (default: agentmail.to)
+    INBOX_ENABLED:                         z.boolean().optional(),                 // Enable inbox (Sim Mailer) on self-hosted (bypasses hosted requirements)
+
+    // E2B Remote Code Execution
+    E2B_ENABLED:                           z.string().optional(),                  // Enable E2B remote code execution
+    E2B_API_KEY:                           z.string().optional(),                  // E2B API key for sandbox creation
+    MOTHERSHIP_E2B_TEMPLATE_ID:             z.string().optional(),                  // Custom E2B template with pre-installed CLI tools for shell execution
+    MOTHERSHIP_E2B_DOC_TEMPLATE_ID:         z.string().optional(),                  // Dedicated E2B template with python-pptx/docx/openpyxl/reportlab for document generation; when set (and E2B enabled), docs compile via Python instead of the JS isolated-vm path
+    E2B_PI_TEMPLATE_ID:                     z.string().optional(),                  // E2B template ID/alias with the Pi CLI + git baked in (Pi Coding Agent cloud mode)
+
+    // Access Control (Permission Groups) - for self-hosted deployments
+    ACCESS_CONTROL_ENABLED:                z.boolean().optional(),                 // Enable access control on self-hosted (bypasses plan requirements)
+
+    // Enterprise Feature Overrides - for self-hosted deployments
+    WHITELABELING_ENABLED:                 z.boolean().optional(),                 // Enable whitelabeling on self-hosted (bypasses hosted requirements)
+    AUDIT_LOGS_ENABLED:                    z.boolean().optional(),                 // Enable audit logs on self-hosted (bypasses hosted requirements)
+    DATA_RETENTION_ENABLED:               z.boolean().optional(),                 // Enable data retention settings on self-hosted (bypasses hosted requirements)
+    DATA_DRAINS_ENABLED:                  z.boolean().optional(),                 // Enable data drains on self-hosted (bypasses hosted requirements)
+    FORKING_ENABLED:                      z.boolean().optional(),                 // Enable workspace forking on self-hosted (bypasses hosted requirements)
+    DEPLOY_AS_BLOCK:                      z.boolean().optional(),                 // Enable deploy-as-block (publish a workflow as a reusable org-wide custom block)
+
+    // Organizations - for self-hosted deployments
+    ORGANIZATIONS_ENABLED:                 z.boolean().optional(),                 // Enable organizations on self-hosted (bypasses plan requirements)
+
+    // Invitations - for self-hosted deployments
+    DISABLE_INVITATIONS:                   z.boolean().optional(),                 // Disable workspace invitations globally (for self-hosted deployments)
+    DISABLE_PUBLIC_API:                    z.boolean().optional(),                 // Disable public API access globally (for self-hosted deployments)
+    MOTHERSHIP_BETA_FEATURES:              z.boolean().optional(),                 // Enable beta Mothership planning/changelog artifact surfaces
+
+    // Development Tools
+    REACT_GRAB_ENABLED:                    z.boolean().optional(),                 // Enable React Grab for UI element debugging in Cursor/AI agents (dev only)
+    REACT_SCAN_ENABLED:                    z.boolean().optional(),                 // Enable React Scan for performance debugging (dev only)
+
+    // SSO Configuration (for script-based registration)
+    SSO_ENABLED:                           z.boolean().optional(),                 // Enable SSO functionality
+    SSO_PROVIDER_TYPE:                     z.enum(['oidc', 'saml']).optional(),    // [REQUIRED] SSO provider type
+    SSO_PROVIDER_ID:                       z.string().optional(),                  // [REQUIRED] SSO provider ID
+    SSO_ISSUER:                            z.string().optional(),                  // [REQUIRED] SSO issuer URL
+    SSO_DOMAIN:                            z.string().optional(),                  // [REQUIRED] SSO email domain
+    SSO_USER_EMAIL:                        z.string().optional(),                  // [REQUIRED] User email for SSO registration
+    SSO_ORGANIZATION_ID:                   z.string().optional(),                  // Organization ID for SSO registration (optional)
+    SSO_TRUSTED_PROVIDER_IDS:              z.string().optional(),                  // Comma-separated SSO provider IDs to trust for automatic account linking when an existing account shares the same email. Use for IdPs that do not assert email_verified. Merged into Better Auth accountLinking.trustedProviders.
+
+    // SSO Mapping Configuration (optional - sensible defaults provided)
+    SSO_MAPPING_ID:                        z.string().optional(),                  // Custom ID claim mapping (default: sub for OIDC, nameidentifier for SAML)
+    SSO_MAPPING_EMAIL:                     z.string().optional(),                  // Custom email claim mapping (default: email for OIDC, emailaddress for SAML)
+    SSO_MAPPING_NAME:                      z.string().optional(),                  // Custom name claim mapping (default: name for both)
+    SSO_MAPPING_IMAGE:                     z.string().optional(),                  // Custom image claim mapping (default: picture for OIDC)
+
+    // SSO OIDC Configuration
+    SSO_OIDC_CLIENT_ID:                    z.string().optional(),                  // [REQUIRED for OIDC] OIDC client ID
+    SSO_OIDC_CLIENT_SECRET:                z.string().optional(),                  // [REQUIRED for OIDC] OIDC client secret
+    SSO_OIDC_SCOPES:                       z.string().optional(),                  // OIDC scopes (default: openid,profile,email)
+    SSO_OIDC_PKCE:                         z.string().optional(),                  // Enable PKCE (default: true)
+    SSO_OIDC_AUTHORIZATION_ENDPOINT:       z.string().optional(),                  // OIDC authorization endpoint (optional, uses discovery)
+    SSO_OIDC_TOKEN_ENDPOINT:               z.string().optional(),                  // OIDC token endpoint (optional, uses discovery)
+    SSO_OIDC_USERINFO_ENDPOINT:            z.string().optional(),                  // OIDC userinfo endpoint (optional, uses discovery)
+    SSO_OIDC_JWKS_ENDPOINT:                z.string().optional(),                  // OIDC JWKS endpoint (optional, uses discovery)
+    SSO_OIDC_DISCOVERY_ENDPOINT:           z.string().optional(),                  // OIDC discovery endpoint (default: {issuer}/.well-known/openid-configuration)
+
+    // SSO SAML Configuration
+    SSO_SAML_ENTRY_POINT:                  z.string().optional(),                  // [REQUIRED for SAML] SAML IdP SSO URL
+    SSO_SAML_CERT:                         z.string().optional(),                  // [REQUIRED for SAML] SAML IdP certificate
+    SSO_SAML_CALLBACK_URL:                 z.string().optional(),                  // SAML callback URL (default: {issuer}/callback)
+    SSO_SAML_SP_METADATA:                  z.string().optional(),                  // SAML SP metadata XML (auto-generated if not provided)
+    SSO_SAML_IDP_METADATA:                 z.string().optional(),                  // SAML IdP metadata XML (optional)
+    SSO_SAML_AUDIENCE:                     z.string().optional(),                  // SAML audience restriction (default: issuer URL)
+    SSO_SAML_WANT_ASSERTIONS_SIGNED:       z.string().optional(),                  // Require signed SAML assertions (default: false)
+    SSO_SAML_SIGNATURE_ALGORITHM:          z.string().optional(),                  // SAML signature algorithm (optional)
+    SSO_SAML_DIGEST_ALGORITHM:             z.string().optional(),                  // SAML digest algorithm (optional)
+    SSO_SAML_IDENTIFIER_FORMAT:            z.string().optional(),                  // SAML identifier format (optional)
+  },
+
+  client: {
+    // Core Application URLs - Required for frontend functionality
+    NEXT_PUBLIC_APP_URL:                   z.string().url(),                       // Base URL of the application (e.g., https://www.sim.ai)
+
+    // Client-side Services
+    NEXT_PUBLIC_SOCKET_URL:                z.string().url().optional(),            // WebSocket server URL for real-time features
+    
+    // Billing
+    NEXT_PUBLIC_BILLING_ENABLED:           z.boolean().optional(),                 // Enable billing enforcement and usage tracking (client-side)
+    
+    // Analytics & Tracking
+    NEXT_PUBLIC_POSTHOG_ENABLED:           z.boolean().optional(),                 // Enable PostHog analytics (client-side)
+    NEXT_PUBLIC_POSTHOG_KEY:               z.string().optional(),                  // PostHog project API key
+
+    // UI Branding & Whitelabeling
+    NEXT_PUBLIC_BRAND_NAME:                z.string().optional(),                  // Custom brand name (defaults to "Sim")
+    NEXT_PUBLIC_BRAND_LOGO_URL:            z.string().url().optional(),            // Custom logo URL
+    NEXT_PUBLIC_BRAND_FAVICON_URL:         z.string().url().optional(),            // Custom favicon URL
+    NEXT_PUBLIC_CUSTOM_CSS_URL:            z.string().url().optional(),            // Custom CSS stylesheet URL
+    NEXT_PUBLIC_SUPPORT_EMAIL:             z.string().email().optional(),          // Custom support email
+
+    NEXT_PUBLIC_E2B_ENABLED:               z.string().optional(),
+    NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS: z.string().optional(),              // Hide Bedrock credential fields when deployment uses AWS default credential chain (IAM roles, instance profiles, ECS task roles, IRSA)
+    NEXT_PUBLIC_AZURE_CONFIGURED:          z.string().optional(),              // Hide Azure credential fields when endpoint/key/version are pre-configured server-side
+    NEXT_PUBLIC_COHERE_CONFIGURED:         z.string().optional(),              // Hide Cohere API key field on Knowledge block when COHERE_API_KEY is pre-configured server-side
+    NEXT_PUBLIC_COPILOT_TRAINING_ENABLED:  z.string().optional(),
+    NEXT_PUBLIC_ENABLE_PLAYGROUND:         z.string().optional(),                  // Enable component playground at /playground
+    NEXT_PUBLIC_DOCUMENTATION_URL:         z.string().url().optional(),            // Custom documentation URL
+    NEXT_PUBLIC_TERMS_URL:                 z.string().url().optional(),            // Custom terms of service URL
+    NEXT_PUBLIC_PRIVACY_URL:               z.string().url().optional(),            // Custom privacy policy URL
+
+    // Theme Customization
+    NEXT_PUBLIC_BRAND_PRIMARY_COLOR:       z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),     // Primary brand color (hex format, e.g., "#33c482")
+    NEXT_PUBLIC_BRAND_PRIMARY_HOVER_COLOR: z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),     // Primary brand hover state (hex format)
+    NEXT_PUBLIC_BRAND_ACCENT_COLOR:        z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),     // Accent brand color (hex format)
+    NEXT_PUBLIC_BRAND_ACCENT_HOVER_COLOR:  z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),     // Accent brand hover state (hex format)
+    NEXT_PUBLIC_BRAND_BACKGROUND_COLOR:    z.string().regex(/^#[0-9A-Fa-f]{6}$/).optional(),     // Brand background color (hex format)
+
+    // Feature Flags
+    NEXT_PUBLIC_SSO_ENABLED:               z.boolean().optional(),                   // Enable SSO login UI components
+    NEXT_PUBLIC_ACCESS_CONTROL_ENABLED:    z.boolean().optional(),                   // Enable access control (permission groups) on self-hosted
+    NEXT_PUBLIC_CUSTOM_BLOCKS_ENABLED:     z.boolean().optional(),                   // Enable custom blocks (deploy-as-block) settings on self-hosted
+    NEXT_PUBLIC_WHITELABELING_ENABLED:     z.boolean().optional(),                   // Enable whitelabeling on self-hosted (bypasses hosted requirements)
+    NEXT_PUBLIC_AUDIT_LOGS_ENABLED:        z.boolean().optional(),                   // Enable audit logs on self-hosted (bypasses hosted requirements)
+    NEXT_PUBLIC_DATA_RETENTION_ENABLED:   z.boolean().optional(),                   // Enable data retention settings on self-hosted (bypasses hosted requirements)
+    NEXT_PUBLIC_DATA_DRAINS_ENABLED:      z.boolean().optional(),                   // Enable data drains on self-hosted (bypasses hosted requirements)
+    NEXT_PUBLIC_FORKING_ENABLED:          z.boolean().optional(),                   // Enable workspace forking on self-hosted (bypasses hosted requirements)
+    NEXT_PUBLIC_WORKFLOW_COLUMNS_ENABLED: z.boolean().optional(),                   // Show the "Workflow" column type in user tables (defaults to false)
+    NEXT_PUBLIC_ORGANIZATIONS_ENABLED:     z.boolean().optional(),                   // Enable organizations on self-hosted (bypasses plan requirements)
+    NEXT_PUBLIC_DISABLE_INVITATIONS:       z.boolean().optional(),                   // Disable workspace invitations globally (for self-hosted deployments)
+    NEXT_PUBLIC_DISABLE_PUBLIC_API:        z.boolean().optional(),                   // Disable public API access UI toggle globally
+    NEXT_PUBLIC_INBOX_ENABLED:             z.boolean().optional(),                   // Enable inbox (Sim Mailer) on self-hosted
+    NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: z.boolean().optional().default(true), // Control visibility of email/password login forms
+    NEXT_PUBLIC_TURNSTILE_SITE_KEY:        z.string().min(1).optional(),           // Cloudflare Turnstile site key for captcha widget
+  },
+
+  // Variables available on both server and client
+  shared: {
+    NODE_ENV:                              z.enum(['development', 'test', 'production']).optional(), // Runtime environment
+    NEXT_TELEMETRY_DISABLED:               z.string().optional(),                                    // Disable Next.js telemetry collection
+  },
+
+  experimental__runtimeEnv: {
+    NEXT_PUBLIC_APP_URL: process.env.NEXT_PUBLIC_APP_URL,
+    NEXT_PUBLIC_BILLING_ENABLED: process.env.NEXT_PUBLIC_BILLING_ENABLED,
+    NEXT_PUBLIC_SOCKET_URL: process.env.NEXT_PUBLIC_SOCKET_URL,
+    NEXT_PUBLIC_BRAND_NAME: process.env.NEXT_PUBLIC_BRAND_NAME,
+    NEXT_PUBLIC_BRAND_LOGO_URL: process.env.NEXT_PUBLIC_BRAND_LOGO_URL,
+    NEXT_PUBLIC_BRAND_FAVICON_URL: process.env.NEXT_PUBLIC_BRAND_FAVICON_URL,
+    NEXT_PUBLIC_CUSTOM_CSS_URL: process.env.NEXT_PUBLIC_CUSTOM_CSS_URL,
+    NEXT_PUBLIC_SUPPORT_EMAIL: process.env.NEXT_PUBLIC_SUPPORT_EMAIL,
+    NEXT_PUBLIC_DOCUMENTATION_URL: process.env.NEXT_PUBLIC_DOCUMENTATION_URL,
+    NEXT_PUBLIC_TERMS_URL: process.env.NEXT_PUBLIC_TERMS_URL,
+    NEXT_PUBLIC_PRIVACY_URL: process.env.NEXT_PUBLIC_PRIVACY_URL,
+    NEXT_PUBLIC_BRAND_PRIMARY_COLOR: process.env.NEXT_PUBLIC_BRAND_PRIMARY_COLOR,
+    NEXT_PUBLIC_BRAND_PRIMARY_HOVER_COLOR: process.env.NEXT_PUBLIC_BRAND_PRIMARY_HOVER_COLOR,
+    NEXT_PUBLIC_BRAND_ACCENT_COLOR: process.env.NEXT_PUBLIC_BRAND_ACCENT_COLOR,
+    NEXT_PUBLIC_BRAND_ACCENT_HOVER_COLOR: process.env.NEXT_PUBLIC_BRAND_ACCENT_HOVER_COLOR,
+    NEXT_PUBLIC_BRAND_BACKGROUND_COLOR: process.env.NEXT_PUBLIC_BRAND_BACKGROUND_COLOR,
+    NEXT_PUBLIC_SSO_ENABLED: process.env.NEXT_PUBLIC_SSO_ENABLED,
+    NEXT_PUBLIC_ACCESS_CONTROL_ENABLED: process.env.NEXT_PUBLIC_ACCESS_CONTROL_ENABLED,
+    NEXT_PUBLIC_CUSTOM_BLOCKS_ENABLED: process.env.NEXT_PUBLIC_CUSTOM_BLOCKS_ENABLED,
+    NEXT_PUBLIC_WHITELABELING_ENABLED: process.env.NEXT_PUBLIC_WHITELABELING_ENABLED,
+    NEXT_PUBLIC_AUDIT_LOGS_ENABLED: process.env.NEXT_PUBLIC_AUDIT_LOGS_ENABLED,
+    NEXT_PUBLIC_DATA_RETENTION_ENABLED: process.env.NEXT_PUBLIC_DATA_RETENTION_ENABLED,
+    NEXT_PUBLIC_DATA_DRAINS_ENABLED: process.env.NEXT_PUBLIC_DATA_DRAINS_ENABLED,
+    NEXT_PUBLIC_FORKING_ENABLED: process.env.NEXT_PUBLIC_FORKING_ENABLED,
+    NEXT_PUBLIC_WORKFLOW_COLUMNS_ENABLED: process.env.NEXT_PUBLIC_WORKFLOW_COLUMNS_ENABLED,
+    NEXT_PUBLIC_ORGANIZATIONS_ENABLED: process.env.NEXT_PUBLIC_ORGANIZATIONS_ENABLED,
+    NEXT_PUBLIC_DISABLE_INVITATIONS: process.env.NEXT_PUBLIC_DISABLE_INVITATIONS,
+    NEXT_PUBLIC_DISABLE_PUBLIC_API: process.env.NEXT_PUBLIC_DISABLE_PUBLIC_API,
+    NEXT_PUBLIC_INBOX_ENABLED: process.env.NEXT_PUBLIC_INBOX_ENABLED,
+    NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: process.env.NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED,
+    NEXT_PUBLIC_TURNSTILE_SITE_KEY: process.env.NEXT_PUBLIC_TURNSTILE_SITE_KEY,
+    NEXT_PUBLIC_E2B_ENABLED: process.env.NEXT_PUBLIC_E2B_ENABLED,
+    NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS: process.env.NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS,
+    NEXT_PUBLIC_AZURE_CONFIGURED: process.env.NEXT_PUBLIC_AZURE_CONFIGURED,
+    NEXT_PUBLIC_COHERE_CONFIGURED: process.env.NEXT_PUBLIC_COHERE_CONFIGURED,
+    NEXT_PUBLIC_COPILOT_TRAINING_ENABLED: process.env.NEXT_PUBLIC_COPILOT_TRAINING_ENABLED,
+    NEXT_PUBLIC_ENABLE_PLAYGROUND: process.env.NEXT_PUBLIC_ENABLE_PLAYGROUND,
+    NEXT_PUBLIC_POSTHOG_ENABLED: process.env.NEXT_PUBLIC_POSTHOG_ENABLED,
+    NEXT_PUBLIC_POSTHOG_KEY: process.env.NEXT_PUBLIC_POSTHOG_KEY,
+    NODE_ENV: process.env.NODE_ENV,
+    NEXT_TELEMETRY_DISABLED: process.env.NEXT_TELEMETRY_DISABLED,
+  },
+})
+
+// Need this utility because t3-env is returning string for boolean values.
+export const isTruthy = (value: string | boolean | number | undefined) =>
+  typeof value === 'string' ? value.toLowerCase() === 'true' || value === '1' : Boolean(value)
+
+// Utility to check if a value is explicitly false (defaults to false only if explicitly set)
+export const isFalsy = (value: string | boolean | number | undefined) =>
+  typeof value === 'string' ? value.toLowerCase() === 'false' || value === '0' : value === false
+
+export { getEnv }
+
+/**
+ * Coerce an env-derived value to a finite number ≥ `min`, falling back to the
+ * provided default when the value is unset, empty, non-finite, or below `min`.
+ * `min` defaults to `0` so configs like `KB_CONFIG_DELAY_BETWEEN_BATCHES=0`
+ * (meaning "no delay / max throughput") are honored. Pass `min: 1` for configs
+ * where zero is invalid (e.g. Redis TTLs, capacity limits).
+ *
+ * `createEnv` is configured with `skipValidation: true`, so values declared as
+ * `z.number()` arrive as raw strings when sourced from `process.env` or Helm.
+ * Use this helper anywhere a numeric env override is consumed to normalize the
+ * type at the boundary instead of relying on JS implicit coercion.
+ */
+export function envNumber(
+  value: number | string | undefined | null,
+  fallback: number,
+  options: { min?: number; integer?: boolean } = {}
+): number {
+  const min = options.min ?? 0
+  if (
+    typeof value === 'number' &&
+    Number.isFinite(value) &&
+    value >= min &&
+    (!options.integer || Number.isInteger(value))
+  ) {
+    return value
+  }
+  if (value === undefined || value === null || value === '') return fallback
+  const parsed = Number(value)
+  return Number.isFinite(parsed) && parsed >= min && (!options.integer || Number.isInteger(parsed))
+    ? parsed
+    : fallback
+}
+
+/**
+ * Coerce an env-derived value to a boolean. Returns `undefined` when unset
+ * so callers can apply context-aware defaults. Required because
+ * `Boolean("false") === true`, so `z.coerce.boolean()` would silently flip
+ * the meaning of `MY_FLAG=false`.
+ */
+export function envBoolean(value: boolean | string | undefined | null): boolean | undefined {
+  if (typeof value === 'boolean') return value
+  if (value === undefined || value === null || value === '') return undefined
+  const normalized = String(value).trim().toLowerCase()
+  return normalized === 'true' || normalized === '1' || normalized === 'yes' || normalized === 'on'
+}

apps/sim/lib/core/config/feature-flags.test.ts

@@ -0,0 +1,212 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type { FeatureFlagContext, FeatureFlagName } from '@/lib/core/config/feature-flags'
+
+const { mockFetch, mockIsPlatformAdmin, envRef, flagRef } = vi.hoisted(() => ({
+  mockFetch: vi.fn(),
+  mockIsPlatformAdmin: vi.fn(),
+  envRef: {
+    APPCONFIG_APPLICATION: 'sim-staging' as string | undefined,
+    APPCONFIG_ENVIRONMENT: 'staging' as string | undefined,
+    FORKING_ENABLED: undefined as boolean | undefined,
+    DEPLOY_AS_BLOCK: undefined as boolean | undefined,
+  },
+  flagRef: { isAppConfigEnabled: false },
+}))
+
+vi.mock('@/lib/core/config/appconfig', () => ({
+  fetchAppConfigProfile: mockFetch,
+}))
+
+vi.mock('@/lib/core/config/env', () => ({
+  isTruthy: (v: unknown) => Boolean(v),
+  get env() {
+    return envRef
+  },
+}))
+
+vi.mock('@/lib/core/config/env-flags', () => ({
+  get isAppConfigEnabled() {
+    return flagRef.isAppConfigEnabled
+  },
+}))
+
+vi.mock('@/lib/permissions/super-user', () => ({
+  isPlatformAdmin: mockIsPlatformAdmin,
+}))
+
+import { getFeatureFlags, isFeatureEnabled } from '@/lib/core/config/feature-flags'
+
+/** Make `getFeatureFlags` resolve to `doc` via the AppConfig path (also exercises parseConfig). */
+function withAppConfig(doc: unknown) {
+  flagRef.isAppConfigEnabled = true
+  mockFetch.mockImplementation((_ids, parse) => Promise.resolve(parse(doc)))
+}
+
+/**
+ * `isFeatureEnabled` only accepts registered `FeatureFlagName`s. These tests
+ * exercise the evaluation logic with throwaway flag names supplied through the
+ * AppConfig document, cast to `FeatureFlagName` through this helper.
+ */
+const enabled = (flag: string, ctx?: FeatureFlagContext) =>
+  isFeatureEnabled(flag as FeatureFlagName, ctx)
+
+describe('getFeatureFlags', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    flagRef.isAppConfigEnabled = false
+  })
+
+  it('derives flags from fallback secrets when AppConfig is disabled, without fetching', async () => {
+    const flags = await getFeatureFlags()
+    // All registered flags should be present, disabled (env vars unset in test env)
+    expect(flags['mothership-beta']).toEqual({ enabled: false })
+    expect(flags['pii-redaction']).toEqual({ enabled: false })
+    expect(flags['pii-granular-redaction']).toEqual({ enabled: false })
+    expect(flags['trigger-eu-region']).toEqual({ enabled: false })
+    expect(mockFetch).not.toHaveBeenCalled()
+  })
+
+  it('reads the feature-flags profile and normalizes the payload when enabled', async () => {
+    withAppConfig({
+      a: { enabled: true },
+      b: { orgIds: ['Org_1', ' org_1 ', '', 'org_2'], userIds: 'nope' },
+      c: 'not-an-object',
+    })
+
+    const flags = await getFeatureFlags()
+    expect(flags.a).toEqual({ enabled: true })
+    expect(flags.b).toEqual({ orgIds: ['Org_1', 'org_1', 'org_2'] })
+    expect(flags.c).toBeUndefined()
+    expect(mockFetch).toHaveBeenCalledWith(
+      { application: 'sim-staging', environment: 'staging', profile: 'feature-flags' },
+      expect.any(Function)
+    )
+  })
+
+  it('falls back to the secret-derived document when the fetch yields null', async () => {
+    flagRef.isAppConfigEnabled = true
+    mockFetch.mockResolvedValue(null)
+    const flags = await getFeatureFlags()
+    expect(flags['mothership-beta']).toEqual({ enabled: false })
+    expect(flags['pii-redaction']).toEqual({ enabled: false })
+    expect(flags['pii-granular-redaction']).toEqual({ enabled: false })
+    expect(flags['trigger-eu-region']).toEqual({ enabled: false })
+  })
+
+  it('degrades gracefully on a malformed document', async () => {
+    withAppConfig('not-an-object')
+    expect(await getFeatureFlags()).toMatchObject({})
+    withAppConfig(null)
+    expect(await getFeatureFlags()).toMatchObject({})
+  })
+})
+
+describe('isFeatureEnabled', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    flagRef.isAppConfigEnabled = false
+    envRef.FORKING_ENABLED = undefined
+    envRef.DEPLOY_AS_BLOCK = undefined
+  })
+
+  describe('workspace-forking flag', () => {
+    it('falls back to FORKING_ENABLED when AppConfig is disabled', async () => {
+      envRef.FORKING_ENABLED = undefined
+      expect(await isFeatureEnabled('workspace-forking', { userId: 'u1', orgId: 'o1' })).toBe(false)
+
+      envRef.FORKING_ENABLED = true
+      expect(await isFeatureEnabled('workspace-forking', { userId: 'u1', orgId: 'o1' })).toBe(true)
+    })
+
+    it('targets specific orgs/users via AppConfig, ignoring the fallback secret', async () => {
+      envRef.FORKING_ENABLED = undefined
+      withAppConfig({ 'workspace-forking': { orgIds: ['o1'], userIds: ['u9'] } })
+
+      expect(await isFeatureEnabled('workspace-forking', { orgId: 'o1' })).toBe(true)
+      expect(await isFeatureEnabled('workspace-forking', { userId: 'u9' })).toBe(true)
+      expect(await isFeatureEnabled('workspace-forking', { orgId: 'o2', userId: 'u1' })).toBe(false)
+    })
+  })
+
+  describe('deploy-as-block flag', () => {
+    it('falls back to DEPLOY_AS_BLOCK when AppConfig is disabled', async () => {
+      envRef.DEPLOY_AS_BLOCK = undefined
+      expect(await isFeatureEnabled('deploy-as-block', { userId: 'u1', orgId: 'o1' })).toBe(false)
+
+      envRef.DEPLOY_AS_BLOCK = true
+      expect(await isFeatureEnabled('deploy-as-block', { userId: 'u1', orgId: 'o1' })).toBe(true)
+    })
+
+    it('targets specific orgs via AppConfig, ignoring the fallback secret', async () => {
+      envRef.DEPLOY_AS_BLOCK = undefined
+      withAppConfig({ 'deploy-as-block': { orgIds: ['o1'] } })
+      expect(await isFeatureEnabled('deploy-as-block', { orgId: 'o1' })).toBe(true)
+      expect(await isFeatureEnabled('deploy-as-block', { orgId: 'o2' })).toBe(false)
+    })
+  })
+
+  it('returns false for an unknown flag', async () => {
+    withAppConfig({})
+    expect(await enabled('missing', { userId: 'u1' })).toBe(false)
+  })
+
+  it('matches the global enabled clause', async () => {
+    withAppConfig({ f: { enabled: true } })
+    expect(await enabled('f')).toBe(true)
+  })
+
+  it('matches the userId allowlist', async () => {
+    withAppConfig({ f: { userIds: ['u1'] } })
+    expect(await enabled('f', { userId: 'u1' })).toBe(true)
+    expect(await enabled('f', { userId: 'u2' })).toBe(false)
+    expect(await enabled('f', {})).toBe(false)
+  })
+
+  it('matches the orgId allowlist', async () => {
+    withAppConfig({ f: { orgIds: ['o1'] } })
+    expect(await enabled('f', { orgId: 'o1' })).toBe(true)
+    expect(await enabled('f', { orgId: 'o2' })).toBe(false)
+  })
+
+  describe('admin clause (lazy resolution)', () => {
+    it('resolves admin from userId when adminEnabled is the deciding clause', async () => {
+      withAppConfig({ f: { adminEnabled: true } })
+      mockIsPlatformAdmin.mockResolvedValue(true)
+      expect(await enabled('f', { userId: 'u1' })).toBe(true)
+      expect(mockIsPlatformAdmin).toHaveBeenCalledWith('u1')
+
+      mockIsPlatformAdmin.mockResolvedValue(false)
+      expect(await enabled('f', { userId: 'u2' })).toBe(false)
+    })
+
+    it('uses the isAdmin override without querying', async () => {
+      withAppConfig({ f: { adminEnabled: true } })
+      expect(await enabled('f', { userId: 'u1', isAdmin: true })).toBe(true)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+
+    it('resolves to false without querying when userId is absent', async () => {
+      withAppConfig({ f: { adminEnabled: true } })
+      expect(await enabled('f', { orgId: 'o1' })).toBe(false)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+
+    it('does not query when an earlier clause already matched', async () => {
+      withAppConfig({ f: { enabled: true, adminEnabled: true } })
+      expect(await enabled('f', { userId: 'u1' })).toBe(true)
+
+      withAppConfig({ g: { userIds: ['u1'], adminEnabled: true } })
+      expect(await enabled('g', { userId: 'u1' })).toBe(true)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+
+    it('does not query when the rule has no adminEnabled clause', async () => {
+      withAppConfig({ f: { userIds: ['u2'] } })
+      expect(await enabled('f', { userId: 'u1' })).toBe(false)
+      expect(mockIsPlatformAdmin).not.toHaveBeenCalled()
+    })
+  })
+})

apps/sim/lib/core/config/feature-flags.ts

@@ -0,0 +1,189 @@
+import { fetchAppConfigProfile } from '@/lib/core/config/appconfig'
+import type { AppConfigGateContext, AppConfigGateRule } from '@/lib/core/config/appconfig-rules'
+import { matchesRule, parseGateConfig } from '@/lib/core/config/appconfig-rules'
+import { env, isTruthy } from '@/lib/core/config/env'
+import { isAppConfigEnabled } from '@/lib/core/config/env-flags'
+
+/**
+ * Name of the AppConfig configuration profile holding the gated feature flags.
+ * Cross-repo contract: must match the `CfnConfigurationProfile` name created by
+ * the infra stack.
+ */
+const FEATURE_FLAGS_PROFILE = 'feature-flags'
+
+/**
+ * A single flag's gating rule. A flag is ON for a context when ANY clause matches:
+ * the global `enabled` default, the org/user allowlists, or `adminEnabled` for
+ * platform admins. An absent clause never matches. Shape shared with the other
+ * AppConfig gating documents via {@link AppConfigGateRule}.
+ */
+export type FeatureFlagRule = AppConfigGateRule
+
+export type FeatureFlagsConfig = Record<string, FeatureFlagRule>
+
+/**
+ * Per-request evaluation context. Pass only the ids you have — a missing id skips
+ * its clause. Admin status is resolved internally from `userId`; `isAdmin` is an
+ * optional fast-path override for callers that already know it (e.g. admin routes).
+ */
+export type FeatureFlagContext = AppConfigGateContext
+
+/**
+ * Registry of known feature flags. Each maps to the secret consulted ONLY when
+ * AppConfig is not the source of truth (self-hosted/OSS, local dev, or hosted
+ * without APPCONFIG_*). A truthy secret turns the flag on globally.
+ *
+ * Gating by org/user/admin is available ONLY through the hosted AppConfig document
+ * — it deliberately cannot be expressed here, so no environment can grant (e.g.)
+ * admin access from a code literal. To add a flag, register its name and the secret
+ * to fall back on.
+ */
+/**
+ * The single definition of a feature flag. Everything about a flag lives in one
+ * place: its name (the registry key), a human-readable `description`, and the
+ * `fallback` secret consulted when AppConfig isn't the source of truth (truthy ⇒ on
+ * globally).
+ *
+ * Gating by org/user/admin is deliberately NOT part of a definition — it lives only
+ * in the hosted AppConfig document, so no environment can grant access from a code
+ * literal.
+ */
+interface FeatureFlagDefinition {
+  description: string
+  /** Env/secret key consulted when AppConfig isn't the source of truth. Truthy ⇒ on. */
+  fallback: keyof typeof env
+}
+
+/** The single registry of known flags. To add a flag, add one entry here. */
+const FEATURE_FLAGS = {
+  'mothership-beta': {
+    description:
+      'Mothership beta plan/changelog artifact surfaces in the copilot VFS and doc compiler. ' +
+      'Note: userId/orgId targeting only works for WorkspaceVfs (resolved in materialize). ' +
+      'getE2BDocFormat, resolveInputFiles, and resolveWorkflowAliasForWorkspace evaluate without ' +
+      'user context — use enabled:true for global rollout rather than per-user targeting.',
+    fallback: 'MOTHERSHIP_BETA_FEATURES',
+  },
+  'table-snapshot-cache': {
+    description:
+      'Mount Sim tables into code sandboxes by reference via a version-keyed CSV snapshot in ' +
+      'object storage (reused across runs until the table mutates) instead of draining the whole ' +
+      'table into web-process heap. resolveInputFiles evaluates without user context — use ' +
+      'enabled:true for global rollout rather than per-user targeting.',
+    fallback: 'TABLE_SNAPSHOT_CACHE',
+  },
+  'pii-redaction': {
+    description:
+      'Redact PII from workflow logs via configurable Data Retention rules (Presidio at the ' +
+      'logger persist choke point) and expose the Data Retention config surfaces. Global on/off ' +
+      'only — evaluated without user/org context so the persist path and config routes always ' +
+      'agree.',
+    fallback: 'PII_REDACTION',
+  },
+  'pii-granular-redaction': {
+    description:
+      'Expose the execution-altering PII redaction stages (redact the workflow input and every ' +
+      'block output in-flight) in the Data Retention config, layered on top of pii-redaction. ' +
+      'Global on/off only — gates the config surfaces (route write + UI). Because stored rules ' +
+      'are the source of truth for the executor, a granular stage can only run once it was ' +
+      'writable, so the executor is never flag-gated at runtime (avoiding a fail-open leak).',
+    fallback: 'PII_GRANULAR_REDACTION',
+  },
+  'trigger-eu-region': {
+    description:
+      'Route Trigger.dev runs to eu-central-1 instead of the default us-east-1. Global on/off ' +
+      'only — resolved without user/org context at every task-trigger call site via ' +
+      'resolveTriggerRegion, so the whole deployment switches regions together.',
+    fallback: 'TRIGGER_EU_REGION',
+  },
+  'workspace-forking': {
+    description:
+      'Runtime rollout gate for workspace forking (fork/promote/rollback), layered on top of ' +
+      'the existing FORKING_ENABLED / Enterprise-plan gate at the shared assertForkingEnabled ' +
+      'choke point. Enforced ONLY where AppConfig is the source of truth (Sim Cloud), so ' +
+      'operators can dark-launch forking to specific orgs/users/admins without touching ' +
+      'self-hosted/local behaviour. Fallback mirrors FORKING_ENABLED for off-AppConfig reads.',
+    fallback: 'FORKING_ENABLED',
+  },
+  'deploy-as-block': {
+    description:
+      'Publish a deployed workflow as a reusable, org-wide custom block (custom name/SVG icon/' +
+      'description; Start inputs become block inputs). Gates the Deploy-modal "Block" tab and the ' +
+      'custom-block publish/list routes. Off-AppConfig falls back to DEPLOY_AS_BLOCK.',
+    fallback: 'DEPLOY_AS_BLOCK',
+  },
+} satisfies Record<string, FeatureFlagDefinition>
+
+/**
+ * The closed set of known feature flags. Derived from the registry, so a flag
+ * cannot exist — or be checked — without a definition (and its mandatory fallback).
+ */
+export type FeatureFlagName = keyof typeof FEATURE_FLAGS
+
+/** Build the fallback document from each flag's secret. Truthy secret ⇒ enabled. */
+function fallbackFlags(): FeatureFlagsConfig {
+  const flags: FeatureFlagsConfig = {}
+  for (const [name, def] of Object.entries(FEATURE_FLAGS) as Array<
+    [string, FeatureFlagDefinition]
+  >) {
+    flags[name] = { enabled: isTruthy(env[def.fallback]) }
+  }
+  return flags
+}
+
+/**
+ * Resolve platform-admin status lazily. Dynamically imported so the DB-backed
+ * helper (and `@sim/db`) stay out of this config module's load graph for callers
+ * that never reach an admin-gated flag.
+ */
+async function resolveAdmin(userId: string): Promise<boolean> {
+  const { isPlatformAdmin } = await import('@/lib/permissions/super-user')
+  return isPlatformAdmin(userId)
+}
+
+/**
+ * The admin clause is resolved last and lazily: a global/userId/orgId match
+ * short-circuits before any DB read, a rule without `adminEnabled` never queries,
+ * and a missing `userId` resolves to `false` without a query.
+ */
+async function evaluate(
+  rule: FeatureFlagRule | undefined,
+  ctx: FeatureFlagContext
+): Promise<boolean> {
+  if (!rule) return false
+  if (matchesRule(rule, ctx, false)) return true
+  if (rule.adminEnabled) {
+    const admin = ctx.isAdmin ?? (ctx.userId ? await resolveAdmin(ctx.userId) : false)
+    if (admin) return true
+  }
+  return false
+}
+
+/**
+ * Resolve the full flag document. Reads from AWS AppConfig on hosted deployments
+ * (cached, ~30s TTL, never blocks after the first fetch), otherwise derives each
+ * flag's on/off state from its registered fallback secret ({@link fallbackFlags}).
+ */
+export async function getFeatureFlags(): Promise<FeatureFlagsConfig> {
+  if (!isAppConfigEnabled) return fallbackFlags()
+
+  const value = await fetchAppConfigProfile(
+    {
+      application: env.APPCONFIG_APPLICATION as string,
+      environment: env.APPCONFIG_ENVIRONMENT as string,
+      profile: FEATURE_FLAGS_PROFILE,
+    },
+    parseGateConfig
+  )
+
+  return value ?? fallbackFlags()
+}
+
+/** Resolve a single flag for a context. Admin status is resolved internally from `userId`. */
+export async function isFeatureEnabled(
+  flag: FeatureFlagName,
+  ctx: FeatureFlagContext = {}
+): Promise<boolean> {
+  const flags = await getFeatureFlags()
+  return evaluate(flags[flag], ctx)
+}

apps/sim/lib/core/config/redis.test.ts

@@ -0,0 +1,249 @@
+import { createEnvMock, createMockRedis } from '@sim/testing'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { MockRedisConstructor } = vi.hoisted(() => ({
+  MockRedisConstructor: vi.fn(),
+}))
+
+const mockRedisInstance = createMockRedis()
+MockRedisConstructor.mockImplementation(
+  class {
+    constructor() {
+      Object.assign(this, mockRedisInstance)
+    }
+  }
+)
+
+vi.mock('@/lib/core/config/env', () => createEnvMock({ REDIS_URL: 'redis://localhost:6379' }))
+vi.mock('ioredis', () => ({
+  default: MockRedisConstructor,
+}))
+
+import {
+  closeRedisConnection,
+  extendLock,
+  getRedisClient,
+  onRedisReconnect,
+  resetForTesting,
+} from '@/lib/core/config/redis'
+
+describe('redis config', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.useFakeTimers()
+    resetForTesting()
+    MockRedisConstructor.mockImplementation(
+      class {
+        constructor() {
+          Object.assign(this, mockRedisInstance)
+        }
+      }
+    )
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  describe('onRedisReconnect', () => {
+    it('should register and invoke reconnect listeners', async () => {
+      const listener = vi.fn()
+      onRedisReconnect(listener)
+
+      getRedisClient()
+
+      mockRedisInstance.ping.mockRejectedValue(new Error('ETIMEDOUT'))
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(listener).toHaveBeenCalledTimes(1)
+    })
+
+    it('should not invoke listeners when PINGs succeed', async () => {
+      const listener = vi.fn()
+      onRedisReconnect(listener)
+
+      getRedisClient()
+      mockRedisInstance.ping.mockResolvedValue('PONG')
+
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(listener).not.toHaveBeenCalled()
+    })
+
+    it('should reset failure count on successful PING', async () => {
+      const listener = vi.fn()
+      onRedisReconnect(listener)
+
+      getRedisClient()
+
+      mockRedisInstance.ping.mockRejectedValueOnce(new Error('timeout'))
+      await vi.advanceTimersByTimeAsync(15_000)
+      mockRedisInstance.ping.mockResolvedValueOnce('PONG')
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      mockRedisInstance.ping.mockRejectedValueOnce(new Error('timeout'))
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(listener).not.toHaveBeenCalled()
+    })
+
+    it('should call disconnect(true) after 2 consecutive PING failures', async () => {
+      getRedisClient()
+
+      mockRedisInstance.ping.mockRejectedValue(new Error('ETIMEDOUT'))
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(mockRedisInstance.disconnect).not.toHaveBeenCalled()
+
+      await vi.advanceTimersByTimeAsync(15_000)
+      expect(mockRedisInstance.disconnect).toHaveBeenCalledWith(true)
+    })
+
+    it('should drop the cached client so the next getRedisClient() builds a fresh one', async () => {
+      getRedisClient()
+      const callsBefore = MockRedisConstructor.mock.calls.length
+
+      mockRedisInstance.ping.mockRejectedValue(new Error('ETIMEDOUT'))
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(mockRedisInstance.disconnect).toHaveBeenCalledWith(true)
+
+      getRedisClient()
+      expect(MockRedisConstructor.mock.calls.length).toBe(callsBefore + 1)
+    })
+
+    it('should restart the PING health check against the new client', async () => {
+      getRedisClient()
+
+      mockRedisInstance.ping.mockRejectedValue(new Error('ETIMEDOUT'))
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(mockRedisInstance.disconnect).toHaveBeenCalledTimes(1)
+
+      getRedisClient()
+
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(mockRedisInstance.disconnect).toHaveBeenCalledTimes(2)
+    })
+
+    it('should handle listener errors gracefully without breaking health check', async () => {
+      const badListener = vi.fn(() => {
+        throw new Error('listener crashed')
+      })
+      const goodListener = vi.fn()
+      onRedisReconnect(badListener)
+      onRedisReconnect(goodListener)
+
+      getRedisClient()
+      mockRedisInstance.ping.mockRejectedValue(new Error('timeout'))
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(badListener).toHaveBeenCalledTimes(1)
+      expect(goodListener).toHaveBeenCalledTimes(1)
+    })
+  })
+
+  describe('closeRedisConnection', () => {
+    it('should clear the PING interval', async () => {
+      getRedisClient()
+
+      mockRedisInstance.quit.mockResolvedValue('OK')
+      await closeRedisConnection()
+
+      mockRedisInstance.ping.mockRejectedValue(new Error('timeout'))
+      await vi.advanceTimersByTimeAsync(15_000 * 5)
+      expect(mockRedisInstance.disconnect).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('extendLock', () => {
+    const lockKey = 'copilot:chat-stream-lock:chat-1'
+    const value = 'stream-abc'
+    const ttlSeconds = 60
+
+    it('returns true when the caller still owns the lock and EXPIRE succeeds', async () => {
+      mockRedisInstance.eval.mockResolvedValueOnce(1)
+
+      const extended = await extendLock(lockKey, value, ttlSeconds)
+
+      expect(extended).toBe(true)
+      expect(mockRedisInstance.eval).toHaveBeenCalledWith(
+        expect.stringContaining('expire'),
+        1,
+        lockKey,
+        value,
+        ttlSeconds
+      )
+    })
+
+    it('returns false when the value does not match (lock owned by another)', async () => {
+      mockRedisInstance.eval.mockResolvedValueOnce(0)
+
+      const extended = await extendLock(lockKey, value, ttlSeconds)
+
+      expect(extended).toBe(false)
+    })
+
+    it('returns true as a no-op when Redis is unavailable', async () => {
+      vi.resetModules()
+      vi.doMock('@/lib/core/config/env', () =>
+        createEnvMock({ REDIS_URL: undefined as unknown as string })
+      )
+      const { extendLock: extendLockNoRedis } = await import('@/lib/core/config/redis')
+
+      const extended = await extendLockNoRedis(lockKey, value, ttlSeconds)
+
+      expect(extended).toBe(true)
+      vi.doUnmock('@/lib/core/config/env')
+    })
+  })
+
+  describe('retryStrategy', () => {
+    function captureRetryStrategy(): (times: number) => number {
+      let capturedConfig: Record<string, unknown> = {}
+      MockRedisConstructor.mockImplementation(
+        class {
+          constructor(_url: string, config: Record<string, unknown>) {
+            capturedConfig = config
+            Object.assign(this, { ping: vi.fn(), on: vi.fn() })
+          }
+        }
+      )
+
+      getRedisClient()
+
+      return capturedConfig.retryStrategy as (times: number) => number
+    }
+
+    it('should use exponential backoff with jitter', () => {
+      const retryStrategy = captureRetryStrategy()
+      expect(retryStrategy).toBeDefined()
+
+      const delay1 = retryStrategy(1)
+      expect(delay1).toBeGreaterThanOrEqual(1000)
+      expect(delay1).toBeLessThanOrEqual(1300)
+
+      const delay3 = retryStrategy(3)
+      expect(delay3).toBeGreaterThanOrEqual(4000)
+      expect(delay3).toBeLessThanOrEqual(5200)
+
+      const delay5 = retryStrategy(5)
+      expect(delay5).toBeGreaterThanOrEqual(10000)
+      expect(delay5).toBeLessThanOrEqual(13000)
+    })
+
+    it('should cap at 30s for attempts beyond 10', () => {
+      const retryStrategy = captureRetryStrategy()
+      expect(retryStrategy(11)).toBe(30000)
+      expect(retryStrategy(100)).toBe(30000)
+    })
+  })
+})

apps/sim/lib/core/config/redis.ts

@@ -0,0 +1,315 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { randomFloat } from '@sim/utils/random'
+import Redis, { type RedisOptions } from 'ioredis'
+import { env } from '@/lib/core/config/env'
+
+const logger = createLogger('Redis')
+
+const redisUrl = env.REDIS_URL
+
+/**
+ * When REDIS_URL targets a bare IP over `rediss://` (e.g. trigger.dev's
+ * PrivateLink VPCE IP), default TLS hostname verification fails — the cert
+ * is issued for the ElastiCache DNS name, not the IP. Override SNI with
+ * REDIS_TLS_SERVERNAME (set to the DNS the cert was issued for).
+ *
+ * For DNS hosts: no override needed, default verification works.
+ */
+function resolveRedisTlsOptions(url: string | undefined): { servername: string } | undefined {
+  if (!url) return undefined
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    return undefined
+  }
+  if (parsed.protocol !== 'rediss:') return undefined
+  const hostIsIp = /^\d{1,3}(\.\d{1,3}){3}$/.test(parsed.hostname)
+  if (!hostIsIp) return undefined
+  if (!env.REDIS_TLS_SERVERNAME) {
+    throw new Error(
+      'REDIS_TLS_SERVERNAME must be set when REDIS_URL targets an IP over rediss://. ' +
+        'TLS cert hostname verification cannot match an IP — set REDIS_TLS_SERVERNAME ' +
+        'to the DNS name the cert was issued for (the ElastiCache primary endpoint).'
+    )
+  }
+  return { servername: env.REDIS_TLS_SERVERNAME }
+}
+
+/**
+ * Shared connection defaults — keepAlive, connectTimeout, enableOfflineQueue,
+ * and TLS SNI when REDIS_URL targets an IP. Every Redis client we open should
+ * spread this; callers add their own retry / timeout policy on top.
+ */
+export function getRedisConnectionDefaults(
+  url: string | undefined
+): Pick<RedisOptions, 'keepAlive' | 'connectTimeout' | 'enableOfflineQueue' | 'tls'> {
+  const tls = resolveRedisTlsOptions(url)
+  return {
+    keepAlive: 1000,
+    connectTimeout: 10000,
+    enableOfflineQueue: true,
+    ...(tls ? { tls } : {}),
+  }
+}
+
+interface RedisState {
+  client: Redis | null
+  pingFailures: number
+  pingInterval: NodeJS.Timeout | null
+  pingInFlight: boolean
+  reconnectListeners: Array<() => void>
+}
+
+const g = globalThis as typeof globalThis & { _redisState?: RedisState }
+if (!g._redisState) {
+  g._redisState = {
+    client: null,
+    pingFailures: 0,
+    pingInterval: null,
+    pingInFlight: false,
+    reconnectListeners: [],
+  }
+}
+const state = g._redisState
+
+const PING_INTERVAL_MS = 15_000
+const MAX_PING_FAILURES = 2
+
+/**
+ * Register a callback that fires when the PING health check forces a reconnect.
+ * Useful for resetting cached adapters that hold a stale Redis reference.
+ */
+export function onRedisReconnect(cb: () => void): void {
+  state.reconnectListeners.push(cb)
+}
+
+function startPingHealthCheck(redis: Redis): void {
+  if (state.pingInterval) return
+
+  state.pingInterval = setInterval(async () => {
+    if (state.pingInFlight) return
+    state.pingInFlight = true
+    try {
+      await redis.ping()
+      state.pingFailures = 0
+    } catch (error) {
+      state.pingFailures++
+      logger.warn('Redis PING failed', {
+        consecutiveFailures: state.pingFailures,
+        error: toError(error).message,
+      })
+
+      if (state.pingFailures >= MAX_PING_FAILURES) {
+        logger.error('Redis PING failed consecutive times — forcing reconnect', {
+          consecutiveFailures: state.pingFailures,
+        })
+        state.pingFailures = 0
+        // Clear before notifying listeners — they may call getRedisClient() and must see the reset state.
+        state.client = null
+        if (state.pingInterval) {
+          clearInterval(state.pingInterval)
+          state.pingInterval = null
+        }
+        for (const cb of state.reconnectListeners) {
+          try {
+            cb()
+          } catch (cbError) {
+            logger.error('Redis reconnect listener error', { error: cbError })
+          }
+        }
+        try {
+          redis.disconnect(true)
+        } catch (disconnectError) {
+          logger.error('Error during forced Redis disconnect', { error: disconnectError })
+        }
+      }
+    } finally {
+      state.pingInFlight = false
+    }
+  }, PING_INTERVAL_MS)
+}
+
+/**
+ * Get a Redis client instance.
+ * Uses connection pooling to reuse connections across requests.
+ *
+ * ioredis handles command queuing internally via `enableOfflineQueue` (default: true),
+ * so commands are queued and executed once connected. No manual connection checks needed.
+ */
+export function getRedisClient(): Redis | null {
+  if (typeof window !== 'undefined') return null
+  if (!redisUrl) return null
+  if (state.client) return state.client
+
+  // Outside the try/catch so config errors aren't silently swallowed.
+  const defaults = getRedisConnectionDefaults(redisUrl)
+
+  try {
+    logger.info('Initializing Redis client')
+
+    state.client = new Redis(redisUrl, {
+      ...defaults,
+      commandTimeout: 5000,
+      maxRetriesPerRequest: 5,
+
+      retryStrategy: (times) => {
+        if (times > 10) {
+          logger.error(`Redis reconnection attempt ${times}`, { nextRetryMs: 30000 })
+          return 30000
+        }
+        const base = Math.min(1000 * 2 ** (times - 1), 10000)
+        const jitter = randomFloat() * base * 0.3
+        const delay = Math.round(base + jitter)
+        logger.warn('Redis reconnecting', { attempt: times, nextRetryMs: delay })
+        return delay
+      },
+
+      reconnectOnError: (err) => {
+        const targetErrors = ['READONLY', 'ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']
+        return targetErrors.some((e) => err.message.includes(e))
+      },
+    })
+
+    state.client.on('connect', () => logger.info('Redis connected'))
+    state.client.on('ready', () => logger.info('Redis ready'))
+    state.client.on('error', (err: Error) => {
+      logger.error('Redis error', { error: err.message, code: (err as any).code })
+    })
+    state.client.on('close', () => logger.warn('Redis connection closed'))
+    state.client.on('end', () => logger.error('Redis connection ended'))
+
+    startPingHealthCheck(state.client)
+
+    return state.client
+  } catch (error) {
+    logger.error('Failed to initialize Redis client', { error })
+    return null
+  }
+}
+
+/**
+ * Lua script for safe lock release.
+ * Only deletes the key if the value matches (ownership verification).
+ * Returns 1 if deleted, 0 if not (value mismatch or key doesn't exist).
+ */
+const RELEASE_LOCK_SCRIPT = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("del", KEYS[1])
+else
+  return 0
+end
+`
+
+/**
+ * Lua script for safe lock TTL extension.
+ * Only refreshes the expiry if the value matches (ownership verification),
+ * so a stale heartbeat from a prior owner cannot extend a lock currently
+ * held by someone else after a TTL eviction.
+ * Returns 1 if the TTL was extended, 0 if not (value mismatch or key gone).
+ */
+const EXTEND_LOCK_SCRIPT = `
+if redis.call("get", KEYS[1]) == ARGV[1] then
+  return redis.call("expire", KEYS[1], ARGV[2])
+else
+  return 0
+end
+`
+
+/**
+ * Acquire a distributed lock using Redis SET NX.
+ * Returns true if lock acquired, false if already held.
+ *
+ * When Redis is not available, returns true (lock "acquired") to allow
+ * single-replica deployments to function without Redis. In multi-replica
+ * deployments without Redis, the idempotency layer prevents duplicate processing.
+ */
+export async function acquireLock(
+  lockKey: string,
+  value: string,
+  expirySeconds: number
+): Promise<boolean> {
+  const redis = getRedisClient()
+  if (!redis) {
+    return true // No-op when Redis unavailable; idempotency layer handles duplicates
+  }
+
+  const result = await redis.set(lockKey, value, 'EX', expirySeconds, 'NX')
+  return result === 'OK'
+}
+
+/**
+ * Release a distributed lock safely.
+ * Only releases if the caller owns the lock (value matches).
+ * Returns true if lock was released, false if not owned or already expired.
+ *
+ * When Redis is not available, returns true (no-op) since no lock was held.
+ */
+export async function releaseLock(lockKey: string, value: string): Promise<boolean> {
+  const redis = getRedisClient()
+  if (!redis) {
+    return true // No-op when Redis unavailable; no lock was actually held
+  }
+
+  const result = await redis.eval(RELEASE_LOCK_SCRIPT, 1, lockKey, value)
+  return result === 1
+}
+
+/**
+ * Extend the TTL of a distributed lock if still owned by the caller.
+ * Returns true if the caller still owns the lock and the TTL was refreshed,
+ * false if the lock has been taken over by another owner or has expired.
+ *
+ * When Redis is not available, returns true (no-op) to match the behavior
+ * of `acquireLock` / `releaseLock`: single-replica deployments without
+ * Redis never held a real lock, so heartbeat success is implicit.
+ */
+export async function extendLock(
+  lockKey: string,
+  value: string,
+  expirySeconds: number
+): Promise<boolean> {
+  const redis = getRedisClient()
+  if (!redis) {
+    return true
+  }
+
+  const result = await redis.eval(EXTEND_LOCK_SCRIPT, 1, lockKey, value, expirySeconds)
+  return result === 1
+}
+
+/**
+ * Close the Redis connection.
+ * Use for graceful shutdown.
+ */
+export async function closeRedisConnection(): Promise<void> {
+  if (state.pingInterval) {
+    clearInterval(state.pingInterval)
+    state.pingInterval = null
+  }
+
+  if (state.client) {
+    try {
+      await state.client.quit()
+    } catch (error) {
+      logger.error('Error closing Redis connection', { error })
+    } finally {
+      state.client = null
+    }
+  }
+}
+
+/**
+ * Reset all module-level state. Only intended for use in tests.
+ */
+export function resetForTesting(): void {
+  if (state.pingInterval) {
+    clearInterval(state.pingInterval)
+    state.pingInterval = null
+  }
+  state.client = null
+  state.pingFailures = 0
+  state.pingInFlight = false
+  state.reconnectListeners.length = 0
+}

apps/sim/lib/core/errors/retryable-infrastructure.ts

@@ -0,0 +1,78 @@
+const RETRYABLE_DB_ERROR_CODES = new Set([
+  '08000',
+  '08001',
+  '08003',
+  '08004',
+  '08006',
+  '08007',
+  '53300',
+  '53400',
+  '57014',
+  '57P01',
+  '57P02',
+  '57P03',
+  '58000',
+  '58030',
+])
+
+const RETRYABLE_NETWORK_ERROR_CODES = new Set([
+  'ETIMEDOUT',
+  'ECONNRESET',
+  'ECONNREFUSED',
+  'EPIPE',
+  'ENETDOWN',
+  'ENETRESET',
+  'ENETUNREACH',
+  'UND_ERR_CONNECT_TIMEOUT',
+  'UND_ERR_SOCKET',
+  'UND_ERR_HEADERS_TIMEOUT',
+  'UND_ERR_BODY_TIMEOUT',
+])
+
+const RETRYABLE_APP_ERROR_CODES = new Set([
+  'SERVICE_OVERLOADED',
+  'RESOURCE_EXHAUSTED',
+  'CONNECTION_POOL_EXHAUSTED',
+])
+
+function getErrorChain(error: unknown): Array<Error & Record<string, unknown>> {
+  const chain: Array<Error & Record<string, unknown>> = []
+  let current: unknown = error
+  for (let depth = 0; depth < 10 && current instanceof Error; depth++) {
+    const candidate = current as Error & Record<string, unknown>
+    chain.push(candidate)
+    current = candidate.cause
+  }
+  return chain
+}
+
+export function describeRetryableInfrastructureError(
+  error: unknown
+): Record<string, unknown> | undefined {
+  for (const candidate of getErrorChain(error)) {
+    const code = typeof candidate.code === 'string' ? candidate.code : undefined
+    const errno = typeof candidate.errno === 'string' ? candidate.errno : undefined
+    const syscall = typeof candidate.syscall === 'string' ? candidate.syscall : undefined
+
+    if (
+      (code && RETRYABLE_DB_ERROR_CODES.has(code)) ||
+      (code && RETRYABLE_NETWORK_ERROR_CODES.has(code)) ||
+      (code && RETRYABLE_APP_ERROR_CODES.has(code)) ||
+      (errno && RETRYABLE_NETWORK_ERROR_CODES.has(errno))
+    ) {
+      return {
+        name: candidate.name,
+        message: candidate.message,
+        code,
+        errno,
+        syscall,
+      }
+    }
+  }
+
+  return undefined
+}
+
+export function isRetryableInfrastructureError(error: unknown): boolean {
+  return Boolean(describeRetryableInfrastructureError(error))
+}

apps/sim/lib/core/execution-limits/index.ts

@@ -0,0 +1 @@
+export * from './types'

apps/sim/lib/core/execution-limits/types.ts

@@ -0,0 +1,159 @@
+import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
+import { env } from '@/lib/core/config/env'
+import { isBillingEnabled } from '@/lib/core/config/env-flags'
+import type { SubscriptionPlan } from '@/lib/core/rate-limiter/types'
+
+interface ExecutionTimeoutConfig {
+  sync: number
+  async: number
+}
+
+const DEFAULT_SYNC_TIMEOUTS_SECONDS = {
+  free: 300,
+  pro: 3000,
+  team: 3000,
+  enterprise: 3000,
+} as const
+
+const DEFAULT_ASYNC_TIMEOUTS_SECONDS = {
+  free: 5400,
+  pro: 5400,
+  team: 5400,
+  enterprise: 5400,
+} as const
+
+function getSyncTimeoutForPlan(plan: SubscriptionPlan): number {
+  const envVarMap: Record<SubscriptionPlan, string | undefined> = {
+    free: env.EXECUTION_TIMEOUT_FREE,
+    pro: env.EXECUTION_TIMEOUT_PRO,
+    team: env.EXECUTION_TIMEOUT_TEAM,
+    enterprise: env.EXECUTION_TIMEOUT_ENTERPRISE,
+  }
+  return (Number.parseInt(envVarMap[plan] || '') || DEFAULT_SYNC_TIMEOUTS_SECONDS[plan]) * 1000
+}
+
+function getAsyncTimeoutForPlan(plan: SubscriptionPlan): number {
+  const envVarMap: Record<SubscriptionPlan, string | undefined> = {
+    free: env.EXECUTION_TIMEOUT_ASYNC_FREE,
+    pro: env.EXECUTION_TIMEOUT_ASYNC_PRO,
+    team: env.EXECUTION_TIMEOUT_ASYNC_TEAM,
+    enterprise: env.EXECUTION_TIMEOUT_ASYNC_ENTERPRISE,
+  }
+  return (Number.parseInt(envVarMap[plan] || '') || DEFAULT_ASYNC_TIMEOUTS_SECONDS[plan]) * 1000
+}
+
+const EXECUTION_TIMEOUTS: Record<SubscriptionPlan, ExecutionTimeoutConfig> = {
+  free: {
+    sync: getSyncTimeoutForPlan('free'),
+    async: getAsyncTimeoutForPlan('free'),
+  },
+  pro: {
+    sync: getSyncTimeoutForPlan('pro'),
+    async: getAsyncTimeoutForPlan('pro'),
+  },
+  team: {
+    sync: getSyncTimeoutForPlan('team'),
+    async: getAsyncTimeoutForPlan('team'),
+  },
+  enterprise: {
+    sync: getSyncTimeoutForPlan('enterprise'),
+    async: getAsyncTimeoutForPlan('enterprise'),
+  },
+}
+
+export function getExecutionTimeout(
+  plan: SubscriptionPlan | string | undefined,
+  type: 'sync' | 'async' = 'sync'
+): number {
+  if (!isBillingEnabled) {
+    return EXECUTION_TIMEOUTS.free[type]
+  }
+  return EXECUTION_TIMEOUTS[getPlanTypeForLimits(plan)][type]
+}
+
+export function getMaxExecutionTimeout(): number {
+  return EXECUTION_TIMEOUTS.enterprise.async
+}
+
+/** Safety buffer added beyond the max execution timeout for execution-lifetime TTLs. */
+export const RESERVATION_TTL_BUFFER_MS = 60_000
+
+/**
+ * TTL (ms) bounding how long a single execution can remain in flight: the max
+ * execution timeout plus a safety buffer. Shared source of truth for the
+ * admission-reservation key and the live progress-marker key so they expire on
+ * the same timeline.
+ */
+export function getExecutionReservationTtlMs(): number {
+  return getMaxExecutionTimeout() + RESERVATION_TTL_BUFFER_MS
+}
+
+export const DEFAULT_EXECUTION_TIMEOUT_MS = EXECUTION_TIMEOUTS.free.sync
+
+export function isTimeoutError(error: unknown): boolean {
+  if (!error) return false
+
+  if (error instanceof Error) {
+    return error.name === 'TimeoutError'
+  }
+
+  if (typeof error === 'object' && 'name' in error) {
+    return (error as { name: string }).name === 'TimeoutError'
+  }
+
+  return false
+}
+
+export function getTimeoutErrorMessage(error: unknown, timeoutMs?: number): string {
+  if (timeoutMs) {
+    const timeoutSeconds = Math.floor(timeoutMs / 1000)
+    const timeoutMinutes = Math.floor(timeoutSeconds / 60)
+    const displayTime =
+      timeoutMinutes > 0
+        ? `${timeoutMinutes} minute${timeoutMinutes > 1 ? 's' : ''}`
+        : `${timeoutSeconds} seconds`
+    return `Execution timed out after ${displayTime}`
+  }
+
+  return 'Execution timed out'
+}
+
+/**
+ * Helper to create an AbortController with timeout handling.
+ * Centralizes the timeout abort pattern used across execution paths.
+ */
+export interface TimeoutAbortController {
+  /** The AbortSignal to pass to execution functions */
+  signal: AbortSignal
+  /** Returns true if the abort was triggered by timeout (not user cancellation) */
+  isTimedOut: () => boolean
+  /** Cleanup function - call in finally block to clear the timeout */
+  cleanup: () => void
+  /** Manually abort the execution (for user cancellation) */
+  abort: () => void
+  /** The timeout duration in milliseconds (undefined if no timeout) */
+  timeoutMs: number | undefined
+}
+
+export function createTimeoutAbortController(timeoutMs?: number): TimeoutAbortController {
+  const abortController = new AbortController()
+  let isTimedOut = false
+  let timeoutId: NodeJS.Timeout | undefined
+
+  if (timeoutMs) {
+    timeoutId = setTimeout(() => {
+      isTimedOut = true
+      abortController.abort()
+    }, timeoutMs)
+  }
+
+  return {
+    signal: abortController.signal,
+    isTimedOut: () => isTimedOut,
+    cleanup: () => {
+      if (timeoutId) clearTimeout(timeoutId)
+    },
+    abort: () => abortController.abort(),
+    timeoutMs,
+  }
+}

apps/sim/lib/core/idempotency/cleanup.ts

@@ -0,0 +1,182 @@
+import { db } from '@sim/db'
+import { idempotencyKey } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { and, count, inArray, like, lt, max, min, sql } from 'drizzle-orm'
+
+const logger = createLogger('IdempotencyCleanup')
+
+export interface CleanupOptions {
+  /**
+   * Maximum age of idempotency keys in seconds before they're considered expired
+   * Default: 7 days (604800 seconds)
+   */
+  maxAgeSeconds?: number
+
+  /**
+   * Maximum number of keys to delete in a single batch
+   * Default: 1000
+   */
+  batchSize?: number
+
+  /**
+   * Specific namespace prefix to clean up (e.g., 'webhook', 'polling')
+   * Keys are prefixed with namespace, so this filters by key prefix
+   */
+  namespace?: string
+}
+
+/**
+ * Clean up expired idempotency keys from the database
+ */
+export async function cleanupExpiredIdempotencyKeys(
+  options: CleanupOptions = {}
+): Promise<{ deleted: number; errors: string[] }> {
+  const {
+    maxAgeSeconds = 7 * 24 * 60 * 60, // 7 days
+    batchSize = 1000,
+    namespace,
+  } = options
+
+  const errors: string[] = []
+  let totalDeleted = 0
+
+  try {
+    const cutoffDate = new Date(Date.now() - maxAgeSeconds * 1000)
+
+    logger.info('Starting idempotency key cleanup', {
+      cutoffDate: cutoffDate.toISOString(),
+      namespace: namespace || 'all',
+      batchSize,
+    })
+
+    let hasMore = true
+    let batchCount = 0
+
+    while (hasMore) {
+      try {
+        // Build where condition - filter by cutoff date and optionally by namespace prefix
+        const whereCondition = namespace
+          ? and(
+              lt(idempotencyKey.createdAt, cutoffDate),
+              like(idempotencyKey.key, `${namespace}:%`)
+            )
+          : lt(idempotencyKey.createdAt, cutoffDate)
+
+        // Find keys to delete with limit
+        const toDelete = await db
+          .select({ key: idempotencyKey.key })
+          .from(idempotencyKey)
+          .where(whereCondition)
+          .limit(batchSize)
+
+        if (toDelete.length === 0) {
+          break
+        }
+
+        // Delete the found records by key
+        const deleteResult = await db
+          .delete(idempotencyKey)
+          .where(
+            inArray(
+              idempotencyKey.key,
+              toDelete.map((item) => item.key)
+            )
+          )
+          .returning({ key: idempotencyKey.key })
+
+        const deletedCount = deleteResult.length
+        totalDeleted += deletedCount
+        batchCount++
+
+        if (deletedCount === 0) {
+          hasMore = false
+          logger.info('No more expired idempotency keys found')
+        } else if (deletedCount < batchSize) {
+          hasMore = false
+          logger.info(`Deleted final batch of ${deletedCount} expired idempotency keys`)
+        } else {
+          logger.info(`Deleted batch ${batchCount}: ${deletedCount} expired idempotency keys`)
+
+          await sleep(100)
+        }
+      } catch (batchError) {
+        const errorMessage = getErrorMessage(batchError, 'Unknown batch error')
+        logger.error(`Error deleting batch ${batchCount + 1}:`, batchError)
+        errors.push(`Batch ${batchCount + 1}: ${errorMessage}`)
+
+        batchCount++
+
+        if (errors.length > 5) {
+          logger.error('Too many batch errors, stopping cleanup')
+          break
+        }
+      }
+    }
+
+    logger.info('Idempotency key cleanup completed', {
+      totalDeleted,
+      batchCount,
+      errors: errors.length,
+    })
+  } catch (error) {
+    const errorMessage = getErrorMessage(error, 'Unknown error')
+    logger.error('Failed to cleanup expired idempotency keys:', error)
+    errors.push(`General error: ${errorMessage}`)
+  }
+
+  return { deleted: totalDeleted, errors }
+}
+
+/**
+ * Get statistics about idempotency key usage
+ * Uses SQL aggregations to avoid loading all keys into memory
+ */
+export async function getIdempotencyKeyStats(): Promise<{
+  totalKeys: number
+  keysByNamespace: Record<string, number>
+  oldestKey: Date | null
+  newestKey: Date | null
+}> {
+  try {
+    // Get total count and date range in a single query
+    const [statsResult] = await db
+      .select({
+        totalKeys: count(),
+        oldestKey: min(idempotencyKey.createdAt),
+        newestKey: max(idempotencyKey.createdAt),
+      })
+      .from(idempotencyKey)
+
+    // Get counts by namespace prefix using SQL substring
+    // Extracts everything before the first ':' as the namespace
+    const namespaceStats = await db
+      .select({
+        namespace: sql<string>`split_part(${idempotencyKey.key}, ':', 1)`.as('namespace'),
+        count: count(),
+      })
+      .from(idempotencyKey)
+      .groupBy(sql`split_part(${idempotencyKey.key}, ':', 1)`)
+
+    const keysByNamespace: Record<string, number> = {}
+    for (const row of namespaceStats) {
+      keysByNamespace[row.namespace || 'unknown'] = row.count
+    }
+
+    return {
+      totalKeys: statsResult?.totalKeys ?? 0,
+      keysByNamespace,
+      oldestKey: statsResult?.oldestKey ?? null,
+      newestKey: statsResult?.newestKey ?? null,
+    }
+  } catch (error) {
+    logger.error('Failed to get idempotency key stats:', error)
+    return {
+      totalKeys: 0,
+      keysByNamespace: {},
+      oldestKey: null,
+      newestKey: null,
+    }
+  }
+}

apps/sim/lib/core/idempotency/index.ts

@@ -0,0 +1,6 @@
+export * from './cleanup'
+export * from './service'
+export {
+  pollingIdempotency,
+  webhookIdempotency,
+} from './service'

apps/sim/lib/core/idempotency/service.test.ts

@@ -0,0 +1,38 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import { IdempotencyService } from '@/lib/core/idempotency/service'
+
+describe('IdempotencyService.createWebhookIdempotencyKey', () => {
+  it('uses Greenhouse-Event-ID when present', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_1',
+      { 'greenhouse-event-id': 'evt-gh-99' },
+      {},
+      'greenhouse'
+    )
+    expect(key).toBe('wh_1:evt-gh-99')
+  })
+
+  it('prefers svix-id for Resend / Svix duplicate delivery deduplication', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_1',
+      { 'svix-id': 'msg_abc123' },
+      { type: 'email.sent' },
+      'resend'
+    )
+    expect(key).toBe('wh_1:msg_abc123')
+  })
+
+  it('prefers Linear-Delivery so repeated updates to the same entity are not treated as one idempotent run', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_linear',
+      { 'linear-delivery': '234d1a4e-b617-4388-90fe-adc3633d6b72' },
+      { action: 'update', data: { id: 'shared-entity-id' } },
+      'linear'
+    )
+    expect(key).toBe('wh_linear:234d1a4e-b617-4388-90fe-adc3633d6b72')
+  })
+})

apps/sim/lib/core/idempotency/service.ts

@@ -0,0 +1,558 @@
+import { db } from '@sim/db'
+import { idempotencyKey } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { generateId } from '@sim/utils/id'
+import { eq, lt } from 'drizzle-orm'
+import { getRedisClient } from '@/lib/core/config/redis'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import { getStorageMethod, type StorageMethod } from '@/lib/core/storage'
+import { extractProviderIdentifierFromBody } from '@/lib/webhooks/providers'
+
+const logger = createLogger('IdempotencyService')
+
+export interface IdempotencyConfig {
+  ttlSeconds?: number
+  namespace?: string
+  /** When true, failed keys are deleted rather than stored so the operation is retried on the next attempt. */
+  retryFailures?: boolean
+  /**
+   * When false, only `{ success, status, error? }` is persisted — not the
+   * operation's return value. Duplicate calls still short-circuit but
+   * resolve to `undefined`. Use when callers don't consume the cached
+   * body (e.g. webhook receivers, where the provider just wants a 2xx).
+   * Defaults to true.
+   */
+  storeResultBody?: boolean
+  /**
+   * Force a specific storage backend regardless of the environment's
+   * auto-detection. Use `'database'` for correctness-critical flows
+   * (money, billing, compliance) where the claim + operation should
+   * fate-share with the Postgres transaction — this closes the narrow
+   * window where the operation commits to DB but `storeResult` to Redis
+   * fails and the retry re-runs the operation. Latency cost is 1–5ms
+   * per call, imperceptible on webhook code paths.
+   *
+   * Leave unset (or set `'redis'`) for latency-sensitive, high-volume
+   * flows like app webhook triggers where the scale benefits of Redis
+   * outweigh the narrow durability window.
+   */
+  forceStorage?: StorageMethod
+}
+
+export interface IdempotencyResult {
+  isFirstTime: boolean
+  normalizedKey: string
+  previousResult?: any
+  storageMethod: StorageMethod
+}
+
+export interface ProcessingResult {
+  success: boolean
+  result?: any
+  error?: string
+  status?: 'in-progress' | 'completed' | 'failed'
+  startedAt?: number
+}
+
+export interface AtomicClaimResult {
+  claimed: boolean
+  existingResult?: ProcessingResult
+  normalizedKey: string
+  storageMethod: StorageMethod
+}
+
+const DEFAULT_TTL = 60 * 60 * 24 * 7
+const REDIS_KEY_PREFIX = 'idempotency:'
+const MAX_WAIT_TIME_MS = getMaxExecutionTimeout()
+const POLL_INTERVAL_MS = 1000
+
+/**
+ * Universal idempotency service for webhooks, triggers, and any other operations
+ * that need duplicate prevention.
+ *
+ * Storage is determined once based on configuration:
+ * - If `forceStorage` is set → that backend unconditionally
+ * - Else if `REDIS_URL` is set → Redis
+ * - Else → PostgreSQL
+ */
+export class IdempotencyService {
+  private config: Required<Omit<IdempotencyConfig, 'forceStorage'>>
+  private storageMethod: StorageMethod
+
+  constructor(config: IdempotencyConfig = {}) {
+    this.config = {
+      ttlSeconds: config.ttlSeconds ?? DEFAULT_TTL,
+      namespace: config.namespace ?? 'default',
+      retryFailures: config.retryFailures ?? false,
+      storeResultBody: config.storeResultBody ?? true,
+    }
+    this.storageMethod = config.forceStorage ?? getStorageMethod()
+    logger.info(`IdempotencyService using ${this.storageMethod} storage`, {
+      namespace: this.config.namespace,
+      forced: Boolean(config.forceStorage),
+    })
+  }
+
+  private normalizeKey(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): string {
+    const base = `${this.config.namespace}:${provider}:${identifier}`
+
+    if (additionalContext && Object.keys(additionalContext).length > 0) {
+      const sortedKeys = Object.keys(additionalContext).sort()
+      const contextStr = sortedKeys.map((key) => `${key}=${additionalContext[key]}`).join('&')
+      return `${base}:${contextStr}`
+    }
+
+    return base
+  }
+
+  async checkIdempotency(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): Promise<IdempotencyResult> {
+    const normalizedKey = this.normalizeKey(provider, identifier, additionalContext)
+
+    if (this.storageMethod === 'redis') {
+      return this.checkIdempotencyRedis(normalizedKey)
+    }
+    return this.checkIdempotencyDb(normalizedKey)
+  }
+
+  private async checkIdempotencyRedis(normalizedKey: string): Promise<IdempotencyResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for idempotency check')
+    }
+
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+    const cachedResult = await redis.get(redisKey)
+
+    if (cachedResult) {
+      logger.debug(`Idempotency hit in Redis: ${normalizedKey}`)
+      return {
+        isFirstTime: false,
+        normalizedKey,
+        previousResult: JSON.parse(cachedResult),
+        storageMethod: 'redis',
+      }
+    }
+
+    logger.debug(`Idempotency miss in Redis: ${normalizedKey}`)
+    return {
+      isFirstTime: true,
+      normalizedKey,
+      storageMethod: 'redis',
+    }
+  }
+
+  private async checkIdempotencyDb(normalizedKey: string): Promise<IdempotencyResult> {
+    const existing = await db
+      .select({ result: idempotencyKey.result, createdAt: idempotencyKey.createdAt })
+      .from(idempotencyKey)
+      .where(eq(idempotencyKey.key, normalizedKey))
+      .limit(1)
+
+    if (existing.length > 0) {
+      const item = existing[0]
+      const isExpired = Date.now() - item.createdAt.getTime() > this.config.ttlSeconds * 1000
+
+      if (!isExpired) {
+        logger.debug(`Idempotency hit in database: ${normalizedKey}`)
+        return {
+          isFirstTime: false,
+          normalizedKey,
+          previousResult: item.result,
+          storageMethod: 'database',
+        }
+      }
+
+      await db
+        .delete(idempotencyKey)
+        .where(eq(idempotencyKey.key, normalizedKey))
+        .catch((err) => logger.warn(`Failed to clean up expired key ${normalizedKey}:`, err))
+    }
+
+    logger.debug(`Idempotency miss in database: ${normalizedKey}`)
+    return {
+      isFirstTime: true,
+      normalizedKey,
+      storageMethod: 'database',
+    }
+  }
+
+  async atomicallyClaim(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): Promise<AtomicClaimResult> {
+    const normalizedKey = this.normalizeKey(provider, identifier, additionalContext)
+    const inProgressResult: ProcessingResult = {
+      success: false,
+      status: 'in-progress',
+      startedAt: Date.now(),
+    }
+
+    if (this.storageMethod === 'redis') {
+      return this.atomicallyClaimRedis(normalizedKey, inProgressResult)
+    }
+    return this.atomicallyClaimDb(normalizedKey, inProgressResult)
+  }
+
+  private async atomicallyClaimRedis(
+    normalizedKey: string,
+    inProgressResult: ProcessingResult
+  ): Promise<AtomicClaimResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for atomic claim')
+    }
+
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+    const claimed = await redis.set(
+      redisKey,
+      JSON.stringify(inProgressResult),
+      'EX',
+      this.config.ttlSeconds,
+      'NX'
+    )
+
+    if (claimed === 'OK') {
+      logger.debug(`Atomically claimed idempotency key in Redis: ${normalizedKey}`)
+      return {
+        claimed: true,
+        normalizedKey,
+        storageMethod: 'redis',
+      }
+    }
+
+    const existingData = await redis.get(redisKey)
+    const existingResult = existingData ? JSON.parse(existingData) : null
+    logger.debug(`Idempotency key already claimed in Redis: ${normalizedKey}`)
+    return {
+      claimed: false,
+      existingResult,
+      normalizedKey,
+      storageMethod: 'redis',
+    }
+  }
+
+  private async atomicallyClaimDb(
+    normalizedKey: string,
+    inProgressResult: ProcessingResult
+  ): Promise<AtomicClaimResult> {
+    const now = new Date()
+    const expiredBefore = new Date(now.getTime() - this.config.ttlSeconds * 1000)
+
+    // `ON CONFLICT DO UPDATE WHERE created_at < expiredBefore` steals the
+    // claim when the existing row has outlived the TTL (e.g. a prior
+    // holder crashed mid-operation and never wrote `completed`/`failed`
+    // or released the key). RETURNING yields a row in two cases:
+    //   (1) fresh INSERT — no prior row existed;
+    //   (2) UPDATE of an expired row — WHERE matched.
+    // An empty RETURNING means conflict with an unexpired row; the
+    // existing holder is still live and we must not steal.
+    const insertResult = await db
+      .insert(idempotencyKey)
+      .values({
+        key: normalizedKey,
+        result: inProgressResult,
+        createdAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [idempotencyKey.key],
+        set: {
+          result: inProgressResult,
+          createdAt: now,
+        },
+        setWhere: lt(idempotencyKey.createdAt, expiredBefore),
+      })
+      .returning({ key: idempotencyKey.key })
+
+    if (insertResult.length > 0) {
+      logger.debug(`Atomically claimed idempotency key in database: ${normalizedKey}`)
+      return {
+        claimed: true,
+        normalizedKey,
+        storageMethod: 'database',
+      }
+    }
+
+    const existing = await db
+      .select({ result: idempotencyKey.result })
+      .from(idempotencyKey)
+      .where(eq(idempotencyKey.key, normalizedKey))
+      .limit(1)
+
+    const existingResult =
+      existing.length > 0 ? (existing[0].result as ProcessingResult) : undefined
+    logger.debug(`Idempotency key already claimed in database: ${normalizedKey}`)
+    return {
+      claimed: false,
+      existingResult,
+      normalizedKey,
+      storageMethod: 'database',
+    }
+  }
+
+  async waitForResult<T>(normalizedKey: string, storageMethod: 'redis' | 'database'): Promise<T> {
+    const startTime = Date.now()
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+
+    while (Date.now() - startTime < MAX_WAIT_TIME_MS) {
+      let currentResult: ProcessingResult | null = null
+
+      if (storageMethod === 'redis') {
+        const redis = getRedisClient()
+        if (!redis) {
+          throw new Error('Redis not available')
+        }
+        const data = await redis.get(redisKey)
+        currentResult = data ? JSON.parse(data) : null
+      } else {
+        const existing = await db
+          .select({ result: idempotencyKey.result })
+          .from(idempotencyKey)
+          .where(eq(idempotencyKey.key, normalizedKey))
+          .limit(1)
+        currentResult = existing.length > 0 ? (existing[0].result as ProcessingResult) : null
+      }
+
+      if (currentResult?.status === 'completed') {
+        logger.debug(`Operation completed, returning result: ${normalizedKey}`)
+        if (currentResult.success === false) {
+          throw new Error(currentResult.error || 'Previous operation failed')
+        }
+        return currentResult.result as T
+      }
+
+      if (currentResult?.status === 'failed') {
+        logger.debug(`Operation failed, throwing error: ${normalizedKey}`)
+        throw new Error(currentResult.error || 'Previous operation failed')
+      }
+
+      await sleep(POLL_INTERVAL_MS)
+    }
+
+    throw new Error(`Timeout waiting for idempotency operation to complete: ${normalizedKey}`)
+  }
+
+  async storeResult(
+    normalizedKey: string,
+    result: ProcessingResult,
+    storageMethod: 'redis' | 'database'
+  ): Promise<void> {
+    if (storageMethod === 'redis') {
+      return this.storeResultRedis(normalizedKey, result)
+    }
+    return this.storeResultDb(normalizedKey, result)
+  }
+
+  private async storeResultRedis(normalizedKey: string, result: ProcessingResult): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for storing result')
+    }
+
+    await redis.setex(
+      `${REDIS_KEY_PREFIX}${normalizedKey}`,
+      this.config.ttlSeconds,
+      JSON.stringify(result)
+    )
+    logger.debug(`Stored idempotency result in Redis: ${normalizedKey}`)
+  }
+
+  private async storeResultDb(normalizedKey: string, result: ProcessingResult): Promise<void> {
+    await db
+      .insert(idempotencyKey)
+      .values({
+        key: normalizedKey,
+        result: result,
+        createdAt: new Date(),
+      })
+      .onConflictDoUpdate({
+        target: [idempotencyKey.key],
+        set: {
+          result: result,
+          createdAt: new Date(),
+        },
+      })
+
+    logger.debug(`Stored idempotency result in database: ${normalizedKey}`)
+  }
+
+  async release(normalizedKey: string, storageMethod: 'redis' | 'database'): Promise<void> {
+    return this.deleteKey(normalizedKey, storageMethod)
+  }
+
+  private async deleteKey(
+    normalizedKey: string,
+    storageMethod: 'redis' | 'database'
+  ): Promise<void> {
+    if (storageMethod === 'redis') {
+      const redis = getRedisClient()
+      if (redis) await redis.del(`${REDIS_KEY_PREFIX}${normalizedKey}`).catch(() => {})
+    } else {
+      await db
+        .delete(idempotencyKey)
+        .where(eq(idempotencyKey.key, normalizedKey))
+        .catch(() => {})
+    }
+  }
+
+  async executeWithIdempotency<T>(
+    provider: string,
+    identifier: string,
+    operation: () => Promise<T>,
+    additionalContext?: Record<string, any>
+  ): Promise<T> {
+    const claimResult = await this.atomicallyClaim(provider, identifier, additionalContext)
+
+    if (!claimResult.claimed) {
+      const existingResult = claimResult.existingResult
+
+      if (existingResult?.status === 'completed') {
+        logger.info(`Returning cached result for: ${claimResult.normalizedKey}`)
+        if (existingResult.success === false) {
+          throw new Error(existingResult.error || 'Previous operation failed')
+        }
+        return existingResult.result as T
+      }
+
+      if (existingResult?.status === 'failed') {
+        if (this.config.retryFailures) {
+          await this.deleteKey(claimResult.normalizedKey, claimResult.storageMethod)
+          return this.executeWithIdempotency(provider, identifier, operation, additionalContext)
+        }
+        logger.info(`Previous operation failed for: ${claimResult.normalizedKey}`)
+        throw new Error(existingResult.error || 'Previous operation failed')
+      }
+
+      if (existingResult?.status === 'in-progress') {
+        logger.info(`Waiting for in-progress operation: ${claimResult.normalizedKey}`)
+        return await this.waitForResult<T>(claimResult.normalizedKey, claimResult.storageMethod)
+      }
+
+      if (existingResult) {
+        return existingResult.result as T
+      }
+
+      throw new Error(`Unexpected state: key claimed but no existing result found`)
+    }
+
+    try {
+      logger.info(`Executing new operation: ${claimResult.normalizedKey}`)
+      const result = await operation()
+
+      await this.storeResult(
+        claimResult.normalizedKey,
+        this.config.storeResultBody
+          ? { success: true, result, status: 'completed' }
+          : { success: true, status: 'completed' },
+        claimResult.storageMethod
+      )
+
+      logger.debug(`Successfully completed operation: ${claimResult.normalizedKey}`)
+      return result
+    } catch (error) {
+      const errorMessage = getErrorMessage(error, 'Unknown error')
+
+      if (this.config.retryFailures) {
+        await this.deleteKey(claimResult.normalizedKey, claimResult.storageMethod)
+      } else {
+        await this.storeResult(
+          claimResult.normalizedKey,
+          { success: false, error: errorMessage, status: 'failed' },
+          claimResult.storageMethod
+        )
+      }
+
+      logger.warn(`Operation failed: ${claimResult.normalizedKey} - ${errorMessage}`)
+      throw error
+    }
+  }
+
+  static createWebhookIdempotencyKey(
+    webhookId: string,
+    headers?: Record<string, string>,
+    body?: any,
+    provider?: string
+  ): string {
+    const normalizedHeaders = headers
+      ? Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v]))
+      : undefined
+
+    const webhookIdHeader =
+      normalizedHeaders?.['x-sim-idempotency-key'] ||
+      normalizedHeaders?.['webhook-id'] ||
+      normalizedHeaders?.['x-webhook-id'] ||
+      normalizedHeaders?.['x-shopify-webhook-id'] ||
+      normalizedHeaders?.['x-github-delivery'] ||
+      normalizedHeaders?.['x-gitlab-event-uuid'] ||
+      normalizedHeaders?.['x-event-id'] ||
+      normalizedHeaders?.['x-teams-notification-id'] ||
+      normalizedHeaders?.['svix-id'] ||
+      normalizedHeaders?.['linear-delivery'] ||
+      normalizedHeaders?.['greenhouse-event-id'] ||
+      normalizedHeaders?.['x-zm-request-id'] ||
+      normalizedHeaders?.['x-atlassian-webhook-identifier'] ||
+      normalizedHeaders?.['idempotency-key']
+
+    if (webhookIdHeader) {
+      return `${webhookId}:${webhookIdHeader}`
+    }
+
+    if (body && provider) {
+      const bodyIdentifier = extractProviderIdentifierFromBody(provider, body)
+      if (bodyIdentifier) {
+        return `${webhookId}:${bodyIdentifier}`
+      }
+    }
+
+    const uniqueId = generateId()
+    logger.warn('No unique identifier found, duplicate executions may occur', {
+      webhookId,
+      provider,
+    })
+    return `${webhookId}:${uniqueId}`
+  }
+}
+
+/**
+ * As a webhook receiver we only need a "we saw this delivery" marker —
+ * the provider's retry just needs a 2xx, not our cached response body.
+ * TTL must exceed the longest provider retry window (Gmail / Pub-Sub: 7d).
+ */
+export const webhookIdempotency = new IdempotencyService({
+  namespace: 'webhook',
+  ttlSeconds: 60 * 60 * 24 * 7, // 7 days
+  storeResultBody: false,
+})
+
+export const pollingIdempotency = new IdempotencyService({
+  namespace: 'polling',
+  ttlSeconds: 60 * 60 * 24 * 3, // 3 days
+  retryFailures: true,
+  storeResultBody: false,
+})
+
+/**
+ * Used by the internal `/api/billing/update-cost` endpoint (copilot,
+ * workspace-chat, MCP, mothership) to dedupe cost-recording calls. Storage
+ * is forced to Postgres: the operation writes AI cost to `user_stats`,
+ * and if Redis evicts the dedup key under memory pressure (high call
+ * volume) or drops it on restart, a retry would double-record usage —
+ * real money. DB storage fate-shares with `user_stats` and is
+ * eviction-proof; ~1-5ms added latency is invisible against LLM call
+ * latency.
+ */
+export const billingIdempotency = new IdempotencyService({
+  namespace: 'billing',
+  ttlSeconds: 60 * 60, // 1 hour
+  forceStorage: 'database',
+})

apps/sim/lib/core/outbox/service.test.ts

@@ -0,0 +1,389 @@
+/**
+ * @vitest-environment node
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+type OutboxRow = {
+  id: string
+  eventType: string
+  payload: unknown
+  status: 'pending' | 'processing' | 'completed' | 'dead_letter'
+  attempts: number
+  maxAttempts: number
+  availableAt: Date
+  lockedAt: Date | null
+  lastError: string | null
+  createdAt: Date
+  processedAt: Date | null
+}
+
+// Hoisted mock state — all tests manipulate these directly.
+const { state, mockDb } = vi.hoisted(() => {
+  const state = {
+    // Rows returned from the FOR UPDATE SKIP LOCKED select in claimBatch.
+    claimedRows: [] as OutboxRow[],
+    // Whether the terminal update (lease CAS) should report a match.
+    leaseHeld: true,
+    // IDs the reaper's UPDATE should return (simulates stuck `processing` rows).
+    reapedRowIds: [] as string[],
+    // Everything written (for assertions).
+    inserts: [] as Array<{ values: unknown }>,
+    updates: [] as Array<{ set: Record<string, unknown>; where?: unknown }>,
+  }
+
+  const makeUpdateChain = () => {
+    const row: { set: Record<string, unknown>; where?: unknown } = { set: {} }
+    const chain: Record<string, unknown> = {}
+    chain.set = vi.fn((s: Record<string, unknown>) => {
+      row.set = s
+      return chain
+    })
+    chain.where = vi.fn((w: unknown) => {
+      row.where = w
+      state.updates.push(row)
+      return chain
+    })
+    chain.returning = vi.fn(async () => {
+      // Terminal UPDATE (lease CAS): has `attempts` + `availableAt`
+      // on retry, or explicit completed/dead_letter. Reaper path sets
+      // status='pending' without attempts/availableAt.
+      const isReaperUpdate =
+        row.set.status === 'pending' && !('attempts' in row.set) && !('availableAt' in row.set)
+
+      if (isReaperUpdate) {
+        return state.reapedRowIds.map((id) => ({ id }))
+      }
+
+      if (
+        row.set.status === 'completed' ||
+        row.set.status === 'dead_letter' ||
+        (row.set.status === 'pending' && 'attempts' in row.set && 'availableAt' in row.set) ||
+        (!('status' in row.set) && 'attempts' in row.set && 'lockedAt' in row.set)
+      ) {
+        return state.leaseHeld ? [{ id: 'evt-1' }] : []
+      }
+
+      return []
+    })
+    return chain
+  }
+
+  const makeSelectChain = () => {
+    const chain: Record<string, unknown> = {}
+    const self = () => chain
+    chain.from = vi.fn(self)
+    chain.where = vi.fn(self)
+    chain.orderBy = vi.fn(self)
+    chain.limit = vi.fn(self)
+    chain.for = vi.fn(async () => state.claimedRows.splice(0, 1))
+    return chain
+  }
+
+  const mockDb = {
+    insert: vi.fn(() => {
+      const chain: Record<string, unknown> = {}
+      chain.values = vi.fn(async (v: unknown) => {
+        state.inserts.push({ values: v })
+      })
+      return chain
+    }),
+    update: vi.fn(() => makeUpdateChain()),
+    select: vi.fn(() => makeSelectChain()),
+    transaction: vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn(mockDb)),
+  }
+
+  return { state, mockDb }
+})
+
+vi.mock('@sim/db', () => ({ db: mockDb }))
+
+vi.mock('@sim/db/schema', () => ({
+  outboxEvent: {
+    id: 'outbox_event.id',
+    eventType: 'outbox_event.event_type',
+    payload: 'outbox_event.payload',
+    status: 'outbox_event.status',
+    attempts: 'outbox_event.attempts',
+    maxAttempts: 'outbox_event.max_attempts',
+    availableAt: 'outbox_event.available_at',
+    lockedAt: 'outbox_event.locked_at',
+    lastError: 'outbox_event.last_error',
+    createdAt: 'outbox_event.created_at',
+    processedAt: 'outbox_event.processed_at',
+    $inferSelect: {} as OutboxRow,
+  },
+}))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }),
+}))
+
+vi.mock('drizzle-orm', () => ({
+  and: vi.fn((...args) => ({ _op: 'and', args })),
+  asc: vi.fn((col) => ({ _op: 'asc', col })),
+  eq: vi.fn((col, val) => ({ _op: 'eq', col, val })),
+  inArray: vi.fn((col, vals) => ({ _op: 'inArray', col, vals })),
+  lte: vi.fn((col, val) => ({ _op: 'lte', col, val })),
+}))
+
+vi.mock('@sim/utils/id', () => ({
+  generateId: vi.fn(() => 'test-event-id'),
+}))
+
+import { enqueueOutboxEvent, processOutboxEvents } from './service'
+
+function makePendingRow(overrides: Partial<OutboxRow> = {}): OutboxRow {
+  return {
+    id: 'evt-1',
+    eventType: 'test.event',
+    payload: { foo: 'bar' },
+    status: 'pending',
+    attempts: 0,
+    maxAttempts: 10,
+    availableAt: new Date(Date.now() - 1000),
+    lockedAt: null,
+    lastError: null,
+    createdAt: new Date(Date.now() - 5000),
+    processedAt: null,
+    ...overrides,
+  }
+}
+
+function resetState() {
+  state.claimedRows = []
+  state.leaseHeld = true
+  state.reapedRowIds = []
+  state.inserts.length = 0
+  state.updates.length = 0
+}
+
+describe('enqueueOutboxEvent', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('inserts a row with the given event type and payload', async () => {
+    const id = await enqueueOutboxEvent(mockDb, 'test.event', { foo: 'bar' })
+    expect(id).toBe('test-event-id')
+    expect(state.inserts[0].values).toMatchObject({
+      id: 'test-event-id',
+      eventType: 'test.event',
+      payload: { foo: 'bar' },
+      maxAttempts: 10,
+    })
+  })
+
+  it('respects maxAttempts override', async () => {
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { maxAttempts: 3 })
+    expect(state.inserts[0].values).toMatchObject({ maxAttempts: 3 })
+  })
+
+  it('respects availableAt override for delayed processing', async () => {
+    const future = new Date(Date.now() + 60_000)
+    await enqueueOutboxEvent(mockDb, 'test.event', {}, { availableAt: future })
+    expect((state.inserts[0].values as { availableAt: Date }).availableAt).toBe(future)
+  })
+})
+
+describe('processOutboxEvents — empty / no handler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('returns zero counts when no events are due', async () => {
+    const result = await processOutboxEvents({})
+    expect(result).toEqual({
+      processed: 0,
+      retried: 0,
+      deadLettered: 0,
+      leaseLost: 0,
+      reaped: 0,
+    })
+  })
+
+  it('dead-letters events with no registered handler', async () => {
+    state.claimedRows = [makePendingRow({ eventType: 'unknown.event' })]
+
+    const result = await processOutboxEvents({})
+
+    expect(result.deadLettered).toBe(1)
+    const terminal = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(terminal).toBeDefined()
+    expect(terminal?.set.lastError).toMatch(/No handler registered/)
+  })
+})
+
+describe('processOutboxEvents — handler success and retry', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('transitions to completed on handler success and passes context to handler', async () => {
+    const handlerCalls: Array<{ payload: unknown; eventId: string; attempts: number }> = []
+    const handler = vi.fn(async (payload: unknown, ctx: { eventId: string; attempts: number }) => {
+      handlerCalls.push({ payload, eventId: ctx.eventId, attempts: ctx.attempts })
+    })
+
+    state.claimedRows = [makePendingRow()]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.processed).toBe(1)
+    expect(handlerCalls).toEqual([{ payload: { foo: 'bar' }, eventId: 'evt-1', attempts: 0 }])
+    const completeUpdate = state.updates.find((u) => u.set.status === 'completed')
+    expect(completeUpdate).toBeDefined()
+  })
+
+  it('schedules retry with exponential backoff on handler failure below maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+
+    const before = Date.now()
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.retried).toBe(1)
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    expect(retryUpdate?.set.attempts).toBe(3)
+    expect(retryUpdate?.set.lastError).toBe('transient failure')
+    // Backoff after nextAttempts=3: 1000 * 2^3 = 8000ms
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 7500)
+    expect(scheduledAt.getTime()).toBeLessThan(before + 10_000)
+  })
+
+  it('dead-letters on failure when attempts reaches maxAttempts', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('permanent failure')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 9, maxAttempts: 10 })]
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.deadLettered).toBe(1)
+    const deadUpdate = state.updates.find((u) => u.set.status === 'dead_letter')
+    expect(deadUpdate).toBeDefined()
+    expect(deadUpdate?.set.attempts).toBe(10)
+    expect(deadUpdate?.set.lastError).toBe('permanent failure')
+  })
+
+  it('caps exponential backoff at 1 hour', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 20, maxAttempts: 100 })]
+
+    const before = Date.now()
+    await processOutboxEvents({ 'test.event': handler })
+
+    const retryUpdate = state.updates.find((u) => u.set.status === 'pending' && 'attempts' in u.set)
+    expect(retryUpdate).toBeDefined()
+    const scheduledAt = retryUpdate?.set.availableAt as Date
+    // 1hr = 3,600,000ms
+    expect(scheduledAt.getTime()).toBeLessThan(before + 3_600_000 + 1000)
+    expect(scheduledAt.getTime()).toBeGreaterThan(before + 3_599_000)
+  })
+})
+
+describe('processOutboxEvents — lease CAS / reaper race', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reports leaseLost when completion UPDATE affects zero rows', async () => {
+    const handler = vi.fn(async () => {
+      // "succeeds" but terminal write will fail the lease CAS
+    })
+
+    state.claimedRows = [makePendingRow()]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.processed).toBe(0)
+  })
+
+  it('reports leaseLost on retry-schedule UPDATE when row was reclaimed', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('transient')
+    })
+
+    state.claimedRows = [makePendingRow({ attempts: 2 })]
+    state.leaseHeld = false
+
+    const result = await processOutboxEvents({ 'test.event': handler })
+
+    expect(result.leaseLost).toBe(1)
+    expect(result.retried).toBe(0)
+  })
+})
+
+describe('processOutboxEvents — handler timeout', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('times out a stuck handler without releasing it for overlapping retry', async () => {
+    const neverResolves = vi.fn(() => new Promise<void>(() => {}))
+
+    state.claimedRows = [makePendingRow({ attempts: 0 })]
+
+    const promise = processOutboxEvents({ 'test.event': neverResolves })
+    // Must exceed DEFAULT_HANDLER_TIMEOUT_MS (90s).
+    await vi.advanceTimersByTimeAsync(90 * 1000 + 1)
+    const result = await promise
+
+    expect(result.leaseLost).toBe(1)
+    const timeoutUpdate = state.updates.find(
+      (u) => !('status' in u.set) && 'attempts' in u.set && 'lockedAt' in u.set
+    )
+    expect(timeoutUpdate?.set.attempts).toBe(1)
+    expect(timeoutUpdate?.set.lastError).toMatch(/timed out/)
+  })
+})
+
+describe('processOutboxEvents — reaper recovery', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetState()
+  })
+
+  it('reaps stuck processing rows back to pending and reports count', async () => {
+    state.reapedRowIds = ['stuck-1', 'stuck-2', 'stuck-3']
+
+    const result = await processOutboxEvents({})
+
+    expect(result.reaped).toBe(3)
+    expect(result.processed).toBe(0)
+
+    // The reaper's UPDATE sets status='pending' with NO attempts / availableAt
+    // fields — that's how runHandler's retry update is distinguished from it.
+    const reaperUpdate = state.updates.find(
+      (u) => u.set.status === 'pending' && !('attempts' in u.set) && !('availableAt' in u.set)
+    )
+    expect(reaperUpdate).toBeDefined()
+    expect(reaperUpdate?.set.lockedAt).toBeNull()
+  })
+
+  it('returns zero reaped when no rows are stuck', async () => {
+    const result = await processOutboxEvents({})
+    expect(result.reaped).toBe(0)
+  })
+})

apps/sim/lib/core/outbox/service.ts

@@ -0,0 +1,580 @@
+import { db } from '@sim/db'
+import { outboxEvent } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { generateId } from '@sim/utils/id'
+import { and, asc, eq, inArray, lte, sql } from 'drizzle-orm'
+
+const logger = createLogger('OutboxService')
+
+const DEFAULT_MAX_ATTEMPTS = 10
+const STUCK_PROCESSING_THRESHOLD_MS = 10 * 60 * 1000 // 10 minutes
+const MAX_BACKOFF_MS = 60 * 60 * 1000 // 1 hour
+const BASE_BACKOFF_MS = 1000 // 1 second, doubled per attempt
+// Kept below the serverless route `maxDuration` (120s) so our in-process
+// timeout fires before the platform kills the invocation and leaves the
+// row stranded in `processing` for the 10-minute reaper window. Also well
+// under `STUCK_PROCESSING_THRESHOLD_MS` so the reaper cannot steal a row
+// a worker is still actively processing.
+const DEFAULT_HANDLER_TIMEOUT_MS = 90 * 1000 // 90 seconds
+
+class OutboxHandlerTimeoutError extends Error {
+  constructor(timeoutMs: number) {
+    super(`Outbox handler timed out after ${timeoutMs}ms`)
+    this.name = 'OutboxHandlerTimeoutError'
+  }
+}
+
+/**
+ * Context passed to every outbox handler. Use `eventId` as the Stripe
+ * (or any external service) idempotency key so that handler retries
+ * collapse on the external side: a second execution of the same event
+ * lands on the same Stripe invoice id / charge id rather than creating
+ * a duplicate. The outbox lease CAS handles our DB side.
+ */
+interface OutboxEventContext {
+  eventId: string
+  eventType: string
+  /** How many times this event has been attempted (zero on first run). */
+  attempts: number
+}
+
+/**
+ * A handler invoked by the outbox worker for events of a given type.
+ * Throwing bumps `attempts` and schedules a retry via exponential
+ * backoff; a successful return transitions the event to `completed`.
+ */
+export type OutboxHandler<T = unknown> = (payload: T, context: OutboxEventContext) => Promise<void>
+
+/**
+ * Map of `eventType` → handler. Register all handlers in one place
+ * and pass them to `processOutboxEvents`.
+ */
+export type OutboxHandlerRegistry = Record<string, OutboxHandler>
+
+export interface EnqueueOptions {
+  /** Total attempts before the event moves to `dead_letter`. Default 10. */
+  maxAttempts?: number
+  /** Earliest time a worker may pick up this event. Default now. */
+  availableAt?: Date
+}
+
+export interface ProcessOutboxResult {
+  processed: number
+  retried: number
+  deadLettered: number
+  leaseLost: number
+  reaped: number
+}
+
+export type ProcessSingleOutboxResult =
+  | 'completed'
+  | 'pending'
+  | 'dead_letter'
+  | 'lease_lost'
+  | 'not_found'
+  | 'processing'
+
+/**
+ * Transactional outbox for reliable "DB write + external system" flows.
+ *
+ * Callers enqueue an event *inside* a `db.transaction` alongside the
+ * primary write; the event row commits or rolls back with the business
+ * data. A polling worker (invoked via the cron endpoint) claims pending
+ * rows with `SELECT ... FOR UPDATE SKIP LOCKED`, marks them as
+ * `processing`, runs the registered handler outside the transaction,
+ * and transitions the event to `completed` / `pending` (retry) /
+ * `dead_letter` (max attempts exceeded).
+ *
+ * Two-phase claim-then-process keeps external API calls out of DB
+ * transactions. A reaper at the top of each run reclaims `processing`
+ * rows whose worker died mid-operation (stale `lockedAt`).
+ *
+ * Enqueue must be called with a `tx` from `db.transaction` so atomicity
+ * with the primary write is preserved. `db` itself is also accepted but
+ * then the caller must guarantee the enqueue and the primary write share
+ * a transaction some other way (or none at all).
+ */
+export async function enqueueOutboxEvent<T>(
+  executor: Pick<typeof db, 'insert'>,
+  eventType: string,
+  payload: T,
+  options: EnqueueOptions = {}
+): Promise<string> {
+  const id = generateId()
+  await executor.insert(outboxEvent).values({
+    id,
+    eventType,
+    payload: payload as never,
+    maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
+    availableAt: options.availableAt ?? new Date(),
+  })
+  logger.info('Enqueued outbox event', { id, eventType })
+  return id
+}
+
+/** Cap on how many dead-lettered rows a single reconciler scan materializes. */
+const DEAD_LETTER_SCAN_LIMIT = 100
+
+/**
+ * Return events currently in `dead_letter` for the given event types (capped at
+ * `DEAD_LETTER_SCAN_LIMIT`). Used by periodic reconcilers to surface stuck work
+ * that exhausted its retries and now needs operator attention — the cap keeps a
+ * runaway backlog from materializing unboundedly into memory each run.
+ */
+export async function findDeadLetteredEvents(
+  eventTypes: string[]
+): Promise<(typeof outboxEvent.$inferSelect)[]> {
+  if (eventTypes.length === 0) return []
+  return db
+    .select()
+    .from(outboxEvent)
+    .where(and(eq(outboxEvent.status, 'dead_letter'), inArray(outboxEvent.eventType, eventTypes)))
+    .limit(DEAD_LETTER_SCAN_LIMIT)
+}
+
+/**
+ * True when an event of the given type whose JSON payload has
+ * `payload->>payloadKey === payloadValue` is still `pending` or `processing`.
+ * Lets a competing writer detect that a DB→external sync is already in flight
+ * for a subject and avoid clobbering the not-yet-pushed DB value.
+ */
+export async function hasInflightOutboxEvent(
+  eventType: string,
+  payloadKey: string,
+  payloadValue: string
+): Promise<boolean> {
+  const [row] = await db
+    .select({ id: outboxEvent.id })
+    .from(outboxEvent)
+    .where(
+      and(
+        eq(outboxEvent.eventType, eventType),
+        inArray(outboxEvent.status, ['pending', 'processing']),
+        sql`${outboxEvent.payload} ->> ${payloadKey} = ${payloadValue}`
+      )
+    )
+    .limit(1)
+  return Boolean(row)
+}
+
+/**
+ * Process one batch of outbox events. Safe to call concurrently from
+ * multiple workers — `SELECT FOR UPDATE SKIP LOCKED` serializes claims.
+ */
+export async function processOutboxEvents(
+  handlers: OutboxHandlerRegistry,
+  options: { batchSize?: number; maxRuntimeMs?: number; minRemainingMs?: number } = {}
+): Promise<ProcessOutboxResult> {
+  const batchSize = options.batchSize ?? 10
+  const deadline = options.maxRuntimeMs ? Date.now() + options.maxRuntimeMs : undefined
+  const minRemainingMs = options.minRemainingMs ?? DEFAULT_HANDLER_TIMEOUT_MS + 5000
+
+  const reaped = await reapStuckProcessingRows()
+
+  let processed = 0
+  let retried = 0
+  let deadLettered = 0
+  let leaseLost = 0
+
+  for (let i = 0; i < batchSize; i++) {
+    if (deadline && Date.now() + minRemainingMs > deadline) break
+
+    const [event] = await claimBatch(1)
+    if (!event) break
+
+    const result = await runHandler(event, handlers)
+    if (result === 'completed') processed++
+    else if (result === 'dead_letter') deadLettered++
+    else if (result === 'lease_lost') leaseLost++
+    else retried++
+  }
+
+  return { processed, retried, deadLettered, leaseLost, reaped }
+}
+
+/**
+ * Process a specific outbox event immediately after its surrounding
+ * transaction commits. Safe to race with the cron worker: the claim uses
+ * `FOR UPDATE SKIP LOCKED`, and non-pending rows are left alone.
+ */
+export async function processOutboxEventById(
+  eventId: string,
+  handlers: OutboxHandlerRegistry
+): Promise<ProcessSingleOutboxResult> {
+  const now = new Date()
+  const event = await db.transaction(async (tx) => {
+    const [row] = await tx
+      .select()
+      .from(outboxEvent)
+      .where(eq(outboxEvent.id, eventId))
+      .limit(1)
+      .for('update', { skipLocked: true })
+
+    if (!row) return null
+    if (row.status !== 'pending') return row.status as ProcessSingleOutboxResult
+    if (row.availableAt > now) return 'pending' as const
+
+    await tx
+      .update(outboxEvent)
+      .set({ status: 'processing', lockedAt: now })
+      .where(eq(outboxEvent.id, eventId))
+
+    return {
+      ...row,
+      status: 'processing' as const,
+      lockedAt: now,
+    }
+  })
+
+  if (!event) {
+    const [current] = await db
+      .select({ status: outboxEvent.status })
+      .from(outboxEvent)
+      .where(eq(outboxEvent.id, eventId))
+      .limit(1)
+    return current ? (current.status as ProcessSingleOutboxResult) : 'not_found'
+  }
+  if (typeof event === 'string') return event
+  return runHandler(event, handlers)
+}
+
+/**
+ * Reaper: move `processing` rows whose worker died (stale `lockedAt`)
+ * back to `pending` so another worker can pick them up. Without this,
+ * a SIGKILL between claim and result-write would permanently strand
+ * the row in `processing`.
+ */
+async function reapStuckProcessingRows(): Promise<number> {
+  const stuckBefore = new Date(Date.now() - STUCK_PROCESSING_THRESHOLD_MS)
+  const result = await db
+    .update(outboxEvent)
+    .set({ status: 'pending', lockedAt: null })
+    .where(and(eq(outboxEvent.status, 'processing'), lte(outboxEvent.lockedAt, stuckBefore)))
+    .returning({ id: outboxEvent.id })
+
+  if (result.length > 0) {
+    logger.warn('Reaped stuck outbox processing rows', {
+      count: result.length,
+      thresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
+    })
+  }
+  return result.length
+}
+
+/**
+ * Phase 1: claim a batch of due pending events.
+ *
+ * `SELECT ... FOR UPDATE SKIP LOCKED` atomically picks rows that no
+ * other worker is currently looking at. We then flip those rows to
+ * `processing` inside the same tx so the claim survives the lock
+ * release — the status change becomes the out-of-band mutual exclusion.
+ */
+async function claimBatch(batchSize: number): Promise<(typeof outboxEvent.$inferSelect)[]> {
+  const now = new Date()
+  return db.transaction(async (tx) => {
+    const rows = await tx
+      .select()
+      .from(outboxEvent)
+      .where(and(eq(outboxEvent.status, 'pending'), lte(outboxEvent.availableAt, now)))
+      .orderBy(asc(outboxEvent.createdAt))
+      .limit(batchSize)
+      .for('update', { skipLocked: true })
+
+    if (rows.length === 0) return []
+
+    await tx
+      .update(outboxEvent)
+      .set({ status: 'processing', lockedAt: now })
+      .where(
+        inArray(
+          outboxEvent.id,
+          rows.map((r) => r.id)
+        )
+      )
+
+    // Return rows with the claim state we just committed. `lockedAt`
+    // on this object is the authoritative lease timestamp used by the
+    // terminal-update lease CAS (see `runHandler`).
+    return rows.map((row) => ({
+      ...row,
+      status: 'processing' as const,
+      lockedAt: now,
+    }))
+  })
+}
+
+/**
+ * Phase 2: invoke the handler for a claimed event, outside any DB
+ * transaction, then transition the row to its terminal or retry state.
+ *
+ * Every terminal UPDATE is guarded by a lease CAS (`WHERE status =
+ * 'processing' AND locked_at = event.lockedAt`). This defends against
+ * the "slow handler + reaper" race: if our handler takes longer than
+ * `STUCK_PROCESSING_THRESHOLD_MS`, the reaper will have reset the row
+ * to `pending` and another worker may have reclaimed it with a fresh
+ * `locked_at`. Our stale terminal write's WHERE clause won't match —
+ * rowCount is 0 — and we log+skip instead of clobbering the new lease.
+ */
+async function runHandler(
+  event: typeof outboxEvent.$inferSelect,
+  handlers: OutboxHandlerRegistry
+): Promise<'completed' | 'pending' | 'dead_letter' | 'lease_lost'> {
+  const handler = handlers[event.eventType]
+
+  if (!handler) {
+    logger.error('No handler registered for outbox event type', {
+      eventId: event.id,
+      eventType: event.eventType,
+    })
+    await updateIfLeaseHeld(event, {
+      status: 'dead_letter',
+      lastError: `No handler registered for event type '${event.eventType}'`,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    return 'dead_letter'
+  }
+
+  try {
+    await runHandlerWithTimeout(handler, event)
+    const updated = await updateIfLeaseHeld(event, {
+      status: 'completed',
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) {
+      logger.warn('Outbox event completion skipped — lease lost (reaped + reclaimed)', {
+        eventId: event.id,
+        eventType: event.eventType,
+      })
+      return 'lease_lost'
+    }
+    logger.info('Outbox event processed', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: event.attempts + 1,
+    })
+    return 'completed'
+  } catch (error) {
+    if (error instanceof OutboxHandlerTimeoutError) {
+      return recordTimedOutAttempt(event, error.message)
+    }
+
+    const nextAttempts = event.attempts + 1
+    const isDead = nextAttempts >= event.maxAttempts
+    const errMsg = toError(error).message
+
+    if (isDead) {
+      const updated = await updateIfLeaseHeld(event, {
+        attempts: nextAttempts,
+        status: 'dead_letter',
+        lastError: errMsg,
+        processedAt: new Date(),
+        lockedAt: null,
+      })
+      if (!updated) {
+        logger.warn('Outbox event dead-letter skipped — lease lost', {
+          eventId: event.id,
+          eventType: event.eventType,
+        })
+        return 'lease_lost'
+      }
+      logger.error('Outbox event dead-lettered after max attempts', {
+        eventId: event.id,
+        eventType: event.eventType,
+        attempts: nextAttempts,
+        error: errMsg,
+      })
+      return 'dead_letter'
+    }
+
+    return scheduleRetry(event, errMsg)
+  }
+}
+
+async function recordTimedOutAttempt(
+  event: typeof outboxEvent.$inferSelect,
+  errMsg: string
+): Promise<'dead_letter' | 'lease_lost'> {
+  const nextAttempts = event.attempts + 1
+  const isDead = nextAttempts >= event.maxAttempts
+
+  if (isDead) {
+    const updated = await updateIfLeaseHeld(event, {
+      attempts: nextAttempts,
+      status: 'dead_letter',
+      lastError: errMsg,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) return 'lease_lost'
+    logger.error('Outbox event dead-lettered after handler timeout max attempts', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: nextAttempts,
+      error: errMsg,
+    })
+    return 'dead_letter'
+  }
+
+  const updated = await updateProcessingIfLeaseHeld(event, {
+    attempts: nextAttempts,
+    lastError: errMsg,
+    lockedAt: new Date(),
+  })
+  if (!updated) return 'lease_lost'
+
+  logger.warn('Outbox event handler timed out; leaving lease for stuck-row reaper', {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: nextAttempts,
+    reaperThresholdMs: STUCK_PROCESSING_THRESHOLD_MS,
+    error: errMsg,
+  })
+  return 'lease_lost'
+}
+
+async function scheduleRetry(
+  event: typeof outboxEvent.$inferSelect,
+  errMsg: string,
+  minimumBackoffMs = 0
+): Promise<'pending' | 'dead_letter' | 'lease_lost'> {
+  const nextAttempts = event.attempts + 1
+  const isDead = nextAttempts >= event.maxAttempts
+
+  if (isDead) {
+    const updated = await updateIfLeaseHeld(event, {
+      attempts: nextAttempts,
+      status: 'dead_letter',
+      lastError: errMsg,
+      processedAt: new Date(),
+      lockedAt: null,
+    })
+    if (!updated) {
+      logger.warn('Outbox event dead-letter skipped — lease lost', {
+        eventId: event.id,
+        eventType: event.eventType,
+      })
+      return 'lease_lost'
+    }
+    logger.error('Outbox event dead-lettered after max attempts', {
+      eventId: event.id,
+      eventType: event.eventType,
+      attempts: nextAttempts,
+      error: errMsg,
+    })
+    return 'dead_letter'
+  }
+
+  const backoffMs = Math.max(
+    minimumBackoffMs,
+    Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** nextAttempts)
+  )
+  const nextAvailableAt = new Date(Date.now() + backoffMs)
+  const updated = await updateIfLeaseHeld(event, {
+    attempts: nextAttempts,
+    status: 'pending',
+    lastError: errMsg,
+    availableAt: nextAvailableAt,
+    lockedAt: null,
+  })
+  if (!updated) {
+    logger.warn('Outbox event retry-schedule skipped — lease lost', {
+      eventId: event.id,
+      eventType: event.eventType,
+    })
+    return 'lease_lost'
+  }
+  logger.warn('Outbox event failed, scheduled retry', {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: nextAttempts,
+    backoffMs,
+    nextAvailableAt: nextAvailableAt.toISOString(),
+    error: errMsg,
+  })
+  return 'pending'
+}
+
+async function updateProcessingIfLeaseHeld(
+  event: typeof outboxEvent.$inferSelect,
+  patch: {
+    attempts: number
+    lastError: string
+    lockedAt: Date
+  }
+): Promise<boolean> {
+  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
+  if (event.lockedAt) {
+    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
+  }
+
+  const result = await db
+    .update(outboxEvent)
+    .set(patch)
+    .where(and(...whereClauses))
+    .returning({ id: outboxEvent.id })
+
+  return result.length > 0
+}
+
+function runHandlerWithTimeout(
+  handler: OutboxHandler,
+  event: typeof outboxEvent.$inferSelect,
+  timeoutMs: number = DEFAULT_HANDLER_TIMEOUT_MS
+): Promise<void> {
+  const context: OutboxEventContext = {
+    eventId: event.id,
+    eventType: event.eventType,
+    attempts: event.attempts,
+  }
+
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new OutboxHandlerTimeoutError(timeoutMs))
+    }, timeoutMs)
+
+    handler(event.payload, context)
+      .then((value) => {
+        clearTimeout(timeout)
+        resolve(value)
+      })
+      .catch((err) => {
+        clearTimeout(timeout)
+        reject(err)
+      })
+  })
+}
+
+/**
+ * Conditional terminal update scoped to the lease acquired at claim
+ * time. Returns true if the UPDATE affected a row, false if the row's
+ * lease was revoked (reaped, reclaimed by another worker). Callers
+ * treat `false` as a "lease lost" signal and skip without retrying —
+ * the newer owner is responsible for the row now.
+ */
+async function updateIfLeaseHeld(
+  event: typeof outboxEvent.$inferSelect,
+  patch: {
+    status: 'completed' | 'pending' | 'dead_letter'
+    attempts?: number
+    lastError?: string | null
+    availableAt?: Date
+    lockedAt: Date | null
+    processedAt?: Date | null
+  }
+): Promise<boolean> {
+  const whereClauses = [eq(outboxEvent.id, event.id), eq(outboxEvent.status, 'processing')]
+  if (event.lockedAt) {
+    whereClauses.push(eq(outboxEvent.lockedAt, event.lockedAt))
+  }
+
+  const result = await db
+    .update(outboxEvent)
+    .set(patch)
+    .where(and(...whereClauses))
+    .returning({ id: outboxEvent.id })
+
+  return result.length > 0
+}

apps/sim/lib/core/rate-limiter/hosted-key/hosted-key-rate-limiter.test.ts

@@ -0,0 +1,864 @@
+import { sleep } from '@sim/utils/helpers'
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenStatus,
+} from '@/lib/core/rate-limiter/storage'
+import { HostedKeyRateLimiter } from './hosted-key-rate-limiter'
+import { HEARTBEAT_REFRESH_INTERVAL_MS, type HostedKeyQueue } from './queue'
+import type { CustomRateLimit, PerRequestRateLimit } from './types'
+
+/** Force the queue wait to give up on the first iteration by reporting a retry time
+ *  larger than the 5-minute MAX_QUEUE_WAIT_MS cap. */
+const RETRY_PAST_CAP_MS = 6 * 60 * 1000
+
+interface MockAdapter {
+  consumeTokens: Mock
+  getTokenStatus: Mock
+  resetBucket: Mock
+}
+
+const createMockAdapter = (): MockAdapter => ({
+  consumeTokens: vi.fn(),
+  getTokenStatus: vi.fn(),
+  resetBucket: vi.fn(),
+})
+
+interface MockQueue {
+  enqueue: Mock
+  checkHead: Mock
+  refreshHeartbeat: Mock
+  dequeue: Mock
+}
+
+/** Stub queue that defaults to "you're at the head, no waiting" — i.e. acts as if the
+ *  queue is empty or Redis is unavailable. Tests override per-call to simulate ordering. */
+const createMockQueue = (): MockQueue => {
+  const queue: MockQueue = {
+    enqueue: vi.fn().mockResolvedValue({ position: 0, enabled: true }),
+    checkHead: vi.fn().mockResolvedValue('head'),
+    refreshHeartbeat: vi.fn().mockResolvedValue(undefined),
+    dequeue: vi.fn().mockResolvedValue(undefined),
+  }
+  return queue
+}
+
+describe('HostedKeyRateLimiter', () => {
+  const testProvider = 'exa'
+  const envKeyPrefix = 'EXA_API_KEY'
+  let mockAdapter: MockAdapter
+  let mockQueue: MockQueue
+  let rateLimiter: HostedKeyRateLimiter
+  let originalEnv: NodeJS.ProcessEnv
+
+  const perRequestRateLimit: PerRequestRateLimit = {
+    mode: 'per_request',
+    requestsPerMinute: 10,
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockAdapter = createMockAdapter()
+    mockQueue = createMockQueue()
+    rateLimiter = new HostedKeyRateLimiter(
+      mockAdapter as RateLimitStorageAdapter,
+      mockQueue as unknown as HostedKeyQueue
+    )
+
+    originalEnv = { ...process.env }
+    process.env.EXA_API_KEY_COUNT = '3'
+    process.env.EXA_API_KEY_1 = 'test-key-1'
+    process.env.EXA_API_KEY_2 = 'test-key-2'
+    process.env.EXA_API_KEY_3 = 'test-key-3'
+  })
+
+  afterEach(() => {
+    process.env = originalEnv
+  })
+
+  describe('acquireKey', () => {
+    it('should return error when no keys are configured', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      process.env.EXA_API_KEY_COUNT = undefined
+      process.env.EXA_API_KEY_1 = undefined
+      process.env.EXA_API_KEY_2 = undefined
+      process.env.EXA_API_KEY_3 = undefined
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.error).toContain('No hosted keys configured')
+    })
+
+    it('should rate limit billing actor when wait exceeds the queue cap', async () => {
+      // resetAt past the 5-minute cap forces the wait loop to bail immediately.
+      const rateLimitedResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(rateLimitedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-123'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.retryAfterMs).toBeDefined()
+      expect(result.error).toContain('Rate limit exceeded')
+    })
+
+    it('should wait for capacity then succeed when bucket refills within the cap', async () => {
+      // First call: bucket empty, refills in 100ms (well under cap).
+      // Second call: bucket has capacity, consumed.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 100),
+      }
+      const allowed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValueOnce(blocked).mockResolvedValueOnce(allowed)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-wait'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(2)
+    })
+
+    it('should allow billing actor within their rate limit', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-123'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.billingActorRateLimited).toBeUndefined()
+      expect(result.key).toBe('test-key-1')
+    })
+
+    it('should distribute requests across keys round-robin style', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      const r1 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+      const r2 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-2'
+      )
+      const r3 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-3'
+      )
+      const r4 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-4'
+      )
+
+      expect(r1.keyIndex).toBe(0)
+      expect(r2.keyIndex).toBe(1)
+      expect(r3.keyIndex).toBe(2)
+      expect(r4.keyIndex).toBe(0) // Wraps back
+    })
+
+    it('should handle partial key availability', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      process.env.EXA_API_KEY_2 = undefined
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(result.envVarName).toBe('EXA_API_KEY_1')
+
+      const r2 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-2'
+      )
+      expect(r2.keyIndex).toBe(2) // Skips missing key 1
+      expect(r2.envVarName).toBe('EXA_API_KEY_3')
+    })
+  })
+
+  describe('FIFO queue ordering', () => {
+    const allowed: ConsumeResult = {
+      allowed: true,
+      tokensRemaining: 9,
+      resetAt: new Date(Date.now() + 60000),
+    }
+
+    it('enqueues every call onto the per-workspace+provider queue', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.enqueue).toHaveBeenCalledWith(
+        testProvider,
+        'workspace-1',
+        expect.any(String)
+      )
+    })
+
+    it('always dequeues at the end of a successful acquisition', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.dequeue).toHaveBeenCalledWith(
+        testProvider,
+        'workspace-1',
+        expect.any(String)
+      )
+    })
+
+    it('always dequeues even when the call fails (no keys configured)', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      process.env.EXA_API_KEY_COUNT = '0'
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.dequeue).toHaveBeenCalled()
+    })
+
+    it('waits at the head of the queue before consuming from the bucket', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      // First two checkHead calls say we're waiting; third says we're up.
+      mockQueue.checkHead
+        .mockResolvedValueOnce('waiting')
+        .mockResolvedValueOnce('waiting')
+        .mockResolvedValueOnce('head')
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(mockQueue.checkHead).toHaveBeenCalledTimes(3)
+      // Bucket is only consumed once we reach the head.
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('refreshes the heartbeat while waiting at the head of the queue', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      // We need the wait loop to iterate long enough for HEARTBEAT_REFRESH_INTERVAL_MS
+      // to elapse. Use fake timers so we don't actually sleep.
+      vi.useFakeTimers()
+      try {
+        // Queue says we're waiting forever — except after some time we're at head.
+        mockQueue.checkHead.mockImplementation(async () => {
+          // Advance past the heartbeat interval each time we poll, then say we're up.
+          vi.advanceTimersByTime(15_000)
+          return mockQueue.checkHead.mock.calls.length >= 2 ? 'head' : 'waiting'
+        })
+
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1'
+        )
+        // Drain pending timers so the sleep() resolves.
+        await vi.runAllTimersAsync()
+        await promise
+
+        expect(mockQueue.refreshHeartbeat).toHaveBeenCalled()
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('returns 429 when the queue wait exceeds the cap', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      mockQueue.checkHead.mockResolvedValue('waiting')
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1'
+        )
+        // Burn past the 5-minute cap.
+        await vi.advanceTimersByTimeAsync(6 * 60 * 1000)
+        const result = await promise
+
+        expect(result.success).toBe(false)
+        expect(result.billingActorRateLimited).toBe(true)
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('treats "missing" status as proceed (queue evicted, fall through to bucket race)', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      mockQueue.checkHead.mockResolvedValueOnce('missing')
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+    })
+  })
+
+  describe('execution-budget-bounded waits', () => {
+    it('bails immediately when the execution signal is already aborted', async () => {
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 100),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1',
+        AbortSignal.abort()
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      // Aborted budget => give up on the first bucket check rather than looping.
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('stops waiting promptly when the signal aborts mid-sleep', async () => {
+      // Bucket reports a long refill, so the wait sleeps up to the heartbeat cap (10s).
+      // Aborting mid-sleep must wake the wait within a tick, not after the full interval.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 10_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      const controller = new AbortController()
+      const start = Date.now()
+      const promise = rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1',
+        controller.signal
+      )
+      // Let the first bucket check run and the sleep begin, then abort.
+      await sleep(20)
+      controller.abort()
+      const result = await promise
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      // Resolved well before the 10s capped sleep would otherwise have elapsed.
+      expect(Date.now() - start).toBeLessThan(2000)
+    })
+
+    it('keeps waiting past the no-signal fallback cap while the signal is live', async () => {
+      // A live (non-aborted) signal means the run still has budget, so the wait must not
+      // 429 at the 5-minute MAX_QUEUE_WAIT_MS fallback. The bucket frees up after ~7 min.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 10_000),
+      }
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1',
+          new AbortController().signal
+        )
+        // Burn well past the 5-minute fallback cap — without a signal this would have 429'd.
+        await vi.advanceTimersByTimeAsync(7 * 60 * 1000)
+        mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+        await vi.advanceTimersByTimeAsync(HEARTBEAT_REFRESH_INTERVAL_MS)
+        const result = await promise
+
+        expect(result.success).toBe(true)
+        expect(result.key).toBe('test-key-1')
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('refreshes the heartbeat during a long low-RPM bucket wait', async () => {
+      // Provider with a long refill (retryAfterMs >> heartbeat TTL). The sleep must be
+      // capped so the heartbeat is renewed and the head is not reaped mid-wait.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1',
+          new AbortController().signal
+        )
+        await vi.advanceTimersByTimeAsync(3 * HEARTBEAT_REFRESH_INTERVAL_MS)
+        mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+        await vi.advanceTimersByTimeAsync(HEARTBEAT_REFRESH_INTERVAL_MS)
+        await promise
+
+        expect(mockQueue.refreshHeartbeat).toHaveBeenCalled()
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+  })
+
+  describe('acquireKey with custom rate limit', () => {
+    const customRateLimit: CustomRateLimit = {
+      mode: 'custom',
+      requestsPerMinute: 5,
+      dimensions: [
+        {
+          name: 'tokens',
+          limitPerMinute: 1000,
+          extractUsage: (_params, response) => (response.tokenCount as number) ?? 0,
+        },
+      ],
+    }
+
+    it('should enforce requestsPerMinute for custom mode when wait exceeds the cap', async () => {
+      const rateLimitedResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(rateLimitedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('Rate limit exceeded')
+    })
+
+    it('should allow request when actor request limit and dimensions have budget', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const budgetAvailable: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(budgetAvailable)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledTimes(1)
+    })
+
+    it('should block request when a dimension wait exceeds the cap', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const depleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(depleted)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('tokens')
+    })
+
+    it('should wait for dimension capacity then succeed when budget refills', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const depleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 100),
+      }
+      const refilled: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValueOnce(depleted).mockResolvedValueOnce(refilled)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-dim-wait'
+      )
+
+      expect(result.success).toBe(true)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledTimes(2)
+    })
+
+    it('should pre-check all dimensions and block on first depleted one', async () => {
+      const multiDimensionConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+          {
+            name: 'search_units',
+            limitPerMinute: 50,
+            extractUsage: (_p, r) => (r.searchUnits as number) ?? 0,
+          },
+        ],
+      }
+
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const tokensBudget: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      const searchUnitsDepleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 100,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.getTokenStatus
+        .mockResolvedValueOnce(tokensBudget)
+        .mockResolvedValueOnce(searchUnitsDepleted)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        multiDimensionConfig,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('search_units')
+    })
+  })
+
+  describe('reportUsage', () => {
+    const customConfig: CustomRateLimit = {
+      mode: 'custom',
+      requestsPerMinute: 5,
+      dimensions: [
+        {
+          name: 'tokens',
+          limitPerMinute: 1000,
+          extractUsage: (_params, response) => (response.tokenCount as number) ?? 0,
+        },
+      ],
+    }
+
+    it('should consume actual tokens from dimension bucket after execution', async () => {
+      const consumeResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 850,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(consumeResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 150 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].name).toBe('tokens')
+      expect(result.dimensions[0].consumed).toBe(150)
+      expect(result.dimensions[0].allowed).toBe(true)
+      expect(result.dimensions[0].tokensRemaining).toBe(850)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        'hosted:exa:actor:workspace-1:tokens',
+        150,
+        expect.objectContaining({ maxTokens: 2000, refillRate: 1000 })
+      )
+    })
+
+    it('should handle overdrawn bucket gracefully (optimistic concurrency)', async () => {
+      const overdrawnResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(overdrawnResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 500 }
+      )
+
+      expect(result.dimensions[0].allowed).toBe(false)
+      expect(result.dimensions[0].consumed).toBe(500)
+    })
+
+    it('should skip consumption when extractUsage returns 0', async () => {
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 0 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].consumed).toBe(0)
+      expect(mockAdapter.consumeTokens).not.toHaveBeenCalled()
+    })
+
+    it('should handle multiple dimensions independently', async () => {
+      const multiConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+          {
+            name: 'search_units',
+            limitPerMinute: 50,
+            extractUsage: (_p, r) => (r.searchUnits as number) ?? 0,
+          },
+        ],
+      }
+
+      const tokensConsumed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 800,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      const searchConsumed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 47,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens
+        .mockResolvedValueOnce(tokensConsumed)
+        .mockResolvedValueOnce(searchConsumed)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        multiConfig,
+        {},
+        { tokenCount: 200, searchUnits: 3 }
+      )
+
+      expect(result.dimensions).toHaveLength(2)
+      expect(result.dimensions[0]).toEqual({
+        name: 'tokens',
+        consumed: 200,
+        allowed: true,
+        tokensRemaining: 800,
+      })
+      expect(result.dimensions[1]).toEqual({
+        name: 'search_units',
+        consumed: 3,
+        allowed: true,
+        tokensRemaining: 47,
+      })
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(2)
+    })
+
+    it('should continue with remaining dimensions if extractUsage throws', async () => {
+      const throwingConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'broken',
+            limitPerMinute: 100,
+            extractUsage: () => {
+              throw new Error('extraction failed')
+            },
+          },
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+        ],
+      }
+
+      const consumeResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 900,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(consumeResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        throwingConfig,
+        {},
+        { tokenCount: 100 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].name).toBe('tokens')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('should handle storage errors gracefully', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('db connection lost'))
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 100 }
+      )
+
+      expect(result.dimensions).toHaveLength(0)
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/hosted-key/hosted-key-rate-limiter.ts

@@ -0,0 +1,695 @@
+import { createLogger } from '@sim/logger'
+import { sleep } from '@sim/utils/helpers'
+import { generateShortId } from '@sim/utils/id'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import {
+  createStorageAdapter,
+  type RateLimitStorageAdapter,
+  type TokenBucketConfig,
+} from '@/lib/core/rate-limiter/storage'
+import { PlatformEvents } from '@/lib/core/telemetry'
+import { getHostedKeyQueue, HEARTBEAT_REFRESH_INTERVAL_MS, type HostedKeyQueue } from './queue'
+import {
+  type AcquireKeyResult,
+  type CustomRateLimit,
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  type HostedKeyRateLimitConfig,
+  type ReportUsageResult,
+  toTokenBucketConfig,
+} from './types'
+
+const logger = createLogger('HostedKeyRateLimiter')
+
+/**
+ * Fallback ceiling on how long a hosted-key acquisition waits for the per-workspace
+ * bucket to refill when no execution `AbortSignal` is available to bound the wait (e.g.
+ * a caller without a wired execution deadline, or Redis no-op mode). When a signal IS
+ * provided, the wait is instead bounded by the surrounding execution budget — the signal
+ * fires when the run hits its plan timeout or is cancelled — and this constant no longer
+ * applies (see {@link ABSOLUTE_MAX_QUEUE_WAIT_MS} for the backstop in that case).
+ */
+const MAX_QUEUE_WAIT_MS = 5 * 60 * 1000
+
+/**
+ * Hard safety ceiling applied even when an execution `AbortSignal` is present, in case the
+ * signal never fires. Matches the longest possible execution budget (enterprise async) so
+ * it never truncates a legitimately long-running background run before its own deadline.
+ */
+const ABSOLUTE_MAX_QUEUE_WAIT_MS = getMaxExecutionTimeout()
+
+/**
+ * Floor on per-iteration sleep when the bucket reports `retryAfterMs <= 0`,
+ * which can happen due to clock skew or sub-millisecond resets. Prevents a
+ * tight retry loop hammering the storage adapter.
+ */
+const MIN_QUEUE_RETRY_DELAY_MS = 50
+
+/**
+ * Poll interval while waiting to reach the head of the FIFO queue. 200ms balances
+ * acquisition latency (worst-case wait for advancement is one poll period) against
+ * Redis load — at this cadence, N waiters generate N×5 EVAL/sec, which is fine for
+ * the typical low-tens contention. Revisit if telemetry shows hot Redis under load.
+ */
+const QUEUE_HEAD_POLL_MS = 200
+
+/**
+ * Sleep for `ms`, resolving early if `signal` aborts. Cleans up its own timer and listener
+ * so neither leaks. Callers don't need to distinguish an early (aborted) return from a normal
+ * one — the surrounding wait loop re-checks its budget immediately after and bails when the
+ * signal has fired. Falls back to a plain sleep when no signal is provided.
+ */
+function interruptibleSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (!signal) return sleep(ms)
+  if (signal.aborted) return Promise.resolve()
+  return new Promise<void>((resolve) => {
+    const onAbort = () => {
+      clearTimeout(timer)
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }
+    const timer = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }, ms)
+    signal.addEventListener('abort', onAbort, { once: true })
+    // Catch an abort that fired between the guard above and addEventListener.
+    if (signal.aborted) onAbort()
+  })
+}
+
+/**
+ * Resolves env var names for a numbered key prefix using a `{PREFIX}_COUNT` env var.
+ * E.g. with `EXA_API_KEY_COUNT=5`, returns `['EXA_API_KEY_1', ..., 'EXA_API_KEY_5']`.
+ */
+function resolveEnvKeys(prefix: string): string[] {
+  const count = Number.parseInt(process.env[`${prefix}_COUNT`] || '0', 10)
+  const names: string[] = []
+  for (let i = 1; i <= count; i++) {
+    names.push(`${prefix}_${i}`)
+  }
+  return names
+}
+
+/** Dimension name for per-billing-actor request rate limiting */
+const ACTOR_REQUESTS_DIMENSION = 'actor_requests'
+
+/**
+ * Information about an available hosted key
+ */
+interface AvailableKey {
+  key: string
+  keyIndex: number
+  envVarName: string
+}
+
+/**
+ * Mutable heartbeat bookkeeping shared across every wait phase of a single `acquireKey`
+ * call. Carrying one `lastHeartbeatAt` across the queue-head wait and the bucket waits
+ * ensures the heartbeat-refresh cadence reflects the *actual* last write to Redis rather
+ * than resetting per phase, which could otherwise let the ticket heartbeat lapse and get
+ * reaped mid-wait (breaking FIFO ordering).
+ */
+interface WaitState {
+  /** Epoch ms of the last heartbeat write to Redis. */
+  lastHeartbeatAt: number
+}
+
+/**
+ * HostedKeyRateLimiter provides:
+ * 1. Per-billing-actor rate limiting (enforced - blocks actors who exceed their limit)
+ * 2. Round-robin key selection (distributes requests evenly across keys)
+ * 3. Post-execution dimension usage tracking for custom rate limits
+ *
+ * The billing actor is typically a workspace ID, meaning rate limits are shared
+ * across all users within the same workspace.
+ */
+export class HostedKeyRateLimiter {
+  private storage: RateLimitStorageAdapter
+  private queue: HostedKeyQueue
+  /** Round-robin counter per provider for even key distribution */
+  private roundRobinCounters = new Map<string, number>()
+
+  constructor(storage?: RateLimitStorageAdapter, queue?: HostedKeyQueue) {
+    this.storage = storage ?? createStorageAdapter()
+    this.queue = queue ?? getHostedKeyQueue()
+  }
+
+  private buildActorStorageKey(provider: string, billingActorId: string): string {
+    return `hosted:${provider}:actor:${billingActorId}:${ACTOR_REQUESTS_DIMENSION}`
+  }
+
+  private buildDimensionStorageKey(
+    provider: string,
+    billingActorId: string,
+    dimensionName: string
+  ): string {
+    return `hosted:${provider}:actor:${billingActorId}:${dimensionName}`
+  }
+
+  private getAvailableKeys(envKeys: string[]): AvailableKey[] {
+    const keys: AvailableKey[] = []
+    for (let i = 0; i < envKeys.length; i++) {
+      const envVarName = envKeys[i]
+      const key = process.env[envVarName]
+      if (key) {
+        keys.push({ key, keyIndex: i, envVarName })
+      }
+    }
+    return keys
+  }
+
+  /**
+   * Build a token bucket config for the per-billing-actor request rate limit.
+   * Works for both `per_request` and `custom` modes since both define `requestsPerMinute`.
+   */
+  private getActorRateLimitConfig(config: HostedKeyRateLimitConfig): TokenBucketConfig | null {
+    if (!config.requestsPerMinute) return null
+    return toTokenBucketConfig(
+      config.requestsPerMinute,
+      config.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+      DEFAULT_WINDOW_MS
+    )
+  }
+
+  /**
+   * Check and consume billing actor request rate limit. Returns null if allowed, or retry info if blocked.
+   */
+  private async checkActorRateLimit(
+    provider: string,
+    billingActorId: string,
+    config: HostedKeyRateLimitConfig
+  ): Promise<{ rateLimited: true; retryAfterMs: number } | null> {
+    const bucketConfig = this.getActorRateLimitConfig(config)
+    if (!bucketConfig) return null
+
+    const storageKey = this.buildActorStorageKey(provider, billingActorId)
+
+    try {
+      const result = await this.storage.consumeTokens(storageKey, 1, bucketConfig)
+      if (!result.allowed) {
+        const retryAfterMs = Math.max(0, result.resetAt.getTime() - Date.now())
+        logger.info(`Billing actor ${billingActorId} rate limited for ${provider}`, {
+          provider,
+          billingActorId,
+          retryAfterMs,
+          tokensRemaining: result.tokensRemaining,
+        })
+        return { rateLimited: true, retryAfterMs }
+      }
+      return null
+    } catch (error) {
+      logger.error(`Error checking billing actor rate limit for ${provider}`, {
+        error,
+        billingActorId,
+      })
+      return null
+    }
+  }
+
+  /**
+   * Pre-check that the billing actor has available budget in all custom dimensions.
+   * Does NOT consume tokens -- just verifies the actor isn't already depleted.
+   * Returns retry info for the most restrictive exhausted dimension, or null if all pass.
+   */
+  private async preCheckDimensions(
+    provider: string,
+    billingActorId: string,
+    config: CustomRateLimit
+  ): Promise<{ rateLimited: true; retryAfterMs: number; dimension: string } | null> {
+    for (const dimension of config.dimensions) {
+      const storageKey = this.buildDimensionStorageKey(provider, billingActorId, dimension.name)
+      const bucketConfig = toTokenBucketConfig(
+        dimension.limitPerMinute,
+        dimension.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+        DEFAULT_WINDOW_MS
+      )
+
+      try {
+        const status = await this.storage.getTokenStatus(storageKey, bucketConfig)
+        if (status.tokensAvailable < 1) {
+          const retryAfterMs = Math.max(0, status.nextRefillAt.getTime() - Date.now())
+          logger.info(
+            `Billing actor ${billingActorId} exhausted dimension ${dimension.name} for ${provider}`,
+            {
+              provider,
+              billingActorId,
+              dimension: dimension.name,
+              tokensAvailable: status.tokensAvailable,
+              retryAfterMs,
+            }
+          )
+          return { rateLimited: true, retryAfterMs, dimension: dimension.name }
+        }
+      } catch (error) {
+        logger.error(`Error pre-checking dimension ${dimension.name} for ${provider}`, {
+          error,
+          billingActorId,
+        })
+      }
+    }
+    return null
+  }
+
+  /**
+   * Acquire an available key via round-robin selection.
+   *
+   * For both modes:
+   *   1. Per-billing-actor request rate limiting (enforced): the call enqueues itself
+   *      onto a per-workspace+provider FIFO queue. Only the head of the queue attempts
+   *      to consume from the token bucket, guaranteeing strict ordering across callers
+   *      within a workspace. Different workspaces have independent queues and don't
+   *      block each other.
+   *   2. Round-robin key selection: cycles through available keys for even distribution
+   *
+   * For `custom` mode additionally:
+   *   3. Pre-checks dimension budgets: head waits on dimension refill the same way it
+   *      waits on actor request capacity.
+   *
+   * The wait is bounded by the surrounding execution budget: when `signal` is provided it
+   * fires at the run's plan timeout (or on cancellation), so a queued call uses its full
+   * available budget rather than a flat cap. When no signal is available the wait falls
+   * back to `MAX_QUEUE_WAIT_MS`. On exhaustion the call returns today's 429 result. The
+   * ticket is removed from the queue on exit regardless of success or failure.
+   *
+   * @param envKeyPrefix - Env var prefix (e.g. 'EXA_API_KEY'). Keys resolved via `{prefix}_COUNT`.
+   * @param billingActorId - The billing actor (typically workspace ID) to rate limit against
+   * @param signal - Optional execution `AbortSignal`; bounds the queue wait to the run's budget.
+   */
+  async acquireKey(
+    provider: string,
+    envKeyPrefix: string,
+    config: HostedKeyRateLimitConfig,
+    billingActorId: string,
+    signal?: AbortSignal
+  ): Promise<AcquireKeyResult> {
+    const ticketId = generateShortId()
+    const startedAt = Date.now()
+    const waitState: WaitState = { lastHeartbeatAt: startedAt }
+    const enqueueResult = await this.queue.enqueue(provider, billingActorId, ticketId)
+
+    try {
+      // Wait for our turn at the head of the queue (no-op when Redis unavailable).
+      const headStatus = await this.waitForQueueHead(
+        provider,
+        billingActorId,
+        ticketId,
+        startedAt,
+        waitState,
+        signal
+      )
+      if (headStatus.timedOut) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'queue_position',
+        })
+        return {
+          success: false,
+          billingActorRateLimited: true,
+          retryAfterMs: MAX_QUEUE_WAIT_MS,
+          error: `Rate limit exceeded — request waited too long in the queue. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+        }
+      }
+
+      let dimensionWaited = false
+
+      if (config.requestsPerMinute) {
+        const rateLimitResult = await this.waitForActorCapacity(
+          provider,
+          billingActorId,
+          ticketId,
+          config,
+          startedAt,
+          waitState,
+          signal
+        )
+        if (rateLimitResult.rateLimited) {
+          return {
+            success: false,
+            billingActorRateLimited: true,
+            retryAfterMs: rateLimitResult.retryAfterMs,
+            error: `Rate limit exceeded. Please wait ${Math.ceil(rateLimitResult.retryAfterMs / 1000)} seconds. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+          }
+        }
+      }
+
+      if (config.mode === 'custom' && config.dimensions.length > 0) {
+        const dimensionResult = await this.waitForDimensionCapacity(
+          provider,
+          billingActorId,
+          ticketId,
+          config,
+          startedAt,
+          waitState,
+          signal
+        )
+        if (dimensionResult.rateLimited) {
+          return {
+            success: false,
+            billingActorRateLimited: true,
+            retryAfterMs: dimensionResult.retryAfterMs,
+            error: `Rate limit exceeded for ${dimensionResult.dimension}. Please wait ${Math.ceil(dimensionResult.retryAfterMs / 1000)} seconds. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+          }
+        }
+        dimensionWaited = dimensionResult.waited
+      }
+
+      const totalWaitedMs = Date.now() - startedAt
+      if (enqueueResult.enabled && (enqueueResult.position > 0 || totalWaitedMs > 100)) {
+        // Attribute the wait to its dominant cause: queue depth takes precedence (it's
+        // reported alongside queuePosition), otherwise the bucket phase that actually slept.
+        const reason: 'queue_position' | 'actor_requests' | 'dimension' =
+          enqueueResult.position > 0
+            ? 'queue_position'
+            : dimensionWaited
+              ? 'dimension'
+              : 'actor_requests'
+        PlatformEvents.hostedKeyQueueWaited({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: totalWaitedMs,
+          attempts: 1,
+          reason,
+          queuePosition: enqueueResult.position,
+        })
+      }
+
+      const envKeys = resolveEnvKeys(envKeyPrefix)
+      const availableKeys = this.getAvailableKeys(envKeys)
+
+      if (availableKeys.length === 0) {
+        logger.warn(`No hosted keys configured for provider ${provider}`)
+        return {
+          success: false,
+          error: `No hosted keys configured for ${provider}`,
+        }
+      }
+
+      const counter = this.roundRobinCounters.get(provider) ?? 0
+      const selected = availableKeys[counter % availableKeys.length]
+      this.roundRobinCounters.set(provider, counter + 1)
+
+      logger.debug(`Selected hosted key for ${provider}`, {
+        provider,
+        keyIndex: selected.keyIndex,
+        envVarName: selected.envVarName,
+      })
+
+      return {
+        success: true,
+        key: selected.key,
+        keyIndex: selected.keyIndex,
+        envVarName: selected.envVarName,
+      }
+    } finally {
+      // Always remove our ticket so the next caller can advance, regardless of whether
+      // we succeeded, hit the cap, or threw. Best-effort; safe to call multiple times.
+      await this.queue.dequeue(provider, billingActorId, ticketId)
+    }
+  }
+
+  /**
+   * Remaining time budget for waiting, in milliseconds. When an execution `AbortSignal` is
+   * present it governs the wait: the budget is exhausted the moment the signal aborts (the
+   * run hit its plan timeout or was cancelled), with {@link ABSOLUTE_MAX_QUEUE_WAIT_MS} as a
+   * backstop should the signal never fire. Without a signal we fall back to the flat
+   * {@link MAX_QUEUE_WAIT_MS} ceiling.
+   */
+  private remainingWaitBudgetMs(startedAt: number, signal?: AbortSignal): number {
+    if (signal?.aborted) return 0
+    const ceiling = signal ? ABSOLUTE_MAX_QUEUE_WAIT_MS : MAX_QUEUE_WAIT_MS
+    return ceiling - (Date.now() - startedAt)
+  }
+
+  /** Refresh the ticket heartbeat if the refresh interval has elapsed since the last write. */
+  private async maybeRefreshHeartbeat(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    waitState: WaitState
+  ): Promise<void> {
+    if (Date.now() - waitState.lastHeartbeatAt >= HEARTBEAT_REFRESH_INTERVAL_MS) {
+      await this.queue.refreshHeartbeat(provider, billingActorId, ticketId)
+      waitState.lastHeartbeatAt = Date.now()
+    }
+  }
+
+  /**
+   * Sleep before the next bucket re-check, refreshing the heartbeat first if due. The sleep
+   * is capped at {@link HEARTBEAT_REFRESH_INTERVAL_MS} so that no single wait can outlive the
+   * heartbeat TTL — even when the bucket reports a long `retryAfterMs` (e.g. low-RPM
+   * providers). Without this cap a multi-second sleep could let the heartbeat lapse, the
+   * head get reaped as dead, and a second caller advance and race us for the bucket. The
+   * sleep also resolves early if `signal` aborts, so a cancelled/timed-out run stops waiting
+   * promptly rather than overshooting by up to the cap.
+   */
+  private async heartbeatAwareSleep(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    desiredMs: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<void> {
+    await this.maybeRefreshHeartbeat(provider, billingActorId, ticketId, waitState)
+    const sleepMs = Math.min(
+      Math.max(MIN_QUEUE_RETRY_DELAY_MS, desiredMs),
+      HEARTBEAT_REFRESH_INTERVAL_MS
+    )
+    await interruptibleSleep(sleepMs, signal)
+  }
+
+  /**
+   * Block until our ticket reaches the head of the queue. Refreshes the heartbeat on a
+   * regular cadence so we don't get reaped as dead. Returns `timedOut: true` once the wait
+   * budget is exhausted before reaching the head (see {@link remainingWaitBudgetMs}).
+   *
+   * No-op when Redis is unavailable (queue.enqueue returns enabled=false and checkHead
+   * always returns 'head').
+   */
+  private async waitForQueueHead(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<{ timedOut: boolean }> {
+    while (true) {
+      const status = await this.queue.checkHead(provider, billingActorId, ticketId)
+      if (status === 'head') return { timedOut: false }
+
+      // 'missing' shouldn't normally happen — the queue list TTL (10min) outlives a typical
+      // wait — but if it does (e.g. Redis flushed mid-wait), treat as "you're up" so the
+      // caller proceeds to the bucket race rather than hanging forever.
+      if (status === 'missing') return { timedOut: false }
+
+      if (this.remainingWaitBudgetMs(startedAt, signal) <= 0) {
+        return { timedOut: true }
+      }
+
+      await this.maybeRefreshHeartbeat(provider, billingActorId, ticketId, waitState)
+      await interruptibleSleep(QUEUE_HEAD_POLL_MS, signal)
+    }
+  }
+
+  /**
+   * Wait for actor request-rate capacity. Called once we're at the head of the FIFO
+   * queue, so other callers can't race us for the next token — they're blocked behind us
+   * at queue level. Re-checks the bucket until the wait budget is exhausted (accounting for
+   * time already spent waiting in the queue). `waited` reports whether the loop ever slept.
+   */
+  private async waitForActorCapacity(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    config: HostedKeyRateLimitConfig,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<
+    { rateLimited: false; waited: boolean } | { rateLimited: true; retryAfterMs: number }
+  > {
+    let waited = false
+
+    while (true) {
+      const result = await this.checkActorRateLimit(provider, billingActorId, config)
+      if (!result) return { rateLimited: false, waited }
+
+      const remaining = this.remainingWaitBudgetMs(startedAt, signal)
+      if (remaining <= 0 || result.retryAfterMs > remaining) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'actor_requests',
+        })
+        return { rateLimited: true, retryAfterMs: result.retryAfterMs }
+      }
+
+      waited = true
+      await this.heartbeatAwareSleep(
+        provider,
+        billingActorId,
+        ticketId,
+        result.retryAfterMs,
+        waitState,
+        signal
+      )
+    }
+  }
+
+  /**
+   * Wait for custom-mode dimension capacity. `preCheckDimensions` is read-only — it does
+   * not consume — so re-running it after a sleep is safe and does not double-charge.
+   * Post-execution `reportUsage` performs the actual consumption. `waited` reports whether
+   * the loop ever slept.
+   */
+  private async waitForDimensionCapacity(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    config: CustomRateLimit,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<
+    | { rateLimited: false; waited: boolean }
+    | { rateLimited: true; retryAfterMs: number; dimension: string }
+  > {
+    let waited = false
+
+    while (true) {
+      const result = await this.preCheckDimensions(provider, billingActorId, config)
+      if (!result) return { rateLimited: false, waited }
+
+      const remaining = this.remainingWaitBudgetMs(startedAt, signal)
+      if (remaining <= 0 || result.retryAfterMs > remaining) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'dimension',
+          dimension: result.dimension,
+        })
+        return {
+          rateLimited: true,
+          retryAfterMs: result.retryAfterMs,
+          dimension: result.dimension,
+        }
+      }
+
+      waited = true
+      await this.heartbeatAwareSleep(
+        provider,
+        billingActorId,
+        ticketId,
+        result.retryAfterMs,
+        waitState,
+        signal
+      )
+    }
+  }
+
+  /**
+   * Report actual usage after successful tool execution (custom mode only).
+   * Calls `extractUsage` on each dimension and consumes the actual token count.
+   * This is the "post-execution" phase of the optimistic two-phase approach.
+   */
+  async reportUsage(
+    provider: string,
+    billingActorId: string,
+    config: CustomRateLimit,
+    params: Record<string, unknown>,
+    response: Record<string, unknown>
+  ): Promise<ReportUsageResult> {
+    const results: ReportUsageResult['dimensions'] = []
+
+    for (const dimension of config.dimensions) {
+      let usage: number
+      try {
+        usage = dimension.extractUsage(params, response)
+      } catch (error) {
+        logger.error(`Failed to extract usage for dimension ${dimension.name}`, {
+          provider,
+          billingActorId,
+          error,
+        })
+        continue
+      }
+
+      if (usage <= 0) {
+        results.push({
+          name: dimension.name,
+          consumed: 0,
+          allowed: true,
+          tokensRemaining: 0,
+        })
+        continue
+      }
+
+      const storageKey = this.buildDimensionStorageKey(provider, billingActorId, dimension.name)
+      const bucketConfig = toTokenBucketConfig(
+        dimension.limitPerMinute,
+        dimension.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+        DEFAULT_WINDOW_MS
+      )
+
+      try {
+        const consumeResult = await this.storage.consumeTokens(storageKey, usage, bucketConfig)
+
+        results.push({
+          name: dimension.name,
+          consumed: usage,
+          allowed: consumeResult.allowed,
+          tokensRemaining: consumeResult.tokensRemaining,
+        })
+
+        if (!consumeResult.allowed) {
+          logger.warn(
+            `Dimension ${dimension.name} overdrawn for ${provider} (optimistic concurrency)`,
+            { provider, billingActorId, usage, tokensRemaining: consumeResult.tokensRemaining }
+          )
+        }
+
+        logger.debug(`Consumed ${usage} from dimension ${dimension.name} for ${provider}`, {
+          provider,
+          billingActorId,
+          usage,
+          allowed: consumeResult.allowed,
+          tokensRemaining: consumeResult.tokensRemaining,
+        })
+      } catch (error) {
+        logger.error(`Failed to consume tokens for dimension ${dimension.name}`, {
+          provider,
+          billingActorId,
+          usage,
+          error,
+        })
+      }
+    }
+
+    return { dimensions: results }
+  }
+}
+
+let cachedInstance: HostedKeyRateLimiter | null = null
+
+/**
+ * Get the singleton HostedKeyRateLimiter instance
+ */
+export function getHostedKeyRateLimiter(): HostedKeyRateLimiter {
+  if (!cachedInstance) {
+    cachedInstance = new HostedKeyRateLimiter()
+  }
+  return cachedInstance
+}
+
+/**
+ * Reset the cached rate limiter (for testing)
+ */
+export function resetHostedKeyRateLimiter(): void {
+  cachedInstance = null
+}

apps/sim/lib/core/rate-limiter/hosted-key/index.ts

@@ -0,0 +1,11 @@
+export {
+  getHostedKeyRateLimiter,
+  HostedKeyRateLimiter,
+  resetHostedKeyRateLimiter,
+} from './hosted-key-rate-limiter'
+export {
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  type HostedKeyRateLimitConfig,
+  toTokenBucketConfig,
+} from './types'

apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts

@@ -0,0 +1,234 @@
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import { HostedKeyQueue } from './queue'
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+interface MockPipeline {
+  rpush: Mock
+  expire: Mock
+  set: Mock
+  lrem: Mock
+  del: Mock
+  exec: Mock
+}
+
+interface MockRedis {
+  multi: Mock
+  set: Mock
+  eval: Mock
+  pipeline: MockPipeline
+}
+
+function createFakeRedis(): MockRedis {
+  const pipeline: MockPipeline = {
+    rpush: vi.fn(),
+    expire: vi.fn(),
+    set: vi.fn(),
+    lrem: vi.fn(),
+    del: vi.fn(),
+    exec: vi.fn(),
+  }
+  // Pipeline methods return the pipeline for chaining.
+  pipeline.rpush.mockReturnValue(pipeline)
+  pipeline.expire.mockReturnValue(pipeline)
+  pipeline.set.mockReturnValue(pipeline)
+  pipeline.lrem.mockReturnValue(pipeline)
+  pipeline.del.mockReturnValue(pipeline)
+
+  return {
+    multi: vi.fn(() => pipeline),
+    set: vi.fn(),
+    eval: vi.fn(),
+    pipeline,
+  }
+}
+
+const provider = 'exa'
+const workspaceId = 'workspace-1'
+const ticketId = 'ticket-1'
+
+describe('HostedKeyQueue', () => {
+  let queue: HostedKeyQueue
+  let mockRedis: MockRedis
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockRedis = createFakeRedis()
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue(mockRedis)
+    queue = new HostedKeyQueue()
+  })
+
+  describe('enqueue', () => {
+    it('returns position 0 when first in line', async () => {
+      // RPUSH returns new list length; first push -> 1.
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 1],
+        [null, 1],
+        [null, 'OK'],
+      ])
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result).toEqual({ position: 0, enabled: true })
+      expect(mockRedis.pipeline.rpush).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        ticketId
+      )
+      expect(mockRedis.pipeline.set).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1',
+        '1',
+        'EX',
+        expect.any(Number)
+      )
+    })
+
+    it('returns higher position when others are ahead', async () => {
+      // Length 5 after push -> position 4.
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 5],
+        [null, 1],
+        [null, 'OK'],
+      ])
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result.position).toBe(4)
+    })
+
+    it('falls back to enabled=false when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result).toEqual({ position: 0, enabled: false })
+    })
+
+    it('falls back to enabled=false on Redis error', async () => {
+      mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost'))
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result.enabled).toBe(false)
+    })
+  })
+
+  describe('checkHead', () => {
+    it('returns "head" when our ticket is at the head', async () => {
+      mockRedis.eval.mockResolvedValueOnce('head')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+
+    it('returns "waiting" when someone else is the head', async () => {
+      mockRedis.eval.mockResolvedValueOnce('waiting')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('waiting')
+    })
+
+    it('returns "missing" when our ticket is not in the queue', async () => {
+      mockRedis.eval.mockResolvedValueOnce('missing')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('missing')
+    })
+
+    it('passes queue list key, heartbeat prefix, and ticketId to the Lua script', async () => {
+      mockRedis.eval.mockResolvedValueOnce('head')
+
+      await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(mockRedis.eval).toHaveBeenCalledWith(
+        expect.stringContaining('lindex'),
+        1,
+        'hosted-queue:exa:workspace-1',
+        'hosted-queue-tkt:exa:workspace-1:',
+        ticketId
+      )
+    })
+
+    it('fails open to "head" on Redis error so callers do not hang', async () => {
+      mockRedis.eval.mockRejectedValueOnce(new Error('boom'))
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+
+    it('returns "head" no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+  })
+
+  describe('refreshHeartbeat', () => {
+    it('writes the heartbeat key with TTL and re-extends the queue list TTL', async () => {
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 'OK'],
+        [null, 1],
+      ])
+
+      await queue.refreshHeartbeat(provider, workspaceId, ticketId)
+
+      expect(mockRedis.pipeline.set).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1',
+        '1',
+        'EX',
+        expect.any(Number)
+      )
+      expect(mockRedis.pipeline.expire).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        expect.any(Number)
+      )
+      expect(mockRedis.pipeline.exec).toHaveBeenCalledTimes(1)
+    })
+
+    it('is a no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      await expect(queue.refreshHeartbeat(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+      expect(mockRedis.multi).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('dequeue', () => {
+    it('removes the ticket from the list and deletes the heartbeat', async () => {
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 1],
+        [null, 1],
+      ])
+
+      await queue.dequeue(provider, workspaceId, ticketId)
+
+      expect(mockRedis.pipeline.lrem).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        1,
+        ticketId
+      )
+      expect(mockRedis.pipeline.del).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1'
+      )
+    })
+
+    it('is a no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      await expect(queue.dequeue(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+      expect(mockRedis.multi).not.toHaveBeenCalled()
+    })
+
+    it('swallows errors so callers do not throw on cleanup', async () => {
+      mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost'))
+
+      await expect(queue.dequeue(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/hosted-key/queue.ts

@@ -0,0 +1,209 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { getRedisClient } from '@/lib/core/config/redis'
+
+const logger = createLogger('HostedKeyQueue')
+
+/**
+ * Per-ticket heartbeat TTL. Refreshed by the head while it's actively waiting
+ * on the bucket. If the holder crashes, the heartbeat key expires, and the next
+ * caller sees the head as dead and removes it (lazy cleanup).
+ */
+const TICKET_HEARTBEAT_TTL_SECONDS = 30
+
+/** How often the head should refresh its heartbeat while waiting. */
+export const HEARTBEAT_REFRESH_INTERVAL_MS = 10_000
+
+/**
+ * TTL on the queue list itself. Set on enqueue and re-extended by the head's heartbeat,
+ * so a long-waiting head can't let the list expire out from under the waiters behind it.
+ * Prevents abandoned queues from sticking around forever in Redis.
+ */
+const QUEUE_LIST_TTL_SECONDS = 600
+
+const queueListKey = (provider: string, billingActorId: string): string =>
+  `hosted-queue:${provider}:${billingActorId}`
+
+const heartbeatKey = (provider: string, billingActorId: string, ticketId: string): string =>
+  `hosted-queue-tkt:${provider}:${billingActorId}:${ticketId}`
+
+/**
+ * Atomically reap any dead head, then return our ticket's status. Combines what
+ * would otherwise be 3 round-trips (reap, LINDEX, LPOS) into one EVAL — meaningful
+ * because callers poll this every ~200ms while waiting in the queue.
+ *
+ * `KEYS[1]` = queue list key. `ARGV[1]` = heartbeat key prefix. `ARGV[2]` = our ticketId.
+ *
+ * Reaping is bounded: at most one dead head is removed per call. If multiple dead
+ * tickets pile up at the head, subsequent polls will clean them one by one. This
+ * keeps the script O(1) rather than O(N) and is sufficient because queue depth
+ * is bounded by concurrent callers per workspace (typically tens).
+ *
+ * Returns one of: "head", "waiting", "missing".
+ */
+const CHECK_HEAD_SCRIPT = `
+local head = redis.call("lindex", KEYS[1], 0)
+if head and redis.call("exists", ARGV[1] .. head) == 0 then
+  redis.call("lrem", KEYS[1], 1, head)
+  head = redis.call("lindex", KEYS[1], 0)
+end
+if not head then
+  return "missing"
+end
+if head == ARGV[2] then
+  return "head"
+end
+if redis.call("lpos", KEYS[1], ARGV[2]) == false then
+  return "missing"
+end
+return "waiting"
+`
+
+export interface EnqueueResult {
+  /** Position at the moment of enqueue (0 = head, you go next). */
+  position: number
+  /** Whether Redis was available — false means we're in no-op mode. */
+  enabled: boolean
+}
+
+/**
+ * Per-workspace+provider FIFO queue for hosted-key acquisitions.
+ *
+ * Callers `enqueue` to claim a position, then `waitForHead` until they're at
+ * the head, then attempt to consume from the token bucket. On success or cap
+ * exceeded, they `dequeue` to make room for the next caller.
+ *
+ * No-op when Redis is unavailable: every method returns "you're the head /
+ * empty / etc." so the rate limiter falls back to plain bucket racing.
+ */
+export class HostedKeyQueue {
+  /**
+   * Push a ticket onto the tail of the queue and write a heartbeat. Returns the
+   * position at enqueue time (0 = head, ready to proceed).
+   */
+  async enqueue(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<EnqueueResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      return { position: 0, enabled: false }
+    }
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+
+    try {
+      const pipeline = redis.multi()
+      pipeline.rpush(listKey, ticketId)
+      pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS)
+      pipeline.set(hbKey, '1', 'EX', TICKET_HEARTBEAT_TTL_SECONDS)
+      const results = await pipeline.exec()
+      // results[0] is the rpush response: [err, length]
+      const length = results?.[0] && typeof results[0][1] === 'number' ? results[0][1] : 1
+      // Position is length - 1 (just-pushed at the tail).
+      return { position: length - 1, enabled: true }
+    } catch (error) {
+      logger.warn(`Queue enqueue failed for ${listKey}`, { error: toError(error).message })
+      return { position: 0, enabled: false }
+    }
+  }
+
+  /**
+   * Check whether `ticketId` is currently at the head of the queue. If the head
+   * is a different ticket but its heartbeat has expired (caller crashed), reap
+   * it and re-check on the next poll.
+   *
+   * Returns:
+   *  - "head": you're at the head, proceed to consume from the bucket
+   *  - "waiting": someone else is the head and they're alive
+   *  - "missing": your ticket isn't in the queue at all (e.g. queue list TTL
+   *    expired); caller should re-enqueue or treat as enabled=false
+   */
+  async checkHead(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<'head' | 'waiting' | 'missing'> {
+    const redis = getRedisClient()
+    if (!redis) {
+      return 'head'
+    }
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbPrefix = `hosted-queue-tkt:${provider}:${billingActorId}:`
+
+    try {
+      const result = (await redis.eval(CHECK_HEAD_SCRIPT, 1, listKey, hbPrefix, ticketId)) as
+        | 'head'
+        | 'waiting'
+        | 'missing'
+      return result
+    } catch (error) {
+      logger.warn(`Queue checkHead failed for ${listKey}`, { error: toError(error).message })
+      // Fail-open: treat as head so the caller proceeds rather than hanging.
+      return 'head'
+    }
+  }
+
+  /**
+   * Refresh the ticket's heartbeat so the head isn't reaped as dead while waiting on the
+   * bucket. Also re-extends the queue list TTL so a wait outliving {@link QUEUE_LIST_TTL_SECONDS}
+   * doesn't let the list expire and collapse FIFO ordering.
+   */
+  async refreshHeartbeat(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) return
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+    try {
+      const pipeline = redis.multi()
+      pipeline.set(hbKey, '1', 'EX', TICKET_HEARTBEAT_TTL_SECONDS)
+      pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS)
+      await pipeline.exec()
+    } catch (error) {
+      logger.warn(`Queue heartbeat refresh failed for ${hbKey}`, {
+        error: toError(error).message,
+      })
+    }
+  }
+
+  /**
+   * Remove a ticket from the queue and its heartbeat key. Best-effort; safe to
+   * call multiple times. LREM count=1 removes at most one matching entry.
+   */
+  async dequeue(provider: string, billingActorId: string, ticketId: string): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) return
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+    try {
+      const pipeline = redis.multi()
+      pipeline.lrem(listKey, 1, ticketId)
+      pipeline.del(hbKey)
+      await pipeline.exec()
+    } catch (error) {
+      logger.warn(`Queue dequeue failed for ${listKey}`, { error: toError(error).message })
+    }
+  }
+}
+
+let cachedQueue: HostedKeyQueue | null = null
+
+export function getHostedKeyQueue(): HostedKeyQueue {
+  if (!cachedQueue) {
+    cachedQueue = new HostedKeyQueue()
+  }
+  return cachedQueue
+}
+
+export function resetHostedKeyQueue(): void {
+  cachedQueue = null
+}

apps/sim/lib/core/rate-limiter/hosted-key/types.ts

@@ -0,0 +1,108 @@
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter/storage'
+
+export type HostedKeyRateLimitMode = 'per_request' | 'custom'
+
+/**
+ * Simple per-request rate limit configuration.
+ * Enforces per-billing-actor rate limiting and distributes requests across keys.
+ */
+export interface PerRequestRateLimit {
+  mode: 'per_request'
+  /** Maximum requests per minute per billing actor (enforced - blocks if exceeded) */
+  requestsPerMinute: number
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+}
+
+/**
+ * Custom rate limit with multiple dimensions (e.g., tokens, search units).
+ * Allows tracking different usage metrics independently.
+ */
+export interface CustomRateLimit {
+  mode: 'custom'
+  /** Maximum requests per minute per billing actor (enforced - blocks if exceeded) */
+  requestsPerMinute: number
+  /** Multiple dimensions to track */
+  dimensions: RateLimitDimension[]
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+}
+
+/**
+ * A single dimension for custom rate limiting.
+ * Each dimension has its own token bucket.
+ */
+interface RateLimitDimension {
+  /** Dimension name (e.g., 'tokens', 'search_units') - used in storage key */
+  name: string
+  /** Limit per minute for this dimension */
+  limitPerMinute: number
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+  /**
+   * Extract usage amount from request params and response.
+   * Called after successful execution to consume the actual usage.
+   */
+  extractUsage: (params: Record<string, unknown>, response: Record<string, unknown>) => number
+}
+
+/** Union of all hosted key rate limit configuration types */
+export type HostedKeyRateLimitConfig = PerRequestRateLimit | CustomRateLimit
+
+/**
+ * Result from acquiring a key from the hosted key rate limiter
+ */
+export interface AcquireKeyResult {
+  /** Whether a key was successfully acquired */
+  success: boolean
+  /** The API key value (if success=true) */
+  key?: string
+  /** Index of the key in the envKeys array */
+  keyIndex?: number
+  /** Environment variable name of the selected key */
+  envVarName?: string
+  /** Error message if no key available */
+  error?: string
+  /** Whether the billing actor was rate limited (exceeded their limit) */
+  billingActorRateLimited?: boolean
+  /** Milliseconds until the billing actor's rate limit resets (if billingActorRateLimited=true) */
+  retryAfterMs?: number
+}
+
+/**
+ * Result from reporting post-execution usage for custom dimensions
+ */
+export interface ReportUsageResult {
+  /** Per-dimension consumption results */
+  dimensions: {
+    name: string
+    consumed: number
+    allowed: boolean
+    tokensRemaining: number
+  }[]
+}
+
+/**
+ * Convert rate limit config to token bucket config for a dimension
+ */
+export function toTokenBucketConfig(
+  limitPerMinute: number,
+  burstMultiplier = 2,
+  windowMs = 60000
+): TokenBucketConfig {
+  return {
+    maxTokens: limitPerMinute * burstMultiplier,
+    refillRate: limitPerMinute,
+    refillIntervalMs: windowMs,
+  }
+}
+
+/**
+ * Default rate limit window in milliseconds (1 minute)
+ */
+export const DEFAULT_WINDOW_MS = 60000
+
+/**
+ * Default burst multiplier
+ */
+export const DEFAULT_BURST_MULTIPLIER = 2

apps/sim/lib/core/rate-limiter/index.ts

@@ -0,0 +1,20 @@
+export {
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  getHostedKeyRateLimiter,
+  type HostedKeyRateLimitConfig,
+  HostedKeyRateLimiter,
+  resetHostedKeyRateLimiter,
+  toTokenBucketConfig,
+} from './hosted-key'
+export { RateLimiter } from './rate-limiter'
+export {
+  DEFAULT_PUBLIC_IP_ROUTE_LIMIT,
+  DEFAULT_USER_ROUTE_LIMIT,
+  enforceIpRateLimit,
+  enforceUserOrIpRateLimit,
+  enforceUserRateLimit,
+} from './route-helpers'
+export type { TokenBucketConfig } from './storage'
+export type { SubscriptionPlan } from './types'
+export { getRateLimit, RATE_LIMITS, RateLimitError } from './types'

apps/sim/lib/core/rate-limiter/rate-limiter.test.ts

@@ -0,0 +1,459 @@
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import { RateLimiter } from './rate-limiter'
+import type { ConsumeResult, RateLimitStorageAdapter, TokenStatus } from './storage'
+import { MANUAL_EXECUTION_LIMIT, RATE_LIMITS, RateLimitError } from './types'
+
+vi.mock('@/lib/core/config/env-flags', () => ({ isBillingEnabled: true }))
+
+interface MockAdapter {
+  consumeTokens: Mock
+  getTokenStatus: Mock
+  resetBucket: Mock
+}
+
+const createMockAdapter = (): MockAdapter => ({
+  consumeTokens: vi.fn(),
+  getTokenStatus: vi.fn(),
+  resetBucket: vi.fn(),
+})
+
+describe('RateLimiter', () => {
+  const testUserId = 'test-user-123'
+  const freeSubscription = { plan: 'free', referenceId: testUserId }
+  let mockAdapter: MockAdapter
+  let rateLimiter: RateLimiter
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockAdapter = createMockAdapter()
+    rateLimiter = new RateLimiter(mockAdapter as RateLimitStorageAdapter)
+  })
+
+  describe('checkRateLimitWithSubscription', () => {
+    it('should allow unlimited requests for manual trigger type', async () => {
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'manual',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(result.resetAt).toBeInstanceOf(Date)
+      expect(mockAdapter.consumeTokens).not.toHaveBeenCalled()
+    })
+
+    it('should consume tokens for API requests', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(mockResult.tokensRemaining)
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+
+    it('should use async bucket for async requests', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.async.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, freeSubscription, 'api', true)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:async`,
+        1,
+        RATE_LIMITS.free.async
+      )
+    })
+
+    it('should use api-endpoint bucket for api-endpoint trigger', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.apiEndpoint.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api-endpoint',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:api-endpoint`,
+        1,
+        RATE_LIMITS.free.apiEndpoint
+      )
+    })
+
+    it('should deny requests when rate limit exceeded', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60000),
+        retryAfterMs: 30000,
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(false)
+      expect(result.remaining).toBe(0)
+      expect(result.retryAfterMs).toBe(30000)
+    })
+
+    it('should use organization key for team subscriptions', async () => {
+      const orgId = 'org-123'
+      const teamSubscription = { plan: 'team', referenceId: orgId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.team.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, teamSubscription, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${orgId}:sync`,
+        1,
+        RATE_LIMITS.team.sync
+      )
+    })
+
+    it('should use user key when team subscription referenceId matches userId', async () => {
+      const directTeamSubscription = { plan: 'team', referenceId: testUserId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.team.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        directTeamSubscription,
+        'api',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.team.sync
+      )
+    })
+
+    it('should allow on storage error (fail open)', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(1)
+    })
+
+    it('should work for all non-manual trigger types', async () => {
+      const triggerTypes = ['api', 'webhook', 'schedule', 'chat'] as const
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      for (const triggerType of triggerTypes) {
+        await rateLimiter.checkRateLimitWithSubscription(
+          testUserId,
+          freeSubscription,
+          triggerType,
+          false
+        )
+        expect(mockAdapter.consumeTokens).toHaveBeenCalled()
+        mockAdapter.consumeTokens.mockClear()
+      }
+    })
+  })
+
+  describe('getRateLimitStatusWithSubscription', () => {
+    it('should return unlimited status for manual trigger type', async () => {
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'manual',
+        false
+      )
+
+      expect(status.requestsPerMinute).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(status.maxBurst).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(status.remaining).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(mockAdapter.getTokenStatus).not.toHaveBeenCalled()
+    })
+
+    it('should return status from storage for API requests', async () => {
+      const mockStatus: TokenStatus = {
+        tokensAvailable: 15,
+        maxTokens: RATE_LIMITS.free.sync.maxTokens,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(mockStatus)
+
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(status.remaining).toBe(15)
+      expect(status.requestsPerMinute).toBe(RATE_LIMITS.free.sync.refillRate)
+      expect(status.maxBurst).toBe(RATE_LIMITS.free.sync.maxTokens)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        RATE_LIMITS.free.sync
+      )
+    })
+  })
+
+  describe('resetRateLimit', () => {
+    it('should reset all bucket types for a user', async () => {
+      mockAdapter.resetBucket.mockResolvedValue(undefined)
+
+      await rateLimiter.resetRateLimit(testUserId)
+
+      expect(mockAdapter.resetBucket).toHaveBeenCalledTimes(3)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:sync`)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:async`)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:api-endpoint`)
+    })
+
+    it('should throw error if reset fails', async () => {
+      mockAdapter.resetBucket.mockRejectedValue(new Error('Reset failed'))
+
+      await expect(rateLimiter.resetRateLimit(testUserId)).rejects.toThrow('Reset failed')
+    })
+  })
+
+  describe('checkRateLimitDirect', () => {
+    const config = { maxTokens: 3, refillRate: 1, refillIntervalMs: 60_000 }
+
+    it('should reflect the storage decision when it succeeds', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 2,
+        resetAt: new Date(),
+      })
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config)
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(2)
+    })
+
+    it('should fail open on storage error by default', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config)
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(1)
+    })
+
+    it('should fail closed on storage error when failClosed is set', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config, {
+        failClosed: true,
+      })
+
+      expect(result.allowed).toBe(false)
+      expect(result.remaining).toBe(0)
+    })
+  })
+
+  describe('subscription plan handling', () => {
+    it('should use pro plan limits', async () => {
+      const proSubscription = { plan: 'pro', referenceId: testUserId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.pro.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, proSubscription, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.pro.sync
+      )
+    })
+
+    it('should use enterprise plan limits', async () => {
+      const enterpriseSubscription = { plan: 'enterprise', referenceId: 'org-enterprise' }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.enterprise.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        enterpriseSubscription,
+        'api',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `org-enterprise:sync`,
+        1,
+        RATE_LIMITS.enterprise.sync
+      )
+    })
+
+    it('should fall back to free plan when subscription is null', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, null, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+  })
+
+  describe('schedule trigger type', () => {
+    it('should use sync bucket for schedule trigger', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'schedule',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+
+    it('should use async bucket for schedule trigger with isAsync true', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'schedule',
+        true
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:async`,
+        1,
+        RATE_LIMITS.free.async
+      )
+    })
+  })
+
+  describe('getRateLimitStatusWithSubscription error handling', () => {
+    it('should return default config on storage error', async () => {
+      mockAdapter.getTokenStatus.mockRejectedValue(new Error('Storage error'))
+
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(status.remaining).toBe(0)
+      expect(status.requestsPerMinute).toBe(RATE_LIMITS.free.sync.refillRate)
+      expect(status.maxBurst).toBe(RATE_LIMITS.free.sync.maxTokens)
+    })
+  })
+})
+
+describe('RateLimitError', () => {
+  it('should create error with default status code 429', () => {
+    const error = new RateLimitError('Rate limit exceeded')
+
+    expect(error.message).toBe('Rate limit exceeded')
+    expect(error.statusCode).toBe(429)
+    expect(error.name).toBe('RateLimitError')
+  })
+
+  it('should create error with custom status code', () => {
+    const error = new RateLimitError('Custom error', 503)
+
+    expect(error.message).toBe('Custom error')
+    expect(error.statusCode).toBe(503)
+  })
+
+  it('should be instanceof Error', () => {
+    const error = new RateLimitError('Test')
+
+    expect(error instanceof Error).toBe(true)
+    expect(error instanceof RateLimitError).toBe(true)
+  })
+
+  it('should have proper stack trace', () => {
+    const error = new RateLimitError('Test error')
+
+    expect(error.stack).toBeDefined()
+    expect(error.stack).toContain('RateLimitError')
+  })
+})

apps/sim/lib/core/rate-limiter/rate-limiter.ts

@@ -0,0 +1,220 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils'
+import { createStorageAdapter, type RateLimitStorageAdapter } from './storage'
+import {
+  getRateLimit,
+  MANUAL_EXECUTION_LIMIT,
+  RATE_LIMIT_WINDOW_MS,
+  type RateLimitCounterType,
+  type SubscriptionPlan,
+  type TriggerType,
+} from './types'
+
+const logger = createLogger('RateLimiter')
+
+interface SubscriptionInfo {
+  plan: string
+  referenceId: string
+}
+
+export interface RateLimitResult {
+  allowed: boolean
+  remaining: number
+  resetAt: Date
+  retryAfterMs?: number
+}
+
+export interface RateLimitStatus {
+  requestsPerMinute: number
+  maxBurst: number
+  remaining: number
+  resetAt: Date
+}
+
+export class RateLimiter {
+  private storage: RateLimitStorageAdapter
+
+  constructor(storage?: RateLimitStorageAdapter) {
+    this.storage = storage ?? createStorageAdapter()
+  }
+
+  private getRateLimitKey(userId: string, subscription: SubscriptionInfo | null): string {
+    if (!subscription) return userId
+
+    if (isOrgScopedSubscription(subscription, userId)) {
+      return subscription.referenceId
+    }
+
+    return userId
+  }
+
+  private getCounterType(triggerType: TriggerType, isAsync: boolean): RateLimitCounterType {
+    if (triggerType === 'api-endpoint') return 'api-endpoint'
+    return isAsync ? 'async' : 'sync'
+  }
+
+  private buildStorageKey(rateLimitKey: string, counterType: RateLimitCounterType): string {
+    return `${rateLimitKey}:${counterType}`
+  }
+
+  private createUnlimitedResult(): RateLimitResult {
+    return {
+      allowed: true,
+      remaining: MANUAL_EXECUTION_LIMIT,
+      resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+    }
+  }
+
+  async checkRateLimitWithSubscription(
+    userId: string,
+    subscription: SubscriptionInfo | null,
+    triggerType: TriggerType = 'manual',
+    isAsync = false
+  ): Promise<RateLimitResult> {
+    try {
+      if (triggerType === 'manual') {
+        return this.createUnlimitedResult()
+      }
+
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const rateLimitKey = this.getRateLimitKey(userId, subscription)
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+      const storageKey = this.buildStorageKey(rateLimitKey, counterType)
+
+      const result = await this.storage.consumeTokens(storageKey, 1, config)
+
+      if (!result.allowed) {
+        logger.info('Rate limit exceeded', {
+          rateLimitKey,
+          counterType,
+          plan,
+          tokensRemaining: result.tokensRemaining,
+        })
+      }
+
+      return {
+        allowed: result.allowed,
+        remaining: result.tokensRemaining,
+        resetAt: result.resetAt,
+        retryAfterMs: result.retryAfterMs,
+      }
+    } catch (error) {
+      logger.error('Rate limit storage error - failing open (allowing request)', {
+        error: toError(error).message,
+        userId,
+        triggerType,
+        isAsync,
+      })
+      return {
+        allowed: true,
+        remaining: 1,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  async getRateLimitStatusWithSubscription(
+    userId: string,
+    subscription: SubscriptionInfo | null,
+    triggerType: TriggerType = 'manual',
+    isAsync = false
+  ): Promise<RateLimitStatus> {
+    try {
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+
+      if (triggerType === 'manual') {
+        return {
+          requestsPerMinute: MANUAL_EXECUTION_LIMIT,
+          maxBurst: MANUAL_EXECUTION_LIMIT,
+          remaining: MANUAL_EXECUTION_LIMIT,
+          resetAt: new Date(Date.now() + config.refillIntervalMs),
+        }
+      }
+
+      const rateLimitKey = this.getRateLimitKey(userId, subscription)
+      const storageKey = this.buildStorageKey(rateLimitKey, counterType)
+
+      const status = await this.storage.getTokenStatus(storageKey, config)
+
+      return {
+        requestsPerMinute: config.refillRate,
+        maxBurst: config.maxTokens,
+        remaining: Math.floor(status.tokensAvailable),
+        resetAt: status.nextRefillAt,
+      }
+    } catch (error) {
+      logger.error('Error getting rate limit status - returning default config', {
+        error: toError(error).message,
+        userId,
+        triggerType,
+        isAsync,
+      })
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+      return {
+        requestsPerMinute: config.refillRate,
+        maxBurst: config.maxTokens,
+        remaining: 0,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  /**
+   * Consume one token from a bucket. On storage failure the default is to fail
+   * open (allow the request) so a limiter outage never takes down normal traffic.
+   * Pass `failClosed: true` for security-critical checks — e.g. a captcha
+   * backstop — where an unenforceable limit must reject rather than admit.
+   */
+  async checkRateLimitDirect(
+    storageKey: string,
+    config: { maxTokens: number; refillRate: number; refillIntervalMs: number },
+    options?: { failClosed?: boolean }
+  ): Promise<RateLimitResult> {
+    try {
+      const result = await this.storage.consumeTokens(storageKey, 1, config)
+      if (!result.allowed) {
+        logger.info('Rate limit exceeded', { storageKey, tokensRemaining: result.tokensRemaining })
+      }
+      return {
+        allowed: result.allowed,
+        remaining: result.tokensRemaining,
+        resetAt: result.resetAt,
+        retryAfterMs: result.retryAfterMs,
+      }
+    } catch (error) {
+      const failClosed = options?.failClosed === true
+      logger.error(
+        `Rate limit storage error - failing ${failClosed ? 'closed (rejecting request)' : 'open (allowing request)'}`,
+        {
+          error: toError(error).message,
+          storageKey,
+        }
+      )
+      return {
+        allowed: !failClosed,
+        remaining: failClosed ? 0 : 1,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  async resetRateLimit(rateLimitKey: string): Promise<void> {
+    try {
+      await Promise.all([
+        this.storage.resetBucket(`${rateLimitKey}:sync`),
+        this.storage.resetBucket(`${rateLimitKey}:async`),
+        this.storage.resetBucket(`${rateLimitKey}:api-endpoint`),
+      ])
+      logger.info(`Reset rate limit for ${rateLimitKey}`)
+    } catch (error) {
+      logger.error('Error resetting rate limit:', error)
+      throw error
+    }
+  }
+}

apps/sim/lib/core/rate-limiter/route-helpers.test.ts

@@ -0,0 +1,192 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest, requestUtilsMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+
+const { mockAdapter } = vi.hoisted(() => ({
+  mockAdapter: {
+    consumeTokens: vi.fn(),
+    getTokenStatus: vi.fn(),
+    resetBucket: vi.fn(),
+  },
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage', async () => {
+  const actual = await vi.importActual<typeof import('@/lib/core/rate-limiter/storage')>(
+    '@/lib/core/rate-limiter/storage'
+  )
+  return {
+    ...actual,
+    createStorageAdapter: () => mockAdapter,
+  }
+})
+
+function passThroughClientIp() {
+  requestUtilsMockFns.mockGetClientIp.mockImplementation(
+    (req: { headers: { get(name: string): string | null } }) =>
+      req.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ||
+      req.headers.get('x-real-ip')?.trim() ||
+      'unknown'
+  )
+}
+
+import { enforceIpRateLimit, enforceUserOrIpRateLimit, enforceUserRateLimit } from './route-helpers'
+
+const consume = mockAdapter.consumeTokens as Mock
+
+describe('route-helpers rate limiting', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('enforceUserRateLimit', () => {
+    it('returns null when the bucket has tokens left', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(Date.now() + 60_000),
+      })
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).toBeNull()
+      expect(consume).toHaveBeenCalledWith(
+        'route:test-bucket:user:user-1',
+        1,
+        expect.objectContaining({ maxTokens: 60, refillRate: 30 })
+      )
+    })
+
+    it('returns a 429 with Retry-After when the bucket is empty', async () => {
+      const resetAt = new Date(Date.now() + 30_000)
+      consume.mockResolvedValueOnce({
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt,
+        retryAfterMs: 30_000,
+      })
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).not.toBeNull()
+      expect(result?.status).toBe(429)
+      expect(result?.headers.get('Retry-After')).toBe('30')
+      expect(result?.headers.get('X-RateLimit-Reset')).toBe(resetAt.toISOString())
+
+      const body = await result?.json()
+      expect(body?.error).toBe('Rate limit exceeded')
+    })
+
+    it('keys buckets per user so different users do not share state', async () => {
+      consume.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+
+      await enforceUserRateLimit('shared-bucket', 'user-a')
+      await enforceUserRateLimit('shared-bucket', 'user-b')
+
+      const keys = consume.mock.calls.map((call) => call[0])
+      expect(keys).toEqual(['route:shared-bucket:user:user-a', 'route:shared-bucket:user:user-b'])
+    })
+
+    it('fails open when the storage layer throws', async () => {
+      consume.mockRejectedValueOnce(new Error('redis down'))
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).toBeNull()
+    })
+  })
+
+  describe('enforceIpRateLimit', () => {
+    beforeEach(() => {
+      passThroughClientIp()
+    })
+
+    it('uses the X-Forwarded-For client IP in the bucket key', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, {
+        'x-forwarded-for': '203.0.113.7, 10.0.0.1',
+      })
+
+      await enforceIpRateLimit('public-bucket', request)
+
+      expect(consume).toHaveBeenCalledWith(
+        'route:public-bucket:ip:203.0.113.7',
+        1,
+        expect.any(Object)
+      )
+    })
+
+    it('folds spoofed `X-Forwarded-For: unknown` into a single shared bucket', async () => {
+      consume.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(),
+      })
+
+      const reqA = createMockRequest('POST', undefined, { 'x-forwarded-for': 'unknown' })
+      const reqB = createMockRequest('POST', undefined, { 'x-forwarded-for': 'unknown' })
+      await enforceIpRateLimit('otp', reqA)
+      await enforceIpRateLimit('otp', reqB)
+
+      const keys = consume.mock.calls.map((call) => call[0])
+      expect(keys).toEqual(['route:otp:ip:unknown', 'route:otp:ip:unknown'])
+    })
+
+    it('returns a 429 with Retry-After on rate limit', async () => {
+      const resetAt = new Date(Date.now() + 60_000)
+      consume.mockResolvedValueOnce({
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt,
+        retryAfterMs: 60_000,
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      const result = await enforceIpRateLimit('public-bucket', request)
+
+      expect(result?.status).toBe(429)
+      expect(result?.headers.get('Retry-After')).toBe('60')
+    })
+  })
+
+  describe('enforceUserOrIpRateLimit', () => {
+    beforeEach(() => {
+      passThroughClientIp()
+    })
+
+    it('keys per-user when userId is present', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      await enforceUserOrIpRateLimit('a2a-test', 'user-1', request)
+
+      expect(consume).toHaveBeenCalledWith('route:a2a-test:user:user-1', 1, expect.any(Object))
+    })
+
+    it('falls back to per-IP when userId is undefined', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      await enforceUserOrIpRateLimit('a2a-test', undefined, request)
+
+      expect(consume).toHaveBeenCalledWith('route:a2a-test:ip:203.0.113.7', 1, expect.any(Object))
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/route-helpers.ts

@@ -0,0 +1,89 @@
+import { createLogger } from '@sim/logger'
+import { type NextRequest, NextResponse } from 'next/server'
+import { RateLimiter } from '@/lib/core/rate-limiter/rate-limiter'
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter/storage'
+import { getClientIp } from '@/lib/core/utils/request'
+
+const logger = createLogger('RouteRateLimit')
+const rateLimiter = new RateLimiter()
+
+/** Default per-user bucket for authenticated tool routes (60 burst, 30/min). */
+export const DEFAULT_USER_ROUTE_LIMIT: TokenBucketConfig = {
+  maxTokens: 60,
+  refillRate: 30,
+  refillIntervalMs: 60_000,
+}
+
+/** Default per-IP bucket for unauthenticated public endpoints (10 burst, 5/min). */
+export const DEFAULT_PUBLIC_IP_ROUTE_LIMIT: TokenBucketConfig = {
+  maxTokens: 10,
+  refillRate: 5,
+  refillIntervalMs: 60_000,
+}
+
+function buildRateLimitResponse(resetAt: Date): NextResponse {
+  const retryAfterSec = Math.max(1, Math.ceil((resetAt.getTime() - Date.now()) / 1000))
+  return NextResponse.json(
+    {
+      error: 'Rate limit exceeded',
+      retryAfter: resetAt.getTime(),
+    },
+    {
+      status: 429,
+      headers: {
+        'Retry-After': String(retryAfterSec),
+        'X-RateLimit-Reset': resetAt.toISOString(),
+      },
+    }
+  )
+}
+
+/**
+ * Apply a per-user token bucket to an authenticated route.
+ * Returns a `NextResponse` on 429, otherwise `null` so the caller can proceed.
+ */
+export async function enforceUserRateLimit(
+  bucketName: string,
+  userId: string,
+  config: TokenBucketConfig = DEFAULT_USER_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  const key = `route:${bucketName}:user:${userId}`
+  const { allowed, resetAt } = await rateLimiter.checkRateLimitDirect(key, config)
+  if (allowed) return null
+  logger.warn('User rate limit exceeded', { bucket: bucketName, userId })
+  return buildRateLimitResponse(resetAt)
+}
+
+/**
+ * Apply a per-IP token bucket to an unauthenticated route. The `unknown` IP
+ * fallback shares one global bucket per route so it cannot be amplified by
+ * `X-Forwarded-For: unknown` spoofing.
+ */
+export async function enforceIpRateLimit(
+  bucketName: string,
+  request: NextRequest,
+  config: TokenBucketConfig = DEFAULT_PUBLIC_IP_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  const ip = getClientIp(request)
+  const key = `route:${bucketName}:ip:${ip}`
+  const { allowed, resetAt } = await rateLimiter.checkRateLimitDirect(key, config)
+  if (allowed) return null
+  logger.warn('IP rate limit exceeded', { bucket: bucketName, ip })
+  return buildRateLimitResponse(resetAt)
+}
+
+/**
+ * Apply a per-user limit when a userId is present, else fall back to per-IP.
+ * Use for routes whose auth path may legitimately resolve without a userId
+ * (e.g. internal JWT calls with `requireWorkflowId: false`) so missing-userId
+ * traffic is still throttled per-IP rather than sharing one global bucket.
+ */
+export async function enforceUserOrIpRateLimit(
+  bucketName: string,
+  userId: string | undefined,
+  request: NextRequest,
+  config: TokenBucketConfig = DEFAULT_USER_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  if (userId) return enforceUserRateLimit(bucketName, userId, config)
+  return enforceIpRateLimit(bucketName, request, config)
+}

apps/sim/lib/core/rate-limiter/storage/adapter.ts

@@ -0,0 +1,25 @@
+export interface TokenBucketConfig {
+  maxTokens: number
+  refillRate: number
+  refillIntervalMs: number
+}
+
+export interface ConsumeResult {
+  allowed: boolean
+  tokensRemaining: number
+  resetAt: Date
+  retryAfterMs?: number
+}
+
+export interface TokenStatus {
+  tokensAvailable: number
+  maxTokens: number
+  lastRefillAt: Date
+  nextRefillAt: Date
+}
+
+export interface RateLimitStorageAdapter {
+  consumeTokens(key: string, tokens: number, config: TokenBucketConfig): Promise<ConsumeResult>
+  getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus>
+  resetBucket(key: string): Promise<void>
+}

apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts

@@ -0,0 +1,136 @@
+import { db } from '@sim/db'
+import { rateLimitBucket } from '@sim/db/schema'
+import { eq, sql } from 'drizzle-orm'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+
+export class DbTokenBucket implements RateLimitStorageAdapter {
+  async consumeTokens(
+    key: string,
+    requestedTokens: number,
+    config: TokenBucketConfig
+  ): Promise<ConsumeResult> {
+    const now = new Date()
+    const nowMs = now.getTime()
+    const nowIso = now.toISOString()
+
+    const result = await db
+      .insert(rateLimitBucket)
+      .values({
+        key,
+        tokens: (config.maxTokens - requestedTokens).toString(),
+        lastRefillAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: rateLimitBucket.key,
+        set: {
+          tokens: sql`
+            CASE
+              WHEN (
+                LEAST(
+                  ${config.maxTokens}::numeric,
+                  ${rateLimitBucket.tokens}::numeric + (
+                    FLOOR(
+                      EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                      / ${config.refillIntervalMs}
+                    ) * ${config.refillRate}
+                  )::numeric
+                )
+              ) >= ${requestedTokens}::numeric
+              THEN LEAST(
+                ${config.maxTokens}::numeric,
+                ${rateLimitBucket.tokens}::numeric + (
+                  FLOOR(
+                    EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                    / ${config.refillIntervalMs}
+                  ) * ${config.refillRate}
+                )::numeric
+              ) - ${requestedTokens}::numeric
+              ELSE -1
+            END
+          `,
+          lastRefillAt: sql`
+            CASE
+              WHEN FLOOR(
+                EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                / ${config.refillIntervalMs}
+              ) > 0
+              THEN ${rateLimitBucket.lastRefillAt} + (
+                FLOOR(
+                  EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                  / ${config.refillIntervalMs}
+                ) * ${config.refillIntervalMs} * INTERVAL '1 millisecond'
+              )
+              ELSE ${rateLimitBucket.lastRefillAt}
+            END
+          `,
+          updatedAt: now,
+        },
+      })
+      .returning({
+        tokens: rateLimitBucket.tokens,
+        lastRefillAt: rateLimitBucket.lastRefillAt,
+      })
+
+    const record = result[0]
+    const tokens = Number.parseFloat(record.tokens)
+    const lastRefillMs = record.lastRefillAt.getTime()
+    const nextRefillAt = new Date(lastRefillMs + config.refillIntervalMs)
+
+    const allowed = tokens >= 0
+
+    return {
+      allowed,
+      tokensRemaining: Math.max(0, tokens),
+      resetAt: nextRefillAt,
+      retryAfterMs: allowed ? undefined : Math.max(0, nextRefillAt.getTime() - nowMs),
+    }
+  }
+
+  async getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus> {
+    const now = new Date()
+
+    const [record] = await db
+      .select({
+        tokens: rateLimitBucket.tokens,
+        lastRefillAt: rateLimitBucket.lastRefillAt,
+      })
+      .from(rateLimitBucket)
+      .where(eq(rateLimitBucket.key, key))
+      .limit(1)
+
+    if (!record) {
+      return {
+        tokensAvailable: config.maxTokens,
+        maxTokens: config.maxTokens,
+        lastRefillAt: now,
+        nextRefillAt: new Date(now.getTime() + config.refillIntervalMs),
+      }
+    }
+
+    const tokens = Number.parseFloat(record.tokens)
+    const elapsed = now.getTime() - record.lastRefillAt.getTime()
+    const intervalsElapsed = Math.floor(elapsed / config.refillIntervalMs)
+    const refillAmount = intervalsElapsed * config.refillRate
+    const tokensAvailable = Math.min(config.maxTokens, tokens + refillAmount)
+    const lastRefillAt = new Date(
+      record.lastRefillAt.getTime() + intervalsElapsed * config.refillIntervalMs
+    )
+
+    return {
+      tokensAvailable,
+      maxTokens: config.maxTokens,
+      lastRefillAt,
+      nextRefillAt: new Date(lastRefillAt.getTime() + config.refillIntervalMs),
+    }
+  }
+
+  async resetBucket(key: string): Promise<void> {
+    await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, key))
+  }
+}

apps/sim/lib/core/rate-limiter/storage/factory.test.ts

@@ -0,0 +1,109 @@
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockGetStorageMethod, reconnectCallbacks } = vi.hoisted(() => {
+  const callbacks: Array<() => void> = []
+  return {
+    mockGetStorageMethod: vi.fn(() => 'db'),
+    reconnectCallbacks: callbacks,
+  }
+})
+
+const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient
+const mockOnRedisReconnect = redisConfigMockFns.mockOnRedisReconnect
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+vi.mock('@/lib/core/storage', () => ({
+  getStorageMethod: mockGetStorageMethod,
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage/db-token-bucket', () => ({
+  DbTokenBucket: vi.fn().mockImplementation(
+    class {
+      type = 'db'
+    }
+  ),
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage/redis-token-bucket', () => ({
+  RedisTokenBucket: vi.fn().mockImplementation(
+    class {
+      type = 'redis'
+    }
+  ),
+}))
+
+import { createStorageAdapter, resetStorageAdapter } from '@/lib/core/rate-limiter/storage/factory'
+
+describe('rate limit storage factory', () => {
+  beforeEach(() => {
+    mockGetRedisClient.mockReset().mockReturnValue(null)
+    mockGetStorageMethod.mockReset().mockReturnValue('db')
+    mockOnRedisReconnect.mockImplementation((cb: () => void) => {
+      reconnectCallbacks.push(cb)
+    })
+    resetStorageAdapter()
+  })
+
+  it('should fall back to DbTokenBucket when Redis is configured but client unavailable', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue(null)
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'db' })
+  })
+
+  it('should use RedisTokenBucket when Redis client is available', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never)
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'redis' })
+  })
+
+  it('should use DbTokenBucket when storage method is db', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'db' })
+  })
+
+  it('should cache the adapter and return same instance', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter1 = createStorageAdapter()
+    const adapter2 = createStorageAdapter()
+    expect(adapter1).toBe(adapter2)
+  })
+
+  it('should register a reconnect listener that resets cached adapter', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter1 = createStorageAdapter()
+
+    /** onRedisReconnect is called once (guarded by reconnectListenerRegistered flag). */
+    expect(reconnectCallbacks.length).toBeGreaterThan(0)
+    const latestCallback = reconnectCallbacks[reconnectCallbacks.length - 1]
+    latestCallback()
+
+    const adapter2 = createStorageAdapter()
+    expect(adapter2).not.toBe(adapter1)
+  })
+
+  it('should re-evaluate storage on next call after reconnect resets cache', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue(null)
+
+    const adapter1 = createStorageAdapter()
+    expect(adapter1).toEqual({ type: 'db' })
+
+    const latestCallback = reconnectCallbacks[reconnectCallbacks.length - 1]
+    latestCallback()
+
+    mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never)
+
+    const adapter2 = createStorageAdapter()
+    expect(adapter2).toEqual({ type: 'redis' })
+  })
+})

apps/sim/lib/core/rate-limiter/storage/factory.ts

@@ -0,0 +1,64 @@
+import { createLogger } from '@sim/logger'
+import { getRedisClient, onRedisReconnect } from '@/lib/core/config/redis'
+import { getStorageMethod, type StorageMethod } from '@/lib/core/storage'
+import type { RateLimitStorageAdapter } from './adapter'
+import { DbTokenBucket } from './db-token-bucket'
+import { RedisTokenBucket } from './redis-token-bucket'
+
+const logger = createLogger('RateLimitStorage')
+
+type FactoryGlobal = typeof globalThis & {
+  _rlCachedAdapter?: RateLimitStorageAdapter | null
+  _rlReconnectListenerRegistered?: boolean
+}
+
+const g = globalThis as FactoryGlobal
+if (!('_rlCachedAdapter' in g)) {
+  g._rlCachedAdapter = null
+  g._rlReconnectListenerRegistered = false
+}
+
+export function createStorageAdapter(): RateLimitStorageAdapter {
+  if (g._rlCachedAdapter) {
+    return g._rlCachedAdapter
+  }
+
+  if (!g._rlReconnectListenerRegistered) {
+    onRedisReconnect(() => {
+      g._rlCachedAdapter = null
+    })
+    g._rlReconnectListenerRegistered = true
+  }
+
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) {
+      logger.warn(
+        'Redis configured but client unavailable - falling back to PostgreSQL for rate limiting'
+      )
+      g._rlCachedAdapter = new DbTokenBucket()
+    } else {
+      logger.info('Rate limiting: Using Redis')
+      g._rlCachedAdapter = new RedisTokenBucket(redis)
+    }
+  } else {
+    logger.info('Rate limiting: Using PostgreSQL')
+    g._rlCachedAdapter = new DbTokenBucket()
+  }
+
+  return g._rlCachedAdapter!
+}
+
+export function getAdapterType(): StorageMethod {
+  return getStorageMethod()
+}
+
+export function resetStorageAdapter(): void {
+  g._rlCachedAdapter = null
+}
+
+export function setStorageAdapter(adapter: RateLimitStorageAdapter): void {
+  g._rlCachedAdapter = adapter
+}

apps/sim/lib/core/rate-limiter/storage/index.ts

@@ -0,0 +1,14 @@
+export type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+export { DbTokenBucket } from './db-token-bucket'
+export {
+  createStorageAdapter,
+  getAdapterType,
+  resetStorageAdapter,
+  setStorageAdapter,
+} from './factory'
+export { RedisTokenBucket } from './redis-token-bucket'

apps/sim/lib/core/rate-limiter/storage/redis-token-bucket.ts

@@ -0,0 +1,135 @@
+import type Redis from 'ioredis'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+
+const CONSUME_SCRIPT = `
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local requested = tonumber(ARGV[2])
+local maxTokens = tonumber(ARGV[3])
+local refillRate = tonumber(ARGV[4])
+local refillIntervalMs = tonumber(ARGV[5])
+local ttl = tonumber(ARGV[6])
+
+local bucket = redis.call('HMGET', key, 'tokens', 'lastRefillAt')
+local tokens = tonumber(bucket[1])
+local lastRefillAt = tonumber(bucket[2])
+
+if tokens == nil then
+  tokens = maxTokens
+  lastRefillAt = now
+end
+
+local elapsed = now - lastRefillAt
+local intervalsElapsed = math.floor(elapsed / refillIntervalMs)
+if intervalsElapsed > 0 then
+  tokens = math.min(maxTokens, tokens + (intervalsElapsed * refillRate))
+  lastRefillAt = lastRefillAt + (intervalsElapsed * refillIntervalMs)
+end
+
+local allowed = 0
+if tokens >= requested then
+  tokens = tokens - requested
+  allowed = 1
+end
+
+redis.call('HSET', key, 'tokens', tokens, 'lastRefillAt', lastRefillAt)
+redis.call('EXPIRE', key, ttl)
+
+local nextRefillAt = lastRefillAt + refillIntervalMs
+
+return {allowed, tokens, lastRefillAt, nextRefillAt}
+`
+
+const STATUS_SCRIPT = `
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local maxTokens = tonumber(ARGV[2])
+local refillRate = tonumber(ARGV[3])
+local refillIntervalMs = tonumber(ARGV[4])
+
+local bucket = redis.call('HMGET', key, 'tokens', 'lastRefillAt')
+local tokens = tonumber(bucket[1])
+local lastRefillAt = tonumber(bucket[2])
+
+if tokens == nil then
+  tokens = maxTokens
+  lastRefillAt = now
+end
+
+local elapsed = now - lastRefillAt
+local intervalsElapsed = math.floor(elapsed / refillIntervalMs)
+if intervalsElapsed > 0 then
+  tokens = math.min(maxTokens, tokens + (intervalsElapsed * refillRate))
+  lastRefillAt = lastRefillAt + (intervalsElapsed * refillIntervalMs)
+end
+
+local nextRefillAt = lastRefillAt + refillIntervalMs
+
+return {tokens, maxTokens, lastRefillAt, nextRefillAt}
+`
+
+export class RedisTokenBucket implements RateLimitStorageAdapter {
+  constructor(private redis: Redis) {}
+
+  async consumeTokens(
+    key: string,
+    tokens: number,
+    config: TokenBucketConfig
+  ): Promise<ConsumeResult> {
+    const now = Date.now()
+    const ttl = Math.ceil((config.refillIntervalMs * 2) / 1000)
+
+    const result = (await this.redis.eval(
+      CONSUME_SCRIPT,
+      1,
+      `ratelimit:tb:${key}`,
+      now,
+      tokens,
+      config.maxTokens,
+      config.refillRate,
+      config.refillIntervalMs,
+      ttl
+    )) as [number, number, number, number]
+
+    const [allowed, remaining, , nextRefill] = result
+
+    return {
+      allowed: allowed === 1,
+      tokensRemaining: remaining,
+      resetAt: new Date(nextRefill),
+      retryAfterMs: allowed === 1 ? undefined : Math.max(0, nextRefill - now),
+    }
+  }
+
+  async getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus> {
+    const now = Date.now()
+
+    const result = (await this.redis.eval(
+      STATUS_SCRIPT,
+      1,
+      `ratelimit:tb:${key}`,
+      now,
+      config.maxTokens,
+      config.refillRate,
+      config.refillIntervalMs
+    )) as [number, number, number, number]
+
+    const [tokensAvailable, maxTokens, lastRefillAt, nextRefillAt] = result
+
+    return {
+      tokensAvailable,
+      maxTokens,
+      lastRefillAt: new Date(lastRefillAt),
+      nextRefillAt: new Date(nextRefillAt),
+    }
+  }
+
+  async resetBucket(key: string): Promise<void> {
+    await this.redis.del(`ratelimit:tb:${key}`)
+  }
+}

apps/sim/lib/core/rate-limiter/types.ts

@@ -0,0 +1,109 @@
+import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
+import { env } from '@/lib/core/config/env'
+import { isBillingEnabled } from '@/lib/core/config/env-flags'
+import type { CoreTriggerType } from '@/stores/logs/filters/types'
+import type { TokenBucketConfig } from './storage'
+
+export type TriggerType = CoreTriggerType | 'api-endpoint'
+
+export type RateLimitCounterType = 'sync' | 'async' | 'api-endpoint'
+
+type RateLimitConfigKey = 'sync' | 'async' | 'apiEndpoint'
+
+export type SubscriptionPlan = 'free' | 'pro' | 'team' | 'enterprise'
+
+export interface RateLimitConfig {
+  sync: TokenBucketConfig
+  async: TokenBucketConfig
+  apiEndpoint: TokenBucketConfig
+}
+
+export const RATE_LIMIT_WINDOW_MS = Number.parseInt(env.RATE_LIMIT_WINDOW_MS) || 60000
+
+export const MANUAL_EXECUTION_LIMIT = Number.parseInt(env.MANUAL_EXECUTION_LIMIT) || 999999
+
+const DEFAULT_RATE_LIMITS = {
+  free: { sync: 50, async: 200, apiEndpoint: 30 },
+  pro: { sync: 150, async: 1000, apiEndpoint: 100 },
+  team: { sync: 300, async: 2500, apiEndpoint: 200 },
+  enterprise: { sync: 600, async: 5000, apiEndpoint: 500 },
+} as const
+
+function toConfigKey(type: RateLimitCounterType): RateLimitConfigKey {
+  return type === 'api-endpoint' ? 'apiEndpoint' : type
+}
+
+function createBucketConfig(ratePerMinute: number, burstMultiplier = 2): TokenBucketConfig {
+  return {
+    maxTokens: ratePerMinute * burstMultiplier,
+    refillRate: ratePerMinute,
+    refillIntervalMs: RATE_LIMIT_WINDOW_MS,
+  }
+}
+
+function getRateLimitForPlan(plan: SubscriptionPlan, type: RateLimitConfigKey): TokenBucketConfig {
+  const envVarMap: Record<SubscriptionPlan, Record<RateLimitConfigKey, string | undefined>> = {
+    free: {
+      sync: env.RATE_LIMIT_FREE_SYNC,
+      async: env.RATE_LIMIT_FREE_ASYNC,
+      apiEndpoint: undefined,
+    },
+    pro: { sync: env.RATE_LIMIT_PRO_SYNC, async: env.RATE_LIMIT_PRO_ASYNC, apiEndpoint: undefined },
+    team: {
+      sync: env.RATE_LIMIT_TEAM_SYNC,
+      async: env.RATE_LIMIT_TEAM_ASYNC,
+      apiEndpoint: undefined,
+    },
+    enterprise: {
+      sync: env.RATE_LIMIT_ENTERPRISE_SYNC,
+      async: env.RATE_LIMIT_ENTERPRISE_ASYNC,
+      apiEndpoint: undefined,
+    },
+  }
+
+  const rate = Number.parseInt(envVarMap[plan][type] || '') || DEFAULT_RATE_LIMITS[plan][type]
+  return createBucketConfig(rate)
+}
+
+export const RATE_LIMITS: Record<SubscriptionPlan, RateLimitConfig> = {
+  free: {
+    sync: getRateLimitForPlan('free', 'sync'),
+    async: getRateLimitForPlan('free', 'async'),
+    apiEndpoint: getRateLimitForPlan('free', 'apiEndpoint'),
+  },
+  pro: {
+    sync: getRateLimitForPlan('pro', 'sync'),
+    async: getRateLimitForPlan('pro', 'async'),
+    apiEndpoint: getRateLimitForPlan('pro', 'apiEndpoint'),
+  },
+  team: {
+    sync: getRateLimitForPlan('team', 'sync'),
+    async: getRateLimitForPlan('team', 'async'),
+    apiEndpoint: getRateLimitForPlan('team', 'apiEndpoint'),
+  },
+  enterprise: {
+    sync: getRateLimitForPlan('enterprise', 'sync'),
+    async: getRateLimitForPlan('enterprise', 'async'),
+    apiEndpoint: getRateLimitForPlan('enterprise', 'apiEndpoint'),
+  },
+}
+
+export function getRateLimit(
+  plan: SubscriptionPlan | string | undefined,
+  type: RateLimitCounterType
+): TokenBucketConfig {
+  const key = toConfigKey(type)
+  if (!isBillingEnabled) {
+    return RATE_LIMITS.free[key]
+  }
+  return RATE_LIMITS[getPlanTypeForLimits(plan)][key]
+}
+
+export class RateLimitError extends Error {
+  statusCode: number
+  constructor(message: string, statusCode = 429) {
+    super(message)
+    this.name = 'RateLimitError'
+    this.statusCode = statusCode
+  }
+}

apps/sim/lib/core/security/csp.test.ts

@@ -0,0 +1,299 @@
+import { createEnvMock, envFlagsMock } from '@sim/testing'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/env', () =>
+  createEnvMock({
+    NEXT_PUBLIC_APP_URL: 'https://example.com',
+    NEXT_PUBLIC_SOCKET_URL: 'https://socket.example.com',
+    OLLAMA_URL: 'http://localhost:11434',
+    S3_BUCKET_NAME: 'test-bucket',
+    AWS_REGION: 'us-east-1',
+    S3_KB_BUCKET_NAME: 'test-kb-bucket',
+    S3_CHAT_BUCKET_NAME: 'test-chat-bucket',
+    NEXT_PUBLIC_BRAND_LOGO_URL: 'https://brand.example.com/logo.png',
+    NEXT_PUBLIC_BRAND_FAVICON_URL: 'https://brand.example.com/favicon.ico',
+    NEXT_PUBLIC_PRIVACY_URL: 'https://legal.example.com/privacy',
+    NEXT_PUBLIC_TERMS_URL: 'https://legal.example.com/terms',
+  })
+)
+
+vi.mock('@/lib/core/config/env-flags', () => envFlagsMock)
+
+import {
+  addCSPSource,
+  buildCSPString,
+  buildTimeCSPDirectives,
+  type CSPDirectives,
+  generateRuntimeCSP,
+  getChatEmbedCSPPolicy,
+  getMainCSPPolicy,
+  getWorkflowExecutionCSPPolicy,
+  removeCSPSource,
+} from './csp'
+
+describe('buildCSPString', () => {
+  it('should build CSP string from directives', () => {
+    const directives: CSPDirectives = {
+      'default-src': ["'self'"],
+      'script-src': ["'self'", "'unsafe-inline'"],
+    }
+
+    const result = buildCSPString(directives)
+
+    expect(result).toContain("default-src 'self'")
+    expect(result).toContain("script-src 'self' 'unsafe-inline'")
+    expect(result).toContain(';')
+  })
+
+  it('should handle empty directives', () => {
+    const directives: CSPDirectives = {}
+    const result = buildCSPString(directives)
+    expect(result).toBe('')
+  })
+
+  it('should skip empty source arrays', () => {
+    const directives: CSPDirectives = {
+      'default-src': ["'self'"],
+      'script-src': [],
+    }
+
+    const result = buildCSPString(directives)
+
+    expect(result).toContain("default-src 'self'")
+    expect(result).not.toContain('script-src')
+  })
+
+  it('should filter out empty string sources', () => {
+    const directives: CSPDirectives = {
+      'default-src': ["'self'", '', '  ', 'https://example.com'],
+    }
+
+    const result = buildCSPString(directives)
+
+    expect(result).toContain("default-src 'self' https://example.com")
+    expect(result).not.toMatch(/\s{2,}/)
+  })
+
+  it('should handle all directive types', () => {
+    const directives: CSPDirectives = {
+      'default-src': ["'self'"],
+      'script-src': ["'self'"],
+      'style-src': ["'self'"],
+      'img-src': ["'self'", 'data:'],
+      'media-src': ["'self'"],
+      'font-src': ["'self'"],
+      'connect-src': ["'self'"],
+      'frame-src': ["'none'"],
+      'frame-ancestors': ["'self'"],
+      'form-action': ["'self'"],
+      'base-uri': ["'self'"],
+      'object-src': ["'none'"],
+    }
+
+    const result = buildCSPString(directives)
+
+    expect(result).toContain("default-src 'self'")
+    expect(result).toContain("script-src 'self'")
+    expect(result).toContain("object-src 'none'")
+  })
+})
+
+describe('getMainCSPPolicy', () => {
+  it('should return a valid CSP policy string', () => {
+    const policy = getMainCSPPolicy()
+
+    expect(policy).toContain("default-src 'self'")
+    expect(policy).toContain('script-src')
+    expect(policy).toContain('style-src')
+    expect(policy).toContain('img-src')
+  })
+
+  it('should include security directives', () => {
+    const policy = getMainCSPPolicy()
+
+    expect(policy).toContain("object-src 'none'")
+    expect(policy).toContain("frame-ancestors 'self'")
+    expect(policy).toContain("form-action 'self'")
+    expect(policy).toContain("base-uri 'self'")
+  })
+
+  it('should include necessary external resources', () => {
+    const policy = getMainCSPPolicy()
+
+    expect(policy).toContain('https://fonts.googleapis.com')
+    expect(policy).toContain('https://fonts.gstatic.com')
+    expect(policy).toContain('https://*.google.com')
+  })
+})
+
+describe('getWorkflowExecutionCSPPolicy', () => {
+  it('should return permissive CSP for workflow execution', () => {
+    const policy = getWorkflowExecutionCSPPolicy()
+
+    expect(policy).toContain('default-src *')
+    expect(policy).toContain("'unsafe-inline'")
+    expect(policy).toContain("'unsafe-eval'")
+    expect(policy).toContain('connect-src *')
+  })
+
+  it('should be more permissive than main CSP', () => {
+    const mainPolicy = getMainCSPPolicy()
+    const execPolicy = getWorkflowExecutionCSPPolicy()
+
+    expect(execPolicy.length).toBeLessThan(mainPolicy.length)
+    expect(execPolicy).toContain('*')
+  })
+})
+
+describe('generateRuntimeCSP', () => {
+  it('should generate CSP with runtime environment variables', () => {
+    const csp = generateRuntimeCSP()
+
+    expect(csp).toContain("default-src 'self'")
+    expect(csp).toContain('https://example.com')
+  })
+
+  it('should include socket URL and WebSocket variant', () => {
+    const csp = generateRuntimeCSP()
+
+    expect(csp).toContain('https://socket.example.com')
+    expect(csp).toContain('wss://socket.example.com')
+  })
+
+  it('should include brand URLs', () => {
+    const csp = generateRuntimeCSP()
+
+    expect(csp).toContain('https://brand.example.com')
+  })
+
+  it('should not have excessive whitespace', () => {
+    const csp = generateRuntimeCSP()
+
+    expect(csp).not.toMatch(/\s{3,}/)
+    expect(csp.trim()).toBe(csp)
+  })
+
+  it('should allow blob URLs for iframe-based PDF previews', () => {
+    const csp = generateRuntimeCSP()
+    const frameSrcDirective = csp
+      .split('; ')
+      .find((directive) => directive.startsWith('frame-src '))
+
+    expect(frameSrcDirective).toBeDefined()
+    expect(frameSrcDirective).toContain('blob:')
+  })
+})
+
+describe('addCSPSource', () => {
+  const originalDirectives = structuredClone(buildTimeCSPDirectives)
+
+  afterEach(() => {
+    Object.keys(buildTimeCSPDirectives).forEach((key) => {
+      const k = key as keyof CSPDirectives
+      buildTimeCSPDirectives[k] = originalDirectives[k]
+    })
+  })
+
+  it('should add a source to an existing directive', () => {
+    const originalLength = buildTimeCSPDirectives['img-src']?.length || 0
+
+    addCSPSource('img-src', 'https://new-source.com')
+
+    expect(buildTimeCSPDirectives['img-src']).toContain('https://new-source.com')
+    expect(buildTimeCSPDirectives['img-src']?.length).toBe(originalLength + 1)
+  })
+
+  it('should not add duplicate sources', () => {
+    addCSPSource('img-src', 'https://duplicate.com')
+    const lengthAfterFirst = buildTimeCSPDirectives['img-src']?.length || 0
+
+    addCSPSource('img-src', 'https://duplicate.com')
+
+    expect(buildTimeCSPDirectives['img-src']?.length).toBe(lengthAfterFirst)
+  })
+
+  it('should create directive array if it does not exist', () => {
+    ;(buildTimeCSPDirectives as any)['worker-src'] = undefined
+
+    addCSPSource('script-src', 'https://worker.example.com')
+
+    expect(buildTimeCSPDirectives['script-src']).toContain('https://worker.example.com')
+  })
+})
+
+describe('removeCSPSource', () => {
+  const originalDirectives = structuredClone(buildTimeCSPDirectives)
+
+  afterEach(() => {
+    Object.keys(buildTimeCSPDirectives).forEach((key) => {
+      const k = key as keyof CSPDirectives
+      buildTimeCSPDirectives[k] = originalDirectives[k]
+    })
+  })
+
+  it('should remove a source from an existing directive', () => {
+    addCSPSource('img-src', 'https://to-remove.com')
+    expect(buildTimeCSPDirectives['img-src']).toContain('https://to-remove.com')
+
+    removeCSPSource('img-src', 'https://to-remove.com')
+
+    expect(buildTimeCSPDirectives['img-src']).not.toContain('https://to-remove.com')
+  })
+
+  it('should handle removing non-existent source gracefully', () => {
+    const originalLength = buildTimeCSPDirectives['img-src']?.length || 0
+
+    removeCSPSource('img-src', 'https://non-existent.com')
+
+    expect(buildTimeCSPDirectives['img-src']?.length).toBe(originalLength)
+  })
+
+  it('should handle removing from non-existent directive gracefully', () => {
+    ;(buildTimeCSPDirectives as any)['worker-src'] = undefined
+
+    expect(() => {
+      removeCSPSource('script-src', 'https://anything.com')
+    }).not.toThrow()
+  })
+})
+
+describe('buildTimeCSPDirectives', () => {
+  it('should have all required security directives', () => {
+    expect(buildTimeCSPDirectives['default-src']).toBeDefined()
+    expect(buildTimeCSPDirectives['object-src']).toContain("'none'")
+    expect(buildTimeCSPDirectives['frame-ancestors']).toContain("'self'")
+    expect(buildTimeCSPDirectives['base-uri']).toContain("'self'")
+  })
+
+  it('should have self as default source', () => {
+    expect(buildTimeCSPDirectives['default-src']).toContain("'self'")
+  })
+
+  it('should allow Google fonts', () => {
+    expect(buildTimeCSPDirectives['style-src']).toContain('https://fonts.googleapis.com')
+    expect(buildTimeCSPDirectives['font-src']).toContain('https://fonts.gstatic.com')
+  })
+
+  it('should allow data: and blob: for images', () => {
+    expect(buildTimeCSPDirectives['img-src']).toContain('data:')
+    expect(buildTimeCSPDirectives['img-src']).toContain('blob:')
+  })
+})
+
+describe('getChatEmbedCSPPolicy', () => {
+  it('allows iframe embedding from any origin', () => {
+    expect(getChatEmbedCSPPolicy()).toContain('frame-ancestors *')
+  })
+
+  it('allows Office.js to load from Microsoft for Excel/Word add-in embedding', () => {
+    const policy = getChatEmbedCSPPolicy()
+    expect(policy).toMatch(/script-src[^;]*https:\/\/appsforoffice\.microsoft\.com/)
+    expect(policy).toMatch(/connect-src[^;]*https:\/\/appsforoffice\.microsoft\.com/)
+  })
+
+  it('does not regress object-src or base-uri restrictions', () => {
+    const policy = getChatEmbedCSPPolicy()
+    expect(policy).toContain("object-src 'none'")
+    expect(policy).toContain("base-uri 'self'")
+  })
+})

apps/sim/lib/core/security/csp.ts

@@ -0,0 +1,285 @@
+import { env, getEnv } from '../config/env'
+import { isDev, isHosted, isReactGrabEnabled } from '../config/env-flags'
+
+/**
+ * Content Security Policy (CSP) configuration builder
+ *
+ * NOTE: This file is loaded by next.config.ts at build time, before @/ path
+ * aliases are resolved. Do NOT import from ../utils/urls (which uses @/ imports).
+ * Keep all URL constants local to this file.
+ */
+
+const DEFAULT_SOCKET_URL = 'http://localhost:3002'
+const DEFAULT_OLLAMA_URL = 'http://localhost:11434'
+
+function toWebSocketUrl(httpUrl: string): string {
+  return httpUrl.replace('http://', 'ws://').replace('https://', 'wss://')
+}
+
+function getHostnameFromUrl(url: string | undefined): string[] {
+  if (!url) return []
+  try {
+    return [`https://${new URL(url).hostname}`]
+  } catch {
+    return []
+  }
+}
+
+export interface CSPDirectives {
+  'default-src'?: string[]
+  'script-src'?: string[]
+  'style-src'?: string[]
+  'img-src'?: string[]
+  'media-src'?: string[]
+  'font-src'?: string[]
+  'connect-src'?: string[]
+  'worker-src'?: string[]
+  'frame-src'?: string[]
+  'frame-ancestors'?: string[]
+  'form-action'?: string[]
+  'base-uri'?: string[]
+  'object-src'?: string[]
+}
+
+/**
+ * Static CSP sources shared between build-time and runtime.
+ * Add new domains here — both paths pick them up automatically.
+ */
+const STATIC_SCRIPT_SRC = [
+  "'self'",
+  "'unsafe-inline'",
+  ...(isDev ? ["'unsafe-eval'"] : []),
+  'https://*.google.com',
+  'https://apis.google.com',
+  'https://challenges.cloudflare.com',
+  // Cal.com booking embed (landing /demo) — embed.js is served from app.cal.com
+  'https://app.cal.com',
+  ...(isReactGrabEnabled ? ['https://unpkg.com'] : []),
+  ...(isHosted
+    ? [
+        'https://www.googletagmanager.com',
+        'https://www.google-analytics.com',
+        'https://analytics.ahrefs.com',
+        // HubSpot tracking (landing pages) — loader plus the
+        // analytics/form-tracking/banner scripts it injects as <script> tags
+        'https://*.hs-scripts.com',
+        'https://*.hs-analytics.net',
+        'https://*.hscollectedforms.net',
+        'https://*.hs-banner.com',
+      ]
+    : []),
+] as const
+
+const STATIC_IMG_SRC = ["'self'", 'data:', 'blob:', 'https:'] as const
+
+const STATIC_CONNECT_SRC = [
+  "'self'",
+  'https://api.browser-use.com',
+  'https://api.elevenlabs.io',
+  'wss://api.elevenlabs.io',
+  'https://api.exa.ai',
+  'https://api.firecrawl.dev',
+  'https://*.googleapis.com',
+  'https://*.amazonaws.com',
+  'https://*.s3.amazonaws.com',
+  'https://*.blob.core.windows.net',
+  'https://*.atlassian.com',
+  'https://*.supabase.co',
+  'https://api.github.com',
+  'https://github.com/*',
+  'https://challenges.cloudflare.com',
+  // Cal.com booking embed (landing /demo) — embed XHR/availability calls
+  'https://app.cal.com',
+  'https://cal.com',
+  ...(isReactGrabEnabled ? ['https://www.react-grab.com'] : []),
+  ...(isDev ? ['ws://localhost:4722'] : []),
+  ...(isHosted
+    ? [
+        'https://www.googletagmanager.com',
+        'https://*.google-analytics.com',
+        'https://*.analytics.google.com',
+        'https://analytics.google.com',
+        'https://www.google.com',
+        'https://analytics.ahrefs.com',
+        'https://*.g.doubleclick.net',
+        // HubSpot tracking — form-tracking API (hscollectedforms.js).
+        // The visitor beacon itself is an image pixel (img-src, already
+        // permitted below), not a connect-src request.
+        'https://*.hscollectedforms.net',
+      ]
+    : []),
+] as const
+
+const STATIC_FRAME_SRC = [
+  "'self'",
+  'blob:',
+  'https://challenges.cloudflare.com',
+  // Cal.com booking embed (landing /demo) — the booking iframe
+  'https://app.cal.com',
+  'https://cal.com',
+  'https://drive.google.com',
+  'https://docs.google.com',
+  'https://*.google.com',
+  'https://www.youtube.com',
+  'https://player.vimeo.com',
+  'https://www.dailymotion.com',
+  'https://player.twitch.tv',
+  'https://clips.twitch.tv',
+  'https://streamable.com',
+  'https://fast.wistia.net',
+  'https://www.tiktok.com',
+  'https://w.soundcloud.com',
+  'https://open.spotify.com',
+  'https://embed.music.apple.com',
+  'https://www.loom.com',
+  'https://www.facebook.com',
+  'https://www.instagram.com',
+  'https://platform.twitter.com',
+  'https://rumble.com',
+  'https://play.vidyard.com',
+  'https://iframe.cloudflarestream.com',
+  'https://www.mixcloud.com',
+  'https://tenor.com',
+  'https://giphy.com',
+  ...(isHosted ? ['https://www.googletagmanager.com'] : []),
+] as const
+
+// Build-time CSP directives (for next.config.ts)
+export const buildTimeCSPDirectives: CSPDirectives = {
+  'default-src': ["'self'"],
+  'script-src': [...STATIC_SCRIPT_SRC],
+  'style-src': ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com'],
+
+  'img-src': [...STATIC_IMG_SRC],
+
+  'media-src': ["'self'", 'blob:'],
+  'worker-src': ["'self'", 'blob:'],
+  'font-src': ["'self'", 'https://fonts.gstatic.com'],
+
+  'connect-src': [
+    ...STATIC_CONNECT_SRC,
+    env.NEXT_PUBLIC_APP_URL || '',
+    ...(env.OLLAMA_URL ? [env.OLLAMA_URL] : isDev ? [DEFAULT_OLLAMA_URL] : []),
+    ...(env.NEXT_PUBLIC_SOCKET_URL
+      ? [env.NEXT_PUBLIC_SOCKET_URL, toWebSocketUrl(env.NEXT_PUBLIC_SOCKET_URL)]
+      : isDev
+        ? [DEFAULT_SOCKET_URL, toWebSocketUrl(DEFAULT_SOCKET_URL)]
+        : []),
+    ...getHostnameFromUrl(env.NEXT_PUBLIC_BRAND_LOGO_URL),
+    ...getHostnameFromUrl(env.NEXT_PUBLIC_PRIVACY_URL),
+    ...getHostnameFromUrl(env.NEXT_PUBLIC_TERMS_URL),
+  ],
+
+  'frame-src': [...STATIC_FRAME_SRC],
+  'frame-ancestors': ["'self'"],
+  'form-action': ["'self'"],
+  'base-uri': ["'self'"],
+  'object-src': ["'none'"],
+}
+
+/**
+ * Build CSP string from directives object
+ */
+export function buildCSPString(directives: CSPDirectives): string {
+  return Object.entries(directives)
+    .map(([directive, sources]) => {
+      if (!sources || sources.length === 0) return ''
+      const validSources = sources.filter((source: string) => source && source.trim() !== '')
+      if (validSources.length === 0) return ''
+      return `${directive} ${validSources.join(' ')}`
+    })
+    .filter(Boolean)
+    .join('; ')
+}
+
+/**
+ * Generate runtime CSP header with dynamic environment variables.
+ * Composes from the same STATIC_* constants as buildTimeCSPDirectives,
+ * but resolves env vars at request time via getEnv() to fix Docker
+ * deployments where build-time values may be stale placeholders.
+ */
+export function generateRuntimeCSP(): string {
+  const appUrl = getEnv('NEXT_PUBLIC_APP_URL') || ''
+
+  const socketUrl = getEnv('NEXT_PUBLIC_SOCKET_URL') || (isDev ? DEFAULT_SOCKET_URL : '')
+  const socketWsUrl = socketUrl ? toWebSocketUrl(socketUrl) : ''
+  const ollamaUrl = getEnv('OLLAMA_URL') || (isDev ? DEFAULT_OLLAMA_URL : '')
+
+  const brandLogoDomains = getHostnameFromUrl(getEnv('NEXT_PUBLIC_BRAND_LOGO_URL'))
+  const privacyDomains = getHostnameFromUrl(getEnv('NEXT_PUBLIC_PRIVACY_URL'))
+  const termsDomains = getHostnameFromUrl(getEnv('NEXT_PUBLIC_TERMS_URL'))
+
+  const runtimeDirectives: CSPDirectives = {
+    ...buildTimeCSPDirectives,
+
+    'img-src': [...STATIC_IMG_SRC],
+
+    'connect-src': [
+      ...STATIC_CONNECT_SRC,
+      appUrl,
+      ollamaUrl,
+      socketUrl,
+      socketWsUrl,
+      ...brandLogoDomains,
+      ...privacyDomains,
+      ...termsDomains,
+    ],
+  }
+
+  return buildCSPString(runtimeDirectives)
+}
+
+/**
+ * Get the main CSP policy string (build-time)
+ */
+export function getMainCSPPolicy(): string {
+  return buildCSPString(buildTimeCSPDirectives)
+}
+
+/**
+ * Permissive CSP for workflow execution endpoints
+ */
+export function getWorkflowExecutionCSPPolicy(): string {
+  return "default-src * 'unsafe-inline' 'unsafe-eval'; connect-src *;"
+}
+
+/**
+ * CSP for embeddable chat pages.
+ * Extends the shared embed policy with Microsoft Office.js sources so the
+ * chat page can serve as an Office (Excel/Word/Outlook) add-in surface
+ * when loaded with `?embed=office`.
+ */
+export function getChatEmbedCSPPolicy(): string {
+  return buildCSPString({
+    ...buildTimeCSPDirectives,
+    'script-src': [...STATIC_SCRIPT_SRC, 'https://appsforoffice.microsoft.com'],
+    'connect-src': [
+      ...(buildTimeCSPDirectives['connect-src'] ?? []),
+      'https://appsforoffice.microsoft.com',
+    ],
+    'frame-ancestors': ['*'],
+  })
+}
+
+/**
+ * Add a source to a specific directive (modifies build-time directives)
+ */
+export function addCSPSource(directive: keyof CSPDirectives, source: string): void {
+  if (!buildTimeCSPDirectives[directive]) {
+    buildTimeCSPDirectives[directive] = []
+  }
+  if (!buildTimeCSPDirectives[directive]!.includes(source)) {
+    buildTimeCSPDirectives[directive]!.push(source)
+  }
+}
+
+/**
+ * Remove a source from a specific directive (modifies build-time directives)
+ */
+export function removeCSPSource(directive: keyof CSPDirectives, source: string): void {
+  if (buildTimeCSPDirectives[directive]) {
+    buildTimeCSPDirectives[directive] = buildTimeCSPDirectives[directive]!.filter(
+      (s: string) => s !== source
+    )
+  }
+}

apps/sim/lib/core/security/deployment-auth.ts

@@ -0,0 +1,202 @@
+import { createLogger } from '@sim/logger'
+import { safeCompare } from '@sim/security/compare'
+import type { NextRequest } from 'next/server'
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter'
+import { RateLimiter } from '@/lib/core/rate-limiter'
+import {
+  type DeploymentAuthKind,
+  deploymentAuthCookieName,
+  isEmailAllowed,
+  validateAuthToken,
+} from '@/lib/core/security/deployment'
+import { decryptSecret } from '@/lib/core/security/encryption'
+import { getClientIp } from '@/lib/core/utils/request'
+
+const logger = createLogger('DeploymentAuth')
+
+const rateLimiter = new RateLimiter()
+
+/**
+ * Throttles unauthenticated password guesses per client IP against a single
+ * deployment, mirroring the OTP/SSO IP limits.
+ */
+const PASSWORD_IP_RATE_LIMIT: TokenBucketConfig = {
+  maxTokens: 10,
+  refillRate: 10,
+  refillIntervalMs: 15 * 60_000,
+}
+
+/**
+ * A password/email-gated resource (a deployed chat or a public file share). Only
+ * the fields the auth check needs — the `password` is the encrypted secret.
+ */
+export interface DeploymentAuthResource {
+  id: string
+  authType: string | null
+  password?: string | null
+  allowedEmails?: unknown
+}
+
+interface DeploymentAuthBody {
+  password?: string
+  email?: string
+  input?: unknown
+}
+
+export interface DeploymentAuthResult {
+  authorized: boolean
+  error?: string
+  status?: number
+  retryAfterMs?: number
+}
+
+/**
+ * Shared password/email/SSO gate for deployed resources. The `cookiePrefix`
+ * selects the auth cookie (`${cookiePrefix}_auth_${id}`) and the rate-limit
+ * namespace so chat deployments and public file shares share one code path. Both
+ * support all four modes: `'public'`, `'password'`, `'email'`, and `'sso'`.
+ */
+export async function validateDeploymentAuth(
+  requestId: string,
+  resource: DeploymentAuthResource,
+  request: NextRequest,
+  parsedBody: DeploymentAuthBody | null | undefined,
+  cookiePrefix: DeploymentAuthKind
+): Promise<DeploymentAuthResult> {
+  const authType = resource.authType || 'public'
+
+  if (authType === 'public') {
+    return { authorized: true }
+  }
+
+  if (authType !== 'sso') {
+    const authCookie = request.cookies.get(deploymentAuthCookieName(cookiePrefix, resource.id))
+
+    if (
+      authCookie &&
+      validateAuthToken(authCookie.value, resource.id, authType, resource.password)
+    ) {
+      return { authorized: true }
+    }
+  }
+
+  if (authType === 'password') {
+    if (request.method === 'GET') {
+      return { authorized: false, error: 'auth_required_password' }
+    }
+
+    try {
+      if (!parsedBody) {
+        return { authorized: false, error: 'Password is required' }
+      }
+
+      const { password, input } = parsedBody
+
+      if (input && !password) {
+        return { authorized: false, error: 'auth_required_password' }
+      }
+
+      if (!password) {
+        return { authorized: false, error: 'Password is required' }
+      }
+
+      if (!resource.password) {
+        logger.error(`[${requestId}] No password set for password-protected ${resource.id}`)
+        return { authorized: false, error: 'Authentication configuration error' }
+      }
+
+      const ip = getClientIp(request)
+      const ipRateLimit = await rateLimiter.checkRateLimitDirect(
+        `${cookiePrefix}-password:ip:${resource.id}:${ip}`,
+        PASSWORD_IP_RATE_LIMIT
+      )
+      if (!ipRateLimit.allowed) {
+        logger.warn(
+          `[${requestId}] Password attempt IP rate limit exceeded for ${resource.id} from ${ip}`
+        )
+        return {
+          authorized: false,
+          error: 'Too many attempts. Please try again later.',
+          status: 429,
+          retryAfterMs: ipRateLimit.retryAfterMs ?? PASSWORD_IP_RATE_LIMIT.refillIntervalMs,
+        }
+      }
+
+      const { decrypted } = await decryptSecret(resource.password)
+      if (!safeCompare(password, decrypted)) {
+        return { authorized: false, error: 'Invalid password' }
+      }
+
+      return { authorized: true }
+    } catch (error) {
+      logger.error(`[${requestId}] Error validating password:`, error)
+      return { authorized: false, error: 'Authentication error' }
+    }
+  }
+
+  if (authType === 'email') {
+    if (request.method === 'GET') {
+      return { authorized: false, error: 'auth_required_email' }
+    }
+
+    try {
+      if (!parsedBody) {
+        return { authorized: false, error: 'Email is required' }
+      }
+
+      const { email, input } = parsedBody
+
+      if (input && !email) {
+        return { authorized: false, error: 'auth_required_email' }
+      }
+
+      if (!email) {
+        return { authorized: false, error: 'Email is required' }
+      }
+
+      const allowedEmails = (resource.allowedEmails as string[]) || []
+
+      if (isEmailAllowed(email, allowedEmails)) {
+        return { authorized: false, error: 'otp_required' }
+      }
+
+      return { authorized: false, error: 'Email not authorized' }
+    } catch (error) {
+      logger.error(`[${requestId}] Error validating email:`, error)
+      return { authorized: false, error: 'Authentication error' }
+    }
+  }
+
+  if (authType === 'sso') {
+    try {
+      if (request.method !== 'GET' && !parsedBody) {
+        return { authorized: false, error: 'SSO authentication is required' }
+      }
+
+      const { getSession } = await import('@/lib/auth')
+      const session = await getSession()
+
+      if (!session || !session.user) {
+        return { authorized: false, error: 'auth_required_sso' }
+      }
+
+      const userEmail = session.user.email
+      if (!userEmail) {
+        return { authorized: false, error: 'SSO session does not contain email' }
+      }
+
+      const allowedEmails = (resource.allowedEmails as string[]) || []
+
+      if (isEmailAllowed(userEmail, allowedEmails)) {
+        return { authorized: true }
+      }
+
+      return { authorized: false, error: 'Your email is not authorized to access this resource' }
+    } catch (error) {
+      logger.error(`[${requestId}] Error validating SSO:`, error)
+      return { authorized: false, error: 'SSO authentication error' }
+    }
+  }
+
+  return { authorized: false, error: 'Unsupported authentication type' }
+}

apps/sim/lib/core/security/deployment.test.ts

@@ -0,0 +1,24 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { isEmailAllowed } from '@/lib/core/security/deployment'
+
+describe('isEmailAllowed', () => {
+  it('matches an exact email regardless of casing on either side', () => {
+    expect(isEmailAllowed('user@acme.com', ['user@acme.com'])).toBe(true)
+    expect(isEmailAllowed('User@Acme.com', ['user@acme.com'])).toBe(true)
+    expect(isEmailAllowed('user@acme.com', ['USER@ACME.COM'])).toBe(true)
+    expect(isEmailAllowed('  User@Acme.com  ', ['user@acme.com'])).toBe(true)
+  })
+
+  it('matches a domain pattern regardless of casing (covers IdP/session emails)', () => {
+    expect(isEmailAllowed('User@Acme.com', ['@acme.com'])).toBe(true)
+    expect(isEmailAllowed('user@acme.com', ['@Acme.com'])).toBe(true)
+  })
+
+  it('rejects emails not on the allow-list', () => {
+    expect(isEmailAllowed('user@evil.com', ['user@acme.com', '@acme.com'])).toBe(false)
+    expect(isEmailAllowed('user@acme.com', [])).toBe(false)
+  })
+})

apps/sim/lib/core/security/deployment.ts

@@ -0,0 +1,134 @@
+import { safeCompare } from '@sim/security/compare'
+import { sha256Hex } from '@sim/security/hash'
+import { hmacSha256Hex } from '@sim/security/hmac'
+import { normalizeEmail } from '@sim/utils/string'
+import type { NextResponse } from 'next/server'
+import { env } from '@/lib/core/config/env'
+import { isDev } from '@/lib/core/config/env-flags'
+
+/**
+ * Shared authentication utilities for deployed chat endpoints.
+ * Handles token generation, validation, and auth cookies. CORS for these
+ * endpoints lives in proxy.ts as the single source of truth.
+ */
+
+function signPayload(payload: string): string {
+  return hmacSha256Hex(payload, env.BETTER_AUTH_SECRET)
+}
+
+function passwordSlot(encryptedPassword?: string | null): string {
+  if (!encryptedPassword) return ''
+  return sha256Hex(encryptedPassword).slice(0, 8)
+}
+
+function generateAuthToken(
+  deploymentId: string,
+  type: string,
+  encryptedPassword?: string | null
+): string {
+  const payload = `${deploymentId}:${type}:${Date.now()}:${passwordSlot(encryptedPassword)}`
+  const sig = signPayload(payload)
+  return Buffer.from(`${payload}:${sig}`).toString('base64')
+}
+
+/**
+ * Validates an HMAC-signed authentication token for a chat deployment.
+ * Includes a password-derived slot so changing the deployment password immediately
+ * invalidates existing sessions.
+ */
+export function validateAuthToken(
+  token: string,
+  deploymentId: string,
+  authType: string,
+  encryptedPassword?: string | null
+): boolean {
+  try {
+    const decoded = Buffer.from(token, 'base64').toString()
+    const lastColon = decoded.lastIndexOf(':')
+    if (lastColon === -1) return false
+
+    const payload = decoded.slice(0, lastColon)
+    const sig = decoded.slice(lastColon + 1)
+
+    const expectedSig = signPayload(payload)
+    if (!safeCompare(sig, expectedSig)) {
+      return false
+    }
+
+    const parts = payload.split(':')
+    if (parts.length < 4) return false
+    const [storedId, storedType, timestamp, storedPwSlot] = parts
+
+    if (storedId !== deploymentId) return false
+
+    // Bind the cookie to the auth type so a token minted under one mode (e.g. a
+    // `public` share, which has an empty password slot) can't satisfy another
+    // mode (e.g. `email` OTP) after the share's auth type is changed.
+    if (storedType !== authType) return false
+
+    const expectedPwSlot = passwordSlot(encryptedPassword)
+    if (storedPwSlot !== expectedPwSlot) return false
+
+    const createdAt = Number.parseInt(timestamp)
+    const expireTime = 24 * 60 * 60 * 1000
+    if (Date.now() - createdAt > expireTime) return false
+
+    return true
+  } catch (_e) {
+    return false
+  }
+}
+
+/** The kind of deployed resource an auth cookie/token belongs to. */
+export type DeploymentAuthKind = 'chat' | 'file'
+
+/** Canonical auth cookie name for a deployed resource (`{kind}_auth_{id}`). */
+export function deploymentAuthCookieName(cookiePrefix: DeploymentAuthKind, id: string): string {
+  return `${cookiePrefix}_auth_${id}`
+}
+
+/**
+ * Sets an authentication cookie for a deployment
+ */
+export function setDeploymentAuthCookie(
+  response: NextResponse,
+  cookiePrefix: DeploymentAuthKind,
+  deploymentId: string,
+  authType: string,
+  encryptedPassword?: string | null
+): void {
+  const token = generateAuthToken(deploymentId, authType, encryptedPassword)
+  response.cookies.set({
+    name: deploymentAuthCookieName(cookiePrefix, deploymentId),
+    value: token,
+    httpOnly: true,
+    secure: !isDev,
+    sameSite: 'lax',
+    path: '/',
+    maxAge: 60 * 60 * 24,
+  })
+}
+
+/**
+ * Checks if an email matches the allowed emails list (exact match or domain
+ * match). Case-insensitive — email addresses are compared lowercased on both
+ * sides, so callers don't need to normalize before calling.
+ */
+export function isEmailAllowed(email: string, allowedEmails: string[]): boolean {
+  const normalizedEmail = normalizeEmail(email)
+  const normalizedAllowed = allowedEmails.map(normalizeEmail)
+
+  if (normalizedAllowed.includes(normalizedEmail)) {
+    return true
+  }
+
+  const atIndex = normalizedEmail.indexOf('@')
+  if (atIndex > 0) {
+    const domain = normalizedEmail.substring(atIndex + 1)
+    if (domain && normalizedAllowed.some((allowed) => allowed === `@${domain}`)) {
+      return true
+    }
+  }
+
+  return false
+}

apps/sim/lib/core/security/encryption.test.ts

@@ -0,0 +1,187 @@
+import { createEnvMock } from '@sim/testing'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/env', () =>
+  createEnvMock({
+    ENCRYPTION_KEY: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+  })
+)
+
+import { env } from '@/lib/core/config/env'
+import { decryptSecret, encryptSecret, generatePassword } from './encryption'
+
+describe('encryptSecret', () => {
+  it('should encrypt a secret and return encrypted value with IV', async () => {
+    const secret = 'my-secret-value'
+    const result = await encryptSecret(secret)
+
+    expect(result.encrypted).toBeDefined()
+    expect(result.iv).toBeDefined()
+    expect(result.encrypted).toContain(':')
+    expect(result.iv).toHaveLength(32)
+  })
+
+  it('should produce different encrypted values for the same input', async () => {
+    const secret = 'same-secret'
+    const result1 = await encryptSecret(secret)
+    const result2 = await encryptSecret(secret)
+
+    expect(result1.encrypted).not.toBe(result2.encrypted)
+    expect(result1.iv).not.toBe(result2.iv)
+  })
+
+  it('should encrypt empty strings', async () => {
+    const result = await encryptSecret('')
+    expect(result.encrypted).toBeDefined()
+    expect(result.iv).toBeDefined()
+  })
+
+  it('should encrypt long secrets', async () => {
+    const longSecret = 'a'.repeat(10000)
+    const result = await encryptSecret(longSecret)
+    expect(result.encrypted).toBeDefined()
+  })
+
+  it('should encrypt secrets with special characters', async () => {
+    const specialSecret = '!@#$%^&*()_+-=[]{}|;\':",.<>?/`~\n\t\r'
+    const result = await encryptSecret(specialSecret)
+    expect(result.encrypted).toBeDefined()
+  })
+
+  it('should encrypt unicode characters', async () => {
+    const unicodeSecret = 'Hello !"#$%&\'()*+,-./0123456789:;<=>?@'
+    const result = await encryptSecret(unicodeSecret)
+    expect(result.encrypted).toBeDefined()
+  })
+})
+
+describe('decryptSecret', () => {
+  it('should decrypt an encrypted secret back to original value', async () => {
+    const originalSecret = 'my-secret-value'
+    const { encrypted } = await encryptSecret(originalSecret)
+    const { decrypted } = await decryptSecret(encrypted)
+
+    expect(decrypted).toBe(originalSecret)
+  })
+
+  it('should decrypt very short secrets', async () => {
+    const { encrypted } = await encryptSecret('a')
+    const { decrypted } = await decryptSecret(encrypted)
+    expect(decrypted).toBe('a')
+  })
+
+  it('should decrypt long secrets', async () => {
+    const longSecret = 'b'.repeat(10000)
+    const { encrypted } = await encryptSecret(longSecret)
+    const { decrypted } = await decryptSecret(encrypted)
+    expect(decrypted).toBe(longSecret)
+  })
+
+  it('should decrypt secrets with special characters', async () => {
+    const specialSecret = '!@#$%^&*()_+-=[]{}|;\':",.<>?/`~\n\t\r'
+    const { encrypted } = await encryptSecret(specialSecret)
+    const { decrypted } = await decryptSecret(encrypted)
+    expect(decrypted).toBe(specialSecret)
+  })
+
+  it('should throw error for invalid encrypted format (missing parts)', async () => {
+    await expect(decryptSecret('invalid')).rejects.toThrow(
+      'Invalid encrypted value format. Expected "iv:encrypted:authTag"'
+    )
+  })
+
+  it('should throw error for invalid encrypted format (only two parts)', async () => {
+    await expect(decryptSecret('part1:part2')).rejects.toThrow(
+      'Invalid encrypted value format. Expected "iv:encrypted:authTag"'
+    )
+  })
+
+  it('should throw error for tampered ciphertext', async () => {
+    const { encrypted } = await encryptSecret('original-secret')
+    const parts = encrypted.split(':')
+    parts[1] = `tampered${parts[1].slice(8)}`
+    const tamperedEncrypted = parts.join(':')
+
+    await expect(decryptSecret(tamperedEncrypted)).rejects.toThrow()
+  })
+
+  it('should throw error for tampered auth tag', async () => {
+    const { encrypted } = await encryptSecret('original-secret')
+    const parts = encrypted.split(':')
+    parts[2] = '00000000000000000000000000000000'
+    const tamperedEncrypted = parts.join(':')
+
+    await expect(decryptSecret(tamperedEncrypted)).rejects.toThrow()
+  })
+
+  it('should throw error for invalid IV', async () => {
+    const { encrypted } = await encryptSecret('original-secret')
+    const parts = encrypted.split(':')
+    parts[0] = '00000000000000000000000000000000'
+    const tamperedEncrypted = parts.join(':')
+
+    await expect(decryptSecret(tamperedEncrypted)).rejects.toThrow()
+  })
+})
+
+describe('generatePassword', () => {
+  it('should generate password with default length of 24', () => {
+    const password = generatePassword()
+    expect(password).toHaveLength(24)
+  })
+
+  it('should generate password with custom length', () => {
+    const password = generatePassword(32)
+    expect(password).toHaveLength(32)
+  })
+
+  it('should generate password with minimum length', () => {
+    const password = generatePassword(1)
+    expect(password).toHaveLength(1)
+  })
+
+  it('should generate different passwords on each call', () => {
+    const passwords = new Set()
+    for (let i = 0; i < 100; i++) {
+      passwords.add(generatePassword())
+    }
+    expect(passwords.size).toBeGreaterThan(90)
+  })
+
+  it('should only contain allowed characters', () => {
+    const allowedChars =
+      'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_-+='
+    const password = generatePassword(1000)
+
+    for (const char of password) {
+      expect(allowedChars).toContain(char)
+    }
+  })
+
+  it('should handle zero length', () => {
+    const password = generatePassword(0)
+    expect(password).toBe('')
+  })
+})
+
+describe('encryption key validation', () => {
+  const originalEncryptionKey = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
+
+  afterEach(() => {
+    ;(env as Record<string, string>).ENCRYPTION_KEY = originalEncryptionKey
+  })
+
+  it('should throw error when ENCRYPTION_KEY is not set', async () => {
+    ;(env as Record<string, string>).ENCRYPTION_KEY = ''
+    await expect(encryptSecret('test')).rejects.toThrow(
+      'ENCRYPTION_KEY must be set to a 64-character hex string (32 bytes)'
+    )
+  })
+
+  it('should throw error when ENCRYPTION_KEY is wrong length', async () => {
+    ;(env as Record<string, string>).ENCRYPTION_KEY = '0123456789abcdef'
+    await expect(encryptSecret('test')).rejects.toThrow(
+      'ENCRYPTION_KEY must be set to a 64-character hex string (32 bytes)'
+    )
+  })
+})

apps/sim/lib/core/security/encryption.ts

@@ -0,0 +1,53 @@
+import { createLogger } from '@sim/logger'
+import { decrypt, encrypt } from '@sim/security/encryption'
+import { toError } from '@sim/utils/errors'
+import { randomInt } from '@sim/utils/random'
+import { env } from '@/lib/core/config/env'
+
+const logger = createLogger('Encryption')
+
+function getEncryptionKey(): Buffer {
+  const key = env.ENCRYPTION_KEY
+  if (!key || key.length !== 64) {
+    throw new Error('ENCRYPTION_KEY must be set to a 64-character hex string (32 bytes)')
+  }
+  return Buffer.from(key, 'hex')
+}
+
+/**
+ * Encrypts a secret using AES-256-GCM with the app's `ENCRYPTION_KEY`.
+ * @param secret - The secret to encrypt
+ * @returns A promise resolving to the encrypted value (`iv:ciphertext:authTag`) and the IV.
+ */
+export async function encryptSecret(secret: string): Promise<{ encrypted: string; iv: string }> {
+  return encrypt(secret, getEncryptionKey())
+}
+
+/**
+ * Decrypts a secret previously produced by {@link encryptSecret}. Logs and
+ * rethrows on malformed input or tampered ciphertext.
+ */
+export async function decryptSecret(encryptedValue: string): Promise<{ decrypted: string }> {
+  try {
+    return await decrypt(encryptedValue, getEncryptionKey())
+  } catch (error) {
+    logger.error('Decryption error:', { error: toError(error).message })
+    throw error
+  }
+}
+
+/**
+ * Generates a secure random password
+ * @param length - The length of the password (default: 24)
+ * @returns A new secure password string
+ */
+export function generatePassword(length = 24): string {
+  const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_-+='
+  let result = ''
+
+  for (let i = 0; i < length; i++) {
+    result += chars.charAt(randomInt(0, chars.length))
+  }
+
+  return result
+}

apps/sim/lib/core/security/input-validation.server.ts

@@ -0,0 +1,720 @@
+import dns from 'dns/promises'
+import http from 'http'
+import https from 'https'
+import type { LookupFunction } from 'net'
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { omit } from '@sim/utils/object'
+import * as ipaddr from 'ipaddr.js'
+import { Agent, type RequestInit as UndiciRequestInit, fetch as undiciFetch } from 'undici'
+import { isHosted, isPrivateDatabaseHostsAllowed } from '@/lib/core/config/env-flags'
+import { type ValidationResult, validateExternalUrl } from '@/lib/core/security/input-validation'
+import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
+
+const logger = createLogger('InputValidation')
+
+/**
+ * Result type for async URL validation with resolved IP
+ */
+export interface AsyncValidationResult extends ValidationResult {
+  resolvedIP?: string
+  originalHostname?: string
+}
+
+/**
+ * Checks if an IP address is private or reserved (not routable on the public internet)
+ * Uses ipaddr.js for robust handling of all IP formats including:
+ * - Octal notation (0177.0.0.1)
+ * - Hex notation (0x7f000001)
+ * - IPv4-mapped IPv6 (::ffff:127.0.0.1)
+ * - IPv4-compatible IPv6 (::a.b.c.d / ::xxxx:xxxx, RFC 4291 §2.5.5.1, deprecated)
+ * - Various edge cases that regex patterns miss
+ */
+export function isPrivateOrReservedIP(ip: string): boolean {
+  try {
+    if (!ipaddr.isValid(ip)) {
+      return true
+    }
+
+    const addr = ipaddr.process(ip)
+    const range = addr.range()
+
+    if (range !== 'unicast') {
+      return true
+    }
+
+    if (addr.kind() === 'ipv6') {
+      const v6 = addr as ipaddr.IPv6
+      const parts = v6.parts
+      const firstSixZero = parts.slice(0, 6).every((p) => p === 0)
+      if (firstSixZero) {
+        const embedded = ipaddr.fromByteArray([
+          (parts[6] >> 8) & 0xff,
+          parts[6] & 0xff,
+          (parts[7] >> 8) & 0xff,
+          parts[7] & 0xff,
+        ])
+        return embedded.range() !== 'unicast'
+      }
+    }
+
+    return false
+  } catch {
+    return true
+  }
+}
+
+/**
+ * Validates a URL and resolves its DNS to prevent SSRF via DNS rebinding
+ *
+ * This function:
+ * 1. Performs basic URL validation (protocol, format)
+ * 2. Resolves the hostname to an IP address
+ * 3. Validates the resolved IP is not private/reserved
+ * 4. Returns the resolved IP for use in the actual request
+ *
+ * @param url - The URL to validate
+ * @param paramName - Name of the parameter for error messages
+ * @returns AsyncValidationResult with resolved IP for DNS pinning
+ */
+export async function validateUrlWithDNS(
+  url: string | null | undefined,
+  paramName = 'url',
+  options: { allowHttp?: boolean } = {}
+): Promise<AsyncValidationResult> {
+  const basicValidation = validateExternalUrl(url, paramName, options)
+  if (!basicValidation.isValid) {
+    return basicValidation
+  }
+
+  const parsedUrl = new URL(url!)
+  const hostname = parsedUrl.hostname
+
+  const hostnameLower = hostname.toLowerCase()
+  const cleanHostname =
+    hostnameLower.startsWith('[') && hostnameLower.endsWith(']')
+      ? hostnameLower.slice(1, -1)
+      : hostnameLower
+
+  let isLocalhost = cleanHostname === 'localhost'
+  if (ipaddr.isValid(cleanHostname)) {
+    const processedIP = ipaddr.process(cleanHostname).toString()
+    if (processedIP === '127.0.0.1' || processedIP === '::1') {
+      isLocalhost = true
+    }
+  }
+
+  try {
+    const { address } = await dns.lookup(cleanHostname, { verbatim: true })
+
+    const resolvedIsLoopback =
+      ipaddr.isValid(address) &&
+      (() => {
+        const ip = ipaddr.process(address).toString()
+        return ip === '127.0.0.1' || ip === '::1'
+      })()
+
+    if (isPrivateOrReservedIP(address) && !(isLocalhost && resolvedIsLoopback && !isHosted)) {
+      logger.warn('URL resolves to blocked IP address', {
+        paramName,
+        hostname,
+        resolvedIP: address,
+      })
+      return {
+        isValid: false,
+        error: `${paramName} resolves to a blocked IP address`,
+      }
+    }
+
+    return {
+      isValid: true,
+      resolvedIP: address,
+      originalHostname: hostname,
+    }
+  } catch (error) {
+    logger.warn('DNS lookup failed for URL', {
+      paramName,
+      hostname,
+      error: toError(error).message,
+    })
+    return {
+      isValid: false,
+      error: `${paramName} hostname could not be resolved`,
+    }
+  }
+}
+
+/**
+ * Validates a database hostname by resolving DNS and checking the resolved IP
+ * against private/reserved ranges to prevent SSRF via database connections.
+ *
+ * Unlike validateHostname (which enforces strict RFC hostname format), this
+ * function is permissive about hostname format to avoid breaking legitimate
+ * database hostnames (e.g. underscores in Docker/K8s service names). It only
+ * blocks localhost and private/reserved IPs.
+ *
+ * Self-hosted operators can set `ALLOW_PRIVATE_DATABASE_HOSTS` to reach databases
+ * on their private network (e.g. a Docker/Swarm service name that resolves to an
+ * internal IP). The opt-in only bypasses the private/reserved/loopback block; DNS
+ * is still resolved so the caller can pin the connection to the resolved IP. The
+ * bypass is never honored on the hosted platform (see {@link isPrivateDatabaseHostsAllowed}).
+ *
+ * @param host - The database hostname to validate
+ * @param paramName - Name of the parameter for error messages
+ * @returns AsyncValidationResult with resolved IP
+ */
+export async function validateDatabaseHost(
+  host: string | null | undefined,
+  paramName = 'host'
+): Promise<AsyncValidationResult> {
+  if (!host) {
+    return { isValid: false, error: `${paramName} is required` }
+  }
+
+  const lowerHost = host.toLowerCase()
+  const cleanHost =
+    lowerHost.startsWith('[') && lowerHost.endsWith(']') ? lowerHost.slice(1, -1) : lowerHost
+
+  if (cleanHost === 'localhost' && !isPrivateDatabaseHostsAllowed) {
+    return { isValid: false, error: `${paramName} cannot be localhost` }
+  }
+
+  if (
+    ipaddr.isValid(cleanHost) &&
+    isPrivateOrReservedIP(cleanHost) &&
+    !isPrivateDatabaseHostsAllowed
+  ) {
+    return { isValid: false, error: `${paramName} cannot be a private IP address` }
+  }
+
+  try {
+    const { address } = await dns.lookup(cleanHost, { verbatim: true })
+
+    if (isPrivateOrReservedIP(address) && !isPrivateDatabaseHostsAllowed) {
+      logger.warn('Database host resolves to blocked IP address', {
+        paramName,
+        hostname: host,
+        resolvedIP: address,
+      })
+      return {
+        isValid: false,
+        error: `${paramName} resolves to a blocked IP address`,
+      }
+    }
+
+    return {
+      isValid: true,
+      resolvedIP: address,
+      originalHostname: host,
+    }
+  } catch (error) {
+    logger.warn('DNS lookup failed for database host', {
+      paramName,
+      hostname: host,
+      error: toError(error).message,
+    })
+    return {
+      isValid: false,
+      error: `${paramName} hostname could not be resolved`,
+    }
+  }
+}
+
+/**
+ * Patterns run against the WHERE clause with string/identifier literals masked
+ * out (so an attacker cannot smuggle `OR 1` or `; DROP` inside a quoted value).
+ *
+ * The connector-literal rules below are intentionally `OR`-only: only an
+ * `OR <truthy>` term broadens a mutation to every row. `AND <number>` is a no-op
+ * for broadening and is also exactly what `BETWEEN low AND high` produces, so
+ * matching it would reject legitimate range filters (e.g. `id BETWEEN 1 AND 10`).
+ */
+const SQL_WHERE_MASKED_PATTERNS: readonly RegExp[] = [
+  /;\s*\w/, // stacked statement
+  /\bunion\s+(?:all\s+)?select\b/i,
+  /\binto\s+(?:out|dump)file\b/i,
+  /--/,
+  /\/\*/,
+  /\*\//,
+  /\b(?:sleep|pg_sleep|benchmark)\s*\(/i,
+  /\b(\w+)\s*=\s*\1\b/i, // same (unquoted) operand both sides: x=x, 1=1
+  /\b\d+(?:\.\d+)?\s*(?:=|==|<>|!=|<=|>=|<|>)\s*\d+(?:\.\d+)?\b/, // constant vs constant: 1=1, 1<2, 2>1
+  /\bor\s+(?:true|false)\b/i, // OR TRUE / OR FALSE
+  /\bor\s+\d+(?:\.\d+)?\b(?!\s*[=<>!+\-*/%])/i, // standalone truthy literal after OR: OR 1, OR 42
+  /^\s*(?:\d+(?:\.\d+)?|true|false)\s*$/i, // bare constant: "1" / "true" / "false"
+]
+
+/**
+ * Patterns run against the raw WHERE clause (need the literal contents intact),
+ * e.g. equality between two identical string literals.
+ */
+const SQL_WHERE_RAW_PATTERNS: readonly RegExp[] = [
+  /(['"])([^'"]*)\1\s*(?:=|==|<>|!=)\s*\1\2\1/, // 'a'='a' / "x"="x"
+]
+
+/**
+ * Replaces the contents of string literals ('...'), double-quoted and
+ * backtick-quoted identifiers with spaces (preserving length) so structural
+ * scans do not treat data inside quotes as SQL. Comments are intentionally left
+ * intact so comment-injection sequences are still detected.
+ */
+function maskSqlStringLiterals(sql: string): string {
+  let out = ''
+  let i = 0
+  while (i < sql.length) {
+    const ch = sql[i]
+    if (ch === "'" || ch === '"' || ch === '`') {
+      out += ' '
+      i++
+      while (i < sql.length && sql[i] !== ch) {
+        if (ch !== '`' && sql[i] === '\\') {
+          out += '  '
+          i += 2
+          continue
+        }
+        out += ' '
+        i++
+      }
+      if (i < sql.length) {
+        out += ' '
+        i++
+      }
+      continue
+    }
+    out += ch
+    i++
+  }
+  return out
+}
+
+/**
+ * Validates a free-form SQL `WHERE` condition for injection and always-true
+ * tautology patterns. Returns a {@link ValidationResult}; callers decide whether
+ * to throw or surface the error.
+ *
+ * IMPORTANT: this is **defense-in-depth, not a security boundary**. A free-form
+ * SQL condition cannot be exhaustively validated against every always-true
+ * expression (e.g. `OR 2 > 1`, `OR (1)`, `OR NOT 0`, `OR length(x) >= 0`). The
+ * real boundary is that the caller supplies their own database credentials and
+ * could run equivalent SQL directly (e.g. via a raw-SQL/execute operation). This
+ * guard stops the easy, obvious ways an injected condition broadens a mutation
+ * to every row; it is not a substitute for constraining untrusted input upstream.
+ *
+ * @param where - The WHERE clause condition (without the `WHERE` keyword)
+ * @param paramName - Label used in the error message
+ */
+export function validateSqlWhereClause(
+  where: string | null | undefined,
+  paramName = 'WHERE clause'
+): ValidationResult {
+  if (typeof where !== 'string' || where.trim().length === 0) {
+    return { isValid: false, error: `${paramName} is required` }
+  }
+
+  const masked = maskSqlStringLiterals(where)
+  const matched =
+    SQL_WHERE_MASKED_PATTERNS.some((pattern) => pattern.test(masked)) ||
+    SQL_WHERE_RAW_PATTERNS.some((pattern) => pattern.test(where))
+
+  if (matched) {
+    return {
+      isValid: false,
+      error: `${paramName} contains a disallowed or always-true expression`,
+    }
+  }
+
+  return { isValid: true }
+}
+
+export interface SecureFetchOptions {
+  method?: string
+  headers?: Record<string, string>
+  body?: string | Buffer | Uint8Array
+  timeout?: number
+  maxRedirects?: number
+  maxResponseBytes?: number
+  signal?: AbortSignal
+  /** Drop the Authorization header when following a redirect, so it is not sent to the redirect target's origin. */
+  stripAuthOnRedirect?: boolean
+}
+
+export class SecureFetchHeaders {
+  private headers: Map<string, string>
+  private setCookies: string[]
+
+  constructor(headers: Record<string, string>, setCookies: string[] = []) {
+    this.headers = new Map(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v]))
+    this.setCookies = setCookies
+  }
+
+  get(name: string): string | null {
+    return this.headers.get(name.toLowerCase()) ?? null
+  }
+
+  /** Returns the raw `Set-Cookie` header values as an array. Each entry is one cookie. */
+  getSetCookie(): string[] {
+    return [...this.setCookies]
+  }
+
+  toRecord(): Record<string, string> {
+    const record: Record<string, string> = {}
+    for (const [key, value] of this.headers) {
+      record[key] = value
+    }
+    return record
+  }
+
+  [Symbol.iterator]() {
+    return this.headers.entries()
+  }
+}
+
+export interface SecureFetchResponse {
+  ok: boolean
+  status: number
+  statusText: string
+  headers: SecureFetchHeaders
+  body: ReadableStream<Uint8Array> | null
+  text: () => Promise<string>
+  json: () => Promise<unknown>
+  arrayBuffer: () => Promise<ArrayBuffer>
+}
+
+const DEFAULT_MAX_REDIRECTS = 5
+
+function isRedirectStatus(status: number): boolean {
+  return status >= 300 && status < 400 && status !== 304
+}
+
+function isRetryableHttpStatus(status: number): boolean {
+  return status === 429 || (status >= 500 && status <= 599)
+}
+
+function resolveRedirectUrl(baseUrl: string, location: string): string {
+  try {
+    return new URL(location, baseUrl).toString()
+  } catch {
+    throw new Error(`Invalid redirect location: ${location}`)
+  }
+}
+
+/**
+ * Creates a DNS lookup function that always returns a pre-resolved IP address.
+ * Use this to prevent DNS rebinding (TOCTOU) attacks when connecting to
+ * user-controlled hostnames via non-HTTP protocols (SMTP, SSH, IMAP, etc.).
+ */
+export function createPinnedLookup(resolvedIP: string): LookupFunction {
+  const isIPv6 = resolvedIP.includes(':')
+  const family = isIPv6 ? 6 : 4
+
+  return (_hostname, options, callback) => {
+    if (options.all) {
+      callback(null, [{ address: resolvedIP, family }])
+    } else {
+      callback(null, resolvedIP, family)
+    }
+  }
+}
+
+/**
+ * Builds a standard `fetch`-compatible function that pins every outbound
+ * connection to `resolvedIP`, preventing DNS-rebinding (TOCTOU) between URL
+ * validation and connection. The original hostname is preserved for TLS SNI and
+ * the `Host` header so it still matches the certificate. This is the single
+ * source of truth for pinned outbound fetches — both the LLM providers and the
+ * MCP transport consume it.
+ *
+ * Pass the returned function as the `fetch` option to the OpenAI/Anthropic SDKs
+ * (or call it directly) after validating the URL with {@link validateUrlWithDNS}
+ * and capturing `resolvedIP`. Because the pinned lookup always returns
+ * `resolvedIP` regardless of hostname, any redirect the server returns also
+ * connects to the validated IP — an attacker cannot rebind a redirect target to
+ * an internal address.
+ *
+ * The `Agent` is captured for the lifetime of the returned function, so repeated
+ * calls (e.g. a provider tool loop) reuse its keep-alive connections.
+ */
+export function createPinnedFetch(resolvedIP: string): typeof fetch {
+  const dispatcher = new Agent({ connect: { lookup: createPinnedLookup(resolvedIP) } })
+
+  const pinned = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+    // double-cast-allowed: DOM RequestInfo/URL and undici fetch input types differ but are structurally compatible at runtime (Node's global fetch IS undici)
+    const undiciInput = input as unknown as Parameters<typeof undiciFetch>[0]
+    // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ
+    const undiciInit: UndiciRequestInit = { ...(init as unknown as UndiciRequestInit), dispatcher }
+    const response = await undiciFetch(undiciInput, undiciInit)
+    // double-cast-allowed: undici Response and DOM Response are structurally compatible at runtime
+    return response as unknown as Response
+  }
+
+  return pinned
+}
+
+/**
+ * Performs a fetch with IP pinning to prevent DNS rebinding attacks.
+ * Uses the pre-resolved IP address while preserving the original hostname for TLS SNI.
+ * Follows redirects securely by validating each redirect target.
+ */
+export async function secureFetchWithPinnedIP(
+  url: string,
+  resolvedIP: string,
+  options: SecureFetchOptions & { allowHttp?: boolean } = {},
+  redirectCount = 0
+): Promise<SecureFetchResponse> {
+  const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS
+  const maxResponseBytes = options.maxResponseBytes
+
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url)
+    const isHttps = parsed.protocol === 'https:'
+    const defaultPort = isHttps ? 443 : 80
+    const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort
+
+    const lookup = createPinnedLookup(resolvedIP)
+
+    const agentOptions: http.AgentOptions = { lookup }
+
+    const agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions)
+
+    const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {}
+
+    const requestOptions: http.RequestOptions = {
+      hostname: parsed.hostname,
+      port,
+      path: parsed.pathname + parsed.search,
+      method: options.method || 'GET',
+      headers: sanitizedHeaders,
+      agent,
+      timeout: options.timeout || 300000,
+    }
+
+    const protocol = isHttps ? https : http
+    const req = protocol.request(requestOptions, (res) => {
+      const statusCode = res.statusCode || 0
+      const location = res.headers.location
+
+      if (isRedirectStatus(statusCode) && location && redirectCount < maxRedirects) {
+        res.resume()
+        const redirectUrl = resolveRedirectUrl(url, location)
+
+        validateUrlWithDNS(redirectUrl, 'redirectUrl', { allowHttp: options.allowHttp })
+          .then((validation) => {
+            if (!validation.isValid) {
+              settledReject(new Error(`Redirect blocked: ${validation.error}`))
+              return
+            }
+            const redirectOptions = options.stripAuthOnRedirect
+              ? {
+                  ...options,
+                  headers: omit(options.headers ?? {}, ['Authorization', 'authorization']),
+                }
+              : options
+            return secureFetchWithPinnedIP(
+              redirectUrl,
+              validation.resolvedIP!,
+              redirectOptions,
+              redirectCount + 1
+            )
+          })
+          .then((response) => {
+            if (response) settledResolve(response)
+          })
+          .catch(settledReject)
+        return
+      }
+
+      if (isRedirectStatus(statusCode) && location && redirectCount >= maxRedirects) {
+        res.resume()
+        settledReject(new Error(`Too many redirects (max: ${maxRedirects})`))
+        return
+      }
+
+      const headersRecord: Record<string, string> = {}
+      let setCookieArray: string[] = []
+      for (const [key, value] of Object.entries(res.headers)) {
+        const lowerKey = key.toLowerCase()
+        if (lowerKey === 'set-cookie') {
+          if (Array.isArray(value)) {
+            setCookieArray = value
+            headersRecord[lowerKey] = value.join(', ')
+          } else if (typeof value === 'string') {
+            setCookieArray = [value]
+            headersRecord[lowerKey] = value
+          }
+        } else if (typeof value === 'string') {
+          headersRecord[lowerKey] = value
+        } else if (Array.isArray(value)) {
+          headersRecord[lowerKey] = value.join(', ')
+        }
+      }
+
+      const contentLength = headersRecord['content-length']
+      if (typeof maxResponseBytes === 'number' && maxResponseBytes > 0 && contentLength) {
+        const parsedLength = Number.parseInt(contentLength, 10)
+        if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) {
+          cleanupAbort()
+          res.destroy()
+          req.destroy()
+          if (isRetryableHttpStatus(statusCode)) {
+            settledResolve({
+              ok: false,
+              status: statusCode,
+              statusText: res.statusMessage || '',
+              headers: new SecureFetchHeaders(headersRecord, setCookieArray),
+              body: null,
+              text: async () => '',
+              json: async () => ({}),
+              arrayBuffer: async () => new ArrayBuffer(0),
+            })
+            return
+          }
+          settledReject(
+            new PayloadSizeLimitError({
+              label: 'response body',
+              maxBytes: maxResponseBytes,
+              observedBytes: parsedLength,
+            })
+          )
+          return
+        }
+      }
+
+      let totalBytes = 0
+      const nodeRes = res
+      const body = new ReadableStream<Uint8Array>({
+        start(controller) {
+          nodeRes.on('data', (chunk: Buffer) => {
+            totalBytes += chunk.length
+            if (
+              typeof maxResponseBytes === 'number' &&
+              maxResponseBytes > 0 &&
+              totalBytes > maxResponseBytes
+            ) {
+              cleanupAbort()
+              controller.error(
+                new PayloadSizeLimitError({
+                  label: 'response body',
+                  maxBytes: maxResponseBytes,
+                  observedBytes: totalBytes,
+                })
+              )
+              nodeRes.destroy()
+              return
+            }
+            controller.enqueue(new Uint8Array(chunk))
+          })
+          nodeRes.on('end', () => {
+            cleanupAbort()
+            controller.close()
+          })
+          nodeRes.on('error', (err) => {
+            cleanupAbort()
+            controller.error(err)
+          })
+        },
+        cancel() {
+          cleanupAbort()
+          nodeRes.destroy()
+        },
+      })
+
+      let bodyBufferPromise: Promise<Buffer> | null = null
+      function readBodyAsBuffer(): Promise<Buffer> {
+        if (!bodyBufferPromise) {
+          bodyBufferPromise = (async () => {
+            const reader = body.getReader()
+            const buffers: Uint8Array[] = []
+            while (true) {
+              const { done, value } = await reader.read()
+              if (done) break
+              if (value) buffers.push(value)
+            }
+            return Buffer.concat(buffers.map((b) => Buffer.from(b)))
+          })()
+        }
+        return bodyBufferPromise
+      }
+
+      settledResolve({
+        ok: statusCode >= 200 && statusCode < 300,
+        status: statusCode,
+        statusText: res.statusMessage || '',
+        headers: new SecureFetchHeaders(headersRecord, setCookieArray),
+        body,
+        text: async () => (await readBodyAsBuffer()).toString('utf-8'),
+        json: async () => JSON.parse((await readBodyAsBuffer()).toString('utf-8')),
+        arrayBuffer: async () => {
+          const buf = await readBodyAsBuffer()
+          return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer
+        },
+      })
+    })
+
+    let onAbort: (() => void) | null = null
+    const cleanupAbort = () => {
+      if (onAbort && options.signal) {
+        options.signal.removeEventListener('abort', onAbort)
+        onAbort = null
+      }
+    }
+    const settledResolve: typeof resolve = (value) => {
+      resolve(value)
+    }
+    const settledReject: typeof reject = (reason) => {
+      cleanupAbort()
+      reject(reason)
+    }
+
+    req.on('error', (error) => {
+      settledReject(error)
+    })
+
+    req.on('timeout', () => {
+      req.destroy()
+      settledReject(new Error(`Request timed out after ${requestOptions.timeout}ms`))
+    })
+
+    if (options.signal) {
+      if (options.signal.aborted) {
+        req.destroy()
+        settledReject(options.signal.reason ?? new Error('Aborted'))
+        return
+      }
+      onAbort = () => {
+        req.destroy()
+        settledReject(options.signal?.reason ?? new Error('Aborted'))
+      }
+      options.signal.addEventListener('abort', onAbort, { once: true })
+    }
+
+    if (options.body) {
+      req.write(options.body)
+    }
+
+    req.end()
+  })
+}
+
+/**
+ * Validates a URL and performs a secure fetch with DNS pinning in one call.
+ * Combines validateUrlWithDNS and secureFetchWithPinnedIP for convenience.
+ *
+ * @param url - The URL to fetch
+ * @param options - Fetch options (method, headers, body, etc.)
+ * @param paramName - Name of the parameter for error messages (default: 'url')
+ * @returns SecureFetchResponse
+ * @throws Error if URL validation fails
+ */
+export async function secureFetchWithValidation(
+  url: string,
+  options: SecureFetchOptions & { allowHttp?: boolean } = {},
+  paramName = 'url'
+): Promise<SecureFetchResponse> {
+  const validation = await validateUrlWithDNS(url, paramName, {
+    allowHttp: options.allowHttp,
+  })
+  if (!validation.isValid) {
+    throw new Error(validation.error)
+  }
+  return secureFetchWithPinnedIP(url, validation.resolvedIP!, options)
+}

apps/sim/lib/core/security/otp.ts

@@ -0,0 +1,250 @@
+import { randomInt } from 'crypto'
+import { db } from '@sim/db'
+import { verification } from '@sim/db/schema'
+import { generateId } from '@sim/utils/id'
+import { and, eq, gt } from 'drizzle-orm'
+import { getRedisClient } from '@/lib/core/config/redis'
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter'
+import { getStorageMethod } from '@/lib/core/storage'
+
+export type DeploymentKind = 'chat' | 'file'
+
+/**
+ * Shared OTP configuration for deployment email-auth gates (chat + public file shares).
+ */
+export const OTP_EXPIRY_SECONDS = 15 * 60
+export const OTP_EXPIRY_MS = OTP_EXPIRY_SECONDS * 1000
+export const MAX_OTP_ATTEMPTS = 5
+
+export const OTP_IP_RATE_LIMIT: TokenBucketConfig = {
+  maxTokens: 10,
+  refillRate: 10,
+  refillIntervalMs: 15 * 60_000,
+}
+
+export const OTP_EMAIL_RATE_LIMIT: TokenBucketConfig = {
+  maxTokens: 3,
+  refillRate: 3,
+  refillIntervalMs: 15 * 60_000,
+}
+
+/**
+ * Key formats are kept per-kind to preserve any in-flight OTPs already issued
+ * against existing chat deployments. The chat Redis key uses the legacy `otp:`
+ * prefix; the chat DB identifier uses `chat-otp:`.
+ */
+const OTP_KEYS = {
+  chat: {
+    redisKey: (email: string, deploymentId: string) => `otp:${email}:${deploymentId}`,
+    dbIdentifier: (email: string, deploymentId: string) => `chat-otp:${deploymentId}:${email}`,
+  },
+  file: {
+    redisKey: (email: string, deploymentId: string) => `otp:file:${email}:${deploymentId}`,
+    dbIdentifier: (email: string, deploymentId: string) => `file-otp:${deploymentId}:${email}`,
+  },
+} as const satisfies Record<
+  DeploymentKind,
+  {
+    redisKey: (email: string, deploymentId: string) => string
+    dbIdentifier: (email: string, deploymentId: string) => string
+  }
+>
+
+/** Returns a cryptographically random 6-digit OTP code. */
+export function generateOTP(): string {
+  return randomInt(100000, 1000000).toString()
+}
+
+/**
+ * OTP values are stored as `"code:attempts"` (e.g. `"654321:0"`).
+ * This keeps the attempt counter in the same key/row as the OTP itself.
+ */
+function encodeOTPValue(otp: string, attempts: number): string {
+  return `${otp}:${attempts}`
+}
+
+export function decodeOTPValue(value: string): { otp: string; attempts: number } {
+  const lastColon = value.lastIndexOf(':')
+  if (lastColon === -1) return { otp: value, attempts: 0 }
+  const attempts = Number.parseInt(value.slice(lastColon + 1), 10)
+  return { otp: value.slice(0, lastColon), attempts: Number.isNaN(attempts) ? 0 : attempts }
+}
+
+/**
+ * Stores an OTP for a deployment+email pair, choosing Redis or the
+ * `verification` table based on the configured storage method.
+ */
+export async function storeOTP(
+  kind: DeploymentKind,
+  deploymentId: string,
+  email: string,
+  otp: string
+): Promise<void> {
+  const keys = OTP_KEYS[kind]
+  const value = encodeOTPValue(otp, 0)
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) throw new Error('Redis configured but client unavailable')
+    await redis.set(keys.redisKey(email, deploymentId), value, 'EX', OTP_EXPIRY_SECONDS)
+    return
+  }
+
+  const now = new Date()
+  const expiresAt = new Date(now.getTime() + OTP_EXPIRY_MS)
+  const identifier = keys.dbIdentifier(email, deploymentId)
+
+  await db.transaction(async (tx) => {
+    await tx.delete(verification).where(eq(verification.identifier, identifier))
+    await tx.insert(verification).values({
+      id: generateId(),
+      identifier,
+      value,
+      expiresAt,
+      createdAt: now,
+      updatedAt: now,
+    })
+  })
+}
+
+export async function getOTP(
+  kind: DeploymentKind,
+  deploymentId: string,
+  email: string
+): Promise<string | null> {
+  const keys = OTP_KEYS[kind]
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) throw new Error('Redis configured but client unavailable')
+    return redis.get(keys.redisKey(email, deploymentId))
+  }
+
+  const now = new Date()
+  const [record] = await db
+    .select({ value: verification.value })
+    .from(verification)
+    .where(
+      and(
+        eq(verification.identifier, keys.dbIdentifier(email, deploymentId)),
+        gt(verification.expiresAt, now)
+      )
+    )
+    .limit(1)
+
+  return record?.value ?? null
+}
+
+/**
+ * Lua script for atomic OTP attempt increment in Redis.
+ * Returns `'LOCKED'` if max attempts reached (key deleted), new encoded value
+ * otherwise, nil if key missing.
+ */
+const ATOMIC_INCREMENT_SCRIPT = `
+local val = redis.call('GET', KEYS[1])
+if not val then return nil end
+local colon = val:find(':([^:]*$)')
+local otp, attempts
+if colon then
+  otp = val:sub(1, colon - 1)
+  attempts = tonumber(val:sub(colon + 1)) or 0
+else
+  otp = val
+  attempts = 0
+end
+attempts = attempts + 1
+if attempts >= tonumber(ARGV[1]) then
+  redis.call('DEL', KEYS[1])
+  return 'LOCKED'
+end
+local newVal = otp .. ':' .. attempts
+local ttl = redis.call('TTL', KEYS[1])
+if ttl > 0 then
+  redis.call('SET', KEYS[1], newVal, 'EX', ttl)
+else
+  redis.call('SET', KEYS[1], newVal)
+end
+return newVal
+`
+
+/**
+ * Atomically increments an OTP's failed-attempt counter. Returns `'locked'`
+ * if the max-attempts threshold was reached (and the OTP was deleted), or
+ * `'incremented'` otherwise. The DB path uses optimistic locking with retry.
+ */
+export async function incrementOTPAttempts(
+  kind: DeploymentKind,
+  deploymentId: string,
+  email: string,
+  currentValue: string
+): Promise<'locked' | 'incremented'> {
+  const keys = OTP_KEYS[kind]
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) throw new Error('Redis configured but client unavailable')
+    const key = keys.redisKey(email, deploymentId)
+    const result = await redis.eval(ATOMIC_INCREMENT_SCRIPT, 1, key, MAX_OTP_ATTEMPTS)
+    if (result === null || result === 'LOCKED') return 'locked'
+    return 'incremented'
+  }
+
+  const identifier = keys.dbIdentifier(email, deploymentId)
+  const MAX_RETRIES = 3
+  let value = currentValue
+
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const { otp, attempts } = decodeOTPValue(value)
+    const newAttempts = attempts + 1
+
+    if (newAttempts >= MAX_OTP_ATTEMPTS) {
+      await db.delete(verification).where(eq(verification.identifier, identifier))
+      return 'locked'
+    }
+
+    const newValue = encodeOTPValue(otp, newAttempts)
+    const updated = await db
+      .update(verification)
+      .set({ value: newValue, updatedAt: new Date() })
+      .where(and(eq(verification.identifier, identifier), eq(verification.value, value)))
+      .returning({ id: verification.id })
+
+    if (updated.length > 0) return 'incremented'
+
+    const fresh = await getOTP(kind, deploymentId, email)
+    if (!fresh) return 'locked'
+    value = fresh
+  }
+
+  /**
+   * Retry exhaustion under heavy DB-path contention: this request did not
+   * succeed in writing its own +1, so the stored count may not reflect it.
+   * Fail closed — invalidate the OTP rather than return `'incremented'` with
+   * a possibly-undercounted attempt total.
+   */
+  await db.delete(verification).where(eq(verification.identifier, identifier))
+  return 'locked'
+}
+
+export async function deleteOTP(
+  kind: DeploymentKind,
+  deploymentId: string,
+  email: string
+): Promise<void> {
+  const keys = OTP_KEYS[kind]
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) throw new Error('Redis configured but client unavailable')
+    await redis.del(keys.redisKey(email, deploymentId))
+    return
+  }
+
+  await db
+    .delete(verification)
+    .where(eq(verification.identifier, keys.dbIdentifier(email, deploymentId)))
+}

apps/sim/lib/core/security/pinned-fetch.server.test.ts

@@ -0,0 +1,126 @@
+/**
+ * @vitest-environment node
+ */
+import { envFlagsMock } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockAgent, mockUndiciFetch, capturedAgentOptions, agentCloses } = vi.hoisted(() => {
+  const capturedAgentOptions: unknown[] = []
+  const agentCloses: unknown[] = []
+  class MockAgent {
+    constructor(options: unknown) {
+      capturedAgentOptions.push(options)
+    }
+    close() {
+      agentCloses.push(this)
+      return Promise.resolve()
+    }
+  }
+  return {
+    mockAgent: MockAgent,
+    mockUndiciFetch: vi.fn(),
+    capturedAgentOptions,
+    agentCloses,
+  }
+})
+
+vi.mock('undici', () => ({ Agent: mockAgent, fetch: mockUndiciFetch }))
+vi.mock('@/lib/core/config/env-flags', () => envFlagsMock)
+
+import { createPinnedFetch } from '@/lib/core/security/input-validation.server'
+
+type LookupCallback = (err: Error | null, address: string, family: number) => void
+type PinnedLookup = (hostname: string, options: { all?: boolean }, callback: LookupCallback) => void
+
+describe('createPinnedFetch', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    capturedAgentOptions.length = 0
+    agentCloses.length = 0
+    mockUndiciFetch.mockResolvedValue(new Response('ok'))
+  })
+
+  it('builds an undici Agent whose pinned lookup always resolves to the validated IP', async () => {
+    createPinnedFetch('203.0.113.10')
+
+    expect(capturedAgentOptions).toHaveLength(1)
+    const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } }
+    expect(typeof connect.lookup).toBe('function')
+
+    const resolved = await new Promise<{ address: string; family: number }>((resolve) => {
+      connect.lookup('rebind.attacker.tld', {}, (_err, address, family) =>
+        resolve({ address, family })
+      )
+    })
+    expect(resolved).toEqual({ address: '203.0.113.10', family: 4 })
+  })
+
+  it('uses IPv6 family when the validated IP is IPv6', async () => {
+    createPinnedFetch('2606:4700:4700::1111')
+    const { connect } = capturedAgentOptions[0] as { connect: { lookup: PinnedLookup } }
+    const resolved = await new Promise<{ address: string; family: number }>((resolve) => {
+      connect.lookup('example.com', {}, (_err, address, family) => resolve({ address, family }))
+    })
+    expect(resolved).toEqual({ address: '2606:4700:4700::1111', family: 6 })
+  })
+
+  it('forwards the pinned dispatcher on every call while preserving init options', async () => {
+    const pinned = createPinnedFetch('203.0.113.10')
+    const controller = new AbortController()
+
+    await pinned('https://myresource.openai.azure.com/openai/v1/responses', {
+      method: 'POST',
+      headers: { 'api-key': 'secret' },
+      body: '{}',
+      signal: controller.signal,
+    })
+
+    expect(mockUndiciFetch).toHaveBeenCalledTimes(1)
+    const [url, init] = mockUndiciFetch.mock.calls[0]
+    expect(url).toBe('https://myresource.openai.azure.com/openai/v1/responses')
+    const typedInit = init as RequestInit & { dispatcher?: unknown }
+    expect(typedInit.dispatcher).toBeInstanceOf(mockAgent)
+    expect(typedInit.method).toBe('POST')
+    expect(typedInit.headers).toEqual({ 'api-key': 'secret' })
+    expect(typedInit.body).toBe('{}')
+    expect(typedInit.signal).toBe(controller.signal)
+  })
+
+  it('handles an undefined init by still attaching the dispatcher', async () => {
+    const pinned = createPinnedFetch('203.0.113.10')
+    await pinned('https://example.com')
+    const init = mockUndiciFetch.mock.calls[0][1] as { dispatcher?: unknown }
+    expect(init.dispatcher).toBeInstanceOf(mockAgent)
+  })
+
+  it('reuses one captured dispatcher across all calls of a single instance', async () => {
+    const pinned = createPinnedFetch('203.0.113.10')
+    await pinned('https://example.com/a')
+    await pinned('https://example.com/b')
+
+    expect(capturedAgentOptions).toHaveLength(1)
+    const d1 = (mockUndiciFetch.mock.calls[0][1] as { dispatcher: unknown }).dispatcher
+    const d2 = (mockUndiciFetch.mock.calls[1][1] as { dispatcher: unknown }).dispatcher
+    expect(d1).toBe(d2)
+  })
+
+  it('creates an independent dispatcher per instance', async () => {
+    const a = createPinnedFetch('203.0.113.10')
+    const b = createPinnedFetch('203.0.113.10')
+    await a('https://example.com/a')
+    await b('https://example.com/b')
+
+    expect(capturedAgentOptions).toHaveLength(2)
+    const d1 = (mockUndiciFetch.mock.calls[0][1] as { dispatcher: unknown }).dispatcher
+    const d2 = (mockUndiciFetch.mock.calls[1][1] as { dispatcher: unknown }).dispatcher
+    expect(d1).not.toBe(d2)
+  })
+
+  it('returns the response produced by undici fetch', async () => {
+    mockUndiciFetch.mockResolvedValueOnce(new Response('pong', { status: 201 }))
+    const pinned = createPinnedFetch('203.0.113.10')
+    const response = await pinned('https://example.com')
+    expect(response.status).toBe(201)
+    expect(await response.text()).toBe('pong')
+  })
+})

apps/sim/lib/core/security/redaction.test.ts

@@ -0,0 +1,771 @@
+import { describe, expect, it } from 'vitest'
+import {
+  isLargeDataKey,
+  isSensitiveKey,
+  REDACTED_MARKER,
+  redactApiKeys,
+  redactSensitiveValues,
+  sanitizeEventData,
+  sanitizeForLogging,
+  TRUNCATED_MARKER,
+} from './redaction'
+
+/**
+ * Security-focused edge case tests for redaction utilities
+ */
+
+describe('REDACTED_MARKER', () => {
+  it.concurrent('should be the standard marker', () => {
+    expect(REDACTED_MARKER).toBe('[REDACTED]')
+  })
+})
+
+describe('TRUNCATED_MARKER', () => {
+  it.concurrent('should be the standard marker', () => {
+    expect(TRUNCATED_MARKER).toBe('[TRUNCATED]')
+  })
+})
+
+describe('isLargeDataKey', () => {
+  it.concurrent('should identify base64 as large data key', () => {
+    expect(isLargeDataKey('base64')).toBe(true)
+  })
+
+  it.concurrent('should not identify other keys as large data', () => {
+    expect(isLargeDataKey('content')).toBe(false)
+    expect(isLargeDataKey('data')).toBe(false)
+    expect(isLargeDataKey('base')).toBe(false)
+  })
+})
+
+describe('isSensitiveKey', () => {
+  describe('exact matches', () => {
+    it.concurrent('should match apiKey variations', () => {
+      expect(isSensitiveKey('apiKey')).toBe(true)
+      expect(isSensitiveKey('api_key')).toBe(true)
+      expect(isSensitiveKey('api-key')).toBe(true)
+      expect(isSensitiveKey('APIKEY')).toBe(true)
+      expect(isSensitiveKey('API_KEY')).toBe(true)
+    })
+
+    it.concurrent('should match token variations', () => {
+      expect(isSensitiveKey('access_token')).toBe(true)
+      expect(isSensitiveKey('refresh_token')).toBe(true)
+      expect(isSensitiveKey('auth_token')).toBe(true)
+      expect(isSensitiveKey('accessToken')).toBe(true)
+    })
+
+    it.concurrent('should match secret variations', () => {
+      expect(isSensitiveKey('client_secret')).toBe(true)
+      expect(isSensitiveKey('clientSecret')).toBe(true)
+      expect(isSensitiveKey('secret')).toBe(true)
+    })
+
+    it.concurrent('should match other sensitive keys', () => {
+      expect(isSensitiveKey('private_key')).toBe(true)
+      expect(isSensitiveKey('authorization')).toBe(true)
+      expect(isSensitiveKey('bearer')).toBe(true)
+      expect(isSensitiveKey('private')).toBe(true)
+      expect(isSensitiveKey('auth')).toBe(true)
+      expect(isSensitiveKey('password')).toBe(true)
+      expect(isSensitiveKey('credential')).toBe(true)
+    })
+  })
+
+  describe('suffix matches', () => {
+    it.concurrent('should match keys ending in secret', () => {
+      expect(isSensitiveKey('clientSecret')).toBe(true)
+      expect(isSensitiveKey('appSecret')).toBe(true)
+      expect(isSensitiveKey('mySecret')).toBe(true)
+    })
+
+    it.concurrent('should match keys ending in password', () => {
+      expect(isSensitiveKey('userPassword')).toBe(true)
+      expect(isSensitiveKey('dbPassword')).toBe(true)
+      expect(isSensitiveKey('adminPassword')).toBe(true)
+    })
+
+    it.concurrent('should match keys ending in token', () => {
+      expect(isSensitiveKey('accessToken')).toBe(true)
+      expect(isSensitiveKey('refreshToken')).toBe(true)
+      expect(isSensitiveKey('bearerToken')).toBe(true)
+    })
+
+    it.concurrent('should match keys ending in credential', () => {
+      expect(isSensitiveKey('userCredential')).toBe(true)
+      expect(isSensitiveKey('dbCredential')).toBe(true)
+    })
+  })
+
+  describe('non-sensitive keys (no false positives)', () => {
+    it.concurrent('should not match keys with sensitive words as prefix only', () => {
+      expect(isSensitiveKey('tokenCount')).toBe(false)
+      expect(isSensitiveKey('tokenizer')).toBe(false)
+      expect(isSensitiveKey('secretKey')).toBe(false)
+      expect(isSensitiveKey('passwordStrength')).toBe(false)
+      expect(isSensitiveKey('authMethod')).toBe(false)
+    })
+
+    it.concurrent('should match keys ending with sensitive words (intentional)', () => {
+      expect(isSensitiveKey('hasSecret')).toBe(true)
+      expect(isSensitiveKey('userPassword')).toBe(true)
+      expect(isSensitiveKey('sessionToken')).toBe(true)
+    })
+
+    it.concurrent('should not match normal field names', () => {
+      expect(isSensitiveKey('name')).toBe(false)
+      expect(isSensitiveKey('email')).toBe(false)
+      expect(isSensitiveKey('id')).toBe(false)
+      expect(isSensitiveKey('value')).toBe(false)
+      expect(isSensitiveKey('data')).toBe(false)
+      expect(isSensitiveKey('count')).toBe(false)
+      expect(isSensitiveKey('status')).toBe(false)
+    })
+  })
+})
+
+describe('redactSensitiveValues', () => {
+  it.concurrent('should redact Bearer tokens', () => {
+    const input = 'Authorization: Bearer abc123xyz456'
+    const result = redactSensitiveValues(input)
+    expect(result).toBe('Authorization: Bearer [REDACTED]')
+    expect(result).not.toContain('abc123xyz456')
+  })
+
+  it.concurrent('should redact Basic auth', () => {
+    const input = 'Authorization: Basic dXNlcjpwYXNz'
+    const result = redactSensitiveValues(input)
+    expect(result).toBe('Authorization: Basic [REDACTED]')
+  })
+
+  it.concurrent('should redact API key prefixes', () => {
+    const input = 'Using key sk-1234567890abcdefghijklmnop'
+    const result = redactSensitiveValues(input)
+    expect(result).toContain('[REDACTED]')
+    expect(result).not.toContain('sk-1234567890abcdefghijklmnop')
+  })
+
+  it.concurrent('should redact JSON-style password fields', () => {
+    const input = 'password: "mysecretpass123"'
+    const result = redactSensitiveValues(input)
+    expect(result).toContain('[REDACTED]')
+    expect(result).not.toContain('mysecretpass123')
+  })
+
+  it.concurrent('should redact JSON-style token fields', () => {
+    const input = 'token: "tokenvalue123"'
+    const result = redactSensitiveValues(input)
+    expect(result).toContain('[REDACTED]')
+    expect(result).not.toContain('tokenvalue123')
+  })
+
+  it.concurrent('should redact JSON-style api_key fields', () => {
+    const input = 'api_key: "key123456"'
+    const result = redactSensitiveValues(input)
+    expect(result).toContain('[REDACTED]')
+    expect(result).not.toContain('key123456')
+  })
+
+  it.concurrent('should not modify safe strings', () => {
+    const input = 'This is a normal string with no secrets'
+    const result = redactSensitiveValues(input)
+    expect(result).toBe(input)
+  })
+
+  it.concurrent('should handle empty strings', () => {
+    expect(redactSensitiveValues('')).toBe('')
+  })
+
+  it.concurrent('should handle null/undefined gracefully', () => {
+    expect(redactSensitiveValues(null as any)).toBe(null)
+    expect(redactSensitiveValues(undefined as any)).toBe(undefined)
+  })
+})
+
+describe('redactApiKeys', () => {
+  describe('object redaction', () => {
+    it.concurrent('should redact sensitive keys in flat objects', () => {
+      const obj = {
+        apiKey: 'secret-key',
+        api_key: 'another-secret',
+        access_token: 'token-value',
+        secret: 'secret-value',
+        password: 'password-value',
+        normalField: 'normal-value',
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.apiKey).toBe('[REDACTED]')
+      expect(result.api_key).toBe('[REDACTED]')
+      expect(result.access_token).toBe('[REDACTED]')
+      expect(result.secret).toBe('[REDACTED]')
+      expect(result.password).toBe('[REDACTED]')
+      expect(result.normalField).toBe('normal-value')
+    })
+
+    it.concurrent('should redact sensitive keys in nested objects', () => {
+      const obj = {
+        config: {
+          apiKey: 'secret-key',
+          normalField: 'normal-value',
+        },
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.config.apiKey).toBe('[REDACTED]')
+      expect(result.config.normalField).toBe('normal-value')
+    })
+
+    it.concurrent('should redact sensitive keys in arrays', () => {
+      const arr = [{ apiKey: 'secret-key-1' }, { apiKey: 'secret-key-2' }]
+
+      const result = redactApiKeys(arr)
+
+      expect(result[0].apiKey).toBe('[REDACTED]')
+      expect(result[1].apiKey).toBe('[REDACTED]')
+    })
+
+    it.concurrent('should handle deeply nested structures', () => {
+      const obj = {
+        users: [
+          {
+            name: 'John',
+            credentials: {
+              apiKey: 'secret-key',
+              username: 'john_doe',
+            },
+          },
+        ],
+        config: {
+          database: {
+            password: 'db-password',
+            host: 'localhost',
+          },
+        },
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.users[0].name).toBe('John')
+      expect(result.users[0].credentials.apiKey).toBe('[REDACTED]')
+      expect(result.users[0].credentials.username).toBe('john_doe')
+      expect(result.config.database.password).toBe('[REDACTED]')
+      expect(result.config.database.host).toBe('localhost')
+    })
+
+    it.concurrent('should truncate base64 fields', () => {
+      const obj = {
+        id: 'file-123',
+        name: 'document.pdf',
+        base64: 'VGhpcyBpcyBhIHZlcnkgbG9uZyBiYXNlNjQgc3RyaW5n...',
+        size: 12345,
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.id).toBe('file-123')
+      expect(result.name).toBe('document.pdf')
+      expect(result.base64).toBe('[TRUNCATED]')
+      expect(result.size).toBe(12345)
+    })
+
+    it.concurrent('should truncate base64 in nested UserFile objects', () => {
+      const obj = {
+        files: [
+          {
+            id: 'file-1',
+            name: 'doc1.pdf',
+            url: 'http://example.com/file1',
+            size: 1000,
+            base64: 'base64content1...',
+          },
+          {
+            id: 'file-2',
+            name: 'doc2.pdf',
+            url: 'http://example.com/file2',
+            size: 2000,
+            base64: 'base64content2...',
+          },
+        ],
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.files[0].id).toBe('file-1')
+      expect(result.files[0].base64).toBe('[TRUNCATED]')
+      expect(result.files[1].base64).toBe('[TRUNCATED]')
+    })
+
+    it.concurrent('should filter UserFile objects to only expose allowed fields', () => {
+      const obj = {
+        processedFiles: [
+          {
+            id: 'file-123',
+            name: 'document.pdf',
+            url: 'http://localhost/api/files/serve/...',
+            size: 12345,
+            type: 'application/pdf',
+            key: 'execution/workspace/workflow/file.pdf',
+            context: 'execution',
+            base64: 'VGhpcyBpcyBhIGJhc2U2NCBzdHJpbmc=',
+          },
+        ],
+      }
+
+      const result = redactApiKeys(obj)
+
+      // Exposed fields should be present
+      expect(result.processedFiles[0].id).toBe('file-123')
+      expect(result.processedFiles[0].name).toBe('document.pdf')
+      expect(result.processedFiles[0].url).toBe('http://localhost/api/files/serve/...')
+      expect(result.processedFiles[0].size).toBe(12345)
+      expect(result.processedFiles[0].type).toBe('application/pdf')
+      expect(result.processedFiles[0].base64).toBe('[TRUNCATED]')
+
+      // Internal fields should be filtered out
+      expect(result.processedFiles[0]).not.toHaveProperty('key')
+      expect(result.processedFiles[0]).not.toHaveProperty('context')
+    })
+  })
+
+  describe('primitive handling', () => {
+    it.concurrent('should return primitives unchanged', () => {
+      expect(redactApiKeys('string')).toBe('string')
+      expect(redactApiKeys(123)).toBe(123)
+      expect(redactApiKeys(true)).toBe(true)
+      expect(redactApiKeys(null)).toBe(null)
+      expect(redactApiKeys(undefined)).toBe(undefined)
+    })
+  })
+
+  describe('no false positives', () => {
+    it.concurrent('should not redact keys with sensitive words as prefix only', () => {
+      const obj = {
+        tokenCount: 100,
+        secretKey: 'not-actually-secret',
+        passwordStrength: 'strong',
+        authMethod: 'oauth',
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.tokenCount).toBe(100)
+      expect(result.secretKey).toBe('not-actually-secret')
+      expect(result.passwordStrength).toBe('strong')
+      expect(result.authMethod).toBe('oauth')
+    })
+  })
+})
+
+describe('sanitizeForLogging', () => {
+  it.concurrent('should truncate long strings', () => {
+    const longString = 'a'.repeat(200)
+    const result = sanitizeForLogging(longString, 50)
+    expect(result.length).toBe(50)
+  })
+
+  it.concurrent('should use default max length of 100', () => {
+    const longString = 'a'.repeat(200)
+    const result = sanitizeForLogging(longString)
+    expect(result.length).toBe(100)
+  })
+
+  it.concurrent('should redact sensitive patterns', () => {
+    const input = 'Bearer abc123xyz456'
+    const result = sanitizeForLogging(input)
+    expect(result).toContain('[REDACTED]')
+    expect(result).not.toContain('abc123xyz456')
+  })
+
+  it.concurrent('should handle empty strings', () => {
+    expect(sanitizeForLogging('')).toBe('')
+  })
+
+  it.concurrent('should not modify safe short strings', () => {
+    const input = 'Safe string'
+    const result = sanitizeForLogging(input)
+    expect(result).toBe(input)
+  })
+})
+
+describe('sanitizeEventData', () => {
+  describe('object sanitization', () => {
+    it.concurrent('should remove sensitive keys entirely', () => {
+      const event = {
+        action: 'login',
+        apiKey: 'secret-key',
+        password: 'secret-pass',
+        userId: '123',
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.action).toBe('login')
+      expect(result.userId).toBe('123')
+      expect(result).not.toHaveProperty('apiKey')
+      expect(result).not.toHaveProperty('password')
+    })
+
+    it.concurrent('should redact sensitive patterns in string values', () => {
+      const event = {
+        message: 'Auth: Bearer abc123token',
+        normal: 'normal value',
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.message).toContain('[REDACTED]')
+      expect(result.message).not.toContain('abc123token')
+      expect(result.normal).toBe('normal value')
+    })
+
+    it.concurrent('should handle nested objects', () => {
+      const event = {
+        user: {
+          id: '123',
+          accessToken: 'secret-token',
+        },
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.user.id).toBe('123')
+      expect(result.user).not.toHaveProperty('accessToken')
+    })
+
+    it.concurrent('should handle arrays', () => {
+      const event = {
+        items: [
+          { id: 1, apiKey: 'key1' },
+          { id: 2, apiKey: 'key2' },
+        ],
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.items[0].id).toBe(1)
+      expect(result.items[0]).not.toHaveProperty('apiKey')
+      expect(result.items[1].id).toBe(2)
+      expect(result.items[1]).not.toHaveProperty('apiKey')
+    })
+  })
+
+  describe('primitive handling', () => {
+    it.concurrent('should return primitives appropriately', () => {
+      expect(sanitizeEventData(null)).toBe(null)
+      expect(sanitizeEventData(undefined)).toBe(undefined)
+      expect(sanitizeEventData(123)).toBe(123)
+      expect(sanitizeEventData(true)).toBe(true)
+    })
+
+    it.concurrent('should redact sensitive patterns in top-level strings', () => {
+      const result = sanitizeEventData('Bearer secrettoken123')
+      expect(result).toContain('[REDACTED]')
+    })
+
+    it.concurrent('should not redact normal strings', () => {
+      const result = sanitizeEventData('normal string')
+      expect(result).toBe('normal string')
+    })
+  })
+
+  describe('no false positives', () => {
+    it.concurrent('should not remove keys with sensitive words in middle', () => {
+      const event = {
+        tokenCount: 500,
+        isAuthenticated: true,
+        hasSecretFeature: false,
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.tokenCount).toBe(500)
+      expect(result.isAuthenticated).toBe(true)
+      expect(result.hasSecretFeature).toBe(false)
+    })
+  })
+})
+
+describe('Security edge cases', () => {
+  describe('redactApiKeys security', () => {
+    it.concurrent('should handle objects with prototype-like key names safely', () => {
+      const obj = {
+        protoField: { isAdmin: true },
+        name: 'test',
+        apiKey: 'secret',
+      }
+      const result = redactApiKeys(obj)
+
+      expect(result.name).toBe('test')
+      expect(result.protoField).toEqual({ isAdmin: true })
+      expect(result.apiKey).toBe('[REDACTED]')
+    })
+
+    it.concurrent('should handle objects with constructor key', () => {
+      const obj = {
+        constructor: 'test-value',
+        normalField: 'normal',
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.constructor).toBe('test-value')
+      expect(result.normalField).toBe('normal')
+    })
+
+    it.concurrent('should handle objects with toString key', () => {
+      const obj = {
+        toString: 'custom-tostring',
+        valueOf: 'custom-valueof',
+        apiKey: 'secret',
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.toString).toBe('custom-tostring')
+      expect(result.valueOf).toBe('custom-valueof')
+      expect(result.apiKey).toBe('[REDACTED]')
+    })
+
+    it.concurrent('should not mutate original object', () => {
+      const original = {
+        apiKey: 'secret-key',
+        nested: {
+          password: 'secret-password',
+        },
+      }
+
+      const originalCopy = structuredClone(original)
+      redactApiKeys(original)
+
+      expect(original).toEqual(originalCopy)
+    })
+
+    it.concurrent('should handle very deeply nested structures', () => {
+      let obj: any = { data: 'value' }
+      for (let i = 0; i < 50; i++) {
+        obj = { nested: obj, apiKey: `secret-${i}` }
+      }
+
+      const result = redactApiKeys(obj)
+
+      expect(result.apiKey).toBe('[REDACTED]')
+      expect(result.nested.apiKey).toBe('[REDACTED]')
+    })
+
+    it.concurrent('should handle arrays with mixed types', () => {
+      const arr = [
+        { apiKey: 'secret' },
+        'string',
+        123,
+        null,
+        undefined,
+        true,
+        [{ password: 'nested' }],
+      ]
+
+      const result = redactApiKeys(arr)
+
+      expect(result[0].apiKey).toBe('[REDACTED]')
+      expect(result[1]).toBe('string')
+      expect(result[2]).toBe(123)
+      expect(result[3]).toBe(null)
+      expect(result[4]).toBe(undefined)
+      expect(result[5]).toBe(true)
+      expect(result[6][0].password).toBe('[REDACTED]')
+    })
+
+    it.concurrent('should handle empty arrays', () => {
+      const result = redactApiKeys([])
+      expect(result).toEqual([])
+    })
+
+    it.concurrent('should handle empty objects', () => {
+      const result = redactApiKeys({})
+      expect(result).toEqual({})
+    })
+  })
+
+  describe('redactSensitiveValues security', () => {
+    it.concurrent('should handle multiple API key patterns in one string', () => {
+      const input = 'Keys: sk-abc123defghijklmnopqr and pk-xyz789abcdefghijklmnop'
+      const result = redactSensitiveValues(input)
+
+      expect(result).not.toContain('sk-abc123defghijklmnopqr')
+      expect(result).not.toContain('pk-xyz789abcdefghijklmnop')
+      expect(result.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(2)
+    })
+
+    it.concurrent('should handle multiline strings with sensitive data', () => {
+      const input = `Line 1: Bearer token123abc456def
+      Line 2: password: "secretpass"
+      Line 3: Normal content`
+
+      const result = redactSensitiveValues(input)
+
+      expect(result).toContain('[REDACTED]')
+      expect(result).not.toContain('token123abc456def')
+      expect(result).not.toContain('secretpass')
+      expect(result).toContain('Normal content')
+    })
+
+    it.concurrent('should handle unicode in strings', () => {
+      const input = 'Bearer abc123'
+      const result = redactSensitiveValues(input)
+
+      expect(result).toContain('[REDACTED]')
+      expect(result).not.toContain('abc123')
+    })
+
+    it.concurrent('should handle very long strings', () => {
+      const longSecret = 'a'.repeat(10000)
+      const input = `Bearer ${longSecret}`
+      const result = redactSensitiveValues(input)
+
+      expect(result).toContain('[REDACTED]')
+      expect(result.length).toBeLessThan(input.length)
+    })
+
+    it.concurrent('should not match partial patterns', () => {
+      const input = 'This is a Bear without er suffix'
+      const result = redactSensitiveValues(input)
+
+      expect(result).toBe(input)
+    })
+
+    it.concurrent('should handle special regex characters safely', () => {
+      const input = 'Test with special chars: $^.*+?()[]{}|'
+      const result = redactSensitiveValues(input)
+
+      expect(result).toBe(input)
+    })
+  })
+
+  describe('sanitizeEventData security', () => {
+    it.concurrent('should strip sensitive keys entirely (not redact)', () => {
+      const event = {
+        action: 'login',
+        apiKey: 'should-be-stripped',
+        password: 'should-be-stripped',
+        userId: '123',
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result).not.toHaveProperty('apiKey')
+      expect(result).not.toHaveProperty('password')
+      expect(Object.keys(result)).not.toContain('apiKey')
+      expect(Object.keys(result)).not.toContain('password')
+    })
+
+    it.concurrent('should handle Symbol keys gracefully', () => {
+      const sym = Symbol('test')
+      const event: any = {
+        [sym]: 'symbol-value',
+        normalKey: 'normal-value',
+      }
+
+      expect(() => sanitizeEventData(event)).not.toThrow()
+    })
+
+    it.concurrent('should handle Date objects as objects', () => {
+      const date = new Date('2024-01-01')
+      const event = {
+        createdAt: date,
+        apiKey: 'secret',
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result.createdAt).toBeDefined()
+      expect(result).not.toHaveProperty('apiKey')
+    })
+
+    it.concurrent('should handle objects with numeric keys', () => {
+      const event: any = {
+        0: 'first',
+        1: 'second',
+        apiKey: 'secret',
+      }
+
+      const result = sanitizeEventData(event)
+
+      expect(result[0]).toBe('first')
+      expect(result[1]).toBe('second')
+      expect(result).not.toHaveProperty('apiKey')
+    })
+  })
+
+  describe('isSensitiveKey security', () => {
+    it.concurrent('should handle case variations', () => {
+      expect(isSensitiveKey('APIKEY')).toBe(true)
+      expect(isSensitiveKey('ApiKey')).toBe(true)
+      expect(isSensitiveKey('apikey')).toBe(true)
+      expect(isSensitiveKey('API_KEY')).toBe(true)
+      expect(isSensitiveKey('api_key')).toBe(true)
+      expect(isSensitiveKey('Api_Key')).toBe(true)
+    })
+
+    it.concurrent('should handle empty string', () => {
+      expect(isSensitiveKey('')).toBe(false)
+    })
+
+    it.concurrent('should handle very long key names', () => {
+      const longKey = `${'a'.repeat(10000)}password`
+      expect(isSensitiveKey(longKey)).toBe(true)
+    })
+
+    it.concurrent('should handle keys with special characters', () => {
+      expect(isSensitiveKey('api-key')).toBe(true)
+      expect(isSensitiveKey('api_key')).toBe(true)
+    })
+
+    it.concurrent('should detect oauth tokens', () => {
+      expect(isSensitiveKey('access_token')).toBe(true)
+      expect(isSensitiveKey('refresh_token')).toBe(true)
+      expect(isSensitiveKey('accessToken')).toBe(true)
+      expect(isSensitiveKey('refreshToken')).toBe(true)
+    })
+
+    it.concurrent('should detect various credential patterns', () => {
+      expect(isSensitiveKey('userCredential')).toBe(true)
+      expect(isSensitiveKey('dbCredential')).toBe(true)
+      expect(isSensitiveKey('appCredential')).toBe(true)
+    })
+  })
+
+  describe('sanitizeForLogging edge cases', () => {
+    it.concurrent('should handle string with only sensitive content', () => {
+      const input = 'Bearer abc123xyz456'
+      const result = sanitizeForLogging(input)
+
+      expect(result).toContain('[REDACTED]')
+      expect(result).not.toContain('abc123xyz456')
+    })
+
+    it.concurrent('should truncate strings to specified length', () => {
+      const longString = 'a'.repeat(200)
+      const result = sanitizeForLogging(longString, 60)
+
+      expect(result.length).toBe(60)
+    })
+
+    it.concurrent('should handle maxLength of 0', () => {
+      const result = sanitizeForLogging('test', 0)
+      expect(result).toBe('')
+    })
+
+    it.concurrent('should handle negative maxLength gracefully', () => {
+      const result = sanitizeForLogging('test', -5)
+      expect(result).toBe('')
+    })
+
+    it.concurrent('should handle maxLength larger than string', () => {
+      const input = 'short'
+      const result = sanitizeForLogging(input, 1000)
+      expect(result).toBe(input)
+    })
+  })
+})

apps/sim/lib/core/security/redaction.ts

@@ -0,0 +1,203 @@
+/**
+ * Centralized redaction utilities for sensitive data
+ */
+
+import { filterUserFileForDisplay, isUserFile } from '@/lib/core/utils/user-file'
+
+export const REDACTED_MARKER = '[REDACTED]'
+export const TRUNCATED_MARKER = '[TRUNCATED]'
+
+const BYPASS_REDACTION_KEYS = new Set(['nextPageToken'])
+
+/** Keys that contain large binary/encoded data that should be truncated in logs */
+const LARGE_DATA_KEYS = new Set(['base64'])
+
+const SENSITIVE_KEY_PATTERNS: RegExp[] = [
+  /^api[_-]?key$/i,
+  /^access[_-]?token$/i,
+  /^refresh[_-]?token$/i,
+  /^client[_-]?secret$/i,
+  /^private[_-]?key$/i,
+  /^auth[_-]?token$/i,
+  /^.*secret$/i,
+  /^.*password$/i,
+  /^.*token$/i,
+  /^.*credential$/i,
+  /^authorization$/i,
+  /^bearer$/i,
+  /^private$/i,
+  /^auth$/i,
+]
+
+/**
+ * Patterns for sensitive values in strings (for redacting values, not keys)
+ * Each pattern has a replacement function
+ */
+const SENSITIVE_VALUE_PATTERNS: Array<{
+  pattern: RegExp
+  replacement: string
+}> = [
+  // Bearer tokens
+  {
+    pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi,
+    replacement: `Bearer ${REDACTED_MARKER}`,
+  },
+  // Basic auth
+  {
+    pattern: /Basic\s+[A-Za-z0-9+/]+=*/gi,
+    replacement: `Basic ${REDACTED_MARKER}`,
+  },
+  // API keys that look like sk-..., pk-..., etc.
+  {
+    pattern: /\b(sk|pk|api|key)[_-][A-Za-z0-9\-._]{20,}\b/gi,
+    replacement: REDACTED_MARKER,
+  },
+  // JSON-style password fields: password: "value" or password: 'value'
+  {
+    pattern: /password['":\s]*['"][^'"]+['"]/gi,
+    replacement: `password: "${REDACTED_MARKER}"`,
+  },
+  // JSON-style token fields: token: "value" or token: 'value'
+  {
+    pattern: /token['":\s]*['"][^'"]+['"]/gi,
+    replacement: `token: "${REDACTED_MARKER}"`,
+  },
+  // JSON-style api_key fields: api_key: "value" or api-key: "value"
+  {
+    pattern: /api[_-]?key['":\s]*['"][^'"]+['"]/gi,
+    replacement: `api_key: "${REDACTED_MARKER}"`,
+  },
+]
+
+export function isSensitiveKey(key: string): boolean {
+  if (BYPASS_REDACTION_KEYS.has(key)) {
+    return false
+  }
+  const lowerKey = key.toLowerCase()
+  return SENSITIVE_KEY_PATTERNS.some((pattern) => pattern.test(lowerKey))
+}
+
+/**
+ * Redacts sensitive patterns from a string value
+ * @param value - The string to redact
+ * @returns The string with sensitive patterns redacted
+ */
+export function redactSensitiveValues(value: string): string {
+  if (!value || typeof value !== 'string') {
+    return value
+  }
+
+  let result = value
+  for (const { pattern, replacement } of SENSITIVE_VALUE_PATTERNS) {
+    result = result.replace(pattern, replacement)
+  }
+  return result
+}
+
+export function isLargeDataKey(key: string): boolean {
+  return LARGE_DATA_KEYS.has(key)
+}
+
+export function redactApiKeys(obj: any): any {
+  if (obj === null || obj === undefined) {
+    return obj
+  }
+
+  if (typeof obj !== 'object') {
+    return obj
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => redactApiKeys(item))
+  }
+
+  if (isUserFile(obj)) {
+    const filtered = filterUserFileForDisplay(obj)
+    const result: Record<string, any> = {}
+    for (const [key, value] of Object.entries(filtered)) {
+      if (isLargeDataKey(key) && typeof value === 'string') {
+        result[key] = TRUNCATED_MARKER
+      } else {
+        result[key] = value
+      }
+    }
+    return result
+  }
+
+  const result: Record<string, any> = {}
+
+  for (const [key, value] of Object.entries(obj)) {
+    if (isSensitiveKey(key)) {
+      result[key] = REDACTED_MARKER
+    } else if (isLargeDataKey(key) && typeof value === 'string') {
+      result[key] = TRUNCATED_MARKER
+    } else if (typeof value === 'object' && value !== null) {
+      result[key] = redactApiKeys(value)
+    } else {
+      result[key] = value
+    }
+  }
+
+  return result
+}
+
+/**
+ * Sanitizes a string for safe logging by truncating and redacting sensitive patterns
+ *
+ * @param value - The string to sanitize
+ * @param maxLength - Maximum length of the output (default: 100)
+ * @returns The sanitized string
+ */
+export function sanitizeForLogging(value: string, maxLength = 100): string {
+  if (!value) return ''
+
+  let sanitized = value.substring(0, maxLength)
+
+  sanitized = redactSensitiveValues(sanitized)
+
+  return sanitized
+}
+
+/**
+ * Sanitizes event data for error reporting/analytics
+ *
+ * @param event - The event data to sanitize
+ * @returns Sanitized event data safe for external reporting
+ */
+export function sanitizeEventData(event: any): any {
+  if (event === null || event === undefined) {
+    return event
+  }
+
+  if (typeof event === 'string') {
+    return redactSensitiveValues(event)
+  }
+
+  if (typeof event !== 'object') {
+    return event
+  }
+
+  if (Array.isArray(event)) {
+    return event.map((item) => sanitizeEventData(item))
+  }
+
+  const sanitized: Record<string, unknown> = {}
+
+  for (const [key, value] of Object.entries(event)) {
+    if (isSensitiveKey(key)) {
+      continue
+    }
+
+    if (typeof value === 'string') {
+      sanitized[key] = redactSensitiveValues(value)
+    } else if (Array.isArray(value)) {
+      sanitized[key] = value.map((v) => sanitizeEventData(v))
+    } else if (value && typeof value === 'object') {
+      sanitized[key] = sanitizeEventData(value)
+    } else {
+      sanitized[key] = value
+    }
+  }
+
+  return sanitized
+}

apps/sim/lib/core/security/same-origin.test.ts

@@ -0,0 +1,32 @@
+/**
+ * @vitest-environment node
+ */
+import type { NextRequest } from 'next/server'
+import { describe, expect, it } from 'vitest'
+import { isCrossSiteSessionRequest } from '@/lib/core/security/same-origin'
+
+function makeRequest(headers: Record<string, string>): NextRequest {
+  return { headers: new Headers(headers) } as unknown as NextRequest
+}
+
+describe('isCrossSiteSessionRequest', () => {
+  it('rejects cross-site requests', () => {
+    expect(isCrossSiteSessionRequest(makeRequest({ 'sec-fetch-site': 'cross-site' }))).toBe(true)
+  })
+
+  it('allows same-origin browser fetches', () => {
+    expect(isCrossSiteSessionRequest(makeRequest({ 'sec-fetch-site': 'same-origin' }))).toBe(false)
+  })
+
+  it('allows same-site fetches (sibling subdomains, e.g. www.<domain> -> <domain>)', () => {
+    expect(isCrossSiteSessionRequest(makeRequest({ 'sec-fetch-site': 'same-site' }))).toBe(false)
+  })
+
+  it('allows user-initiated requests (Sec-Fetch-Site: none)', () => {
+    expect(isCrossSiteSessionRequest(makeRequest({ 'sec-fetch-site': 'none' }))).toBe(false)
+  })
+
+  it('allows requests with no Sec-Fetch-Site header (older clients)', () => {
+    expect(isCrossSiteSessionRequest(makeRequest({}))).toBe(false)
+  })
+})

apps/sim/lib/core/security/same-origin.ts

@@ -0,0 +1,23 @@
+import type { NextRequest } from 'next/server'
+
+/**
+ * Returns true when a request is provably cross-site — a browser fetch driven
+ * from a different site than our own. Used to reject session-cookie CSRF on
+ * state-changing routes.
+ *
+ * `Sec-Fetch-Site` is browser-set and a forbidden header, so page JavaScript
+ * cannot forge it. A cross-site browser request (the CSRF threat) always reports
+ * `cross-site`. We deliberately accept `same-origin`, `same-site`, and `none`:
+ * the app is served across sibling subdomains (e.g. `www.<domain>` calling
+ * `<domain>`), so a legitimate `same-site` fetch must NOT be blocked — rejecting
+ * it 403s real "Run" requests on those origins. An absent header (older clients)
+ * is also allowed; the conventional CSRF posture is to reject only a provable
+ * cross-site request.
+ *
+ * This is CSRF protection only. It does not defend against a non-browser client
+ * that forges headers directly (no header check can); that surface is covered by
+ * the credit and execution rate-limit gates.
+ */
+export function isCrossSiteSessionRequest(req: NextRequest): boolean {
+  return req.headers.get('sec-fetch-site') === 'cross-site'
+}

apps/sim/lib/core/security/turnstile.ts

@@ -0,0 +1,105 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { env } from '@/lib/core/config/env'
+
+const TURNSTILE_SITEVERIFY_URL = 'https://challenges.cloudflare.com/turnstile/v0/siteverify'
+const TURNSTILE_TIMEOUT_MS = 5_000
+
+const logger = createLogger('TurnstileVerify')
+
+interface TurnstileSiteverifyResponse {
+  success: boolean
+  'error-codes'?: string[]
+  challenge_ts?: string
+  hostname?: string
+  action?: string
+  cdata?: string
+  metadata?: { interactive?: boolean }
+}
+
+export interface VerifyTurnstileResult {
+  success: boolean
+  errorCodes?: string[]
+  /** True when the siteverify request itself failed (network/timeout). */
+  transportError?: boolean
+}
+
+export interface VerifyTurnstileOptions {
+  token: string | null | undefined
+  remoteIp?: string
+  /** Rejects the token when Cloudflare's reported hostname differs. */
+  expectedHostname?: string
+  idempotencyKey?: string
+}
+
+/**
+ * Verifies a Turnstile token against Cloudflare's siteverify endpoint. Tokens
+ * are single-use and expire after 300 seconds; never cache the result.
+ */
+export async function verifyTurnstileToken({
+  token,
+  remoteIp,
+  expectedHostname,
+  idempotencyKey,
+}: VerifyTurnstileOptions): Promise<VerifyTurnstileResult> {
+  const secret = env.TURNSTILE_SECRET_KEY
+  if (!secret) {
+    logger.warn('Turnstile verification called without TURNSTILE_SECRET_KEY configured')
+    return { success: false, errorCodes: ['missing-input-secret'] }
+  }
+
+  if (!token) {
+    return { success: false, errorCodes: ['missing-input-response'] }
+  }
+
+  const body = new URLSearchParams()
+  body.set('secret', secret)
+  body.set('response', token)
+  if (remoteIp && remoteIp !== 'unknown') body.set('remoteip', remoteIp)
+  if (idempotencyKey) body.set('idempotency_key', idempotencyKey)
+
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), TURNSTILE_TIMEOUT_MS)
+
+  try {
+    const response = await fetch(TURNSTILE_SITEVERIFY_URL, {
+      method: 'POST',
+      body,
+      signal: controller.signal,
+    })
+
+    if (!response.ok) {
+      logger.warn('Turnstile siteverify returned non-2xx', { status: response.status })
+      return { success: false, transportError: true }
+    }
+
+    const data = (await response.json()) as TurnstileSiteverifyResponse
+
+    if (!data.success) {
+      return { success: false, errorCodes: data['error-codes'] }
+    }
+
+    if (expectedHostname && data.hostname && data.hostname !== expectedHostname) {
+      logger.warn('Turnstile hostname mismatch', {
+        expected: expectedHostname,
+        actual: data.hostname,
+      })
+      return { success: false, errorCodes: ['hostname-mismatch'] }
+    }
+
+    return { success: true }
+  } catch (err) {
+    const error = toError(err)
+    logger.warn('Turnstile siteverify request failed', {
+      aborted: error.name === 'AbortError',
+      error: error.message,
+    })
+    return { success: false, transportError: true }
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+export function isTurnstileConfigured(): boolean {
+  return Boolean(env.TURNSTILE_SECRET_KEY)
+}

apps/sim/lib/core/security/url-safety.test.ts

@@ -0,0 +1,61 @@
+/**
+ * @vitest-environment jsdom
+ */
+import { describe, expect, it } from 'vitest'
+import { isAllowedExternalUrl, sanitizeRenderedHyperlinks } from '@/lib/core/security/url-safety'
+
+describe('isAllowedExternalUrl', () => {
+  it('allows http, https, and mailto URLs', () => {
+    expect(isAllowedExternalUrl('https://example.com/deck')).toBe(true)
+    expect(isAllowedExternalUrl('http://example.com/deck')).toBe(true)
+    expect(isAllowedExternalUrl('mailto:support@example.com')).toBe(true)
+  })
+
+  it('rejects scriptable, data, and relative URLs', () => {
+    expect(isAllowedExternalUrl('javascript:alert(1)')).toBe(false)
+    expect(isAllowedExternalUrl('data:text/html,<script>alert(1)</script>')).toBe(false)
+    expect(isAllowedExternalUrl('/workspace/files')).toBe(false)
+  })
+})
+
+describe('sanitizeRenderedHyperlinks', () => {
+  function containerWithAnchor(href: string): HTMLDivElement {
+    const container = document.createElement('div')
+    const anchor = document.createElement('a')
+    anchor.setAttribute('href', href)
+    anchor.textContent = 'link'
+    container.appendChild(anchor)
+    return container
+  }
+
+  it('strips javascript: hrefs from a docx-preview hyperlink rendering', () => {
+    const container = containerWithAnchor(
+      "javascript:document.body.setAttribute('data-xss-fired','1')"
+    )
+    sanitizeRenderedHyperlinks(container)
+    const anchor = container.querySelector('a')
+    expect(anchor?.hasAttribute('href')).toBe(false)
+  })
+
+  it('strips data: and vbscript: hrefs', () => {
+    for (const href of ['data:text/html,<script>alert(1)</script>', 'vbscript:msgbox(1)']) {
+      const container = containerWithAnchor(href)
+      sanitizeRenderedHyperlinks(container)
+      expect(container.querySelector('a')?.hasAttribute('href')).toBe(false)
+    }
+  })
+
+  it('preserves same-document bookmark anchors', () => {
+    const container = containerWithAnchor('#section-2')
+    sanitizeRenderedHyperlinks(container)
+    expect(container.querySelector('a')?.getAttribute('href')).toBe('#section-2')
+  })
+
+  it('keeps allowed external links and adds rel=noopener noreferrer', () => {
+    const container = containerWithAnchor('https://example.com/report')
+    sanitizeRenderedHyperlinks(container)
+    const anchor = container.querySelector('a')
+    expect(anchor?.getAttribute('href')).toBe('https://example.com/report')
+    expect(anchor?.getAttribute('rel')).toBe('noopener noreferrer')
+  })
+})

apps/sim/lib/core/security/url-safety.ts

@@ -0,0 +1,37 @@
+/**
+ * URL safety utilities for external hyperlinks/media in untrusted document content
+ * (PPTX, DOCX, and other previews rendered into the app origin).
+ */
+
+const ALLOWED_PROTOCOLS = new Set(['http:', 'https:', 'mailto:'])
+
+/**
+ * Returns true only for absolute URLs with an allowed protocol.
+ */
+export function isAllowedExternalUrl(url: string): boolean {
+  try {
+    const parsed = new URL(url)
+    return ALLOWED_PROTOCOLS.has(parsed.protocol.toLowerCase())
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Neutralizes anchors rendered from untrusted document content (e.g. docx-preview,
+ * which copies an external-relationship `Target` straight into `href` with no scheme
+ * check). Same-document fragment links (`#bookmark`) are left intact; anything else
+ * that isn't http/https/mailto has its `href` stripped so the anchor can't navigate.
+ * Surviving external links get `rel="noopener noreferrer"` to block tabnabbing.
+ */
+export function sanitizeRenderedHyperlinks(root: ParentNode): void {
+  for (const anchor of root.querySelectorAll('a[href]')) {
+    const href = anchor.getAttribute('href') ?? ''
+    if (href.startsWith('#')) continue
+    if (isAllowedExternalUrl(href)) {
+      anchor.setAttribute('rel', 'noopener noreferrer')
+      continue
+    }
+    anchor.removeAttribute('href')
+  }
+}

apps/sim/lib/core/storage/index.ts

@@ -0,0 +1,8 @@
+export {
+  getStorageMethod,
+  isDatabaseStorage,
+  isRedisStorage,
+  requireRedis,
+  resetStorageMethod,
+  type StorageMethod,
+} from './storage'

apps/sim/lib/core/storage/storage.ts

@@ -0,0 +1,79 @@
+import { createLogger } from '@sim/logger'
+import { getRedisClient } from '@/lib/core/config/redis'
+
+const logger = createLogger('Storage')
+
+export type StorageMethod = 'redis' | 'database'
+
+let cachedStorageMethod: StorageMethod | null = null
+
+/**
+ * Determine the storage method once based on configuration.
+ * This decision is made at first call and cached for the lifetime of the process.
+ *
+ * - If REDIS_URL is configured and client initializes → 'redis'
+ * - If REDIS_URL is not configured → 'database'
+ *
+ * Transient failures do NOT change the storage method.
+ * If Redis is configured but fails, operations will fail (not fallback to DB).
+ */
+export function getStorageMethod(): StorageMethod {
+  if (cachedStorageMethod) {
+    return cachedStorageMethod
+  }
+
+  const redis = getRedisClient()
+
+  if (redis) {
+    cachedStorageMethod = 'redis'
+    logger.info('Storage method: Redis')
+  } else {
+    cachedStorageMethod = 'database'
+    logger.info('Storage method: PostgreSQL')
+  }
+
+  return cachedStorageMethod
+}
+
+/**
+ * Check if Redis is the configured storage method.
+ * Use this for conditional logic that depends on storage type.
+ */
+export function isRedisStorage(): boolean {
+  return getStorageMethod() === 'redis'
+}
+
+/**
+ * Check if database is the configured storage method.
+ */
+export function isDatabaseStorage(): boolean {
+  return getStorageMethod() === 'database'
+}
+
+/**
+ * Get Redis client, but only if Redis is the configured storage method.
+ * Throws if Redis is configured but client is unavailable.
+ *
+ * Use this instead of getRedisClient() directly when you need to ensure
+ * consistency with the storage method decision.
+ */
+export function requireRedis() {
+  if (!isRedisStorage()) {
+    throw new Error('Redis storage not configured')
+  }
+
+  const redis = getRedisClient()
+  if (!redis) {
+    throw new Error('Redis client unavailable')
+  }
+
+  return redis
+}
+
+/**
+ * Reset the cached storage method.
+ * Only use for testing purposes.
+ */
+export function resetStorageMethod(): void {
+  cachedStorageMethod = null
+}

apps/sim/lib/core/utils.test.ts

@@ -0,0 +1,346 @@
+import { cn } from '@sim/emcn'
+import { createEnvMock } from '@sim/testing'
+import {
+  formatDate,
+  formatDateTime,
+  formatDuration,
+  formatTime,
+  getTimezoneAbbreviation,
+} from '@sim/utils/formatting'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { getRotatingApiKey } from '@/lib/core/config/api-keys'
+import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
+import { convertScheduleOptionsToCron } from '@/lib/core/utils/scheduling'
+import { getInvalidCharacters, isValidName, validateName } from '@/lib/core/utils/validation'
+
+vi.mock('crypto', () => ({
+  createCipheriv: vi.fn().mockReturnValue({
+    update: vi.fn().mockReturnValue('encrypted-data'),
+    final: vi.fn().mockReturnValue('final-data'),
+    getAuthTag: vi.fn().mockReturnValue({
+      toString: vi.fn().mockReturnValue('auth-tag'),
+    }),
+  }),
+  createDecipheriv: vi.fn().mockReturnValue({
+    update: vi.fn().mockReturnValue('decrypted-data'),
+    final: vi.fn().mockReturnValue('final-data'),
+    setAuthTag: vi.fn(),
+  }),
+  randomBytes: vi.fn().mockReturnValue({
+    toString: vi.fn().mockReturnValue('random-iv'),
+  }),
+}))
+
+vi.mock('@/lib/core/config/env', () =>
+  createEnvMock({
+    ENCRYPTION_KEY: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+    OPENAI_API_KEY_1: 'test-openai-key-1',
+    OPENAI_API_KEY_2: 'test-openai-key-2',
+    OPENAI_API_KEY_3: 'test-openai-key-3',
+    ANTHROPIC_API_KEY_1: 'test-anthropic-key-1',
+    ANTHROPIC_API_KEY_2: 'test-anthropic-key-2',
+    ANTHROPIC_API_KEY_3: 'test-anthropic-key-3',
+    GEMINI_API_KEY_1: 'test-gemini-key-1',
+    GEMINI_API_KEY_2: 'test-gemini-key-2',
+    GEMINI_API_KEY_3: 'test-gemini-key-3',
+    XAI_API_KEY_1: 'test-xai-key-1',
+    XAI_API_KEY_2: 'test-xai-key-2',
+    XAI_API_KEY_3: 'test-xai-key-3',
+  })
+)
+
+afterEach(() => {
+  vi.clearAllMocks()
+})
+
+describe('cn (class name utility)', () => {
+  it.concurrent('should merge class names correctly', () => {
+    const result = cn('class1', 'class2')
+    expect(result).toBe('class1 class2')
+  })
+
+  it.concurrent('should handle conditional classes', () => {
+    const isActive = true
+    const result = cn('base', isActive && 'active')
+    expect(result).toBe('base active')
+  })
+
+  it.concurrent('should handle falsy values', () => {
+    const result = cn('base', false && 'hidden', null, undefined, 0, '')
+    expect(result).toBe('base')
+  })
+
+  it.concurrent('should handle arrays of class names', () => {
+    const result = cn('base', ['class1', 'class2'])
+    expect(result).toContain('base')
+    expect(result).toContain('class1')
+    expect(result).toContain('class2')
+  })
+})
+
+describe('encryption and decryption', () => {
+  it.concurrent('should encrypt secrets correctly', async () => {
+    const result = await encryptSecret('my-secret')
+    expect(result).toHaveProperty('encrypted')
+    expect(result).toHaveProperty('iv')
+    expect(result.encrypted).toContain('random-iv')
+    expect(result.encrypted).toContain('encrypted-data')
+    expect(result.encrypted).toContain('final-data')
+    expect(result.encrypted).toContain('auth-tag')
+  })
+
+  it.concurrent('should decrypt secrets correctly', async () => {
+    const result = await decryptSecret('iv:encrypted:authTag')
+    expect(result).toHaveProperty('decrypted')
+    expect(result.decrypted).toBe('decrypted-datafinal-data')
+  })
+
+  it.concurrent('should throw error for invalid decrypt format', async () => {
+    await expect(decryptSecret('invalid-format')).rejects.toThrow('Invalid encrypted value format')
+  })
+})
+
+describe('convertScheduleOptionsToCron', () => {
+  it.concurrent('should convert minutes schedule to cron', () => {
+    const result = convertScheduleOptionsToCron('minutes', { minutesInterval: '5' })
+    expect(result).toBe('*/5 * * * *')
+  })
+
+  it.concurrent('should convert hourly schedule to cron', () => {
+    const result = convertScheduleOptionsToCron('hourly', { hourlyMinute: '30' })
+    expect(result).toBe('30 * * * *')
+  })
+
+  it.concurrent('should convert daily schedule to cron', () => {
+    const result = convertScheduleOptionsToCron('daily', { dailyTime: '15:30' })
+    expect(result).toBe('15 30 * * *')
+  })
+
+  it.concurrent('should convert weekly schedule to cron', () => {
+    const result = convertScheduleOptionsToCron('weekly', {
+      weeklyDay: 'MON',
+      weeklyDayTime: '09:30',
+    })
+    expect(result).toBe('09 30 * * 1')
+  })
+
+  it.concurrent('should convert monthly schedule to cron', () => {
+    const result = convertScheduleOptionsToCron('monthly', {
+      monthlyDay: '15',
+      monthlyTime: '12:00',
+    })
+    expect(result).toBe('12 00 15 * *')
+  })
+
+  it.concurrent('should use custom cron expression directly', () => {
+    const customCron = '*/15 9-17 * * 1-5'
+    const result = convertScheduleOptionsToCron('custom', { cronExpression: customCron })
+    expect(result).toBe(customCron)
+  })
+
+  it.concurrent('should throw error for unsupported schedule type', () => {
+    expect(() => convertScheduleOptionsToCron('invalid', {})).toThrow('Unsupported schedule type')
+  })
+
+  it.concurrent('should use default values when options are not provided', () => {
+    const result = convertScheduleOptionsToCron('daily', {})
+    expect(result).toBe('00 09 * * *')
+  })
+})
+
+describe('date formatting functions', () => {
+  it.concurrent('should format datetime correctly', () => {
+    const date = new Date('2023-05-15T14:30:00')
+    const result = formatDateTime(date)
+    expect(result).toMatch(/May 15, 2023/)
+    expect(result).toMatch(/2:30 PM|14:30/)
+  })
+
+  it.concurrent('should format date correctly', () => {
+    const date = new Date('2023-05-15T14:30:00')
+    const result = formatDate(date)
+    expect(result).toMatch(/May 15, 2023/)
+    expect(result).not.toMatch(/2:30|14:30/)
+  })
+
+  it.concurrent('should format time correctly', () => {
+    const date = new Date('2023-05-15T14:30:00')
+    const result = formatTime(date)
+    expect(result).toMatch(/2:30 PM|14:30/)
+    expect(result).not.toMatch(/2023|May/)
+  })
+})
+
+describe('formatDuration', () => {
+  it.concurrent('should format milliseconds correctly', () => {
+    const result = formatDuration(500)
+    expect(result).toBe('500ms')
+  })
+
+  it.concurrent('should format seconds correctly', () => {
+    const result = formatDuration(5000)
+    expect(result).toBe('5s')
+  })
+
+  it.concurrent('should format minutes and seconds correctly', () => {
+    const result = formatDuration(125000) // 2m 5s
+    expect(result).toBe('2m 5s')
+  })
+
+  it.concurrent('should format hours, minutes correctly', () => {
+    const result = formatDuration(3725000) // 1h 2m 5s
+    expect(result).toBe('1h 2m')
+  })
+})
+
+describe('getTimezoneAbbreviation', () => {
+  it.concurrent('should return UTC for UTC timezone', () => {
+    const result = getTimezoneAbbreviation('UTC')
+    expect(result).toBe('UTC')
+  })
+
+  it.concurrent('should return PST/PDT for Los Angeles timezone', () => {
+    const winterDate = new Date('2023-01-15') // Standard time
+    const summerDate = new Date('2023-07-15') // Daylight time
+
+    const winterResult = getTimezoneAbbreviation('America/Los_Angeles', winterDate)
+    const summerResult = getTimezoneAbbreviation('America/Los_Angeles', summerDate)
+
+    expect(['PST', 'PDT']).toContain(winterResult)
+    expect(['PST', 'PDT']).toContain(summerResult)
+  })
+
+  it.concurrent('should return JST for Tokyo timezone (no DST)', () => {
+    const winterDate = new Date('2023-01-15')
+    const summerDate = new Date('2023-07-15')
+
+    const winterResult = getTimezoneAbbreviation('Asia/Tokyo', winterDate)
+    const summerResult = getTimezoneAbbreviation('Asia/Tokyo', summerDate)
+
+    expect(winterResult).toBe('JST')
+    expect(summerResult).toBe('JST')
+  })
+
+  it.concurrent('should return full timezone name for unknown timezones', () => {
+    const result = getTimezoneAbbreviation('Unknown/Timezone')
+    expect(result).toBe('Unknown/Timezone')
+  })
+})
+
+describe('validateName', () => {
+  it.concurrent('should remove invalid characters', () => {
+    const result = validateName('test@#$%name')
+    expect(result).toBe('testname')
+  })
+
+  it.concurrent('should keep valid characters', () => {
+    const result = validateName('test_name_123')
+    expect(result).toBe('test_name_123')
+  })
+
+  it.concurrent('should keep spaces', () => {
+    const result = validateName('test name')
+    expect(result).toBe('test name')
+  })
+
+  it.concurrent('should handle empty string', () => {
+    const result = validateName('')
+    expect(result).toBe('')
+  })
+
+  it.concurrent('should handle string with only invalid characters', () => {
+    const result = validateName('@#$%')
+    expect(result).toBe('')
+  })
+
+  it.concurrent('should handle mixed valid and invalid characters', () => {
+    const result = validateName('my-workflow@2023!')
+    expect(result).toBe('myworkflow2023')
+  })
+
+  it.concurrent('should collapse multiple spaces into single spaces', () => {
+    const result = validateName('test    multiple     spaces')
+    expect(result).toBe('test multiple spaces')
+  })
+
+  it.concurrent('should handle mixed whitespace and invalid characters', () => {
+    const result = validateName('test@#$  name')
+    expect(result).toBe('test name')
+  })
+})
+
+describe('isValidName', () => {
+  it.concurrent('should return true for valid names', () => {
+    expect(isValidName('test_name')).toBe(true)
+    expect(isValidName('test123')).toBe(true)
+    expect(isValidName('test name')).toBe(true)
+    expect(isValidName('TestName')).toBe(true)
+    expect(isValidName('')).toBe(true)
+  })
+
+  it.concurrent('should return false for invalid names', () => {
+    expect(isValidName('test@name')).toBe(false)
+    expect(isValidName('test-name')).toBe(false)
+    expect(isValidName('test#name')).toBe(false)
+    expect(isValidName('test$name')).toBe(false)
+    expect(isValidName('test%name')).toBe(false)
+  })
+})
+
+describe('getInvalidCharacters', () => {
+  it.concurrent('should return empty array for valid names', () => {
+    const result = getInvalidCharacters('test_name_123')
+    expect(result).toEqual([])
+  })
+
+  it.concurrent('should return invalid characters', () => {
+    const result = getInvalidCharacters('test@#$name')
+    expect(result).toEqual(['@', '#', '$'])
+  })
+
+  it.concurrent('should return unique invalid characters', () => {
+    const result = getInvalidCharacters('test@@##name')
+    expect(result).toEqual(['@', '#'])
+  })
+
+  it.concurrent('should handle empty string', () => {
+    const result = getInvalidCharacters('')
+    expect(result).toEqual([])
+  })
+
+  it.concurrent('should handle string with only invalid characters', () => {
+    const result = getInvalidCharacters('@#$%')
+    expect(result).toEqual(['@', '#', '$', '%'])
+  })
+})
+
+describe('getRotatingApiKey', () => {
+  it.concurrent('should return OpenAI API key based on current minute', () => {
+    const result = getRotatingApiKey('openai')
+    expect(result).toMatch(/^test-openai-key-[1-3]$/)
+  })
+
+  it.concurrent('should return Anthropic API key based on current minute', () => {
+    const result = getRotatingApiKey('anthropic')
+    expect(result).toMatch(/^test-anthropic-key-[1-3]$/)
+  })
+
+  it.concurrent('should return Gemini API key based on current minute', () => {
+    const result = getRotatingApiKey('gemini')
+    expect(result).toMatch(/^test-gemini-key-[1-3]$/)
+  })
+
+  it.concurrent('should return xAI API key based on current minute', () => {
+    const result = getRotatingApiKey('xai')
+    expect(result).toMatch(/^test-xai-key-[1-3]$/)
+  })
+
+  it.concurrent('should throw error for unsupported provider', () => {
+    expect(() => getRotatingApiKey('unsupported')).toThrow('No rotation implemented for provider')
+  })
+
+  it.concurrent('should rotate keys based on minute modulo', () => {
+    const result = getRotatingApiKey('openai')
+    expect(['test-openai-key-1', 'test-openai-key-2', 'test-openai-key-3']).toContain(result)
+  })
+})

apps/sim/lib/core/utils/arrays.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from 'vitest'
+import { normalizeStringArray } from '@/lib/core/utils/arrays'
+
+describe('array normalization utilities', () => {
+  it('normalizes string arrays loaded from untyped state', () => {
+    expect(normalizeStringArray(['output-1', 2, 'output-2', null])).toEqual([
+      'output-1',
+      'output-2',
+    ])
+    expect(normalizeStringArray('output-1')).toEqual([])
+    expect(normalizeStringArray(undefined)).toEqual([])
+  })
+})

apps/sim/lib/core/utils/arrays.ts

@@ -0,0 +1,10 @@
+/**
+ * Normalizes optional string-list values loaded from untyped persisted state.
+ */
+export function normalizeStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) {
+    return []
+  }
+
+  return value.filter((item): item is string => typeof item === 'string')
+}

apps/sim/lib/core/utils/background.test.ts

@@ -0,0 +1,36 @@
+/**
+ * @vitest-environment node
+ */
+import { sleep } from '@sim/utils/helpers'
+import { describe, expect, it, vi } from 'vitest'
+import { runDetached } from '@/lib/core/utils/background'
+
+const flushMicrotasks = () => sleep(0)
+
+describe('runDetached', () => {
+  it('runs the work without the caller awaiting it', async () => {
+    const work = vi.fn().mockResolvedValue(undefined)
+
+    runDetached('test', work)
+
+    await flushMicrotasks()
+    expect(work).toHaveBeenCalledTimes(1)
+  })
+
+  it('swallows rejections so they do not surface as unhandled', async () => {
+    const work = vi.fn().mockRejectedValue(new Error('boom'))
+
+    expect(() => runDetached('test', work)).not.toThrow()
+    await flushMicrotasks()
+    expect(work).toHaveBeenCalledTimes(1)
+  })
+
+  it('swallows synchronous throws from work', async () => {
+    const work = vi.fn(() => {
+      throw new Error('sync boom')
+    })
+
+    expect(() => runDetached('test', work)).not.toThrow()
+    await flushMicrotasks()
+  })
+})

apps/sim/lib/core/utils/background.ts

@@ -0,0 +1,26 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+
+const logger = createLogger('BackgroundTask')
+
+/**
+ * Runs work detached from the HTTP response so a caller (e.g. a cron job with a
+ * short request timeout) receives an immediate response while processing
+ * continues on the long-lived server process.
+ *
+ * `withRouteHandler` only wraps awaited work in its try/catch, so a detached
+ * promise must catch its own rejection or it surfaces as an `unhandledRejection`.
+ * The request-scoped AsyncLocalStorage context (request ID) is captured when the
+ * work is scheduled and preserved across the detached continuation, so loggers
+ * inside `work` keep the originating request ID.
+ *
+ * @param label - Identifier used in the failure log line.
+ * @param work - The async work to run in the background.
+ */
+export function runDetached(label: string, work: () => Promise<unknown>): void {
+  void Promise.resolve()
+    .then(work)
+    .catch((error) => {
+      logger.error(`Background task failed: ${label}`, toError(error))
+    })
+}

apps/sim/lib/core/utils/browser-polyfills.ts

@@ -0,0 +1,47 @@
+/**
+ * Polyfills for `Promise.withResolvers` (Safari < 17.4, Chrome < 119) and
+ * `URL.parse` (Safari < 18, Chrome < 126), which pdf.js 5.x calls at
+ * module-evaluation time. Without them, importing `react-pdf`/`pdfjs-dist`
+ * throws before anything renders, so this module must be imported for its
+ * side effects BEFORE those imports. The pdf.js worker runs in a separate
+ * context these polyfills cannot reach; it is covered by serving pdf.js's
+ * self-polyfilling legacy worker build (see pdf-viewer.tsx).
+ *
+ * Typed locally because the repo TS lib is ES2022, which predates both APIs.
+ */
+
+interface PromiseWithResolversResult<T> {
+  promise: Promise<T>
+  resolve: (value: T | PromiseLike<T>) => void
+  reject: (reason?: unknown) => void
+}
+
+const promiseCtor = Promise as typeof Promise & {
+  withResolvers?: <T>() => PromiseWithResolversResult<T>
+}
+
+if (typeof promiseCtor.withResolvers !== 'function') {
+  promiseCtor.withResolvers = <T>(): PromiseWithResolversResult<T> => {
+    let resolve!: (value: T | PromiseLike<T>) => void
+    let reject!: (reason?: unknown) => void
+    const promise = new Promise<T>((res, rej) => {
+      resolve = res
+      reject = rej
+    })
+    return { promise, resolve, reject }
+  }
+}
+
+const urlCtor = URL as typeof URL & {
+  parse?: (url: string | URL, base?: string | URL) => URL | null
+}
+
+if (typeof urlCtor.parse !== 'function') {
+  urlCtor.parse = (url: string | URL, base?: string | URL): URL | null => {
+    try {
+      return new URL(url, base)
+    } catch {
+      return null
+    }
+  }
+}

apps/sim/lib/core/utils/browser-storage.test.ts

@@ -0,0 +1,88 @@
+/**
+ * @vitest-environment jsdom
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { MothershipHandoffStorage, STORAGE_KEYS } from '@/lib/core/utils/browser-storage'
+import type { ChatContext } from '@/stores/panel'
+
+const WS = 'ws-1'
+
+describe('MothershipHandoffStorage', () => {
+  beforeEach(() => {
+    localStorage.clear()
+  })
+
+  it('round-trips a handoff and trims the message, preserving contexts', () => {
+    const contexts: ChatContext[] = [{ kind: 'logs', executionId: 'run-1', label: 'My Flow' }]
+    expect(MothershipHandoffStorage.store({ message: '  fix it  ', contexts }, WS)).toBe(true)
+
+    expect(MothershipHandoffStorage.consume(WS)).toEqual({ message: 'fix it', contexts })
+  })
+
+  it('is one-shot — a second consume returns null', () => {
+    MothershipHandoffStorage.store({ message: 'fix it' }, WS)
+
+    expect(MothershipHandoffStorage.consume(WS)).not.toBeNull()
+    expect(MothershipHandoffStorage.consume(WS)).toBeNull()
+  })
+
+  it('refuses to store without a message or workspace', () => {
+    expect(MothershipHandoffStorage.store({ message: '   ' }, WS)).toBe(false)
+    expect(MothershipHandoffStorage.store({ message: 'fix it' }, '')).toBe(false)
+    expect(MothershipHandoffStorage.consume(WS)).toBeNull()
+  })
+
+  it('leaves a handoff owned by another workspace untouched for its owner', () => {
+    MothershipHandoffStorage.store({ message: 'fix it' }, WS)
+
+    // A different workspace must not claim it, and must not clear it.
+    expect(MothershipHandoffStorage.consume('ws-other')).toBeNull()
+    expect(localStorage.getItem(STORAGE_KEYS.MOTHERSHIP_HANDOFF)).not.toBeNull()
+
+    // The owning workspace still consumes it.
+    expect(MothershipHandoffStorage.consume(WS)).toEqual({ message: 'fix it', contexts: undefined })
+  })
+
+  it('tombstones a corrupted entry (missing timestamp) instead of leaving it forever', () => {
+    localStorage.setItem(
+      STORAGE_KEYS.MOTHERSHIP_HANDOFF,
+      JSON.stringify({ message: 'fix it', workspaceId: WS })
+    )
+
+    expect(MothershipHandoffStorage.consume(WS)).toBeNull()
+    expect(localStorage.getItem(STORAGE_KEYS.MOTHERSHIP_HANDOFF)).toBeNull()
+  })
+
+  it('tombstones a legacy entry (message + timestamp, no workspaceId) rather than firing it', () => {
+    // The old pre-scoping format could be sitting in storage across a deploy —
+    // it must be discarded, not attributed to the current workspace.
+    vi.useFakeTimers()
+    try {
+      vi.setSystemTime(new Date('2026-01-01T00:00:00Z'))
+      localStorage.setItem(
+        STORAGE_KEYS.MOTHERSHIP_HANDOFF,
+        JSON.stringify({ message: 'fix it', timestamp: Date.now() })
+      )
+
+      expect(MothershipHandoffStorage.consume(WS)).toBeNull()
+      expect(localStorage.getItem(STORAGE_KEYS.MOTHERSHIP_HANDOFF)).toBeNull()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('drops and clears a handoff older than maxAge', () => {
+    vi.useFakeTimers()
+    try {
+      vi.setSystemTime(new Date('2026-01-01T00:00:00Z'))
+      MothershipHandoffStorage.store({ message: 'fix it' }, WS)
+
+      vi.advanceTimersByTime(61 * 1000)
+
+      expect(MothershipHandoffStorage.consume(WS)).toBeNull()
+      expect(localStorage.getItem(STORAGE_KEYS.MOTHERSHIP_HANDOFF)).toBeNull()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

apps/sim/lib/core/utils/browser-storage.ts

@@ -0,0 +1,382 @@
+/**
+ * Safe localStorage utilities with SSR support
+ * Provides clean error handling and type safety for browser storage operations
+ */
+
+import { createLogger } from '@sim/logger'
+import type { ChatContext } from '@/stores/panel'
+
+const logger = createLogger('BrowserStorage')
+
+/**
+ * Safe localStorage operations with fallbacks
+ */
+export class BrowserStorage {
+  /**
+   * Safely gets an item from localStorage
+   * @param key - The storage key
+   * @param defaultValue - The default value to return if key doesn't exist or access fails
+   * @returns The stored value or default value
+   */
+  static getItem<T = string>(key: string, defaultValue: T): T {
+    if (typeof window === 'undefined') {
+      return defaultValue
+    }
+
+    try {
+      const item = window.localStorage.getItem(key)
+      if (item === null) {
+        return defaultValue
+      }
+
+      try {
+        return JSON.parse(item) as T
+      } catch {
+        return item as T
+      }
+    } catch (error) {
+      logger.warn(`Failed to get localStorage item "${key}":`, error)
+      return defaultValue
+    }
+  }
+
+  /**
+   * Safely sets an item in localStorage
+   * @param key - The storage key
+   * @param value - The value to store
+   * @returns True if successful, false otherwise
+   */
+  static setItem<T>(key: string, value: T): boolean {
+    if (typeof window === 'undefined') {
+      return false
+    }
+
+    try {
+      const serializedValue = typeof value === 'string' ? value : JSON.stringify(value)
+      window.localStorage.setItem(key, serializedValue)
+      return true
+    } catch (error) {
+      logger.warn(`Failed to set localStorage item "${key}":`, error)
+      return false
+    }
+  }
+
+  /**
+   * Safely removes an item from localStorage
+   * @param key - The storage key to remove
+   * @returns True if successful, false otherwise
+   */
+  static removeItem(key: string): boolean {
+    if (typeof window === 'undefined') {
+      return false
+    }
+
+    try {
+      window.localStorage.removeItem(key)
+      return true
+    } catch (error) {
+      logger.warn(`Failed to remove localStorage item "${key}":`, error)
+      return false
+    }
+  }
+
+  /**
+   * Check if localStorage is available
+   * @returns True if localStorage is available and accessible
+   */
+  static isAvailable(): boolean {
+    if (typeof window === 'undefined') {
+      return false
+    }
+
+    try {
+      const testKey = '__test_localStorage_availability__'
+      window.localStorage.setItem(testKey, 'test')
+      window.localStorage.removeItem(testKey)
+      return true
+    } catch {
+      return false
+    }
+  }
+}
+
+export const STORAGE_KEYS = {
+  LANDING_PAGE_PROMPT: 'sim_landing_page_prompt',
+  LANDING_PAGE_WORKFLOW_SEED: 'sim_landing_page_workflow_seed',
+  WORKSPACE_RECENCY: 'sim_workspace_recency',
+  MOTHERSHIP_HANDOFF: 'sim_mothership_handoff',
+} as const
+
+export class WorkspaceRecencyStorage {
+  private static readonly KEY = STORAGE_KEYS.WORKSPACE_RECENCY
+
+  static touch(workspaceId: string): void {
+    const map = WorkspaceRecencyStorage.getAll()
+    map[workspaceId] = Date.now()
+    BrowserStorage.setItem(WorkspaceRecencyStorage.KEY, map)
+  }
+
+  static getAll(): Record<string, number> {
+    return BrowserStorage.getItem<Record<string, number>>(WorkspaceRecencyStorage.KEY, {})
+  }
+
+  static getMostRecent(): string | null {
+    const map = WorkspaceRecencyStorage.getAll()
+    const entries = Object.entries(map)
+    if (entries.length === 0) return null
+    entries.sort((a, b) => b[1] - a[1])
+    return entries[0][0]
+  }
+
+  static remove(workspaceId: string): void {
+    const map = WorkspaceRecencyStorage.getAll()
+    delete map[workspaceId]
+    BrowserStorage.setItem(WorkspaceRecencyStorage.KEY, map)
+  }
+
+  /**
+   * Removes localStorage entries for workspace IDs not in the provided list.
+   * Call from effects or event handlers, not during render.
+   */
+  static prune(validIds: Set<string>): void {
+    const map = WorkspaceRecencyStorage.getAll()
+    let pruned = false
+    for (const id of Object.keys(map)) {
+      if (!validIds.has(id)) {
+        delete map[id]
+        pruned = true
+      }
+    }
+    if (pruned) {
+      BrowserStorage.setItem(WorkspaceRecencyStorage.KEY, map)
+    }
+  }
+
+  /**
+   * Sorts workspaces by recency (most recent first).
+   * Workspaces without a recorded timestamp are placed after tracked ones.
+   * Pure function safe for use in render-phase computations.
+   */
+  static sortByRecency<T extends { id: string }>(workspaces: T[]): T[] {
+    const map = WorkspaceRecencyStorage.getAll()
+    return [...workspaces].sort((a, b) => {
+      const aTime = map[a.id] ?? 0
+      const bTime = map[b.id] ?? 0
+      return bTime - aTime
+    })
+  }
+}
+
+/**
+ * Specialized utility for managing the landing page prompt
+ */
+export class LandingPromptStorage {
+  private static readonly KEY = STORAGE_KEYS.LANDING_PAGE_PROMPT
+
+  /**
+   * Store a prompt from the landing page
+   * @param prompt - The prompt text to store
+   * @returns True if successful, false otherwise
+   */
+  static store(prompt: string): boolean {
+    if (!prompt || prompt.trim().length === 0) {
+      return false
+    }
+
+    const data = {
+      prompt: prompt.trim(),
+      timestamp: Date.now(),
+    }
+
+    return BrowserStorage.setItem(LandingPromptStorage.KEY, data)
+  }
+
+  /**
+   * Retrieve and consume the stored prompt
+   * @param maxAge - Maximum age of the prompt in milliseconds (default: 24 hours)
+   * @returns The stored prompt or null if not found/expired
+   */
+  static consume(maxAge: number = 24 * 60 * 60 * 1000): string | null {
+    const data = BrowserStorage.getItem<{ prompt: string; timestamp: number } | null>(
+      LandingPromptStorage.KEY,
+      null
+    )
+
+    if (!data || !data.prompt || !data.timestamp) {
+      return null
+    }
+
+    const age = Date.now() - data.timestamp
+    if (age > maxAge) {
+      LandingPromptStorage.clear()
+      return null
+    }
+
+    LandingPromptStorage.clear()
+    return data.prompt
+  }
+
+  /**
+   * Check if there's a stored prompt without consuming it
+   * @param maxAge - Maximum age of the prompt in milliseconds (default: 24 hours)
+   * @returns True if there's a valid prompt, false otherwise
+   */
+  static hasPrompt(maxAge: number = 24 * 60 * 60 * 1000): boolean {
+    const data = BrowserStorage.getItem<{ prompt: string; timestamp: number } | null>(
+      LandingPromptStorage.KEY,
+      null
+    )
+
+    if (!data || !data.prompt || !data.timestamp) {
+      return false
+    }
+
+    const age = Date.now() - data.timestamp
+    if (age > maxAge) {
+      LandingPromptStorage.clear()
+      return false
+    }
+
+    return true
+  }
+
+  /**
+   * Clear the stored prompt
+   * @returns True if successful, false otherwise
+   */
+  static clear(): boolean {
+    return BrowserStorage.removeItem(LandingPromptStorage.KEY)
+  }
+}
+
+export interface LandingWorkflowSeed {
+  templateId: string
+  workflowName: string
+  workflowDescription?: string
+  workflowJson: string
+}
+
+/**
+ * Specialized utility for managing a landing-page workflow seed.
+ * Stores a workflow export JSON so it can be imported after signup.
+ */
+export class LandingWorkflowSeedStorage {
+  private static readonly KEY = STORAGE_KEYS.LANDING_PAGE_WORKFLOW_SEED
+
+  static store(seed: LandingWorkflowSeed): boolean {
+    if (!seed.templateId || !seed.workflowName || !seed.workflowJson) {
+      return false
+    }
+
+    return BrowserStorage.setItem(LandingWorkflowSeedStorage.KEY, {
+      ...seed,
+      timestamp: Date.now(),
+    })
+  }
+
+  static consume(maxAge: number = 24 * 60 * 60 * 1000): LandingWorkflowSeed | null {
+    const data = BrowserStorage.getItem<(LandingWorkflowSeed & { timestamp: number }) | null>(
+      LandingWorkflowSeedStorage.KEY,
+      null
+    )
+
+    if (!data || !data.templateId || !data.workflowName || !data.timestamp || !data.workflowJson) {
+      return null
+    }
+
+    if (Date.now() - data.timestamp > maxAge) {
+      LandingWorkflowSeedStorage.clear()
+      return null
+    }
+
+    LandingWorkflowSeedStorage.clear()
+    const { timestamp: _timestamp, ...seed } = data
+    return seed
+  }
+
+  static clear(): boolean {
+    return BrowserStorage.removeItem(LandingWorkflowSeedStorage.KEY)
+  }
+}
+
+export interface MothershipHandoff {
+  /** The message to auto-send to Chat once the home surface mounts. */
+  message: string
+  /** Structured contexts to attach — e.g. a `logs` mention tagging a run. */
+  contexts?: ChatContext[]
+}
+
+/**
+ * One-shot handoff that seeds an auto-sent Chat (mothership) message when the
+ * user is routed to the workspace home from elsewhere in the app — e.g. the
+ * "Troubleshoot in Chat" action on an errored log, which tags the failed run
+ * and asks Sim to fix it.
+ *
+ * The home surface consumes this exactly once on mount. A short max-age guards
+ * against a stale handoff firing on a later, unrelated visit, and `consume`
+ * always clears the entry so it can never be replayed.
+ */
+export class MothershipHandoffStorage {
+  private static readonly KEY = STORAGE_KEYS.MOTHERSHIP_HANDOFF
+
+  /**
+   * Store a handoff to be auto-sent on the next home-surface mount, scoped to
+   * the workspace it targets so a different workspace never claims it.
+   * @returns True if stored, false when the message or workspace is empty.
+   */
+  static store(handoff: MothershipHandoff, workspaceId: string): boolean {
+    const message = handoff.message.trim()
+    if (!message || !workspaceId) {
+      return false
+    }
+
+    return BrowserStorage.setItem(MothershipHandoffStorage.KEY, {
+      message,
+      contexts: handoff.contexts,
+      workspaceId,
+      timestamp: Date.now(),
+    })
+  }
+
+  /**
+   * Retrieve and consume the stored handoff for `workspaceId`. A handoff owned
+   * by a different workspace is left untouched for its owner — its tagged run
+   * only resolves in its own workspace, so misfiring it elsewhere would drop the
+   * context. The owner (and any legacy/corrupt entry) is tombstoned via `clear`
+   * before the validity/expiry checks so it fires at most once and never lingers.
+   * @param maxAge - Maximum age in milliseconds (default: 60 seconds)
+   */
+  static consume(workspaceId: string, maxAge: number = 60 * 1000): MothershipHandoff | null {
+    const data = BrowserStorage.getItem<{
+      message?: string
+      contexts?: ChatContext[]
+      workspaceId?: string
+      timestamp?: number
+    } | null>(MothershipHandoffStorage.KEY, null)
+
+    if (!data) {
+      return null
+    }
+
+    if (data.workspaceId && data.workspaceId !== workspaceId) {
+      return null
+    }
+
+    MothershipHandoffStorage.clear()
+
+    if (
+      !data.workspaceId ||
+      !data.message ||
+      !data.timestamp ||
+      Date.now() - data.timestamp > maxAge
+    ) {
+      return null
+    }
+
+    return { message: data.message, contexts: data.contexts }
+  }
+
+  static clear(): boolean {
+    return BrowserStorage.removeItem(MothershipHandoffStorage.KEY)
+  }
+}

apps/sim/lib/core/utils/concurrency.ts

@@ -0,0 +1,35 @@
+/**
+ * Maps over `items` with at most `limit` concurrent invocations of `fn`,
+ * preserving input order in the result. Use to bound fan-out (e.g. per-row
+ * object-storage reads) so a large batch doesn't issue every request at once.
+ *
+ * Contract: `fn` MUST NOT reject. Results are awaited via `Promise.all`, so a
+ * single rejection fails the entire batch (e.g. one bad row would break a whole
+ * logs export/list page). Callers that materialize per-row data pass a total
+ * mapper (one that catches and returns a degraded value rather than throwing);
+ * keep it that way. If a future caller needs per-item isolation, add an
+ * allSettled-style variant rather than letting a throwing mapper through here.
+ */
+export async function mapWithConcurrency<T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>
+): Promise<R[]> {
+  const results = new Array<R>(items.length)
+  const workerCount = Math.max(1, Math.min(limit, items.length))
+  let cursor = 0
+
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = cursor++
+      if (index >= items.length) return
+      results[index] = await fn(items[index], index)
+    }
+  }
+
+  await Promise.all(Array.from({ length: workerCount }, worker))
+  return results
+}
+
+/** Default bound for per-row object-storage materialization fan-out. */
+export const MATERIALIZE_CONCURRENCY = 20

apps/sim/lib/core/utils/csv.ts

@@ -0,0 +1,7 @@
+/**
+ * Prefixes a single quote to values starting with a spreadsheet formula trigger
+ * (`=`, `+`, `-`, `@`, tab, CR), neutralizing CSV injection in Excel/Sheets.
+ */
+export function neutralizeCsvFormula(value: string): string {
+  return /^[=+\-@\t\r]/.test(value) ? `'${value}` : value
+}

apps/sim/lib/core/utils/display-filters.ts

@@ -0,0 +1,182 @@
+import { filterUserFileForDisplay, isUserFile } from '@/lib/core/utils/user-file'
+
+const MAX_STRING_LENGTH = 15000
+const MAX_DEPTH = 50
+
+function truncateString(value: string, maxLength = MAX_STRING_LENGTH): string {
+  if (value.length <= maxLength) {
+    return value
+  }
+  return `${value.substring(0, maxLength)}... [truncated ${value.length - maxLength} chars]`
+}
+
+function filterUserFile(data: any): any {
+  if (isUserFile(data)) {
+    return filterUserFileForDisplay(data)
+  }
+  return data
+}
+
+const DISPLAY_FILTERS = [filterUserFile]
+
+export function filterForDisplay(data: any): any {
+  const seen = new Set<object>()
+  return filterForDisplayInternal(data, seen, 0)
+}
+
+function getObjectType(data: unknown): string {
+  return Object.prototype.toString.call(data).slice(8, -1)
+}
+
+function filterForDisplayInternal(data: any, seen: Set<object>, depth: number): any {
+  try {
+    if (data === null || data === undefined) {
+      return data
+    }
+
+    const dataType = typeof data
+
+    if (dataType === 'string') {
+      // Remove null bytes which are not allowed in PostgreSQL JSONB
+      const sanitized = data.includes('\u0000') ? data.replace(/\u0000/g, '') : data
+      return truncateString(sanitized)
+    }
+
+    if (dataType === 'number') {
+      if (Number.isNaN(data)) {
+        return '[NaN]'
+      }
+      if (!Number.isFinite(data)) {
+        return data > 0 ? '[Infinity]' : '[-Infinity]'
+      }
+      return data
+    }
+
+    if (dataType === 'boolean') {
+      return data
+    }
+
+    if (dataType === 'bigint') {
+      return `[BigInt: ${data.toString()}]`
+    }
+
+    if (dataType === 'symbol') {
+      return `[Symbol: ${data.toString()}]`
+    }
+
+    if (dataType === 'function') {
+      return `[Function: ${data.name || 'anonymous'}]`
+    }
+
+    if (dataType !== 'object') {
+      return '[Unknown Type]'
+    }
+
+    // True circular reference: object is an ancestor in the current path
+    if (seen.has(data)) {
+      return '[Circular Reference]'
+    }
+
+    if (depth > MAX_DEPTH) {
+      return '[Max Depth Exceeded]'
+    }
+
+    const objectType = getObjectType(data)
+
+    switch (objectType) {
+      case 'Date': {
+        const timestamp = (data as Date).getTime()
+        if (Number.isNaN(timestamp)) {
+          return '[Invalid Date]'
+        }
+        return (data as Date).toISOString()
+      }
+
+      case 'RegExp':
+        return (data as RegExp).toString()
+
+      case 'URL':
+        return (data as URL).toString()
+
+      case 'Error': {
+        const err = data as Error
+        return {
+          name: err.name,
+          message: truncateString(err.message),
+          stack: err.stack ? truncateString(err.stack) : undefined,
+        }
+      }
+
+      case 'ArrayBuffer':
+        return `[ArrayBuffer: ${(data as ArrayBuffer).byteLength} bytes]`
+
+      case 'Map': {
+        seen.add(data)
+        const obj: Record<string, any> = {}
+        for (const [key, value] of (data as Map<any, any>).entries()) {
+          const keyStr = typeof key === 'string' ? key : String(key)
+          obj[keyStr] = filterForDisplayInternal(value, seen, depth + 1)
+        }
+        seen.delete(data)
+        return obj
+      }
+
+      case 'Set': {
+        seen.add(data)
+        const result = Array.from(data as Set<any>).map((item) =>
+          filterForDisplayInternal(item, seen, depth + 1)
+        )
+        seen.delete(data)
+        return result
+      }
+
+      case 'WeakMap':
+        return '[WeakMap]'
+
+      case 'WeakSet':
+        return '[WeakSet]'
+
+      case 'WeakRef':
+        return '[WeakRef]'
+
+      case 'Promise':
+        return '[Promise]'
+    }
+
+    if (ArrayBuffer.isView(data)) {
+      return `[${objectType}: ${(data as ArrayBufferView).byteLength} bytes]`
+    }
+
+    // Add to current path before processing children
+    seen.add(data)
+
+    for (const filterFn of DISPLAY_FILTERS) {
+      const filtered = filterFn(data)
+      if (filtered !== data) {
+        const result = filterForDisplayInternal(filtered, seen, depth + 1)
+        seen.delete(data)
+        return result
+      }
+    }
+
+    if (Array.isArray(data)) {
+      const result = data.map((item) => filterForDisplayInternal(item, seen, depth + 1))
+      seen.delete(data)
+      return result
+    }
+
+    const result: Record<string, any> = {}
+    for (const key of Object.keys(data)) {
+      try {
+        result[key] = filterForDisplayInternal(data[key], seen, depth + 1)
+      } catch {
+        result[key] = '[Error accessing property]'
+      }
+    }
+    // Remove from current path after processing children
+    seen.delete(data)
+    return result
+  } catch {
+    return '[Unserializable]'
+  }
+}

apps/sim/lib/core/utils/http-error.ts

@@ -0,0 +1,19 @@
+/**
+ * Base class for domain errors that map to a specific HTTP status when they
+ * bubble up unhandled through `withRouteHandler`. Modeled after NestJS
+ * `HttpException` / Spring `ResponseStatusException`: subclasses declare a
+ * concrete `statusCode`, and the centralized route wrapper uses an
+ * `instanceof HttpError` check (not duck-typing on a `statusCode` property)
+ * to decide whether to forward the error's `message` to the client.
+ *
+ * Using a class check prevents third-party errors that happen to carry a
+ * `statusCode`-shaped field from being treated as typed HTTP errors and
+ * leaking internal details.
+ *
+ * Subclasses MUST ensure that `message` is safe to expose to clients — no
+ * stack traces, secrets, file paths, ORM internals, or upstream provider
+ * details.
+ */
+export abstract class HttpError extends Error {
+  abstract readonly statusCode: number
+}

apps/sim/lib/core/utils/logging.ts

@@ -0,0 +1,21 @@
+import { truncate } from '@sim/utils/string'
+
+/**
+ * Sanitize URLs for logging by stripping query/hash and truncating.
+ */
+export function sanitizeUrlForLog(url: string, maxLength = 120): string {
+  if (!url) return ''
+
+  const trimmed = url.trim()
+  try {
+    const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:/.test(trimmed)
+    const parsed = new URL(trimmed, hasProtocol ? undefined : 'http://localhost')
+    const origin = parsed.origin === 'null' ? '' : parsed.origin
+    const sanitized = `${origin}${parsed.pathname}`
+    const result = sanitized || parsed.pathname || trimmed
+    return truncate(result, maxLength)
+  } catch {
+    const withoutQuery = trimmed.split('?')[0].split('#')[0]
+    return truncate(withoutQuery, maxLength)
+  }
+}

apps/sim/lib/core/utils/multipart.test.ts

@@ -0,0 +1,203 @@
+/**
+ * @vitest-environment node
+ */
+import type { Readable } from 'node:stream'
+import { describe, expect, it } from 'vitest'
+import { isMultipartError, type MultipartError, readMultipart } from '@/lib/core/utils/multipart'
+
+type Part =
+  | { name: string; value: string }
+  | { name: string; filename: string; value: string; contentType?: string }
+
+const BOUNDARY = '----testboundary1234'
+
+function buildBody(parts: Part[], boundary = BOUNDARY): Buffer {
+  const segments: Buffer[] = []
+  for (const part of parts) {
+    let header = `--${boundary}\r\nContent-Disposition: form-data; name="${part.name}"`
+    if ('filename' in part) {
+      header += `; filename="${part.filename}"\r\nContent-Type: ${part.contentType ?? 'text/csv'}`
+    }
+    header += '\r\n\r\n'
+    segments.push(Buffer.from(header, 'utf8'), Buffer.from(part.value, 'utf8'), Buffer.from('\r\n'))
+  }
+  segments.push(Buffer.from(`--${boundary}--\r\n`, 'utf8'))
+  return Buffer.concat(segments)
+}
+
+function toWebStream(body: Buffer, chunkSize?: number): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      if (chunkSize) {
+        for (let i = 0; i < body.length; i += chunkSize) {
+          controller.enqueue(new Uint8Array(body.subarray(i, i + chunkSize)))
+        }
+      } else {
+        controller.enqueue(new Uint8Array(body))
+      }
+      controller.close()
+    },
+  })
+}
+
+function makeRequest(
+  parts: Part[],
+  opts?: { chunkSize?: number; contentType?: string; boundary?: string }
+) {
+  const boundary = opts?.boundary ?? BOUNDARY
+  return {
+    headers: new Headers({
+      'content-type': opts?.contentType ?? `multipart/form-data; boundary=${boundary}`,
+    }),
+    body: toWebStream(buildBody(parts, boundary), opts?.chunkSize),
+  }
+}
+
+async function readStream(stream: Readable): Promise<string> {
+  const chunks: Buffer[] = []
+  for await (const chunk of stream) chunks.push(Buffer.from(chunk))
+  return Buffer.concat(chunks).toString('utf8')
+}
+
+function expectCode(error: unknown, code: MultipartError['code']) {
+  expect(isMultipartError(error)).toBe(true)
+  expect((error as MultipartError).code).toBe(code)
+}
+
+describe('readMultipart', () => {
+  it('parses text fields (before the file) and exposes the file stream', async () => {
+    const csv = 'name,age\nAlice,30\n'
+    const request = makeRequest([
+      { name: 'workspaceId', value: 'ws-1' },
+      { name: 'file', filename: 'data.csv', value: csv },
+    ])
+
+    const { fields, file } = await readMultipart(request, {
+      maxFileBytes: 1024,
+      requiredFieldsBeforeFile: ['workspaceId'],
+    })
+
+    expect(fields.workspaceId).toBe('ws-1')
+    expect(file?.filename).toBe('data.csv')
+    expect(file?.fieldName).toBe('file')
+    expect(await readStream(file!.stream)).toBe(csv)
+  })
+
+  it('handles a body delivered in tiny chunks (split mid-boundary)', async () => {
+    const csv = 'name,age\nAlice,30\nBob,40\n'
+    const request = makeRequest(
+      [
+        { name: 'workspaceId', value: 'ws-1' },
+        { name: 'file', filename: 'data.csv', value: csv },
+      ],
+      { chunkSize: 3 }
+    )
+
+    const { file } = await readMultipart(request, { maxFileBytes: 1024 })
+    expect(await readStream(file!.stream)).toBe(csv)
+  })
+
+  it('rejects FIELD_AFTER_FILE when a required field comes after the file', async () => {
+    const request = makeRequest([
+      { name: 'file', filename: 'data.csv', value: 'name\nAlice\n' },
+      { name: 'workspaceId', value: 'ws-1' },
+    ])
+
+    await readMultipart(request, {
+      maxFileBytes: 1024,
+      requiredFieldsBeforeFile: ['workspaceId'],
+    }).then(
+      () => {
+        throw new Error('expected rejection')
+      },
+      (err) => expectCode(err, 'FIELD_AFTER_FILE')
+    )
+  })
+
+  it('rejects NO_FILE when the body has no file part', async () => {
+    const request = makeRequest([{ name: 'workspaceId', value: 'ws-1' }])
+    await readMultipart(request, { maxFileBytes: 1024 }).then(
+      () => {
+        throw new Error('expected rejection')
+      },
+      (err) => expectCode(err, 'NO_FILE')
+    )
+  })
+
+  it('rejects NOT_MULTIPART for a non-multipart content type', async () => {
+    const request = {
+      headers: new Headers({ 'content-type': 'application/json' }),
+      body: toWebStream(Buffer.from('{}')),
+    }
+    await readMultipart(request, { maxFileBytes: 1024 }).then(
+      () => {
+        throw new Error('expected rejection')
+      },
+      (err) => expectCode(err, 'NOT_MULTIPART')
+    )
+  })
+
+  it('errors the file stream with FILE_TOO_LARGE when the cap is exceeded', async () => {
+    const request = makeRequest([
+      { name: 'workspaceId', value: 'ws-1' },
+      { name: 'file', filename: 'big.csv', value: 'x'.repeat(500) },
+    ])
+
+    const { file } = await readMultipart(request, { maxFileBytes: 50 })
+    await readStream(file!.stream).then(
+      () => {
+        throw new Error('expected stream error')
+      },
+      (err) => expectCode(err, 'FILE_TOO_LARGE')
+    )
+  })
+
+  it('rejects when the signal is already aborted', async () => {
+    const controller = new AbortController()
+    controller.abort()
+    const request = makeRequest([
+      { name: 'workspaceId', value: 'ws-1' },
+      { name: 'file', filename: 'data.csv', value: 'name\nAlice\n' },
+    ])
+
+    await expect(
+      readMultipart(request, { maxFileBytes: 1024, signal: controller.signal })
+    ).rejects.toBeTruthy()
+  })
+
+  it('destroys the file stream when the signal aborts mid-upload (after resolve)', async () => {
+    const controller = new AbortController()
+    // A body that delivers the file-part header but never closes, so the file stream stays open
+    // after readMultipart resolves — mimicking a client still uploading.
+    let enqueue!: (b: Buffer) => void
+    const body = new ReadableStream<Uint8Array>({
+      start(c) {
+        enqueue = (b) => c.enqueue(new Uint8Array(b))
+      },
+    })
+    const head = Buffer.concat([
+      Buffer.from(
+        `--${BOUNDARY}\r\nContent-Disposition: form-data; name="workspaceId"\r\n\r\nws-1\r\n`
+      ),
+      Buffer.from(
+        `--${BOUNDARY}\r\nContent-Disposition: form-data; name="file"; filename="data.csv"\r\nContent-Type: text/csv\r\n\r\n`
+      ),
+      Buffer.from('name,age\n'),
+    ])
+    const request = {
+      headers: new Headers({ 'content-type': `multipart/form-data; boundary=${BOUNDARY}` }),
+      body,
+    }
+    enqueue(head)
+
+    const parsed = await readMultipart(request, {
+      maxFileBytes: 1024,
+      requiredFieldsBeforeFile: ['workspaceId'],
+      signal: controller.signal,
+    })
+    expect(parsed.file).toBeTruthy()
+
+    controller.abort()
+    await expect(readStream(parsed.file!.stream)).rejects.toBeTruthy()
+  })
+})

apps/sim/lib/core/utils/multipart.ts

@@ -0,0 +1,249 @@
+import { Readable } from 'node:stream'
+import type { ReadableStream as NodeReadableStream } from 'node:stream/web'
+import { getErrorMessage } from '@sim/utils/errors'
+import busboy from 'busboy'
+
+/**
+ * Streaming multipart/form-data reader built on `busboy`.
+ *
+ * Unlike `request.formData()` (undici), this never buffers the whole request
+ * body in memory and does not depend on a correct `content-length`/boundary —
+ * it parses the request as it streams off the socket. The single file part is
+ * surfaced as an un-drained Node {@link Readable} so the caller can run auth /
+ * create-table work BEFORE consuming the (potentially huge) file bytes.
+ *
+ * @see readMultipart
+ */
+
+/** Error codes surfaced by {@link readMultipart} and the returned file stream. */
+export type MultipartErrorCode =
+  | 'NOT_MULTIPART'
+  | 'NO_BODY'
+  | 'FILE_TOO_LARGE'
+  | 'FIELD_AFTER_FILE'
+  | 'NO_FILE'
+  | 'PARSE_ERROR'
+
+/**
+ * Error thrown by {@link readMultipart} (for pre-file failures) or emitted on
+ * the returned file stream (for failures during consumption, e.g.
+ * `FILE_TOO_LARGE`). Callers map `code` to an HTTP status.
+ */
+export class MultipartError extends Error {
+  readonly code: MultipartErrorCode
+
+  constructor(code: MultipartErrorCode, message: string) {
+    super(message)
+    this.name = 'MultipartError'
+    this.code = code
+  }
+}
+
+export function isMultipartError(error: unknown): error is MultipartError {
+  return error instanceof MultipartError
+}
+
+export interface MultipartFilePart {
+  /** The multipart field name that carried the file (expected: `file`). */
+  fieldName: string
+  filename: string
+  mimeType: string
+  /**
+   * The file bytes. The caller MUST fully consume or `destroy()` this stream
+   * (use a `finally`) or the request will hang. On overflow of `maxFileBytes`
+   * the stream is destroyed with a {@link MultipartError} (`FILE_TOO_LARGE`).
+   */
+  stream: Readable
+}
+
+export interface ParsedMultipart {
+  /** Text fields that arrived before the file part, keyed by field name. */
+  fields: Record<string, string>
+  /** The single file part, or `null` if the body had no file part. */
+  file: MultipartFilePart | null
+}
+
+export interface ReadMultipartOptions {
+  /** Per-file byte cap. Overflow destroys the file stream with `FILE_TOO_LARGE`. */
+  maxFileBytes: number
+  /**
+   * Field names that must arrive before the file part. If the file part is
+   * seen while any are still missing, the parse rejects with `FIELD_AFTER_FILE`.
+   */
+  requiredFieldsBeforeFile?: string[]
+  /** Field name expected to carry the file. Defaults to `file`. */
+  fileFieldName?: string
+  /** Abort signal — cancels parsing and destroys the underlying stream. */
+  signal?: AbortSignal
+}
+
+interface MultipartRequest {
+  headers: Headers
+  body: ReadableStream<Uint8Array> | null
+}
+
+/**
+ * Parse a `multipart/form-data` request as a stream. Resolves as soon as the
+ * file-part header is seen (text fields collected up to that point are in
+ * `fields`); the file bytes are NOT yet consumed — the caller drives
+ * `result.file.stream`.
+ *
+ * Pre-file failures reject the returned promise; failures that happen while the
+ * file streams (size limit, mid-body parse errors, abort) are surfaced as an
+ * error on `result.file.stream`.
+ */
+export function readMultipart(
+  request: MultipartRequest,
+  options: ReadMultipartOptions
+): Promise<ParsedMultipart> {
+  const { maxFileBytes, requiredFieldsBeforeFile = [], fileFieldName = 'file', signal } = options
+
+  return new Promise<ParsedMultipart>((resolve, reject) => {
+    const contentType = request.headers.get('content-type')
+    if (!contentType || !contentType.toLowerCase().includes('multipart/form-data')) {
+      reject(new MultipartError('NOT_MULTIPART', 'Expected multipart/form-data request'))
+      return
+    }
+    if (!request.body) {
+      reject(new MultipartError('NO_BODY', 'Request has no body'))
+      return
+    }
+
+    let bb: busboy.Busboy
+    try {
+      bb = busboy({
+        headers: { 'content-type': contentType },
+        limits: { fileSize: maxFileBytes, files: 1 },
+      })
+    } catch (err) {
+      reject(new MultipartError('NOT_MULTIPART', getErrorMessage(err, 'Invalid multipart request')))
+      return
+    }
+
+    // double-cast-allowed: the web ReadableStream on request.body isn't structurally assignable to the Node type Readable.fromWeb expects
+    const nodeStream = Readable.fromWeb(request.body as unknown as NodeReadableStream<Uint8Array>)
+    const fields: Record<string, string> = {}
+    let settled = false
+    let fileSeen = false
+
+    const onAbort = () => {
+      const reason = signal?.reason instanceof Error ? signal.reason : new Error('Aborted')
+      nodeStream.destroy(reason)
+      bb.destroy()
+      if (!settled) {
+        settled = true
+        reject(reason)
+      }
+    }
+
+    const cleanup = () => {
+      signal?.removeEventListener('abort', onAbort)
+    }
+
+    const settle = (fn: () => void) => {
+      if (settled) return
+      settled = true
+      cleanup()
+      fn()
+    }
+
+    if (signal?.aborted) {
+      // `destroy()` with no reason emits 'close', not an unhandled 'error'.
+      nodeStream.destroy()
+      settled = true
+      reject(signal.reason instanceof Error ? signal.reason : new Error('Aborted'))
+      return
+    }
+    signal?.addEventListener('abort', onAbort, { once: true })
+
+    bb.on('field', (name, value) => {
+      fields[name] = value
+    })
+
+    bb.on('file', (name, stream, info) => {
+      if (settled || fileSeen) {
+        stream.resume()
+        return
+      }
+      fileSeen = true
+
+      if (name !== fileFieldName) {
+        stream.resume()
+        nodeStream.destroy()
+        settle(() =>
+          reject(
+            new MultipartError('NO_FILE', `Expected file field "${fileFieldName}", got "${name}"`)
+          )
+        )
+        return
+      }
+
+      const missing = requiredFieldsBeforeFile.filter((field) => !(field in fields))
+      if (missing.length > 0) {
+        stream.resume()
+        nodeStream.destroy()
+        settle(() =>
+          reject(
+            new MultipartError(
+              'FIELD_AFTER_FILE',
+              `Field(s) must precede the file in the request body: ${missing.join(', ')}`
+            )
+          )
+        )
+        return
+      }
+
+      stream.once('limit', () => {
+        stream.destroy(
+          new MultipartError('FILE_TOO_LARGE', `File exceeds maximum size of ${maxFileBytes} bytes`)
+        )
+      })
+
+      settle(() => {
+        // settle() detached the pre-file abort handler. Re-arm one scoped to the file stream so a
+        // client disconnect mid-upload tears it down — otherwise the caller's consume loop hangs
+        // until maxDuration. Detach when the stream closes so it can't fire afterward.
+        if (signal) {
+          const onStreamAbort = () => {
+            const reason = signal.reason instanceof Error ? signal.reason : new Error('Aborted')
+            stream.destroy(reason)
+            nodeStream.destroy(reason)
+            bb.destroy()
+          }
+          if (signal.aborted) onStreamAbort()
+          else {
+            signal.addEventListener('abort', onStreamAbort, { once: true })
+            stream.once('close', () => signal.removeEventListener('abort', onStreamAbort))
+          }
+        }
+        resolve({
+          fields,
+          file: { fieldName: name, filename: info.filename, mimeType: info.mimeType, stream },
+        })
+      })
+    })
+
+    bb.on('error', (err) => {
+      const message = getErrorMessage(err, 'Failed to parse multipart body')
+      settle(() => reject(new MultipartError('PARSE_ERROR', message)))
+    })
+
+    bb.on('close', () => {
+      if (!fileSeen) {
+        settle(() => reject(new MultipartError('NO_FILE', 'No file part in multipart body')))
+      }
+    })
+
+    nodeStream.on('error', (err) => {
+      settle(() =>
+        reject(
+          err instanceof MultipartError
+            ? err
+            : new MultipartError('PARSE_ERROR', getErrorMessage(err, 'Failed to read request body'))
+        )
+      )
+    })
+
+    nodeStream.pipe(bb)
+  })
+}

apps/sim/lib/core/utils/platform.ts

@@ -0,0 +1,8 @@
+/**
+ * Detects whether the current platform is macOS/iOS.
+ * Returns false during SSR.
+ */
+export function isMacPlatform(): boolean {
+  if (typeof window === 'undefined') return false
+  return /Mac|iPhone|iPod|iPad/i.test(navigator.userAgent)
+}

apps/sim/lib/core/utils/react-node-text.ts

@@ -0,0 +1,14 @@
+import { isValidElement, type ReactNode } from 'react'
+
+/**
+ * Recursively extracts plain text content from a React node tree.
+ */
+export function extractTextContent(node: ReactNode): string {
+  if (typeof node === 'string') return node
+  if (typeof node === 'number') return String(node)
+  if (!node) return ''
+  if (Array.isArray(node)) return node.map(extractTextContent).join('')
+  if (isValidElement(node))
+    return extractTextContent((node.props as { children?: ReactNode }).children)
+  return ''
+}

apps/sim/lib/core/utils/records.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import {
+  normalizeRecord,
+  normalizeRecordMap,
+  normalizeStringRecord,
+  normalizeWorkflowVariables,
+} from '@/lib/core/utils/records'
+
+describe('record normalization utilities', () => {
+  it('normalizes unknown values to object records', () => {
+    expect(normalizeRecord({ value: 1 })).toEqual({ value: 1 })
+    expect(normalizeRecord([])).toEqual({})
+    expect(normalizeRecord('not-a-record')).toEqual({})
+  })
+
+  it('normalizes string records for environment-like values', () => {
+    expect(
+      normalizeStringRecord({
+        TOKEN: 'secret',
+        RETRIES: 3,
+        ENABLED: true,
+        EMPTY: null,
+      })
+    ).toEqual({
+      TOKEN: 'secret',
+      RETRIES: '3',
+      ENABLED: 'true',
+    })
+    expect(normalizeStringRecord([])).toEqual({})
+  })
+
+  it('normalizes record maps by dropping malformed entries', () => {
+    expect(
+      normalizeRecordMap({
+        valid: { type: 'string' },
+        invalid: [],
+      })
+    ).toEqual({
+      valid: { type: 'string' },
+    })
+  })
+
+  it('normalizes legacy workflow variable arrays into records', () => {
+    const variableWithId = { id: 'var-1', name: 'brand', type: 'plain', value: 'myfitness' }
+    const variableWithName = { name: 'channel', type: 'plain', value: 'whatsapp' }
+
+    expect(normalizeWorkflowVariables([variableWithId, variableWithName, []])).toEqual({
+      'var-1': variableWithId,
+      channel: variableWithName,
+    })
+    expect(normalizeWorkflowVariables({ existing: variableWithId })).toEqual({
+      existing: variableWithId,
+    })
+    expect(normalizeWorkflowVariables('not-a-record')).toEqual({})
+  })
+})

apps/sim/lib/core/utils/records.ts

@@ -0,0 +1,80 @@
+import { isPlainRecord } from '@sim/utils/object'
+
+export type UnknownRecord = Record<string, unknown>
+export type StringRecord = Record<string, string>
+
+/**
+ * Normalizes optional execution context maps to the record shape expected by
+ * internal API contracts.
+ */
+export function normalizeRecord(value: unknown): UnknownRecord {
+  return isPlainRecord(value) ? value : {}
+}
+
+/**
+ * Normalizes environment-like maps to string values, matching process/env
+ * semantics at execution boundaries.
+ */
+export function normalizeStringRecord(value: unknown): StringRecord {
+  if (!isPlainRecord(value)) {
+    return {}
+  }
+
+  const normalized: StringRecord = {}
+  for (const [key, entryValue] of Object.entries(value)) {
+    if (entryValue === undefined || entryValue === null) {
+      continue
+    }
+    normalized[key] = typeof entryValue === 'string' ? entryValue : String(entryValue)
+  }
+  return normalized
+}
+
+/**
+ * Normalizes record-of-record maps such as block output schema maps.
+ */
+export function normalizeRecordMap(value: unknown): Record<string, UnknownRecord> {
+  if (!isPlainRecord(value)) {
+    return {}
+  }
+
+  const normalized: Record<string, UnknownRecord> = {}
+  for (const [key, entryValue] of Object.entries(value)) {
+    if (isPlainRecord(entryValue)) {
+      normalized[key] = entryValue
+    }
+  }
+  return normalized
+}
+
+/**
+ * Workflow variables are stored as a record in current state, while some
+ * legacy and imported snapshots can carry an array of variable objects.
+ */
+export function normalizeWorkflowVariables(value: unknown): UnknownRecord {
+  if (isPlainRecord(value)) {
+    return value
+  }
+
+  if (!Array.isArray(value)) {
+    return {}
+  }
+
+  const normalized: UnknownRecord = {}
+  for (const variable of value) {
+    if (!isPlainRecord(variable)) {
+      continue
+    }
+
+    const id = typeof variable.id === 'string' && variable.id.trim() ? variable.id : undefined
+    const name =
+      typeof variable.name === 'string' && variable.name.trim() ? variable.name : undefined
+    const key = id ?? name
+
+    if (key) {
+      normalized[key] = variable
+    }
+  }
+
+  return normalized
+}

apps/sim/lib/core/utils/request.ts

@@ -0,0 +1,27 @@
+import { getRequestContext } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
+/**
+ * Generate a short request ID for correlation. If called inside a request
+ * context (see `withRouteHandler` and `runWithRequestContext`), returns the
+ * active request's ID so inline `[${requestId}]` log prefixes align with
+ * the auto-attached `{requestId=...}` logger metadata.
+ */
+export function generateRequestId(): string {
+  return getRequestContext()?.requestId ?? generateId().slice(0, 8)
+}
+
+/**
+ * Extract the client IP from a request, checking `x-forwarded-for` then `x-real-ip`.
+ */
+export function getClientIp(request: { headers: { get(name: string): string | null } }): string {
+  return (
+    request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ||
+    request.headers.get('x-real-ip')?.trim() ||
+    'unknown'
+  )
+}
+
+/**
+ * No-operation function for use as default callback
+ */
+export const noop = () => {}

apps/sim/lib/core/utils/response-format.test.ts

@@ -0,0 +1,58 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { extractFieldValues, traverseObjectPath } from '@/lib/core/utils/response-format'
+import {
+  LARGE_ARRAY_MANIFEST_VERSION,
+  type LargeArrayManifest,
+} from '@/lib/execution/payloads/large-array-manifest-metadata'
+
+function createManifest(totalCount = 100_000): LargeArrayManifest {
+  return {
+    __simLargeArrayManifest: true,
+    version: LARGE_ARRAY_MANIFEST_VERSION,
+    kind: 'array',
+    totalCount,
+    chunkCount: 1,
+    byteSize: 12 * 1024 * 1024,
+    chunks: [
+      {
+        count: totalCount,
+        byteSize: 12 * 1024 * 1024,
+        ref: {
+          __simLargeValueRef: true,
+          version: 1,
+          id: 'lv_ABCDEFGHIJKL',
+          kind: 'array',
+          size: 12 * 1024 * 1024,
+          executionId: 'execution-1',
+        },
+      },
+    ],
+    preview: [{ key: 'SIM-0' }],
+  }
+}
+
+describe('response format traversal', () => {
+  it('returns whole large array manifest metadata without materializing chunks', () => {
+    const manifest = createManifest()
+
+    expect(traverseObjectPath({ output: { rows: manifest } }, 'output.rows')).toEqual(manifest)
+  })
+
+  it('returns manifest totalCount for length selections', () => {
+    const manifest = createManifest()
+
+    expect(traverseObjectPath({ output: { rows: manifest } }, 'output.rows.length')).toBe(100_000)
+    expect(
+      extractFieldValues({ output: { rows: manifest } }, ['block-1_output.rows.length'], 'block-1')
+    ).toEqual({ 'output.rows.length': 100_000 })
+  })
+
+  it('does not perform indexed manifest reads in sync traversal', () => {
+    const manifest = createManifest()
+
+    expect(traverseObjectPath({ output: { rows: manifest } }, 'output.rows.0.key')).toBeUndefined()
+  })
+})

apps/sim/lib/core/utils/response-format.ts

@@ -0,0 +1,242 @@
+import { createLogger } from '@sim/logger'
+import { materializeLargeValueRefSyncOrThrow } from '@/lib/execution/payloads/cache'
+import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
+import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
+
+const logger = createLogger('ResponseFormatUtils')
+
+export interface Field {
+  name: string
+  type: string
+  description?: string
+}
+
+/**
+ * Helper function to extract fields from JSON Schema
+ * Handles both legacy format with fields array and new JSON Schema format
+ */
+export function extractFieldsFromSchema(schema: any): Field[] {
+  if (!schema || typeof schema !== 'object') {
+    return []
+  }
+
+  // Handle legacy format with fields array
+  if (Array.isArray(schema.fields)) {
+    return schema.fields
+  }
+
+  // Handle new JSON Schema format
+  const schemaObj = schema.schema || schema
+  if (!schemaObj || !schemaObj.properties || typeof schemaObj.properties !== 'object') {
+    return []
+  }
+
+  // Extract fields from schema properties
+  return Object.entries(schemaObj.properties).map(([name, prop]: [string, any]) => {
+    // Handle array format like ['string', 'array']
+    if (Array.isArray(prop)) {
+      return {
+        name,
+        type: prop.includes('array') ? 'array' : prop[0] || 'string',
+        description: undefined,
+      }
+    }
+
+    // Handle object format like { type: 'string', description: '...' }
+    return {
+      name,
+      type: prop.type || 'string',
+      description: prop.description,
+    }
+  })
+}
+
+/**
+ * Helper function to safely parse response format
+ * Handles both string and object formats
+ */
+export function parseResponseFormatSafely(
+  responseFormatValue: any,
+  blockId: string,
+  options?: { allowReferences?: boolean }
+): any {
+  if (!responseFormatValue) {
+    return null
+  }
+
+  const allowReferences = options?.allowReferences ?? false
+
+  try {
+    if (typeof responseFormatValue === 'string') {
+      const trimmedValue = responseFormatValue.trim()
+      if (trimmedValue === '') return null
+      if (allowReferences && trimmedValue.startsWith('<') && trimmedValue.includes('>')) {
+        return trimmedValue
+      }
+      return JSON.parse(trimmedValue)
+    }
+    return responseFormatValue
+  } catch (error) {
+    logger.warn(`Failed to parse response format for block ${blockId}:`, error)
+    return null
+  }
+}
+
+/**
+ * Extract field values from a parsed JSON object based on selected output paths
+ * Used for both workspace and chat client field extraction
+ */
+export function extractFieldValues(
+  parsedContent: any,
+  selectedOutputs: string[],
+  blockId: string
+): Record<string, any> {
+  const extractedValues: Record<string, any> = {}
+
+  for (const outputId of selectedOutputs) {
+    const blockIdForOutput = extractBlockIdFromOutputId(outputId)
+
+    if (blockIdForOutput !== blockId) {
+      continue
+    }
+
+    const path = extractPathFromOutputId(outputId, blockIdForOutput)
+
+    if (path) {
+      const current = traverseObjectPathInternal(parsedContent, path)
+      if (current !== undefined) {
+        extractedValues[path] = current
+      }
+    }
+  }
+
+  return extractedValues
+}
+
+/**
+ * Format extracted field values for display
+ * Returns formatted string representation of field values
+ */
+export function formatFieldValues(extractedValues: Record<string, any>): string {
+  const formattedValues: string[] = []
+
+  for (const [fieldName, value] of Object.entries(extractedValues)) {
+    const formattedValue = typeof value === 'string' ? value : JSON.stringify(value)
+    formattedValues.push(formattedValue)
+  }
+
+  return formattedValues.join('\n')
+}
+
+/**
+ * Extract block ID from output ID
+ * Handles both formats: "blockId" and "blockId_path" or "blockId.path"
+ */
+export function extractBlockIdFromOutputId(outputId: string): string {
+  return outputId.includes('_') ? outputId.split('_')[0] : outputId.split('.')[0]
+}
+
+/**
+ * Extract path from output ID after the block ID
+ */
+export function extractPathFromOutputId(outputId: string, blockId: string): string {
+  return outputId.substring(blockId.length + 1)
+}
+
+/**
+ * Parse JSON content from output safely
+ * Handles both string and object formats with proper error handling
+ */
+export function parseOutputContentSafely(output: any): any {
+  if (!output?.content) {
+    return output
+  }
+
+  if (typeof output.content === 'string') {
+    try {
+      return JSON.parse(output.content)
+    } catch (e) {
+      // Fallback to original structure if parsing fails
+      return output
+    }
+  }
+
+  return output
+}
+
+/**
+ * Check if a set of output IDs contains response format selections for a specific block
+ */
+export function hasResponseFormatSelection(selectedOutputs: string[], blockId: string): boolean {
+  return selectedOutputs.some((outputId) => {
+    const blockIdForOutput = extractBlockIdFromOutputId(outputId)
+    return blockIdForOutput === blockId && outputId.includes('_')
+  })
+}
+
+/**
+ * Get selected field names for a specific block from output IDs
+ */
+export function getSelectedFieldNames(selectedOutputs: string[], blockId: string): string[] {
+  return selectedOutputs
+    .filter((outputId) => {
+      const blockIdForOutput = extractBlockIdFromOutputId(outputId)
+      return blockIdForOutput === blockId && outputId.includes('_')
+    })
+    .map((outputId) => extractPathFromOutputId(outputId, blockId))
+}
+
+/**
+ * Internal helper to traverse an object path without parsing
+ * @param obj The object to traverse
+ * @param path The dot-separated path (e.g., "result.data.value")
+ * @returns The value at the path, or undefined if path doesn't exist
+ */
+function traverseObjectPathInternal(obj: any, path: string): any {
+  if (!path) return obj
+
+  let current = obj
+  const parts = path.split('.')
+
+  for (const part of parts) {
+    if (isLargeValueRef(current)) {
+      current = materializeLargeValueRefSyncOrThrow(current)
+    }
+
+    if (isLargeArrayManifest(current)) {
+      if (part === 'length' || part === 'totalCount') {
+        current = current.totalCount
+        continue
+      }
+      if (part === 'chunkCount' || part === 'byteSize' || part === 'preview') {
+        current = current[part]
+        continue
+      }
+      return undefined
+    }
+
+    if (current?.[part] !== undefined) {
+      current = current[part]
+    } else {
+      return undefined
+    }
+  }
+
+  if (isLargeValueRef(current)) {
+    return current
+  }
+
+  return current
+}
+
+/**
+ * Traverses an object path safely, returning undefined if any part doesn't exist
+ * Automatically handles parsing of output content if needed
+ * @param obj The object to traverse (may contain unparsed content)
+ * @param path The dot-separated path (e.g., "result.data.value")
+ * @returns The value at the path, or undefined if path doesn't exist
+ */
+export function traverseObjectPath(obj: any, path: string): any {
+  const parsed = parseOutputContentSafely(obj)
+  return traverseObjectPathInternal(parsed, path)
+}

apps/sim/lib/core/utils/restore-name.test.ts

@@ -0,0 +1,32 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+import { generateRestoreName } from '@/lib/core/utils/restore-name'
+
+describe('generateRestoreName', () => {
+  it('returns original when unused', async () => {
+    const nameExists = vi.fn().mockResolvedValue(false)
+    await expect(generateRestoreName('doc.pdf', nameExists, { hasExtension: true })).resolves.toBe(
+      'doc.pdf'
+    )
+    expect(nameExists).toHaveBeenCalledWith('doc.pdf')
+  })
+
+  it('uses _restored suffix when original is taken', async () => {
+    const nameExists = vi
+      .fn()
+      .mockImplementation(async (name: string) => name === 'Notes' || name === 'Notes_restored')
+
+    await expect(generateRestoreName('Notes', nameExists)).resolves.toMatch(
+      /^Notes_restored_[a-f0-9]{6}$/
+    )
+  })
+
+  it('throws after max hash attempts when all candidates exist', async () => {
+    const nameExists = vi.fn().mockResolvedValue(true)
+    await expect(generateRestoreName('x', nameExists)).rejects.toThrow(
+      'Could not generate a unique restore name'
+    )
+  })
+})

apps/sim/lib/core/utils/restore-name.ts

@@ -0,0 +1,44 @@
+import { randomBytes } from 'crypto'
+
+const HASH_ATTEMPTS = 8
+
+/**
+ * Generates a unique name for a restored entity by trying in order:
+ * 1. The original name
+ * 2. `name_restored` (inserted before file extension when `hasExtension` is true)
+ * 3. `name_restored_{6-char hex}` — retries random suffixes until one is free
+ */
+export async function generateRestoreName(
+  originalName: string,
+  nameExists: (name: string) => Promise<boolean>,
+  options?: { hasExtension?: boolean }
+): Promise<string> {
+  if (!(await nameExists(originalName))) {
+    return originalName
+  }
+
+  const restoredName = addSuffix(originalName, '_restored', options?.hasExtension)
+  if (!(await nameExists(restoredName))) {
+    return restoredName
+  }
+
+  for (let i = 0; i < HASH_ATTEMPTS; i++) {
+    const hash = randomBytes(3).toString('hex')
+    const candidate = addSuffix(originalName, `_restored_${hash}`, options?.hasExtension)
+    if (!(await nameExists(candidate))) {
+      return candidate
+    }
+  }
+
+  throw new Error(`Could not generate a unique restore name for "${originalName}"`)
+}
+
+function addSuffix(name: string, suffix: string, hasExtension?: boolean): string {
+  if (hasExtension) {
+    const dotIndex = name.lastIndexOf('.')
+    if (dotIndex > 0) {
+      return `${name.slice(0, dotIndex)}${suffix}${name.slice(dotIndex)}`
+    }
+  }
+  return `${name}${suffix}`
+}

apps/sim/lib/core/utils/scheduling.ts

@@ -0,0 +1,51 @@
+/**
+ * Converts schedule options to a cron expression
+ */
+export function convertScheduleOptionsToCron(
+  scheduleType: string,
+  options: Record<string, string>
+): string {
+  switch (scheduleType) {
+    case 'minutes': {
+      const interval = options.minutesInterval || '15'
+      // For example, if options.minutesStartingAt is provided, use that as the start minute.
+      return `*/${interval} * * * *`
+    }
+    case 'hourly': {
+      // When scheduling hourly, take the specified minute offset
+      return `${options.hourlyMinute || '00'} * * * *`
+    }
+    case 'daily': {
+      // Expected dailyTime in HH:MM
+      const [minute, hour] = (options.dailyTime || '00:09').split(':')
+      return `${minute || '00'} ${hour || '09'} * * *`
+    }
+    case 'weekly': {
+      // Expected weeklyDay as MON, TUE, etc. and weeklyDayTime in HH:MM
+      const dayMap: Record<string, number> = {
+        MON: 1,
+        TUE: 2,
+        WED: 3,
+        THU: 4,
+        FRI: 5,
+        SAT: 6,
+        SUN: 0,
+      }
+      const day = dayMap[options.weeklyDay || 'MON']
+      const [minute, hour] = (options.weeklyDayTime || '00:09').split(':')
+      return `${minute || '00'} ${hour || '09'} * * ${day}`
+    }
+    case 'monthly': {
+      // Expected monthlyDay and monthlyTime in HH:MM
+      const day = options.monthlyDay || '1'
+      const [minute, hour] = (options.monthlyTime || '00:09').split(':')
+      return `${minute || '00'} ${hour || '09'} ${day} * *`
+    }
+    case 'custom': {
+      // Use the provided cron expression directly
+      return options.cronExpression
+    }
+    default:
+      throw new Error('Unsupported schedule type')
+  }
+}

apps/sim/lib/core/utils/smooth-bottom-chase.ts

@@ -0,0 +1,92 @@
+/**
+ * Fraction of the remaining gap to close per frame while chasing the bottom —
+ * an exponential glide (originating in the subagent viewport's stick-to-bottom,
+ * see BoundedViewport in agent-group.tsx) instead of snapping `scrollTop` to
+ * `scrollHeight` on every content append. Closes ~90% of any gap within ~18
+ * frames (~300ms) — deliberately lazier than the subagent viewport's 0.18 so a
+ * large content burst reads as a calm upward drift of the transcript rather
+ * than a lurch.
+ */
+export const SMOOTH_CHASE_RATE = 0.12
+
+/** Gap (px) below which the chase parks until new growth reopens it. */
+export const CHASE_REST_GAP = 0.5
+
+export interface SmoothBottomChaseTarget {
+  /** Current scroll offset. */
+  getTop: () => number
+  /** Scroll offset at which the viewport bottom meets the content bottom. */
+  getBottomTop: () => number
+  /** Apply a new scroll offset. */
+  setTop: (top: number) => void
+}
+
+export interface SmoothBottomChaseHandle {
+  /** True while a chase frame is scheduled (gap still closing). */
+  isActive: () => boolean
+  /** Start the loop if parked. Call after content growth. */
+  kick: () => void
+  cancel: () => void
+}
+
+/**
+ * Eased stick-to-bottom chase over any scrollable target (a DOM element or an
+ * editor API like Monaco's). Each frame closes {@link SMOOTH_CHASE_RATE} of the
+ * remaining gap and self-parks at {@link CHASE_REST_GAP}; content growth
+ * restarts it via `kick()`.
+ *
+ * Self-interrupting: chase writes only ever move the offset down, and content
+ * growth leaves it where the last write put it — so an offset that moved UP
+ * since the last write can only be a user scrolling away, and the loop parks
+ * instead of fighting them. `shouldContinue` layers any caller-owned stickiness
+ * on top (checked every frame).
+ */
+export function createSmoothBottomChase(
+  target: SmoothBottomChaseTarget,
+  shouldContinue: () => boolean = () => true
+): SmoothBottomChaseHandle {
+  let raf: number | null = null
+  let lastTop: number | null = null
+
+  const park = () => {
+    if (raf !== null) cancelAnimationFrame(raf)
+    raf = null
+    lastTop = null
+  }
+
+  const step = () => {
+    // `raf` deliberately keeps this (already-fired) frame's id while the step
+    // body runs: canceling a fired handle is a no-op, and a non-null `raf`
+    // means `isActive()` stays true and a reentrant `kick()` — e.g. from a
+    // target whose `setTop` fires synchronous scroll listeners, like Monaco's
+    // onDidScrollChange — cannot start a second parallel chain.
+    if (!shouldContinue()) {
+      park()
+      return
+    }
+    const top = target.getTop()
+    if (lastTop !== null && top < lastTop - 1) {
+      park()
+      return
+    }
+    const gap = target.getBottomTop() - top
+    if (gap <= CHASE_REST_GAP) {
+      park()
+      return
+    }
+    target.setTop(top + Math.max(1, gap * SMOOTH_CHASE_RATE))
+    // A synchronous side-effect of `setTop` may have called `cancel()`; honor
+    // it instead of re-queuing over it.
+    if (raf === null) return
+    lastTop = target.getTop()
+    raf = requestAnimationFrame(step)
+  }
+
+  return {
+    isActive: () => raf !== null,
+    kick: () => {
+      if (raf === null) raf = requestAnimationFrame(step)
+    },
+    cancel: park,
+  }
+}

apps/sim/lib/core/utils/sse.test.ts

@@ -0,0 +1,658 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+import {
+  encodeSSE,
+  readSSEEvents,
+  readSSELines,
+  readSSEStream,
+  SSE_HEADERS,
+} from '@/lib/core/utils/sse'
+
+function createStreamFromChunks(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
+  let index = 0
+  return new ReadableStream({
+    pull(controller) {
+      if (index < chunks.length) {
+        controller.enqueue(chunks[index])
+        index++
+      } else {
+        controller.close()
+      }
+    },
+  })
+}
+
+function createSSEChunk(data: object): Uint8Array {
+  return new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`)
+}
+
+describe('SSE_HEADERS', () => {
+  it.concurrent('should have correct Content-Type', () => {
+    expect(SSE_HEADERS['Content-Type']).toBe('text/event-stream')
+  })
+
+  it.concurrent('should have correct Cache-Control', () => {
+    expect(SSE_HEADERS['Cache-Control']).toBe('no-cache')
+  })
+
+  it.concurrent('should have Connection keep-alive', () => {
+    expect(SSE_HEADERS.Connection).toBe('keep-alive')
+  })
+
+  it.concurrent('should disable buffering', () => {
+    expect(SSE_HEADERS['X-Accel-Buffering']).toBe('no')
+  })
+})
+
+describe('encodeSSE', () => {
+  it.concurrent('should encode data as SSE format', () => {
+    const data = { chunk: 'hello' }
+    const result = encodeSSE(data)
+    const decoded = new TextDecoder().decode(result)
+    expect(decoded).toBe('data: {"chunk":"hello"}\n\n')
+  })
+
+  it.concurrent('should handle complex objects', () => {
+    const data = { chunk: 'test', nested: { value: 123 } }
+    const result = encodeSSE(data)
+    const decoded = new TextDecoder().decode(result)
+    expect(decoded).toBe('data: {"chunk":"test","nested":{"value":123}}\n\n')
+  })
+
+  it.concurrent('should handle strings with special characters', () => {
+    const data = { chunk: 'Hello, 世界! 🌍' }
+    const result = encodeSSE(data)
+    const decoded = new TextDecoder().decode(result)
+    expect(decoded).toContain('Hello, 世界! 🌍')
+  })
+})
+
+describe('readSSEStream', () => {
+  it.concurrent('should accumulate content from chunks', async () => {
+    const chunks = [
+      createSSEChunk({ chunk: 'Hello' }),
+      createSSEChunk({ chunk: ' World' }),
+      createSSEChunk({ done: true }),
+    ]
+    const stream = createStreamFromChunks(chunks)
+
+    const result = await readSSEStream(stream)
+    expect(result).toBe('Hello World')
+  })
+
+  it.concurrent('should call onChunk callback for each chunk', async () => {
+    const onChunk = vi.fn()
+    const chunks = [createSSEChunk({ chunk: 'A' }), createSSEChunk({ chunk: 'B' })]
+    const stream = createStreamFromChunks(chunks)
+
+    await readSSEStream(stream, { onChunk })
+
+    expect(onChunk).toHaveBeenCalledTimes(2)
+    expect(onChunk).toHaveBeenNthCalledWith(1, 'A')
+    expect(onChunk).toHaveBeenNthCalledWith(2, 'B')
+  })
+
+  it.concurrent('should call onAccumulated callback with accumulated content', async () => {
+    const onAccumulated = vi.fn()
+    const chunks = [createSSEChunk({ chunk: 'A' }), createSSEChunk({ chunk: 'B' })]
+    const stream = createStreamFromChunks(chunks)
+
+    await readSSEStream(stream, { onAccumulated })
+
+    expect(onAccumulated).toHaveBeenCalledTimes(2)
+    expect(onAccumulated).toHaveBeenNthCalledWith(1, 'A')
+    expect(onAccumulated).toHaveBeenNthCalledWith(2, 'AB')
+  })
+
+  it.concurrent('should skip [DONE] messages', async () => {
+    const encoder = new TextEncoder()
+    const chunks = [createSSEChunk({ chunk: 'content' }), encoder.encode('data: [DONE]\n\n')]
+    const stream = createStreamFromChunks(chunks)
+
+    const result = await readSSEStream(stream)
+    expect(result).toBe('content')
+  })
+
+  it.concurrent('should skip lines with error field', async () => {
+    const chunks = [
+      createSSEChunk({ error: 'Something went wrong' }),
+      createSSEChunk({ chunk: 'valid content' }),
+    ]
+    const stream = createStreamFromChunks(chunks)
+
+    const result = await readSSEStream(stream)
+    expect(result).toBe('valid content')
+  })
+
+  it.concurrent('should handle abort signal', async () => {
+    const controller = new AbortController()
+    controller.abort()
+
+    const chunks = [createSSEChunk({ chunk: 'content' })]
+    const stream = createStreamFromChunks(chunks)
+
+    const result = await readSSEStream(stream, { signal: controller.signal })
+    expect(result).toBe('')
+  })
+
+  it.concurrent('should skip unparseable lines', async () => {
+    const encoder = new TextEncoder()
+    const chunks = [encoder.encode('data: invalid-json\n\n'), createSSEChunk({ chunk: 'valid' })]
+    const stream = createStreamFromChunks(chunks)
+
+    const result = await readSSEStream(stream)
+    expect(result).toBe('valid')
+  })
+
+  describe('multi-byte UTF-8 character handling', () => {
+    it.concurrent('should handle Turkish characters split across chunks', async () => {
+      const text = 'Merhaba dünya! Öğretmen şarkı söyledi.'
+      const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+      const bytes = new TextEncoder().encode(fullData)
+
+      const splitPoint = Math.floor(bytes.length / 2)
+      const chunk1 = bytes.slice(0, splitPoint)
+      const chunk2 = bytes.slice(splitPoint)
+
+      const stream = createStreamFromChunks([chunk1, chunk2])
+      const result = await readSSEStream(stream)
+      expect(result).toBe(text)
+    })
+
+    it.concurrent('should handle emoji split across chunks', async () => {
+      const text = 'Hello 🚀 World 🌍 Test 🎯'
+      const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+      const bytes = new TextEncoder().encode(fullData)
+
+      const emojiIndex = fullData.indexOf('🚀')
+      const byteOffset = new TextEncoder().encode(fullData.slice(0, emojiIndex)).length
+      const splitPoint = byteOffset + 2
+
+      const chunk1 = bytes.slice(0, splitPoint)
+      const chunk2 = bytes.slice(splitPoint)
+
+      const stream = createStreamFromChunks([chunk1, chunk2])
+      const result = await readSSEStream(stream)
+      expect(result).toBe(text)
+    })
+
+    it.concurrent('should handle CJK characters split across chunks', async () => {
+      const text = '你好世界！日本語テスト。한국어도 됩니다.'
+      const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+      const bytes = new TextEncoder().encode(fullData)
+
+      const third = Math.floor(bytes.length / 3)
+      const chunk1 = bytes.slice(0, third)
+      const chunk2 = bytes.slice(third, third * 2)
+      const chunk3 = bytes.slice(third * 2)
+
+      const stream = createStreamFromChunks([chunk1, chunk2, chunk3])
+      const result = await readSSEStream(stream)
+      expect(result).toBe(text)
+    })
+
+    it.concurrent('should handle mixed multi-byte content split at byte boundaries', async () => {
+      const text = 'Ö is Turkish, 中 is Chinese, 🎉 is emoji'
+      const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+      const bytes = new TextEncoder().encode(fullData)
+
+      const chunks: Uint8Array[] = []
+      for (let i = 0; i < bytes.length; i += 3) {
+        chunks.push(bytes.slice(i, Math.min(i + 3, bytes.length)))
+      }
+
+      const stream = createStreamFromChunks(chunks)
+      const result = await readSSEStream(stream)
+      expect(result).toBe(text)
+    })
+
+    it.concurrent('should handle SSE message split across chunks', async () => {
+      const encoder = new TextEncoder()
+      const message1 = { chunk: 'First' }
+      const message2 = { chunk: 'Second' }
+
+      const fullText = `data: ${JSON.stringify(message1)}\n\ndata: ${JSON.stringify(message2)}\n\n`
+      const bytes = encoder.encode(fullText)
+
+      const delimiterIndex = fullText.indexOf('\n\n') + 1
+      const byteOffset = encoder.encode(fullText.slice(0, delimiterIndex)).length
+
+      const chunk1 = bytes.slice(0, byteOffset)
+      const chunk2 = bytes.slice(byteOffset)
+
+      const stream = createStreamFromChunks([chunk1, chunk2])
+      const result = await readSSEStream(stream)
+      expect(result).toBe('FirstSecond')
+    })
+
+    it.concurrent('should handle 2-byte UTF-8 character (Ö) split at byte boundary', async () => {
+      const text = 'AÖB'
+      const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+      const bytes = new TextEncoder().encode(fullData)
+
+      const textStart = fullData.indexOf('"') + 1 + text.indexOf('Ö')
+      const byteOffset = new TextEncoder().encode(fullData.slice(0, textStart)).length
+
+      const chunk1 = bytes.slice(0, byteOffset + 1)
+      const chunk2 = bytes.slice(byteOffset + 1)
+
+      const stream = createStreamFromChunks([chunk1, chunk2])
+      const result = await readSSEStream(stream)
+      expect(result).toBe(text)
+    })
+
+    it.concurrent(
+      'should handle 3-byte UTF-8 character (中) split at byte boundaries',
+      async () => {
+        const text = 'A中B'
+        const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+        const bytes = new TextEncoder().encode(fullData)
+
+        const textStart = fullData.indexOf('"') + 1 + text.indexOf('中')
+        const byteOffset = new TextEncoder().encode(fullData.slice(0, textStart)).length
+
+        const chunk1 = bytes.slice(0, byteOffset + 1)
+        const chunk2 = bytes.slice(byteOffset + 1, byteOffset + 2)
+        const chunk3 = bytes.slice(byteOffset + 2)
+
+        const stream = createStreamFromChunks([chunk1, chunk2, chunk3])
+        const result = await readSSEStream(stream)
+        expect(result).toBe(text)
+      }
+    )
+
+    it.concurrent(
+      'should handle 4-byte UTF-8 character (🚀) split at byte boundaries',
+      async () => {
+        const text = 'A🚀B'
+        const fullData = `data: ${JSON.stringify({ chunk: text })}\n\n`
+        const bytes = new TextEncoder().encode(fullData)
+
+        const textStart = fullData.indexOf('"') + 1 + text.indexOf('🚀')
+        const byteOffset = new TextEncoder().encode(fullData.slice(0, textStart)).length
+
+        const chunk1 = bytes.slice(0, byteOffset + 1)
+        const chunk2 = bytes.slice(byteOffset + 1, byteOffset + 2)
+        const chunk3 = bytes.slice(byteOffset + 2, byteOffset + 3)
+        const chunk4 = bytes.slice(byteOffset + 3)
+
+        const stream = createStreamFromChunks([chunk1, chunk2, chunk3, chunk4])
+        const result = await readSSEStream(stream)
+        expect(result).toBe(text)
+      }
+    )
+  })
+
+  describe('SSE message buffering', () => {
+    it.concurrent('should handle incomplete SSE message waiting for more data', async () => {
+      const encoder = new TextEncoder()
+
+      const chunk1 = encoder.encode('data: {"chu')
+      const chunk2 = encoder.encode('nk":"hello"}\n\n')
+
+      const stream = createStreamFromChunks([chunk1, chunk2])
+      const result = await readSSEStream(stream)
+      expect(result).toBe('hello')
+    })
+
+    it.concurrent('should handle multiple complete messages in one chunk', async () => {
+      const encoder = new TextEncoder()
+
+      const multiMessage = 'data: {"chunk":"A"}\n\ndata: {"chunk":"B"}\n\ndata: {"chunk":"C"}\n\n'
+      const chunk = encoder.encode(multiMessage)
+
+      const stream = createStreamFromChunks([chunk])
+      const result = await readSSEStream(stream)
+      expect(result).toBe('ABC')
+    })
+
+    it.concurrent('should handle message that ends exactly at chunk boundary', async () => {
+      const chunks = [createSSEChunk({ chunk: 'First' }), createSSEChunk({ chunk: 'Second' })]
+      const stream = createStreamFromChunks(chunks)
+
+      const result = await readSSEStream(stream)
+      expect(result).toBe('FirstSecond')
+    })
+  })
+})
+
+function streamFromStringChunks(chunks: string[]): ReadableStream<Uint8Array> {
+  const encoder = new TextEncoder()
+  return createStreamFromChunks(chunks.map((c) => encoder.encode(c)))
+}
+
+describe('readSSEEvents', () => {
+  it('parses `\\n\\n`-framed events', async () => {
+    const stream = streamFromStringChunks([
+      'data: {"n":1}\n\n',
+      'data: {"n":2}\n\n',
+      'data: {"n":3}\n\n',
+    ])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1, 2, 3])
+  })
+
+  it('parses `\\n`-framed events', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\ndata: {"n":2}\ndata: {"n":3}\n'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1, 2, 3])
+  })
+
+  it('reassembles events split across chunk boundaries', async () => {
+    const stream = streamFromStringChunks(['data: {"ms', 'g":"hel', 'lo"}\n\n'])
+    const events: Array<{ msg: string }> = []
+    await readSSEEvents<{ msg: string }>(stream, {
+      onEvent: (e) => {
+        events.push(e)
+      },
+    })
+    expect(events).toEqual([{ msg: 'hello' }])
+  })
+
+  it('emits a final data: line that has no trailing newline (stream tail)', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\n', 'data: {"n":2}'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1, 2])
+  })
+
+  it('flushes a multi-byte character in the final unterminated line', async () => {
+    const encoder = new TextEncoder()
+    const euro = encoder.encode('€')
+    const chunk1 = new Uint8Array([...encoder.encode('data: {"s":"'), euro[0], euro[1]])
+    const chunk2 = new Uint8Array([euro[2], ...encoder.encode('"}')])
+    const stream = createStreamFromChunks([chunk1, chunk2])
+    const events: Array<{ s: string }> = []
+    await readSSEEvents<{ s: string }>(stream, {
+      onEvent: (e) => {
+        events.push(e)
+      },
+    })
+    expect(events).toEqual([{ s: '€' }])
+  })
+
+  it('skips the [DONE] sentinel', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1])
+  })
+
+  it('accepts `data:` with and without a leading space', async () => {
+    const stream = streamFromStringChunks(['data:{"n":1}\n\n', 'data: {"n":2}\n\n'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1, 2])
+  })
+
+  it('strips trailing carriage returns (\\r\\n framing)', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\r\n\r\n', 'data: {"n":2}\r\n\r\n'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([1, 2])
+  })
+
+  it('routes unparseable payloads to onParseError and continues', async () => {
+    const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n'])
+    const events: number[] = []
+    const onParseError = vi.fn()
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+      onParseError,
+    })
+    expect(events).toEqual([2])
+    expect(onParseError).toHaveBeenCalledTimes(1)
+    expect(onParseError).toHaveBeenCalledWith('not-json', expect.any(Error))
+  })
+
+  it('stops early when onEvent returns true', async () => {
+    const stream = streamFromStringChunks([
+      'data: {"n":1}\n\n',
+      'data: {"n":2}\n\n',
+      'data: {"n":3}\n\n',
+    ])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: (e) => {
+        events.push(e.n)
+        return e.n === 2
+      },
+    })
+    expect(events).toEqual([1, 2])
+  })
+
+  it('does not process events once the signal is aborted', async () => {
+    const controller = new AbortController()
+    const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: {"n":2}\n\n'])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      signal: controller.signal,
+      onEvent: (e) => {
+        events.push(e.n)
+        controller.abort()
+      },
+    })
+    expect(events).toEqual([1])
+  })
+
+  it('returns immediately when the signal is already aborted', async () => {
+    const controller = new AbortController()
+    controller.abort()
+    const stream = streamFromStringChunks(['data: {"n":1}\n\n'])
+    const onEvent = vi.fn()
+    await readSSEEvents(stream, { signal: controller.signal, onEvent })
+    expect(onEvent).not.toHaveBeenCalled()
+  })
+
+  it('releases the reader lock for a stream source', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\n\n'])
+    await readSSEEvents<{ n: number }>(stream, { onEvent: () => {} })
+    expect(() => stream.getReader()).not.toThrow()
+  })
+
+  it('does not release the lock for a reader source', async () => {
+    const stream = streamFromStringChunks(['data: {"n":1}\n\n'])
+    const reader = stream.getReader()
+    await readSSEEvents<{ n: number }>(reader, { onEvent: () => {} })
+    expect(() => stream.getReader()).toThrow()
+    reader.releaseLock()
+  })
+
+  it('accepts a Response source', async () => {
+    const response = new Response(streamFromStringChunks(['data: {"n":7}\n\n']))
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(response, {
+      onEvent: (e) => {
+        events.push(e.n)
+      },
+    })
+    expect(events).toEqual([7])
+  })
+
+  it('silently skips unparseable payloads when no onParseError is provided', async () => {
+    const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n'])
+    const events: number[] = []
+    await expect(
+      readSSEEvents<{ n: number }>(stream, {
+        onEvent: (e) => {
+          events.push(e.n)
+        },
+      })
+    ).resolves.toBeUndefined()
+    expect(events).toEqual([2])
+  })
+
+  it('surfaces a fatal parse error when onParseError throws', async () => {
+    const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n'])
+    const events: number[] = []
+    await expect(
+      readSSEEvents<{ n: number }>(stream, {
+        onEvent: (e) => {
+          events.push(e.n)
+        },
+        onParseError: () => {
+          throw new Error('boom')
+        },
+      })
+    ).rejects.toThrow('boom')
+    expect(events).toEqual([])
+  })
+
+  it('stops early when onEvent resolves true asynchronously', async () => {
+    const stream = streamFromStringChunks([
+      'data: {"n":1}\n\n',
+      'data: {"n":2}\n\n',
+      'data: {"n":3}\n\n',
+    ])
+    const events: number[] = []
+    await readSSEEvents<{ n: number }>(stream, {
+      onEvent: async (e) => {
+        events.push(e.n)
+        return e.n === 2
+      },
+    })
+    expect(events).toEqual([1, 2])
+  })
+
+  it('throws "No response body" for a Response without a body', async () => {
+    const response = new Response(null)
+    await expect(readSSEEvents(response, { onEvent: () => {} })).rejects.toThrow('No response body')
+  })
+})
+
+describe('readSSELines', () => {
+  it('delivers raw (un-parsed) data payloads', async () => {
+    const stream = streamFromStringChunks(['data: raw-one\n\n', 'data: {"keep":"asString"}\n\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      onData: (raw) => {
+        lines.push(raw)
+      },
+    })
+    expect(lines).toEqual(['raw-one', '{"keep":"asString"}'])
+  })
+
+  it('skips [DONE] and blank separator lines', async () => {
+    const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: [DONE]\n\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      onData: (raw) => {
+        lines.push(raw)
+      },
+    })
+    expect(lines).toEqual(['a', 'b'])
+  })
+
+  it('preserves the raw payload verbatim (no JSON parsing)', async () => {
+    const stream = streamFromStringChunks(['data: {"unterminated\n\n', 'data:no-space\n\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      onData: (raw) => {
+        lines.push(raw)
+      },
+    })
+    expect(lines).toEqual(['{"unterminated', 'no-space'])
+  })
+
+  it('strips a trailing carriage return from each line', async () => {
+    const stream = streamFromStringChunks(['data: one\r\n\r\ndata: two\r\n\r\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      onData: (raw) => {
+        lines.push(raw)
+      },
+    })
+    expect(lines).toEqual(['one', 'two'])
+  })
+
+  it('stops early when onData returns true', async () => {
+    const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      onData: (raw) => {
+        lines.push(raw)
+        return raw === 'b'
+      },
+    })
+    expect(lines).toEqual(['a', 'b'])
+  })
+
+  it('does not deliver any line when the signal is already aborted', async () => {
+    const controller = new AbortController()
+    controller.abort()
+    const stream = streamFromStringChunks(['data: a\n\n'])
+    const onData = vi.fn()
+    await readSSELines(stream, { signal: controller.signal, onData })
+    expect(onData).not.toHaveBeenCalled()
+  })
+
+  it('stops between events in the same chunk once aborted mid-stream', async () => {
+    const controller = new AbortController()
+    const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n'])
+    const lines: string[] = []
+    await readSSELines(stream, {
+      signal: controller.signal,
+      onData: (raw) => {
+        lines.push(raw)
+        if (raw === 'a') controller.abort()
+      },
+    })
+    expect(lines).toEqual(['a'])
+  })
+
+  it('releases the lock for a stream source', async () => {
+    const stream = streamFromStringChunks(['data: a\n\n'])
+    await readSSELines(stream, { onData: () => {} })
+    expect(() => stream.getReader()).not.toThrow()
+  })
+
+  it('does not release the lock for a reader source', async () => {
+    const stream = streamFromStringChunks(['data: a\n\n'])
+    const reader = stream.getReader()
+    await readSSELines(reader, { onData: () => {} })
+    expect(() => stream.getReader()).toThrow()
+    reader.releaseLock()
+  })
+
+  it('releases the lock for a stream source even when onData throws', async () => {
+    const stream = streamFromStringChunks(['data: a\n\n'])
+    await expect(
+      readSSELines(stream, {
+        onData: () => {
+          throw new Error('handler failed')
+        },
+      })
+    ).rejects.toThrow('handler failed')
+    expect(() => stream.getReader()).not.toThrow()
+  })
+})