chore: import upstream snapshot with attribution

apps/sim/lib/core/rate-limiter/hosted-key/hosted-key-rate-limiter.test.ts

@@ -0,0 +1,864 @@
+import { sleep } from '@sim/utils/helpers'
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenStatus,
+} from '@/lib/core/rate-limiter/storage'
+import { HostedKeyRateLimiter } from './hosted-key-rate-limiter'
+import { HEARTBEAT_REFRESH_INTERVAL_MS, type HostedKeyQueue } from './queue'
+import type { CustomRateLimit, PerRequestRateLimit } from './types'
+
+/** Force the queue wait to give up on the first iteration by reporting a retry time
+ *  larger than the 5-minute MAX_QUEUE_WAIT_MS cap. */
+const RETRY_PAST_CAP_MS = 6 * 60 * 1000
+
+interface MockAdapter {
+  consumeTokens: Mock
+  getTokenStatus: Mock
+  resetBucket: Mock
+}
+
+const createMockAdapter = (): MockAdapter => ({
+  consumeTokens: vi.fn(),
+  getTokenStatus: vi.fn(),
+  resetBucket: vi.fn(),
+})
+
+interface MockQueue {
+  enqueue: Mock
+  checkHead: Mock
+  refreshHeartbeat: Mock
+  dequeue: Mock
+}
+
+/** Stub queue that defaults to "you're at the head, no waiting" — i.e. acts as if the
+ *  queue is empty or Redis is unavailable. Tests override per-call to simulate ordering. */
+const createMockQueue = (): MockQueue => {
+  const queue: MockQueue = {
+    enqueue: vi.fn().mockResolvedValue({ position: 0, enabled: true }),
+    checkHead: vi.fn().mockResolvedValue('head'),
+    refreshHeartbeat: vi.fn().mockResolvedValue(undefined),
+    dequeue: vi.fn().mockResolvedValue(undefined),
+  }
+  return queue
+}
+
+describe('HostedKeyRateLimiter', () => {
+  const testProvider = 'exa'
+  const envKeyPrefix = 'EXA_API_KEY'
+  let mockAdapter: MockAdapter
+  let mockQueue: MockQueue
+  let rateLimiter: HostedKeyRateLimiter
+  let originalEnv: NodeJS.ProcessEnv
+
+  const perRequestRateLimit: PerRequestRateLimit = {
+    mode: 'per_request',
+    requestsPerMinute: 10,
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockAdapter = createMockAdapter()
+    mockQueue = createMockQueue()
+    rateLimiter = new HostedKeyRateLimiter(
+      mockAdapter as RateLimitStorageAdapter,
+      mockQueue as unknown as HostedKeyQueue
+    )
+
+    originalEnv = { ...process.env }
+    process.env.EXA_API_KEY_COUNT = '3'
+    process.env.EXA_API_KEY_1 = 'test-key-1'
+    process.env.EXA_API_KEY_2 = 'test-key-2'
+    process.env.EXA_API_KEY_3 = 'test-key-3'
+  })
+
+  afterEach(() => {
+    process.env = originalEnv
+  })
+
+  describe('acquireKey', () => {
+    it('should return error when no keys are configured', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      process.env.EXA_API_KEY_COUNT = undefined
+      process.env.EXA_API_KEY_1 = undefined
+      process.env.EXA_API_KEY_2 = undefined
+      process.env.EXA_API_KEY_3 = undefined
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.error).toContain('No hosted keys configured')
+    })
+
+    it('should rate limit billing actor when wait exceeds the queue cap', async () => {
+      // resetAt past the 5-minute cap forces the wait loop to bail immediately.
+      const rateLimitedResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(rateLimitedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-123'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.retryAfterMs).toBeDefined()
+      expect(result.error).toContain('Rate limit exceeded')
+    })
+
+    it('should wait for capacity then succeed when bucket refills within the cap', async () => {
+      // First call: bucket empty, refills in 100ms (well under cap).
+      // Second call: bucket has capacity, consumed.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 100),
+      }
+      const allowed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValueOnce(blocked).mockResolvedValueOnce(allowed)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-wait'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(2)
+    })
+
+    it('should allow billing actor within their rate limit', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-123'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.billingActorRateLimited).toBeUndefined()
+      expect(result.key).toBe('test-key-1')
+    })
+
+    it('should distribute requests across keys round-robin style', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      const r1 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+      const r2 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-2'
+      )
+      const r3 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-3'
+      )
+      const r4 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-4'
+      )
+
+      expect(r1.keyIndex).toBe(0)
+      expect(r2.keyIndex).toBe(1)
+      expect(r3.keyIndex).toBe(2)
+      expect(r4.keyIndex).toBe(0) // Wraps back
+    })
+
+    it('should handle partial key availability', async () => {
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+
+      process.env.EXA_API_KEY_2 = undefined
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(result.envVarName).toBe('EXA_API_KEY_1')
+
+      const r2 = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-2'
+      )
+      expect(r2.keyIndex).toBe(2) // Skips missing key 1
+      expect(r2.envVarName).toBe('EXA_API_KEY_3')
+    })
+  })
+
+  describe('FIFO queue ordering', () => {
+    const allowed: ConsumeResult = {
+      allowed: true,
+      tokensRemaining: 9,
+      resetAt: new Date(Date.now() + 60000),
+    }
+
+    it('enqueues every call onto the per-workspace+provider queue', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.enqueue).toHaveBeenCalledWith(
+        testProvider,
+        'workspace-1',
+        expect.any(String)
+      )
+    })
+
+    it('always dequeues at the end of a successful acquisition', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.dequeue).toHaveBeenCalledWith(
+        testProvider,
+        'workspace-1',
+        expect.any(String)
+      )
+    })
+
+    it('always dequeues even when the call fails (no keys configured)', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      process.env.EXA_API_KEY_COUNT = '0'
+
+      await rateLimiter.acquireKey(testProvider, envKeyPrefix, perRequestRateLimit, 'workspace-1')
+
+      expect(mockQueue.dequeue).toHaveBeenCalled()
+    })
+
+    it('waits at the head of the queue before consuming from the bucket', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      // First two checkHead calls say we're waiting; third says we're up.
+      mockQueue.checkHead
+        .mockResolvedValueOnce('waiting')
+        .mockResolvedValueOnce('waiting')
+        .mockResolvedValueOnce('head')
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(mockQueue.checkHead).toHaveBeenCalledTimes(3)
+      // Bucket is only consumed once we reach the head.
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('refreshes the heartbeat while waiting at the head of the queue', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+
+      // We need the wait loop to iterate long enough for HEARTBEAT_REFRESH_INTERVAL_MS
+      // to elapse. Use fake timers so we don't actually sleep.
+      vi.useFakeTimers()
+      try {
+        // Queue says we're waiting forever — except after some time we're at head.
+        mockQueue.checkHead.mockImplementation(async () => {
+          // Advance past the heartbeat interval each time we poll, then say we're up.
+          vi.advanceTimersByTime(15_000)
+          return mockQueue.checkHead.mock.calls.length >= 2 ? 'head' : 'waiting'
+        })
+
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1'
+        )
+        // Drain pending timers so the sleep() resolves.
+        await vi.runAllTimersAsync()
+        await promise
+
+        expect(mockQueue.refreshHeartbeat).toHaveBeenCalled()
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('returns 429 when the queue wait exceeds the cap', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      mockQueue.checkHead.mockResolvedValue('waiting')
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1'
+        )
+        // Burn past the 5-minute cap.
+        await vi.advanceTimersByTimeAsync(6 * 60 * 1000)
+        const result = await promise
+
+        expect(result.success).toBe(false)
+        expect(result.billingActorRateLimited).toBe(true)
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('treats "missing" status as proceed (queue evicted, fall through to bucket race)', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue(allowed)
+      mockQueue.checkHead.mockResolvedValueOnce('missing')
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+    })
+  })
+
+  describe('execution-budget-bounded waits', () => {
+    it('bails immediately when the execution signal is already aborted', async () => {
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 100),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1',
+        AbortSignal.abort()
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      // Aborted budget => give up on the first bucket check rather than looping.
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('stops waiting promptly when the signal aborts mid-sleep', async () => {
+      // Bucket reports a long refill, so the wait sleeps up to the heartbeat cap (10s).
+      // Aborting mid-sleep must wake the wait within a tick, not after the full interval.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 10_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      const controller = new AbortController()
+      const start = Date.now()
+      const promise = rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        perRequestRateLimit,
+        'workspace-1',
+        controller.signal
+      )
+      // Let the first bucket check run and the sleep begin, then abort.
+      await sleep(20)
+      controller.abort()
+      const result = await promise
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      // Resolved well before the 10s capped sleep would otherwise have elapsed.
+      expect(Date.now() - start).toBeLessThan(2000)
+    })
+
+    it('keeps waiting past the no-signal fallback cap while the signal is live', async () => {
+      // A live (non-aborted) signal means the run still has budget, so the wait must not
+      // 429 at the 5-minute MAX_QUEUE_WAIT_MS fallback. The bucket frees up after ~7 min.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 10_000),
+      }
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1',
+          new AbortController().signal
+        )
+        // Burn well past the 5-minute fallback cap — without a signal this would have 429'd.
+        await vi.advanceTimersByTimeAsync(7 * 60 * 1000)
+        mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+        await vi.advanceTimersByTimeAsync(HEARTBEAT_REFRESH_INTERVAL_MS)
+        const result = await promise
+
+        expect(result.success).toBe(true)
+        expect(result.key).toBe('test-key-1')
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
+    it('refreshes the heartbeat during a long low-RPM bucket wait', async () => {
+      // Provider with a long refill (retryAfterMs >> heartbeat TTL). The sleep must be
+      // capped so the heartbeat is renewed and the head is not reaped mid-wait.
+      const blocked: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      const allowedResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60_000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(blocked)
+
+      vi.useFakeTimers()
+      try {
+        const promise = rateLimiter.acquireKey(
+          testProvider,
+          envKeyPrefix,
+          perRequestRateLimit,
+          'workspace-1',
+          new AbortController().signal
+        )
+        await vi.advanceTimersByTimeAsync(3 * HEARTBEAT_REFRESH_INTERVAL_MS)
+        mockAdapter.consumeTokens.mockResolvedValue(allowedResult)
+        await vi.advanceTimersByTimeAsync(HEARTBEAT_REFRESH_INTERVAL_MS)
+        await promise
+
+        expect(mockQueue.refreshHeartbeat).toHaveBeenCalled()
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+  })
+
+  describe('acquireKey with custom rate limit', () => {
+    const customRateLimit: CustomRateLimit = {
+      mode: 'custom',
+      requestsPerMinute: 5,
+      dimensions: [
+        {
+          name: 'tokens',
+          limitPerMinute: 1000,
+          extractUsage: (_params, response) => (response.tokenCount as number) ?? 0,
+        },
+      ],
+    }
+
+    it('should enforce requestsPerMinute for custom mode when wait exceeds the cap', async () => {
+      const rateLimitedResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(rateLimitedResult)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('Rate limit exceeded')
+    })
+
+    it('should allow request when actor request limit and dimensions have budget', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const budgetAvailable: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(budgetAvailable)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(true)
+      expect(result.key).toBe('test-key-1')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledTimes(1)
+    })
+
+    it('should block request when a dimension wait exceeds the cap', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const depleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(depleted)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('tokens')
+    })
+
+    it('should wait for dimension capacity then succeed when budget refills', async () => {
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 4,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const depleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 100),
+      }
+      const refilled: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValueOnce(depleted).mockResolvedValueOnce(refilled)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        customRateLimit,
+        'workspace-dim-wait'
+      )
+
+      expect(result.success).toBe(true)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledTimes(2)
+    })
+
+    it('should pre-check all dimensions and block on first depleted one', async () => {
+      const multiDimensionConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+          {
+            name: 'search_units',
+            limitPerMinute: 50,
+            extractUsage: (_p, r) => (r.searchUnits as number) ?? 0,
+          },
+        ],
+      }
+
+      const allowedConsume: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(allowedConsume)
+
+      const tokensBudget: TokenStatus = {
+        tokensAvailable: 500,
+        maxTokens: 2000,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      const searchUnitsDepleted: TokenStatus = {
+        tokensAvailable: 0,
+        maxTokens: 100,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + RETRY_PAST_CAP_MS),
+      }
+      mockAdapter.getTokenStatus
+        .mockResolvedValueOnce(tokensBudget)
+        .mockResolvedValueOnce(searchUnitsDepleted)
+
+      const result = await rateLimiter.acquireKey(
+        testProvider,
+        envKeyPrefix,
+        multiDimensionConfig,
+        'workspace-1'
+      )
+
+      expect(result.success).toBe(false)
+      expect(result.billingActorRateLimited).toBe(true)
+      expect(result.error).toContain('search_units')
+    })
+  })
+
+  describe('reportUsage', () => {
+    const customConfig: CustomRateLimit = {
+      mode: 'custom',
+      requestsPerMinute: 5,
+      dimensions: [
+        {
+          name: 'tokens',
+          limitPerMinute: 1000,
+          extractUsage: (_params, response) => (response.tokenCount as number) ?? 0,
+        },
+      ],
+    }
+
+    it('should consume actual tokens from dimension bucket after execution', async () => {
+      const consumeResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 850,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(consumeResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 150 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].name).toBe('tokens')
+      expect(result.dimensions[0].consumed).toBe(150)
+      expect(result.dimensions[0].allowed).toBe(true)
+      expect(result.dimensions[0].tokensRemaining).toBe(850)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        'hosted:exa:actor:workspace-1:tokens',
+        150,
+        expect.objectContaining({ maxTokens: 2000, refillRate: 1000 })
+      )
+    })
+
+    it('should handle overdrawn bucket gracefully (optimistic concurrency)', async () => {
+      const overdrawnResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(overdrawnResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 500 }
+      )
+
+      expect(result.dimensions[0].allowed).toBe(false)
+      expect(result.dimensions[0].consumed).toBe(500)
+    })
+
+    it('should skip consumption when extractUsage returns 0', async () => {
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 0 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].consumed).toBe(0)
+      expect(mockAdapter.consumeTokens).not.toHaveBeenCalled()
+    })
+
+    it('should handle multiple dimensions independently', async () => {
+      const multiConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+          {
+            name: 'search_units',
+            limitPerMinute: 50,
+            extractUsage: (_p, r) => (r.searchUnits as number) ?? 0,
+          },
+        ],
+      }
+
+      const tokensConsumed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 800,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      const searchConsumed: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 47,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens
+        .mockResolvedValueOnce(tokensConsumed)
+        .mockResolvedValueOnce(searchConsumed)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        multiConfig,
+        {},
+        { tokenCount: 200, searchUnits: 3 }
+      )
+
+      expect(result.dimensions).toHaveLength(2)
+      expect(result.dimensions[0]).toEqual({
+        name: 'tokens',
+        consumed: 200,
+        allowed: true,
+        tokensRemaining: 800,
+      })
+      expect(result.dimensions[1]).toEqual({
+        name: 'search_units',
+        consumed: 3,
+        allowed: true,
+        tokensRemaining: 47,
+      })
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(2)
+    })
+
+    it('should continue with remaining dimensions if extractUsage throws', async () => {
+      const throwingConfig: CustomRateLimit = {
+        mode: 'custom',
+        requestsPerMinute: 10,
+        dimensions: [
+          {
+            name: 'broken',
+            limitPerMinute: 100,
+            extractUsage: () => {
+              throw new Error('extraction failed')
+            },
+          },
+          {
+            name: 'tokens',
+            limitPerMinute: 1000,
+            extractUsage: (_p, r) => (r.tokenCount as number) ?? 0,
+          },
+        ],
+      }
+
+      const consumeResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 900,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(consumeResult)
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        throwingConfig,
+        {},
+        { tokenCount: 100 }
+      )
+
+      expect(result.dimensions).toHaveLength(1)
+      expect(result.dimensions[0].name).toBe('tokens')
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledTimes(1)
+    })
+
+    it('should handle storage errors gracefully', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('db connection lost'))
+
+      const result = await rateLimiter.reportUsage(
+        testProvider,
+        'workspace-1',
+        customConfig,
+        {},
+        { tokenCount: 100 }
+      )
+
+      expect(result.dimensions).toHaveLength(0)
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/hosted-key/hosted-key-rate-limiter.ts

@@ -0,0 +1,695 @@
+import { createLogger } from '@sim/logger'
+import { sleep } from '@sim/utils/helpers'
+import { generateShortId } from '@sim/utils/id'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import {
+  createStorageAdapter,
+  type RateLimitStorageAdapter,
+  type TokenBucketConfig,
+} from '@/lib/core/rate-limiter/storage'
+import { PlatformEvents } from '@/lib/core/telemetry'
+import { getHostedKeyQueue, HEARTBEAT_REFRESH_INTERVAL_MS, type HostedKeyQueue } from './queue'
+import {
+  type AcquireKeyResult,
+  type CustomRateLimit,
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  type HostedKeyRateLimitConfig,
+  type ReportUsageResult,
+  toTokenBucketConfig,
+} from './types'
+
+const logger = createLogger('HostedKeyRateLimiter')
+
+/**
+ * Fallback ceiling on how long a hosted-key acquisition waits for the per-workspace
+ * bucket to refill when no execution `AbortSignal` is available to bound the wait (e.g.
+ * a caller without a wired execution deadline, or Redis no-op mode). When a signal IS
+ * provided, the wait is instead bounded by the surrounding execution budget — the signal
+ * fires when the run hits its plan timeout or is cancelled — and this constant no longer
+ * applies (see {@link ABSOLUTE_MAX_QUEUE_WAIT_MS} for the backstop in that case).
+ */
+const MAX_QUEUE_WAIT_MS = 5 * 60 * 1000
+
+/**
+ * Hard safety ceiling applied even when an execution `AbortSignal` is present, in case the
+ * signal never fires. Matches the longest possible execution budget (enterprise async) so
+ * it never truncates a legitimately long-running background run before its own deadline.
+ */
+const ABSOLUTE_MAX_QUEUE_WAIT_MS = getMaxExecutionTimeout()
+
+/**
+ * Floor on per-iteration sleep when the bucket reports `retryAfterMs <= 0`,
+ * which can happen due to clock skew or sub-millisecond resets. Prevents a
+ * tight retry loop hammering the storage adapter.
+ */
+const MIN_QUEUE_RETRY_DELAY_MS = 50
+
+/**
+ * Poll interval while waiting to reach the head of the FIFO queue. 200ms balances
+ * acquisition latency (worst-case wait for advancement is one poll period) against
+ * Redis load — at this cadence, N waiters generate N×5 EVAL/sec, which is fine for
+ * the typical low-tens contention. Revisit if telemetry shows hot Redis under load.
+ */
+const QUEUE_HEAD_POLL_MS = 200
+
+/**
+ * Sleep for `ms`, resolving early if `signal` aborts. Cleans up its own timer and listener
+ * so neither leaks. Callers don't need to distinguish an early (aborted) return from a normal
+ * one — the surrounding wait loop re-checks its budget immediately after and bails when the
+ * signal has fired. Falls back to a plain sleep when no signal is provided.
+ */
+function interruptibleSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (!signal) return sleep(ms)
+  if (signal.aborted) return Promise.resolve()
+  return new Promise<void>((resolve) => {
+    const onAbort = () => {
+      clearTimeout(timer)
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }
+    const timer = setTimeout(() => {
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }, ms)
+    signal.addEventListener('abort', onAbort, { once: true })
+    // Catch an abort that fired between the guard above and addEventListener.
+    if (signal.aborted) onAbort()
+  })
+}
+
+/**
+ * Resolves env var names for a numbered key prefix using a `{PREFIX}_COUNT` env var.
+ * E.g. with `EXA_API_KEY_COUNT=5`, returns `['EXA_API_KEY_1', ..., 'EXA_API_KEY_5']`.
+ */
+function resolveEnvKeys(prefix: string): string[] {
+  const count = Number.parseInt(process.env[`${prefix}_COUNT`] || '0', 10)
+  const names: string[] = []
+  for (let i = 1; i <= count; i++) {
+    names.push(`${prefix}_${i}`)
+  }
+  return names
+}
+
+/** Dimension name for per-billing-actor request rate limiting */
+const ACTOR_REQUESTS_DIMENSION = 'actor_requests'
+
+/**
+ * Information about an available hosted key
+ */
+interface AvailableKey {
+  key: string
+  keyIndex: number
+  envVarName: string
+}
+
+/**
+ * Mutable heartbeat bookkeeping shared across every wait phase of a single `acquireKey`
+ * call. Carrying one `lastHeartbeatAt` across the queue-head wait and the bucket waits
+ * ensures the heartbeat-refresh cadence reflects the *actual* last write to Redis rather
+ * than resetting per phase, which could otherwise let the ticket heartbeat lapse and get
+ * reaped mid-wait (breaking FIFO ordering).
+ */
+interface WaitState {
+  /** Epoch ms of the last heartbeat write to Redis. */
+  lastHeartbeatAt: number
+}
+
+/**
+ * HostedKeyRateLimiter provides:
+ * 1. Per-billing-actor rate limiting (enforced - blocks actors who exceed their limit)
+ * 2. Round-robin key selection (distributes requests evenly across keys)
+ * 3. Post-execution dimension usage tracking for custom rate limits
+ *
+ * The billing actor is typically a workspace ID, meaning rate limits are shared
+ * across all users within the same workspace.
+ */
+export class HostedKeyRateLimiter {
+  private storage: RateLimitStorageAdapter
+  private queue: HostedKeyQueue
+  /** Round-robin counter per provider for even key distribution */
+  private roundRobinCounters = new Map<string, number>()
+
+  constructor(storage?: RateLimitStorageAdapter, queue?: HostedKeyQueue) {
+    this.storage = storage ?? createStorageAdapter()
+    this.queue = queue ?? getHostedKeyQueue()
+  }
+
+  private buildActorStorageKey(provider: string, billingActorId: string): string {
+    return `hosted:${provider}:actor:${billingActorId}:${ACTOR_REQUESTS_DIMENSION}`
+  }
+
+  private buildDimensionStorageKey(
+    provider: string,
+    billingActorId: string,
+    dimensionName: string
+  ): string {
+    return `hosted:${provider}:actor:${billingActorId}:${dimensionName}`
+  }
+
+  private getAvailableKeys(envKeys: string[]): AvailableKey[] {
+    const keys: AvailableKey[] = []
+    for (let i = 0; i < envKeys.length; i++) {
+      const envVarName = envKeys[i]
+      const key = process.env[envVarName]
+      if (key) {
+        keys.push({ key, keyIndex: i, envVarName })
+      }
+    }
+    return keys
+  }
+
+  /**
+   * Build a token bucket config for the per-billing-actor request rate limit.
+   * Works for both `per_request` and `custom` modes since both define `requestsPerMinute`.
+   */
+  private getActorRateLimitConfig(config: HostedKeyRateLimitConfig): TokenBucketConfig | null {
+    if (!config.requestsPerMinute) return null
+    return toTokenBucketConfig(
+      config.requestsPerMinute,
+      config.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+      DEFAULT_WINDOW_MS
+    )
+  }
+
+  /**
+   * Check and consume billing actor request rate limit. Returns null if allowed, or retry info if blocked.
+   */
+  private async checkActorRateLimit(
+    provider: string,
+    billingActorId: string,
+    config: HostedKeyRateLimitConfig
+  ): Promise<{ rateLimited: true; retryAfterMs: number } | null> {
+    const bucketConfig = this.getActorRateLimitConfig(config)
+    if (!bucketConfig) return null
+
+    const storageKey = this.buildActorStorageKey(provider, billingActorId)
+
+    try {
+      const result = await this.storage.consumeTokens(storageKey, 1, bucketConfig)
+      if (!result.allowed) {
+        const retryAfterMs = Math.max(0, result.resetAt.getTime() - Date.now())
+        logger.info(`Billing actor ${billingActorId} rate limited for ${provider}`, {
+          provider,
+          billingActorId,
+          retryAfterMs,
+          tokensRemaining: result.tokensRemaining,
+        })
+        return { rateLimited: true, retryAfterMs }
+      }
+      return null
+    } catch (error) {
+      logger.error(`Error checking billing actor rate limit for ${provider}`, {
+        error,
+        billingActorId,
+      })
+      return null
+    }
+  }
+
+  /**
+   * Pre-check that the billing actor has available budget in all custom dimensions.
+   * Does NOT consume tokens -- just verifies the actor isn't already depleted.
+   * Returns retry info for the most restrictive exhausted dimension, or null if all pass.
+   */
+  private async preCheckDimensions(
+    provider: string,
+    billingActorId: string,
+    config: CustomRateLimit
+  ): Promise<{ rateLimited: true; retryAfterMs: number; dimension: string } | null> {
+    for (const dimension of config.dimensions) {
+      const storageKey = this.buildDimensionStorageKey(provider, billingActorId, dimension.name)
+      const bucketConfig = toTokenBucketConfig(
+        dimension.limitPerMinute,
+        dimension.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+        DEFAULT_WINDOW_MS
+      )
+
+      try {
+        const status = await this.storage.getTokenStatus(storageKey, bucketConfig)
+        if (status.tokensAvailable < 1) {
+          const retryAfterMs = Math.max(0, status.nextRefillAt.getTime() - Date.now())
+          logger.info(
+            `Billing actor ${billingActorId} exhausted dimension ${dimension.name} for ${provider}`,
+            {
+              provider,
+              billingActorId,
+              dimension: dimension.name,
+              tokensAvailable: status.tokensAvailable,
+              retryAfterMs,
+            }
+          )
+          return { rateLimited: true, retryAfterMs, dimension: dimension.name }
+        }
+      } catch (error) {
+        logger.error(`Error pre-checking dimension ${dimension.name} for ${provider}`, {
+          error,
+          billingActorId,
+        })
+      }
+    }
+    return null
+  }
+
+  /**
+   * Acquire an available key via round-robin selection.
+   *
+   * For both modes:
+   *   1. Per-billing-actor request rate limiting (enforced): the call enqueues itself
+   *      onto a per-workspace+provider FIFO queue. Only the head of the queue attempts
+   *      to consume from the token bucket, guaranteeing strict ordering across callers
+   *      within a workspace. Different workspaces have independent queues and don't
+   *      block each other.
+   *   2. Round-robin key selection: cycles through available keys for even distribution
+   *
+   * For `custom` mode additionally:
+   *   3. Pre-checks dimension budgets: head waits on dimension refill the same way it
+   *      waits on actor request capacity.
+   *
+   * The wait is bounded by the surrounding execution budget: when `signal` is provided it
+   * fires at the run's plan timeout (or on cancellation), so a queued call uses its full
+   * available budget rather than a flat cap. When no signal is available the wait falls
+   * back to `MAX_QUEUE_WAIT_MS`. On exhaustion the call returns today's 429 result. The
+   * ticket is removed from the queue on exit regardless of success or failure.
+   *
+   * @param envKeyPrefix - Env var prefix (e.g. 'EXA_API_KEY'). Keys resolved via `{prefix}_COUNT`.
+   * @param billingActorId - The billing actor (typically workspace ID) to rate limit against
+   * @param signal - Optional execution `AbortSignal`; bounds the queue wait to the run's budget.
+   */
+  async acquireKey(
+    provider: string,
+    envKeyPrefix: string,
+    config: HostedKeyRateLimitConfig,
+    billingActorId: string,
+    signal?: AbortSignal
+  ): Promise<AcquireKeyResult> {
+    const ticketId = generateShortId()
+    const startedAt = Date.now()
+    const waitState: WaitState = { lastHeartbeatAt: startedAt }
+    const enqueueResult = await this.queue.enqueue(provider, billingActorId, ticketId)
+
+    try {
+      // Wait for our turn at the head of the queue (no-op when Redis unavailable).
+      const headStatus = await this.waitForQueueHead(
+        provider,
+        billingActorId,
+        ticketId,
+        startedAt,
+        waitState,
+        signal
+      )
+      if (headStatus.timedOut) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'queue_position',
+        })
+        return {
+          success: false,
+          billingActorRateLimited: true,
+          retryAfterMs: MAX_QUEUE_WAIT_MS,
+          error: `Rate limit exceeded — request waited too long in the queue. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+        }
+      }
+
+      let dimensionWaited = false
+
+      if (config.requestsPerMinute) {
+        const rateLimitResult = await this.waitForActorCapacity(
+          provider,
+          billingActorId,
+          ticketId,
+          config,
+          startedAt,
+          waitState,
+          signal
+        )
+        if (rateLimitResult.rateLimited) {
+          return {
+            success: false,
+            billingActorRateLimited: true,
+            retryAfterMs: rateLimitResult.retryAfterMs,
+            error: `Rate limit exceeded. Please wait ${Math.ceil(rateLimitResult.retryAfterMs / 1000)} seconds. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+          }
+        }
+      }
+
+      if (config.mode === 'custom' && config.dimensions.length > 0) {
+        const dimensionResult = await this.waitForDimensionCapacity(
+          provider,
+          billingActorId,
+          ticketId,
+          config,
+          startedAt,
+          waitState,
+          signal
+        )
+        if (dimensionResult.rateLimited) {
+          return {
+            success: false,
+            billingActorRateLimited: true,
+            retryAfterMs: dimensionResult.retryAfterMs,
+            error: `Rate limit exceeded for ${dimensionResult.dimension}. Please wait ${Math.ceil(dimensionResult.retryAfterMs / 1000)} seconds. If you're getting throttled frequently, consider adding your own API key under Settings > BYOK to avoid shared rate limits.`,
+          }
+        }
+        dimensionWaited = dimensionResult.waited
+      }
+
+      const totalWaitedMs = Date.now() - startedAt
+      if (enqueueResult.enabled && (enqueueResult.position > 0 || totalWaitedMs > 100)) {
+        // Attribute the wait to its dominant cause: queue depth takes precedence (it's
+        // reported alongside queuePosition), otherwise the bucket phase that actually slept.
+        const reason: 'queue_position' | 'actor_requests' | 'dimension' =
+          enqueueResult.position > 0
+            ? 'queue_position'
+            : dimensionWaited
+              ? 'dimension'
+              : 'actor_requests'
+        PlatformEvents.hostedKeyQueueWaited({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: totalWaitedMs,
+          attempts: 1,
+          reason,
+          queuePosition: enqueueResult.position,
+        })
+      }
+
+      const envKeys = resolveEnvKeys(envKeyPrefix)
+      const availableKeys = this.getAvailableKeys(envKeys)
+
+      if (availableKeys.length === 0) {
+        logger.warn(`No hosted keys configured for provider ${provider}`)
+        return {
+          success: false,
+          error: `No hosted keys configured for ${provider}`,
+        }
+      }
+
+      const counter = this.roundRobinCounters.get(provider) ?? 0
+      const selected = availableKeys[counter % availableKeys.length]
+      this.roundRobinCounters.set(provider, counter + 1)
+
+      logger.debug(`Selected hosted key for ${provider}`, {
+        provider,
+        keyIndex: selected.keyIndex,
+        envVarName: selected.envVarName,
+      })
+
+      return {
+        success: true,
+        key: selected.key,
+        keyIndex: selected.keyIndex,
+        envVarName: selected.envVarName,
+      }
+    } finally {
+      // Always remove our ticket so the next caller can advance, regardless of whether
+      // we succeeded, hit the cap, or threw. Best-effort; safe to call multiple times.
+      await this.queue.dequeue(provider, billingActorId, ticketId)
+    }
+  }
+
+  /**
+   * Remaining time budget for waiting, in milliseconds. When an execution `AbortSignal` is
+   * present it governs the wait: the budget is exhausted the moment the signal aborts (the
+   * run hit its plan timeout or was cancelled), with {@link ABSOLUTE_MAX_QUEUE_WAIT_MS} as a
+   * backstop should the signal never fire. Without a signal we fall back to the flat
+   * {@link MAX_QUEUE_WAIT_MS} ceiling.
+   */
+  private remainingWaitBudgetMs(startedAt: number, signal?: AbortSignal): number {
+    if (signal?.aborted) return 0
+    const ceiling = signal ? ABSOLUTE_MAX_QUEUE_WAIT_MS : MAX_QUEUE_WAIT_MS
+    return ceiling - (Date.now() - startedAt)
+  }
+
+  /** Refresh the ticket heartbeat if the refresh interval has elapsed since the last write. */
+  private async maybeRefreshHeartbeat(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    waitState: WaitState
+  ): Promise<void> {
+    if (Date.now() - waitState.lastHeartbeatAt >= HEARTBEAT_REFRESH_INTERVAL_MS) {
+      await this.queue.refreshHeartbeat(provider, billingActorId, ticketId)
+      waitState.lastHeartbeatAt = Date.now()
+    }
+  }
+
+  /**
+   * Sleep before the next bucket re-check, refreshing the heartbeat first if due. The sleep
+   * is capped at {@link HEARTBEAT_REFRESH_INTERVAL_MS} so that no single wait can outlive the
+   * heartbeat TTL — even when the bucket reports a long `retryAfterMs` (e.g. low-RPM
+   * providers). Without this cap a multi-second sleep could let the heartbeat lapse, the
+   * head get reaped as dead, and a second caller advance and race us for the bucket. The
+   * sleep also resolves early if `signal` aborts, so a cancelled/timed-out run stops waiting
+   * promptly rather than overshooting by up to the cap.
+   */
+  private async heartbeatAwareSleep(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    desiredMs: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<void> {
+    await this.maybeRefreshHeartbeat(provider, billingActorId, ticketId, waitState)
+    const sleepMs = Math.min(
+      Math.max(MIN_QUEUE_RETRY_DELAY_MS, desiredMs),
+      HEARTBEAT_REFRESH_INTERVAL_MS
+    )
+    await interruptibleSleep(sleepMs, signal)
+  }
+
+  /**
+   * Block until our ticket reaches the head of the queue. Refreshes the heartbeat on a
+   * regular cadence so we don't get reaped as dead. Returns `timedOut: true` once the wait
+   * budget is exhausted before reaching the head (see {@link remainingWaitBudgetMs}).
+   *
+   * No-op when Redis is unavailable (queue.enqueue returns enabled=false and checkHead
+   * always returns 'head').
+   */
+  private async waitForQueueHead(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<{ timedOut: boolean }> {
+    while (true) {
+      const status = await this.queue.checkHead(provider, billingActorId, ticketId)
+      if (status === 'head') return { timedOut: false }
+
+      // 'missing' shouldn't normally happen — the queue list TTL (10min) outlives a typical
+      // wait — but if it does (e.g. Redis flushed mid-wait), treat as "you're up" so the
+      // caller proceeds to the bucket race rather than hanging forever.
+      if (status === 'missing') return { timedOut: false }
+
+      if (this.remainingWaitBudgetMs(startedAt, signal) <= 0) {
+        return { timedOut: true }
+      }
+
+      await this.maybeRefreshHeartbeat(provider, billingActorId, ticketId, waitState)
+      await interruptibleSleep(QUEUE_HEAD_POLL_MS, signal)
+    }
+  }
+
+  /**
+   * Wait for actor request-rate capacity. Called once we're at the head of the FIFO
+   * queue, so other callers can't race us for the next token — they're blocked behind us
+   * at queue level. Re-checks the bucket until the wait budget is exhausted (accounting for
+   * time already spent waiting in the queue). `waited` reports whether the loop ever slept.
+   */
+  private async waitForActorCapacity(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    config: HostedKeyRateLimitConfig,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<
+    { rateLimited: false; waited: boolean } | { rateLimited: true; retryAfterMs: number }
+  > {
+    let waited = false
+
+    while (true) {
+      const result = await this.checkActorRateLimit(provider, billingActorId, config)
+      if (!result) return { rateLimited: false, waited }
+
+      const remaining = this.remainingWaitBudgetMs(startedAt, signal)
+      if (remaining <= 0 || result.retryAfterMs > remaining) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'actor_requests',
+        })
+        return { rateLimited: true, retryAfterMs: result.retryAfterMs }
+      }
+
+      waited = true
+      await this.heartbeatAwareSleep(
+        provider,
+        billingActorId,
+        ticketId,
+        result.retryAfterMs,
+        waitState,
+        signal
+      )
+    }
+  }
+
+  /**
+   * Wait for custom-mode dimension capacity. `preCheckDimensions` is read-only — it does
+   * not consume — so re-running it after a sleep is safe and does not double-charge.
+   * Post-execution `reportUsage` performs the actual consumption. `waited` reports whether
+   * the loop ever slept.
+   */
+  private async waitForDimensionCapacity(
+    provider: string,
+    billingActorId: string,
+    ticketId: string,
+    config: CustomRateLimit,
+    startedAt: number,
+    waitState: WaitState,
+    signal?: AbortSignal
+  ): Promise<
+    | { rateLimited: false; waited: boolean }
+    | { rateLimited: true; retryAfterMs: number; dimension: string }
+  > {
+    let waited = false
+
+    while (true) {
+      const result = await this.preCheckDimensions(provider, billingActorId, config)
+      if (!result) return { rateLimited: false, waited }
+
+      const remaining = this.remainingWaitBudgetMs(startedAt, signal)
+      if (remaining <= 0 || result.retryAfterMs > remaining) {
+        PlatformEvents.hostedKeyQueueWaitExceeded({
+          provider,
+          workspaceId: billingActorId,
+          waitedMs: Date.now() - startedAt,
+          reason: 'dimension',
+          dimension: result.dimension,
+        })
+        return {
+          rateLimited: true,
+          retryAfterMs: result.retryAfterMs,
+          dimension: result.dimension,
+        }
+      }
+
+      waited = true
+      await this.heartbeatAwareSleep(
+        provider,
+        billingActorId,
+        ticketId,
+        result.retryAfterMs,
+        waitState,
+        signal
+      )
+    }
+  }
+
+  /**
+   * Report actual usage after successful tool execution (custom mode only).
+   * Calls `extractUsage` on each dimension and consumes the actual token count.
+   * This is the "post-execution" phase of the optimistic two-phase approach.
+   */
+  async reportUsage(
+    provider: string,
+    billingActorId: string,
+    config: CustomRateLimit,
+    params: Record<string, unknown>,
+    response: Record<string, unknown>
+  ): Promise<ReportUsageResult> {
+    const results: ReportUsageResult['dimensions'] = []
+
+    for (const dimension of config.dimensions) {
+      let usage: number
+      try {
+        usage = dimension.extractUsage(params, response)
+      } catch (error) {
+        logger.error(`Failed to extract usage for dimension ${dimension.name}`, {
+          provider,
+          billingActorId,
+          error,
+        })
+        continue
+      }
+
+      if (usage <= 0) {
+        results.push({
+          name: dimension.name,
+          consumed: 0,
+          allowed: true,
+          tokensRemaining: 0,
+        })
+        continue
+      }
+
+      const storageKey = this.buildDimensionStorageKey(provider, billingActorId, dimension.name)
+      const bucketConfig = toTokenBucketConfig(
+        dimension.limitPerMinute,
+        dimension.burstMultiplier ?? DEFAULT_BURST_MULTIPLIER,
+        DEFAULT_WINDOW_MS
+      )
+
+      try {
+        const consumeResult = await this.storage.consumeTokens(storageKey, usage, bucketConfig)
+
+        results.push({
+          name: dimension.name,
+          consumed: usage,
+          allowed: consumeResult.allowed,
+          tokensRemaining: consumeResult.tokensRemaining,
+        })
+
+        if (!consumeResult.allowed) {
+          logger.warn(
+            `Dimension ${dimension.name} overdrawn for ${provider} (optimistic concurrency)`,
+            { provider, billingActorId, usage, tokensRemaining: consumeResult.tokensRemaining }
+          )
+        }
+
+        logger.debug(`Consumed ${usage} from dimension ${dimension.name} for ${provider}`, {
+          provider,
+          billingActorId,
+          usage,
+          allowed: consumeResult.allowed,
+          tokensRemaining: consumeResult.tokensRemaining,
+        })
+      } catch (error) {
+        logger.error(`Failed to consume tokens for dimension ${dimension.name}`, {
+          provider,
+          billingActorId,
+          usage,
+          error,
+        })
+      }
+    }
+
+    return { dimensions: results }
+  }
+}
+
+let cachedInstance: HostedKeyRateLimiter | null = null
+
+/**
+ * Get the singleton HostedKeyRateLimiter instance
+ */
+export function getHostedKeyRateLimiter(): HostedKeyRateLimiter {
+  if (!cachedInstance) {
+    cachedInstance = new HostedKeyRateLimiter()
+  }
+  return cachedInstance
+}
+
+/**
+ * Reset the cached rate limiter (for testing)
+ */
+export function resetHostedKeyRateLimiter(): void {
+  cachedInstance = null
+}

apps/sim/lib/core/rate-limiter/hosted-key/index.ts

@@ -0,0 +1,11 @@
+export {
+  getHostedKeyRateLimiter,
+  HostedKeyRateLimiter,
+  resetHostedKeyRateLimiter,
+} from './hosted-key-rate-limiter'
+export {
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  type HostedKeyRateLimitConfig,
+  toTokenBucketConfig,
+} from './types'

apps/sim/lib/core/rate-limiter/hosted-key/queue.test.ts

@@ -0,0 +1,234 @@
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import { HostedKeyQueue } from './queue'
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+interface MockPipeline {
+  rpush: Mock
+  expire: Mock
+  set: Mock
+  lrem: Mock
+  del: Mock
+  exec: Mock
+}
+
+interface MockRedis {
+  multi: Mock
+  set: Mock
+  eval: Mock
+  pipeline: MockPipeline
+}
+
+function createFakeRedis(): MockRedis {
+  const pipeline: MockPipeline = {
+    rpush: vi.fn(),
+    expire: vi.fn(),
+    set: vi.fn(),
+    lrem: vi.fn(),
+    del: vi.fn(),
+    exec: vi.fn(),
+  }
+  // Pipeline methods return the pipeline for chaining.
+  pipeline.rpush.mockReturnValue(pipeline)
+  pipeline.expire.mockReturnValue(pipeline)
+  pipeline.set.mockReturnValue(pipeline)
+  pipeline.lrem.mockReturnValue(pipeline)
+  pipeline.del.mockReturnValue(pipeline)
+
+  return {
+    multi: vi.fn(() => pipeline),
+    set: vi.fn(),
+    eval: vi.fn(),
+    pipeline,
+  }
+}
+
+const provider = 'exa'
+const workspaceId = 'workspace-1'
+const ticketId = 'ticket-1'
+
+describe('HostedKeyQueue', () => {
+  let queue: HostedKeyQueue
+  let mockRedis: MockRedis
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockRedis = createFakeRedis()
+    redisConfigMockFns.mockGetRedisClient.mockReturnValue(mockRedis)
+    queue = new HostedKeyQueue()
+  })
+
+  describe('enqueue', () => {
+    it('returns position 0 when first in line', async () => {
+      // RPUSH returns new list length; first push -> 1.
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 1],
+        [null, 1],
+        [null, 'OK'],
+      ])
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result).toEqual({ position: 0, enabled: true })
+      expect(mockRedis.pipeline.rpush).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        ticketId
+      )
+      expect(mockRedis.pipeline.set).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1',
+        '1',
+        'EX',
+        expect.any(Number)
+      )
+    })
+
+    it('returns higher position when others are ahead', async () => {
+      // Length 5 after push -> position 4.
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 5],
+        [null, 1],
+        [null, 'OK'],
+      ])
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result.position).toBe(4)
+    })
+
+    it('falls back to enabled=false when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result).toEqual({ position: 0, enabled: false })
+    })
+
+    it('falls back to enabled=false on Redis error', async () => {
+      mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost'))
+
+      const result = await queue.enqueue(provider, workspaceId, ticketId)
+
+      expect(result.enabled).toBe(false)
+    })
+  })
+
+  describe('checkHead', () => {
+    it('returns "head" when our ticket is at the head', async () => {
+      mockRedis.eval.mockResolvedValueOnce('head')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+
+    it('returns "waiting" when someone else is the head', async () => {
+      mockRedis.eval.mockResolvedValueOnce('waiting')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('waiting')
+    })
+
+    it('returns "missing" when our ticket is not in the queue', async () => {
+      mockRedis.eval.mockResolvedValueOnce('missing')
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('missing')
+    })
+
+    it('passes queue list key, heartbeat prefix, and ticketId to the Lua script', async () => {
+      mockRedis.eval.mockResolvedValueOnce('head')
+
+      await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(mockRedis.eval).toHaveBeenCalledWith(
+        expect.stringContaining('lindex'),
+        1,
+        'hosted-queue:exa:workspace-1',
+        'hosted-queue-tkt:exa:workspace-1:',
+        ticketId
+      )
+    })
+
+    it('fails open to "head" on Redis error so callers do not hang', async () => {
+      mockRedis.eval.mockRejectedValueOnce(new Error('boom'))
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+
+    it('returns "head" no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      const status = await queue.checkHead(provider, workspaceId, ticketId)
+
+      expect(status).toBe('head')
+    })
+  })
+
+  describe('refreshHeartbeat', () => {
+    it('writes the heartbeat key with TTL and re-extends the queue list TTL', async () => {
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 'OK'],
+        [null, 1],
+      ])
+
+      await queue.refreshHeartbeat(provider, workspaceId, ticketId)
+
+      expect(mockRedis.pipeline.set).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1',
+        '1',
+        'EX',
+        expect.any(Number)
+      )
+      expect(mockRedis.pipeline.expire).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        expect.any(Number)
+      )
+      expect(mockRedis.pipeline.exec).toHaveBeenCalledTimes(1)
+    })
+
+    it('is a no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      await expect(queue.refreshHeartbeat(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+      expect(mockRedis.multi).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('dequeue', () => {
+    it('removes the ticket from the list and deletes the heartbeat', async () => {
+      mockRedis.pipeline.exec.mockResolvedValueOnce([
+        [null, 1],
+        [null, 1],
+      ])
+
+      await queue.dequeue(provider, workspaceId, ticketId)
+
+      expect(mockRedis.pipeline.lrem).toHaveBeenCalledWith(
+        'hosted-queue:exa:workspace-1',
+        1,
+        ticketId
+      )
+      expect(mockRedis.pipeline.del).toHaveBeenCalledWith(
+        'hosted-queue-tkt:exa:workspace-1:ticket-1'
+      )
+    })
+
+    it('is a no-op when Redis is unavailable', async () => {
+      redisConfigMockFns.mockGetRedisClient.mockReturnValueOnce(null)
+
+      await expect(queue.dequeue(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+      expect(mockRedis.multi).not.toHaveBeenCalled()
+    })
+
+    it('swallows errors so callers do not throw on cleanup', async () => {
+      mockRedis.pipeline.exec.mockRejectedValueOnce(new Error('connection lost'))
+
+      await expect(queue.dequeue(provider, workspaceId, ticketId)).resolves.toBeUndefined()
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/hosted-key/queue.ts

@@ -0,0 +1,209 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { getRedisClient } from '@/lib/core/config/redis'
+
+const logger = createLogger('HostedKeyQueue')
+
+/**
+ * Per-ticket heartbeat TTL. Refreshed by the head while it's actively waiting
+ * on the bucket. If the holder crashes, the heartbeat key expires, and the next
+ * caller sees the head as dead and removes it (lazy cleanup).
+ */
+const TICKET_HEARTBEAT_TTL_SECONDS = 30
+
+/** How often the head should refresh its heartbeat while waiting. */
+export const HEARTBEAT_REFRESH_INTERVAL_MS = 10_000
+
+/**
+ * TTL on the queue list itself. Set on enqueue and re-extended by the head's heartbeat,
+ * so a long-waiting head can't let the list expire out from under the waiters behind it.
+ * Prevents abandoned queues from sticking around forever in Redis.
+ */
+const QUEUE_LIST_TTL_SECONDS = 600
+
+const queueListKey = (provider: string, billingActorId: string): string =>
+  `hosted-queue:${provider}:${billingActorId}`
+
+const heartbeatKey = (provider: string, billingActorId: string, ticketId: string): string =>
+  `hosted-queue-tkt:${provider}:${billingActorId}:${ticketId}`
+
+/**
+ * Atomically reap any dead head, then return our ticket's status. Combines what
+ * would otherwise be 3 round-trips (reap, LINDEX, LPOS) into one EVAL — meaningful
+ * because callers poll this every ~200ms while waiting in the queue.
+ *
+ * `KEYS[1]` = queue list key. `ARGV[1]` = heartbeat key prefix. `ARGV[2]` = our ticketId.
+ *
+ * Reaping is bounded: at most one dead head is removed per call. If multiple dead
+ * tickets pile up at the head, subsequent polls will clean them one by one. This
+ * keeps the script O(1) rather than O(N) and is sufficient because queue depth
+ * is bounded by concurrent callers per workspace (typically tens).
+ *
+ * Returns one of: "head", "waiting", "missing".
+ */
+const CHECK_HEAD_SCRIPT = `
+local head = redis.call("lindex", KEYS[1], 0)
+if head and redis.call("exists", ARGV[1] .. head) == 0 then
+  redis.call("lrem", KEYS[1], 1, head)
+  head = redis.call("lindex", KEYS[1], 0)
+end
+if not head then
+  return "missing"
+end
+if head == ARGV[2] then
+  return "head"
+end
+if redis.call("lpos", KEYS[1], ARGV[2]) == false then
+  return "missing"
+end
+return "waiting"
+`
+
+export interface EnqueueResult {
+  /** Position at the moment of enqueue (0 = head, you go next). */
+  position: number
+  /** Whether Redis was available — false means we're in no-op mode. */
+  enabled: boolean
+}
+
+/**
+ * Per-workspace+provider FIFO queue for hosted-key acquisitions.
+ *
+ * Callers `enqueue` to claim a position, then `waitForHead` until they're at
+ * the head, then attempt to consume from the token bucket. On success or cap
+ * exceeded, they `dequeue` to make room for the next caller.
+ *
+ * No-op when Redis is unavailable: every method returns "you're the head /
+ * empty / etc." so the rate limiter falls back to plain bucket racing.
+ */
+export class HostedKeyQueue {
+  /**
+   * Push a ticket onto the tail of the queue and write a heartbeat. Returns the
+   * position at enqueue time (0 = head, ready to proceed).
+   */
+  async enqueue(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<EnqueueResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      return { position: 0, enabled: false }
+    }
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+
+    try {
+      const pipeline = redis.multi()
+      pipeline.rpush(listKey, ticketId)
+      pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS)
+      pipeline.set(hbKey, '1', 'EX', TICKET_HEARTBEAT_TTL_SECONDS)
+      const results = await pipeline.exec()
+      // results[0] is the rpush response: [err, length]
+      const length = results?.[0] && typeof results[0][1] === 'number' ? results[0][1] : 1
+      // Position is length - 1 (just-pushed at the tail).
+      return { position: length - 1, enabled: true }
+    } catch (error) {
+      logger.warn(`Queue enqueue failed for ${listKey}`, { error: toError(error).message })
+      return { position: 0, enabled: false }
+    }
+  }
+
+  /**
+   * Check whether `ticketId` is currently at the head of the queue. If the head
+   * is a different ticket but its heartbeat has expired (caller crashed), reap
+   * it and re-check on the next poll.
+   *
+   * Returns:
+   *  - "head": you're at the head, proceed to consume from the bucket
+   *  - "waiting": someone else is the head and they're alive
+   *  - "missing": your ticket isn't in the queue at all (e.g. queue list TTL
+   *    expired); caller should re-enqueue or treat as enabled=false
+   */
+  async checkHead(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<'head' | 'waiting' | 'missing'> {
+    const redis = getRedisClient()
+    if (!redis) {
+      return 'head'
+    }
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbPrefix = `hosted-queue-tkt:${provider}:${billingActorId}:`
+
+    try {
+      const result = (await redis.eval(CHECK_HEAD_SCRIPT, 1, listKey, hbPrefix, ticketId)) as
+        | 'head'
+        | 'waiting'
+        | 'missing'
+      return result
+    } catch (error) {
+      logger.warn(`Queue checkHead failed for ${listKey}`, { error: toError(error).message })
+      // Fail-open: treat as head so the caller proceeds rather than hanging.
+      return 'head'
+    }
+  }
+
+  /**
+   * Refresh the ticket's heartbeat so the head isn't reaped as dead while waiting on the
+   * bucket. Also re-extends the queue list TTL so a wait outliving {@link QUEUE_LIST_TTL_SECONDS}
+   * doesn't let the list expire and collapse FIFO ordering.
+   */
+  async refreshHeartbeat(
+    provider: string,
+    billingActorId: string,
+    ticketId: string
+  ): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) return
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+    try {
+      const pipeline = redis.multi()
+      pipeline.set(hbKey, '1', 'EX', TICKET_HEARTBEAT_TTL_SECONDS)
+      pipeline.expire(listKey, QUEUE_LIST_TTL_SECONDS)
+      await pipeline.exec()
+    } catch (error) {
+      logger.warn(`Queue heartbeat refresh failed for ${hbKey}`, {
+        error: toError(error).message,
+      })
+    }
+  }
+
+  /**
+   * Remove a ticket from the queue and its heartbeat key. Best-effort; safe to
+   * call multiple times. LREM count=1 removes at most one matching entry.
+   */
+  async dequeue(provider: string, billingActorId: string, ticketId: string): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) return
+
+    const listKey = queueListKey(provider, billingActorId)
+    const hbKey = heartbeatKey(provider, billingActorId, ticketId)
+    try {
+      const pipeline = redis.multi()
+      pipeline.lrem(listKey, 1, ticketId)
+      pipeline.del(hbKey)
+      await pipeline.exec()
+    } catch (error) {
+      logger.warn(`Queue dequeue failed for ${listKey}`, { error: toError(error).message })
+    }
+  }
+}
+
+let cachedQueue: HostedKeyQueue | null = null
+
+export function getHostedKeyQueue(): HostedKeyQueue {
+  if (!cachedQueue) {
+    cachedQueue = new HostedKeyQueue()
+  }
+  return cachedQueue
+}
+
+export function resetHostedKeyQueue(): void {
+  cachedQueue = null
+}

apps/sim/lib/core/rate-limiter/hosted-key/types.ts

@@ -0,0 +1,108 @@
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter/storage'
+
+export type HostedKeyRateLimitMode = 'per_request' | 'custom'
+
+/**
+ * Simple per-request rate limit configuration.
+ * Enforces per-billing-actor rate limiting and distributes requests across keys.
+ */
+export interface PerRequestRateLimit {
+  mode: 'per_request'
+  /** Maximum requests per minute per billing actor (enforced - blocks if exceeded) */
+  requestsPerMinute: number
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+}
+
+/**
+ * Custom rate limit with multiple dimensions (e.g., tokens, search units).
+ * Allows tracking different usage metrics independently.
+ */
+export interface CustomRateLimit {
+  mode: 'custom'
+  /** Maximum requests per minute per billing actor (enforced - blocks if exceeded) */
+  requestsPerMinute: number
+  /** Multiple dimensions to track */
+  dimensions: RateLimitDimension[]
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+}
+
+/**
+ * A single dimension for custom rate limiting.
+ * Each dimension has its own token bucket.
+ */
+interface RateLimitDimension {
+  /** Dimension name (e.g., 'tokens', 'search_units') - used in storage key */
+  name: string
+  /** Limit per minute for this dimension */
+  limitPerMinute: number
+  /** Burst multiplier for token bucket max capacity. Default: 2 */
+  burstMultiplier?: number
+  /**
+   * Extract usage amount from request params and response.
+   * Called after successful execution to consume the actual usage.
+   */
+  extractUsage: (params: Record<string, unknown>, response: Record<string, unknown>) => number
+}
+
+/** Union of all hosted key rate limit configuration types */
+export type HostedKeyRateLimitConfig = PerRequestRateLimit | CustomRateLimit
+
+/**
+ * Result from acquiring a key from the hosted key rate limiter
+ */
+export interface AcquireKeyResult {
+  /** Whether a key was successfully acquired */
+  success: boolean
+  /** The API key value (if success=true) */
+  key?: string
+  /** Index of the key in the envKeys array */
+  keyIndex?: number
+  /** Environment variable name of the selected key */
+  envVarName?: string
+  /** Error message if no key available */
+  error?: string
+  /** Whether the billing actor was rate limited (exceeded their limit) */
+  billingActorRateLimited?: boolean
+  /** Milliseconds until the billing actor's rate limit resets (if billingActorRateLimited=true) */
+  retryAfterMs?: number
+}
+
+/**
+ * Result from reporting post-execution usage for custom dimensions
+ */
+export interface ReportUsageResult {
+  /** Per-dimension consumption results */
+  dimensions: {
+    name: string
+    consumed: number
+    allowed: boolean
+    tokensRemaining: number
+  }[]
+}
+
+/**
+ * Convert rate limit config to token bucket config for a dimension
+ */
+export function toTokenBucketConfig(
+  limitPerMinute: number,
+  burstMultiplier = 2,
+  windowMs = 60000
+): TokenBucketConfig {
+  return {
+    maxTokens: limitPerMinute * burstMultiplier,
+    refillRate: limitPerMinute,
+    refillIntervalMs: windowMs,
+  }
+}
+
+/**
+ * Default rate limit window in milliseconds (1 minute)
+ */
+export const DEFAULT_WINDOW_MS = 60000
+
+/**
+ * Default burst multiplier
+ */
+export const DEFAULT_BURST_MULTIPLIER = 2

apps/sim/lib/core/rate-limiter/index.ts

@@ -0,0 +1,20 @@
+export {
+  DEFAULT_BURST_MULTIPLIER,
+  DEFAULT_WINDOW_MS,
+  getHostedKeyRateLimiter,
+  type HostedKeyRateLimitConfig,
+  HostedKeyRateLimiter,
+  resetHostedKeyRateLimiter,
+  toTokenBucketConfig,
+} from './hosted-key'
+export { RateLimiter } from './rate-limiter'
+export {
+  DEFAULT_PUBLIC_IP_ROUTE_LIMIT,
+  DEFAULT_USER_ROUTE_LIMIT,
+  enforceIpRateLimit,
+  enforceUserOrIpRateLimit,
+  enforceUserRateLimit,
+} from './route-helpers'
+export type { TokenBucketConfig } from './storage'
+export type { SubscriptionPlan } from './types'
+export { getRateLimit, RATE_LIMITS, RateLimitError } from './types'

apps/sim/lib/core/rate-limiter/rate-limiter.test.ts

@@ -0,0 +1,459 @@
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+import { RateLimiter } from './rate-limiter'
+import type { ConsumeResult, RateLimitStorageAdapter, TokenStatus } from './storage'
+import { MANUAL_EXECUTION_LIMIT, RATE_LIMITS, RateLimitError } from './types'
+
+vi.mock('@/lib/core/config/env-flags', () => ({ isBillingEnabled: true }))
+
+interface MockAdapter {
+  consumeTokens: Mock
+  getTokenStatus: Mock
+  resetBucket: Mock
+}
+
+const createMockAdapter = (): MockAdapter => ({
+  consumeTokens: vi.fn(),
+  getTokenStatus: vi.fn(),
+  resetBucket: vi.fn(),
+})
+
+describe('RateLimiter', () => {
+  const testUserId = 'test-user-123'
+  const freeSubscription = { plan: 'free', referenceId: testUserId }
+  let mockAdapter: MockAdapter
+  let rateLimiter: RateLimiter
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockAdapter = createMockAdapter()
+    rateLimiter = new RateLimiter(mockAdapter as RateLimitStorageAdapter)
+  })
+
+  describe('checkRateLimitWithSubscription', () => {
+    it('should allow unlimited requests for manual trigger type', async () => {
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'manual',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(result.resetAt).toBeInstanceOf(Date)
+      expect(mockAdapter.consumeTokens).not.toHaveBeenCalled()
+    })
+
+    it('should consume tokens for API requests', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(mockResult.tokensRemaining)
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+
+    it('should use async bucket for async requests', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.async.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, freeSubscription, 'api', true)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:async`,
+        1,
+        RATE_LIMITS.free.async
+      )
+    })
+
+    it('should use api-endpoint bucket for api-endpoint trigger', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.apiEndpoint.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api-endpoint',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:api-endpoint`,
+        1,
+        RATE_LIMITS.free.apiEndpoint
+      )
+    })
+
+    it('should deny requests when rate limit exceeded', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt: new Date(Date.now() + 60000),
+        retryAfterMs: 30000,
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(false)
+      expect(result.remaining).toBe(0)
+      expect(result.retryAfterMs).toBe(30000)
+    })
+
+    it('should use organization key for team subscriptions', async () => {
+      const orgId = 'org-123'
+      const teamSubscription = { plan: 'team', referenceId: orgId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.team.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, teamSubscription, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${orgId}:sync`,
+        1,
+        RATE_LIMITS.team.sync
+      )
+    })
+
+    it('should use user key when team subscription referenceId matches userId', async () => {
+      const directTeamSubscription = { plan: 'team', referenceId: testUserId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.team.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        directTeamSubscription,
+        'api',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.team.sync
+      )
+    })
+
+    it('should allow on storage error (fail open)', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(1)
+    })
+
+    it('should work for all non-manual trigger types', async () => {
+      const triggerTypes = ['api', 'webhook', 'schedule', 'chat'] as const
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      for (const triggerType of triggerTypes) {
+        await rateLimiter.checkRateLimitWithSubscription(
+          testUserId,
+          freeSubscription,
+          triggerType,
+          false
+        )
+        expect(mockAdapter.consumeTokens).toHaveBeenCalled()
+        mockAdapter.consumeTokens.mockClear()
+      }
+    })
+  })
+
+  describe('getRateLimitStatusWithSubscription', () => {
+    it('should return unlimited status for manual trigger type', async () => {
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'manual',
+        false
+      )
+
+      expect(status.requestsPerMinute).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(status.maxBurst).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(status.remaining).toBe(MANUAL_EXECUTION_LIMIT)
+      expect(mockAdapter.getTokenStatus).not.toHaveBeenCalled()
+    })
+
+    it('should return status from storage for API requests', async () => {
+      const mockStatus: TokenStatus = {
+        tokensAvailable: 15,
+        maxTokens: RATE_LIMITS.free.sync.maxTokens,
+        lastRefillAt: new Date(),
+        nextRefillAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.getTokenStatus.mockResolvedValue(mockStatus)
+
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(status.remaining).toBe(15)
+      expect(status.requestsPerMinute).toBe(RATE_LIMITS.free.sync.refillRate)
+      expect(status.maxBurst).toBe(RATE_LIMITS.free.sync.maxTokens)
+      expect(mockAdapter.getTokenStatus).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        RATE_LIMITS.free.sync
+      )
+    })
+  })
+
+  describe('resetRateLimit', () => {
+    it('should reset all bucket types for a user', async () => {
+      mockAdapter.resetBucket.mockResolvedValue(undefined)
+
+      await rateLimiter.resetRateLimit(testUserId)
+
+      expect(mockAdapter.resetBucket).toHaveBeenCalledTimes(3)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:sync`)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:async`)
+      expect(mockAdapter.resetBucket).toHaveBeenCalledWith(`${testUserId}:api-endpoint`)
+    })
+
+    it('should throw error if reset fails', async () => {
+      mockAdapter.resetBucket.mockRejectedValue(new Error('Reset failed'))
+
+      await expect(rateLimiter.resetRateLimit(testUserId)).rejects.toThrow('Reset failed')
+    })
+  })
+
+  describe('checkRateLimitDirect', () => {
+    const config = { maxTokens: 3, refillRate: 1, refillIntervalMs: 60_000 }
+
+    it('should reflect the storage decision when it succeeds', async () => {
+      mockAdapter.consumeTokens.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 2,
+        resetAt: new Date(),
+      })
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config)
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(2)
+    })
+
+    it('should fail open on storage error by default', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config)
+
+      expect(result.allowed).toBe(true)
+      expect(result.remaining).toBe(1)
+    })
+
+    it('should fail closed on storage error when failClosed is set', async () => {
+      mockAdapter.consumeTokens.mockRejectedValue(new Error('Storage error'))
+
+      const result = await rateLimiter.checkRateLimitDirect('public:contact:ip', config, {
+        failClosed: true,
+      })
+
+      expect(result.allowed).toBe(false)
+      expect(result.remaining).toBe(0)
+    })
+  })
+
+  describe('subscription plan handling', () => {
+    it('should use pro plan limits', async () => {
+      const proSubscription = { plan: 'pro', referenceId: testUserId }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.pro.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, proSubscription, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.pro.sync
+      )
+    })
+
+    it('should use enterprise plan limits', async () => {
+      const enterpriseSubscription = { plan: 'enterprise', referenceId: 'org-enterprise' }
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.enterprise.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        enterpriseSubscription,
+        'api',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `org-enterprise:sync`,
+        1,
+        RATE_LIMITS.enterprise.sync
+      )
+    })
+
+    it('should fall back to free plan when subscription is null', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: RATE_LIMITS.free.sync.maxTokens - 1,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(testUserId, null, 'api', false)
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+  })
+
+  describe('schedule trigger type', () => {
+    it('should use sync bucket for schedule trigger', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'schedule',
+        false
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:sync`,
+        1,
+        RATE_LIMITS.free.sync
+      )
+    })
+
+    it('should use async bucket for schedule trigger with isAsync true', async () => {
+      const mockResult: ConsumeResult = {
+        allowed: true,
+        tokensRemaining: 10,
+        resetAt: new Date(Date.now() + 60000),
+      }
+      mockAdapter.consumeTokens.mockResolvedValue(mockResult)
+
+      await rateLimiter.checkRateLimitWithSubscription(
+        testUserId,
+        freeSubscription,
+        'schedule',
+        true
+      )
+
+      expect(mockAdapter.consumeTokens).toHaveBeenCalledWith(
+        `${testUserId}:async`,
+        1,
+        RATE_LIMITS.free.async
+      )
+    })
+  })
+
+  describe('getRateLimitStatusWithSubscription error handling', () => {
+    it('should return default config on storage error', async () => {
+      mockAdapter.getTokenStatus.mockRejectedValue(new Error('Storage error'))
+
+      const status = await rateLimiter.getRateLimitStatusWithSubscription(
+        testUserId,
+        freeSubscription,
+        'api',
+        false
+      )
+
+      expect(status.remaining).toBe(0)
+      expect(status.requestsPerMinute).toBe(RATE_LIMITS.free.sync.refillRate)
+      expect(status.maxBurst).toBe(RATE_LIMITS.free.sync.maxTokens)
+    })
+  })
+})
+
+describe('RateLimitError', () => {
+  it('should create error with default status code 429', () => {
+    const error = new RateLimitError('Rate limit exceeded')
+
+    expect(error.message).toBe('Rate limit exceeded')
+    expect(error.statusCode).toBe(429)
+    expect(error.name).toBe('RateLimitError')
+  })
+
+  it('should create error with custom status code', () => {
+    const error = new RateLimitError('Custom error', 503)
+
+    expect(error.message).toBe('Custom error')
+    expect(error.statusCode).toBe(503)
+  })
+
+  it('should be instanceof Error', () => {
+    const error = new RateLimitError('Test')
+
+    expect(error instanceof Error).toBe(true)
+    expect(error instanceof RateLimitError).toBe(true)
+  })
+
+  it('should have proper stack trace', () => {
+    const error = new RateLimitError('Test error')
+
+    expect(error.stack).toBeDefined()
+    expect(error.stack).toContain('RateLimitError')
+  })
+})

apps/sim/lib/core/rate-limiter/rate-limiter.ts

@@ -0,0 +1,220 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils'
+import { createStorageAdapter, type RateLimitStorageAdapter } from './storage'
+import {
+  getRateLimit,
+  MANUAL_EXECUTION_LIMIT,
+  RATE_LIMIT_WINDOW_MS,
+  type RateLimitCounterType,
+  type SubscriptionPlan,
+  type TriggerType,
+} from './types'
+
+const logger = createLogger('RateLimiter')
+
+interface SubscriptionInfo {
+  plan: string
+  referenceId: string
+}
+
+export interface RateLimitResult {
+  allowed: boolean
+  remaining: number
+  resetAt: Date
+  retryAfterMs?: number
+}
+
+export interface RateLimitStatus {
+  requestsPerMinute: number
+  maxBurst: number
+  remaining: number
+  resetAt: Date
+}
+
+export class RateLimiter {
+  private storage: RateLimitStorageAdapter
+
+  constructor(storage?: RateLimitStorageAdapter) {
+    this.storage = storage ?? createStorageAdapter()
+  }
+
+  private getRateLimitKey(userId: string, subscription: SubscriptionInfo | null): string {
+    if (!subscription) return userId
+
+    if (isOrgScopedSubscription(subscription, userId)) {
+      return subscription.referenceId
+    }
+
+    return userId
+  }
+
+  private getCounterType(triggerType: TriggerType, isAsync: boolean): RateLimitCounterType {
+    if (triggerType === 'api-endpoint') return 'api-endpoint'
+    return isAsync ? 'async' : 'sync'
+  }
+
+  private buildStorageKey(rateLimitKey: string, counterType: RateLimitCounterType): string {
+    return `${rateLimitKey}:${counterType}`
+  }
+
+  private createUnlimitedResult(): RateLimitResult {
+    return {
+      allowed: true,
+      remaining: MANUAL_EXECUTION_LIMIT,
+      resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+    }
+  }
+
+  async checkRateLimitWithSubscription(
+    userId: string,
+    subscription: SubscriptionInfo | null,
+    triggerType: TriggerType = 'manual',
+    isAsync = false
+  ): Promise<RateLimitResult> {
+    try {
+      if (triggerType === 'manual') {
+        return this.createUnlimitedResult()
+      }
+
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const rateLimitKey = this.getRateLimitKey(userId, subscription)
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+      const storageKey = this.buildStorageKey(rateLimitKey, counterType)
+
+      const result = await this.storage.consumeTokens(storageKey, 1, config)
+
+      if (!result.allowed) {
+        logger.info('Rate limit exceeded', {
+          rateLimitKey,
+          counterType,
+          plan,
+          tokensRemaining: result.tokensRemaining,
+        })
+      }
+
+      return {
+        allowed: result.allowed,
+        remaining: result.tokensRemaining,
+        resetAt: result.resetAt,
+        retryAfterMs: result.retryAfterMs,
+      }
+    } catch (error) {
+      logger.error('Rate limit storage error - failing open (allowing request)', {
+        error: toError(error).message,
+        userId,
+        triggerType,
+        isAsync,
+      })
+      return {
+        allowed: true,
+        remaining: 1,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  async getRateLimitStatusWithSubscription(
+    userId: string,
+    subscription: SubscriptionInfo | null,
+    triggerType: TriggerType = 'manual',
+    isAsync = false
+  ): Promise<RateLimitStatus> {
+    try {
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+
+      if (triggerType === 'manual') {
+        return {
+          requestsPerMinute: MANUAL_EXECUTION_LIMIT,
+          maxBurst: MANUAL_EXECUTION_LIMIT,
+          remaining: MANUAL_EXECUTION_LIMIT,
+          resetAt: new Date(Date.now() + config.refillIntervalMs),
+        }
+      }
+
+      const rateLimitKey = this.getRateLimitKey(userId, subscription)
+      const storageKey = this.buildStorageKey(rateLimitKey, counterType)
+
+      const status = await this.storage.getTokenStatus(storageKey, config)
+
+      return {
+        requestsPerMinute: config.refillRate,
+        maxBurst: config.maxTokens,
+        remaining: Math.floor(status.tokensAvailable),
+        resetAt: status.nextRefillAt,
+      }
+    } catch (error) {
+      logger.error('Error getting rate limit status - returning default config', {
+        error: toError(error).message,
+        userId,
+        triggerType,
+        isAsync,
+      })
+      const plan = (subscription?.plan || 'free') as SubscriptionPlan
+      const counterType = this.getCounterType(triggerType, isAsync)
+      const config = getRateLimit(plan, counterType)
+      return {
+        requestsPerMinute: config.refillRate,
+        maxBurst: config.maxTokens,
+        remaining: 0,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  /**
+   * Consume one token from a bucket. On storage failure the default is to fail
+   * open (allow the request) so a limiter outage never takes down normal traffic.
+   * Pass `failClosed: true` for security-critical checks — e.g. a captcha
+   * backstop — where an unenforceable limit must reject rather than admit.
+   */
+  async checkRateLimitDirect(
+    storageKey: string,
+    config: { maxTokens: number; refillRate: number; refillIntervalMs: number },
+    options?: { failClosed?: boolean }
+  ): Promise<RateLimitResult> {
+    try {
+      const result = await this.storage.consumeTokens(storageKey, 1, config)
+      if (!result.allowed) {
+        logger.info('Rate limit exceeded', { storageKey, tokensRemaining: result.tokensRemaining })
+      }
+      return {
+        allowed: result.allowed,
+        remaining: result.tokensRemaining,
+        resetAt: result.resetAt,
+        retryAfterMs: result.retryAfterMs,
+      }
+    } catch (error) {
+      const failClosed = options?.failClosed === true
+      logger.error(
+        `Rate limit storage error - failing ${failClosed ? 'closed (rejecting request)' : 'open (allowing request)'}`,
+        {
+          error: toError(error).message,
+          storageKey,
+        }
+      )
+      return {
+        allowed: !failClosed,
+        remaining: failClosed ? 0 : 1,
+        resetAt: new Date(Date.now() + RATE_LIMIT_WINDOW_MS),
+      }
+    }
+  }
+
+  async resetRateLimit(rateLimitKey: string): Promise<void> {
+    try {
+      await Promise.all([
+        this.storage.resetBucket(`${rateLimitKey}:sync`),
+        this.storage.resetBucket(`${rateLimitKey}:async`),
+        this.storage.resetBucket(`${rateLimitKey}:api-endpoint`),
+      ])
+      logger.info(`Reset rate limit for ${rateLimitKey}`)
+    } catch (error) {
+      logger.error('Error resetting rate limit:', error)
+      throw error
+    }
+  }
+}

apps/sim/lib/core/rate-limiter/route-helpers.test.ts

@@ -0,0 +1,192 @@
+/**
+ * @vitest-environment node
+ */
+import { createMockRequest, requestUtilsMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
+
+const { mockAdapter } = vi.hoisted(() => ({
+  mockAdapter: {
+    consumeTokens: vi.fn(),
+    getTokenStatus: vi.fn(),
+    resetBucket: vi.fn(),
+  },
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage', async () => {
+  const actual = await vi.importActual<typeof import('@/lib/core/rate-limiter/storage')>(
+    '@/lib/core/rate-limiter/storage'
+  )
+  return {
+    ...actual,
+    createStorageAdapter: () => mockAdapter,
+  }
+})
+
+function passThroughClientIp() {
+  requestUtilsMockFns.mockGetClientIp.mockImplementation(
+    (req: { headers: { get(name: string): string | null } }) =>
+      req.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ||
+      req.headers.get('x-real-ip')?.trim() ||
+      'unknown'
+  )
+}
+
+import { enforceIpRateLimit, enforceUserOrIpRateLimit, enforceUserRateLimit } from './route-helpers'
+
+const consume = mockAdapter.consumeTokens as Mock
+
+describe('route-helpers rate limiting', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('enforceUserRateLimit', () => {
+    it('returns null when the bucket has tokens left', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(Date.now() + 60_000),
+      })
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).toBeNull()
+      expect(consume).toHaveBeenCalledWith(
+        'route:test-bucket:user:user-1',
+        1,
+        expect.objectContaining({ maxTokens: 60, refillRate: 30 })
+      )
+    })
+
+    it('returns a 429 with Retry-After when the bucket is empty', async () => {
+      const resetAt = new Date(Date.now() + 30_000)
+      consume.mockResolvedValueOnce({
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt,
+        retryAfterMs: 30_000,
+      })
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).not.toBeNull()
+      expect(result?.status).toBe(429)
+      expect(result?.headers.get('Retry-After')).toBe('30')
+      expect(result?.headers.get('X-RateLimit-Reset')).toBe(resetAt.toISOString())
+
+      const body = await result?.json()
+      expect(body?.error).toBe('Rate limit exceeded')
+    })
+
+    it('keys buckets per user so different users do not share state', async () => {
+      consume.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+
+      await enforceUserRateLimit('shared-bucket', 'user-a')
+      await enforceUserRateLimit('shared-bucket', 'user-b')
+
+      const keys = consume.mock.calls.map((call) => call[0])
+      expect(keys).toEqual(['route:shared-bucket:user:user-a', 'route:shared-bucket:user:user-b'])
+    })
+
+    it('fails open when the storage layer throws', async () => {
+      consume.mockRejectedValueOnce(new Error('redis down'))
+
+      const result = await enforceUserRateLimit('test-bucket', 'user-1')
+
+      expect(result).toBeNull()
+    })
+  })
+
+  describe('enforceIpRateLimit', () => {
+    beforeEach(() => {
+      passThroughClientIp()
+    })
+
+    it('uses the X-Forwarded-For client IP in the bucket key', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, {
+        'x-forwarded-for': '203.0.113.7, 10.0.0.1',
+      })
+
+      await enforceIpRateLimit('public-bucket', request)
+
+      expect(consume).toHaveBeenCalledWith(
+        'route:public-bucket:ip:203.0.113.7',
+        1,
+        expect.any(Object)
+      )
+    })
+
+    it('folds spoofed `X-Forwarded-For: unknown` into a single shared bucket', async () => {
+      consume.mockResolvedValue({
+        allowed: true,
+        tokensRemaining: 9,
+        resetAt: new Date(),
+      })
+
+      const reqA = createMockRequest('POST', undefined, { 'x-forwarded-for': 'unknown' })
+      const reqB = createMockRequest('POST', undefined, { 'x-forwarded-for': 'unknown' })
+      await enforceIpRateLimit('otp', reqA)
+      await enforceIpRateLimit('otp', reqB)
+
+      const keys = consume.mock.calls.map((call) => call[0])
+      expect(keys).toEqual(['route:otp:ip:unknown', 'route:otp:ip:unknown'])
+    })
+
+    it('returns a 429 with Retry-After on rate limit', async () => {
+      const resetAt = new Date(Date.now() + 60_000)
+      consume.mockResolvedValueOnce({
+        allowed: false,
+        tokensRemaining: 0,
+        resetAt,
+        retryAfterMs: 60_000,
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      const result = await enforceIpRateLimit('public-bucket', request)
+
+      expect(result?.status).toBe(429)
+      expect(result?.headers.get('Retry-After')).toBe('60')
+    })
+  })
+
+  describe('enforceUserOrIpRateLimit', () => {
+    beforeEach(() => {
+      passThroughClientIp()
+    })
+
+    it('keys per-user when userId is present', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      await enforceUserOrIpRateLimit('a2a-test', 'user-1', request)
+
+      expect(consume).toHaveBeenCalledWith('route:a2a-test:user:user-1', 1, expect.any(Object))
+    })
+
+    it('falls back to per-IP when userId is undefined', async () => {
+      consume.mockResolvedValueOnce({
+        allowed: true,
+        tokensRemaining: 59,
+        resetAt: new Date(),
+      })
+      const request = createMockRequest('POST', undefined, { 'x-forwarded-for': '203.0.113.7' })
+
+      await enforceUserOrIpRateLimit('a2a-test', undefined, request)
+
+      expect(consume).toHaveBeenCalledWith('route:a2a-test:ip:203.0.113.7', 1, expect.any(Object))
+    })
+  })
+})

apps/sim/lib/core/rate-limiter/route-helpers.ts

@@ -0,0 +1,89 @@
+import { createLogger } from '@sim/logger'
+import { type NextRequest, NextResponse } from 'next/server'
+import { RateLimiter } from '@/lib/core/rate-limiter/rate-limiter'
+import type { TokenBucketConfig } from '@/lib/core/rate-limiter/storage'
+import { getClientIp } from '@/lib/core/utils/request'
+
+const logger = createLogger('RouteRateLimit')
+const rateLimiter = new RateLimiter()
+
+/** Default per-user bucket for authenticated tool routes (60 burst, 30/min). */
+export const DEFAULT_USER_ROUTE_LIMIT: TokenBucketConfig = {
+  maxTokens: 60,
+  refillRate: 30,
+  refillIntervalMs: 60_000,
+}
+
+/** Default per-IP bucket for unauthenticated public endpoints (10 burst, 5/min). */
+export const DEFAULT_PUBLIC_IP_ROUTE_LIMIT: TokenBucketConfig = {
+  maxTokens: 10,
+  refillRate: 5,
+  refillIntervalMs: 60_000,
+}
+
+function buildRateLimitResponse(resetAt: Date): NextResponse {
+  const retryAfterSec = Math.max(1, Math.ceil((resetAt.getTime() - Date.now()) / 1000))
+  return NextResponse.json(
+    {
+      error: 'Rate limit exceeded',
+      retryAfter: resetAt.getTime(),
+    },
+    {
+      status: 429,
+      headers: {
+        'Retry-After': String(retryAfterSec),
+        'X-RateLimit-Reset': resetAt.toISOString(),
+      },
+    }
+  )
+}
+
+/**
+ * Apply a per-user token bucket to an authenticated route.
+ * Returns a `NextResponse` on 429, otherwise `null` so the caller can proceed.
+ */
+export async function enforceUserRateLimit(
+  bucketName: string,
+  userId: string,
+  config: TokenBucketConfig = DEFAULT_USER_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  const key = `route:${bucketName}:user:${userId}`
+  const { allowed, resetAt } = await rateLimiter.checkRateLimitDirect(key, config)
+  if (allowed) return null
+  logger.warn('User rate limit exceeded', { bucket: bucketName, userId })
+  return buildRateLimitResponse(resetAt)
+}
+
+/**
+ * Apply a per-IP token bucket to an unauthenticated route. The `unknown` IP
+ * fallback shares one global bucket per route so it cannot be amplified by
+ * `X-Forwarded-For: unknown` spoofing.
+ */
+export async function enforceIpRateLimit(
+  bucketName: string,
+  request: NextRequest,
+  config: TokenBucketConfig = DEFAULT_PUBLIC_IP_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  const ip = getClientIp(request)
+  const key = `route:${bucketName}:ip:${ip}`
+  const { allowed, resetAt } = await rateLimiter.checkRateLimitDirect(key, config)
+  if (allowed) return null
+  logger.warn('IP rate limit exceeded', { bucket: bucketName, ip })
+  return buildRateLimitResponse(resetAt)
+}
+
+/**
+ * Apply a per-user limit when a userId is present, else fall back to per-IP.
+ * Use for routes whose auth path may legitimately resolve without a userId
+ * (e.g. internal JWT calls with `requireWorkflowId: false`) so missing-userId
+ * traffic is still throttled per-IP rather than sharing one global bucket.
+ */
+export async function enforceUserOrIpRateLimit(
+  bucketName: string,
+  userId: string | undefined,
+  request: NextRequest,
+  config: TokenBucketConfig = DEFAULT_USER_ROUTE_LIMIT
+): Promise<NextResponse | null> {
+  if (userId) return enforceUserRateLimit(bucketName, userId, config)
+  return enforceIpRateLimit(bucketName, request, config)
+}

apps/sim/lib/core/rate-limiter/storage/adapter.ts

@@ -0,0 +1,25 @@
+export interface TokenBucketConfig {
+  maxTokens: number
+  refillRate: number
+  refillIntervalMs: number
+}
+
+export interface ConsumeResult {
+  allowed: boolean
+  tokensRemaining: number
+  resetAt: Date
+  retryAfterMs?: number
+}
+
+export interface TokenStatus {
+  tokensAvailable: number
+  maxTokens: number
+  lastRefillAt: Date
+  nextRefillAt: Date
+}
+
+export interface RateLimitStorageAdapter {
+  consumeTokens(key: string, tokens: number, config: TokenBucketConfig): Promise<ConsumeResult>
+  getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus>
+  resetBucket(key: string): Promise<void>
+}

apps/sim/lib/core/rate-limiter/storage/db-token-bucket.ts

@@ -0,0 +1,136 @@
+import { db } from '@sim/db'
+import { rateLimitBucket } from '@sim/db/schema'
+import { eq, sql } from 'drizzle-orm'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+
+export class DbTokenBucket implements RateLimitStorageAdapter {
+  async consumeTokens(
+    key: string,
+    requestedTokens: number,
+    config: TokenBucketConfig
+  ): Promise<ConsumeResult> {
+    const now = new Date()
+    const nowMs = now.getTime()
+    const nowIso = now.toISOString()
+
+    const result = await db
+      .insert(rateLimitBucket)
+      .values({
+        key,
+        tokens: (config.maxTokens - requestedTokens).toString(),
+        lastRefillAt: now,
+        updatedAt: now,
+      })
+      .onConflictDoUpdate({
+        target: rateLimitBucket.key,
+        set: {
+          tokens: sql`
+            CASE
+              WHEN (
+                LEAST(
+                  ${config.maxTokens}::numeric,
+                  ${rateLimitBucket.tokens}::numeric + (
+                    FLOOR(
+                      EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                      / ${config.refillIntervalMs}
+                    ) * ${config.refillRate}
+                  )::numeric
+                )
+              ) >= ${requestedTokens}::numeric
+              THEN LEAST(
+                ${config.maxTokens}::numeric,
+                ${rateLimitBucket.tokens}::numeric + (
+                  FLOOR(
+                    EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                    / ${config.refillIntervalMs}
+                  ) * ${config.refillRate}
+                )::numeric
+              ) - ${requestedTokens}::numeric
+              ELSE -1
+            END
+          `,
+          lastRefillAt: sql`
+            CASE
+              WHEN FLOOR(
+                EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                / ${config.refillIntervalMs}
+              ) > 0
+              THEN ${rateLimitBucket.lastRefillAt} + (
+                FLOOR(
+                  EXTRACT(EPOCH FROM (${nowIso}::timestamp - ${rateLimitBucket.lastRefillAt})) * 1000
+                  / ${config.refillIntervalMs}
+                ) * ${config.refillIntervalMs} * INTERVAL '1 millisecond'
+              )
+              ELSE ${rateLimitBucket.lastRefillAt}
+            END
+          `,
+          updatedAt: now,
+        },
+      })
+      .returning({
+        tokens: rateLimitBucket.tokens,
+        lastRefillAt: rateLimitBucket.lastRefillAt,
+      })
+
+    const record = result[0]
+    const tokens = Number.parseFloat(record.tokens)
+    const lastRefillMs = record.lastRefillAt.getTime()
+    const nextRefillAt = new Date(lastRefillMs + config.refillIntervalMs)
+
+    const allowed = tokens >= 0
+
+    return {
+      allowed,
+      tokensRemaining: Math.max(0, tokens),
+      resetAt: nextRefillAt,
+      retryAfterMs: allowed ? undefined : Math.max(0, nextRefillAt.getTime() - nowMs),
+    }
+  }
+
+  async getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus> {
+    const now = new Date()
+
+    const [record] = await db
+      .select({
+        tokens: rateLimitBucket.tokens,
+        lastRefillAt: rateLimitBucket.lastRefillAt,
+      })
+      .from(rateLimitBucket)
+      .where(eq(rateLimitBucket.key, key))
+      .limit(1)
+
+    if (!record) {
+      return {
+        tokensAvailable: config.maxTokens,
+        maxTokens: config.maxTokens,
+        lastRefillAt: now,
+        nextRefillAt: new Date(now.getTime() + config.refillIntervalMs),
+      }
+    }
+
+    const tokens = Number.parseFloat(record.tokens)
+    const elapsed = now.getTime() - record.lastRefillAt.getTime()
+    const intervalsElapsed = Math.floor(elapsed / config.refillIntervalMs)
+    const refillAmount = intervalsElapsed * config.refillRate
+    const tokensAvailable = Math.min(config.maxTokens, tokens + refillAmount)
+    const lastRefillAt = new Date(
+      record.lastRefillAt.getTime() + intervalsElapsed * config.refillIntervalMs
+    )
+
+    return {
+      tokensAvailable,
+      maxTokens: config.maxTokens,
+      lastRefillAt,
+      nextRefillAt: new Date(lastRefillAt.getTime() + config.refillIntervalMs),
+    }
+  }
+
+  async resetBucket(key: string): Promise<void> {
+    await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, key))
+  }
+}

apps/sim/lib/core/rate-limiter/storage/factory.test.ts

@@ -0,0 +1,109 @@
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockGetStorageMethod, reconnectCallbacks } = vi.hoisted(() => {
+  const callbacks: Array<() => void> = []
+  return {
+    mockGetStorageMethod: vi.fn(() => 'db'),
+    reconnectCallbacks: callbacks,
+  }
+})
+
+const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient
+const mockOnRedisReconnect = redisConfigMockFns.mockOnRedisReconnect
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+vi.mock('@/lib/core/storage', () => ({
+  getStorageMethod: mockGetStorageMethod,
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage/db-token-bucket', () => ({
+  DbTokenBucket: vi.fn().mockImplementation(
+    class {
+      type = 'db'
+    }
+  ),
+}))
+
+vi.mock('@/lib/core/rate-limiter/storage/redis-token-bucket', () => ({
+  RedisTokenBucket: vi.fn().mockImplementation(
+    class {
+      type = 'redis'
+    }
+  ),
+}))
+
+import { createStorageAdapter, resetStorageAdapter } from '@/lib/core/rate-limiter/storage/factory'
+
+describe('rate limit storage factory', () => {
+  beforeEach(() => {
+    mockGetRedisClient.mockReset().mockReturnValue(null)
+    mockGetStorageMethod.mockReset().mockReturnValue('db')
+    mockOnRedisReconnect.mockImplementation((cb: () => void) => {
+      reconnectCallbacks.push(cb)
+    })
+    resetStorageAdapter()
+  })
+
+  it('should fall back to DbTokenBucket when Redis is configured but client unavailable', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue(null)
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'db' })
+  })
+
+  it('should use RedisTokenBucket when Redis client is available', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never)
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'redis' })
+  })
+
+  it('should use DbTokenBucket when storage method is db', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter = createStorageAdapter()
+    expect(adapter).toEqual({ type: 'db' })
+  })
+
+  it('should cache the adapter and return same instance', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter1 = createStorageAdapter()
+    const adapter2 = createStorageAdapter()
+    expect(adapter1).toBe(adapter2)
+  })
+
+  it('should register a reconnect listener that resets cached adapter', () => {
+    mockGetStorageMethod.mockReturnValue('db')
+
+    const adapter1 = createStorageAdapter()
+
+    /** onRedisReconnect is called once (guarded by reconnectListenerRegistered flag). */
+    expect(reconnectCallbacks.length).toBeGreaterThan(0)
+    const latestCallback = reconnectCallbacks[reconnectCallbacks.length - 1]
+    latestCallback()
+
+    const adapter2 = createStorageAdapter()
+    expect(adapter2).not.toBe(adapter1)
+  })
+
+  it('should re-evaluate storage on next call after reconnect resets cache', () => {
+    mockGetStorageMethod.mockReturnValue('redis')
+    mockGetRedisClient.mockReturnValue(null)
+
+    const adapter1 = createStorageAdapter()
+    expect(adapter1).toEqual({ type: 'db' })
+
+    const latestCallback = reconnectCallbacks[reconnectCallbacks.length - 1]
+    latestCallback()
+
+    mockGetRedisClient.mockReturnValue({ ping: vi.fn() } as never)
+
+    const adapter2 = createStorageAdapter()
+    expect(adapter2).toEqual({ type: 'redis' })
+  })
+})

apps/sim/lib/core/rate-limiter/storage/factory.ts

@@ -0,0 +1,64 @@
+import { createLogger } from '@sim/logger'
+import { getRedisClient, onRedisReconnect } from '@/lib/core/config/redis'
+import { getStorageMethod, type StorageMethod } from '@/lib/core/storage'
+import type { RateLimitStorageAdapter } from './adapter'
+import { DbTokenBucket } from './db-token-bucket'
+import { RedisTokenBucket } from './redis-token-bucket'
+
+const logger = createLogger('RateLimitStorage')
+
+type FactoryGlobal = typeof globalThis & {
+  _rlCachedAdapter?: RateLimitStorageAdapter | null
+  _rlReconnectListenerRegistered?: boolean
+}
+
+const g = globalThis as FactoryGlobal
+if (!('_rlCachedAdapter' in g)) {
+  g._rlCachedAdapter = null
+  g._rlReconnectListenerRegistered = false
+}
+
+export function createStorageAdapter(): RateLimitStorageAdapter {
+  if (g._rlCachedAdapter) {
+    return g._rlCachedAdapter
+  }
+
+  if (!g._rlReconnectListenerRegistered) {
+    onRedisReconnect(() => {
+      g._rlCachedAdapter = null
+    })
+    g._rlReconnectListenerRegistered = true
+  }
+
+  const storageMethod = getStorageMethod()
+
+  if (storageMethod === 'redis') {
+    const redis = getRedisClient()
+    if (!redis) {
+      logger.warn(
+        'Redis configured but client unavailable - falling back to PostgreSQL for rate limiting'
+      )
+      g._rlCachedAdapter = new DbTokenBucket()
+    } else {
+      logger.info('Rate limiting: Using Redis')
+      g._rlCachedAdapter = new RedisTokenBucket(redis)
+    }
+  } else {
+    logger.info('Rate limiting: Using PostgreSQL')
+    g._rlCachedAdapter = new DbTokenBucket()
+  }
+
+  return g._rlCachedAdapter!
+}
+
+export function getAdapterType(): StorageMethod {
+  return getStorageMethod()
+}
+
+export function resetStorageAdapter(): void {
+  g._rlCachedAdapter = null
+}
+
+export function setStorageAdapter(adapter: RateLimitStorageAdapter): void {
+  g._rlCachedAdapter = adapter
+}

apps/sim/lib/core/rate-limiter/storage/index.ts

@@ -0,0 +1,14 @@
+export type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+export { DbTokenBucket } from './db-token-bucket'
+export {
+  createStorageAdapter,
+  getAdapterType,
+  resetStorageAdapter,
+  setStorageAdapter,
+} from './factory'
+export { RedisTokenBucket } from './redis-token-bucket'

apps/sim/lib/core/rate-limiter/storage/redis-token-bucket.ts

@@ -0,0 +1,135 @@
+import type Redis from 'ioredis'
+import type {
+  ConsumeResult,
+  RateLimitStorageAdapter,
+  TokenBucketConfig,
+  TokenStatus,
+} from './adapter'
+
+const CONSUME_SCRIPT = `
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local requested = tonumber(ARGV[2])
+local maxTokens = tonumber(ARGV[3])
+local refillRate = tonumber(ARGV[4])
+local refillIntervalMs = tonumber(ARGV[5])
+local ttl = tonumber(ARGV[6])
+
+local bucket = redis.call('HMGET', key, 'tokens', 'lastRefillAt')
+local tokens = tonumber(bucket[1])
+local lastRefillAt = tonumber(bucket[2])
+
+if tokens == nil then
+  tokens = maxTokens
+  lastRefillAt = now
+end
+
+local elapsed = now - lastRefillAt
+local intervalsElapsed = math.floor(elapsed / refillIntervalMs)
+if intervalsElapsed > 0 then
+  tokens = math.min(maxTokens, tokens + (intervalsElapsed * refillRate))
+  lastRefillAt = lastRefillAt + (intervalsElapsed * refillIntervalMs)
+end
+
+local allowed = 0
+if tokens >= requested then
+  tokens = tokens - requested
+  allowed = 1
+end
+
+redis.call('HSET', key, 'tokens', tokens, 'lastRefillAt', lastRefillAt)
+redis.call('EXPIRE', key, ttl)
+
+local nextRefillAt = lastRefillAt + refillIntervalMs
+
+return {allowed, tokens, lastRefillAt, nextRefillAt}
+`
+
+const STATUS_SCRIPT = `
+local key = KEYS[1]
+local now = tonumber(ARGV[1])
+local maxTokens = tonumber(ARGV[2])
+local refillRate = tonumber(ARGV[3])
+local refillIntervalMs = tonumber(ARGV[4])
+
+local bucket = redis.call('HMGET', key, 'tokens', 'lastRefillAt')
+local tokens = tonumber(bucket[1])
+local lastRefillAt = tonumber(bucket[2])
+
+if tokens == nil then
+  tokens = maxTokens
+  lastRefillAt = now
+end
+
+local elapsed = now - lastRefillAt
+local intervalsElapsed = math.floor(elapsed / refillIntervalMs)
+if intervalsElapsed > 0 then
+  tokens = math.min(maxTokens, tokens + (intervalsElapsed * refillRate))
+  lastRefillAt = lastRefillAt + (intervalsElapsed * refillIntervalMs)
+end
+
+local nextRefillAt = lastRefillAt + refillIntervalMs
+
+return {tokens, maxTokens, lastRefillAt, nextRefillAt}
+`
+
+export class RedisTokenBucket implements RateLimitStorageAdapter {
+  constructor(private redis: Redis) {}
+
+  async consumeTokens(
+    key: string,
+    tokens: number,
+    config: TokenBucketConfig
+  ): Promise<ConsumeResult> {
+    const now = Date.now()
+    const ttl = Math.ceil((config.refillIntervalMs * 2) / 1000)
+
+    const result = (await this.redis.eval(
+      CONSUME_SCRIPT,
+      1,
+      `ratelimit:tb:${key}`,
+      now,
+      tokens,
+      config.maxTokens,
+      config.refillRate,
+      config.refillIntervalMs,
+      ttl
+    )) as [number, number, number, number]
+
+    const [allowed, remaining, , nextRefill] = result
+
+    return {
+      allowed: allowed === 1,
+      tokensRemaining: remaining,
+      resetAt: new Date(nextRefill),
+      retryAfterMs: allowed === 1 ? undefined : Math.max(0, nextRefill - now),
+    }
+  }
+
+  async getTokenStatus(key: string, config: TokenBucketConfig): Promise<TokenStatus> {
+    const now = Date.now()
+
+    const result = (await this.redis.eval(
+      STATUS_SCRIPT,
+      1,
+      `ratelimit:tb:${key}`,
+      now,
+      config.maxTokens,
+      config.refillRate,
+      config.refillIntervalMs
+    )) as [number, number, number, number]
+
+    const [tokensAvailable, maxTokens, lastRefillAt, nextRefillAt] = result
+
+    return {
+      tokensAvailable,
+      maxTokens,
+      lastRefillAt: new Date(lastRefillAt),
+      nextRefillAt: new Date(nextRefillAt),
+    }
+  }
+
+  async resetBucket(key: string): Promise<void> {
+    await this.redis.del(`ratelimit:tb:${key}`)
+  }
+}

apps/sim/lib/core/rate-limiter/types.ts

@@ -0,0 +1,109 @@
+import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
+import { env } from '@/lib/core/config/env'
+import { isBillingEnabled } from '@/lib/core/config/env-flags'
+import type { CoreTriggerType } from '@/stores/logs/filters/types'
+import type { TokenBucketConfig } from './storage'
+
+export type TriggerType = CoreTriggerType | 'api-endpoint'
+
+export type RateLimitCounterType = 'sync' | 'async' | 'api-endpoint'
+
+type RateLimitConfigKey = 'sync' | 'async' | 'apiEndpoint'
+
+export type SubscriptionPlan = 'free' | 'pro' | 'team' | 'enterprise'
+
+export interface RateLimitConfig {
+  sync: TokenBucketConfig
+  async: TokenBucketConfig
+  apiEndpoint: TokenBucketConfig
+}
+
+export const RATE_LIMIT_WINDOW_MS = Number.parseInt(env.RATE_LIMIT_WINDOW_MS) || 60000
+
+export const MANUAL_EXECUTION_LIMIT = Number.parseInt(env.MANUAL_EXECUTION_LIMIT) || 999999
+
+const DEFAULT_RATE_LIMITS = {
+  free: { sync: 50, async: 200, apiEndpoint: 30 },
+  pro: { sync: 150, async: 1000, apiEndpoint: 100 },
+  team: { sync: 300, async: 2500, apiEndpoint: 200 },
+  enterprise: { sync: 600, async: 5000, apiEndpoint: 500 },
+} as const
+
+function toConfigKey(type: RateLimitCounterType): RateLimitConfigKey {
+  return type === 'api-endpoint' ? 'apiEndpoint' : type
+}
+
+function createBucketConfig(ratePerMinute: number, burstMultiplier = 2): TokenBucketConfig {
+  return {
+    maxTokens: ratePerMinute * burstMultiplier,
+    refillRate: ratePerMinute,
+    refillIntervalMs: RATE_LIMIT_WINDOW_MS,
+  }
+}
+
+function getRateLimitForPlan(plan: SubscriptionPlan, type: RateLimitConfigKey): TokenBucketConfig {
+  const envVarMap: Record<SubscriptionPlan, Record<RateLimitConfigKey, string | undefined>> = {
+    free: {
+      sync: env.RATE_LIMIT_FREE_SYNC,
+      async: env.RATE_LIMIT_FREE_ASYNC,
+      apiEndpoint: undefined,
+    },
+    pro: { sync: env.RATE_LIMIT_PRO_SYNC, async: env.RATE_LIMIT_PRO_ASYNC, apiEndpoint: undefined },
+    team: {
+      sync: env.RATE_LIMIT_TEAM_SYNC,
+      async: env.RATE_LIMIT_TEAM_ASYNC,
+      apiEndpoint: undefined,
+    },
+    enterprise: {
+      sync: env.RATE_LIMIT_ENTERPRISE_SYNC,
+      async: env.RATE_LIMIT_ENTERPRISE_ASYNC,
+      apiEndpoint: undefined,
+    },
+  }
+
+  const rate = Number.parseInt(envVarMap[plan][type] || '') || DEFAULT_RATE_LIMITS[plan][type]
+  return createBucketConfig(rate)
+}
+
+export const RATE_LIMITS: Record<SubscriptionPlan, RateLimitConfig> = {
+  free: {
+    sync: getRateLimitForPlan('free', 'sync'),
+    async: getRateLimitForPlan('free', 'async'),
+    apiEndpoint: getRateLimitForPlan('free', 'apiEndpoint'),
+  },
+  pro: {
+    sync: getRateLimitForPlan('pro', 'sync'),
+    async: getRateLimitForPlan('pro', 'async'),
+    apiEndpoint: getRateLimitForPlan('pro', 'apiEndpoint'),
+  },
+  team: {
+    sync: getRateLimitForPlan('team', 'sync'),
+    async: getRateLimitForPlan('team', 'async'),
+    apiEndpoint: getRateLimitForPlan('team', 'apiEndpoint'),
+  },
+  enterprise: {
+    sync: getRateLimitForPlan('enterprise', 'sync'),
+    async: getRateLimitForPlan('enterprise', 'async'),
+    apiEndpoint: getRateLimitForPlan('enterprise', 'apiEndpoint'),
+  },
+}
+
+export function getRateLimit(
+  plan: SubscriptionPlan | string | undefined,
+  type: RateLimitCounterType
+): TokenBucketConfig {
+  const key = toConfigKey(type)
+  if (!isBillingEnabled) {
+    return RATE_LIMITS.free[key]
+  }
+  return RATE_LIMITS[getPlanTypeForLimits(plan)][key]
+}
+
+export class RateLimitError extends Error {
+  statusCode: number
+  constructor(message: string, statusCode = 429) {
+    super(message)
+    this.name = 'RateLimitError'
+    this.statusCode = statusCode
+  }
+}