chore: import upstream snapshot with attribution

apps/sim/lib/core/idempotency/cleanup.ts

@@ -0,0 +1,182 @@
+import { db } from '@sim/db'
+import { idempotencyKey } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { and, count, inArray, like, lt, max, min, sql } from 'drizzle-orm'
+
+const logger = createLogger('IdempotencyCleanup')
+
+export interface CleanupOptions {
+  /**
+   * Maximum age of idempotency keys in seconds before they're considered expired
+   * Default: 7 days (604800 seconds)
+   */
+  maxAgeSeconds?: number
+
+  /**
+   * Maximum number of keys to delete in a single batch
+   * Default: 1000
+   */
+  batchSize?: number
+
+  /**
+   * Specific namespace prefix to clean up (e.g., 'webhook', 'polling')
+   * Keys are prefixed with namespace, so this filters by key prefix
+   */
+  namespace?: string
+}
+
+/**
+ * Clean up expired idempotency keys from the database
+ */
+export async function cleanupExpiredIdempotencyKeys(
+  options: CleanupOptions = {}
+): Promise<{ deleted: number; errors: string[] }> {
+  const {
+    maxAgeSeconds = 7 * 24 * 60 * 60, // 7 days
+    batchSize = 1000,
+    namespace,
+  } = options
+
+  const errors: string[] = []
+  let totalDeleted = 0
+
+  try {
+    const cutoffDate = new Date(Date.now() - maxAgeSeconds * 1000)
+
+    logger.info('Starting idempotency key cleanup', {
+      cutoffDate: cutoffDate.toISOString(),
+      namespace: namespace || 'all',
+      batchSize,
+    })
+
+    let hasMore = true
+    let batchCount = 0
+
+    while (hasMore) {
+      try {
+        // Build where condition - filter by cutoff date and optionally by namespace prefix
+        const whereCondition = namespace
+          ? and(
+              lt(idempotencyKey.createdAt, cutoffDate),
+              like(idempotencyKey.key, `${namespace}:%`)
+            )
+          : lt(idempotencyKey.createdAt, cutoffDate)
+
+        // Find keys to delete with limit
+        const toDelete = await db
+          .select({ key: idempotencyKey.key })
+          .from(idempotencyKey)
+          .where(whereCondition)
+          .limit(batchSize)
+
+        if (toDelete.length === 0) {
+          break
+        }
+
+        // Delete the found records by key
+        const deleteResult = await db
+          .delete(idempotencyKey)
+          .where(
+            inArray(
+              idempotencyKey.key,
+              toDelete.map((item) => item.key)
+            )
+          )
+          .returning({ key: idempotencyKey.key })
+
+        const deletedCount = deleteResult.length
+        totalDeleted += deletedCount
+        batchCount++
+
+        if (deletedCount === 0) {
+          hasMore = false
+          logger.info('No more expired idempotency keys found')
+        } else if (deletedCount < batchSize) {
+          hasMore = false
+          logger.info(`Deleted final batch of ${deletedCount} expired idempotency keys`)
+        } else {
+          logger.info(`Deleted batch ${batchCount}: ${deletedCount} expired idempotency keys`)
+
+          await sleep(100)
+        }
+      } catch (batchError) {
+        const errorMessage = getErrorMessage(batchError, 'Unknown batch error')
+        logger.error(`Error deleting batch ${batchCount + 1}:`, batchError)
+        errors.push(`Batch ${batchCount + 1}: ${errorMessage}`)
+
+        batchCount++
+
+        if (errors.length > 5) {
+          logger.error('Too many batch errors, stopping cleanup')
+          break
+        }
+      }
+    }
+
+    logger.info('Idempotency key cleanup completed', {
+      totalDeleted,
+      batchCount,
+      errors: errors.length,
+    })
+  } catch (error) {
+    const errorMessage = getErrorMessage(error, 'Unknown error')
+    logger.error('Failed to cleanup expired idempotency keys:', error)
+    errors.push(`General error: ${errorMessage}`)
+  }
+
+  return { deleted: totalDeleted, errors }
+}
+
+/**
+ * Get statistics about idempotency key usage
+ * Uses SQL aggregations to avoid loading all keys into memory
+ */
+export async function getIdempotencyKeyStats(): Promise<{
+  totalKeys: number
+  keysByNamespace: Record<string, number>
+  oldestKey: Date | null
+  newestKey: Date | null
+}> {
+  try {
+    // Get total count and date range in a single query
+    const [statsResult] = await db
+      .select({
+        totalKeys: count(),
+        oldestKey: min(idempotencyKey.createdAt),
+        newestKey: max(idempotencyKey.createdAt),
+      })
+      .from(idempotencyKey)
+
+    // Get counts by namespace prefix using SQL substring
+    // Extracts everything before the first ':' as the namespace
+    const namespaceStats = await db
+      .select({
+        namespace: sql<string>`split_part(${idempotencyKey.key}, ':', 1)`.as('namespace'),
+        count: count(),
+      })
+      .from(idempotencyKey)
+      .groupBy(sql`split_part(${idempotencyKey.key}, ':', 1)`)
+
+    const keysByNamespace: Record<string, number> = {}
+    for (const row of namespaceStats) {
+      keysByNamespace[row.namespace || 'unknown'] = row.count
+    }
+
+    return {
+      totalKeys: statsResult?.totalKeys ?? 0,
+      keysByNamespace,
+      oldestKey: statsResult?.oldestKey ?? null,
+      newestKey: statsResult?.newestKey ?? null,
+    }
+  } catch (error) {
+    logger.error('Failed to get idempotency key stats:', error)
+    return {
+      totalKeys: 0,
+      keysByNamespace: {},
+      oldestKey: null,
+      newestKey: null,
+    }
+  }
+}

apps/sim/lib/core/idempotency/index.ts

@@ -0,0 +1,6 @@
+export * from './cleanup'
+export * from './service'
+export {
+  pollingIdempotency,
+  webhookIdempotency,
+} from './service'

apps/sim/lib/core/idempotency/service.test.ts

@@ -0,0 +1,38 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import { IdempotencyService } from '@/lib/core/idempotency/service'
+
+describe('IdempotencyService.createWebhookIdempotencyKey', () => {
+  it('uses Greenhouse-Event-ID when present', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_1',
+      { 'greenhouse-event-id': 'evt-gh-99' },
+      {},
+      'greenhouse'
+    )
+    expect(key).toBe('wh_1:evt-gh-99')
+  })
+
+  it('prefers svix-id for Resend / Svix duplicate delivery deduplication', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_1',
+      { 'svix-id': 'msg_abc123' },
+      { type: 'email.sent' },
+      'resend'
+    )
+    expect(key).toBe('wh_1:msg_abc123')
+  })
+
+  it('prefers Linear-Delivery so repeated updates to the same entity are not treated as one idempotent run', () => {
+    const key = IdempotencyService.createWebhookIdempotencyKey(
+      'wh_linear',
+      { 'linear-delivery': '234d1a4e-b617-4388-90fe-adc3633d6b72' },
+      { action: 'update', data: { id: 'shared-entity-id' } },
+      'linear'
+    )
+    expect(key).toBe('wh_linear:234d1a4e-b617-4388-90fe-adc3633d6b72')
+  })
+})

apps/sim/lib/core/idempotency/service.ts

@@ -0,0 +1,558 @@
+import { db } from '@sim/db'
+import { idempotencyKey } from '@sim/db/schema'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { generateId } from '@sim/utils/id'
+import { eq, lt } from 'drizzle-orm'
+import { getRedisClient } from '@/lib/core/config/redis'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import { getStorageMethod, type StorageMethod } from '@/lib/core/storage'
+import { extractProviderIdentifierFromBody } from '@/lib/webhooks/providers'
+
+const logger = createLogger('IdempotencyService')
+
+export interface IdempotencyConfig {
+  ttlSeconds?: number
+  namespace?: string
+  /** When true, failed keys are deleted rather than stored so the operation is retried on the next attempt. */
+  retryFailures?: boolean
+  /**
+   * When false, only `{ success, status, error? }` is persisted — not the
+   * operation's return value. Duplicate calls still short-circuit but
+   * resolve to `undefined`. Use when callers don't consume the cached
+   * body (e.g. webhook receivers, where the provider just wants a 2xx).
+   * Defaults to true.
+   */
+  storeResultBody?: boolean
+  /**
+   * Force a specific storage backend regardless of the environment's
+   * auto-detection. Use `'database'` for correctness-critical flows
+   * (money, billing, compliance) where the claim + operation should
+   * fate-share with the Postgres transaction — this closes the narrow
+   * window where the operation commits to DB but `storeResult` to Redis
+   * fails and the retry re-runs the operation. Latency cost is 1–5ms
+   * per call, imperceptible on webhook code paths.
+   *
+   * Leave unset (or set `'redis'`) for latency-sensitive, high-volume
+   * flows like app webhook triggers where the scale benefits of Redis
+   * outweigh the narrow durability window.
+   */
+  forceStorage?: StorageMethod
+}
+
+export interface IdempotencyResult {
+  isFirstTime: boolean
+  normalizedKey: string
+  previousResult?: any
+  storageMethod: StorageMethod
+}
+
+export interface ProcessingResult {
+  success: boolean
+  result?: any
+  error?: string
+  status?: 'in-progress' | 'completed' | 'failed'
+  startedAt?: number
+}
+
+export interface AtomicClaimResult {
+  claimed: boolean
+  existingResult?: ProcessingResult
+  normalizedKey: string
+  storageMethod: StorageMethod
+}
+
+const DEFAULT_TTL = 60 * 60 * 24 * 7
+const REDIS_KEY_PREFIX = 'idempotency:'
+const MAX_WAIT_TIME_MS = getMaxExecutionTimeout()
+const POLL_INTERVAL_MS = 1000
+
+/**
+ * Universal idempotency service for webhooks, triggers, and any other operations
+ * that need duplicate prevention.
+ *
+ * Storage is determined once based on configuration:
+ * - If `forceStorage` is set → that backend unconditionally
+ * - Else if `REDIS_URL` is set → Redis
+ * - Else → PostgreSQL
+ */
+export class IdempotencyService {
+  private config: Required<Omit<IdempotencyConfig, 'forceStorage'>>
+  private storageMethod: StorageMethod
+
+  constructor(config: IdempotencyConfig = {}) {
+    this.config = {
+      ttlSeconds: config.ttlSeconds ?? DEFAULT_TTL,
+      namespace: config.namespace ?? 'default',
+      retryFailures: config.retryFailures ?? false,
+      storeResultBody: config.storeResultBody ?? true,
+    }
+    this.storageMethod = config.forceStorage ?? getStorageMethod()
+    logger.info(`IdempotencyService using ${this.storageMethod} storage`, {
+      namespace: this.config.namespace,
+      forced: Boolean(config.forceStorage),
+    })
+  }
+
+  private normalizeKey(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): string {
+    const base = `${this.config.namespace}:${provider}:${identifier}`
+
+    if (additionalContext && Object.keys(additionalContext).length > 0) {
+      const sortedKeys = Object.keys(additionalContext).sort()
+      const contextStr = sortedKeys.map((key) => `${key}=${additionalContext[key]}`).join('&')
+      return `${base}:${contextStr}`
+    }
+
+    return base
+  }
+
+  async checkIdempotency(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): Promise<IdempotencyResult> {
+    const normalizedKey = this.normalizeKey(provider, identifier, additionalContext)
+
+    if (this.storageMethod === 'redis') {
+      return this.checkIdempotencyRedis(normalizedKey)
+    }
+    return this.checkIdempotencyDb(normalizedKey)
+  }
+
+  private async checkIdempotencyRedis(normalizedKey: string): Promise<IdempotencyResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for idempotency check')
+    }
+
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+    const cachedResult = await redis.get(redisKey)
+
+    if (cachedResult) {
+      logger.debug(`Idempotency hit in Redis: ${normalizedKey}`)
+      return {
+        isFirstTime: false,
+        normalizedKey,
+        previousResult: JSON.parse(cachedResult),
+        storageMethod: 'redis',
+      }
+    }
+
+    logger.debug(`Idempotency miss in Redis: ${normalizedKey}`)
+    return {
+      isFirstTime: true,
+      normalizedKey,
+      storageMethod: 'redis',
+    }
+  }
+
+  private async checkIdempotencyDb(normalizedKey: string): Promise<IdempotencyResult> {
+    const existing = await db
+      .select({ result: idempotencyKey.result, createdAt: idempotencyKey.createdAt })
+      .from(idempotencyKey)
+      .where(eq(idempotencyKey.key, normalizedKey))
+      .limit(1)
+
+    if (existing.length > 0) {
+      const item = existing[0]
+      const isExpired = Date.now() - item.createdAt.getTime() > this.config.ttlSeconds * 1000
+
+      if (!isExpired) {
+        logger.debug(`Idempotency hit in database: ${normalizedKey}`)
+        return {
+          isFirstTime: false,
+          normalizedKey,
+          previousResult: item.result,
+          storageMethod: 'database',
+        }
+      }
+
+      await db
+        .delete(idempotencyKey)
+        .where(eq(idempotencyKey.key, normalizedKey))
+        .catch((err) => logger.warn(`Failed to clean up expired key ${normalizedKey}:`, err))
+    }
+
+    logger.debug(`Idempotency miss in database: ${normalizedKey}`)
+    return {
+      isFirstTime: true,
+      normalizedKey,
+      storageMethod: 'database',
+    }
+  }
+
+  async atomicallyClaim(
+    provider: string,
+    identifier: string,
+    additionalContext?: Record<string, any>
+  ): Promise<AtomicClaimResult> {
+    const normalizedKey = this.normalizeKey(provider, identifier, additionalContext)
+    const inProgressResult: ProcessingResult = {
+      success: false,
+      status: 'in-progress',
+      startedAt: Date.now(),
+    }
+
+    if (this.storageMethod === 'redis') {
+      return this.atomicallyClaimRedis(normalizedKey, inProgressResult)
+    }
+    return this.atomicallyClaimDb(normalizedKey, inProgressResult)
+  }
+
+  private async atomicallyClaimRedis(
+    normalizedKey: string,
+    inProgressResult: ProcessingResult
+  ): Promise<AtomicClaimResult> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for atomic claim')
+    }
+
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+    const claimed = await redis.set(
+      redisKey,
+      JSON.stringify(inProgressResult),
+      'EX',
+      this.config.ttlSeconds,
+      'NX'
+    )
+
+    if (claimed === 'OK') {
+      logger.debug(`Atomically claimed idempotency key in Redis: ${normalizedKey}`)
+      return {
+        claimed: true,
+        normalizedKey,
+        storageMethod: 'redis',
+      }
+    }
+
+    const existingData = await redis.get(redisKey)
+    const existingResult = existingData ? JSON.parse(existingData) : null
+    logger.debug(`Idempotency key already claimed in Redis: ${normalizedKey}`)
+    return {
+      claimed: false,
+      existingResult,
+      normalizedKey,
+      storageMethod: 'redis',
+    }
+  }
+
+  private async atomicallyClaimDb(
+    normalizedKey: string,
+    inProgressResult: ProcessingResult
+  ): Promise<AtomicClaimResult> {
+    const now = new Date()
+    const expiredBefore = new Date(now.getTime() - this.config.ttlSeconds * 1000)
+
+    // `ON CONFLICT DO UPDATE WHERE created_at < expiredBefore` steals the
+    // claim when the existing row has outlived the TTL (e.g. a prior
+    // holder crashed mid-operation and never wrote `completed`/`failed`
+    // or released the key). RETURNING yields a row in two cases:
+    //   (1) fresh INSERT — no prior row existed;
+    //   (2) UPDATE of an expired row — WHERE matched.
+    // An empty RETURNING means conflict with an unexpired row; the
+    // existing holder is still live and we must not steal.
+    const insertResult = await db
+      .insert(idempotencyKey)
+      .values({
+        key: normalizedKey,
+        result: inProgressResult,
+        createdAt: now,
+      })
+      .onConflictDoUpdate({
+        target: [idempotencyKey.key],
+        set: {
+          result: inProgressResult,
+          createdAt: now,
+        },
+        setWhere: lt(idempotencyKey.createdAt, expiredBefore),
+      })
+      .returning({ key: idempotencyKey.key })
+
+    if (insertResult.length > 0) {
+      logger.debug(`Atomically claimed idempotency key in database: ${normalizedKey}`)
+      return {
+        claimed: true,
+        normalizedKey,
+        storageMethod: 'database',
+      }
+    }
+
+    const existing = await db
+      .select({ result: idempotencyKey.result })
+      .from(idempotencyKey)
+      .where(eq(idempotencyKey.key, normalizedKey))
+      .limit(1)
+
+    const existingResult =
+      existing.length > 0 ? (existing[0].result as ProcessingResult) : undefined
+    logger.debug(`Idempotency key already claimed in database: ${normalizedKey}`)
+    return {
+      claimed: false,
+      existingResult,
+      normalizedKey,
+      storageMethod: 'database',
+    }
+  }
+
+  async waitForResult<T>(normalizedKey: string, storageMethod: 'redis' | 'database'): Promise<T> {
+    const startTime = Date.now()
+    const redisKey = `${REDIS_KEY_PREFIX}${normalizedKey}`
+
+    while (Date.now() - startTime < MAX_WAIT_TIME_MS) {
+      let currentResult: ProcessingResult | null = null
+
+      if (storageMethod === 'redis') {
+        const redis = getRedisClient()
+        if (!redis) {
+          throw new Error('Redis not available')
+        }
+        const data = await redis.get(redisKey)
+        currentResult = data ? JSON.parse(data) : null
+      } else {
+        const existing = await db
+          .select({ result: idempotencyKey.result })
+          .from(idempotencyKey)
+          .where(eq(idempotencyKey.key, normalizedKey))
+          .limit(1)
+        currentResult = existing.length > 0 ? (existing[0].result as ProcessingResult) : null
+      }
+
+      if (currentResult?.status === 'completed') {
+        logger.debug(`Operation completed, returning result: ${normalizedKey}`)
+        if (currentResult.success === false) {
+          throw new Error(currentResult.error || 'Previous operation failed')
+        }
+        return currentResult.result as T
+      }
+
+      if (currentResult?.status === 'failed') {
+        logger.debug(`Operation failed, throwing error: ${normalizedKey}`)
+        throw new Error(currentResult.error || 'Previous operation failed')
+      }
+
+      await sleep(POLL_INTERVAL_MS)
+    }
+
+    throw new Error(`Timeout waiting for idempotency operation to complete: ${normalizedKey}`)
+  }
+
+  async storeResult(
+    normalizedKey: string,
+    result: ProcessingResult,
+    storageMethod: 'redis' | 'database'
+  ): Promise<void> {
+    if (storageMethod === 'redis') {
+      return this.storeResultRedis(normalizedKey, result)
+    }
+    return this.storeResultDb(normalizedKey, result)
+  }
+
+  private async storeResultRedis(normalizedKey: string, result: ProcessingResult): Promise<void> {
+    const redis = getRedisClient()
+    if (!redis) {
+      throw new Error('Redis not available for storing result')
+    }
+
+    await redis.setex(
+      `${REDIS_KEY_PREFIX}${normalizedKey}`,
+      this.config.ttlSeconds,
+      JSON.stringify(result)
+    )
+    logger.debug(`Stored idempotency result in Redis: ${normalizedKey}`)
+  }
+
+  private async storeResultDb(normalizedKey: string, result: ProcessingResult): Promise<void> {
+    await db
+      .insert(idempotencyKey)
+      .values({
+        key: normalizedKey,
+        result: result,
+        createdAt: new Date(),
+      })
+      .onConflictDoUpdate({
+        target: [idempotencyKey.key],
+        set: {
+          result: result,
+          createdAt: new Date(),
+        },
+      })
+
+    logger.debug(`Stored idempotency result in database: ${normalizedKey}`)
+  }
+
+  async release(normalizedKey: string, storageMethod: 'redis' | 'database'): Promise<void> {
+    return this.deleteKey(normalizedKey, storageMethod)
+  }
+
+  private async deleteKey(
+    normalizedKey: string,
+    storageMethod: 'redis' | 'database'
+  ): Promise<void> {
+    if (storageMethod === 'redis') {
+      const redis = getRedisClient()
+      if (redis) await redis.del(`${REDIS_KEY_PREFIX}${normalizedKey}`).catch(() => {})
+    } else {
+      await db
+        .delete(idempotencyKey)
+        .where(eq(idempotencyKey.key, normalizedKey))
+        .catch(() => {})
+    }
+  }
+
+  async executeWithIdempotency<T>(
+    provider: string,
+    identifier: string,
+    operation: () => Promise<T>,
+    additionalContext?: Record<string, any>
+  ): Promise<T> {
+    const claimResult = await this.atomicallyClaim(provider, identifier, additionalContext)
+
+    if (!claimResult.claimed) {
+      const existingResult = claimResult.existingResult
+
+      if (existingResult?.status === 'completed') {
+        logger.info(`Returning cached result for: ${claimResult.normalizedKey}`)
+        if (existingResult.success === false) {
+          throw new Error(existingResult.error || 'Previous operation failed')
+        }
+        return existingResult.result as T
+      }
+
+      if (existingResult?.status === 'failed') {
+        if (this.config.retryFailures) {
+          await this.deleteKey(claimResult.normalizedKey, claimResult.storageMethod)
+          return this.executeWithIdempotency(provider, identifier, operation, additionalContext)
+        }
+        logger.info(`Previous operation failed for: ${claimResult.normalizedKey}`)
+        throw new Error(existingResult.error || 'Previous operation failed')
+      }
+
+      if (existingResult?.status === 'in-progress') {
+        logger.info(`Waiting for in-progress operation: ${claimResult.normalizedKey}`)
+        return await this.waitForResult<T>(claimResult.normalizedKey, claimResult.storageMethod)
+      }
+
+      if (existingResult) {
+        return existingResult.result as T
+      }
+
+      throw new Error(`Unexpected state: key claimed but no existing result found`)
+    }
+
+    try {
+      logger.info(`Executing new operation: ${claimResult.normalizedKey}`)
+      const result = await operation()
+
+      await this.storeResult(
+        claimResult.normalizedKey,
+        this.config.storeResultBody
+          ? { success: true, result, status: 'completed' }
+          : { success: true, status: 'completed' },
+        claimResult.storageMethod
+      )
+
+      logger.debug(`Successfully completed operation: ${claimResult.normalizedKey}`)
+      return result
+    } catch (error) {
+      const errorMessage = getErrorMessage(error, 'Unknown error')
+
+      if (this.config.retryFailures) {
+        await this.deleteKey(claimResult.normalizedKey, claimResult.storageMethod)
+      } else {
+        await this.storeResult(
+          claimResult.normalizedKey,
+          { success: false, error: errorMessage, status: 'failed' },
+          claimResult.storageMethod
+        )
+      }
+
+      logger.warn(`Operation failed: ${claimResult.normalizedKey} - ${errorMessage}`)
+      throw error
+    }
+  }
+
+  static createWebhookIdempotencyKey(
+    webhookId: string,
+    headers?: Record<string, string>,
+    body?: any,
+    provider?: string
+  ): string {
+    const normalizedHeaders = headers
+      ? Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v]))
+      : undefined
+
+    const webhookIdHeader =
+      normalizedHeaders?.['x-sim-idempotency-key'] ||
+      normalizedHeaders?.['webhook-id'] ||
+      normalizedHeaders?.['x-webhook-id'] ||
+      normalizedHeaders?.['x-shopify-webhook-id'] ||
+      normalizedHeaders?.['x-github-delivery'] ||
+      normalizedHeaders?.['x-gitlab-event-uuid'] ||
+      normalizedHeaders?.['x-event-id'] ||
+      normalizedHeaders?.['x-teams-notification-id'] ||
+      normalizedHeaders?.['svix-id'] ||
+      normalizedHeaders?.['linear-delivery'] ||
+      normalizedHeaders?.['greenhouse-event-id'] ||
+      normalizedHeaders?.['x-zm-request-id'] ||
+      normalizedHeaders?.['x-atlassian-webhook-identifier'] ||
+      normalizedHeaders?.['idempotency-key']
+
+    if (webhookIdHeader) {
+      return `${webhookId}:${webhookIdHeader}`
+    }
+
+    if (body && provider) {
+      const bodyIdentifier = extractProviderIdentifierFromBody(provider, body)
+      if (bodyIdentifier) {
+        return `${webhookId}:${bodyIdentifier}`
+      }
+    }
+
+    const uniqueId = generateId()
+    logger.warn('No unique identifier found, duplicate executions may occur', {
+      webhookId,
+      provider,
+    })
+    return `${webhookId}:${uniqueId}`
+  }
+}
+
+/**
+ * As a webhook receiver we only need a "we saw this delivery" marker —
+ * the provider's retry just needs a 2xx, not our cached response body.
+ * TTL must exceed the longest provider retry window (Gmail / Pub-Sub: 7d).
+ */
+export const webhookIdempotency = new IdempotencyService({
+  namespace: 'webhook',
+  ttlSeconds: 60 * 60 * 24 * 7, // 7 days
+  storeResultBody: false,
+})
+
+export const pollingIdempotency = new IdempotencyService({
+  namespace: 'polling',
+  ttlSeconds: 60 * 60 * 24 * 3, // 3 days
+  retryFailures: true,
+  storeResultBody: false,
+})
+
+/**
+ * Used by the internal `/api/billing/update-cost` endpoint (copilot,
+ * workspace-chat, MCP, mothership) to dedupe cost-recording calls. Storage
+ * is forced to Postgres: the operation writes AI cost to `user_stats`,
+ * and if Redis evicts the dedup key under memory pressure (high call
+ * volume) or drops it on restart, a retry would double-record usage —
+ * real money. DB storage fate-shares with `user_stats` and is
+ * eviction-proof; ~1-5ms added latency is invisible against LLM call
+ * latency.
+ */
+export const billingIdempotency = new IdempotencyService({
+  namespace: 'billing',
+  ttlSeconds: 60 * 60, // 1 hour
+  forceStorage: 'database',
+})