chore: import upstream snapshot with attribution

apps/sim/lib/concurrency/__tests__/leader-lock.test.ts

@@ -0,0 +1,215 @@
+/**
+ * @vitest-environment node
+ */
+import { redisConfigMock, redisConfigMockFns } from '@sim/testing'
+import { sleep } from '@sim/utils/helpers'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/redis', () => redisConfigMock)
+
+import { withLeaderLock } from '@/lib/concurrency/leader-lock'
+
+beforeEach(() => {
+  vi.clearAllMocks()
+  redisConfigMockFns.mockAcquireLock.mockResolvedValue(true)
+  redisConfigMockFns.mockReleaseLock.mockResolvedValue(true)
+})
+
+describe('withLeaderLock', () => {
+  it('runs onLeader exactly once when lock acquired', async () => {
+    const onLeader = vi.fn(async () => 'leader-result')
+    const onFollower = vi.fn(async () => null)
+
+    const result = await withLeaderLock<string>({
+      key: 'k',
+      onLeader,
+      onFollower,
+    })
+
+    expect(result).toBe('leader-result')
+    expect(onLeader).toHaveBeenCalledTimes(1)
+    expect(onFollower).not.toHaveBeenCalled()
+    expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledTimes(1)
+  })
+
+  it('passes a fresh owner token to acquireLock and releaseLock', async () => {
+    await withLeaderLock<string>({
+      key: 'k',
+      onLeader: async () => 'x',
+      onFollower: async () => null,
+    })
+
+    const [acquireKey, acquireValue] = redisConfigMockFns.mockAcquireLock.mock.calls[0]!
+    const [releaseKey, releaseValue] = redisConfigMockFns.mockReleaseLock.mock.calls[0]!
+
+    expect(acquireKey).toBe('k')
+    expect(releaseKey).toBe('k')
+    expect(acquireValue).toBe(releaseValue)
+    expect(typeof acquireValue).toBe('string')
+    expect((acquireValue as string).length).toBeGreaterThan(0)
+  })
+
+  it('falls back to uncoordinated leader when acquireLock throws', async () => {
+    redisConfigMockFns.mockAcquireLock.mockRejectedValueOnce(new Error('redis down'))
+
+    const onLeader = vi.fn(async () => 'fallback')
+    const onFollower = vi.fn(async () => null)
+
+    const result = await withLeaderLock<string>({
+      key: 'k',
+      onLeader,
+      onFollower,
+    })
+
+    expect(result).toBe('fallback')
+    expect(onLeader).toHaveBeenCalledTimes(1)
+    expect(onFollower).not.toHaveBeenCalled()
+    expect(redisConfigMockFns.mockReleaseLock).not.toHaveBeenCalled()
+  })
+
+  it('does not propagate releaseLock errors out of the leader path', async () => {
+    redisConfigMockFns.mockReleaseLock.mockRejectedValueOnce(new Error('redis blip'))
+
+    const result = await withLeaderLock<string>({
+      key: 'k',
+      onLeader: async () => 'leader-value',
+      onFollower: async () => null,
+    })
+
+    expect(result).toBe('leader-value')
+  })
+
+  it('releases the lock even when onLeader throws', async () => {
+    const onLeader = vi.fn(async () => {
+      throw new Error('boom')
+    })
+
+    await expect(
+      withLeaderLock<string>({
+        key: 'k',
+        onLeader,
+        onFollower: async () => null,
+      })
+    ).rejects.toThrow('boom')
+
+    expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledTimes(1)
+  })
+
+  it('follower polls onFollower until it returns non-null', async () => {
+    redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
+
+    let polls = 0
+    const onFollower = vi.fn(async () => {
+      polls += 1
+      if (polls >= 2) return 'available'
+      return null
+    })
+
+    const result = await withLeaderLock<string>({
+      key: 'k',
+      pollIntervalMs: 5,
+      maxWaitMs: 1000,
+      onLeader: async () => 'should-not-run',
+      onFollower,
+    })
+
+    expect(result).toBe('available')
+    expect(onFollower.mock.calls.length).toBeGreaterThanOrEqual(2)
+  })
+
+  it('follower does a final read after timeout to catch a just-finished leader', async () => {
+    redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
+
+    /**
+     * The intent: after the in-loop poll deadline is reached, the follower
+     * does exactly one more (last-chance) `onFollower` call to catch a leader
+     * that finished between the previous poll and the timeout. Using fake
+     * timers makes the timing deterministic — pollInterval=10 and maxWait=15
+     * cause two in-loop polls (T+10, T+20) and one last-chance read (T+20),
+     * but the schedule is driven by mocked time, not the CI wall clock.
+     */
+    vi.useFakeTimers()
+    try {
+      let polls = 0
+      const onFollower = vi.fn(async () => {
+        polls += 1
+        if (polls <= 2) return null
+        return 'late-leader'
+      })
+
+      const promise = withLeaderLock<string>({
+        key: 'k',
+        pollIntervalMs: 10,
+        maxWaitMs: 15,
+        onLeader: async () => 'should-not-run',
+        onFollower,
+      })
+
+      await vi.advanceTimersByTimeAsync(30)
+      const result = await promise
+
+      expect(result).toBe('late-leader')
+      expect(onFollower).toHaveBeenCalledTimes(3)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('follower returns null after timeout', async () => {
+    redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
+
+    vi.useFakeTimers()
+    try {
+      const onFollower = vi.fn(async () => null)
+      const promise = withLeaderLock<string>({
+        key: 'k',
+        pollIntervalMs: 10,
+        maxWaitMs: 25,
+        onLeader: async () => 'should-not-run',
+        onFollower,
+      })
+
+      await vi.advanceTimersByTimeAsync(50)
+      const result = await promise
+
+      expect(result).toBeNull()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('only one of N concurrent callers acquires the lock', async () => {
+    // Track which calls won the lock: first one returns true, rest return false.
+    let acquired = false
+    redisConfigMockFns.mockAcquireLock.mockImplementation(async () => {
+      if (acquired) return false
+      acquired = true
+      return true
+    })
+    redisConfigMockFns.mockReleaseLock.mockImplementation(async () => {
+      acquired = false
+      return true
+    })
+
+    let leaderRuns = 0
+
+    const callers = Array.from({ length: 5 }, () =>
+      withLeaderLock<string>({
+        key: 'shared',
+        pollIntervalMs: 5,
+        maxWaitMs: 200,
+        onLeader: async () => {
+          leaderRuns += 1
+          await sleep(20)
+          return 'leader-value'
+        },
+        onFollower: async () => (acquired ? null : 'follower-saw-released'),
+      })
+    )
+
+    const results = await Promise.all(callers)
+    expect(leaderRuns).toBe(1)
+    expect(results.filter((r) => r === 'leader-value').length).toBe(1)
+    expect(results.filter((r) => r === 'follower-saw-released').length).toBeGreaterThan(0)
+  })
+})

apps/sim/lib/concurrency/__tests__/singleflight.test.ts

@@ -0,0 +1,163 @@
+/**
+ * @vitest-environment node
+ */
+import { sleep } from '@sim/utils/helpers'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+  __resetCoalesceLocallyForTests,
+  CoalesceSettleTimeoutError,
+  coalesceLocally,
+} from '@/lib/concurrency/singleflight'
+
+afterEach(() => {
+  __resetCoalesceLocallyForTests()
+  vi.restoreAllMocks()
+})
+
+describe('coalesceLocally', () => {
+  it('invokes fn once when N callers race on the same key', async () => {
+    const fn = vi.fn(async () => {
+      await sleep(5)
+      return 'value'
+    })
+
+    const results = await Promise.all(
+      Array.from({ length: 10 }, () => coalesceLocally('shared', fn))
+    )
+
+    expect(fn).toHaveBeenCalledTimes(1)
+    expect(results).toEqual(Array.from({ length: 10 }, () => 'value'))
+  })
+
+  it('returns the same promise instance to concurrent callers', () => {
+    const fn = async () => {
+      await sleep(10)
+      return 1
+    }
+    const a = coalesceLocally('same-key', fn)
+    const b = coalesceLocally('same-key', fn)
+    expect(a).toBe(b)
+  })
+
+  it('clears the cache after success so the next call invokes fn again', async () => {
+    let count = 0
+    const fn = async () => {
+      count += 1
+      return count
+    }
+
+    expect(await coalesceLocally('once', fn)).toBe(1)
+    expect(await coalesceLocally('once', fn)).toBe(2)
+  })
+
+  it('clears the cache after rejection so the next call invokes fn again', async () => {
+    let count = 0
+    const fn = async () => {
+      count += 1
+      throw new Error(`fail ${count}`)
+    }
+
+    await expect(coalesceLocally('rejection', fn)).rejects.toThrow('fail 1')
+    await expect(coalesceLocally('rejection', fn)).rejects.toThrow('fail 2')
+  })
+
+  it('surfaces a synchronously-thrown fn error and evicts the entry', async () => {
+    const fn = vi.fn((): Promise<string> => {
+      throw new Error('sync boom')
+    })
+
+    // The real error must surface (not a TDZ ReferenceError from the evict
+    // closure) and the entry must be evicted so the next call retries.
+    await expect(coalesceLocally('sync-throw', fn)).rejects.toThrow('sync boom')
+    await expect(coalesceLocally('sync-throw', fn)).rejects.toThrow('sync boom')
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not coalesce across distinct keys', async () => {
+    const fn = vi.fn(async () => 'value')
+    await Promise.all([coalesceLocally('a', fn), coalesceLocally('b', fn)])
+    expect(fn).toHaveBeenCalledTimes(2)
+  })
+
+  it('rejects all awaiters and evicts the entry when the producer misses the settle deadline', async () => {
+    vi.useFakeTimers()
+    try {
+      let resolveHung: (value: string) => void
+      const hung = vi.fn(
+        () =>
+          new Promise<string>((resolve) => {
+            resolveHung = resolve
+          })
+      )
+
+      const a = coalesceLocally('wedged', hung)
+      const b = coalesceLocally('wedged', hung)
+      const aAssertion = expect(a).rejects.toBeInstanceOf(CoalesceSettleTimeoutError)
+      const bAssertion = expect(b).rejects.toBeInstanceOf(CoalesceSettleTimeoutError)
+
+      await vi.advanceTimersByTimeAsync(30_000)
+      await aAssertion
+      await bAssertion
+      expect(hung).toHaveBeenCalledTimes(1)
+
+      const fresh = vi.fn(async () => 'recovered')
+      await expect(coalesceLocally('wedged', fresh)).resolves.toBe('recovered')
+      expect(fresh).toHaveBeenCalledTimes(1)
+
+      resolveHung!('late')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('a timed-out producer settling late does not evict its successor', async () => {
+    vi.useFakeTimers()
+    try {
+      let resolveOld: (value: string) => void
+      const old = coalesceLocally(
+        'late-settle',
+        () =>
+          new Promise<string>((resolve) => {
+            resolveOld = resolve
+          }),
+        1_000
+      )
+      const oldAssertion = expect(old).rejects.toBeInstanceOf(CoalesceSettleTimeoutError)
+      await vi.advanceTimersByTimeAsync(1_000)
+      await oldAssertion
+
+      let resolveNew: (value: string) => void
+      const successor = coalesceLocally(
+        'late-settle',
+        () =>
+          new Promise<string>((resolve) => {
+            resolveNew = resolve
+          })
+      )
+
+      resolveOld!('late')
+      await vi.advanceTimersByTimeAsync(0)
+
+      const joined = coalesceLocally('late-settle', async () => 'should-not-run')
+      expect(joined).toBe(successor)
+
+      resolveNew!('new-value')
+      await expect(successor).resolves.toBe('new-value')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('does not fire the deadline for producers that settle in time', async () => {
+    vi.useFakeTimers()
+    try {
+      const value = await coalesceLocally('prompt', async () => 'ok', 1_000)
+      expect(value).toBe('ok')
+
+      await vi.advanceTimersByTimeAsync(2_000)
+      await expect(coalesceLocally('prompt', async () => 'again', 1_000)).resolves.toBe('again')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+})

apps/sim/lib/concurrency/leader-lock.ts

@@ -0,0 +1,73 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { generateShortId } from '@sim/utils/id'
+import { acquireLock, releaseLock } from '@/lib/core/config/redis'
+
+const logger = createLogger('LeaderLock')
+
+const DEFAULT_TTL_SEC = 10
+const DEFAULT_POLL_INTERVAL_MS = 100
+const DEFAULT_MAX_WAIT_MS = 3_000
+
+export interface LeaderLockOptions<T> {
+  key: string
+  ttlSec?: number
+  pollIntervalMs?: number
+  maxWaitMs?: number
+  onLeader: () => Promise<T | null>
+  onFollower: () => Promise<T | null>
+}
+
+export async function withLeaderLock<T>(opts: LeaderLockOptions<T>): Promise<T | null> {
+  const {
+    key,
+    ttlSec = DEFAULT_TTL_SEC,
+    pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+    maxWaitMs = DEFAULT_MAX_WAIT_MS,
+    onLeader,
+    onFollower,
+  } = opts
+
+  const ownerToken = generateShortId()
+
+  let acquired = false
+  try {
+    acquired = await acquireLock(key, ownerToken, ttlSec)
+  } catch (error) {
+    logger.warn('Lock acquisition failed; running leader path uncoordinated', {
+      key,
+      error: toError(error).message,
+    })
+    return onLeader()
+  }
+
+  if (acquired) {
+    try {
+      return await onLeader()
+    } finally {
+      try {
+        await releaseLock(key, ownerToken)
+      } catch (error) {
+        logger.warn('Lock release failed (will expire via TTL)', {
+          key,
+          error: toError(error).message,
+        })
+      }
+    }
+  }
+
+  const deadline = Date.now() + maxWaitMs
+  while (Date.now() < deadline) {
+    await sleep(pollIntervalMs)
+    const value = await onFollower()
+    if (value !== null) return value
+  }
+
+  // The leader may have persisted between our final poll and now; one last check.
+  const lastChance = await onFollower()
+  if (lastChance !== null) return lastChance
+
+  logger.warn('Follower timed out waiting for leader', { key, maxWaitMs })
+  return null
+}

apps/sim/lib/concurrency/singleflight.ts

@@ -0,0 +1,71 @@
+const inflight = new Map<string, Promise<unknown>>()
+
+/**
+ * Default deadline for a coalesced producer to settle. Joiners share the
+ * producer's promise, so without a deadline a single hung producer wedges
+ * every future caller for that key until process restart.
+ */
+const DEFAULT_SETTLE_TIMEOUT_MS = 30_000
+
+/**
+ * Thrown to all awaiters when a coalesced producer fails to settle within
+ * its deadline. The entry is evicted first, so the next caller mints a
+ * fresh producer instead of joining the wedged one.
+ */
+export class CoalesceSettleTimeoutError extends Error {
+  constructor(key: string, timeoutMs: number) {
+    super(`Coalesced producer for "${key}" did not settle within ${timeoutMs}ms`)
+    this.name = 'CoalesceSettleTimeoutError'
+  }
+}
+
+/**
+ * Deduplicates concurrent async work by key within this process: the first
+ * caller runs `fn`, every concurrent caller for the same key shares its
+ * promise. The entry is evicted when the producer settles (either way) or
+ * when the settle deadline fires, whichever comes first. The underlying
+ * `fn` is not cancelled on timeout — it keeps running detached, but no new
+ * caller will join it.
+ */
+export function coalesceLocally<T>(
+  key: string,
+  fn: () => Promise<T>,
+  settleTimeoutMs: number = DEFAULT_SETTLE_TIMEOUT_MS
+): Promise<T> {
+  const existing = inflight.get(key) as Promise<T> | undefined
+  if (existing) return existing
+
+  let timer: ReturnType<typeof setTimeout> | undefined
+  const evict = () => {
+    if (inflight.get(key) === guarded) inflight.delete(key)
+  }
+
+  const guarded: Promise<T> = Promise.race([
+    (async () => {
+      try {
+        // Defer fn() to a microtask so a synchronous throw surfaces as a
+        // rejection after `guarded` and the timer are initialized. Calling it
+        // inline would run the finally below during construction, touching
+        // `guarded` in its temporal dead zone and masking fn's real error.
+        return await Promise.resolve().then(fn)
+      } finally {
+        clearTimeout(timer)
+        evict()
+      }
+    })(),
+    new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        evict()
+        reject(new CoalesceSettleTimeoutError(key, settleTimeoutMs))
+      }, settleTimeoutMs)
+      timer.unref?.()
+    }),
+  ])
+
+  inflight.set(key, guarded)
+  return guarded
+}
+
+export function __resetCoalesceLocallyForTests(): void {
+  inflight.clear()
+}