import { describe, expect, it, vi } from "vitest"; import { redisTest } from "@internal/testcontainers"; import { MollifierBuffer } from "@trigger.dev/redis-worker"; vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); import { runStaleSweepOnce, startStaleSweepInterval, } from "~/v3/mollifier/mollifierStaleSweep.server"; import { MollifierStaleSweepState } from "~/v3/mollifier/mollifierStaleSweepState.server"; const SNAPSHOT = { taskIdentifier: "hello-world", payload: '{"x":1}', payloadType: "application/json", traceContext: {}, }; // In-memory fake state for unit tests that don't have a Redis container. // The testcontainer tests use a real MollifierStaleSweepState against // the test Redis instead. function makeFakeState() { let cursor = 0; let orgList: string[] = []; const counts = new Map(); let visited = new Set(); return { readCursor: async () => cursor, writeCursor: async (v: number) => { cursor = v; }, rebuildOrgList: async (orgs: string[]) => { orgList = [...orgs]; }, readOrgListSlice: async (start: number, count: number) => ({ orgs: orgList.slice(start, start + count), total: orgList.length, }), setEnvStaleCount: async (envId: string, count: number) => { if (count > 0) counts.set(envId, count); else counts.delete(envId); }, readAllEnvStaleCounts: async () => new Map(counts), markEnvVisited: async (envId: string) => { visited.add(envId); }, reconcileVisited: async () => { for (const envId of [...counts.keys()]) { if (!visited.has(envId)) counts.delete(envId); } visited = new Set(); }, clearAll: async () => { cursor = 0; orgList = []; counts.clear(); visited = new Set(); }, close: async () => {}, }; } function spyDeps() { // Counter ticks — metric carries no `envId` label (high-cardinality) // so the spy is a simple call count. Per-env detail lives on the // structured warn log and the snapshot map. let staleEntryCount = 0; const snapshots: Array> = []; const warnings: Array<{ message: string; fields: Record }> = []; return { get staleEntryCount() { return staleEntryCount; }, snapshots, warnings, deps: { recordStaleEntry: () => { staleEntryCount += 1; }, reportStaleEntrySnapshot: (snapshot: Map) => { // Clone so post-sweep assertions see what was reported *at that // call site*, not whatever subsequent passes mutate the source // map into. snapshots.push(new Map(snapshot)); }, logger: { warn: (message: string, fields: Record) => { warnings.push({ message, fields }); }, }, }, }; } describe("runStaleSweepOnce — unit", () => { it("returns zeros when the buffer is null", async () => { // Mirrors the prod gate: if TRIGGER_MOLLIFIER_ENABLED=0 the buffer // singleton is null and the sweep is a no-op. We don't want it to // emit a metric (or throw) just because mollifier is disabled. const spies = spyDeps(); const result = await runStaleSweepOnce( { staleThresholdMs: 1000 }, { ...spies.deps, getBuffer: () => null, state: makeFakeState() } ); expect(result).toEqual({ orgsScanned: 0, envsScanned: 0, entriesScanned: 0, staleCount: 0, }); expect(spies.staleEntryCount).toBe(0); expect(spies.warnings).toEqual([]); const snapshots = spies.snapshots; // An empty snapshot is still reported so any previously-paging env // (from a prior sweep before mollifier was disabled) clears. expect(snapshots).toHaveLength(1); expect(snapshots[0].size).toBe(0); }); it("surfaces readOrgListSlice failures and leaves durable state untouched", async () => { // Regression: previously a Redis read failure inside // `readOrgListSlice` returned `{ orgs: [], total: 0 }` and the // sweep treated that as a clean empty cycle — writing cursor=0, // reconciling visited envs against the empty result, and CLEARING // the stale-entry gauge. That silenced the very alerts the sweep // exists to raise. The fix re-throws; the caller (this function // and the interval wrapper above it) must NOT mutate cursor or // counts when readOrgListSlice fails. const state = makeFakeState(); // Seed durable state so we can assert it isn't touched on failure. await state.writeCursor(42); await state.setEnvStaleCount("env_seed", 7); await state.rebuildOrgList(["org_pre"]); // Inject a failure on the very next slice read. const readErr = new Error("Redis read failed"); let readAttempts = 0; const failingState = { ...state, readOrgListSlice: async (start: number, count: number) => { readAttempts += 1; throw readErr; }, }; const spies = spyDeps(); const buffer = { listOrgs: async () => ["org_pre"], listEnvsForOrg: async () => [], listEntriesForEnv: async () => [], } as unknown as MollifierBuffer; await expect( runStaleSweepOnce( { staleThresholdMs: 60_000, maxOrgsPerPass: 10 }, { ...spies.deps, state: failingState, getBuffer: () => buffer, now: () => Date.now(), } ) ).rejects.toThrow("Redis read failed"); expect(readAttempts).toBe(1); // Cursor untouched (still the seeded 42, not reset to 0). expect(await state.readCursor()).toBe(42); // Counts hash untouched — the seeded env's count survives the // failed cycle so the gauge keeps reporting its last-known value. const counts = await state.readAllEnvStaleCounts(); expect(counts.get("env_seed")).toBe(7); // No snapshot was reported because the function threw before // reaching reportStaleEntrySnapshot. expect(spies.snapshots).toHaveLength(0); expect(spies.staleEntryCount).toBe(0); }); }); describe("runStaleSweepOnce — testcontainers", () => { redisTest( "flags every entry whose dwell exceeds the stale threshold", { timeout: 20_000 }, async ({ redisOptions }) => { const buffer = new MollifierBuffer({ redisOptions }); try { // Three entries across two envs in the same org. The sweep below // runs against a `now` advanced by 5 minutes, so all three have // dwell ~5min and ALL THREE are stale against a 1-minute // threshold — there is no "fresh" entry in this scenario. The // assertions below pin the all-three-stale shape. await buffer.accept({ runId: "run_stale_a", envId: "env_a", orgId: "org_1", payload: JSON.stringify(SNAPSHOT), }); await buffer.accept({ runId: "run_stale_b", envId: "env_b", orgId: "org_1", payload: JSON.stringify(SNAPSHOT), }); await buffer.accept({ runId: "run_stale_c", envId: "env_a", orgId: "org_1", payload: JSON.stringify(SNAPSHOT), }); // Yank the system clock forward 5 minutes for the sweep — way // past the threshold below. The `now` deps seam lets us drive // the threshold without actually waiting in real time. const futureNow = Date.now() + 5 * 60 * 1000; const spies = spyDeps(); const state = new MollifierStaleSweepState({ redisOptions }); try { const result = await runStaleSweepOnce( { staleThresholdMs: 60 * 1000 }, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, } ); expect(result.envsScanned).toBe(2); expect(result.entriesScanned).toBe(3); expect(result.staleCount).toBe(3); // All three entries exceed the threshold; each emits one // counter tick + one warning. expect(spies.staleEntryCount).toBe(3); expect(spies.warnings).toHaveLength(3); for (const w of spies.warnings) { expect(w.message).toBe("mollifier.stale_entry"); expect(w.fields.staleThresholdMs).toBe(60 * 1000); expect(w.fields.dwellMs).toBeGreaterThan(60 * 1000); } // Snapshot drives the alertable gauge — env_a has 2 stale // entries, env_b has 1. Per-env detail is still passed to // `reportStaleEntrySnapshot` for forensic value even though the // gauge itself aggregates the total. expect(spies.snapshots).toHaveLength(1); expect(Object.fromEntries(spies.snapshots[0])).toEqual({ env_a: 2, env_b: 1, }); } finally { await state.close(); } } finally { await buffer.close(); } } ); redisTest( "snapshot omits envs that have entries but none stale (durable hash HDEL's zeros)", { timeout: 20_000 }, async ({ redisOptions }) => { // Critical for alert behaviour: a previous sweep flagged env_a // stale, alert fired, drainer caught up. The next sweep must // remove env_a from the durable counts hash so the gauge drops // below the alert threshold instead of staying latched at the // last stale value. With the sharded design the snapshot is // sourced from the HASH directly — visiting an env with zero // stale entries HDEL's it, so it's simply absent from the // snapshot (telemetry sums values, so absence is equivalent to // zero for the gauge). const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_just_arrived", envId: "env_a", orgId: "org_1", payload: JSON.stringify(SNAPSHOT), }); const spies = spyDeps(); await runStaleSweepOnce( { staleThresholdMs: 60 * 1000 }, { ...spies.deps, getBuffer: () => buffer, state } ); expect(spies.snapshots).toHaveLength(1); // env_a has entries but none stale → not in the snapshot. expect(spies.snapshots[0].has("env_a")).toBe(false); } finally { await state.close(); await buffer.close(); } } ); redisTest( "leaves fresh entries alone (dwell below threshold)", { timeout: 20_000 }, async ({ redisOptions }) => { // Regression guard for the inequality direction. A bug that flipped // `dwellMs > threshold` to `dwellMs >= threshold` would flag every // entry the first time the sweep runs after a perfectly synchronised // accept call — the dashboard would page on every burst. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_fresh_only", envId: "env_a", orgId: "org_1", payload: JSON.stringify(SNAPSHOT), }); const spies = spyDeps(); const result = await runStaleSweepOnce( { staleThresholdMs: 60 * 1000 }, { ...spies.deps, getBuffer: () => buffer, state } ); expect(result.staleCount).toBe(0); expect(spies.staleEntryCount).toBe(0); expect(spies.warnings).toEqual([]); } finally { await state.close(); await buffer.close(); } } ); redisTest( "shards work across ticks: cursor advances by maxOrgsPerPass and wraps after a full cycle", { timeout: 30_000 }, async ({ redisOptions }) => { // Without sharding the sweep walks every org/env every tick — at // any meaningful backlog that runs longer than the tick interval // and the next tick gets dropped by the inFlight guard. Sharding // splits the work: each tick visits at most `maxOrgsPerPass` orgs, // advances a durable cursor, and resumes from there next tick. // Over `ceil(N / cap)` ticks the cycle covers every org. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { for (let i = 0; i < 5; i++) { await buffer.accept({ runId: `run_shard_${i}`, envId: `env_shard_${i}`, orgId: `org_shard_${i}`, payload: JSON.stringify(SNAPSHOT), }); } const futureNow = Date.now() + 5 * 60 * 1000; const spies = spyDeps(); const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 2 }; const baseDeps = { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }; // Tick 1: cursor starts at 0, scans 2 orgs. const r1 = await runStaleSweepOnce(cfg, baseDeps); expect(r1.orgsScanned).toBe(2); expect(spies.snapshots[0].size).toBe(2); // Tick 2: cursor was 2, scans 2 more orgs. const r2 = await runStaleSweepOnce(cfg, baseDeps); expect(r2.orgsScanned).toBe(2); // Snapshot is the durable HASH — accumulates across ticks. expect(spies.snapshots[1].size).toBe(4); // Tick 3: cursor was 4, scans the last 1 org and wraps to 0. const r3 = await runStaleSweepOnce(cfg, baseDeps); expect(r3.orgsScanned).toBe(1); expect(spies.snapshots[2].size).toBe(5); // Tick 4: cycle complete, cursor is back at 0 — starts over. const r4 = await runStaleSweepOnce(cfg, baseDeps); expect(r4.orgsScanned).toBe(2); } finally { await state.close(); await buffer.close(); } } ); redisTest( "clears an env from the durable snapshot on revisit when it has entries but none currently stale", { timeout: 30_000 }, async ({ redisOptions }) => { // Stale state in the durable hash must be HDEL'd, not just left // stale, when a previously-flagged env no longer has any entries // whose dwell exceeds the threshold (drainer caught up, alert // condition cleared). The same `entry` flips from stale to // not-stale between two sweep ticks by varying the sweep's `now` // — tick 1 uses a future clock so the entry is flagged stale; // tick 2 uses real time so the same entry has near-zero dwell and // is no longer stale. The env stays in the active set throughout // (queue still has an entry), so the cursor revisits it and the // hash field is cleared. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_drain", envId: "env_drain", orgId: "org_drain", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const spies = spyDeps(); const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 10 }; // Tick 1 with future clock: entry's dwell is 5min vs 1min // threshold → flagged stale. await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }); expect(spies.snapshots[0].get("env_drain")).toBe(1); // Tick 2 with real time: same entry, but its dwell is now ~ms // vs the same 1min threshold → not stale. The env is revisited // (cursor wrapped to 0 after tick 1, only 1 org in the list), // setEnvStaleCount called with 0 → HDEL. await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => buffer, state, }); expect(spies.snapshots[1].has("env_drain")).toBe(false); } finally { await state.close(); await buffer.close(); } } ); redisTest( "evicts fully-drained envs from the counts hash at cycle wrap (no permanent alert)", { timeout: 30_000 }, async ({ redisOptions }) => { // Devin's BUG report on PR #3754: an env that drains completely // between sweep ticks disappears from `mollifier:org-envs:${orgId}` // entirely, so the inner loop at runStaleSweepOnce never visits it // and `setEnvStaleCount(envId, 0)` (which HDELs the field) is // never called. The counts hash retains the env's last-known // stale count forever, the gauge stays elevated, and the // recommended alert `> 0 for 5m` fires indefinitely. // // Fix: at cycle wrap (cursor returned to 0) HDEL any env in the // counts hash that wasn't visited during the just-completed cycle. // Verified here by: // 1. Flagging env_will_drain stale, confirming it's in the hash // 2. Draining its only entry — now invisible to listEnvsForOrg // 3. Running a sweep tick that triggers cycle wrap // 4. Asserting the env is no longer in the snapshot const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_will_drain", envId: "env_will_drain", orgId: "org_will_drain", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 10 }; const spies = spyDeps(); // Tick 1: env_will_drain is flagged stale → enters counts hash. // Cursor wraps to 0 (only 1 org in the list). await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }); expect(spies.snapshots[0].get("env_will_drain")).toBe(1); // Drain the only entry. mollifier:queue:env_will_drain is now // empty, and the buffer's atomic Lua removes env_will_drain // from `mollifier:org-envs:org_will_drain` (and removes the org // from `mollifier:orgs` since it has no other envs). The env is // now invisible to listEnvsForOrg. const popped = await buffer.pop("env_will_drain"); expect(popped?.runId).toBe("run_will_drain"); // Tick 2: cursor was 0 after tick 1's wrap, so this rebuilds // the org list (now empty) and immediately wraps again. The // wrap-handler must HDEL env_will_drain from the counts hash // because it wasn't in the visited set for this cycle. await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }); expect(spies.snapshots[1].has("env_will_drain")).toBe(false); // And the durable hash is genuinely empty, not just absent from // this snapshot. expect((await state.readAllEnvStaleCounts()).size).toBe(0); } finally { await state.close(); await buffer.close(); } } ); redisTest("scans across multiple orgs", { timeout: 20_000 }, async ({ redisOptions }) => { // The drainer pops with org-level fairness, so the sweep must // walk every org/env to surface stale entries across all of them // — not just stop at the first env it finds. If a future refactor // collapsed listOrgs/listEnvsForOrg into a single env-flat list, // this test catches a regression there. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_x", envId: "env_x", orgId: "org_x", payload: JSON.stringify(SNAPSHOT), }); await buffer.accept({ runId: "run_y", envId: "env_y", orgId: "org_y", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const spies = spyDeps(); const result = await runStaleSweepOnce( { staleThresholdMs: 60 * 1000 }, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow } ); expect(result.orgsScanned).toBe(2); expect(result.envsScanned).toBe(2); expect(result.staleCount).toBe(2); } finally { await state.close(); await buffer.close(); } }); redisTest( "state survives process restart: a second state instance picks up the cursor and counts", { timeout: 30_000 }, async ({ redisOptions }) => { // This is the headline reason the sweep state is durable in Redis // instead of process-local — a webapp restart mid-cycle must not // re-emit the gauge as fresh-zero for previously-flagged envs nor // restart the cursor walk from scratch. Simulated here by closing // state1 (its Redis client quits cleanly) and constructing state2 // against the same Redis. The cursor + counts that state1 wrote // are visible to state2 on its first tick. const buffer = new MollifierBuffer({ redisOptions }); const state1 = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_a", envId: "env_a", orgId: "org_a", payload: JSON.stringify(SNAPSHOT), }); await buffer.accept({ runId: "run_b", envId: "env_b", orgId: "org_b", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 1 }; const spies1 = spyDeps(); // Tick 1 with state1: visits 1 of 2 orgs. await runStaleSweepOnce(cfg, { ...spies1.deps, getBuffer: () => buffer, state: state1, now: () => futureNow, }); expect(spies1.snapshots[0].size).toBe(1); } finally { // Simulate webapp restart: state1's Redis client closes cleanly. await state1.close(); } // New process boots, constructs a fresh state pointing at the // same Redis. The cycle's frozen org_list, the cursor, and the // counts hash are all preserved — state2 picks up at the second // org of the cycle. const state2 = new MollifierStaleSweepState({ redisOptions }); try { const futureNow = Date.now() + 5 * 60 * 1000; const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 1 }; const spies2 = spyDeps(); await runStaleSweepOnce(cfg, { ...spies2.deps, getBuffer: () => buffer, state: state2, now: () => futureNow, }); // Snapshot now has BOTH envs: the one tick 1 flagged (still in // the counts hash from state1) plus the one tick 2 just flagged. // A non-durable design would show only the second. expect(spies2.snapshots[0].size).toBe(2); } finally { await state2.close(); await buffer.close(); } } ); redisTest( "cycle wrap rebuilds the org list, so orgs that joined mid-cycle get visited on the next cycle", { timeout: 30_000 }, async ({ redisOptions }) => { // The docstring promises "orgs joining mid-cycle wait until the // next cycle to be visited." The mechanism is rebuildOrgList at // cursor=0: a fresh snapshot of buffer.listOrgs() replaces the // previous frozen LIST. Verified here by adding a third org // between cycles and asserting it shows up only in the next // cycle's snapshot. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_init_a", envId: "env_init_a", orgId: "org_init_a", payload: JSON.stringify(SNAPSHOT), }); await buffer.accept({ runId: "run_init_b", envId: "env_init_b", orgId: "org_init_b", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const spies = spyDeps(); const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 10 }; const baseDeps = { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }; // Tick 1: cycle 1. Visits both initial orgs; cursor wraps to 0. await runStaleSweepOnce(cfg, baseDeps); expect(spies.snapshots[0].size).toBe(2); // Mid-flight: a third org joins the buffer. It must NOT have // been part of cycle 1's frozen LIST. await buffer.accept({ runId: "run_mid", envId: "env_mid", orgId: "org_mid", payload: JSON.stringify(SNAPSHOT), }); // Tick 2: cycle 2 begins (cursor was 0 after tick 1's wrap). // rebuildOrgList captures all 3 orgs; this tick visits all 3. const r2 = await runStaleSweepOnce(cfg, baseDeps); expect(r2.orgsScanned).toBe(3); expect(spies.snapshots[1].size).toBe(3); expect(spies.snapshots[1].has("env_mid")).toBe(true); } finally { await state.close(); await buffer.close(); } } ); redisTest( "empty buffer (no orgs) advances cleanly with zero work and an empty snapshot", { timeout: 30_000 }, async ({ redisOptions }) => { // `mollifier:orgs` is empty (no entries ever accepted, or every // entry has been drained). The sweep must handle the boundary: // rebuildOrgList with [], readOrgListSlice returns total=0, // the org loop is skipped, and the cursor stays at 0 instead of // tripping the wrap math. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { const spies = spyDeps(); const result = await runStaleSweepOnce( { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 10 }, { ...spies.deps, getBuffer: () => buffer, state } ); expect(result).toEqual({ orgsScanned: 0, envsScanned: 0, entriesScanned: 0, staleCount: 0, }); expect(spies.snapshots).toHaveLength(1); expect(spies.snapshots[0].size).toBe(0); // Cursor stayed at 0 — nothing to advance through. expect(await state.readCursor()).toBe(0); } finally { await state.close(); await buffer.close(); } } ); redisTest( "buffer-null branch wipes the durable state so a re-enable starts fresh", { timeout: 30_000 }, async ({ redisOptions }) => { // The unit test above asserts the snapshot is empty when the // buffer is null, but doesn't verify the durable state was // actually cleared. Without clearAll the next re-enable would // resume on a stale cursor + carry over a stale counts hash. const buffer = new MollifierBuffer({ redisOptions }); const state = new MollifierStaleSweepState({ redisOptions }); try { await buffer.accept({ runId: "run_seed", envId: "env_seed", orgId: "org_seed", payload: JSON.stringify(SNAPSHOT), }); const futureNow = Date.now() + 5 * 60 * 1000; const cfg = { staleThresholdMs: 60 * 1000, maxOrgsPerPass: 10 }; const spies = spyDeps(); // Tick 1: populate state. await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => buffer, state, now: () => futureNow, }); expect(spies.snapshots[0].size).toBe(1); expect((await state.readAllEnvStaleCounts()).size).toBe(1); // Tick 2: mollifier flips OFF — getBuffer returns null. The // sweep must clear the durable state. await runStaleSweepOnce(cfg, { ...spies.deps, getBuffer: () => null, state, }); expect(spies.snapshots[1].size).toBe(0); expect((await state.readAllEnvStaleCounts()).size).toBe(0); expect(await state.readCursor()).toBe(0); } finally { await state.close(); await buffer.close(); } } ); }); describe("MollifierStaleSweepState — direct unit tests", () => { redisTest( "readCursor returns 0 when the key is absent", { timeout: 20_000 }, async ({ redisOptions }) => { const state = new MollifierStaleSweepState({ redisOptions }); try { expect(await state.readCursor()).toBe(0); } finally { await state.close(); } } ); redisTest( "writeCursor + readCursor round-trip; readCursor parses a non-numeric value as 0", { timeout: 20_000 }, async ({ redisOptions }) => { const state = new MollifierStaleSweepState({ redisOptions }); try { await state.writeCursor(42); expect(await state.readCursor()).toBe(42); // Defensive: a corrupted/garbage value must not throw or // propagate NaN into the sweep's cursor arithmetic. await state["redis"].set("mollifier:stale_sweep:cursor", "not-a-number"); expect(await state.readCursor()).toBe(0); } finally { await state.close(); } } ); redisTest( "rebuildOrgList replaces the previous list (DEL + RPUSH, in order)", { timeout: 20_000 }, async ({ redisOptions }) => { const state = new MollifierStaleSweepState({ redisOptions }); try { await state.rebuildOrgList(["org_a", "org_b", "org_c"]); let slice = await state.readOrgListSlice(0, 10); expect(slice.total).toBe(3); expect(slice.orgs).toEqual(["org_a", "org_b", "org_c"]); // Replacement, not append. await state.rebuildOrgList(["org_x"]); slice = await state.readOrgListSlice(0, 10); expect(slice.total).toBe(1); expect(slice.orgs).toEqual(["org_x"]); // Empty rebuild leaves the list empty (DEL fires, no RPUSH). await state.rebuildOrgList([]); slice = await state.readOrgListSlice(0, 10); expect(slice.total).toBe(0); expect(slice.orgs).toEqual([]); } finally { await state.close(); } } ); redisTest( "setEnvStaleCount HSETs when count > 0 and HDELs when count === 0", { timeout: 20_000 }, async ({ redisOptions }) => { const state = new MollifierStaleSweepState({ redisOptions }); try { await state.setEnvStaleCount("env_a", 3); await state.setEnvStaleCount("env_b", 1); let counts = await state.readAllEnvStaleCounts(); expect(Object.fromEntries(counts)).toEqual({ env_a: 3, env_b: 1 }); // Zero clears the field (HDEL), not stores 0. await state.setEnvStaleCount("env_a", 0); counts = await state.readAllEnvStaleCounts(); expect(Object.fromEntries(counts)).toEqual({ env_b: 1 }); expect(counts.has("env_a")).toBe(false); } finally { await state.close(); } } ); redisTest( "clearAll DELs cursor, org_list, and counts in one call", { timeout: 20_000 }, async ({ redisOptions }) => { const state = new MollifierStaleSweepState({ redisOptions }); try { await state.writeCursor(7); await state.rebuildOrgList(["org_a", "org_b"]); await state.setEnvStaleCount("env_a", 5); await state.clearAll(); expect(await state.readCursor()).toBe(0); expect((await state.readOrgListSlice(0, 10)).total).toBe(0); expect((await state.readAllEnvStaleCounts()).size).toBe(0); } finally { await state.close(); } } ); }); describe("startStaleSweepInterval — lifecycle", () => { it("stop() waits for an in-flight tick to finish before closing the state", async () => { // Devin's BUG report on PR #3754: `stop()` previously called // `deps.state.close()` immediately after `clearInterval`, but the // `tick` function only checks `stopped` at entry. A tick that was // already past that check would keep making `state.*` Redis calls // against a now-closed ioredis client, throw, get caught by tick's // own try/catch, and log a `mollifier.stale_sweep.failed` warning // for every graceful shutdown. // // The fix tracks the current tick promise so `stop()` can await it // before closing. This test pins that order by gating one of the // tick's state calls on a Deferred — until we resolve it, the tick // can't progress, and `stop()` must hang in the meantime. let resolveGate: () => void = () => {}; const gate = new Promise((r) => { resolveGate = r; }); const callOrder: string[] = []; let closeCalled = false; const state = { readCursor: async () => { callOrder.push("readCursor:start"); await gate; callOrder.push("readCursor:end"); return 0; }, writeCursor: async () => { callOrder.push("writeCursor"); }, rebuildOrgList: async () => { callOrder.push("rebuildOrgList"); }, readOrgListSlice: async () => { callOrder.push("readOrgListSlice"); // Return zero orgs so the org loop is a no-op — we only care // about ordering of state calls vs close, not the work. return { orgs: [] as string[], total: 0 }; }, setEnvStaleCount: async () => { callOrder.push("setEnvStaleCount"); }, readAllEnvStaleCounts: async () => { callOrder.push("readAllEnvStaleCounts"); return new Map(); }, markEnvVisited: async () => { callOrder.push("markEnvVisited"); }, reconcileVisited: async () => { callOrder.push("reconcileVisited"); }, clearAll: async () => { callOrder.push("clearAll"); }, close: async () => { callOrder.push("close"); closeCalled = true; }, }; const fakeBuffer = { listOrgs: async () => [], listEnvsForOrg: async () => [], listEntriesForEnv: async () => [], } as any; const handle = startStaleSweepInterval( { intervalMs: 20, staleThresholdMs: 60_000, maxOrgsPerPass: 10, }, { state, getBuffer: () => fakeBuffer, recordStaleEntry: () => {}, reportStaleEntrySnapshot: () => {}, logger: { warn: () => {} }, now: () => Date.now(), } ); // Wait for the interval to fire one tick. The tick will start, call // readCursor, and then block on `gate`. await new Promise((r) => setTimeout(r, 80)); expect(callOrder).toContain("readCursor:start"); expect(closeCalled).toBe(false); // Call stop() concurrently — its promise MUST NOT resolve while the // tick is still mid-flight. let stopResolved = false; const stopPromise = handle.stop().then(() => { stopResolved = true; }); await new Promise((r) => setTimeout(r, 50)); expect(stopResolved).toBe(false); expect(closeCalled).toBe(false); // Release the gate. The tick can now finish, and only then should // stop() resolve and close the state. resolveGate(); await stopPromise; expect(stopResolved).toBe(true); expect(closeCalled).toBe(true); // The tick's readCursor:end MUST appear before the close — otherwise // we closed the Redis client out from under an in-flight tick. expect(callOrder.indexOf("readCursor:end")).toBeGreaterThan(-1); expect(callOrder.indexOf("close")).toBeGreaterThan(callOrder.indexOf("readCursor:end")); }); });