import { describe, expect, onTestFinished, vi } from "vitest"; // db.server + splitMode are mocked so the idempotency dedup client resolves to // the container prisma passed into the concern (split stays off). vi.mock("~/db.server", () => ({ prisma: {}, $replica: {}, runOpsNewPrisma: {}, runOpsLegacyPrisma: {}, })); vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); vi.mock("~/services/platform.v3.server", async (importOriginal) => { const actual = (await importOriginal()) as Record; return { ...actual, getEntitlement: vi.fn(), }; }); import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests"; import { containerTest } from "@internal/testcontainers"; import { trace } from "@opentelemetry/api"; import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; import type { ValidationResult } from "~/runEngine/types"; import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server"; import { MOCK_SPAN_ID, MOCK_TRACE_ID, MockPayloadProcessor, MockTraceEventConcern, MockTriggerTaskValidator, } from "./triggerTaskTestHelpers"; vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); describe("RunEngineTriggerTaskService", () => { // ─── Mollifier integration ────────────────────────────────────────────────── // // These tests pin the call-site behaviour of the mollifier hooks inside // RunEngineTriggerTaskService.call. They use the optional DI ports // (`evaluateGate`, `getMollifierBuffer`) added on the service constructor — // production wiring is unchanged (defaults to the live module-level imports). // Each test's regression intent lives in its own setup comment. class CapturingMollifierBuffer { public accepted: Array<{ runId: string; envId: string; orgId: string; payload: string }> = []; async accept(input: { runId: string; envId: string; orgId: string; payload: string }) { this.accepted.push(input); return true; } async pop() { return null; } async ack() {} async requeue() {} async fail() { return false; } async getEntry() { return null; } async listEnvs(): Promise { return []; } async getEntryTtlSeconds(): Promise { return -1; } async evaluateTrip() { return { tripped: false, count: 0 }; } async close() {} } containerTest( "mollifier · validation throws before the gate is consulted; no buffer write", async ({ prisma, redisOptions }) => { const engine = new RunEngine({ prisma, worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, queue: { redis: redisOptions }, runLock: { redis: redisOptions }, machines: { defaultMachine: "small-1x", machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, }, baseCostInCents: 0.0005, }, tracer: trace.getTracer("test", "0.0.0"), }); onTestFinished(() => engine.quit()); const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); const taskIdentifier = "test-task"; await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); // Validator that fails on maxAttempts. Any validation throw must abort // the call BEFORE the gate runs — otherwise the gate could leak a // buffer write for an invalid request. class FailingMaxAttemptsValidator extends MockTriggerTaskValidator { validateMaxAttempts(): ValidationResult { return { ok: false, error: new Error("synthetic max-attempts failure") }; } } const buffer = new CapturingMollifierBuffer(); const evaluateGateSpy = vi.fn(async () => ({ action: "mollify" as const, decision: { divert: true as const, reason: "per_env_rate" as const, count: 99, threshold: 1, windowMs: 200, holdMs: 500, }, })); const triggerTaskService = new RunEngineTriggerTaskService({ engine, prisma, payloadProcessor: new MockPayloadProcessor(), queueConcern: new DefaultQueueManager(prisma, engine), idempotencyKeyConcern: new IdempotencyKeyConcern( prisma, engine, new MockTraceEventConcern() ), validator: new FailingMaxAttemptsValidator(), traceEventConcern: new MockTraceEventConcern(), tracer: trace.getTracer("test", "0.0.0"), metadataMaximumSize: 1024 * 1024, evaluateGate: evaluateGateSpy, getMollifierBuffer: () => buffer as never, isMollifierGloballyEnabled: () => true, }); await expect( triggerTaskService.call({ taskId: taskIdentifier, environment: authenticatedEnvironment, body: { payload: { test: "x" } }, }) ).rejects.toThrow(/synthetic max-attempts failure/); // Critical: the gate must NEVER be consulted when validation fails. // If this assertion fires, validation has been re-ordered after the // mollifier gate — a regression that would let invalid triggers land // in the buffer. expect(evaluateGateSpy).not.toHaveBeenCalled(); expect(buffer.accepted).toHaveLength(0); } ); containerTest( "mollifier · mollify action writes to buffer and returns synthetic result (no Postgres row)", async ({ prisma, redisOptions }) => { // When the gate decides mollify, the call site // invokes `mollifyTrigger` which writes the engine.trigger snapshot // to the buffer and returns a synthesised `MollifySyntheticResult` // (run.friendlyId + notice + isCached:false). `engine.trigger` is // NEVER invoked on this path — the run materialises in Postgres // later, when the drainer replays the snapshot. The replay is // covered by `mollifierDrainerHandler.test.ts`; this test pins the // call-site integration: synthetic result + buffer write + no // Postgres side effect. const engine = new RunEngine({ prisma, worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, queue: { redis: redisOptions }, runLock: { redis: redisOptions }, machines: { defaultMachine: "small-1x", machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, }, baseCostInCents: 0.0005, }, tracer: trace.getTracer("test", "0.0.0"), }); onTestFinished(() => engine.quit()); const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); const taskIdentifier = "test-task"; await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); // Buffer override records the time of the accept call so we can // assert that traceRun fired strictly before the buffer was // touched. If a future change re-introduces the "skip traceRun on // mollify" shortcut, traceConcern.traceRunEnteredAt stays // undefined and the ordering assertion fails. class TimestampedBuffer extends CapturingMollifierBuffer { public acceptedAt: number | undefined; override async accept(input: { runId: string; envId: string; orgId: string; payload: string; }) { this.acceptedAt = Date.now(); return await super.accept(input); } } const buffer = new TimestampedBuffer(); const trippedDecision = { divert: true as const, reason: "per_env_rate" as const, count: 150, threshold: 100, windowMs: 200, holdMs: 500, }; const traceConcern = new MockTraceEventConcern(); const triggerTaskService = new RunEngineTriggerTaskService({ engine, prisma, payloadProcessor: new MockPayloadProcessor(), queueConcern: new DefaultQueueManager(prisma, engine), idempotencyKeyConcern: new IdempotencyKeyConcern( prisma, engine, new MockTraceEventConcern() ), validator: new MockTriggerTaskValidator(), traceEventConcern: traceConcern, tracer: trace.getTracer("test", "0.0.0"), metadataMaximumSize: 1024 * 1024, evaluateGate: async () => ({ action: "mollify", decision: trippedDecision }), getMollifierBuffer: () => buffer as never, isMollifierGloballyEnabled: () => true, }); const result = await triggerTaskService.call({ taskId: taskIdentifier, environment: authenticatedEnvironment, body: { payload: { hello: "world" } }, }); // Pre-modifier span creation: traceRun must run BEFORE the buffer // is touched. Customer-visible effect — the run span lands in // ClickHouse from the moment the trigger returns, even when the // drainer is offline, so buffered runs are visible in the trace // view immediately rather than only after drain. expect(traceConcern.traceRunEnteredAt).toBeDefined(); expect(buffer.acceptedAt).toBeDefined(); expect(traceConcern.traceRunEnteredAt!).toBeLessThanOrEqual(buffer.acceptedAt!); // Synthetic result is returned with the `mollifier.queued` notice // (the call-site casts the synthetic shape to `TriggerTaskServiceResult`; // at runtime the `notice` and `isCached: false` fields are present // and read by the api.v1.tasks.$taskId.trigger.ts route handler). expect(result).toBeDefined(); expect(result?.run.friendlyId).toBeDefined(); const synthetic = result as unknown as { run: { friendlyId: string }; isCached: false; notice: { code: string; message: string; docs: string }; }; expect(synthetic.isCached).toBe(false); expect(synthetic.notice.code).toBe("mollifier.queued"); expect(synthetic.notice.message).toBeTypeOf("string"); expect(synthetic.notice.docs).toBeTypeOf("string"); // The mollify branch must flag `isMollified: true` on the result so // the trigger route can skip `saveRequestIdempotency`. Caching the // synthetic runId in the request-idempotency table would mean a // lost-response SDK retry (same `x-trigger-request-idempotency-key` // header) hits a PG miss in `handleRequestIdempotency` and falls // through to a fresh trigger — producing a duplicate buffer entry // for trigger calls without a task-level idempotency key. The // bounded behaviour (accept retry-as-fresh-trigger during the // buffer window) is the deliberate choice; a stale-cache lookup // returning null is not. expect(result?.isMollified).toBe(true); // buffer.accept ran — Redis has the canonical engine.trigger snapshot // under the synthesised friendlyId. The drainer will read this and // replay it through engine.trigger to materialise the run. expect(buffer.accepted).toHaveLength(1); expect(buffer.accepted[0]!.runId).toBe(result!.run.friendlyId); expect(buffer.accepted[0]!.envId).toBe(authenticatedEnvironment.id); expect(buffer.accepted[0]!.orgId).toBe(authenticatedEnvironment.organizationId); // Payload is a JSON-serialised MollifierSnapshot (the engine.trigger // input). Schema is internal to the engine, so we only assert that // it parses and references the friendlyId — anything more specific // would couple the mollifier-layer test to engine-layer fields. const snapshot = JSON.parse(buffer.accepted[0]!.payload) as { traceId?: string; spanId?: string; traceContext?: { traceparent?: string }; }; // Regression guard for the dashboard trace-tree bug: the mollifier // snapshot MUST carry a W3C `traceparent` in `traceContext`, // seeded from the same span traceRun opened. Without it, the // drainer replays through engine.trigger with empty traceContext // and every downstream `recordRunDebugLog` // (QUEUED/EXECUTING/FINISHED/run:notify…) gets a fresh traceId + // null parentId — the run-detail page can only show the root // span. Both the mollify and pass-through paths now flow through // `traceEventConcern.traceRun`; this assertion pins the // seeding-from-the-run-span contract. expect(snapshot.traceContext?.traceparent).toMatch( /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/ ); expect(snapshot.traceContext!.traceparent).toContain(snapshot.traceId); expect(snapshot.traceContext!.traceparent).toContain(snapshot.spanId); // The snapshot inherits the *run span's* traceId/spanId (from the // event handed in by traceRun), not a separately-generated OTel // span. This is what lets the drainer's `mollifier.drained` span // and downstream engine.trigger materialisation parent on the // same ClickHouse trace the customer sees from the moment trigger // returns. expect(snapshot.traceId).toBe(MOCK_TRACE_ID); expect(snapshot.spanId).toBe(MOCK_SPAN_ID); // Postgres has NOT been written: engine.trigger was never called on // the mollify path. The run materialises only when the drainer // replays the snapshot. Regression intent: if a future change makes // the mollify branch fall through to engine.trigger (re-introducing // phase-1 dual-write), this assertion fails loudly. const pgRun = await prisma.taskRun.findFirst({ where: { friendlyId: result!.run.friendlyId }, }); expect(pgRun).toBeNull(); } ); containerTest( "mollifier · pass_through action does NOT call buffer.accept", async ({ prisma, redisOptions }) => { const engine = new RunEngine({ prisma, worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, queue: { redis: redisOptions }, runLock: { redis: redisOptions }, machines: { defaultMachine: "small-1x", machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, }, baseCostInCents: 0.0005, }, tracer: trace.getTracer("test", "0.0.0"), }); onTestFinished(() => engine.quit()); const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); const taskIdentifier = "test-task"; await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); const buffer = new CapturingMollifierBuffer(); const getBufferSpy = vi.fn(() => buffer as never); const triggerTaskService = new RunEngineTriggerTaskService({ engine, prisma, payloadProcessor: new MockPayloadProcessor(), queueConcern: new DefaultQueueManager(prisma, engine), idempotencyKeyConcern: new IdempotencyKeyConcern( prisma, engine, new MockTraceEventConcern() ), validator: new MockTriggerTaskValidator(), traceEventConcern: new MockTraceEventConcern(), tracer: trace.getTracer("test", "0.0.0"), metadataMaximumSize: 1024 * 1024, evaluateGate: async () => ({ action: "pass_through" }), getMollifierBuffer: getBufferSpy, isMollifierGloballyEnabled: () => true, }); const result = await triggerTaskService.call({ taskId: taskIdentifier, environment: authenticatedEnvironment, body: { payload: { test: "x" } }, }); expect(result).toBeDefined(); // Postgres has the run, no buffer side-effects expect(buffer.accepted).toHaveLength(0); // getMollifierBuffer must not be called either — the call site short-circuits // before touching the singleton when the gate says pass_through. expect(getBufferSpy).not.toHaveBeenCalled(); // Pass-through must NOT set `isMollified` — `result.run` is a real // PG row, and the trigger route's `saveRequestIdempotency` is // safe to call. Setting the flag here would silently skip the // request-idempotency cache for every non-mollified trigger on a // mollifier-enabled org, breaking lost-response retry dedup. expect(result?.isMollified).toBeFalsy(); } ); containerTest( "mollifier · idempotency-key match short-circuits BEFORE the gate is consulted", async ({ prisma, redisOptions }) => { // SCENARIO: a trigger arrives with an idempotency key matching an // already-created run. `IdempotencyKeyConcern.handleTriggerRequest` // (line 236 of triggerTask.server.ts) detects the match BEFORE the // mollifier gate runs and returns `{ isCached: true, run }`. The // service early-returns. The gate is never consulted, buffer.accept // never fires, no orphan entry is created. // // Regression intent: if IdempotencyKeyConcern were re-ordered to run // AFTER evaluateGate, every idempotent retry on a flagged org would // produce an orphan buffer entry — the audit-trail invariant ("every // buffered runId has a matching TaskRun") would silently start failing // for retries. This test pins the current order. const engine = new RunEngine({ prisma, worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, queue: { redis: redisOptions }, runLock: { redis: redisOptions }, machines: { defaultMachine: "small-1x", machines: { "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, }, baseCostInCents: 0.0005, }, tracer: trace.getTracer("test", "0.0.0"), }); onTestFinished(() => engine.quit()); const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); const taskIdentifier = "test-task"; await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier); const idempotencyKeyConcern = new IdempotencyKeyConcern( prisma, engine, new MockTraceEventConcern() ); // Setup: normal trigger to create the cached run (no mollifier). const baseline = new RunEngineTriggerTaskService({ engine, prisma, payloadProcessor: new MockPayloadProcessor(), queueConcern: new DefaultQueueManager(prisma, engine), idempotencyKeyConcern, validator: new MockTriggerTaskValidator(), traceEventConcern: new MockTraceEventConcern(), tracer: trace.getTracer("test", "0.0.0"), metadataMaximumSize: 1024 * 1024, }); const first = await baseline.call({ taskId: taskIdentifier, environment: authenticatedEnvironment, body: { payload: { test: "x" }, options: { idempotencyKey: "regression-key-5" } }, }); expect(first?.isCached).toBe(false); // Action: same idempotency key, with a mollify-stub gate that WOULD // create an orphan if reached. The concern must short-circuit first. const buffer = new CapturingMollifierBuffer(); const evaluateGateSpy = vi.fn(async () => ({ action: "mollify" as const, decision: { divert: true as const, reason: "per_env_rate" as const, count: 150, threshold: 100, windowMs: 200, holdMs: 500, }, })); const mollifierService = new RunEngineTriggerTaskService({ engine, prisma, payloadProcessor: new MockPayloadProcessor(), queueConcern: new DefaultQueueManager(prisma, engine), idempotencyKeyConcern, validator: new MockTriggerTaskValidator(), traceEventConcern: new MockTraceEventConcern(), tracer: trace.getTracer("test", "0.0.0"), metadataMaximumSize: 1024 * 1024, evaluateGate: evaluateGateSpy, getMollifierBuffer: () => buffer as never, isMollifierGloballyEnabled: () => true, }); const cached = await mollifierService.call({ taskId: taskIdentifier, environment: authenticatedEnvironment, body: { payload: { test: "x" }, options: { idempotencyKey: "regression-key-5" } }, }); // Customer sees the cached run, isCached=true expect(cached).toBeDefined(); expect(cached?.isCached).toBe(true); expect(cached?.run.friendlyId).toBe(first?.run.friendlyId); // Critical: the gate must NEVER be consulted on a cached-idempotency replay. expect(evaluateGateSpy).not.toHaveBeenCalled(); expect(buffer.accepted).toHaveLength(0); } ); });