chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,507 @@
+import { describe, expect, onTestFinished, vi } from "vitest";
+
+// db.server + splitMode are mocked so the idempotency dedup client resolves to
+// the container prisma passed into the concern (split stays off).
+vi.mock("~/db.server", () => ({
+  prisma: {},
+  $replica: {},
+  runOpsNewPrisma: {},
+  runOpsLegacyPrisma: {},
+}));
+
+vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
+
+vi.mock("~/services/platform.v3.server", async (importOriginal) => {
+  const actual = (await importOriginal()) as Record<string, unknown>;
+  return {
+    ...actual,
+    getEntitlement: vi.fn(),
+  };
+});
+
+import { RunEngine } from "@internal/run-engine";
+import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests";
+import { containerTest } from "@internal/testcontainers";
+import { trace } from "@opentelemetry/api";
+import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
+import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
+import type { ValidationResult } from "~/runEngine/types";
+import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server";
+import {
+  MOCK_SPAN_ID,
+  MOCK_TRACE_ID,
+  MockPayloadProcessor,
+  MockTraceEventConcern,
+  MockTriggerTaskValidator,
+} from "./triggerTaskTestHelpers";
+
+vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 });
+
+describe("RunEngineTriggerTaskService", () => {
+  // ─── Mollifier integration ──────────────────────────────────────────────────
+  //
+  // These tests pin the call-site behaviour of the mollifier hooks inside
+  // RunEngineTriggerTaskService.call. They use the optional DI ports
+  // (`evaluateGate`, `getMollifierBuffer`) added on the service constructor —
+  // production wiring is unchanged (defaults to the live module-level imports).
+  // Each test's regression intent lives in its own setup comment.
+
+  class CapturingMollifierBuffer {
+    public accepted: Array<{ runId: string; envId: string; orgId: string; payload: string }> = [];
+    async accept(input: { runId: string; envId: string; orgId: string; payload: string }) {
+      this.accepted.push(input);
+      return true;
+    }
+    async pop() {
+      return null;
+    }
+    async ack() {}
+    async requeue() {}
+    async fail() {
+      return false;
+    }
+    async getEntry() {
+      return null;
+    }
+    async listEnvs(): Promise<string[]> {
+      return [];
+    }
+    async getEntryTtlSeconds(): Promise<number> {
+      return -1;
+    }
+    async evaluateTrip() {
+      return { tripped: false, count: 0 };
+    }
+    async close() {}
+  }
+
+  containerTest(
+    "mollifier · validation throws before the gate is consulted; no buffer write",
+    async ({ prisma, redisOptions }) => {
+      const engine = new RunEngine({
+        prisma,
+        worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
+        queue: { redis: redisOptions },
+        runLock: { redis: redisOptions },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+      });
+      onTestFinished(() => engine.quit());
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+      const taskIdentifier = "test-task";
+      await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
+
+      // Validator that fails on maxAttempts. Any validation throw must abort
+      // the call BEFORE the gate runs — otherwise the gate could leak a
+      // buffer write for an invalid request.
+      class FailingMaxAttemptsValidator extends MockTriggerTaskValidator {
+        validateMaxAttempts(): ValidationResult {
+          return { ok: false, error: new Error("synthetic max-attempts failure") };
+        }
+      }
+
+      const buffer = new CapturingMollifierBuffer();
+      const evaluateGateSpy = vi.fn(async () => ({
+        action: "mollify" as const,
+        decision: {
+          divert: true as const,
+          reason: "per_env_rate" as const,
+          count: 99,
+          threshold: 1,
+          windowMs: 200,
+          holdMs: 500,
+        },
+      }));
+
+      const triggerTaskService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: new DefaultQueueManager(prisma, engine),
+        idempotencyKeyConcern: new IdempotencyKeyConcern(
+          prisma,
+          engine,
+          new MockTraceEventConcern()
+        ),
+        validator: new FailingMaxAttemptsValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024,
+        evaluateGate: evaluateGateSpy,
+        getMollifierBuffer: () => buffer as never,
+        isMollifierGloballyEnabled: () => true,
+      });
+
+      await expect(
+        triggerTaskService.call({
+          taskId: taskIdentifier,
+          environment: authenticatedEnvironment,
+          body: { payload: { test: "x" } },
+        })
+      ).rejects.toThrow(/synthetic max-attempts failure/);
+
+      // Critical: the gate must NEVER be consulted when validation fails.
+      // If this assertion fires, validation has been re-ordered after the
+      // mollifier gate — a regression that would let invalid triggers land
+      // in the buffer.
+      expect(evaluateGateSpy).not.toHaveBeenCalled();
+      expect(buffer.accepted).toHaveLength(0);
+    }
+  );
+
+  containerTest(
+    "mollifier · mollify action writes to buffer and returns synthetic result (no Postgres row)",
+    async ({ prisma, redisOptions }) => {
+      // When the gate decides mollify, the call site
+      // invokes `mollifyTrigger` which writes the engine.trigger snapshot
+      // to the buffer and returns a synthesised `MollifySyntheticResult`
+      // (run.friendlyId + notice + isCached:false). `engine.trigger` is
+      // NEVER invoked on this path — the run materialises in Postgres
+      // later, when the drainer replays the snapshot. The replay is
+      // covered by `mollifierDrainerHandler.test.ts`; this test pins the
+      // call-site integration: synthetic result + buffer write + no
+      // Postgres side effect.
+      const engine = new RunEngine({
+        prisma,
+        worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
+        queue: { redis: redisOptions },
+        runLock: { redis: redisOptions },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+      });
+      onTestFinished(() => engine.quit());
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+      const taskIdentifier = "test-task";
+      await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
+
+      // Buffer override records the time of the accept call so we can
+      // assert that traceRun fired strictly before the buffer was
+      // touched. If a future change re-introduces the "skip traceRun on
+      // mollify" shortcut, traceConcern.traceRunEnteredAt stays
+      // undefined and the ordering assertion fails.
+      class TimestampedBuffer extends CapturingMollifierBuffer {
+        public acceptedAt: number | undefined;
+        override async accept(input: {
+          runId: string;
+          envId: string;
+          orgId: string;
+          payload: string;
+        }) {
+          this.acceptedAt = Date.now();
+          return await super.accept(input);
+        }
+      }
+      const buffer = new TimestampedBuffer();
+      const trippedDecision = {
+        divert: true as const,
+        reason: "per_env_rate" as const,
+        count: 150,
+        threshold: 100,
+        windowMs: 200,
+        holdMs: 500,
+      };
+      const traceConcern = new MockTraceEventConcern();
+
+      const triggerTaskService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: new DefaultQueueManager(prisma, engine),
+        idempotencyKeyConcern: new IdempotencyKeyConcern(
+          prisma,
+          engine,
+          new MockTraceEventConcern()
+        ),
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: traceConcern,
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024,
+        evaluateGate: async () => ({ action: "mollify", decision: trippedDecision }),
+        getMollifierBuffer: () => buffer as never,
+        isMollifierGloballyEnabled: () => true,
+      });
+
+      const result = await triggerTaskService.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: { payload: { hello: "world" } },
+      });
+
+      // Pre-modifier span creation: traceRun must run BEFORE the buffer
+      // is touched. Customer-visible effect — the run span lands in
+      // ClickHouse from the moment the trigger returns, even when the
+      // drainer is offline, so buffered runs are visible in the trace
+      // view immediately rather than only after drain.
+      expect(traceConcern.traceRunEnteredAt).toBeDefined();
+      expect(buffer.acceptedAt).toBeDefined();
+      expect(traceConcern.traceRunEnteredAt!).toBeLessThanOrEqual(buffer.acceptedAt!);
+
+      // Synthetic result is returned with the `mollifier.queued` notice
+      // (the call-site casts the synthetic shape to `TriggerTaskServiceResult`;
+      // at runtime the `notice` and `isCached: false` fields are present
+      // and read by the api.v1.tasks.$taskId.trigger.ts route handler).
+      expect(result).toBeDefined();
+      expect(result?.run.friendlyId).toBeDefined();
+      const synthetic = result as unknown as {
+        run: { friendlyId: string };
+        isCached: false;
+        notice: { code: string; message: string; docs: string };
+      };
+      expect(synthetic.isCached).toBe(false);
+      expect(synthetic.notice.code).toBe("mollifier.queued");
+      expect(synthetic.notice.message).toBeTypeOf("string");
+      expect(synthetic.notice.docs).toBeTypeOf("string");
+
+      // The mollify branch must flag `isMollified: true` on the result so
+      // the trigger route can skip `saveRequestIdempotency`. Caching the
+      // synthetic runId in the request-idempotency table would mean a
+      // lost-response SDK retry (same `x-trigger-request-idempotency-key`
+      // header) hits a PG miss in `handleRequestIdempotency` and falls
+      // through to a fresh trigger — producing a duplicate buffer entry
+      // for trigger calls without a task-level idempotency key. The
+      // bounded behaviour (accept retry-as-fresh-trigger during the
+      // buffer window) is the deliberate choice; a stale-cache lookup
+      // returning null is not.
+      expect(result?.isMollified).toBe(true);
+
+      // buffer.accept ran — Redis has the canonical engine.trigger snapshot
+      // under the synthesised friendlyId. The drainer will read this and
+      // replay it through engine.trigger to materialise the run.
+      expect(buffer.accepted).toHaveLength(1);
+      expect(buffer.accepted[0]!.runId).toBe(result!.run.friendlyId);
+      expect(buffer.accepted[0]!.envId).toBe(authenticatedEnvironment.id);
+      expect(buffer.accepted[0]!.orgId).toBe(authenticatedEnvironment.organizationId);
+      // Payload is a JSON-serialised MollifierSnapshot (the engine.trigger
+      // input). Schema is internal to the engine, so we only assert that
+      // it parses and references the friendlyId — anything more specific
+      // would couple the mollifier-layer test to engine-layer fields.
+      const snapshot = JSON.parse(buffer.accepted[0]!.payload) as {
+        traceId?: string;
+        spanId?: string;
+        traceContext?: { traceparent?: string };
+      };
+
+      // Regression guard for the dashboard trace-tree bug: the mollifier
+      // snapshot MUST carry a W3C `traceparent` in `traceContext`,
+      // seeded from the same span traceRun opened. Without it, the
+      // drainer replays through engine.trigger with empty traceContext
+      // and every downstream `recordRunDebugLog`
+      // (QUEUED/EXECUTING/FINISHED/run:notify…) gets a fresh traceId +
+      // null parentId — the run-detail page can only show the root
+      // span. Both the mollify and pass-through paths now flow through
+      // `traceEventConcern.traceRun`; this assertion pins the
+      // seeding-from-the-run-span contract.
+      expect(snapshot.traceContext?.traceparent).toMatch(
+        /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/
+      );
+      expect(snapshot.traceContext!.traceparent).toContain(snapshot.traceId);
+      expect(snapshot.traceContext!.traceparent).toContain(snapshot.spanId);
+      // The snapshot inherits the *run span's* traceId/spanId (from the
+      // event handed in by traceRun), not a separately-generated OTel
+      // span. This is what lets the drainer's `mollifier.drained` span
+      // and downstream engine.trigger materialisation parent on the
+      // same ClickHouse trace the customer sees from the moment trigger
+      // returns.
+      expect(snapshot.traceId).toBe(MOCK_TRACE_ID);
+      expect(snapshot.spanId).toBe(MOCK_SPAN_ID);
+
+      // Postgres has NOT been written: engine.trigger was never called on
+      // the mollify path. The run materialises only when the drainer
+      // replays the snapshot. Regression intent: if a future change makes
+      // the mollify branch fall through to engine.trigger (re-introducing
+      // phase-1 dual-write), this assertion fails loudly.
+      const pgRun = await prisma.taskRun.findFirst({
+        where: { friendlyId: result!.run.friendlyId },
+      });
+      expect(pgRun).toBeNull();
+    }
+  );
+
+  containerTest(
+    "mollifier · pass_through action does NOT call buffer.accept",
+    async ({ prisma, redisOptions }) => {
+      const engine = new RunEngine({
+        prisma,
+        worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
+        queue: { redis: redisOptions },
+        runLock: { redis: redisOptions },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+      });
+      onTestFinished(() => engine.quit());
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+      const taskIdentifier = "test-task";
+      await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
+
+      const buffer = new CapturingMollifierBuffer();
+      const getBufferSpy = vi.fn(() => buffer as never);
+
+      const triggerTaskService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: new DefaultQueueManager(prisma, engine),
+        idempotencyKeyConcern: new IdempotencyKeyConcern(
+          prisma,
+          engine,
+          new MockTraceEventConcern()
+        ),
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024,
+        evaluateGate: async () => ({ action: "pass_through" }),
+        getMollifierBuffer: getBufferSpy,
+        isMollifierGloballyEnabled: () => true,
+      });
+
+      const result = await triggerTaskService.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: { payload: { test: "x" } },
+      });
+
+      expect(result).toBeDefined();
+      // Postgres has the run, no buffer side-effects
+      expect(buffer.accepted).toHaveLength(0);
+      // getMollifierBuffer must not be called either — the call site short-circuits
+      // before touching the singleton when the gate says pass_through.
+      expect(getBufferSpy).not.toHaveBeenCalled();
+      // Pass-through must NOT set `isMollified` — `result.run` is a real
+      // PG row, and the trigger route's `saveRequestIdempotency` is
+      // safe to call. Setting the flag here would silently skip the
+      // request-idempotency cache for every non-mollified trigger on a
+      // mollifier-enabled org, breaking lost-response retry dedup.
+      expect(result?.isMollified).toBeFalsy();
+    }
+  );
+
+  containerTest(
+    "mollifier · idempotency-key match short-circuits BEFORE the gate is consulted",
+    async ({ prisma, redisOptions }) => {
+      // SCENARIO: a trigger arrives with an idempotency key matching an
+      // already-created run. `IdempotencyKeyConcern.handleTriggerRequest`
+      // (line 236 of triggerTask.server.ts) detects the match BEFORE the
+      // mollifier gate runs and returns `{ isCached: true, run }`. The
+      // service early-returns. The gate is never consulted, buffer.accept
+      // never fires, no orphan entry is created.
+      //
+      // Regression intent: if IdempotencyKeyConcern were re-ordered to run
+      // AFTER evaluateGate, every idempotent retry on a flagged org would
+      // produce an orphan buffer entry — the audit-trail invariant ("every
+      // buffered runId has a matching TaskRun") would silently start failing
+      // for retries. This test pins the current order.
+
+      const engine = new RunEngine({
+        prisma,
+        worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
+        queue: { redis: redisOptions },
+        runLock: { redis: redisOptions },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+      });
+      onTestFinished(() => engine.quit());
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+      const taskIdentifier = "test-task";
+      await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
+
+      const idempotencyKeyConcern = new IdempotencyKeyConcern(
+        prisma,
+        engine,
+        new MockTraceEventConcern()
+      );
+
+      // Setup: normal trigger to create the cached run (no mollifier).
+      const baseline = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: new DefaultQueueManager(prisma, engine),
+        idempotencyKeyConcern,
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024,
+      });
+      const first = await baseline.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: { payload: { test: "x" }, options: { idempotencyKey: "regression-key-5" } },
+      });
+      expect(first?.isCached).toBe(false);
+
+      // Action: same idempotency key, with a mollify-stub gate that WOULD
+      // create an orphan if reached. The concern must short-circuit first.
+      const buffer = new CapturingMollifierBuffer();
+      const evaluateGateSpy = vi.fn(async () => ({
+        action: "mollify" as const,
+        decision: {
+          divert: true as const,
+          reason: "per_env_rate" as const,
+          count: 150,
+          threshold: 100,
+          windowMs: 200,
+          holdMs: 500,
+        },
+      }));
+
+      const mollifierService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: new DefaultQueueManager(prisma, engine),
+        idempotencyKeyConcern,
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024,
+        evaluateGate: evaluateGateSpy,
+        getMollifierBuffer: () => buffer as never,
+        isMollifierGloballyEnabled: () => true,
+      });
+
+      const cached = await mollifierService.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: { payload: { test: "x" }, options: { idempotencyKey: "regression-key-5" } },
+      });
+
+      // Customer sees the cached run, isCached=true
+      expect(cached).toBeDefined();
+      expect(cached?.isCached).toBe(true);
+      expect(cached?.run.friendlyId).toBe(first?.run.friendlyId);
+
+      // Critical: the gate must NEVER be consulted on a cached-idempotency replay.
+      expect(evaluateGateSpy).not.toHaveBeenCalled();
+      expect(buffer.accepted).toHaveLength(0);
+    }
+  );
+});