chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,109 @@
+// Repro for the checkpoint WAIT_FOR_BATCH replica-lag stall (createCheckpoint.server.ts:148-181).
+//
+// The service resolves the batch via `runStore.findBatchTaskRunByFriendlyId(friendlyId, envId)` with
+// NO client, so the read is served from the REPLICA. Its decision hinges on `batchRun.resumedAt`:
+//   resumedAt set  -> return keepRunAlive:true  (batch already resumed; the run must keep executing)
+//   resumedAt null -> fall through -> create the checkpoint -> SUSPEND the run
+// If the batch just resumed (primary has resumedAt), but the replica still lags (resumedAt null), the
+// service suspends a run whose batch already completed -> it stalls until a sweep. The sibling
+// WAIT_FOR_TASK arm reads the primary (this._prisma); only the batch arm defaults to the replica.
+//
+// This is invisible to the normal single-DB harness (no lag). We reintroduce the lag with the shared
+// `laggingReplica` primitive: the store's replica is frozen at the pre-resume snapshot while the
+// primary advances. RED = the current (client-less, replica) read -> SUSPEND. GREEN = threading the
+// primary (the one-line fix) -> KEEP_ALIVE.
+
+import { laggingReplica, postgresTest } from "@internal/testcontainers";
+import type { PrismaClient } from "@trigger.dev/database";
+import { describe, expect } from "vitest";
+import { PostgresRunStore } from "./PostgresRunStore.js";
+import type { ReadClient } from "./types.js";
+
+type BatchRow = { resumedAt: Date | null } | null;
+
+// A line-for-line mirror of the service's WAIT_FOR_BATCH pre-check. `readClient` models the fix: the
+// buggy code passes nothing (replica default); the fix threads the primary.
+async function precheckWaitForBatch(
+  store: PostgresRunStore,
+  batchFriendlyId: string,
+  environmentId: string,
+  readClient?: ReadClient
+): Promise<"DROP_RUN" | "KEEP_ALIVE" | "SUSPEND"> {
+  const batchRun = (await store.findBatchTaskRunByFriendlyId(
+    batchFriendlyId,
+    environmentId,
+    undefined,
+    readClient
+  )) as BatchRow;
+  if (!batchRun) return "DROP_RUN"; // keepRunAlive:false
+  if (batchRun.resumedAt) return "KEEP_ALIVE"; // batch already resumed -> run continues
+  return "SUSPEND"; // falls through -> checkpoint created -> run suspended
+}
+
+async function seedEnvironment(prisma: PrismaClient, suffix: string) {
+  const organization = await prisma.organization.create({
+    data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
+  });
+  const project = await prisma.project.create({
+    data: {
+      name: `Project ${suffix}`,
+      slug: `project-${suffix}`,
+      externalRef: `proj_${suffix}`,
+      organizationId: organization.id,
+    },
+  });
+  const environment = await prisma.runtimeEnvironment.create({
+    data: {
+      type: "DEVELOPMENT",
+      slug: "dev",
+      projectId: project.id,
+      organizationId: organization.id,
+      apiKey: `tr_dev_${suffix}`,
+      pkApiKey: `pk_dev_${suffix}`,
+      shortcode: `short_${suffix}`,
+    },
+  });
+  return { organization, project, environment };
+}
+
+describe("checkpoint WAIT_FOR_BATCH under replica lag", () => {
+  postgresTest(
+    "a batch resumed on the primary but stale on the replica suspends an already-resumed run",
+    async ({ prisma }) => {
+      const { environment } = await seedEnvironment(prisma, "ckpt_lag");
+      const friendlyId = "batch_ckpt_lag";
+      const batch = await prisma.batchTaskRun.create({
+        data: { friendlyId, runtimeEnvironmentId: environment.id },
+      });
+
+      // Snapshot the batch as the replica still sees it (pre-resume: resumedAt = null).
+      const staleBatch = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batch.id } });
+      expect(staleBatch.resumedAt).toBeNull();
+
+      // The batch completes and resumes the parent: primary now has resumedAt set...
+      await prisma.batchTaskRun.update({
+        where: { id: batch.id },
+        data: { resumedAt: new Date() },
+      });
+
+      // ...but the replica lags, frozen at the pre-resume snapshot.
+      const replica = laggingReplica(prisma, [
+        { model: "batchTaskRun", mode: "frozen", rows: [staleBatch] },
+      ]);
+      const store = new PostgresRunStore({
+        prisma,
+        readOnlyPrisma: replica.client,
+        schemaVariant: "legacy",
+      });
+
+      // RED - the current service call (no client -> replica): stale null -> SUSPEND an already-resumed run.
+      const buggy = await precheckWaitForBatch(store, friendlyId, environment.id);
+      expect(replica.wasHit("batchTaskRun")).toBe(true); // proves it read the (stale) replica
+      expect(buggy).toBe("SUSPEND"); // the stall bug
+
+      // GREEN - the fix (thread the primary): sees resumedAt -> keep the run alive.
+      const fixed = await precheckWaitForBatch(store, friendlyId, environment.id, prisma);
+      expect(fixed).toBe("KEEP_ALIVE");
+    }
+  );
+});