chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "@internal/run-store",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"main": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"@triggerdotdev/source": "./src/index.ts",
|
||||
"import": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"default": "./dist/src/index.js"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@internal/run-ops-database": "workspace:*",
|
||||
"@trigger.dev/core": "workspace:*",
|
||||
"@trigger.dev/database": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internal/testcontainers": "workspace:*",
|
||||
"rimraf": "6.0.1"
|
||||
},
|
||||
"scripts": {
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.build.json",
|
||||
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
|
||||
"build": "pnpm run clean && tsc -p tsconfig.build.json",
|
||||
"dev": "tsc --watch -p tsconfig.build.json"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
// RED→GREEN repro for the run-ops split BASELINE BLOCKER:
|
||||
// RoutingRunStore cross-DB PROBE reads forward the caller's control-plane `client` into the #new
|
||||
// sub-store probe, so #new queries the CONTROL-PLANE DB instead of its own (5434) and never finds a
|
||||
// run-ops id-resident batch/attempt → returns null. Live effect: batchSystem.#tryCompleteBatch calls
|
||||
// `runStore.findBatchTaskRunById(batchId, undefined, this.$.prisma)` → null → "batch doesn't exist"
|
||||
// → the batch waitpoint is never completed → every `batchTriggerAndWait` parent hangs forever.
|
||||
//
|
||||
// `heteroRunOpsPostgresTest` gives a REAL split topology: prisma17 = real RunOpsPrismaClient over the
|
||||
// dedicated subset schema (#new / 5434), prisma14 = full legacy schema on a SEPARATE physical PG
|
||||
// container (#legacy / control-plane). NEVER mocked. The repro seeds a run-ops batch (and a run-ops id
|
||||
// attempt) on #new and probes via the router passing the LEGACY client as the read client — exactly
|
||||
// as the live caller does. RED before the fix (router forwards the client → #new reads control-plane
|
||||
// → null); GREEN after (router drops the client → #new reads its own DB → finds the row).
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id LENGTH (runOpsResidency.ts): 25 chars → cuid → LEGACY,
|
||||
// a v1 body (26 chars, version "1" at index 25) → NEW.
|
||||
const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema)
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / prisma17, dedicated subset schema)
|
||||
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function makeLegacyStore(prisma14: PrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
}
|
||||
|
||||
// Real production split topology: #new = dedicated subset on prisma17, #legacy = full schema on
|
||||
// prisma14 — two physically distinct DBs.
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = makeLegacyStore(prisma14);
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
legacyStore,
|
||||
newStore,
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — cross-DB probe reads must NOT forward the caller's control-plane client", () => {
|
||||
// findBatchTaskRunById — the live batchTriggerAndWait hang: #tryCompleteBatch probes with the
|
||||
// control-plane client, which the router forwarded into #new → #new read the wrong DB → null.
|
||||
heteroRunOpsPostgresTest(
|
||||
"findBatchTaskRunById FINDS a run-ops batch on #new even when probed with the LEGACY (control-plane) client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "batchprobe_new");
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id → #new
|
||||
|
||||
// Seed the batch directly on #new (5434), exactly where a runEngine-routed run-ops batch lives.
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_probe_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 3,
|
||||
successfulRunCount: 3,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
// Probe EXACTLY as batchSystem.#tryCompleteBatch does: pass the control-plane client.
|
||||
// RED before fix: null (probed control-plane). GREEN after: resolved from #new's own DB.
|
||||
const found = await router.findBatchTaskRunById(batchId, undefined, prisma14 as never);
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(batchId);
|
||||
expect(found!.successfulRunCount).toBe(3);
|
||||
}
|
||||
);
|
||||
|
||||
// Control: a cuid batch on #legacy is still found through the router when probed with the same
|
||||
// (legacy) client — proving the fix does not regress the legacy cohort.
|
||||
heteroRunOpsPostgresTest(
|
||||
"findBatchTaskRunById control: a cuid batch on #legacy is still found",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "batchprobe_leg");
|
||||
const batchId = `batch_${CUID_25}`; // cuid → #legacy
|
||||
|
||||
await prisma14.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_probe_leg",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 1,
|
||||
successfulRunCount: 1,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
const found = await router.findBatchTaskRunById(batchId, undefined, prisma14 as never);
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(batchId);
|
||||
}
|
||||
);
|
||||
|
||||
// findBatchTaskRunByFriendlyId — same anti-pattern (env-scoped friendlyId probe).
|
||||
heteroRunOpsPostgresTest(
|
||||
"findBatchTaskRunByFriendlyId FINDS a run-ops batch on #new despite the LEGACY client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "batchfid_new");
|
||||
const batchId = `batch_${NEW_ID_26}`;
|
||||
const friendlyId = "batch_fid_new";
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
const found = await router.findBatchTaskRunByFriendlyId(
|
||||
friendlyId,
|
||||
env.environment.id,
|
||||
undefined,
|
||||
prisma14 as never
|
||||
);
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(batchId);
|
||||
}
|
||||
);
|
||||
|
||||
// findBatchTaskRunByIdempotencyKey — same anti-pattern (env + idempotency-key probe).
|
||||
heteroRunOpsPostgresTest(
|
||||
"findBatchTaskRunByIdempotencyKey FINDS a run-ops batch on #new despite the LEGACY client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "batchidem_new");
|
||||
const batchId = `batch_${NEW_ID_26}`;
|
||||
const idempotencyKey = "idem_batch_new";
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_idem_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
const found = await router.findBatchTaskRunByIdempotencyKey(
|
||||
env.environment.id,
|
||||
idempotencyKey,
|
||||
undefined,
|
||||
prisma14 as never
|
||||
);
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(batchId);
|
||||
}
|
||||
);
|
||||
|
||||
// findTaskRunAttempt — same anti-pattern. A classifiable taskRunId routes to the owning store
|
||||
// (#new for a run-ops run) but the control-plane client was still forwarded into it.
|
||||
heteroRunOpsPostgresTest(
|
||||
"findTaskRunAttempt FINDS a run-ops id attempt on #new even when probed with the LEGACY client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "attempt_new");
|
||||
const runId = `run_${NEW_ID_26}`; // run-ops run → #new
|
||||
const attemptId = `attempt_${NEW_ID_26}`;
|
||||
|
||||
// The attempt's owning run lives on #new (the FK is co-resident on the dedicated schema).
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id: runId,
|
||||
engine: "V2",
|
||||
status: "EXECUTING",
|
||||
friendlyId: "run_attempt_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${runId}`,
|
||||
spanId: `span_${runId}`,
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
await prisma17.taskRunAttempt.create({
|
||||
data: {
|
||||
id: attemptId,
|
||||
number: 1,
|
||||
friendlyId: "attempt_fid_new",
|
||||
taskRunId: runId,
|
||||
backgroundWorkerId: `bw_${NEW_ID_26}`,
|
||||
backgroundWorkerTaskId: `bwt_${NEW_ID_26}`,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
queueId: `queue_${NEW_ID_26}`,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
// Probe with the LEGACY client, mirroring callers that pass the control-plane handle.
|
||||
const found = await router.findTaskRunAttempt(
|
||||
{ where: { taskRunId: runId } },
|
||||
prisma14 as never
|
||||
);
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(attemptId);
|
||||
}
|
||||
);
|
||||
|
||||
// Split-OFF guard: with a single store configured, the probe finds the batch with or without a
|
||||
// passed client (the one configured store reads its own DB either way) — no behavior change.
|
||||
heteroRunOpsPostgresTest(
|
||||
"split-OFF: a single-store router finds the batch with or without a passed client",
|
||||
async ({ prisma17 }) => {
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
// Single-DB config: both slots point at the same dedicated store (split effectively OFF).
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: newStore });
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "splitoff_new");
|
||||
const batchId = `batch_${NEW_ID_26}`;
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_splitoff",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
status: "PENDING",
|
||||
},
|
||||
});
|
||||
|
||||
const withoutClient = await router.findBatchTaskRunById(batchId);
|
||||
const withClient = await router.findBatchTaskRunById(batchId, undefined, prisma17 as never);
|
||||
expect(withoutClient?.id).toBe(batchId);
|
||||
expect(withClient?.id).toBe(batchId);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
// ProjectAlert.taskRunId/taskRunAttemptId FKs point INTO the run subgraph. A run-ops run lives ONLY
|
||||
// on the dedicated run-ops DB (prisma17), so `projectAlert.create({ taskRunId: <run-ops id> })` on
|
||||
// control-plane (prisma14) violates the FK and the alert is silently dropped. After the FK drop +
|
||||
// @relation removal the create succeeds; the read path resolves the run via runStore.findRun.
|
||||
// Asserts the create succeeds: it fails with an FK violation before the fix and succeeds after.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
|
||||
// v1 internal id (26 chars, version "1" at index 25) → NEW (lives only on the dedicated run-ops DB).
|
||||
const NEW_ID_26 = "k".repeat(24) + "01";
|
||||
|
||||
async function seedControlPlaneAlertPrereqs(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "PRODUCTION",
|
||||
slug: "prod",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_prod_${suffix}`,
|
||||
pkApiKey: `pk_prod_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
const channel = await prisma.projectAlertChannel.create({
|
||||
data: {
|
||||
friendlyId: `alert_channel_${suffix}`,
|
||||
type: "EMAIL",
|
||||
name: "Email",
|
||||
properties: { type: "EMAIL", email: "alerts@example.com" },
|
||||
alertTypes: ["TASK_RUN"],
|
||||
projectId: project.id,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment, channel };
|
||||
}
|
||||
|
||||
describe("ProjectAlert control-plane → run-subgraph FK reconciliation", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"creating a TASK_RUN alert with a run-ops id taskRunId (run only on the run-ops DB) succeeds on control-plane",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const suffix = "alert-runops";
|
||||
const { project, environment, channel } = await seedControlPlaneAlertPrereqs(
|
||||
prisma14,
|
||||
suffix
|
||||
);
|
||||
|
||||
// The run exists ONLY on the dedicated run-ops DB (prisma17), never on control-plane.
|
||||
await (prisma17 as RunOpsPrismaClient).taskRun.create({
|
||||
data: {
|
||||
id: NEW_ID_26,
|
||||
friendlyId: `run_${suffix}`,
|
||||
engine: "V2",
|
||||
status: "COMPLETED_WITH_ERRORS",
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceId: `trace_${suffix}`,
|
||||
spanId: `span_${suffix}`,
|
||||
queue: "task/my-task",
|
||||
runtimeEnvironmentId: environment.id,
|
||||
projectId: project.id,
|
||||
organizationId: project.organizationId,
|
||||
environmentType: "PRODUCTION",
|
||||
},
|
||||
});
|
||||
|
||||
// Control-plane has no TaskRun row for NEW_ID_26. With the FK present this throws P2003;
|
||||
// after the FK is dropped + the @relation removed it succeeds.
|
||||
const alert = await prisma14.projectAlert.create({
|
||||
data: {
|
||||
friendlyId: `alert_${suffix}`,
|
||||
channelId: channel.id,
|
||||
projectId: project.id,
|
||||
environmentId: environment.id,
|
||||
status: "PENDING",
|
||||
type: "TASK_RUN",
|
||||
taskRunId: NEW_ID_26,
|
||||
},
|
||||
});
|
||||
|
||||
expect(alert.taskRunId).toBe(NEW_ID_26);
|
||||
|
||||
// The scalar round-trips and can be re-read off the control-plane row (the read path resolves
|
||||
// the actual run via runStore.findRun against the run-ops DB).
|
||||
const reread = await prisma14.projectAlert.findUniqueOrThrow({ where: { id: alert.id } });
|
||||
expect(reread.taskRunId).toBe(NEW_ID_26);
|
||||
},
|
||||
120_000
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"creating a TASK_RUN_ATTEMPT alert with a run-ops id taskRunAttemptId (attempt only on the run-ops DB) succeeds on control-plane",
|
||||
async ({ prisma14 }) => {
|
||||
const suffix = "alert-run-ops id-attempt";
|
||||
const { project, environment, channel } = await seedControlPlaneAlertPrereqs(
|
||||
prisma14,
|
||||
suffix
|
||||
);
|
||||
|
||||
// A run-ops id attempt id with no matching control-plane TaskRunAttempt row. With the FK present
|
||||
// this throws P2003; after the FK is dropped it succeeds.
|
||||
const attemptId = "a".repeat(24) + "01";
|
||||
const alert = await prisma14.projectAlert.create({
|
||||
data: {
|
||||
friendlyId: `alert_${suffix}`,
|
||||
channelId: channel.id,
|
||||
projectId: project.id,
|
||||
environmentId: environment.id,
|
||||
status: "PENDING",
|
||||
type: "TASK_RUN_ATTEMPT",
|
||||
taskRunAttemptId: attemptId,
|
||||
},
|
||||
});
|
||||
|
||||
expect(alert.taskRunAttemptId).toBe(attemptId);
|
||||
},
|
||||
120_000
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
// Cross-generation Prisma error normalization LOCK.
|
||||
//
|
||||
// The store can be backed by the run-ops `@internal/run-ops-database` client, a SEPARATELY
|
||||
// generated Prisma client with its OWN `PrismaClientKnownRequestError` class object (distinct
|
||||
// module identity from `@trigger.dev/database`'s, even at the same version). A P2002 raised by
|
||||
// the run-ops client is therefore NOT `instanceof` the control-plane
|
||||
// `Prisma.PrismaClientKnownRequestError` — so the webapp's uniform P2002→422 conversion
|
||||
// (`error instanceof Prisma.PrismaClientKnownRequestError`) is skipped and a raw 500 escapes.
|
||||
//
|
||||
// PostgresRunStore normalizes at its write boundary: a routed NEW-client P2002 surfaces such
|
||||
// that a control-plane `instanceof` catch (the 422 path) sees it. This test drives a REAL
|
||||
// duplicate-key on the REAL run-ops-generation client (prisma17) through the store and asserts
|
||||
// the surfaced error is recognized by the control-plane class — the exact predicate every
|
||||
// routed-write caller uses. Fails before the normalization (raw foreign error ⇒ instanceof false).
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import { Prisma } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type { CreateBatchTaskRunData } from "./types.js";
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function batchData(overrides: Partial<CreateBatchTaskRunData> = {}): CreateBatchTaskRunData {
|
||||
return {
|
||||
id: `batch_${"x".repeat(24)}`,
|
||||
friendlyId: "batch_dup_friendly",
|
||||
runtimeEnvironmentId: "env_cgerr",
|
||||
status: "PENDING",
|
||||
runCount: 1,
|
||||
expectedCount: 1,
|
||||
batchVersion: "runengine:v2",
|
||||
sealed: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("PostgresRunStore — cross-generation Prisma error normalization", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"a routed NEW-client P2002 surfaces as a control-plane instanceof Prisma.PrismaClientKnownRequestError",
|
||||
async ({ prisma17 }) => {
|
||||
const store = makeDedicatedStore(prisma17);
|
||||
|
||||
// First create succeeds; second collides on the unique friendlyId → NEW-generation P2002.
|
||||
await store.createBatchTaskRun(batchData({ id: `batch_${"a".repeat(24)}` }));
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await store.createBatchTaskRun(batchData({ id: `batch_${"b".repeat(24)}` }));
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
|
||||
// The control-plane `instanceof` catch (the P2002→422 path the webapp uses) must see it.
|
||||
expect(caught instanceof Prisma.PrismaClientKnownRequestError).toBe(true);
|
||||
const known = caught as Prisma.PrismaClientKnownRequestError;
|
||||
expect(known.code).toBe("P2002");
|
||||
// code/message/meta are preserved through the normalization.
|
||||
expect(typeof known.message).toBe("string");
|
||||
expect(known.message.length).toBeGreaterThan(0);
|
||||
expect(known.clientVersion).toBeTruthy();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"a NEW-client P2002 inside runInTransaction is also normalized to the control-plane class",
|
||||
async ({ prisma17 }) => {
|
||||
const store = makeDedicatedStore(prisma17);
|
||||
|
||||
await store.createBatchTaskRun(batchData({ id: `batch_${"c".repeat(24)}` }));
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await store.runInTransaction(undefined, async (txStore) => {
|
||||
await txStore.createBatchTaskRun(batchData({ id: `batch_${"d".repeat(24)}` }));
|
||||
});
|
||||
} catch (error) {
|
||||
caught = error;
|
||||
}
|
||||
|
||||
expect(caught instanceof Prisma.PrismaClientKnownRequestError).toBe(true);
|
||||
expect((caught as Prisma.PrismaClientKnownRequestError).code).toBe("P2002");
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"a successful NEW-client write is untouched by the normalization wrapper",
|
||||
async ({ prisma17 }) => {
|
||||
const store = makeDedicatedStore(prisma17);
|
||||
|
||||
const created = await store.createBatchTaskRun(batchData({ id: `batch_${"e".repeat(24)}` }));
|
||||
|
||||
expect(created.id).toBe(`batch_${"e".repeat(24)}`);
|
||||
expect(created.friendlyId).toBe("batch_dup_friendly");
|
||||
}
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,440 @@
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type {
|
||||
CreateRunInput,
|
||||
RunAssociatedWaitpointInput,
|
||||
RunStoreSchemaVariant,
|
||||
} from "./types.js";
|
||||
|
||||
// The store's structural client accepts either backing Prisma client; the two generated
|
||||
// clients are nominally distinct so we widen at the boundary, exactly as buildRunStore does.
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// On the dedicated subset schema there are no Organization/Project/RuntimeEnvironment models and
|
||||
// the run-ops rows carry FK-free scalar ids, so we mint synthetic ids; on legacy we seed the real
|
||||
// owning rows the FKs require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildAssociatedWaitpoint(params: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
}): RunAssociatedWaitpointInput {
|
||||
return {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "RUN",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
associatedWaitpoint?: RunAssociatedWaitpointInput;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
associatedWaitpoint: params.associatedWaitpoint,
|
||||
};
|
||||
}
|
||||
|
||||
async function seedPendingWaitpoint(
|
||||
prisma: AnyClient,
|
||||
params: { id: string; friendlyId: string; projectId: string; environmentId: string }
|
||||
) {
|
||||
return (prisma as PrismaClient).waitpoint.create({
|
||||
data: {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeStore(prisma: AnyClient, schemaVariant: RunStoreSchemaVariant) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma as never,
|
||||
readOnlyPrisma: prisma as never,
|
||||
schemaVariant,
|
||||
});
|
||||
}
|
||||
|
||||
// --- group-A on TaskRun: associatedWaitpoint -------------------------------------------------
|
||||
|
||||
// Runs the run-engine-shaped completeAttemptSuccess call: a caller select that includes the
|
||||
// group-A `associatedWaitpoint` relation key, exactly as runAttemptSystem does.
|
||||
async function runAssociatedWaitpointScenario(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
const runId = `run_${suffix}`;
|
||||
const waitpointId = `wp_assoc_${suffix}`;
|
||||
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_${suffix}`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: waitpointId,
|
||||
friendlyId: `waitpoint_assoc_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
// The actual run-engine call shape (runAttemptSystem.completeRunAttemptSuccess).
|
||||
const completed = await store.completeAttemptSuccess(
|
||||
runId,
|
||||
{
|
||||
completedAt: new Date(),
|
||||
output: '{"done":true}',
|
||||
outputType: "application/json",
|
||||
usageDurationMs: 100,
|
||||
costInCents: 1,
|
||||
snapshot: {
|
||||
executionStatus: "FINISHED",
|
||||
description: "Attempt succeeded",
|
||||
runStatus: "COMPLETED_SUCCESSFULLY",
|
||||
attemptNumber: 1,
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
},
|
||||
},
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
status: true,
|
||||
associatedWaitpoint: {
|
||||
select: { id: true },
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
// findRun with the same group-A select (the read path).
|
||||
const found = await store.findRun(
|
||||
{ id: runId },
|
||||
{ select: { id: true, associatedWaitpoint: { select: { id: true } } } }
|
||||
);
|
||||
|
||||
return { runId, waitpointId, completed, found };
|
||||
}
|
||||
|
||||
// --- group-A on TaskRunExecutionSnapshot: completedWaitpoints --------------------------------
|
||||
|
||||
async function runCompletedWaitpointsScenario(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
const runId = `run_${suffix}`;
|
||||
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_${suffix}`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
const w1 = `wp_${suffix}_1`;
|
||||
const w2 = `wp_${suffix}_2`;
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w1,
|
||||
friendlyId: `waitpoint_${suffix}_1`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w2,
|
||||
friendlyId: `waitpoint_${suffix}_2`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
const snapshot = await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING_WITH_WAITPOINTS", description: "with waitpoints" },
|
||||
completedWaitpoints: [
|
||||
{ id: w2, index: 1 },
|
||||
{ id: w1, index: 0 },
|
||||
],
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
});
|
||||
|
||||
// The run-engine call shape for fetching a snapshot's completed waitpoints.
|
||||
const fetched = await store.findExecutionSnapshot({
|
||||
where: { id: snapshot.id },
|
||||
include: { completedWaitpoints: true },
|
||||
});
|
||||
|
||||
return { snapshotId: snapshot.id, w1, w2, fetched };
|
||||
}
|
||||
|
||||
// --- connection back-ref on Waitpoint: blockingTaskRuns --------------------------------------
|
||||
|
||||
async function runBlockingTaskRunsScenario(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
const runId = `run_${suffix}`;
|
||||
const waitpointId = `wp_block_${suffix}`;
|
||||
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_${suffix}`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: waitpointId,
|
||||
friendlyId: `waitpoint_block_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
// Block the run on the waitpoint (writes the TaskRunWaitpoint block edge + connection).
|
||||
await store.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [waitpointId],
|
||||
projectId: env.project.id,
|
||||
});
|
||||
|
||||
// The run-engine call shape (engine.getWaitpoint).
|
||||
const waitpoint = await store.findWaitpoint({
|
||||
where: { id: waitpointId },
|
||||
include: {
|
||||
blockingTaskRuns: {
|
||||
select: {
|
||||
taskRun: {
|
||||
select: { id: true, friendlyId: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return { runId, waitpointId, waitpoint, friendlyId: `run_friendly_${suffix}` };
|
||||
}
|
||||
|
||||
describe("PostgresRunStore dedicated caller-select adapter (P2-store-bodies-2)", () => {
|
||||
// associatedWaitpoint (TaskRun group-A) — RED on dedicated before this task, GREEN after.
|
||||
heteroRunOpsPostgresTest(
|
||||
"completeAttemptSuccess + findRun honor associatedWaitpoint on the DEDICATED client",
|
||||
async ({ prisma17 }) => {
|
||||
const r = await runAssociatedWaitpointScenario(prisma17, "dedicated", "ded_aw");
|
||||
|
||||
expect(r.completed.id).toBe(r.runId);
|
||||
expect(r.completed.status).toBe("COMPLETED_SUCCESSFULLY");
|
||||
// honor the caller sub-select { id: true } only
|
||||
expect(r.completed.associatedWaitpoint).not.toBeNull();
|
||||
expect(r.completed.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
expect(Object.keys(r.completed.associatedWaitpoint!)).toEqual(["id"]);
|
||||
|
||||
expect(r.found).not.toBeNull();
|
||||
expect(r.found!.associatedWaitpoint).not.toBeNull();
|
||||
expect(r.found!.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"completeAttemptSuccess + findRun honor associatedWaitpoint on the LEGACY client",
|
||||
async ({ prisma14 }) => {
|
||||
const r = await runAssociatedWaitpointScenario(prisma14, "legacy", "leg_aw");
|
||||
|
||||
expect(r.completed.id).toBe(r.runId);
|
||||
expect(r.completed.associatedWaitpoint).not.toBeNull();
|
||||
expect(r.completed.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
expect(r.found!.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
}
|
||||
);
|
||||
|
||||
// completedWaitpoints (snapshot group-A) — RED on dedicated before, GREEN after.
|
||||
heteroRunOpsPostgresTest(
|
||||
"findExecutionSnapshot honors completedWaitpoints on the DEDICATED client",
|
||||
async ({ prisma17 }) => {
|
||||
const r = await runCompletedWaitpointsScenario(prisma17, "dedicated", "ded_cw");
|
||||
|
||||
expect(r.fetched).not.toBeNull();
|
||||
expect(r.fetched!.id).toBe(r.snapshotId);
|
||||
expect(r.fetched!.completedWaitpoints.map((w) => w.id).sort()).toEqual([r.w1, r.w2].sort());
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"findExecutionSnapshot honors completedWaitpoints on the LEGACY client",
|
||||
async ({ prisma14 }) => {
|
||||
const r = await runCompletedWaitpointsScenario(prisma14, "legacy", "leg_cw");
|
||||
|
||||
expect(r.fetched).not.toBeNull();
|
||||
expect(r.fetched!.completedWaitpoints.map((w) => w.id).sort()).toEqual([r.w1, r.w2].sort());
|
||||
}
|
||||
);
|
||||
|
||||
// blockingTaskRuns connection back-ref (Waitpoint group-A) — RED on dedicated before, GREEN after.
|
||||
heteroRunOpsPostgresTest(
|
||||
"findWaitpoint honors blockingTaskRuns back-ref on the DEDICATED client",
|
||||
async ({ prisma17 }) => {
|
||||
const r = await runBlockingTaskRunsScenario(prisma17, "dedicated", "ded_bk");
|
||||
|
||||
expect(r.waitpoint).not.toBeNull();
|
||||
expect(r.waitpoint!.id).toBe(r.waitpointId);
|
||||
const blocking = r.waitpoint!.blockingTaskRuns;
|
||||
expect(blocking.length).toBe(1);
|
||||
expect(blocking[0].taskRun.id).toBe(r.runId);
|
||||
expect(blocking[0].taskRun.friendlyId).toBe(r.friendlyId);
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"findWaitpoint honors blockingTaskRuns back-ref on the LEGACY client",
|
||||
async ({ prisma14 }) => {
|
||||
const r = await runBlockingTaskRunsScenario(prisma14, "legacy", "leg_bk");
|
||||
|
||||
expect(r.waitpoint).not.toBeNull();
|
||||
const blocking = r.waitpoint!.blockingTaskRuns;
|
||||
expect(blocking.length).toBe(1);
|
||||
expect(blocking[0].taskRun.id).toBe(r.runId);
|
||||
expect(blocking[0].taskRun.friendlyId).toBe(r.friendlyId);
|
||||
}
|
||||
);
|
||||
|
||||
// The dedicated finders forward include/select to a subset client typed as the full schema, so a
|
||||
// control-plane-only relation would throw an opaque Prisma 500 for NEW data (invisible to tsc). The
|
||||
// boundary guard rejects the known keys with a clear message; the legacy (full-schema) store is a no-op.
|
||||
heteroRunOpsPostgresTest(
|
||||
"dedicated batch/attempt finders reject control-plane-only includes with a clear error",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const dedicated = makeStore(prisma17, "dedicated");
|
||||
const legacy = makeStore(prisma14, "legacy");
|
||||
|
||||
await expect(
|
||||
dedicated.findBatchTaskRunById("batch_x", { include: { runsBlocked: true } as never })
|
||||
).rejects.toThrow(/not available on the dedicated run-ops subset/);
|
||||
await expect(
|
||||
dedicated.findTaskRunAttempt({
|
||||
where: { id: "att_x" },
|
||||
include: { backgroundWorker: true },
|
||||
} as never)
|
||||
).rejects.toThrow(/not available on the dedicated run-ops subset/);
|
||||
|
||||
// A subset-present include passes the guard and resolves normally (null for a missing batch).
|
||||
expect(
|
||||
await dedicated.findBatchTaskRunById("batch_missing", { include: { items: true } as never })
|
||||
).toBeNull();
|
||||
// Legacy store: guard is a no-op; the full schema has the relation, so no guard throw.
|
||||
expect(
|
||||
await legacy.findBatchTaskRunById("batch_missing", {
|
||||
include: { runsBlocked: true } as never,
|
||||
})
|
||||
).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,378 @@
|
||||
import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type { CreateRunInput, RunAssociatedWaitpointInput } from "./types.js";
|
||||
|
||||
// The store's structural client accepts either backing Prisma client; the two generated
|
||||
// clients are nominally distinct so we widen at the boundary, exactly as buildRunStore does.
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// The dedicated subset schema has no Organization/Project/RuntimeEnvironment models and the
|
||||
// run-ops TaskRun/Waitpoint carry FK-free scalar ids, so on that variant we mint synthetic
|
||||
// ids; on legacy we seed the real owning rows the FKs require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: "legacy" | "dedicated",
|
||||
slugSuffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${slugSuffix}` },
|
||||
project: { id: `proj_${slugSuffix}` },
|
||||
environment: { id: `env_${slugSuffix}` },
|
||||
};
|
||||
}
|
||||
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${slugSuffix}`,
|
||||
slug: `project-${slugSuffix}`,
|
||||
externalRef: `proj_${slugSuffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${slugSuffix}`,
|
||||
pkApiKey: `pk_dev_${slugSuffix}`,
|
||||
shortcode: `short_${slugSuffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildAssociatedWaitpoint(params: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
}): RunAssociatedWaitpointInput {
|
||||
return {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "RUN",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
taskIdentifier: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
associatedWaitpoint?: RunAssociatedWaitpointInput;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: "trace_1",
|
||||
spanId: "span_1",
|
||||
runTags: ["alpha", "beta"],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
associatedWaitpoint: params.associatedWaitpoint,
|
||||
};
|
||||
}
|
||||
|
||||
async function seedPendingWaitpoint(
|
||||
prisma: AnyClient,
|
||||
params: { id: string; friendlyId: string; projectId: string; environmentId: string }
|
||||
) {
|
||||
return (prisma as PrismaClient).waitpoint.create({
|
||||
data: {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Strip the prisma-managed / connection-volatile columns so two waitpoint rows born on
|
||||
// different physical DBs (and via different code paths) compare field-for-field.
|
||||
function normalizeWaitpoint(row: Record<string, unknown> | null) {
|
||||
if (!row) return row;
|
||||
const r = { ...row };
|
||||
delete r.createdAt;
|
||||
delete r.updatedAt;
|
||||
return r;
|
||||
}
|
||||
|
||||
// Runs the same createRun + snapshot scenario against any (client, schemaVariant) pair and
|
||||
// returns the observable shapes the interface contract promises, so the legacy and dedicated
|
||||
// runs can be asserted equivalent.
|
||||
async function runScenario(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: "legacy" | "dedicated",
|
||||
suffix: string
|
||||
) {
|
||||
const store = new PostgresRunStore({
|
||||
prisma: prisma as never,
|
||||
readOnlyPrisma: prisma as never,
|
||||
schemaVariant,
|
||||
});
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
|
||||
const runId = `run_dual_${suffix}`;
|
||||
const created = await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: `wp_assoc_${suffix}`,
|
||||
friendlyId: `waitpoint_assoc_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
// The run read that pulls the associatedWaitpoint back (rewriteDebouncedRun shape).
|
||||
const rewritten = await store.rewriteDebouncedRun(runId, {
|
||||
payload: '{"hello":"again"}',
|
||||
payloadType: "application/json",
|
||||
});
|
||||
|
||||
// Two pending waitpoints to complete via a snapshot.
|
||||
const w1 = `wp_${suffix}_1`;
|
||||
const w2 = `wp_${suffix}_2`;
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w1,
|
||||
friendlyId: `waitpoint_${suffix}_1`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w2,
|
||||
friendlyId: `waitpoint_${suffix}_2`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
const ids = {
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
};
|
||||
|
||||
const snapshot = await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING_WITH_WAITPOINTS", description: "with waitpoints" },
|
||||
completedWaitpoints: [
|
||||
{ id: w2, index: 1 },
|
||||
{ id: w1, index: 0 },
|
||||
],
|
||||
...ids,
|
||||
});
|
||||
|
||||
const latest = await store.findLatestExecutionSnapshot(runId);
|
||||
const joinIds = await store.findSnapshotCompletedWaitpointIds(snapshot.id);
|
||||
|
||||
return {
|
||||
runId,
|
||||
created,
|
||||
rewritten,
|
||||
snapshot,
|
||||
latest,
|
||||
joinIds,
|
||||
waitpointId: `wp_assoc_${suffix}`,
|
||||
w1,
|
||||
w2,
|
||||
};
|
||||
}
|
||||
|
||||
function assertScenario(r: Awaited<ReturnType<typeof runScenario>>) {
|
||||
// createRun returns the run with its associatedWaitpoint hydrated.
|
||||
expect(r.created.id).toBe(r.runId);
|
||||
expect(r.created.associatedWaitpoint).not.toBeNull();
|
||||
expect(r.created.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
expect(r.created.associatedWaitpoint!.type).toBe("RUN");
|
||||
expect(r.created.associatedWaitpoint!.completedByTaskRunId).toBe(r.runId);
|
||||
|
||||
// The run read hydrates the same associatedWaitpoint.
|
||||
expect(r.rewritten.associatedWaitpoint).not.toBeNull();
|
||||
expect(r.rewritten.associatedWaitpoint!.id).toBe(r.waitpointId);
|
||||
|
||||
// The snapshot create derives completedWaitpointOrder by index (w1 index 0, w2 index 1).
|
||||
expect(r.snapshot.completedWaitpointOrder).toEqual([r.w1, r.w2]);
|
||||
|
||||
// The join read returns both completed waitpoints (set-equal).
|
||||
expect([...r.joinIds].sort()).toEqual([r.w1, r.w2].sort());
|
||||
|
||||
// findLatest hydrates completedWaitpoints (set-equal) and the (null) checkpoint.
|
||||
expect(r.latest).not.toBeNull();
|
||||
expect(r.latest!.id).toBe(r.snapshot.id);
|
||||
expect(r.latest!.checkpoint).toBeNull();
|
||||
expect(r.latest!.completedWaitpoints.map((w) => w.id).sort()).toEqual([r.w1, r.w2].sort());
|
||||
}
|
||||
|
||||
describe("PostgresRunStore dual-schema (P2-store-bodies)", () => {
|
||||
// Legacy variant over the full @trigger.dev/database schema — existing behavior must hold.
|
||||
heteroPostgresTest(
|
||||
"createRun + snapshot relation ops work on the LEGACY client (schemaVariant=legacy)",
|
||||
async ({ prisma14 }) => {
|
||||
const r = await runScenario(prisma14, "legacy", "leg");
|
||||
assertScenario(r);
|
||||
}
|
||||
);
|
||||
|
||||
// Dedicated variant over the @internal/run-ops-database SUBSET schema — RED before this
|
||||
// task (Prisma validation error on associatedWaitpoint/completedWaitpoints), GREEN after.
|
||||
heteroRunOpsPostgresTest(
|
||||
"createRun + snapshot relation ops work on the DEDICATED RunOpsPrismaClient (schemaVariant=dedicated)",
|
||||
async ({ prisma17 }) => {
|
||||
const r = await runScenario(prisma17, "dedicated", "ded");
|
||||
assertScenario(r);
|
||||
}
|
||||
);
|
||||
|
||||
// Cross-variant equivalence: the observable return contract is the same regardless of which
|
||||
// backing schema produced it.
|
||||
heteroRunOpsPostgresTest(
|
||||
"legacy and dedicated produce equivalent return shapes",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const legacy = await runScenario(prisma14, "legacy", "xleg");
|
||||
const dedicated = await runScenario(prisma17, "dedicated", "xded");
|
||||
|
||||
// Associated waitpoint: normalize per-DB volatile columns, the rest must match.
|
||||
const legW = normalizeWaitpoint(
|
||||
legacy.created.associatedWaitpoint as unknown as Record<string, unknown>
|
||||
);
|
||||
const dedW = normalizeWaitpoint(
|
||||
dedicated.created.associatedWaitpoint as unknown as Record<string, unknown>
|
||||
);
|
||||
// Both carry the same friendlyId/type/status/completedByTaskRunId-shaped contract;
|
||||
// ids differ by suffix so compare the structural keys that must agree.
|
||||
expect(legW!.type).toEqual(dedW!.type);
|
||||
expect(legW!.status).toEqual(dedW!.status);
|
||||
expect((legW as Record<string, unknown>).outputType).toEqual(
|
||||
(dedW as Record<string, unknown>).outputType
|
||||
);
|
||||
|
||||
// completedWaitpointOrder derivation is variant-independent.
|
||||
expect(legacy.snapshot.completedWaitpointOrder.length).toEqual(
|
||||
dedicated.snapshot.completedWaitpointOrder.length
|
||||
);
|
||||
expect(legacy.latest!.completedWaitpoints.length).toEqual(
|
||||
dedicated.latest!.completedWaitpoints.length
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// expireRunsBatch dedicated-fixture: RED before fix (Prisma.join mis-binds on dedicated client
|
||||
// → 42601-class error), GREEN after (= ANY(ids::text[]) path).
|
||||
heteroRunOpsPostgresTest(
|
||||
"expireRunsBatch sets EXPIRED on the DEDICATED RunOpsPrismaClient (schemaVariant=dedicated)",
|
||||
async ({ prisma17 }) => {
|
||||
const store = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
|
||||
// Dedicated subset has no Organization/Project/RuntimeEnvironment tables — use synthetic ids.
|
||||
const orgId = "org_expbatch_ded";
|
||||
const projId = "proj_expbatch_ded";
|
||||
const envId = "env_expbatch_ded";
|
||||
|
||||
const runId1 = "run_expbatch_ded_1";
|
||||
const runId2 = "run_expbatch_ded_2";
|
||||
|
||||
for (const id of [runId1, runId2]) {
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: `friendly_${id}`,
|
||||
runtimeEnvironmentId: envId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: orgId,
|
||||
projectId: projId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${id}`,
|
||||
spanId: `span_${id}`,
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const now = new Date("2026-06-01T12:00:00.000Z");
|
||||
const error = {
|
||||
type: "STRING_ERROR" as const,
|
||||
raw: "Run expired because the TTL was reached",
|
||||
};
|
||||
|
||||
const count = await store.expireRunsBatch([runId1, runId2], { error, now });
|
||||
|
||||
expect(count).toBe(2);
|
||||
|
||||
for (const id of [runId1, runId2]) {
|
||||
const row = await prisma17.taskRun.findUniqueOrThrow({
|
||||
where: { id },
|
||||
select: { status: true, completedAt: true, expiredAt: true, updatedAt: true },
|
||||
});
|
||||
expect(row.status).toBe("EXPIRED");
|
||||
expect(row.completedAt).toEqual(now);
|
||||
expect(row.expiredAt).toEqual(now);
|
||||
expect(row.updatedAt).toEqual(now);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,715 @@
|
||||
// Cross-DB WRITE ATOMICITY against the REAL dedicated split topology.
|
||||
//
|
||||
// Under the run-ops split, several engine operations that were atomic-by-`prisma.$transaction` in
|
||||
// single-DB make TWO distinct RunStore writes (e.g. startAttempt + createExecutionSnapshot, or
|
||||
// promotePendingVersionRuns + createExecutionSnapshot). When the run is run-ops id (#new), `RoutingRunStore`
|
||||
// routes each write to the NEW store but DROPS the caller's control-plane `tx` — so the two writes
|
||||
// execute as independent auto-commit statements on the NEW DB, OUTSIDE any shared transaction. A crash
|
||||
// between them leaves partial state (a run EXECUTING with no matching snapshot; promoted-but-no-snapshot).
|
||||
//
|
||||
// `heteroRunOpsPostgresTest` gives the REAL production split: prisma17 = a real `RunOpsPrismaClient`
|
||||
// over the @internal/run-ops-database SUBSET schema (#new), prisma14 = the full control-plane schema on
|
||||
// a SEPARATE physical PG container (#legacy). No mocks.
|
||||
//
|
||||
// The first test EMPIRICALLY DEMONSTRATES the regression (two un-wrapped routed writes persist partial
|
||||
// state on a mid-pair failure). The remaining tests prove `RoutingRunStore.runInTransaction(runId, fn)`
|
||||
// wraps the co-resident multi-write unit in ONE `#new` transaction so a failure between the two writes
|
||||
// rolls BOTH back — no partial state.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { CreateRunInput, RunStore, RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by the version char: no marker → cuid → LEGACY, v1 body → run-ops id → NEW.
|
||||
const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / control-plane DB, full schema)
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / dedicated run-ops DB, subset schema)
|
||||
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function makeLegacyStore(prisma14: PrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
}
|
||||
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = makeLegacyStore(prisma14);
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
legacyStore,
|
||||
newStore,
|
||||
};
|
||||
}
|
||||
|
||||
// Seed a run-ops run on #new (its create nests the initial RUN_CREATED snapshot) and return its ids.
|
||||
async function seedRunOpsRun(
|
||||
router: RunStore,
|
||||
prisma17: RunOpsPrismaClient,
|
||||
suffix: string
|
||||
): Promise<{ runId: string; env: { project: { id: string }; environment: { id: string } } }> {
|
||||
const env = await seedEnvironment(prisma17, "dedicated", suffix);
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_${suffix}`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
return { runId, env };
|
||||
}
|
||||
|
||||
const ATTEMPT_SELECT = { id: true, status: true, attemptNumber: true } as const;
|
||||
|
||||
function snapshotInput(
|
||||
runId: string,
|
||||
env: { project: { id: string }; environment: { id: string } }
|
||||
) {
|
||||
return {
|
||||
run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING" as const, description: "Attempt created, starting" },
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
projectId: env.project.id,
|
||||
organizationId: env.project.id,
|
||||
};
|
||||
}
|
||||
|
||||
describe("cross-DB write atomicity (startAttempt + createExecutionSnapshot)", () => {
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// RED demonstration: the BROKEN behaviour. Two separate routed writes (as the engine made them
|
||||
// before the fix) on a run-ops run leave PARTIAL state on a mid-pair failure — the run is EXECUTING
|
||||
// but no EXECUTING snapshot exists. This is the regression vs single-DB.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
heteroRunOpsPostgresTest(
|
||||
"BROKEN baseline: two un-wrapped routed writes persist partial state on a mid-pair failure",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const { runId, env } = await seedRunOpsRun(router, prisma17, "broken_atomic");
|
||||
|
||||
// Simulate the OLD engine pattern: startAttempt then a failure BEFORE createExecutionSnapshot,
|
||||
// each as an independent routed (auto-commit) write — no shared transaction.
|
||||
await expect(
|
||||
(async () => {
|
||||
await router.startAttempt(
|
||||
runId,
|
||||
{ attemptNumber: 1, executedAt: new Date(), isWarmStart: false },
|
||||
{ select: ATTEMPT_SELECT }
|
||||
);
|
||||
throw new Error("boom between writes");
|
||||
// eslint-disable-next-line no-unreachable
|
||||
await router.createExecutionSnapshot(snapshotInput(runId, env));
|
||||
})()
|
||||
).rejects.toThrow("boom between writes");
|
||||
|
||||
// The first write was auto-committed: the run is EXECUTING but there is NO EXECUTING snapshot.
|
||||
const run = await prisma17.taskRun.findFirstOrThrow({ where: { id: runId } });
|
||||
expect(run.status).toBe("EXECUTING"); // partial state PERSISTED — the bug
|
||||
const execSnap = await prisma17.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING" },
|
||||
});
|
||||
expect(execSnap).toBeNull(); // no snapshot → run executing without a snapshot
|
||||
}
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// FIX: runInTransaction wraps the co-resident multi-write unit in ONE #new transaction. A failure
|
||||
// BETWEEN the two writes rolls the FIRST write back — no partial state.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
heteroRunOpsPostgresTest(
|
||||
"runInTransaction rolls back startAttempt when a failure is injected before the snapshot write (run-ops id → #new)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const { runId, env } = await seedRunOpsRun(router, prisma17, "rollback_new");
|
||||
|
||||
await expect(
|
||||
router.runInTransaction(runId, async (store, tx) => {
|
||||
await store.startAttempt(
|
||||
runId,
|
||||
{ attemptNumber: 1, executedAt: new Date(), isWarmStart: false },
|
||||
{ select: ATTEMPT_SELECT },
|
||||
tx
|
||||
);
|
||||
// Inject the failure AFTER the first write, BEFORE the snapshot write.
|
||||
throw new Error("boom between writes");
|
||||
// eslint-disable-next-line no-unreachable
|
||||
await store.createExecutionSnapshot(snapshotInput(runId, env), tx);
|
||||
})
|
||||
).rejects.toThrow("boom between writes");
|
||||
|
||||
// Both writes rolled back: run is still PENDING and no EXECUTING snapshot exists.
|
||||
const run = await prisma17.taskRun.findFirstOrThrow({ where: { id: runId } });
|
||||
expect(run.status).toBe("PENDING");
|
||||
expect(run.attemptNumber).toBeNull();
|
||||
const execSnap = await prisma17.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING" },
|
||||
});
|
||||
expect(execSnap).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"runInTransaction commits BOTH writes atomically on success (run-ops id → #new)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const { runId, env } = await seedRunOpsRun(router, prisma17, "commit_new");
|
||||
|
||||
const result = await router.runInTransaction(runId, async (store, tx) => {
|
||||
const run = await store.startAttempt(
|
||||
runId,
|
||||
{ attemptNumber: 1, executedAt: new Date(), isWarmStart: false },
|
||||
{ select: ATTEMPT_SELECT },
|
||||
tx
|
||||
);
|
||||
const snapshot = await store.createExecutionSnapshot(snapshotInput(runId, env), tx);
|
||||
return { run, snapshot };
|
||||
});
|
||||
|
||||
expect(result.run.status).toBe("EXECUTING");
|
||||
expect(result.snapshot.executionStatus).toBe("EXECUTING");
|
||||
|
||||
// Both persisted on #new.
|
||||
const run = await prisma17.taskRun.findFirstOrThrow({ where: { id: runId } });
|
||||
expect(run.status).toBe("EXECUTING");
|
||||
expect(run.attemptNumber).toBe(1);
|
||||
const execSnap = await prisma17.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING" },
|
||||
});
|
||||
expect(execSnap).not.toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// The same atomic guarantee for a cuid run on #legacy — the owning store is #legacy and the inner
|
||||
// writes share its transaction.
|
||||
heteroRunOpsPostgresTest(
|
||||
"runInTransaction rolls back BOTH writes on a cuid run (#legacy)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "rollback_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_rollback_leg`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
await expect(
|
||||
router.runInTransaction(runId, async (store, tx) => {
|
||||
await store.startAttempt(
|
||||
runId,
|
||||
{ attemptNumber: 1, executedAt: new Date(), isWarmStart: false },
|
||||
{ select: ATTEMPT_SELECT },
|
||||
tx
|
||||
);
|
||||
throw new Error("boom between writes");
|
||||
})
|
||||
).rejects.toThrow("boom between writes");
|
||||
|
||||
const run = await prisma14.taskRun.findFirstOrThrow({ where: { id: runId } });
|
||||
expect(run.status).toBe("PENDING");
|
||||
expect(run.attemptNumber).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// A run's blocking edges may straddle both DBs mid-drain, so clearBlockingWaitpoints routes the
|
||||
// taskRunId-keyed delete through the both-stores fan-out. The #new leg can't join a control-plane
|
||||
// tx, but the #legacy leg CAN — so the caller's tx (e.g. attemptFailed) must still be honored for
|
||||
// the legacy edges, keeping them atomic with the caller's operation instead of auto-committing.
|
||||
async function seedLegacyBlockingEdge(
|
||||
prisma14: PrismaClient,
|
||||
env: { project: { id: string }; environment: { id: string } },
|
||||
runId: string,
|
||||
suffix: string
|
||||
): Promise<void> {
|
||||
const waitpoint = await prisma14.waitpoint.create({
|
||||
data: {
|
||||
friendlyId: `wp_${suffix}`,
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${suffix}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
},
|
||||
});
|
||||
await prisma14.taskRunWaitpoint.create({
|
||||
data: { taskRunId: runId, waitpointId: waitpoint.id, projectId: env.project.id },
|
||||
});
|
||||
}
|
||||
|
||||
describe("fan-out deleteManyTaskRunWaitpoints honors the caller's tx on the #legacy leg", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"rolls the #legacy edge delete back when the caller's control-plane tx rolls back",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "del_tx_rb");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_del_tx_rb",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
await seedLegacyBlockingEdge(prisma14, env, runId, "del_tx_rb");
|
||||
|
||||
await expect(
|
||||
prisma14.$transaction(async (tx) => {
|
||||
await router.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx);
|
||||
throw new Error("rollback");
|
||||
})
|
||||
).rejects.toThrow("rollback");
|
||||
|
||||
const remaining = await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId } });
|
||||
expect(remaining).toBe(1);
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"still deletes the #legacy edge when the caller's tx commits",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "del_tx_commit");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_del_tx_commit",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
await seedLegacyBlockingEdge(prisma14, env, runId, "del_tx_commit");
|
||||
|
||||
await prisma14.$transaction(async (tx) => {
|
||||
await router.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx);
|
||||
});
|
||||
|
||||
const remaining = await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId } });
|
||||
expect(remaining).toBe(0);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// createExecutionSnapshot writes the snapshot row and its completed-waitpoint join rows. These MUST
|
||||
// commit together: with the flag off, `/snapshots/since` is served from a lagging read replica, so a
|
||||
// snapshot that commits before its `_completedWaitpoints` rows can be read waitpoint-less, and the
|
||||
// runner's EXECUTING branch no-ops on an empty completedWaitpoints -> the resume is lost -> hang.
|
||||
describe("createExecutionSnapshot writes the snapshot and its completed-waitpoint links atomically", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"rolls the snapshot back if the completed-waitpoint insert fails (no waitpoint-less snapshot persists)",
|
||||
async ({ prisma14 }) => {
|
||||
const legacy = makeLegacyStore(prisma14);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "ces_atomic");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await legacy.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_ces_atomic",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
const waitpoint = await prisma14.waitpoint.create({
|
||||
data: {
|
||||
friendlyId: "wp_ces_atomic",
|
||||
type: "MANUAL",
|
||||
status: "COMPLETED",
|
||||
idempotencyKey: "idem-ces_atomic",
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Force the completed-waitpoint join insert to fail mid-write.
|
||||
await prisma14.$executeRawUnsafe('DROP TABLE "_completedWaitpoints"');
|
||||
|
||||
await expect(
|
||||
// Pass the base client as `tx` - exactly how the engine threads its base prisma through
|
||||
// (continueRunIfUnblocked -> executionSnapshotSystem.createExecutionSnapshot(prisma, ...)).
|
||||
// It is NOT an interactive transaction, so the store must still open its own to stay atomic.
|
||||
legacy.createExecutionSnapshot(
|
||||
{
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: {
|
||||
executionStatus: "EXECUTING_WITH_WAITPOINTS",
|
||||
description: "Run was blocked by a waitpoint.",
|
||||
},
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.project.id,
|
||||
completedWaitpoints: [{ id: waitpoint.id, index: 0 }],
|
||||
},
|
||||
prisma14
|
||||
)
|
||||
).rejects.toThrow();
|
||||
|
||||
// The snapshot must NOT persist without its links, or a replica can serve it waitpoint-less.
|
||||
const snap = await prisma14.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING_WITH_WAITPOINTS" },
|
||||
});
|
||||
expect(snap).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// RoutingRunStore.createExecutionSnapshot accepts a caller tx but must forward it to the OWNING store
|
||||
// only when that store is #legacy: a control-plane tx can't wrap a #new (cross-DB) write, but it can
|
||||
// (and should) wrap a legacy-resident snapshot so it stays atomic with the caller's operation.
|
||||
describe("createExecutionSnapshot honors the caller's tx on the #legacy owning store", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"rolls the snapshot back when a legacy run's caller tx rolls back",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "ces_rb");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_ces_rb",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
await expect(
|
||||
prisma14.$transaction(async (tx) => {
|
||||
await router.createExecutionSnapshot(snapshotInput(runId, env), tx);
|
||||
throw new Error("rollback");
|
||||
})
|
||||
).rejects.toThrow("rollback");
|
||||
|
||||
const snap = await prisma14.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING" },
|
||||
});
|
||||
expect(snap).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"persists the snapshot when the legacy caller tx commits",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "ces_commit");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_ces_commit",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
await prisma14.$transaction(async (tx) => {
|
||||
await router.createExecutionSnapshot(snapshotInput(runId, env), tx);
|
||||
});
|
||||
|
||||
const snap = await prisma14.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING" },
|
||||
});
|
||||
expect(snap).not.toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// On the dedicated subset schema the associated (RUN-type) waitpoint is created as a SEPARATE
|
||||
// waitpoint.create after taskRun.create (the legacy schema nests it atomically). The pair must commit
|
||||
// together, or a crash / lagging read leaves a run with no completion waitpoint and its parent never resumes.
|
||||
function assocWaitpoint(
|
||||
env: { project: { id: string }; environment: { id: string } },
|
||||
suffix: string
|
||||
) {
|
||||
return {
|
||||
id: `wp_${suffix}`,
|
||||
friendlyId: `waitpoint_${suffix}`,
|
||||
type: "RUN" as const,
|
||||
status: "PENDING" as const,
|
||||
idempotencyKey: `idem_${suffix}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
};
|
||||
}
|
||||
|
||||
describe("createRun / createFailedRun write the run and its associated waitpoint atomically (dedicated)", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"createRun rolls the run back if the associated-waitpoint create fails",
|
||||
async ({ prisma17 }) => {
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "cr_atomic");
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
const input = {
|
||||
...buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_cr_atomic",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
}),
|
||||
associatedWaitpoint: assocWaitpoint(env, "cr_atomic"),
|
||||
};
|
||||
|
||||
// Force #createAssociatedWaitpoint (waitpoint.create) to fail after taskRun.create.
|
||||
await prisma17.$executeRawUnsafe('DROP TABLE "Waitpoint"');
|
||||
|
||||
await expect(newStore.createRun(input)).rejects.toThrow();
|
||||
|
||||
const run = await prisma17.taskRun.findFirst({ where: { id: runId } });
|
||||
expect(run).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"createFailedRun rolls the run back if the associated-waitpoint create fails",
|
||||
async ({ prisma17 }) => {
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "cf_atomic");
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
const base = buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_cf_atomic",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
});
|
||||
const input = { data: base.data, associatedWaitpoint: assocWaitpoint(env, "cf_atomic") };
|
||||
|
||||
await prisma17.$executeRawUnsafe('DROP TABLE "Waitpoint"');
|
||||
|
||||
await expect(newStore.createFailedRun(input)).rejects.toThrow();
|
||||
|
||||
const run = await prisma17.taskRun.findFirst({ where: { id: runId } });
|
||||
expect(run).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// The dedicated (#new) leg connects completed waitpoints through the `CompletedWaitpoint` join table
|
||||
// (createMany), where the legacy leg uses the implicit `_completedWaitpoints` M2M. Both must commit the
|
||||
// snapshot and its links together: a snapshot that commits before its links can be read waitpoint-less
|
||||
// from a lagging replica, and the runner's EXECUTING branch no-ops on an empty set -> the resume hangs.
|
||||
describe("createExecutionSnapshot / lockRunToWorker write the snapshot and its links atomically (dedicated)", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"createExecutionSnapshot rolls the snapshot back if the CompletedWaitpoint insert fails",
|
||||
async ({ prisma17 }) => {
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "ces_ded");
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
await newStore.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_ces_ded",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// Force the dedicated join insert (completedWaitpoint.createMany) to fail mid-write.
|
||||
await prisma17.$executeRawUnsafe('DROP TABLE "CompletedWaitpoint"');
|
||||
|
||||
await expect(
|
||||
// Base client as `tx` = how the engine threads its base prisma through
|
||||
// (continueRunIfUnblocked -> executionSnapshotSystem.createExecutionSnapshot(prisma, ...)).
|
||||
// It is NOT an interactive transaction, so the store must still open its own to stay atomic.
|
||||
newStore.createExecutionSnapshot(
|
||||
{
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: {
|
||||
executionStatus: "EXECUTING_WITH_WAITPOINTS",
|
||||
description: "Run was blocked by a waitpoint.",
|
||||
},
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.project.id,
|
||||
completedWaitpoints: [{ id: `wp_${NEW_ID_26}`, index: 0 }],
|
||||
},
|
||||
prisma17 as never
|
||||
)
|
||||
).rejects.toThrow();
|
||||
|
||||
const snap = await prisma17.taskRunExecutionSnapshot.findFirst({
|
||||
where: { runId, executionStatus: "EXECUTING_WITH_WAITPOINTS" },
|
||||
});
|
||||
expect(snap).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"lockRunToWorker rolls the snapshot and run lock back if the CompletedWaitpoint insert fails",
|
||||
async ({ prisma17 }) => {
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "lock_ded");
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
await newStore.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_lock_ded",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
const prior = await prisma17.taskRunExecutionSnapshot.findFirstOrThrow({ where: { runId } });
|
||||
|
||||
await prisma17.$executeRawUnsafe('DROP TABLE "CompletedWaitpoint"');
|
||||
|
||||
const snapshotId = `snap_${NEW_ID_26}`;
|
||||
await expect(
|
||||
// lockedById/lockedToVersionId/lockedQueueId are FK-free scalars on the dedicated subset, so
|
||||
// synthetic ids are fine; the base client as `tx` mirrors the dequeue path (no interactive tx).
|
||||
newStore.lockRunToWorker(
|
||||
runId,
|
||||
{
|
||||
lockedAt: new Date(),
|
||||
lockedById: `bwt_${NEW_ID_26}`,
|
||||
lockedToVersionId: `bw_${NEW_ID_26}`,
|
||||
lockedQueueId: `queue_${NEW_ID_26}`,
|
||||
startedAt: new Date(),
|
||||
baseCostInCents: 5,
|
||||
machinePreset: "small-1x",
|
||||
taskVersion: "20260601.1",
|
||||
sdkVersion: "3.0.0",
|
||||
cliVersion: "3.0.0",
|
||||
maxDurationInSeconds: null,
|
||||
snapshot: {
|
||||
id: snapshotId,
|
||||
previousSnapshotId: prior.id,
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.project.id,
|
||||
completedWaitpointIds: [`wp_${NEW_ID_26}`],
|
||||
completedWaitpointOrder: [`wp_${NEW_ID_26}`],
|
||||
},
|
||||
},
|
||||
prisma17 as never
|
||||
)
|
||||
).rejects.toThrow();
|
||||
|
||||
const snap = await prisma17.taskRunExecutionSnapshot.findUnique({
|
||||
where: { id: snapshotId },
|
||||
});
|
||||
expect(snap).toBeNull();
|
||||
// The whole lock write must roll back, not just the status: no lock columns may leak through.
|
||||
const run = await prisma17.taskRun.findUniqueOrThrow({ where: { id: runId } });
|
||||
expect(run.status).not.toBe("DEQUEUED");
|
||||
expect(run.lockedAt).toBeNull();
|
||||
expect(run.lockedById).toBeNull();
|
||||
expect(run.lockedToVersionId).toBeNull();
|
||||
expect(run.lockedQueueId).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
// REGRESSION suite for the run-ops split "control-plane tx/client forwarded into a NEW-resident
|
||||
// store" bug class on the BatchTaskRun write/probe path. When the router resolves the owning store
|
||||
// to #new but forwards the caller's control-plane handle, #new issues its statement against the
|
||||
// CONTROL-PLANE DB where the run-ops id row does not exist → "No record was found" (update), wrong-DB row
|
||||
// (create), or wrong count. Covers updateBatchTaskRun (commit 62ae880af), createBatchTaskRun and
|
||||
// countBatchTaskRunItems (this sweep). `heteroRunOpsPostgresTest` is the REAL two-DB split topology
|
||||
// (prisma17 = dedicated #new, prisma14 = legacy #legacy); NEVER mocked.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id LENGTH (runOpsResidency.ts): 25 chars → cuid → LEGACY,
|
||||
// a v1 body (26 chars, version "1" at index 25) → NEW.
|
||||
const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema)
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / prisma17, dedicated subset schema)
|
||||
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
// BatchTaskRunItem.taskRunId has an FK into TaskRun on the dedicated schema, so seed the referenced
|
||||
// run before creating an item that points at it.
|
||||
async function seedDedicatedRun(prisma17: RunOpsPrismaClient, envId: string, runId: string) {
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id: runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: `run_${runId}`,
|
||||
runtimeEnvironmentId: envId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: "org_cntitems_new",
|
||||
projectId: "proj_cntitems_new",
|
||||
taskIdentifier: "batch-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `t_${runId}`,
|
||||
spanId: `s_${runId}`,
|
||||
queue: "task/batch-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function makeLegacyStore(prisma14: PrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
}
|
||||
|
||||
// Real production split topology: #new = dedicated subset on prisma17, #legacy = full schema on
|
||||
// prisma14 — two physically distinct DBs.
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = makeLegacyStore(prisma14);
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
legacyStore,
|
||||
newStore,
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — BatchTaskRun writes/probes must NOT forward the control-plane tx/client into NEW", () => {
|
||||
// ===========================================================================================
|
||||
// updateBatchTaskRun (commit 62ae880af) — the batch-completion residency regression.
|
||||
// ===========================================================================================
|
||||
|
||||
// The live `batchSystem.#tryCompleteBatch` shape: a run-ops batch on #new is updated to COMPLETED
|
||||
// while the control-plane client is passed as `tx`. RED on the pre-62ae880af code (the router
|
||||
// forwarded tx → #new ran the UPDATE on the control-plane DB → "No record was found for an
|
||||
// update"); GREEN now (tx dropped for NEW → the row updates on #new's own DB).
|
||||
heteroRunOpsPostgresTest(
|
||||
"updateBatchTaskRun marks a run-ops batch on #new COMPLETED even when the control-plane client is passed as tx",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "updbatch_new");
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id → #new
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_upd_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 2,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
|
||||
// Pass the LEGACY (control-plane) client as `tx`, EXACTLY as #tryCompleteBatch does.
|
||||
const updated = await router.updateBatchTaskRun(
|
||||
{ where: { id: batchId }, data: { status: "COMPLETED" }, select: { id: true } },
|
||||
prisma14 as never
|
||||
);
|
||||
expect(updated.id).toBe(batchId);
|
||||
|
||||
// The row on #new (its own DB) is genuinely COMPLETED — not a phantom update on the wrong DB.
|
||||
const onNew = await prisma17.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onNew?.status).toBe("COMPLETED");
|
||||
}
|
||||
);
|
||||
|
||||
// Control: a cuid batch on #legacy still updates through the router when the same (legacy) client
|
||||
// is passed as tx — the tx IS forwarded for LEGACY (same physical DB), so atomicity is preserved.
|
||||
heteroRunOpsPostgresTest(
|
||||
"updateBatchTaskRun control: a cuid batch on #legacy still updates with the control-plane tx forwarded",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "updbatch_leg");
|
||||
const batchId = `batch_${CUID_25}`; // cuid → #legacy
|
||||
|
||||
await prisma14.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_upd_leg",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 1,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
|
||||
const updated = await router.updateBatchTaskRun(
|
||||
{ where: { id: batchId }, data: { status: "COMPLETED" }, select: { id: true } },
|
||||
prisma14 as never
|
||||
);
|
||||
expect(updated.id).toBe(batchId);
|
||||
const onLegacy = await prisma14.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onLegacy?.status).toBe("COMPLETED");
|
||||
}
|
||||
);
|
||||
|
||||
// ===========================================================================================
|
||||
// createBatchTaskRun (this sweep) — same anti-pattern on the create path.
|
||||
// ===========================================================================================
|
||||
|
||||
// A run-ops batch routed to #new with a forwarded control-plane tx must still be created on #new's
|
||||
// OWN DB, not the control-plane DB (which would strand the batch away from its co-resident child
|
||||
// runs/items). Forwarding tx unconditionally would land the row on prisma14.
|
||||
heteroRunOpsPostgresTest(
|
||||
"createBatchTaskRun lands a run-ops batch on #new even when the control-plane client is passed as tx",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "crbatch_new");
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id → #new
|
||||
|
||||
const created = await router.createBatchTaskRun(
|
||||
{
|
||||
id: batchId,
|
||||
friendlyId: "batch_cr_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 1,
|
||||
},
|
||||
prisma14 as never // control-plane tx
|
||||
);
|
||||
expect(created.id).toBe(batchId);
|
||||
|
||||
// Resident on #new (its own DB), absent from #legacy — co-located with its run-ops child runs.
|
||||
const onNew = await prisma17.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onNew).not.toBeNull();
|
||||
const onLegacy = await prisma14.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onLegacy).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// Control: a cuid batch is created on #legacy with the same control-plane tx forwarded (same DB).
|
||||
heteroRunOpsPostgresTest(
|
||||
"createBatchTaskRun control: a cuid batch lands on #legacy with the control-plane tx forwarded",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "crbatch_leg");
|
||||
const batchId = `batch_${CUID_25}`; // cuid → #legacy
|
||||
|
||||
const created = await router.createBatchTaskRun(
|
||||
{
|
||||
id: batchId,
|
||||
friendlyId: "batch_cr_leg",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 1,
|
||||
},
|
||||
prisma14 as never
|
||||
);
|
||||
expect(created.id).toBe(batchId);
|
||||
const onLegacy = await prisma14.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onLegacy).not.toBeNull();
|
||||
const onNew = await prisma17.batchTaskRun.findUnique({ where: { id: batchId } });
|
||||
expect(onNew).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// ===========================================================================================
|
||||
// countBatchTaskRunItems (this sweep) — same anti-pattern on a routed probe read.
|
||||
// ===========================================================================================
|
||||
|
||||
// A run-ops batch's items live on #new; counting them with the control-plane client forwarded would
|
||||
// count on the wrong DB (→ 0). The routed store must read its OWN DB and return the real count.
|
||||
heteroRunOpsPostgresTest(
|
||||
"countBatchTaskRunItems counts a run-ops batch's items on #new even when the control-plane client is passed",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "cntitems_new");
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id → #new
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_cnt_new",
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
runCount: 2,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
const runA = `run_${NEW_ID_26.slice(0, -3)}ra1`;
|
||||
const runB = `run_${NEW_ID_26.slice(0, -3)}rb1`;
|
||||
await seedDedicatedRun(prisma17, env.environment.id, runA);
|
||||
await seedDedicatedRun(prisma17, env.environment.id, runB);
|
||||
await prisma17.batchTaskRunItem.create({
|
||||
data: { batchTaskRunId: batchId, taskRunId: runA, status: "COMPLETED" },
|
||||
});
|
||||
await prisma17.batchTaskRunItem.create({
|
||||
data: { batchTaskRunId: batchId, taskRunId: runB, status: "PENDING" },
|
||||
});
|
||||
|
||||
// Pass the LEGACY (control-plane) client; the routed #new store must ignore it and read its own DB.
|
||||
expect(
|
||||
await router.countBatchTaskRunItems({ batchTaskRunId: batchId }, prisma14 as never)
|
||||
).toBe(2);
|
||||
expect(
|
||||
await router.countBatchTaskRunItems(
|
||||
{ batchTaskRunId: batchId, status: "COMPLETED" },
|
||||
prisma14 as never
|
||||
)
|
||||
).toBe(1);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
// Repro for the checkpoint WAIT_FOR_BATCH replica-lag stall (createCheckpoint.server.ts:148-181).
|
||||
//
|
||||
// The service resolves the batch via `runStore.findBatchTaskRunByFriendlyId(friendlyId, envId)` with
|
||||
// NO client, so the read is served from the REPLICA. Its decision hinges on `batchRun.resumedAt`:
|
||||
// resumedAt set -> return keepRunAlive:true (batch already resumed; the run must keep executing)
|
||||
// resumedAt null -> fall through -> create the checkpoint -> SUSPEND the run
|
||||
// If the batch just resumed (primary has resumedAt), but the replica still lags (resumedAt null), the
|
||||
// service suspends a run whose batch already completed -> it stalls until a sweep. The sibling
|
||||
// WAIT_FOR_TASK arm reads the primary (this._prisma); only the batch arm defaults to the replica.
|
||||
//
|
||||
// This is invisible to the normal single-DB harness (no lag). We reintroduce the lag with the shared
|
||||
// `laggingReplica` primitive: the store's replica is frozen at the pre-resume snapshot while the
|
||||
// primary advances. RED = the current (client-less, replica) read -> SUSPEND. GREEN = threading the
|
||||
// primary (the one-line fix) -> KEEP_ALIVE.
|
||||
|
||||
import { laggingReplica, postgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type { ReadClient } from "./types.js";
|
||||
|
||||
type BatchRow = { resumedAt: Date | null } | null;
|
||||
|
||||
// A line-for-line mirror of the service's WAIT_FOR_BATCH pre-check. `readClient` models the fix: the
|
||||
// buggy code passes nothing (replica default); the fix threads the primary.
|
||||
async function precheckWaitForBatch(
|
||||
store: PostgresRunStore,
|
||||
batchFriendlyId: string,
|
||||
environmentId: string,
|
||||
readClient?: ReadClient
|
||||
): Promise<"DROP_RUN" | "KEEP_ALIVE" | "SUSPEND"> {
|
||||
const batchRun = (await store.findBatchTaskRunByFriendlyId(
|
||||
batchFriendlyId,
|
||||
environmentId,
|
||||
undefined,
|
||||
readClient
|
||||
)) as BatchRow;
|
||||
if (!batchRun) return "DROP_RUN"; // keepRunAlive:false
|
||||
if (batchRun.resumedAt) return "KEEP_ALIVE"; // batch already resumed -> run continues
|
||||
return "SUSPEND"; // falls through -> checkpoint created -> run suspended
|
||||
}
|
||||
|
||||
async function seedEnvironment(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
describe("checkpoint WAIT_FOR_BATCH under replica lag", () => {
|
||||
postgresTest(
|
||||
"a batch resumed on the primary but stale on the replica suspends an already-resumed run",
|
||||
async ({ prisma }) => {
|
||||
const { environment } = await seedEnvironment(prisma, "ckpt_lag");
|
||||
const friendlyId = "batch_ckpt_lag";
|
||||
const batch = await prisma.batchTaskRun.create({
|
||||
data: { friendlyId, runtimeEnvironmentId: environment.id },
|
||||
});
|
||||
|
||||
// Snapshot the batch as the replica still sees it (pre-resume: resumedAt = null).
|
||||
const staleBatch = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batch.id } });
|
||||
expect(staleBatch.resumedAt).toBeNull();
|
||||
|
||||
// The batch completes and resumes the parent: primary now has resumedAt set...
|
||||
await prisma.batchTaskRun.update({
|
||||
where: { id: batch.id },
|
||||
data: { resumedAt: new Date() },
|
||||
});
|
||||
|
||||
// ...but the replica lags, frozen at the pre-resume snapshot.
|
||||
const replica = laggingReplica(prisma, [
|
||||
{ model: "batchTaskRun", mode: "frozen", rows: [staleBatch] },
|
||||
]);
|
||||
const store = new PostgresRunStore({
|
||||
prisma,
|
||||
readOnlyPrisma: replica.client,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
|
||||
// RED - the current service call (no client -> replica): stale null -> SUSPEND an already-resumed run.
|
||||
const buggy = await precheckWaitForBatch(store, friendlyId, environment.id);
|
||||
expect(replica.wasHit("batchTaskRun")).toBe(true); // proves it read the (stale) replica
|
||||
expect(buggy).toBe("SUSPEND"); // the stall bug
|
||||
|
||||
// GREEN - the fix (thread the primary): sees resumedAt -> keep the run alive.
|
||||
const fixed = await precheckWaitForBatch(store, friendlyId, environment.id, prisma);
|
||||
expect(fixed).toBe("KEEP_ALIVE");
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
// A TYPE-LEVEL test: it must COMPILE. It proves a RunOpsPrismaClient can back a
|
||||
// PostgresRunStore that satisfies the RunStore interface, alongside the legacy client.
|
||||
import { expectTypeOf, it } from "vitest";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type { RunStore } from "./types.js";
|
||||
|
||||
it("both clients can back a RunStore", () => {
|
||||
// These are type-only assertions; no runtime DB needed.
|
||||
const legacy = null as unknown as PrismaClient;
|
||||
const dedicated = null as unknown as RunOpsPrismaClient;
|
||||
const a: RunStore = new PostgresRunStore({ prisma: legacy, readOnlyPrisma: legacy });
|
||||
const b: RunStore = new PostgresRunStore({ prisma: dedicated, readOnlyPrisma: dedicated });
|
||||
expectTypeOf(a).toMatchTypeOf<RunStore>();
|
||||
expectTypeOf(b).toMatchTypeOf<RunStore>();
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./types.js";
|
||||
export * from "./PostgresRunStore.js";
|
||||
export * from "./runOpsStore.js";
|
||||
export * from "./readReplicaClient.js";
|
||||
@@ -0,0 +1,26 @@
|
||||
// A writer and a read-replica Prisma client are structurally identical at runtime (a replica is a
|
||||
// `new PrismaClient(...)` too, so it also exposes `$transaction`). The routing layer therefore
|
||||
// cannot tell a caller-passed replica from a writer by shape, yet it must — a writer/tx read has to
|
||||
// reach the owning store's PRIMARY (read-your-writes) while a replica read stays on a replica (read
|
||||
// scaling). So the client builder brands replica handles and the routing store reads the brand.
|
||||
export const READ_REPLICA_BRAND: unique symbol = Symbol.for("trigger.dev/run-store/read-replica");
|
||||
|
||||
// Brand a replica client (returns the same object). MUST only be called on a genuine replica: the
|
||||
// routing layer trusts the brand to mean "do not escalate this read to the primary". An unbranded
|
||||
// replica just escalates as before — a scaling miss, never a correctness bug.
|
||||
export function markReadReplicaClient<T extends object>(client: T): T {
|
||||
try {
|
||||
(client as Record<symbol, unknown>)[READ_REPLICA_BRAND] = true;
|
||||
} catch {
|
||||
// Frozen/exotic clients may reject the assignment; only costs the optimization, not correctness.
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
export function isReadReplicaClient(client: unknown): boolean {
|
||||
return (
|
||||
!!client &&
|
||||
typeof client === "object" &&
|
||||
(client as Record<symbol, unknown>)[READ_REPLICA_BRAND] === true
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// updateManyBatchTaskRunItems routed by where.id first, but BatchTaskRunItem.id is a cuid (always
|
||||
// classifies LEGACY). For a NEW-residency batch the items live on #new, so completing an item by
|
||||
// {id, batchTaskRunId, status} updated #legacy, matched 0 rows, and the caller treated count===0 as
|
||||
// "already completed" -> tryCompleteBatchV3 never fired -> the item stayed PENDING and the parent's
|
||||
// batchTriggerAndWait hung. Fix: route by batchTaskRunId (residency-encoding) first, like
|
||||
// countBatchTaskRunItems. Real two-DB topology; never mocked.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // run-ops id -> NEW (#new / prisma17)
|
||||
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
return new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
}
|
||||
|
||||
// BatchTaskRunItem.taskRunId FKs into TaskRun on the dedicated schema, so seed the run first.
|
||||
async function seedDedicatedRun(prisma17: RunOpsPrismaClient, envId: string, runId: string) {
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id: runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: `run_${runId}`,
|
||||
runtimeEnvironmentId: envId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: "org_batchitem",
|
||||
projectId: "proj_batchitem",
|
||||
taskIdentifier: "batch-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `t_${runId}`,
|
||||
spanId: `s_${runId}`,
|
||||
queue: "task/batch-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("run-ops split — completing a batch item routes by the batch id, not the item cuid", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"updateManyBatchTaskRunItems completes a NEW-resident item addressed by {id, batchTaskRunId}",
|
||||
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
|
||||
const router = makeSplitRouter(prisma14, prisma17);
|
||||
const envId = "env_batchitem";
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id -> #new
|
||||
const runId = `run_${NEW_ID_26.slice(0, -3)}ri1`;
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_item_new",
|
||||
runtimeEnvironmentId: envId,
|
||||
runCount: 1,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
await seedDedicatedRun(prisma17, envId, runId);
|
||||
// item.id is an auto cuid (classifies LEGACY) — the mis-routing key, exactly as in production.
|
||||
const item = await prisma17.batchTaskRunItem.create({
|
||||
data: { batchTaskRunId: batchId, taskRunId: runId, status: "PENDING" },
|
||||
});
|
||||
|
||||
const result = await router.updateManyBatchTaskRunItems({
|
||||
where: { id: item.id, batchTaskRunId: batchId, status: "PENDING" },
|
||||
data: { status: "COMPLETED" },
|
||||
});
|
||||
|
||||
// RED: routed by the item cuid -> #legacy -> 0 rows -> batch never completes -> parent hangs.
|
||||
// GREEN: routed by batchTaskRunId (NEW) -> #new -> the item completes.
|
||||
expect(result.count).toBe(1);
|
||||
const onNew = await prisma17.batchTaskRunItem.findUnique({ where: { id: item.id } });
|
||||
expect(onNew?.status).toBe("COMPLETED");
|
||||
// Legacy was never touched: no phantom/double-routed item update on #legacy.
|
||||
expect(await prisma14.batchTaskRunItem.count({ where: { batchTaskRunId: batchId } })).toBe(0);
|
||||
}
|
||||
);
|
||||
|
||||
// createBatchTaskRunItem must co-locate the item with its BATCH (so the completion count, routed by
|
||||
// batchTaskRunId, finds it), not with the child run. Routing by taskRunId would place a divergent-
|
||||
// residency item on the child's DB -> invisible to the count -> the batch never completes -> parent hangs.
|
||||
heteroRunOpsPostgresTest(
|
||||
"createBatchTaskRunItem places the item on the batch's DB, routing by batchTaskRunId not taskRunId",
|
||||
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
|
||||
const router = makeSplitRouter(prisma14, prisma17);
|
||||
const envId = "env_batchitem";
|
||||
const batchId = `batch_${NEW_ID_26}`; // run-ops id -> #new
|
||||
const runId = "c".repeat(25); // cuid -> classifies LEGACY (the divergent routing key)
|
||||
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
friendlyId: "batch_create_new",
|
||||
runtimeEnvironmentId: envId,
|
||||
runCount: 1,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
// The run physically lives on #new (the batch's DB) so the item's FKs resolve there.
|
||||
await seedDedicatedRun(prisma17, envId, runId);
|
||||
|
||||
await router.createBatchTaskRunItem({
|
||||
batchTaskRunId: batchId,
|
||||
taskRunId: runId,
|
||||
status: "PENDING",
|
||||
});
|
||||
|
||||
// GREEN: routed by batchTaskRunId (NEW) -> item on #new, visible to the batch-completion count.
|
||||
// RED: routed by the cuid taskRunId -> #legacy -> the create FK-fails / the count never sees it.
|
||||
expect(await prisma17.batchTaskRunItem.count({ where: { batchTaskRunId: batchId } })).toBe(1);
|
||||
expect(await prisma14.batchTaskRunItem.count({ where: { batchTaskRunId: batchId } })).toBe(0);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,162 @@
|
||||
// createExecutionSnapshot's legacy branch records completed waitpoints via Prisma `connect` on the
|
||||
// implicit _completedWaitpoints M2M, whose FK to Waitpoint rejects a cross-DB (NEW-resident) token.
|
||||
// A LEGACY parent doing triggerAndWait on a NEW child completes on the NEW token; the resume snapshot
|
||||
// (routed to #legacy) then connects that NEW token -> FK violation -> the legacy parent hangs forever.
|
||||
// Fix: drop _completedWaitpoints_B_fkey (migration); with the FK gone the connect records the cross-DB
|
||||
// link (app-enforced integrity). Real two-DB topology; never mocked.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { CreateRunInput } from "./types.js";
|
||||
|
||||
const CUID_25 = "c".repeat(25); // LEGACY run -> #legacy (prisma14)
|
||||
|
||||
function makeRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
return new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
}
|
||||
|
||||
async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "PRODUCTION",
|
||||
slug: `prod-${suffix}`,
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_prod_${suffix}`,
|
||||
pkApiKey: `pk_prod_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
maximumConcurrencyLimit: 10,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(p: {
|
||||
runId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: p.runId,
|
||||
engine: "V2",
|
||||
status: "EXECUTING",
|
||||
friendlyId: "run_cwc",
|
||||
runtimeEnvironmentId: p.runtimeEnvironmentId,
|
||||
environmentType: "PRODUCTION",
|
||||
organizationId: p.organizationId,
|
||||
projectId: p.projectId,
|
||||
taskIdentifier: "cwc-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
context: {},
|
||||
traceContext: {},
|
||||
traceId: `trace_${p.runId}`,
|
||||
spanId: `span_${p.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/cwc-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: p.runtimeEnvironmentId,
|
||||
environmentType: "PRODUCTION",
|
||||
projectId: p.projectId,
|
||||
organizationId: p.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — a LEGACY snapshot can record a cross-DB completed waitpoint (NEW-resident token)", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"createExecutionSnapshot connects a NEW-resident completed token to a LEGACY run's snapshot",
|
||||
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
|
||||
const router = makeRouter(prisma14, prisma17);
|
||||
// The harness builds the schema with `prisma db push`, which re-creates the FK that migration
|
||||
// 20260705230000 drops in prod. Mirror the drop on this clone so the connect exercises prod state.
|
||||
await prisma14.$executeRawUnsafe(
|
||||
`ALTER TABLE "_completedWaitpoints" DROP CONSTRAINT IF EXISTS "_completedWaitpoints_B_fkey"`
|
||||
);
|
||||
const env = await seedEnvironmentLegacy(prisma14, "cwc");
|
||||
const runId = `run_${CUID_25}`; // LEGACY run -> #legacy
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
const created = await router.findLatestExecutionSnapshot(runId);
|
||||
|
||||
// A completed token minted co-located with a NEW child -> its Waitpoint row lives on #new.
|
||||
const tokenId = "waitpoint_" + "y".repeat(20);
|
||||
await prisma17.waitpoint.create({
|
||||
data: {
|
||||
id: tokenId,
|
||||
friendlyId: "wp_cwc",
|
||||
type: "MANUAL",
|
||||
status: "COMPLETED",
|
||||
completedAt: new Date(),
|
||||
idempotencyKey: `idem_${tokenId}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
},
|
||||
});
|
||||
|
||||
const snap = await router.createExecutionSnapshot(
|
||||
{
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING_WITH_WAITPOINTS", description: "resumed" },
|
||||
previousSnapshotId: created!.id,
|
||||
completedWaitpoints: [{ id: tokenId, index: 0 }],
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "PRODUCTION",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
},
|
||||
prisma14
|
||||
);
|
||||
|
||||
// RED: the connect hits _completedWaitpoints_B_fkey (token not on #legacy) -> throws -> parent hangs.
|
||||
// GREEN (FK dropped): the cross-DB completed-waitpoint link is recorded.
|
||||
const link = (await prisma14.$queryRaw`
|
||||
SELECT "B" FROM "_completedWaitpoints" WHERE "A" = ${snap.id}
|
||||
`) as { B: string }[];
|
||||
expect(link.map((r) => r.B)).toEqual([tokenId]);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
// blockRunWithWaitpointEdges' legacy branch joined `FROM "Waitpoint" w`, so a LEGACY run blocking on
|
||||
// a NEW-resident (run-ops) token — whose Waitpoint row lives on #new — matched 0 rows and wrote NO
|
||||
// blocking edge. countPendingWaitpoints then fans out, still sees the token PENDING, so the run is
|
||||
// suspended believing it's blocked while nothing will ever resume it -> silent hang. The fix sources
|
||||
// the edge rows from the id array via `unnest` (FK-free, mirroring the dedicated branch); a migration
|
||||
// drops the _WaitpointRunConnections -> Waitpoint FK so the cross-DB connection can be recorded too.
|
||||
// Real two-DB topology; never mocked.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
|
||||
const CUID_25 = "c".repeat(25); // LEGACY run -> #legacy (prisma14, full schema)
|
||||
|
||||
function makeRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
return new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
}
|
||||
|
||||
async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function taskRunData(opts: {
|
||||
id: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}) {
|
||||
return {
|
||||
id: opts.id,
|
||||
engine: "V2" as const,
|
||||
status: "EXECUTING" as const,
|
||||
friendlyId: `run_${opts.id}`,
|
||||
runtimeEnvironmentId: opts.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
organizationId: opts.organizationId,
|
||||
projectId: opts.projectId,
|
||||
taskIdentifier: "xdb-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${opts.id}`,
|
||||
spanId: `span_${opts.id}`,
|
||||
queue: "task/xdb-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — a LEGACY run blocking on a NEW-resident token gets its blocking edge (cross-DB)", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"blockRunWithWaitpointEdges writes the edge for a cross-DB token instead of stranding the run",
|
||||
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
|
||||
const router = makeRouter(prisma14, prisma17);
|
||||
// The harness builds the schema with `prisma db push`, which re-creates the FKs that the
|
||||
// run-ops split migrations (20260705210000 / 20260705220000) drop in prod. Mirror those drops on
|
||||
// this clone so the cross-DB insert exercises the real prod state (FKs gone, app-enforced).
|
||||
await prisma14.$executeRawUnsafe(
|
||||
`ALTER TABLE "TaskRunWaitpoint" DROP CONSTRAINT IF EXISTS "TaskRunWaitpoint_waitpointId_fkey"`
|
||||
);
|
||||
await prisma14.$executeRawUnsafe(
|
||||
`ALTER TABLE "_WaitpointRunConnections" DROP CONSTRAINT IF EXISTS "_WaitpointRunConnections_B_fkey"`
|
||||
);
|
||||
const seed = await seedEnvironmentLegacy(prisma14, "xdbblock");
|
||||
const runId = `run_${CUID_25}`; // LEGACY run, resident on #legacy
|
||||
await prisma14.taskRun.create({
|
||||
data: taskRunData({
|
||||
id: runId,
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
}),
|
||||
});
|
||||
|
||||
// The token was minted co-located with a run-ops run, so its Waitpoint row lives on #new.
|
||||
const tokenId = "waitpoint_" + "x".repeat(20);
|
||||
await prisma17.waitpoint.create({
|
||||
data: {
|
||||
id: tokenId,
|
||||
friendlyId: "wp_xdb",
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${tokenId}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: seed.project.id,
|
||||
environmentId: seed.environment.id,
|
||||
},
|
||||
});
|
||||
|
||||
await router.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [tokenId],
|
||||
projectId: seed.project.id,
|
||||
});
|
||||
|
||||
// RED: the legacy `FROM "Waitpoint"` join matches 0 rows -> no edge -> the run is stranded.
|
||||
// GREEN: `unnest` writes the edge from the id directly.
|
||||
expect(
|
||||
await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId, waitpointId: tokenId } })
|
||||
).toBe(1);
|
||||
// The historical connection is recorded too (needs the dropped B-fkey migration).
|
||||
const conn = (await prisma14.$queryRaw`
|
||||
SELECT "B" FROM "_WaitpointRunConnections" WHERE "A" = ${runId}
|
||||
`) as unknown[];
|
||||
expect(conn.length).toBe(1);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,186 @@
|
||||
// DOCUMENTS the bounded concurrent-flip-window cross-DB idempotency
|
||||
// duplicate. This is a known, bounded (<=mintCache TTL, 30s default) edge, NOT a closed gap.
|
||||
//
|
||||
// During the flip window, two CONCURRENT same-(env, idempotencyKey, taskIdentifier) ROOT triggers can
|
||||
// land on instances with DIVERGENT cached mint-kinds: the stale instance mints a cuid run on #legacy,
|
||||
// the flipped instance a run-ops run on #new. The dedup probe (probe-before-mint) only catches an
|
||||
// ALREADY-COMMITTED run; two truly-simultaneous mints both miss, then both create. The per-DB unique
|
||||
// constraint on (runtimeEnvironmentId, idempotencyKey, taskIdentifier) is PER PHYSICAL DB, so it
|
||||
// cannot reject the second insert that lands on the OTHER DB. This test proves both creates SUCCEED
|
||||
// (the duplicate is real) and the NEW-first read fan-out collapses subsequent reads to one run
|
||||
// (the duplicate is bounded — see the cross-DB dedup tie-break test). A cross-DB write guard is
|
||||
// intentionally not added here; that is a deliberate policy decision left to the operator.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id length (no internal underscore): 25 -> cuid -> LEGACY,
|
||||
// 27 -> run-ops id -> NEW.
|
||||
const cuidLegacy = (seed: string) => (seed + "c".repeat(25)).slice(0, 25);
|
||||
const runOpsNew = (seed: string) =>
|
||||
(seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01";
|
||||
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
idempotencyKey: string;
|
||||
taskIdentifier: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
idempotencyKey: params.idempotencyKey,
|
||||
idempotencyKeyExpiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
return new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
}
|
||||
|
||||
describe("RoutingRunStore — mint-on-flip bounded concurrent-window cross-DB duplicate (DOCUMENTED, not guarded)", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"two divergent-cache root mints of the SAME (env, key) BOTH succeed, landing one-per-DB (per-DB unique cannot catch it)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const router = makeSplitRouter(prisma14, prisma17);
|
||||
// One logical environment shared across both physical DBs (same scalar envId on each).
|
||||
const seed = await seedEnvironment(prisma14, "legacy", "flipwin");
|
||||
const environmentId = seed.environment.id;
|
||||
const idempotencyKey = "flip-window-key";
|
||||
const taskIdentifier = "my-task";
|
||||
|
||||
const staleCuidRunId = cuidLegacy("rfl"); // stale instance mints cuid -> #legacy
|
||||
const flippedRunOpsRunId = runOpsNew("rfn"); // flipped instance mints run-ops id -> #new
|
||||
|
||||
// Both concurrent mints commit. The second does NOT throw a unique violation: the constraint is
|
||||
// PER-DB and these land on different physical DBs.
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: staleCuidRunId,
|
||||
friendlyId: "run_flip_legacy",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: flippedRunOpsRunId,
|
||||
friendlyId: "run_flip_new",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})
|
||||
);
|
||||
|
||||
// The duplicate is REAL: a row for the same key physically exists on BOTH DBs.
|
||||
expect(await prisma14.taskRun.findFirst({ where: { id: staleCuidRunId } })).not.toBeNull();
|
||||
expect(
|
||||
await prisma17.taskRun.findFirst({ where: { id: flippedRunOpsRunId } })
|
||||
).not.toBeNull();
|
||||
|
||||
// The duplicate is BOUNDED: subsequent reads via the id-less probe collapse to exactly ONE run,
|
||||
// deterministically the NEW one (NEW-first fan-out) — the same tie-break the cross-DB dedup
|
||||
// read locks. So at most one of the two divergent mints is observable after the window closes.
|
||||
const found = (await router.findRun({
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})) as Record<string, any> | null;
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(flippedRunOpsRunId);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { RunStore } from "./types.js";
|
||||
|
||||
// forWaitpointCompletion is async: it picks a preferred store from the id-shape + pins, then
|
||||
// PROBES findWaitpoint to resolve where the token ACTUALLY lives (drain can relocate a cuid
|
||||
// waitpoint onto NEW, or a run-ops token can be pinned LEGACY), falling back to the other store.
|
||||
// So the slots here are fakes whose only behaviour is "do I hold this waitpoint id?".
|
||||
function fakeStore(slot: string, heldIds: Set<string>): RunStore {
|
||||
return {
|
||||
__slot: slot,
|
||||
async findWaitpoint(args: { where?: { id?: string } }) {
|
||||
const id = args.where?.id;
|
||||
return id !== undefined && heldIds.has(id) ? ({ id } as never) : null;
|
||||
},
|
||||
} as unknown as RunStore;
|
||||
}
|
||||
|
||||
const RUN_OPS_ID = "waitpoint_" + "a".repeat(24) + "01";
|
||||
const CUID_ID = "waitpoint_" + "a".repeat(25);
|
||||
const UNCLASSIFIABLE_ID = "waitpoint_" + "a".repeat(26);
|
||||
|
||||
// Both stores hold the id under test unless a case overrides, so the resolver returns the
|
||||
// preferred store and the assertion is purely about the preference rule.
|
||||
function buildRouter(opts?: { newHolds?: string[]; legacyHolds?: string[] }): {
|
||||
router: RoutingRunStore;
|
||||
newStore: RunStore;
|
||||
legacyStore: RunStore;
|
||||
} {
|
||||
const all = [RUN_OPS_ID, CUID_ID, UNCLASSIFIABLE_ID];
|
||||
const newStore = fakeStore("new", new Set(opts?.newHolds ?? all));
|
||||
const legacyStore = fakeStore("legacy", new Set(opts?.legacyHolds ?? all));
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
newStore,
|
||||
legacyStore,
|
||||
};
|
||||
}
|
||||
|
||||
describe("RoutingRunStore.forWaitpointCompletion", () => {
|
||||
it("resolves a run-ops waitpointId with no pins to the NEW slot", async () => {
|
||||
const { router, newStore } = buildRouter();
|
||||
expect(await router.forWaitpointCompletion(RUN_OPS_ID, { routeKind: "MANUAL" })).toBe(newStore);
|
||||
});
|
||||
|
||||
it("resolves a cuid waitpointId with no pins to the LEGACY slot", async () => {
|
||||
const { router, legacyStore } = buildRouter();
|
||||
expect(await router.forWaitpointCompletion(CUID_ID, { routeKind: "MANUAL" })).toBe(legacyStore);
|
||||
});
|
||||
|
||||
it("pins to LEGACY when isCrossTreeIdempotency is true, even for a run-ops id", async () => {
|
||||
const { router, legacyStore } = buildRouter();
|
||||
expect(
|
||||
await router.forWaitpointCompletion(RUN_OPS_ID, {
|
||||
routeKind: "IDEMPOTENCY_REUSE",
|
||||
isCrossTreeIdempotency: true,
|
||||
})
|
||||
).toBe(legacyStore);
|
||||
});
|
||||
|
||||
it("pins to LEGACY when treeOwnerResidency is LEGACY, even for a run-ops id", async () => {
|
||||
const { router, legacyStore } = buildRouter();
|
||||
expect(
|
||||
await router.forWaitpointCompletion(RUN_OPS_ID, {
|
||||
routeKind: "MANUAL",
|
||||
treeOwnerResidency: "LEGACY",
|
||||
})
|
||||
).toBe(legacyStore);
|
||||
});
|
||||
|
||||
it("pins to LEGACY when hasLegacyParent is true, even for a run-ops id", async () => {
|
||||
const { router, legacyStore } = buildRouter();
|
||||
expect(
|
||||
await router.forWaitpointCompletion(RUN_OPS_ID, {
|
||||
routeKind: "RUN",
|
||||
hasLegacyParent: true,
|
||||
})
|
||||
).toBe(legacyStore);
|
||||
});
|
||||
|
||||
it("falls back to the OTHER store when the preferred store does not hold the token", async () => {
|
||||
// run-ops id prefers NEW, but the token actually lives on LEGACY (drain/relocation): the
|
||||
// probe must fall through to LEGACY rather than route by id-shape alone and miss it.
|
||||
const { router, legacyStore } = buildRouter({ newHolds: [], legacyHolds: [RUN_OPS_ID] });
|
||||
expect(await router.forWaitpointCompletion(RUN_OPS_ID, { routeKind: "MANUAL" })).toBe(
|
||||
legacyStore
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves an unclassifiable id to LEGACY-preferred (never throws)", async () => {
|
||||
// #classifySafe treats an unclassifiable id as LEGACY; with both stores empty the preferred
|
||||
// (LEGACY) is returned. The completion path must not blow up on an odd-length id.
|
||||
const { router, legacyStore } = buildRouter({ newHolds: [], legacyHolds: [] });
|
||||
expect(await router.forWaitpointCompletion(UNCLASSIFIABLE_ID, { routeKind: "MANUAL" })).toBe(
|
||||
legacyStore
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PostgresRunStore.forWaitpointCompletion", () => {
|
||||
it("returns the same store instance without classifying, even for an unclassifiable id", async () => {
|
||||
// No prisma client touched: forWaitpointCompletion is a pure `return this`.
|
||||
const store = new PostgresRunStore({} as never);
|
||||
expect(await store.forWaitpointCompletion(UNCLASSIFIABLE_ID, { routeKind: "MANUAL" })).toBe(
|
||||
store
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,456 @@
|
||||
// Idempotency cross-DB dedup LOCK against the REAL two-physical-DB split.
|
||||
//
|
||||
// The trigger hot path dedupes before minting via the id-less probe
|
||||
// `runStore.findRun({ runtimeEnvironmentId, idempotencyKey, taskIdentifier },
|
||||
// { include: { associatedWaitpoint: true } }, dedupClient)`
|
||||
// (apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts). The existing run may live
|
||||
// on EITHER physical DB (a cuid run on #legacy minted before the org flipped to run-ops id; a run-ops run on
|
||||
// #new after). The PG unique key is PER-DB and cannot enforce cross-DB uniqueness, so dedup must be
|
||||
// correct at the routing layer. RoutingRunStore.findRun drops the caller
|
||||
// dedupClient and, for an id-less where, fans out NEW→LEGACY (#findRunUnrouted).
|
||||
// Highest risk: `associatedWaitpoint` hydration — the scalar-only #new store strips the relation and
|
||||
// rehydrates from Waitpoint.completedByTaskRunId, whereas #legacy uses the Prisma include; the andWait
|
||||
// idempotent hit reads existingRun.associatedWaitpoint.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type {
|
||||
CreateRunInput,
|
||||
RunAssociatedWaitpointInput,
|
||||
RunStoreSchemaVariant,
|
||||
} from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id LENGTH after stripping a single leading `<prefix>_`:
|
||||
// 25 chars → cuid → LEGACY, a v1 body (version "1" at index 25) → run-ops id → NEW. So a classifiable id
|
||||
// must carry NO internal underscore. These mint a distinct id of the right length from a short seed.
|
||||
function cuidLegacy(seed: string): string {
|
||||
return (seed + "c".repeat(25)).slice(0, 25); // 25 chars, no underscore → LEGACY
|
||||
}
|
||||
function runOpsNew(seed: string): string {
|
||||
return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01";
|
||||
}
|
||||
|
||||
// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops
|
||||
// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows
|
||||
// the kept FKs require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildAssociatedWaitpoint(params: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
}): RunAssociatedWaitpointInput {
|
||||
return {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "RUN",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `wpidem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
idempotencyKey: string;
|
||||
taskIdentifier: string;
|
||||
associatedWaitpoint?: RunAssociatedWaitpointInput;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
idempotencyKey: params.idempotencyKey,
|
||||
idempotencyKeyExpiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
associatedWaitpoint: params.associatedWaitpoint,
|
||||
};
|
||||
}
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function makeLegacyStore(prisma14: PrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
}
|
||||
|
||||
// The REAL production split topology: #new = dedicated subset on prisma17, #legacy = full schema on
|
||||
// prisma14. Two physically-distinct DBs, dedicated schema on #new.
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = makeLegacyStore(prisma14);
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
legacyStore,
|
||||
newStore,
|
||||
};
|
||||
}
|
||||
|
||||
// The EXACT dedup probe the trigger hot path issues: id-less
|
||||
// where keyed on (runtimeEnvironmentId, idempotencyKey, taskIdentifier), include associatedWaitpoint.
|
||||
function dedupProbe(
|
||||
router: RoutingRunStore,
|
||||
params: { runtimeEnvironmentId: string; idempotencyKey: string; taskIdentifier: string }
|
||||
) {
|
||||
return router.findRun(
|
||||
{
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
idempotencyKey: params.idempotencyKey,
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
},
|
||||
{ include: { associatedWaitpoint: true } }
|
||||
);
|
||||
}
|
||||
|
||||
describe("RoutingRunStore — cross-DB idempotency dedup probe", () => {
|
||||
// the matching run + its associated waitpoint live on #legacy (cuid, full schema). The
|
||||
// probe fans out NEW (miss) → LEGACY (hit) and must hydrate the waitpoint via the legacy include.
|
||||
heteroRunOpsPostgresTest(
|
||||
"a cuid run on #legacy is found by the id-less probe with associatedWaitpoint hydrated",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "cg2_a");
|
||||
const runId = cuidLegacy("ral"); // 25 chars → LEGACY home
|
||||
const waitpointId = cuidLegacy("wal");
|
||||
const idempotencyKey = "cg2-key-a";
|
||||
const taskIdentifier = "my-task";
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_cg2_a`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: waitpointId,
|
||||
friendlyId: `waitpoint_cg2_a`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
// It must NOT have landed on #new (the cuid id routes to LEGACY).
|
||||
expect(await prisma17.taskRun.findFirst({ where: { id: runId } })).toBeNull();
|
||||
expect(await prisma14.taskRun.findFirst({ where: { id: runId } })).not.toBeNull();
|
||||
|
||||
const found = (await dedupProbe(router, {
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})) as Record<string, any> | null;
|
||||
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(runId);
|
||||
expect(found!.idempotencyKey).toBe(idempotencyKey);
|
||||
// The load-bearing assertion: the andWait idempotent hit reads existingRun.associatedWaitpoint.
|
||||
expect(found!.associatedWaitpoint).not.toBeNull();
|
||||
expect(found!.associatedWaitpoint.id).toBe(waitpointId);
|
||||
expect(found!.associatedWaitpoint.type).toBe("RUN");
|
||||
expect(found!.associatedWaitpoint.completedByTaskRunId).toBe(runId);
|
||||
}
|
||||
);
|
||||
|
||||
// the matching run + its associated waitpoint live on #new (run-ops id, dedicated subset). The
|
||||
// probe hits the NEW leg first; the SCALAR-ONLY store must strip the `associatedWaitpoint` relation
|
||||
// and re-hydrate it from `Waitpoint.completedByTaskRunId`.
|
||||
heteroRunOpsPostgresTest(
|
||||
"a run-ops run on #new is found by the id-less probe with associatedWaitpoint hydrated from scalar",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "cg2_b");
|
||||
const runId = runOpsNew("rbn"); // v1 body → NEW home
|
||||
const waitpointId = runOpsNew("wbn");
|
||||
const idempotencyKey = "cg2-key-b";
|
||||
const taskIdentifier = "my-task";
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_cg2_b`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: waitpointId,
|
||||
friendlyId: `waitpoint_cg2_b`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
// It must NOT have landed on #legacy (the run-ops id routes to NEW).
|
||||
expect(await prisma14.taskRun.findFirst({ where: { id: runId } })).toBeNull();
|
||||
expect(await prisma17.taskRun.findFirst({ where: { id: runId } })).not.toBeNull();
|
||||
|
||||
const found = (await dedupProbe(router, {
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})) as Record<string, any> | null;
|
||||
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(runId);
|
||||
expect(found!.idempotencyKey).toBe(idempotencyKey);
|
||||
expect(found!.associatedWaitpoint).not.toBeNull();
|
||||
expect(found!.associatedWaitpoint.id).toBe(waitpointId);
|
||||
expect(found!.associatedWaitpoint.type).toBe("RUN");
|
||||
expect(found!.associatedWaitpoint.completedByTaskRunId).toBe(runId);
|
||||
}
|
||||
);
|
||||
|
||||
// duplicate-guard contract: a run with the SAME (env, idempotencyKey, taskIdentifier)
|
||||
// exists on BOTH DBs. The per-DB unique constraint allows one row each (it cannot enforce cross-DB
|
||||
// uniqueness); the probe MUST still resolve to exactly ONE run, deterministically the NEW (run-ops id)
|
||||
// one per #findRunUnrouted (NEW-first). The duplicate itself is prevented upstream by
|
||||
// probe-before-mint plus the per-DB unique constraint; this locks the read tie-break contract.
|
||||
heteroRunOpsPostgresTest(
|
||||
"the same (env, key) on BOTH DBs resolves deterministically to the NEW run",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
// ONE logical environment id shared by both DBs (the run-ops envId is the same scalar on each).
|
||||
const legacySeed = await seedEnvironment(prisma14, "legacy", "cg2_c");
|
||||
const environmentId = legacySeed.environment.id;
|
||||
const idempotencyKey = "cg2-key-c";
|
||||
const taskIdentifier = "my-task";
|
||||
|
||||
const legacyRunId = cuidLegacy("rcl"); // cuid → LEGACY
|
||||
const newRunId = runOpsNew("rcn"); // run-ops id → NEW
|
||||
const legacyWaitpointId = cuidLegacy("wcl");
|
||||
const newWaitpointId = runOpsNew("wcn");
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyRunId,
|
||||
friendlyId: `run_friendly_cg2_c_l`,
|
||||
organizationId: legacySeed.organization.id,
|
||||
projectId: legacySeed.project.id,
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: legacyWaitpointId,
|
||||
friendlyId: `waitpoint_cg2_c_l`,
|
||||
projectId: legacySeed.project.id,
|
||||
environmentId,
|
||||
}),
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
// #new is the dedicated subset (FK-free scalar ids), so the same environmentId scalar is
|
||||
// valid there with no owning rows needed.
|
||||
runId: newRunId,
|
||||
friendlyId: `run_friendly_cg2_c_n`,
|
||||
organizationId: legacySeed.organization.id,
|
||||
projectId: legacySeed.project.id,
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
associatedWaitpoint: buildAssociatedWaitpoint({
|
||||
id: newWaitpointId,
|
||||
friendlyId: `waitpoint_cg2_c_n`,
|
||||
projectId: legacySeed.project.id,
|
||||
environmentId,
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
// Sanity: both physical DBs really do carry a row for this key.
|
||||
expect(await prisma14.taskRun.findFirst({ where: { id: legacyRunId } })).not.toBeNull();
|
||||
expect(await prisma17.taskRun.findFirst({ where: { id: newRunId } })).not.toBeNull();
|
||||
|
||||
const found = (await dedupProbe(router, {
|
||||
runtimeEnvironmentId: environmentId,
|
||||
idempotencyKey,
|
||||
taskIdentifier,
|
||||
})) as Record<string, any> | null;
|
||||
|
||||
// Exactly ONE run, deterministically the NEW one (NEW-first fan-out), with its
|
||||
// own DB's associated waitpoint hydrated.
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(newRunId);
|
||||
expect(found!.associatedWaitpoint).not.toBeNull();
|
||||
expect(found!.associatedWaitpoint.id).toBe(newWaitpointId);
|
||||
}
|
||||
);
|
||||
|
||||
// Negative: no row on either DB → null (so the trigger path proceeds to mint a fresh run).
|
||||
heteroRunOpsPostgresTest(
|
||||
"miss: an unknown (env, key) returns null from the cross-DB probe",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "cg2_miss");
|
||||
|
||||
const found = await dedupProbe(router, {
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey: "cg2-key-does-not-exist",
|
||||
taskIdentifier: "my-task",
|
||||
});
|
||||
|
||||
expect(found).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// Standalone idempotent hit (no associatedWaitpoint): the include key must still be PRESENT in the
|
||||
// result and be null, on BOTH DB homes — the andWait path lazily creates the waitpoint when this is
|
||||
// falsy, so a MISSING key (undefined) vs null must not differ.
|
||||
heteroRunOpsPostgresTest(
|
||||
"standalone: a run with no associatedWaitpoint hydrates the include key as null on #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma17, "dedicated", "cg2_sa_n");
|
||||
const runId = runOpsNew("rsan"); // run-ops id → NEW
|
||||
const idempotencyKey = "cg2-key-sa-n";
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_cg2_sa_n`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: "my-task",
|
||||
// no associatedWaitpoint
|
||||
})
|
||||
);
|
||||
|
||||
const found = (await dedupProbe(router, {
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: "my-task",
|
||||
})) as Record<string, any> | null;
|
||||
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(runId);
|
||||
expect("associatedWaitpoint" in found!).toBe(true);
|
||||
expect(found!.associatedWaitpoint).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroRunOpsPostgresTest(
|
||||
"standalone: a run with no associatedWaitpoint hydrates the include key as null on #legacy",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedEnvironment(prisma14, "legacy", "cg2_sa_l");
|
||||
const runId = cuidLegacy("rsal"); // cuid → LEGACY
|
||||
const idempotencyKey = "cg2-key-sa-l";
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_cg2_sa_l`,
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: "my-task",
|
||||
})
|
||||
);
|
||||
|
||||
const found = (await dedupProbe(router, {
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
idempotencyKey,
|
||||
taskIdentifier: "my-task",
|
||||
})) as Record<string, any> | null;
|
||||
|
||||
expect(found).not.toBeNull();
|
||||
expect(found!.id).toBe(runId);
|
||||
expect("associatedWaitpoint" in found!).toBe(true);
|
||||
expect(found!.associatedWaitpoint).toBeNull();
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,900 @@
|
||||
// MIXED-RESIDENCY MATRIX — systematic LOCK that every RoutingRunStore fan-out / partition / merge /
|
||||
// dedup method behaves correctly when cuid (#legacy) AND run-ops id (#new) data COEXIST in the SAME call,
|
||||
// against the REAL two-physical-DB split (heteroRunOpsPostgresTest: prisma14 = full/legacy on PG14,
|
||||
// prisma17 = RunOpsPrismaClient / dedicated subset on PG17). NEVER mocked.
|
||||
//
|
||||
// Existing tests exercise these methods one residency at a time or for a single specific bug. This
|
||||
// file is the cross-residency matrix: each case seeds BOTH a cuid row on #legacy AND a run-ops id row on
|
||||
// #new in one environment, then drives the wired router and asserts the merge/partition is correct.
|
||||
// The matrix MUST go RED if a fan-out leg is dropped or a NEW-wins dedup regresses (see the reverted
|
||||
// mutation probes recorded in the task report).
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id LENGTH after stripping a leading `<prefix>_`
|
||||
// (runOpsResidency.ts): 25 chars (no internal underscore) → cuid → LEGACY, a v1 body (version "1" at index 25) → run-ops id → NEW.
|
||||
// These mint a distinct classifiable id of the right length from a short seed.
|
||||
function cuidLegacy(seed: string): string {
|
||||
return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY (#legacy / prisma14)
|
||||
}
|
||||
function runOpsNew(seed: string): string {
|
||||
return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01";
|
||||
}
|
||||
|
||||
// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (run-ops rows
|
||||
// carry FK-free scalar ids), so mint synthetic owning ids. On legacy seed the real rows the kept FKs
|
||||
// require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
taskIdentifier?: string;
|
||||
status?: "PENDING" | "EXECUTING";
|
||||
spanId?: string;
|
||||
batchId?: string;
|
||||
createdAt?: Date;
|
||||
idempotencyKey?: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: params.status ?? "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: params.taskIdentifier ?? "my-task",
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: params.spanId ?? `span_${params.runId}`,
|
||||
runTags: [],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: params.createdAt ?? new Date("2024-01-01T00:00:00.000Z"),
|
||||
...(params.batchId && { batchId: params.batchId }),
|
||||
...(params.idempotencyKey && {
|
||||
idempotencyKey: params.idempotencyKey,
|
||||
idempotencyKeyExpiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
|
||||
}),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: params.status ?? "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function seedPendingWaitpoint(
|
||||
prisma: AnyClient,
|
||||
params: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
type?: "MANUAL" | "RUN" | "DATETIME";
|
||||
status?: "PENDING" | "COMPLETED";
|
||||
completedByTaskRunId?: string;
|
||||
completedByBatchId?: string;
|
||||
}
|
||||
) {
|
||||
return (prisma as PrismaClient).waitpoint.create({
|
||||
data: {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: params.type ?? "MANUAL",
|
||||
status: params.status ?? "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
...(params.completedByTaskRunId && { completedByTaskRunId: params.completedByTaskRunId }),
|
||||
...(params.completedByBatchId && { completedByBatchId: params.completedByBatchId }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
}
|
||||
|
||||
function makeLegacyStore(prisma14: PrismaClient) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
}
|
||||
|
||||
// The REAL production split topology: #new = dedicated subset on prisma17, #legacy = full schema on
|
||||
// prisma14. Two physically-distinct DBs, dedicated subset schema on #new.
|
||||
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
|
||||
const legacyStore = makeLegacyStore(prisma14);
|
||||
const newStore = makeDedicatedStore(prisma17);
|
||||
return {
|
||||
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
|
||||
legacyStore,
|
||||
newStore,
|
||||
};
|
||||
}
|
||||
|
||||
// Seed ONE logical environment whose scalar env/project/org ids are shared by both physical DBs (the
|
||||
// run-ops scalar ids are identical on each), with real owning rows on #legacy and synthetic ids on
|
||||
// #new. Returns the shared scalar ids used by every mixed-residency seed.
|
||||
async function seedSharedEnv(prisma14: PrismaClient, suffix: string) {
|
||||
const legacy = await seedEnvironment(prisma14, "legacy", suffix);
|
||||
return {
|
||||
organizationId: legacy.organization.id,
|
||||
projectId: legacy.project.id,
|
||||
runtimeEnvironmentId: legacy.environment.id,
|
||||
environmentId: legacy.environment.id,
|
||||
};
|
||||
}
|
||||
|
||||
describe("RoutingRunStore — mixed-residency matrix (cuid #legacy + run-ops id #new coexisting)", () => {
|
||||
// ── Case 1: findRuns by a MIXED bounded id-set (#findRunsByIdSet, runOpsStore.ts:294) ──
|
||||
// A list-hydrate id set spans cuid (legacy) + run-ops id (new) ids plus a run-ops id absent from legacy.
|
||||
// Both resident runs returned; take/skip applied GLOBALLY post-merge; orderBy honored; the absent
|
||||
// run-ops id short-circuits (never probed on LEGACY, :309).
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 1: findRuns by a mixed id-set returns both DBs' runs, ordered, take/skip global",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m1");
|
||||
|
||||
const legacyId = cuidLegacy("m1l"); // cuid → #legacy
|
||||
const newId = runOpsNew("m1n"); // run-ops id → #new
|
||||
const ghostRunOpsId = runOpsNew("m1g"); // run-ops id, NEVER created → tests the LEGACY short-circuit
|
||||
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyId,
|
||||
friendlyId: "run_m1_legacy",
|
||||
createdAt: new Date("2024-01-02T00:00:00.000Z"),
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: newId,
|
||||
friendlyId: "run_m1_new",
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
...env,
|
||||
})
|
||||
);
|
||||
|
||||
// Physical residency sanity: each landed on its own DB only.
|
||||
expect(await prisma14.taskRun.findUnique({ where: { id: legacyId } })).not.toBeNull();
|
||||
expect(await prisma17.taskRun.findUnique({ where: { id: legacyId } })).toBeNull();
|
||||
expect(await prisma17.taskRun.findUnique({ where: { id: newId } })).not.toBeNull();
|
||||
expect(await prisma14.taskRun.findUnique({ where: { id: newId } })).toBeNull();
|
||||
|
||||
// Full merge, ordered by createdAt asc → newId (Jan 1) before legacyId (Jan 2).
|
||||
const all = await router.findRuns({
|
||||
where: { id: { in: [legacyId, newId, ghostRunOpsId] } },
|
||||
select: { id: true, createdAt: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
expect(all.map((r) => r.id)).toEqual([newId, legacyId]);
|
||||
|
||||
// take=1 after the merge → only the first (newId). Proves take is applied GLOBALLY, not per-leg
|
||||
// (a per-leg take=1 would return one row from EACH DB → both ids).
|
||||
const firstOnly = await router.findRuns({
|
||||
where: { id: { in: [legacyId, newId, ghostRunOpsId] } },
|
||||
select: { id: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
take: 1,
|
||||
});
|
||||
expect(firstOnly.map((r) => r.id)).toEqual([newId]);
|
||||
|
||||
// skip=1 take=1 → the second (legacyId).
|
||||
const second = await router.findRuns({
|
||||
where: { id: { in: [legacyId, newId, ghostRunOpsId] } },
|
||||
select: { id: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
skip: 1,
|
||||
take: 1,
|
||||
});
|
||||
expect(second.map((r) => r.id)).toEqual([legacyId]);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 1b: NEW-wins on id collision in #findRunsByIdSet ──
|
||||
// The copy→fence window can leave the same id on both DBs. The id-set path queries NEW first; an id
|
||||
// already found on NEW must NOT be re-fetched from LEGACY, so the NEW copy wins.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 1b: findRuns by id-set with a colliding id resolves to the NEW copy",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m1b");
|
||||
|
||||
// A cuid id (LEGACY id-shape) that exists on BOTH DBs with a distinguishing field.
|
||||
const collidingId = cuidLegacy("m1b");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: collidingId, friendlyId: "run_m1b_legacy", ...env })
|
||||
); // → #legacy (cuid)
|
||||
// Force the same id onto #new with a different taskIdentifier so we can tell the copies apart.
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id: collidingId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: "run_m1b_new",
|
||||
runtimeEnvironmentId: env.environmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: env.organizationId,
|
||||
projectId: env.projectId,
|
||||
taskIdentifier: "new-copy-wins",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: "t",
|
||||
spanId: "s",
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const rows = await router.findRuns({
|
||||
where: { id: { in: [collidingId] } },
|
||||
select: { id: true, taskIdentifier: true },
|
||||
});
|
||||
expect(rows).toHaveLength(1); // deduped, not double-reported
|
||||
expect((rows[0] as any).taskIdentifier).toBe("new-copy-wins"); // NEW wins
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 2: findRuns by an OPEN predicate (#findRunsOpen, runOpsStore.ts:319) ──
|
||||
// No id set → query BOTH stores, union, dedup by id (NEW wins). Filter by a shared scalar
|
||||
// (runtimeEnvironmentId + status) that matches rows on both DBs.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 2: findRuns by an open predicate unions rows from both DBs (NEW-wins dedup)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m2");
|
||||
|
||||
const legacyId = cuidLegacy("m2l");
|
||||
const newId = runOpsNew("m2n");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyId,
|
||||
friendlyId: "run_m2_legacy",
|
||||
status: "EXECUTING",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: newId, friendlyId: "run_m2_new", status: "EXECUTING", ...env })
|
||||
);
|
||||
// A PENDING run on each DB that must be FILTERED OUT by the status predicate.
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: cuidLegacy("m2lp"),
|
||||
friendlyId: "run_m2_legacy_pending",
|
||||
status: "PENDING",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: runOpsNew("m2np"),
|
||||
friendlyId: "run_m2_new_pending",
|
||||
status: "PENDING",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
|
||||
const executing = await router.findRuns({
|
||||
where: { runtimeEnvironmentId: env.environmentId, status: "EXECUTING" },
|
||||
select: { id: true },
|
||||
orderBy: { id: "asc" },
|
||||
});
|
||||
expect(executing.map((r) => r.id).sort()).toEqual([legacyId, newId].sort());
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 3: expireRunsBatch with a MIXED id list (runOpsStore.ts:474) ──
|
||||
// Partitions run-ops id→NEW / cuid→LEGACY; each leg called only when non-empty; counts summed; each row
|
||||
// updated on its OWN DB only.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 3: expireRunsBatch partitions a mixed id list per-DB and sums the count",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m3");
|
||||
|
||||
const legacyId = cuidLegacy("m3l");
|
||||
const newId = runOpsNew("m3n");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: legacyId, friendlyId: "run_m3_legacy", ...env })
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: newId, friendlyId: "run_m3_new", ...env })
|
||||
);
|
||||
|
||||
const now = new Date("2024-03-03T00:00:00.000Z");
|
||||
const count = await router.expireRunsBatch([legacyId, newId], {
|
||||
error: { type: "STRING_ERROR", raw: "expired" },
|
||||
now,
|
||||
});
|
||||
expect(count).toBe(2); // one updated on each DB, summed
|
||||
|
||||
// Each row is EXPIRED on its OWN DB only.
|
||||
expect((await prisma14.taskRun.findUnique({ where: { id: legacyId } }))?.status).toBe(
|
||||
"EXPIRED"
|
||||
);
|
||||
expect((await prisma17.taskRun.findUnique({ where: { id: newId } }))?.status).toBe("EXPIRED");
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 4: clearIdempotencyKey fan-out arm (byFriendlyIds, runOpsStore.ts:358) ──
|
||||
// byFriendlyIds spans mixed residency → fan out to both, sum the count, each row cleared on its home.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 4: clearIdempotencyKey byFriendlyIds clears across both DBs and sums the count",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m4");
|
||||
|
||||
const legacyId = cuidLegacy("m4l");
|
||||
const newId = runOpsNew("m4n");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyId,
|
||||
friendlyId: "run_m4_legacy",
|
||||
idempotencyKey: "m4-key-legacy",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: newId,
|
||||
friendlyId: "run_m4_new",
|
||||
idempotencyKey: "m4-key-new",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
|
||||
const { count } = await router.clearIdempotencyKey({
|
||||
byFriendlyIds: ["run_m4_legacy", "run_m4_new"],
|
||||
});
|
||||
expect(count).toBe(2); // one cleared on each DB, summed
|
||||
|
||||
expect((await prisma14.taskRun.findUnique({ where: { id: legacyId } }))?.idempotencyKey).toBe(
|
||||
null
|
||||
);
|
||||
expect((await prisma17.taskRun.findUnique({ where: { id: newId } }))?.idempotencyKey).toBe(
|
||||
null
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 5: countPendingWaitpoints scattered across both DBs (runOpsStore.ts:731) ──
|
||||
// A run's pending waitpoints can be split across both stores mid-drain → count on each and sum.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 5: countPendingWaitpoints sums PENDING waitpoints scattered across both DBs",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m5");
|
||||
|
||||
const legacyWp = cuidLegacy("m5l"); // PENDING on #legacy
|
||||
const newWp = runOpsNew("m5n"); // PENDING on #new
|
||||
const completedWp = runOpsNew("m5c"); // COMPLETED on #new → must NOT be counted
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: legacyWp,
|
||||
friendlyId: "wp_m5_legacy",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: newWp,
|
||||
friendlyId: "wp_m5_new",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: completedWp,
|
||||
friendlyId: "wp_m5_completed",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
status: "COMPLETED",
|
||||
});
|
||||
|
||||
// Both PENDING ones counted (one per DB); the COMPLETED one excluded.
|
||||
expect(await router.countPendingWaitpoints([legacyWp, newWp, completedWp])).toBe(2);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 6: findManyWaitpoints { id: { in: [...mixed...] } } (runOpsStore.ts:793) ──
|
||||
// Merge waitpoints from both DBs for a mixed id set.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 6: findManyWaitpoints merges a mixed id set from both DBs",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m6");
|
||||
|
||||
const legacyWp = cuidLegacy("m6l");
|
||||
const newWp = runOpsNew("m6n");
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: legacyWp,
|
||||
friendlyId: "wp_m6_legacy",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: newWp,
|
||||
friendlyId: "wp_m6_new",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
|
||||
const found = await router.findManyWaitpoints({ where: { id: { in: [legacyWp, newWp] } } });
|
||||
expect(found.map((w) => w.id).sort()).toEqual([legacyWp, newWp].sort());
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 8: findExecutionSnapshot / findManyExecutionSnapshots OPEN (no runId) where ──
|
||||
// A by-snapshot-id-only lookup (snapshot ids are non-classifiable cuids) must fan out NEW→LEGACY
|
||||
// (findExecutionSnapshot, :675) / merge both (findManyExecutionSnapshots, :688). Seed a snapshot on
|
||||
// EACH DB (one run-ops run on #new, one cuid run on #legacy) and read with a no-runId where.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 8: findExecutionSnapshot/findManyExecutionSnapshots with an open where reach both DBs",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m8");
|
||||
|
||||
const legacyRun = cuidLegacy("m8l");
|
||||
const newRun = runOpsNew("m8n");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: legacyRun, friendlyId: "run_m8_legacy", ...env })
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: newRun, friendlyId: "run_m8_new", ...env })
|
||||
);
|
||||
|
||||
const snapEnv = {
|
||||
environmentId: env.environmentId,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
projectId: env.projectId,
|
||||
organizationId: env.organizationId,
|
||||
};
|
||||
const legacySnap = await router.createExecutionSnapshot({
|
||||
run: { id: legacyRun, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING", description: "m8 legacy snap" },
|
||||
...snapEnv,
|
||||
});
|
||||
const newSnap = await router.createExecutionSnapshot({
|
||||
run: { id: newRun, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING", description: "m8 new snap" },
|
||||
...snapEnv,
|
||||
});
|
||||
|
||||
// findExecutionSnapshot with a no-runId where targeting the LEGACY snapshot id: NEW miss → LEGACY hit.
|
||||
const foundLegacy = await router.findExecutionSnapshot({ where: { id: legacySnap.id } });
|
||||
expect(foundLegacy?.id).toBe(legacySnap.id);
|
||||
// And the NEW snapshot id resolves on the NEW leg.
|
||||
const foundNew = await router.findExecutionSnapshot({ where: { id: newSnap.id } });
|
||||
expect(foundNew?.id).toBe(newSnap.id);
|
||||
|
||||
// findManyExecutionSnapshots open where (both ids) merges both DBs.
|
||||
const many = await router.findManyExecutionSnapshots({
|
||||
where: { id: { in: [legacySnap.id, newSnap.id] } },
|
||||
});
|
||||
expect(many.map((s) => s.id).sort()).toEqual([legacySnap.id, newSnap.id].sort());
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 9a: findRun with an UNCLASSIFIABLE where (spanId) on a #legacy run (#findRunUnrouted, :213) ──
|
||||
// A run-ops run on #new and a cuid run on #legacy each carry a distinct spanId. A spanId where can't
|
||||
// be id-classified → fan out NEW-first then LEGACY. The legacy-resident run must be found.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 9a: findRun by spanId fans out and finds a #legacy run (NEW miss → LEGACY hit)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m9a");
|
||||
|
||||
const legacyRun = cuidLegacy("m9al");
|
||||
const newRun = runOpsNew("m9an");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyRun,
|
||||
friendlyId: "run_m9a_legacy",
|
||||
spanId: "span_m9a_legacy",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: newRun,
|
||||
friendlyId: "run_m9a_new",
|
||||
spanId: "span_m9a_new",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
|
||||
const onLegacy = (await router.findRun(
|
||||
{ spanId: "span_m9a_legacy" },
|
||||
{ select: { id: true } }
|
||||
)) as Record<string, any> | null;
|
||||
expect(onLegacy?.id).toBe(legacyRun);
|
||||
|
||||
const onNew = (await router.findRun(
|
||||
{ spanId: "span_m9a_new" },
|
||||
{ select: { id: true } }
|
||||
)) as Record<string, any> | null;
|
||||
expect(onNew?.id).toBe(newRun);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 9b: findRunOrThrow with an UNCLASSIFIABLE where (spanId) on a #legacy run (:593) ──
|
||||
// The throwing twin must match findRun's fan-out: an unclassifiable where whose only matching run
|
||||
// lives on #legacy must NOT throw. A NEW-only fallback would miss the legacy run and throw.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 9b: findRunOrThrow by spanId fans out and finds a #legacy run without throwing",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m9b");
|
||||
|
||||
const legacyRun = cuidLegacy("m9bl");
|
||||
const newRun = runOpsNew("m9bn");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: legacyRun,
|
||||
friendlyId: "run_m9b_legacy",
|
||||
spanId: "span_m9b_legacy",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: newRun,
|
||||
friendlyId: "run_m9b_new",
|
||||
spanId: "span_m9b_new",
|
||||
...env,
|
||||
})
|
||||
);
|
||||
|
||||
const onLegacy = (await router.findRunOrThrow(
|
||||
{ spanId: "span_m9b_legacy" },
|
||||
{ select: { id: true } }
|
||||
)) as Record<string, any>;
|
||||
expect(onLegacy.id).toBe(legacyRun);
|
||||
|
||||
const onNew = (await router.findRunOrThrow(
|
||||
{ spanId: "span_m9b_new" },
|
||||
{ select: { id: true } }
|
||||
)) as Record<string, any>;
|
||||
expect(onNew.id).toBe(newRun);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 7: findManyTaskRunWaitpoints with edges whose relations STRADDLE DBs (runOpsStore.ts:876) ──
|
||||
// An edge co-locates with its RUN, but its `waitpoint`/`taskRun` relations can live on the OTHER DB
|
||||
// (a cuid token blocking a run-ops run, and vice versa). The per-leg scalar query is stripped of the
|
||||
// relation keys; the router re-hydrates `waitpoint`/`taskRun` across BOTH DBs. Exercises BOTH
|
||||
// straddle directions in one read by querying both edges via { taskRunId: { in } }.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 7: findManyTaskRunWaitpoints rehydrates waitpoint/taskRun relations across both DBs (both straddle directions)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m7");
|
||||
|
||||
// Direction A: run-ops run on #new, blocked on a cuid token that lives ONLY on #legacy. Edge on #new.
|
||||
const newRun = runOpsNew("m7nr");
|
||||
const legacyToken = cuidLegacy("m7lt");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: newRun, friendlyId: "run_m7_new", ...env })
|
||||
);
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: legacyToken,
|
||||
friendlyId: "wp_m7_legacy_token",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
// Write the edge on #new (the run's DB) directly — the cuid token is absent from #new, so the
|
||||
// edge's `waitpoint` must be re-hydrated from #legacy.
|
||||
await prisma17.$executeRawUnsafe(
|
||||
`INSERT INTO "TaskRunWaitpoint" ("id","taskRunId","waitpointId","projectId","createdAt","updatedAt") VALUES (gen_random_uuid(),'${newRun}','${legacyToken}','${env.projectId}',NOW(),NOW())`
|
||||
);
|
||||
|
||||
// Direction B: cuid run on #legacy, blocked on a run-ops token mirrored onto BOTH DBs (drain
|
||||
// window). The #legacy copy is a STALE placeholder (PENDING) that satisfies the legacy edge FK;
|
||||
// the AUTHORITATIVE #new copy is COMPLETED. Edge on #legacy. Hydration re-resolves cross-DB and
|
||||
// NEW-wins the dedup → the edge's waitpoint must read the #new (COMPLETED) copy, not the local mirror.
|
||||
const legacyRun = cuidLegacy("m7lr");
|
||||
const newToken = runOpsNew("m7nt");
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: legacyRun, friendlyId: "run_m7_legacy", ...env })
|
||||
);
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: newToken,
|
||||
friendlyId: "wp_m7_legacy_mirror",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
status: "PENDING",
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: newToken,
|
||||
friendlyId: "wp_m7_new_token",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
status: "COMPLETED",
|
||||
});
|
||||
await prisma14.$executeRawUnsafe(
|
||||
`INSERT INTO "TaskRunWaitpoint" ("id","taskRunId","waitpointId","projectId","createdAt","updatedAt") VALUES (gen_random_uuid(),'${legacyRun}','${newToken}','${env.projectId}',NOW(),NOW())`
|
||||
);
|
||||
|
||||
// Edges sanity: each edge lives on its run's DB only.
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: newRun } })).toBe(1);
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { taskRunId: newRun } })).toBe(0);
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { taskRunId: legacyRun } })).toBe(1);
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: legacyRun } })).toBe(0);
|
||||
|
||||
// One read spanning both runs: both edges returned (deduped by id), and each edge's `waitpoint`
|
||||
// + `taskRun` re-hydrated from whichever DB holds them.
|
||||
const edges = (await router.findManyTaskRunWaitpoints({
|
||||
where: { taskRunId: { in: [newRun, legacyRun] } },
|
||||
select: {
|
||||
id: true,
|
||||
taskRunId: true,
|
||||
waitpointId: true,
|
||||
waitpoint: { select: { id: true, status: true } },
|
||||
taskRun: { select: { id: true } },
|
||||
},
|
||||
})) as Array<Record<string, any>>;
|
||||
|
||||
expect(edges).toHaveLength(2);
|
||||
const byRun = new Map(edges.map((e) => [e.taskRunId as string, e]));
|
||||
|
||||
// Direction A edge: waitpoint hydrated from #legacy (cuid token), taskRun is the #new run.
|
||||
const aEdge = byRun.get(newRun)!;
|
||||
expect(aEdge.waitpoint?.id).toBe(legacyToken);
|
||||
expect(aEdge.waitpoint?.status).toBe("PENDING");
|
||||
expect(aEdge.taskRun?.id).toBe(newRun);
|
||||
|
||||
// Direction B edge: waitpoint hydrated from the AUTHORITATIVE #new copy (COMPLETED), proving the
|
||||
// relation was re-resolved cross-DB and NEW won the dedup over the stale local #legacy mirror.
|
||||
const bEdge = byRun.get(legacyRun)!;
|
||||
expect(bEdge.waitpoint?.id).toBe(newToken);
|
||||
expect(bEdge.waitpoint?.status).toBe("COMPLETED");
|
||||
expect(bEdge.taskRun?.id).toBe(legacyRun);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 7b: the "blocking waitpoint not found on either DB" HARD ERROR (runOpsStore.ts:917) ──
|
||||
// An edge whose `waitpointId` resolves on NEITHER DB must throw rather than leave a null status that
|
||||
// would strand (hang) or wrongly unblock the run.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 7b: findManyTaskRunWaitpoints throws when a blocking waitpoint is on neither DB",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m7b");
|
||||
|
||||
const newRun = runOpsNew("m7br");
|
||||
const ghostToken = runOpsNew("m7bg"); // never created on either DB
|
||||
await router.createRun(
|
||||
buildCreateRunInput({ runId: newRun, friendlyId: "run_m7b_new", ...env })
|
||||
);
|
||||
await prisma17.$executeRawUnsafe(
|
||||
`INSERT INTO "TaskRunWaitpoint" ("id","taskRunId","waitpointId","projectId","createdAt","updatedAt") VALUES (gen_random_uuid(),'${newRun}','${ghostToken}','${env.projectId}',NOW(),NOW())`
|
||||
);
|
||||
|
||||
await expect(
|
||||
router.findManyTaskRunWaitpoints({
|
||||
where: { taskRunId: newRun },
|
||||
select: { id: true, waitpointId: true, waitpoint: { select: { status: true } } },
|
||||
})
|
||||
).rejects.toThrow(/not found on either run-ops DB/);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 10: findBatchTaskRunById / findBatchTaskRunByFriendlyId NEW-then-LEGACY probe (:1124,:1137) ──
|
||||
// A batch resident on #legacy AND a run-ops-id batch landed on #new (the control-plane window mints
|
||||
// cuid ids, but a run-ops batch resides on #new) are BOTH found via the probe, regardless of id-shape.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 10: findBatchTaskRunById/byFriendlyId probe NEW then LEGACY and find batches on either DB",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m10");
|
||||
|
||||
const legacyBatch = cuidLegacy("m10l"); // cuid → #legacy
|
||||
const newBatch = runOpsNew("m10n"); // run-ops id → #new
|
||||
await prisma14.batchTaskRun.create({
|
||||
data: {
|
||||
id: legacyBatch,
|
||||
friendlyId: "batch_m10_legacy",
|
||||
runtimeEnvironmentId: env.environmentId,
|
||||
runCount: 1,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
await prisma17.batchTaskRun.create({
|
||||
data: {
|
||||
id: newBatch,
|
||||
friendlyId: "batch_m10_new",
|
||||
runtimeEnvironmentId: env.environmentId,
|
||||
runCount: 1,
|
||||
status: "PROCESSING",
|
||||
},
|
||||
});
|
||||
|
||||
// by id: each found on its own DB via the NEW-then-LEGACY probe.
|
||||
expect((await router.findBatchTaskRunById(legacyBatch))?.id).toBe(legacyBatch);
|
||||
expect((await router.findBatchTaskRunById(newBatch))?.id).toBe(newBatch);
|
||||
|
||||
// by friendlyId (env-scoped): same probe order, both resolved.
|
||||
expect(
|
||||
(await router.findBatchTaskRunByFriendlyId("batch_m10_legacy", env.environmentId))?.id
|
||||
).toBe(legacyBatch);
|
||||
expect(
|
||||
(await router.findBatchTaskRunByFriendlyId("batch_m10_new", env.environmentId))?.id
|
||||
).toBe(newBatch);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 11a: updateManyWaitpoints with a NO-ID (batch) where fans out to both and sums (:822) ──
|
||||
// A batch where (no single routable id, e.g. completedByTaskRunId IS NULL + status PENDING) must
|
||||
// apply on BOTH DBs and sum the count.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 11a: updateManyWaitpoints with a no-id where updates both DBs and sums the count",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m11a");
|
||||
|
||||
const legacyWp = cuidLegacy("m11al");
|
||||
const newWp = runOpsNew("m11an");
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: legacyWp,
|
||||
friendlyId: "wp_m11a_legacy",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: newWp,
|
||||
friendlyId: "wp_m11a_new",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
|
||||
const { count } = await router.updateManyWaitpoints({
|
||||
where: { status: "PENDING", projectId: env.projectId },
|
||||
data: { status: "COMPLETED" },
|
||||
});
|
||||
expect(count).toBe(2); // one per DB, summed
|
||||
|
||||
expect((await prisma14.waitpoint.findUnique({ where: { id: legacyWp } }))?.status).toBe(
|
||||
"COMPLETED"
|
||||
);
|
||||
expect((await prisma17.waitpoint.findUnique({ where: { id: newWp } }))?.status).toBe(
|
||||
"COMPLETED"
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// ── Case 11b: deleteManyTaskRunWaitpoints by taskRunId fans out to both and sums (:944) ──
|
||||
// A run's edges can straddle DBs mid-drain; a delete keyed by taskRunId (not waitpointId) must
|
||||
// delete from BOTH DBs and sum the count.
|
||||
heteroRunOpsPostgresTest(
|
||||
"case 11b: deleteManyTaskRunWaitpoints by taskRunId deletes edges on both DBs and sums",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = makeSplitRouter(prisma14, prisma17);
|
||||
const env = await seedSharedEnv(prisma14, "m11b");
|
||||
|
||||
// ONE logical run id whose edges happen to exist on BOTH DBs (the straddle the fan-out guards).
|
||||
// The edge is FK-free on #new (unnest path) and FK-bound on #legacy, so seed a co-resident
|
||||
// waitpoint + run on #legacy for its edge, and write the #new edge directly.
|
||||
const runId = runOpsNew("m11br");
|
||||
const legacyToken = cuidLegacy("m11bt");
|
||||
await router.createRun(buildCreateRunInput({ runId, friendlyId: "run_m11b", ...env }));
|
||||
// #legacy needs the run + token present for the FK-bound edge insert.
|
||||
await prisma14.taskRun.create({
|
||||
data: {
|
||||
id: runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: "run_m11b_legacy_mirror",
|
||||
runtimeEnvironmentId: env.environmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: env.organizationId,
|
||||
projectId: env.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: "t",
|
||||
spanId: "s_m11b",
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
});
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: legacyToken,
|
||||
friendlyId: "wp_m11b_legacy",
|
||||
projectId: env.projectId,
|
||||
environmentId: env.environmentId,
|
||||
});
|
||||
await prisma14.$executeRawUnsafe(
|
||||
`INSERT INTO "TaskRunWaitpoint" ("id","taskRunId","waitpointId","projectId","createdAt","updatedAt") VALUES (gen_random_uuid(),'${runId}','${legacyToken}','${env.projectId}',NOW(),NOW())`
|
||||
);
|
||||
// #new edge (FK-free) pointing at a run-ops token absent locally — drain straddle.
|
||||
const newToken = runOpsNew("m11bn");
|
||||
await prisma17.$executeRawUnsafe(
|
||||
`INSERT INTO "TaskRunWaitpoint" ("id","taskRunId","waitpointId","projectId","createdAt","updatedAt") VALUES (gen_random_uuid(),'${runId}','${newToken}','${env.projectId}',NOW(),NOW())`
|
||||
);
|
||||
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(1);
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(1);
|
||||
|
||||
const { count } = await router.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } });
|
||||
expect(count).toBe(2); // one edge deleted on each DB, summed
|
||||
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(0);
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(0);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,317 @@
|
||||
// RED→GREEN repro for the run-ops split READ-AFTER-WRITE hole:
|
||||
// RoutingRunStore.findRun/findRunOrThrow dropped the caller's client and always routed the read to
|
||||
// the owning store's REPLICA (readOnlyPrisma). Read-after-write callers
|
||||
// (api.v1.sessions / api.v1.tasks.$taskId.trigger) deliberately pass the control-plane WRITER
|
||||
// (`prisma`) to read back a run they just committed and beat replica lag. Routed to the lagging
|
||||
// replica the read returned null → "Triggered run X not found" → HTTP 500.
|
||||
//
|
||||
// The fix keys on the passed client's IDENTITY: a WRITER (has `$transaction`) means read-your-writes
|
||||
// → route to the OWNING store's own writer (findRunOnPrimary), for BOTH residencies, WITHOUT leaking
|
||||
// a control-plane client into a NEW-DB query (each store reads its OWN writer). A replica / nothing
|
||||
// keeps the default (owning store's replica).
|
||||
//
|
||||
// `heteroRunOpsPostgresTest` gives a REAL split topology: prisma17 = RunOpsPrismaClient over the
|
||||
// dedicated subset schema (#new / 5434), prisma14 = full legacy schema on a SEPARATE physical PG
|
||||
// container (#legacy / control-plane). NEVER mocked. Replica lag is simulated by backing each store's
|
||||
// `readOnlyPrisma` with a recording proxy whose taskRun reads return EMPTY (a lagging replica has not
|
||||
// yet seen the fresh row) while recording that it was hit — so a replica-routed read MISSES and a
|
||||
// writer-routed read FINDS. Seeds/writes always go through the real writer.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by internal-id LENGTH: 25 chars → cuid → LEGACY, 27 → run-ops id → NEW.
|
||||
const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema)
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / prisma17, dedicated subset schema)
|
||||
|
||||
// A recording "replica" that has NOT yet caught up: its taskRun reads always come back empty and
|
||||
// record that they ran, so a replica-routed read misses the just-written row. Everything else
|
||||
// forwards to the real client. `hit` flips true iff a taskRun read was routed here.
|
||||
function laggingReplica<C extends AnyClient>(real: C): { client: C; wasHit: () => boolean } {
|
||||
let hit = false;
|
||||
const laggingTaskRun = new Proxy((real as any).taskRun, {
|
||||
get(target, prop) {
|
||||
if (prop === "findFirst" || prop === "findMany") {
|
||||
return async () => {
|
||||
hit = true;
|
||||
return prop === "findMany" ? [] : null;
|
||||
};
|
||||
}
|
||||
if (prop === "findFirstOrThrow") {
|
||||
return async () => {
|
||||
hit = true;
|
||||
throw new Error("lagging replica: row not visible");
|
||||
};
|
||||
}
|
||||
return (target as any)[prop];
|
||||
},
|
||||
});
|
||||
const client = new Proxy(real, {
|
||||
get(target, prop) {
|
||||
if (prop === "taskRun") {
|
||||
return laggingTaskRun;
|
||||
}
|
||||
return (target as any)[prop];
|
||||
},
|
||||
}) as C;
|
||||
return { client, wasHit: () => hit };
|
||||
}
|
||||
|
||||
async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function seedEnvironmentDedicated(suffix: string) {
|
||||
return {
|
||||
organization: { id: `org_${suffix}` },
|
||||
project: { id: `proj_${suffix}` },
|
||||
environment: { id: `env_${suffix}` },
|
||||
};
|
||||
}
|
||||
|
||||
function taskRunData(opts: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}) {
|
||||
return {
|
||||
id: opts.id,
|
||||
engine: "V2" as const,
|
||||
status: "PENDING" as const,
|
||||
friendlyId: opts.friendlyId,
|
||||
runtimeEnvironmentId: opts.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
organizationId: opts.organizationId,
|
||||
projectId: opts.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${opts.id}`,
|
||||
spanId: `span_${opts.id}`,
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — read-after-write reads the OWNING store's WRITER, not its lagging replica", () => {
|
||||
// (a) LEGACY-resident (cuid) run: the run was just committed to the control-plane writer; the
|
||||
// control-plane replica lags. Passing the control-plane WRITER as the read-your-writes client must
|
||||
// resolve the run via the owning (legacy) writer, NOT the replica.
|
||||
heteroRunOpsPostgresTest(
|
||||
"LEGACY cuid: read-after-write via the control-plane WRITER finds the fresh run despite replica lag",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const legacyReplica = laggingReplica(prisma14);
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: legacyReplica.client,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
const seed = await seedEnvironmentLegacy(prisma14, "raw_leg");
|
||||
const runId = `run_${CUID_25}`; // cuid → LEGACY
|
||||
await prisma14.taskRun.create({
|
||||
data: taskRunData({
|
||||
id: runId,
|
||||
friendlyId: "run_raw_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
}),
|
||||
});
|
||||
|
||||
// FAIL-BEFORE proof: a plain replica read (no client) hits the lagging replica → miss.
|
||||
const viaReplica = await router.findRun(
|
||||
{ id: runId },
|
||||
{ select: { friendlyId: true } }
|
||||
// no client → default replica
|
||||
);
|
||||
expect(viaReplica).toBeNull();
|
||||
expect(legacyReplica.wasHit()).toBe(true);
|
||||
|
||||
// PASS-AFTER: read-your-writes with the control-plane WRITER resolves the fresh run.
|
||||
const legacyReplica2 = laggingReplica(prisma14);
|
||||
const legacyStore2 = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: legacyReplica2.client,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const router2 = new RoutingRunStore({ new: newStore, legacy: legacyStore2 });
|
||||
const viaWriter = await router2.findRun(
|
||||
{ id: runId },
|
||||
{ select: { friendlyId: true } },
|
||||
prisma14 // control-plane WRITER → read-your-writes
|
||||
);
|
||||
expect(viaWriter).not.toBeNull();
|
||||
expect((viaWriter as { friendlyId: string }).friendlyId).toBe("run_raw_leg");
|
||||
// The read hit the WRITER, never the replica.
|
||||
expect(legacyReplica2.wasHit()).toBe(false);
|
||||
|
||||
// findRunOrThrow: same behavior — writer resolves, replica would have thrown.
|
||||
const legacyReplica3 = laggingReplica(prisma14);
|
||||
const legacyStore3 = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: legacyReplica3.client,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const router3 = new RoutingRunStore({ new: newStore, legacy: legacyStore3 });
|
||||
const orThrow = await router3.findRunOrThrow(
|
||||
{ id: runId },
|
||||
{ select: { friendlyId: true } },
|
||||
prisma14
|
||||
);
|
||||
expect((orThrow as { friendlyId: string }).friendlyId).toBe("run_raw_leg");
|
||||
expect(legacyReplica3.wasHit()).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
// (b) NEW-resident (run-ops id) run: born on the NEW DB (5434). The NEW replica lags. Passing the NEW
|
||||
// WRITER as the read-your-writes client must resolve the run via the NEW writer, NOT its replica —
|
||||
// and (proving the constraint that motivated the original client-drop) the control-plane writer is
|
||||
// never leaked into the NEW query: each store reads its OWN writer.
|
||||
heteroRunOpsPostgresTest(
|
||||
"NEW run-ops id: read-after-write via the NEW WRITER finds the fresh run despite NEW replica lag",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newReplica = laggingReplica(prisma17);
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: newReplica.client as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
const seed = seedEnvironmentDedicated("raw_new");
|
||||
const runId = `run_${NEW_ID_26}`; // run-ops id → NEW
|
||||
await prisma17.taskRun.create({
|
||||
data: taskRunData({
|
||||
id: runId,
|
||||
friendlyId: "run_raw_new",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
}),
|
||||
});
|
||||
|
||||
// FAIL-BEFORE proof: a plain replica read hits the lagging NEW replica → miss.
|
||||
const viaReplica = await router.findRun({ id: runId }, { select: { friendlyId: true } });
|
||||
expect(viaReplica).toBeNull();
|
||||
expect(newReplica.wasHit()).toBe(true);
|
||||
|
||||
// PASS-AFTER: read-your-writes with the NEW WRITER resolves the fresh run on the NEW DB.
|
||||
const newReplica2 = laggingReplica(prisma17);
|
||||
const newStore2 = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: newReplica2.client as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const router2 = new RoutingRunStore({ new: newStore2, legacy: legacyStore });
|
||||
const viaWriter = await router2.findRun(
|
||||
{ id: runId },
|
||||
{ select: { friendlyId: true } },
|
||||
prisma17 as never // NEW WRITER → read-your-writes
|
||||
);
|
||||
expect(viaWriter).not.toBeNull();
|
||||
expect((viaWriter as { friendlyId: string }).friendlyId).toBe("run_raw_new");
|
||||
// The read hit the NEW WRITER, never the NEW replica.
|
||||
expect(newReplica2.wasHit()).toBe(false);
|
||||
|
||||
// Even passing the LEGACY (control-plane) WRITER as the read-your-writes signal resolves the
|
||||
// run-ops run: the router routes by residency to the NEW store's OWN writer, never forwarding the
|
||||
// control-plane client into the NEW DB. (This is the exact live shape — sessions/trigger pass
|
||||
// the control-plane `prisma`, and the run may be NEW-resident under split-ON.)
|
||||
const newReplica3 = laggingReplica(prisma17);
|
||||
const newStore3 = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: newReplica3.client as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const router3 = new RoutingRunStore({ new: newStore3, legacy: legacyStore });
|
||||
const viaControlPlaneWriter = await router3.findRun(
|
||||
{ id: runId },
|
||||
{ select: { friendlyId: true } },
|
||||
prisma14 // control-plane WRITER (writer identity) — router routes to NEW's own writer
|
||||
);
|
||||
expect((viaControlPlaneWriter as { friendlyId: string }).friendlyId).toBe("run_raw_new");
|
||||
expect(newReplica3.wasHit()).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
// Guard: a plain replica read (no client, or a replica client) still routes to the replica — the
|
||||
// fix must not turn every read into a primary read (which would defeat replica offload).
|
||||
heteroRunOpsPostgresTest(
|
||||
"plain reads still route to the replica (no read-your-writes escalation)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const legacyReplica = laggingReplica(prisma14);
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: legacyReplica.client,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
const seed = await seedEnvironmentLegacy(prisma14, "plain_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await prisma14.taskRun.create({
|
||||
data: taskRunData({
|
||||
id: runId,
|
||||
friendlyId: "run_plain_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
}),
|
||||
});
|
||||
|
||||
await router.findRun({ id: runId }, { select: { friendlyId: true } });
|
||||
// No writer passed → the read went to the replica, exactly as before the fix.
|
||||
expect(legacyReplica.wasHit()).toBe(true);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,377 @@
|
||||
// RED→GREEN repro for the routed-read CLIENT DROP: RoutingRunStore's runId-routed / fan-out reads
|
||||
// accept `client?: ReadClient` but dropped it, so the sub-store fell back to its REPLICA. The
|
||||
// run-engine passes its writer (`tx ?? this.$.prisma`) into these reads for read-your-writes
|
||||
// consistency (dequeue re-reads the just-written QUEUED snapshot), so the drop surfaces in cloud as
|
||||
// TASK_DEQUEUED_INVALID_STATE / "No execution snapshot found for TaskRun ...". The fix routes a
|
||||
// caller-passed client to the OWNING store's OWN primary (never forwarded verbatim — it is bound to
|
||||
// the control-plane DB); no client keeps the replica default.
|
||||
//
|
||||
// Deterministic harness: `heteroPostgresTest` hands two PHYSICALLY separate postgres containers
|
||||
// over the same full schema. The owning store WRITES to one and its `readOnlyPrisma` points at the
|
||||
// other, which stays EMPTY (a replica with unbounded lag) — so a replica-routed read MISSES and a
|
||||
// primary-routed read finds the row, with no replica==primary aliasing to mask the drop.
|
||||
|
||||
import { heteroPostgresTest } from "@internal/testcontainers";
|
||||
import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import { markReadReplicaClient } from "./readReplicaClient.js";
|
||||
import type { CreateRunInput } from "./types.js";
|
||||
|
||||
// ownerEngine classifies by the version char: a 25-char cuid → LEGACY; a valid run-ops v1 body
|
||||
// (26 chars: base32hex core + region char + version "1") → NEW.
|
||||
const CUID_25 = "c".repeat(25);
|
||||
const RUN_OPS_ID_BODY = generateRunOpsId();
|
||||
|
||||
// Router topology where the OWNING store (the one the test's run ids route to) writes to `writer`
|
||||
// but reads by default from `lagging` — a physically separate, never-written DB. The other store
|
||||
// lives entirely on the lagging DB so fan-out legs can't accidentally see rows. Both DBs carry the
|
||||
// full schema (the forwarding under test is residency-agnostic; dedicated-subset parity is covered
|
||||
// by the sibling suites), so both stores use the "legacy" variant.
|
||||
function splitTopology(
|
||||
residency: "LEGACY" | "NEW",
|
||||
writer: PrismaClient,
|
||||
lagging: PrismaClient
|
||||
): { owningStore: PostgresRunStore; router: RoutingRunStore } {
|
||||
const owningStore = new PostgresRunStore({
|
||||
prisma: writer,
|
||||
readOnlyPrisma: lagging,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const otherStore = new PostgresRunStore({
|
||||
prisma: lagging,
|
||||
readOnlyPrisma: lagging,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const router = new RoutingRunStore(
|
||||
residency === "LEGACY"
|
||||
? { new: otherStore, legacy: owningStore }
|
||||
: { new: owningStore, legacy: otherStore }
|
||||
);
|
||||
return { owningStore, router };
|
||||
}
|
||||
|
||||
async function seedEnvironment(prisma: PrismaClient, suffix: string) {
|
||||
const organization = await prisma.organization.create({
|
||||
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
|
||||
});
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
name: `Project ${suffix}`,
|
||||
slug: `project-${suffix}`,
|
||||
externalRef: `proj_${suffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await prisma.runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${suffix}`,
|
||||
pkApiKey: `pk_dev_${suffix}`,
|
||||
shortcode: `short_${suffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: "my-task",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceContext: {},
|
||||
traceId: `trace_${params.runId}`,
|
||||
spanId: `span_${params.runId}`,
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("run-ops split — routed reads honor a caller-passed client via the owning store's PRIMARY", () => {
|
||||
// The outage path: dequeue writes the QUEUED snapshot then re-reads it via
|
||||
// `getLatestExecutionSnapshot(this.$.prisma, ...)`. The router must not downgrade that read to
|
||||
// the replica. Covers the whole snapshot read family on the LEGACY (cuid) routing arm.
|
||||
heteroPostgresTest(
|
||||
"LEGACY cuid: snapshot reads with a client resolve on the owning primary; without, on the replica",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = splitTopology("LEGACY", prisma14, prisma17);
|
||||
const seed = await seedEnvironment(prisma14, "snap_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_snap_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// findLatestExecutionSnapshot — client passed → owning primary finds the fresh snapshot.
|
||||
const latest = await router.findLatestExecutionSnapshot(runId, prisma14);
|
||||
expect(latest).not.toBeNull();
|
||||
expect(latest?.executionStatus).toBe("RUN_CREATED");
|
||||
// No client → the (empty) replica, unchanged behavior.
|
||||
expect(await router.findLatestExecutionSnapshot(runId)).toBeNull();
|
||||
|
||||
const snapshotId = latest!.id;
|
||||
|
||||
// findExecutionSnapshot (runId-routed, the warm-restart shape).
|
||||
const one = await router.findExecutionSnapshot(
|
||||
{ where: { runId, isValid: true }, select: { id: true } },
|
||||
prisma14
|
||||
);
|
||||
expect(one?.id).toBe(snapshotId);
|
||||
expect(await router.findExecutionSnapshot({ where: { runId, isValid: true } })).toBeNull();
|
||||
|
||||
// findManyExecutionSnapshots (runId-routed).
|
||||
const many = await router.findManyExecutionSnapshots(
|
||||
{ where: { runId }, select: { id: true } },
|
||||
prisma14
|
||||
);
|
||||
expect(many.map((s) => s.id)).toEqual([snapshotId]);
|
||||
expect(await router.findManyExecutionSnapshots({ where: { runId } })).toEqual([]);
|
||||
|
||||
// findSnapshotCompletedWaitpointIds (the resume-payload join read). Seed a completed
|
||||
// waitpoint + its `_completedWaitpoints` join on the writer only.
|
||||
const waitpoint = await prisma14.waitpoint.create({
|
||||
data: {
|
||||
friendlyId: "waitpoint_snap_leg",
|
||||
type: "MANUAL",
|
||||
status: "COMPLETED",
|
||||
idempotencyKey: "idem_snap_leg",
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: seed.project.id,
|
||||
environmentId: seed.environment.id,
|
||||
},
|
||||
});
|
||||
// Link via the Prisma relation API (not a raw insert into the implicit join table) so a
|
||||
// relation rename fails at compile time rather than silently seeding nothing.
|
||||
await prisma14.taskRunExecutionSnapshot.update({
|
||||
where: { id: snapshotId },
|
||||
data: { completedWaitpoints: { connect: { id: waitpoint.id } } },
|
||||
});
|
||||
expect(await router.findSnapshotCompletedWaitpointIds(snapshotId, prisma14)).toEqual([
|
||||
waitpoint.id,
|
||||
]);
|
||||
expect(await router.findSnapshotCompletedWaitpointIds(snapshotId)).toEqual([]);
|
||||
}
|
||||
);
|
||||
|
||||
// NEW (run-ops id) routing arm. The caller's client here is the CONTROL-PLANE writer — the wrong
|
||||
// physical DB for a NEW-resident run — so this also pins that the client is never forwarded
|
||||
// verbatim: the read must resolve on the owning NEW store's OWN primary.
|
||||
heteroPostgresTest(
|
||||
"NEW run-ops id: a control-plane client routes the snapshot read to the NEW store's OWN primary",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
// Owning (NEW) store writes to prisma14; the control-plane/other store is prisma17.
|
||||
const { router } = splitTopology("NEW", prisma14, prisma17);
|
||||
const seed = await seedEnvironment(prisma14, "snap_new");
|
||||
const runId = `run_${RUN_OPS_ID_BODY}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_snap_new",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// Control-plane writer (prisma17 side of this topology) passed as the client: the row can
|
||||
// only be found on the NEW store's own primary (prisma14) — verbatim forwarding would miss.
|
||||
const latest = await router.findLatestExecutionSnapshot(runId, prisma17);
|
||||
expect(latest).not.toBeNull();
|
||||
expect(latest?.executionStatus).toBe("RUN_CREATED");
|
||||
// No client → the NEW store's (empty) replica.
|
||||
expect(await router.findLatestExecutionSnapshot(runId)).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"LEGACY cuid: findRuns fan-out and batch friendlyId probe honor a caller client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = splitTopology("LEGACY", prisma14, prisma17);
|
||||
const seed = await seedEnvironment(prisma14, "runs_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_runs_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// findRuns (bounded id-set fan-out).
|
||||
const rows = await router.findRuns(
|
||||
{ where: { id: { in: [runId] } }, select: { id: true } },
|
||||
prisma14
|
||||
);
|
||||
expect(rows.map((r) => r.id)).toEqual([runId]);
|
||||
expect(
|
||||
await router.findRuns({ where: { id: { in: [runId] } }, select: { id: true } })
|
||||
).toEqual([]);
|
||||
|
||||
// findBatchTaskRunByFriendlyId (env-scoped fan-out probe; the one batch read that defaults
|
||||
// to the replica).
|
||||
const batch = await prisma14.batchTaskRun.create({
|
||||
data: {
|
||||
id: `batch_${CUID_25}`,
|
||||
friendlyId: "batch_runs_leg",
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
},
|
||||
});
|
||||
const viaPrimary = await router.findBatchTaskRunByFriendlyId(
|
||||
batch.friendlyId,
|
||||
seed.environment.id,
|
||||
undefined,
|
||||
prisma14
|
||||
);
|
||||
expect(viaPrimary?.id).toBe(batch.id);
|
||||
expect(
|
||||
await router.findBatchTaskRunByFriendlyId(batch.friendlyId, seed.environment.id)
|
||||
).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// Read scaling: a caller that passes an explicit READ REPLICA (e.g. `$replica`) must NOT be
|
||||
// escalated to the primary — only true read-your-writes (a writer/tx) should. A replica is a
|
||||
// full PrismaClient at runtime (it has `$transaction` too), so shape can't distinguish it; the
|
||||
// client builder brands it and the router honors the brand. Proven here by branding a client and
|
||||
// showing the read stays on the owning store's (empty) replica — same as passing no client —
|
||||
// while an unbranded writer escalates and finds the fresh row.
|
||||
heteroPostgresTest(
|
||||
"a branded read-replica client stays on the replica; a writer escalates to the primary",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = splitTopology("LEGACY", prisma14, prisma17);
|
||||
const seed = await seedEnvironment(prisma14, "replica_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_replica_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// The router discards the caller's client object (it can't cross DBs) and reads the brand
|
||||
// only, so a branded marker faithfully stands in for a passed `$replica`.
|
||||
const replicaClient = markReadReplicaClient({} as unknown as PrismaClient);
|
||||
|
||||
// findRun (readYourWrites path): branded replica → owning replica (empty) → miss.
|
||||
expect(
|
||||
await router.findRun({ id: runId }, { select: { id: true } }, replicaClient)
|
||||
).toBeNull();
|
||||
// Control: an unbranded writer escalates to the owning primary → finds the fresh row.
|
||||
expect((await router.findRun({ id: runId }, { select: { id: true } }, prisma14))?.id).toBe(
|
||||
runId
|
||||
);
|
||||
|
||||
// findLatestExecutionSnapshot (#ownPrimary path): same replica-stays-on-replica invariant.
|
||||
expect(await router.findLatestExecutionSnapshot(runId, replicaClient)).toBeNull();
|
||||
expect((await router.findLatestExecutionSnapshot(runId, prisma14))?.executionStatus).toBe(
|
||||
"RUN_CREATED"
|
||||
);
|
||||
// No client behaves identically to the branded replica.
|
||||
expect(await router.findLatestExecutionSnapshot(runId)).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
heteroPostgresTest(
|
||||
"LEGACY cuid: waitpoint reads honor a caller client",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const { router } = splitTopology("LEGACY", prisma14, prisma17);
|
||||
const seed = await seedEnvironment(prisma14, "wp_leg");
|
||||
const runId = `run_${CUID_25}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_wp_leg",
|
||||
organizationId: seed.organization.id,
|
||||
projectId: seed.project.id,
|
||||
runtimeEnvironmentId: seed.environment.id,
|
||||
})
|
||||
);
|
||||
const waitpoint = await prisma14.waitpoint.create({
|
||||
data: {
|
||||
id: `waitpoint_${CUID_25}`,
|
||||
friendlyId: "waitpoint_wp_leg",
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: "idem_wp_leg",
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: seed.project.id,
|
||||
environmentId: seed.environment.id,
|
||||
},
|
||||
});
|
||||
await prisma14.taskRunWaitpoint.create({
|
||||
data: { taskRunId: runId, waitpointId: waitpoint.id, projectId: seed.project.id },
|
||||
});
|
||||
|
||||
// findWaitpoint (by id, resolve + scalar read).
|
||||
const found = await router.findWaitpoint({ where: { id: waitpoint.id } }, prisma14);
|
||||
expect(found?.id).toBe(waitpoint.id);
|
||||
expect(await router.findWaitpoint({ where: { id: waitpoint.id } })).toBeNull();
|
||||
|
||||
// findManyWaitpoints (both-store fan-out).
|
||||
const manyOnPrimary = await router.findManyWaitpoints(
|
||||
{ where: { id: { in: [waitpoint.id] } } },
|
||||
prisma14
|
||||
);
|
||||
expect(manyOnPrimary.map((w) => w.id)).toEqual([waitpoint.id]);
|
||||
expect(await router.findManyWaitpoints({ where: { id: { in: [waitpoint.id] } } })).toEqual(
|
||||
[]
|
||||
);
|
||||
|
||||
// countPendingWaitpoints (both-store fan-out sum).
|
||||
expect(await router.countPendingWaitpoints([waitpoint.id], prisma14)).toBe(1);
|
||||
expect(await router.countPendingWaitpoints([waitpoint.id])).toBe(0);
|
||||
|
||||
// findManyTaskRunWaitpoints (the blocked-run edge fan-out).
|
||||
const edges = await router.findManyTaskRunWaitpoints(
|
||||
{ where: { taskRunId: runId } },
|
||||
prisma14
|
||||
);
|
||||
expect(edges).toHaveLength(1);
|
||||
expect(edges[0]?.waitpointId).toBe(waitpoint.id);
|
||||
expect(await router.findManyTaskRunWaitpoints({ where: { taskRunId: runId } })).toEqual([]);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
// Managed resume reads a run's completed waitpoints by SNAPSHOT id
|
||||
// (getExecutionSnapshotsSince -> getSnapshotWaitpointIds -> findSnapshotCompletedWaitpointIds).
|
||||
// Snapshot ids are @default(cuid()), so #routeOrNew(snapshotId) always classifies LEGACY. For a
|
||||
// NEW-residency run the snapshot's CompletedWaitpoint join rows live on #new, so routing to #legacy
|
||||
// returns [] and the resumed run sees zero completed waitpoints and hangs. The fix fans out across
|
||||
// both stores and merges, like findWaitpointCompletedSnapshotIds. Real two-DB topology; never mocked.
|
||||
|
||||
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
|
||||
// cuid-shaped snapshot id -> classifies LEGACY (the always-wrong routing key).
|
||||
const SNAPSHOT_CUID = "c".repeat(25);
|
||||
const WAITPOINT_ID = "waitpoint_" + "n".repeat(20);
|
||||
|
||||
describe("run-ops split — completed waitpoints for a cuid snapshot are found on the owning store, not misrouted to legacy", () => {
|
||||
heteroRunOpsPostgresTest(
|
||||
"findSnapshotCompletedWaitpointIds finds a NEW-resident join despite the cuid snapshot id",
|
||||
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const legacyStore = new PostgresRunStore({
|
||||
prisma: prisma14,
|
||||
readOnlyPrisma: prisma14,
|
||||
schemaVariant: "legacy",
|
||||
});
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
// A NEW-residency run's completed-waitpoint join lives on #new; its snapshot id is a cuid.
|
||||
await prisma17.completedWaitpoint.create({
|
||||
data: { snapshotId: SNAPSHOT_CUID, waitpointId: WAITPOINT_ID },
|
||||
});
|
||||
|
||||
const ids = await router.findSnapshotCompletedWaitpointIds(SNAPSHOT_CUID);
|
||||
|
||||
// RED: the cuid snapshot id routes to #legacy -> [] -> resumed run never completes its waitpoints.
|
||||
// GREEN: fan-out finds the #new-resident join.
|
||||
expect(ids).toEqual([WAITPOINT_ID]);
|
||||
}
|
||||
);
|
||||
|
||||
// WithPresence reports, in one read, whether the snapshot is visible on the reader (so a multi-reader
|
||||
// replica can distinguish "no waitpoints" from "reader has not applied the snapshot yet").
|
||||
heteroRunOpsPostgresTest(
|
||||
"findSnapshotCompletedWaitpointIdsWithPresence reports present+ids for a dedicated snapshot, absent otherwise",
|
||||
async ({ prisma17 }: { prisma17: RunOpsPrismaClient }) => {
|
||||
const newStore = new PostgresRunStore({
|
||||
prisma: prisma17 as never,
|
||||
readOnlyPrisma: prisma17 as never,
|
||||
schemaVariant: "dedicated",
|
||||
});
|
||||
const runId = "run_" + "k".repeat(24) + "01";
|
||||
await prisma17.taskRun.create({
|
||||
data: {
|
||||
id: runId,
|
||||
friendlyId: "run_pres",
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
taskIdentifier: "t",
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
traceId: "tr",
|
||||
spanId: "sp",
|
||||
queue: "q",
|
||||
runtimeEnvironmentId: "env_pres",
|
||||
projectId: "proj_pres",
|
||||
},
|
||||
});
|
||||
const snap = await prisma17.taskRunExecutionSnapshot.create({
|
||||
data: {
|
||||
id: "c".repeat(25),
|
||||
engine: "V2",
|
||||
executionStatus: "EXECUTING",
|
||||
description: "continue",
|
||||
runStatus: "PENDING",
|
||||
runId,
|
||||
environmentId: "env_pres",
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: "proj_pres",
|
||||
organizationId: "org_pres",
|
||||
},
|
||||
});
|
||||
await prisma17.completedWaitpoint.create({
|
||||
data: { snapshotId: snap.id, waitpointId: WAITPOINT_ID },
|
||||
});
|
||||
|
||||
expect(await newStore.findSnapshotCompletedWaitpointIdsWithPresence(snap.id)).toEqual({
|
||||
present: true,
|
||||
ids: [WAITPOINT_ID],
|
||||
});
|
||||
// A snapshot the reader does not have -> present:false, so its empty ids are not trusted as authoritative.
|
||||
expect(
|
||||
await newStore.findSnapshotCompletedWaitpointIdsWithPresence("c".repeat(24) + "zz")
|
||||
).toEqual({ present: false, ids: [] });
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,363 @@
|
||||
// RunStore run-ops persistence — snapshots, against the REAL dedicated split topology.
|
||||
//
|
||||
// `heteroRunOpsPostgresTest` gives prisma14 = the full control-plane schema (#legacy) and
|
||||
// prisma17 = a real `RunOpsPrismaClient` over the @internal/run-ops-database SUBSET schema (#new).
|
||||
// These were previously on the weaker `heteroPostgresTest` (full schema on BOTH sides), which could
|
||||
// not catch dedicated-subset behaviour differences — the entire point of the split. On the subset
|
||||
// there are no Organization/Project/RuntimeEnvironment models and no implicit M2M join tables
|
||||
// (`_completedWaitpoints` is the explicit `CompletedWaitpoint` model), so the snapshot store must
|
||||
// behave identically whether backed by the legacy implicit M2M or the dedicated explicit join.
|
||||
//
|
||||
// The assertions still compare the store's behaviour across the two physical DBs (control-plane vs
|
||||
// dedicated): a snapshot created + read through the store yields the same observable result on both.
|
||||
|
||||
import { heteroRunOpsPostgresTest, HETERO_PINNED_ICU_COLLATION } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops
|
||||
// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows
|
||||
// the kept FKs require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
slugSuffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${slugSuffix}` },
|
||||
project: { id: `proj_${slugSuffix}` },
|
||||
environment: { id: `env_${slugSuffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${slugSuffix}`,
|
||||
slug: `project-${slugSuffix}`,
|
||||
externalRef: `proj_${slugSuffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${slugSuffix}`,
|
||||
pkApiKey: `pk_dev_${slugSuffix}`,
|
||||
shortcode: `short_${slugSuffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
// ownerEngine classifies by the version char after stripping a single leading `<prefix>_`: a v1 body
|
||||
// → run-ops id → NEW (#new / dedicated run-ops DB subset), 25 chars → cuid → LEGACY (#legacy / full schema).
|
||||
const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW residency, exercises the dedicated store
|
||||
const CUID_25 = "c".repeat(25); // → LEGACY residency, exercises the full-schema store
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
taskIdentifier: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: "trace_1",
|
||||
spanId: "span_1",
|
||||
runTags: ["alpha", "beta"],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function seedPendingWaitpoint(
|
||||
prisma: AnyClient,
|
||||
params: { id: string; friendlyId: string; projectId: string; environmentId: string }
|
||||
) {
|
||||
return (prisma as PrismaClient).waitpoint.create({
|
||||
data: {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeStore(prisma: AnyClient, schemaVariant: RunStoreSchemaVariant) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma as never,
|
||||
readOnlyPrisma: prisma as never,
|
||||
schemaVariant,
|
||||
});
|
||||
}
|
||||
|
||||
// Strip the prisma-managed / per-DB id fields so two rows born on different physical DBs
|
||||
// (legacy full schema vs dedicated subset) compare field-for-field for behavioural parity.
|
||||
function normalizeSnapshot(row: Record<string, unknown>) {
|
||||
const r = { ...row };
|
||||
delete r.id;
|
||||
delete r.runId;
|
||||
delete r.previousSnapshotId;
|
||||
delete r.createdAt;
|
||||
delete r.updatedAt;
|
||||
delete r.environmentId;
|
||||
delete r.projectId;
|
||||
delete r.organizationId;
|
||||
return r;
|
||||
}
|
||||
|
||||
describe("RunStore run-ops persistence — snapshots", () => {
|
||||
// an identical run + ≥2 snapshots (one invalid, one valid) seeded on #legacy (full schema)
|
||||
// and #new (dedicated subset) yield a deep-equal `findLatestExecutionSnapshot` row, and it is the
|
||||
// valid one — proving the dedicated store's group-A hydration does not perturb the scalar columns.
|
||||
heteroRunOpsPostgresTest(
|
||||
"snapshot findLatest is behaviourally identical across #legacy and #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const seed = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_latest_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
const ids = {
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT" as const,
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
};
|
||||
|
||||
// An invalid snapshot (error set) that must NOT be returned by findLatest.
|
||||
await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING", description: "invalid one" },
|
||||
error: "boom",
|
||||
...ids,
|
||||
});
|
||||
// The valid snapshot created last — this is the one findLatest must return.
|
||||
const valid = await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING_WITH_WAITPOINTS", description: "valid latest" },
|
||||
...ids,
|
||||
});
|
||||
return { store, validId: valid.id };
|
||||
};
|
||||
|
||||
const legacyRunId = `run_${CUID_25}`; // → #legacy (full schema)
|
||||
const newRunId = `run_${NEW_ID_26}`; // → #new (dedicated subset)
|
||||
const seed14 = await seed(prisma14, "legacy", legacyRunId, "sa14");
|
||||
const seed17 = await seed(prisma17, "dedicated", newRunId, "sa17");
|
||||
|
||||
const latest14 = await seed14.store.findLatestExecutionSnapshot(legacyRunId);
|
||||
const latest17 = await seed17.store.findLatestExecutionSnapshot(newRunId);
|
||||
|
||||
expect(latest14).not.toBeNull();
|
||||
expect(latest17).not.toBeNull();
|
||||
// The valid snapshot wins over the earlier invalid one.
|
||||
expect(latest14!.id).toBe(seed14.validId);
|
||||
expect(latest17!.id).toBe(seed17.validId);
|
||||
expect(latest14!.isValid).toBe(true);
|
||||
expect(latest14!.description).toBe("valid latest");
|
||||
expect(latest17!.isValid).toBe(true);
|
||||
expect(latest17!.description).toBe("valid latest");
|
||||
|
||||
// Compare the persisted columns (drop relation arrays + per-DB ids). The dedicated store
|
||||
// hydrates `completedWaitpoints` from the explicit CompletedWaitpoint join, the legacy store
|
||||
// from the implicit M2M — both stripped here, leaving the scalar columns to compare.
|
||||
const strip = (
|
||||
row: NonNullable<Awaited<ReturnType<PostgresRunStore["findLatestExecutionSnapshot"]>>>
|
||||
) => {
|
||||
const { completedWaitpoints, checkpoint, ...rest } = row;
|
||||
return normalizeSnapshot(rest as Record<string, unknown>);
|
||||
};
|
||||
expect(strip(latest14!)).toEqual(strip(latest17!));
|
||||
}
|
||||
);
|
||||
|
||||
// completedWaitpoints round-trips through the join (implicit `_completedWaitpoints` on legacy,
|
||||
// explicit `CompletedWaitpoint` on the dedicated subset), and the derived completedWaitpointOrder
|
||||
// preserves the supplied index order, on both stores.
|
||||
heteroRunOpsPostgresTest(
|
||||
"completedWaitpoints round-trip preserves order across #legacy and #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const run = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_cw_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
const w1 = `wp_${suffix}_1`;
|
||||
const w2 = `wp_${suffix}_2`;
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w1,
|
||||
friendlyId: `waitpoint_${suffix}_1`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w2,
|
||||
friendlyId: `waitpoint_${suffix}_2`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
const snapshot = await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: {
|
||||
executionStatus: "EXECUTING_WITH_WAITPOINTS",
|
||||
description: "with waitpoints",
|
||||
},
|
||||
completedWaitpoints: [
|
||||
{ id: w1, index: 0 },
|
||||
{ id: w2, index: 1 },
|
||||
],
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
});
|
||||
|
||||
const joinIds = await store.findSnapshotCompletedWaitpointIds(snapshot.id);
|
||||
return { w1, w2, joinIds, order: snapshot.completedWaitpointOrder };
|
||||
};
|
||||
|
||||
const r14 = await run(prisma14, "legacy", `run_${CUID_25}`, "sb14");
|
||||
const r17 = await run(prisma17, "dedicated", `run_${NEW_ID_26}`, "sb17");
|
||||
|
||||
// The join links the snapshot to both waitpoints (set-equal) on both stores.
|
||||
expect([...r14.joinIds].sort()).toEqual([r14.w1, r14.w2].sort());
|
||||
expect([...r17.joinIds].sort()).toEqual([r17.w1, r17.w2].sort());
|
||||
|
||||
// The derived order column reflects the supplied index order, identically per store.
|
||||
expect(r14.order).toEqual([r14.w1, r14.w2]);
|
||||
expect(r17.order).toEqual([r17.w1, r17.w2]);
|
||||
}
|
||||
);
|
||||
|
||||
// a collation-sensitive ORDER BY over a text column pinned to the shared ICU collation
|
||||
// (`und-x-icu`, present on both the #legacy container and the #new container) returns the
|
||||
// identical sequence of snapshot descriptions on #legacy and #new. The pin keeps the comparison a
|
||||
// proof of the split rather than of a default-collation difference between the two DBs.
|
||||
heteroRunOpsPostgresTest(
|
||||
"snapshot ORDER BY pinned to the shared ICU collation is identical across #legacy and #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const descriptions = ["Zebra", "apple", "Apple", "éclair", "banana", "_underscore"];
|
||||
|
||||
const seed = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_order_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
for (const description of descriptions) {
|
||||
await store.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING", description },
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
await seed(prisma14, "legacy", `run_${CUID_25}`, "sc14");
|
||||
await seed(prisma17, "dedicated", `run_${NEW_ID_26}`, "sc17");
|
||||
|
||||
const orderedDescriptions = async (client: AnyClient) => {
|
||||
const rows = await (client as PrismaClient).$queryRawUnsafe<{ description: string }[]>(
|
||||
`SELECT "description" FROM "TaskRunExecutionSnapshot" WHERE "description" != 'Run was created' ORDER BY "description" COLLATE "${HETERO_PINNED_ICU_COLLATION}" ASC`
|
||||
);
|
||||
return rows.map((r) => r.description);
|
||||
};
|
||||
|
||||
const ordered14 = await orderedDescriptions(prisma14);
|
||||
const ordered17 = await orderedDescriptions(prisma17);
|
||||
|
||||
expect(ordered14).toEqual(ordered17);
|
||||
expect(ordered14).toHaveLength(descriptions.length);
|
||||
}
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,622 @@
|
||||
// RunStore run-ops persistence — waitpoints, against the REAL dedicated split topology.
|
||||
//
|
||||
// `heteroRunOpsPostgresTest` gives prisma14 = the full control-plane schema (#legacy) and
|
||||
// prisma17 = a real `RunOpsPrismaClient` over the @internal/run-ops-database SUBSET schema (#new).
|
||||
// These were previously on the weaker `heteroPostgresTest` (full schema on BOTH sides), which could
|
||||
// not catch dedicated-subset behaviour differences — the entire point of the split. On the subset
|
||||
// there are no Organization/Project/RuntimeEnvironment models and the implicit M2M join tables
|
||||
// (`_WaitpointRunConnections`) are replaced by the explicit FK-free `WaitpointRunConnection` model,
|
||||
// so the store's blocking/completion paths must behave identically whether backed by the legacy
|
||||
// implicit M2M or the dedicated explicit join.
|
||||
|
||||
import { heteroRunOpsPostgresTest, HETERO_PINNED_ICU_COLLATION } from "@internal/testcontainers";
|
||||
import type { PrismaClient } from "@trigger.dev/database";
|
||||
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
|
||||
import { describe, expect } from "vitest";
|
||||
import { PostgresRunStore } from "./PostgresRunStore.js";
|
||||
import { RoutingRunStore } from "./runOpsStore.js";
|
||||
import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js";
|
||||
|
||||
type AnyClient = PrismaClient | RunOpsPrismaClient;
|
||||
|
||||
// ownerEngine classifies by the version char after stripping a single leading `<prefix>_`: a v1 body
|
||||
// → run-ops id → NEW (#new / dedicated subset), 25 chars → cuid → LEGACY (#legacy / full schema).
|
||||
const NEW_ID_26 = "k".repeat(24) + "01";
|
||||
const CUID_25 = "c".repeat(25);
|
||||
|
||||
// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops
|
||||
// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows
|
||||
// the kept FKs require.
|
||||
async function seedEnvironment(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
slugSuffix: string
|
||||
) {
|
||||
if (schemaVariant === "dedicated") {
|
||||
return {
|
||||
organization: { id: `org_${slugSuffix}` },
|
||||
project: { id: `proj_${slugSuffix}` },
|
||||
environment: { id: `env_${slugSuffix}` },
|
||||
};
|
||||
}
|
||||
const organization = await (prisma as PrismaClient).organization.create({
|
||||
data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` },
|
||||
});
|
||||
const project = await (prisma as PrismaClient).project.create({
|
||||
data: {
|
||||
name: `Project ${slugSuffix}`,
|
||||
slug: `project-${slugSuffix}`,
|
||||
externalRef: `proj_${slugSuffix}`,
|
||||
organizationId: organization.id,
|
||||
},
|
||||
});
|
||||
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
|
||||
data: {
|
||||
type: "DEVELOPMENT",
|
||||
slug: "dev",
|
||||
projectId: project.id,
|
||||
organizationId: organization.id,
|
||||
apiKey: `tr_dev_${slugSuffix}`,
|
||||
pkApiKey: `pk_dev_${slugSuffix}`,
|
||||
shortcode: `short_${slugSuffix}`,
|
||||
},
|
||||
});
|
||||
return { organization, project, environment };
|
||||
}
|
||||
|
||||
function buildCreateRunInput(params: {
|
||||
runId: string;
|
||||
friendlyId: string;
|
||||
taskIdentifier: string;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
parentTaskRunId?: string;
|
||||
rootTaskRunId?: string;
|
||||
}): CreateRunInput {
|
||||
return {
|
||||
data: {
|
||||
id: params.runId,
|
||||
engine: "V2",
|
||||
status: "PENDING",
|
||||
friendlyId: params.friendlyId,
|
||||
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
organizationId: params.organizationId,
|
||||
projectId: params.projectId,
|
||||
taskIdentifier: params.taskIdentifier,
|
||||
payload: '{"hello":"world"}',
|
||||
payloadType: "application/json",
|
||||
context: { foo: "bar" },
|
||||
traceContext: { trace: "ctx" },
|
||||
traceId: "trace_1",
|
||||
spanId: "span_1",
|
||||
runTags: ["alpha", "beta"],
|
||||
queue: "task/my-task",
|
||||
isTest: false,
|
||||
taskEventStore: "taskEvent",
|
||||
depth: 0,
|
||||
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
||||
...(params.parentTaskRunId && { parentTaskRunId: params.parentTaskRunId }),
|
||||
...(params.rootTaskRunId && { rootTaskRunId: params.rootTaskRunId }),
|
||||
},
|
||||
snapshot: {
|
||||
engine: "V2",
|
||||
executionStatus: "RUN_CREATED",
|
||||
description: "Run was created",
|
||||
runStatus: "PENDING",
|
||||
environmentId: params.runtimeEnvironmentId,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: params.projectId,
|
||||
organizationId: params.organizationId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function seedPendingWaitpoint(
|
||||
prisma: AnyClient,
|
||||
params: {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
type?: "MANUAL" | "RUN";
|
||||
completedByTaskRunId?: string;
|
||||
}
|
||||
) {
|
||||
return (prisma as PrismaClient).waitpoint.create({
|
||||
data: {
|
||||
id: params.id,
|
||||
friendlyId: params.friendlyId,
|
||||
type: params.type ?? "MANUAL",
|
||||
status: "PENDING",
|
||||
idempotencyKey: `idem_${params.id}`,
|
||||
userProvidedIdempotencyKey: false,
|
||||
projectId: params.projectId,
|
||||
environmentId: params.environmentId,
|
||||
...(params.completedByTaskRunId && { completedByTaskRunId: params.completedByTaskRunId }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeStore(prisma: AnyClient, schemaVariant: RunStoreSchemaVariant) {
|
||||
return new PostgresRunStore({
|
||||
prisma: prisma as never,
|
||||
readOnlyPrisma: prisma as never,
|
||||
schemaVariant,
|
||||
});
|
||||
}
|
||||
|
||||
// Count the run↔waitpoint connection rows for (runId, waitpointId), reading from whichever physical
|
||||
// connection table the store writes: the implicit `_WaitpointRunConnections` M2M on #legacy, the
|
||||
// explicit FK-free `WaitpointRunConnection` model on the dedicated #new subset.
|
||||
async function countConnection(
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
waitpointId: string
|
||||
): Promise<number> {
|
||||
const rows =
|
||||
schemaVariant === "dedicated"
|
||||
? await (prisma as PrismaClient).$queryRawUnsafe<{ count: bigint }[]>(
|
||||
`SELECT COUNT(*)::bigint as count FROM "WaitpointRunConnection" WHERE "taskRunId" = '${runId}' AND "waitpointId" = '${waitpointId}'`
|
||||
)
|
||||
: await (prisma as PrismaClient).$queryRawUnsafe<{ count: bigint }[]>(
|
||||
`SELECT COUNT(*)::bigint as count FROM "_WaitpointRunConnections" WHERE "A" = '${runId}' AND "B" = '${waitpointId}'`
|
||||
);
|
||||
return Number(rows.at(0)?.count ?? 0);
|
||||
}
|
||||
|
||||
// Strip per-DB / prisma-managed fields so completed waitpoint rows compare field-for-field.
|
||||
function normalizeWaitpoint(row: Record<string, unknown>) {
|
||||
const r = { ...row };
|
||||
delete r.id;
|
||||
delete r.friendlyId;
|
||||
delete r.idempotencyKey;
|
||||
delete r.completedAt;
|
||||
delete r.createdAt;
|
||||
delete r.updatedAt;
|
||||
delete r.projectId;
|
||||
delete r.environmentId;
|
||||
return r;
|
||||
}
|
||||
|
||||
describe("RunStore run-ops persistence — waitpoints", () => {
|
||||
// a PENDING waitpoint blocked then completed via the store yields a behaviourally-identical
|
||||
// completed row on #legacy (full schema) and #new (dedicated subset).
|
||||
heteroRunOpsPostgresTest(
|
||||
"waitpoint complete is behaviourally identical across #legacy and #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const completedAt = new Date("2024-02-02T00:00:00.000Z");
|
||||
|
||||
const run = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_wa_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
const w = `wp_${suffix}`;
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w,
|
||||
friendlyId: `waitpoint_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
await store.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [w],
|
||||
projectId: env.project.id,
|
||||
});
|
||||
await store.updateManyWaitpoints({
|
||||
where: { id: w },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
output: '{"done":true}',
|
||||
outputType: "application/json",
|
||||
completedAt,
|
||||
},
|
||||
});
|
||||
|
||||
return store.findWaitpoint({ where: { id: w } });
|
||||
};
|
||||
|
||||
const wp14 = await run(prisma14, "legacy", `run_${CUID_25}`, "wa14");
|
||||
const wp17 = await run(prisma17, "dedicated", `run_${NEW_ID_26}`, "wa17");
|
||||
|
||||
expect(wp14).not.toBeNull();
|
||||
expect(wp17).not.toBeNull();
|
||||
expect(wp14!.status).toBe("COMPLETED");
|
||||
expect(wp17!.status).toBe("COMPLETED");
|
||||
expect(wp14!.completedAt?.toISOString()).toBe(completedAt.toISOString());
|
||||
expect(wp17!.completedAt?.toISOString()).toBe(completedAt.toISOString());
|
||||
expect(normalizeWaitpoint(wp14 as Record<string, unknown>)).toEqual(
|
||||
normalizeWaitpoint(wp17 as Record<string, unknown>)
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// the blocking CTE writes exactly one TaskRunWaitpoint + one connection edge (the implicit
|
||||
// `_WaitpointRunConnections` on #legacy, the explicit `WaitpointRunConnection` on #new), is
|
||||
// idempotent on a re-run (ON CONFLICT DO NOTHING), and countPendingWaitpoints (the separate MVCC
|
||||
// statement) flips 1 → 0 across the completion — identically on both stores.
|
||||
heteroRunOpsPostgresTest(
|
||||
"blocking CTE round-trips idempotently and pending-count reflects completion",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const run = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
runId: string,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: `run_friendly_wb_${suffix}`,
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
const w = `wp_${suffix}`;
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w,
|
||||
friendlyId: `waitpoint_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
const countEdges = async () => {
|
||||
const trw = await store.findManyTaskRunWaitpoints({ where: { taskRunId: runId } });
|
||||
const conn = await countConnection(prisma, schemaVariant, runId, w);
|
||||
return { trw: trw.length, conn };
|
||||
};
|
||||
|
||||
// Pass an explicit batchIndex so the `@@unique([taskRunId, waitpointId, batchIndex])`
|
||||
// index engages and the CTE's `ON CONFLICT DO NOTHING` genuinely dedupes the
|
||||
// TaskRunWaitpoint row. (With a NULL batchIndex, NULLs are distinct in the unique
|
||||
// index, so dedup is handled by a SQL-only partial index that the migration does not
|
||||
// ship into the test clone — out of scope for this round-trip proof.)
|
||||
const block = () =>
|
||||
store.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [w],
|
||||
projectId: env.project.id,
|
||||
batchIndex: 0,
|
||||
});
|
||||
|
||||
await block();
|
||||
const afterFirst = await countEdges();
|
||||
const pendingBefore = await store.countPendingWaitpoints([w]);
|
||||
|
||||
// Second call: ON CONFLICT DO NOTHING keeps it at exactly one of each.
|
||||
await block();
|
||||
const afterSecond = await countEdges();
|
||||
|
||||
await store.updateManyWaitpoints({ where: { id: w }, data: { status: "COMPLETED" } });
|
||||
const pendingAfter = await store.countPendingWaitpoints([w]);
|
||||
|
||||
return { afterFirst, afterSecond, pendingBefore, pendingAfter };
|
||||
};
|
||||
|
||||
for (const variant of [
|
||||
{
|
||||
prisma: prisma14,
|
||||
schemaVariant: "legacy" as const,
|
||||
runId: `run_${CUID_25}`,
|
||||
suffix: "wb14",
|
||||
},
|
||||
{
|
||||
prisma: prisma17,
|
||||
schemaVariant: "dedicated" as const,
|
||||
runId: `run_${NEW_ID_26}`,
|
||||
suffix: "wb17",
|
||||
},
|
||||
]) {
|
||||
const r = await run(variant.prisma, variant.schemaVariant, variant.runId, variant.suffix);
|
||||
expect(r.afterFirst).toEqual({ trw: 1, conn: 1 });
|
||||
expect(r.afterSecond).toEqual({ trw: 1, conn: 1 });
|
||||
expect(r.pendingBefore).toBe(1);
|
||||
expect(r.pendingAfter).toBe(0);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// a small V2 dependency subgraph (parent → child blocked on a RUN-type waitpoint completed by
|
||||
// the child) traversed via the store reads produces an identically ordered closure id sequence on
|
||||
// #legacy and #new. The load-bearing assertion is ordering parity; the order step is pinned to the
|
||||
// shared ICU collation (`und-x-icu`, present on both containers).
|
||||
heteroRunOpsPostgresTest(
|
||||
"V2 dependency closure ordering is identical across #legacy and #new",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const buildClosure = async (
|
||||
prisma: AnyClient,
|
||||
schemaVariant: RunStoreSchemaVariant,
|
||||
suffix: string
|
||||
) => {
|
||||
const store = makeStore(prisma, schemaVariant);
|
||||
const env = await seedEnvironment(prisma, schemaVariant, suffix);
|
||||
|
||||
const parentId = "run_parent";
|
||||
const childId = "run_child";
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: parentId,
|
||||
friendlyId: `run_parent_friendly_${suffix}`,
|
||||
taskIdentifier: "parent-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
await store.createRun(
|
||||
buildCreateRunInput({
|
||||
runId: childId,
|
||||
friendlyId: `run_child_friendly_${suffix}`,
|
||||
taskIdentifier: "child-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
parentTaskRunId: parentId,
|
||||
rootTaskRunId: parentId,
|
||||
})
|
||||
);
|
||||
|
||||
// A RUN-type waitpoint completed by the child, blocking the parent. The id is
|
||||
// version-independent (each DB clone is isolated) so the closure id sequence is
|
||||
// directly comparable across the two stores — the friendlyId carries the per-DB suffix
|
||||
// to satisfy its global-unique constraint.
|
||||
const w = "wp_run_closure";
|
||||
await seedPendingWaitpoint(prisma, {
|
||||
id: w,
|
||||
friendlyId: `waitpoint_run_${suffix}`,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
type: "RUN",
|
||||
completedByTaskRunId: childId,
|
||||
});
|
||||
await store.blockRunWithWaitpointEdges({
|
||||
runId: parentId,
|
||||
waitpointIds: [w],
|
||||
projectId: env.project.id,
|
||||
});
|
||||
|
||||
// Traverse: parent → its blocking edges → the blocking waitpoints → the run that
|
||||
// completes each. Order the closure with explicit COLLATE on the text id step.
|
||||
const edges = await store.findManyTaskRunWaitpoints({ where: { taskRunId: parentId } });
|
||||
const orderedWaitpointIds = (
|
||||
await (prisma as PrismaClient).$queryRawUnsafe<{ id: string }[]>(
|
||||
`SELECT "id" FROM "Waitpoint" WHERE "id" IN (${edges
|
||||
.map((e) => `'${e.waitpointId}'`)
|
||||
.join(",")}) ORDER BY "id" COLLATE "${HETERO_PINNED_ICU_COLLATION}" ASC`
|
||||
)
|
||||
).map((r) => r.id);
|
||||
const waitpoints = await store.findManyWaitpoints({
|
||||
where: { id: { in: orderedWaitpointIds } },
|
||||
});
|
||||
const completingRunIds = waitpoints
|
||||
.map((wp) => wp.completedByTaskRunId)
|
||||
.filter((id): id is string => Boolean(id));
|
||||
const completingRuns = await store.findRuns({
|
||||
where: { id: { in: completingRunIds } },
|
||||
orderBy: { id: "asc" },
|
||||
});
|
||||
|
||||
return [parentId, ...orderedWaitpointIds, ...completingRuns.map((r) => r.id)];
|
||||
};
|
||||
|
||||
const closure14 = await buildClosure(prisma14, "legacy", "wc14");
|
||||
const closure17 = await buildClosure(prisma17, "dedicated", "wc17");
|
||||
|
||||
expect(closure14).toEqual(closure17);
|
||||
expect(closure14).toEqual(["run_parent", "wp_run_closure", "run_child"]);
|
||||
}
|
||||
);
|
||||
|
||||
// single-DB passthrough — both router stores are the same #legacy store over one client. A
|
||||
// snapshot create + waitpoint block + complete via the router round-trips on that client and never
|
||||
// touches the dedicated #new DB (prisma17, the SUBSET schema).
|
||||
heteroRunOpsPostgresTest(
|
||||
"single-DB binds one client for run-ops (passthrough)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const store = makeStore(prisma14, "legacy");
|
||||
const router = new RoutingRunStore({ new: store, legacy: store });
|
||||
|
||||
const env = await seedEnvironment(prisma14, "legacy", "wd14");
|
||||
|
||||
// NEW_ID_26-length id → NEW residency, exercising the route; both slots are the same store so
|
||||
// it still lands on prisma14.
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_passthrough_wd",
|
||||
taskIdentifier: "passthrough-task",
|
||||
organizationId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
runtimeEnvironmentId: env.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
const w = "wp_passthrough_wd";
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: w,
|
||||
friendlyId: "waitpoint_passthrough_wd",
|
||||
projectId: env.project.id,
|
||||
environmentId: env.environment.id,
|
||||
});
|
||||
|
||||
const snapshot = await router.createExecutionSnapshot({
|
||||
run: { id: runId, status: "EXECUTING", attemptNumber: 1 },
|
||||
snapshot: { executionStatus: "EXECUTING_WITH_WAITPOINTS", description: "passthrough" },
|
||||
completedWaitpoints: [{ id: w, index: 0 }],
|
||||
environmentId: env.environment.id,
|
||||
environmentType: "DEVELOPMENT",
|
||||
projectId: env.project.id,
|
||||
organizationId: env.organization.id,
|
||||
});
|
||||
await router.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [w],
|
||||
projectId: env.project.id,
|
||||
});
|
||||
await router.updateManyWaitpoints({ where: { id: w }, data: { status: "COMPLETED" } });
|
||||
|
||||
const latest = await router.findLatestExecutionSnapshot(runId);
|
||||
expect(latest?.id).toBe(snapshot.id);
|
||||
const joinIds = await router.findSnapshotCompletedWaitpointIds(snapshot.id);
|
||||
expect(joinIds).toEqual([w]);
|
||||
expect(await router.countPendingWaitpoints([w])).toBe(0);
|
||||
|
||||
// Everything landed on the one #legacy client; the dedicated #new DB was never touched.
|
||||
expect(await prisma14.taskRun.findUnique({ where: { id: runId } })).not.toBeNull();
|
||||
expect(await prisma17.taskRun.findUnique({ where: { id: runId } })).toBeNull();
|
||||
expect(await prisma17.waitpoint.findUnique({ where: { id: w } })).toBeNull();
|
||||
}
|
||||
);
|
||||
|
||||
// the silent-hang case, against the REAL split. A NEW (run-ops id) run is blocked on
|
||||
// a LEGACY (cuid) token, so its block edge lives on #new (co-located with the run) while the token's
|
||||
// id-shape says LEGACY. Completing that token must FAN OUT the waitpointId edge read across both DBs
|
||||
// and find the edge on #new — routing by the token's id-shape (LEGACY) returns zero edges and the
|
||||
// run hangs forever. The token is mirrored onto both DBs (the drain window), so #resolveWaitpointStore
|
||||
// would resolve it to LEGACY and miss the NEW edge without the fan-out.
|
||||
heteroRunOpsPostgresTest(
|
||||
"completing a LEGACY token finds a NEW run's edge across both DBs (no silent hang)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = makeStore(prisma17, "dedicated");
|
||||
const legacyStore = makeStore(prisma14, "legacy");
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
// The NEW run + its (synthetic) env live on the dedicated #new subset (prisma17).
|
||||
const env17 = await seedEnvironment(prisma17, "dedicated", "we17");
|
||||
const runId = `run_${NEW_ID_26}`; // run-ops id → NEW residency
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_friendly_we",
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env17.organization.id,
|
||||
projectId: env17.project.id,
|
||||
runtimeEnvironmentId: env17.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
// A LEGACY (cuid) token, mirrored onto BOTH DBs as during drain. The edge can only be
|
||||
// written on #new (the run's DB) because the dedicated block insert sources the edge rows
|
||||
// from the waitpointId array directly (FK-free).
|
||||
const token = "w".repeat(25); // cuid-length → LEGACY id-shape
|
||||
const env14 = await seedEnvironment(prisma14, "legacy", "we14");
|
||||
await seedPendingWaitpoint(prisma14, {
|
||||
id: token,
|
||||
friendlyId: "waitpoint_we_legacy",
|
||||
projectId: env14.project.id,
|
||||
environmentId: env14.environment.id,
|
||||
});
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: token,
|
||||
friendlyId: "waitpoint_we_new",
|
||||
projectId: env17.project.id,
|
||||
environmentId: env17.environment.id,
|
||||
});
|
||||
|
||||
// The edge is written on #new only (co-located with the run).
|
||||
await newStore.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [token],
|
||||
projectId: env17.project.id,
|
||||
});
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { waitpointId: token } })).toBe(0);
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { waitpointId: token } })).toBe(1);
|
||||
|
||||
// The completion fan-out (the read completeWaitpoint uses) must find the NEW-DB edge even
|
||||
// though the token classifies LEGACY. Pre-fix this returned [] (LEGACY-only) → silent hang.
|
||||
const affected = await router.findManyTaskRunWaitpoints({
|
||||
where: { waitpointId: token },
|
||||
select: { taskRunId: true },
|
||||
});
|
||||
expect(affected.map((e) => e.taskRunId)).toEqual([runId]);
|
||||
}
|
||||
);
|
||||
|
||||
// replay / partial-completion safety, against the REAL split. There is NO cross-DB
|
||||
// transaction, so a completion can flip the token on one DB while the edge-clear lands on the other
|
||||
// (or a job is retried). The unblock recomputes the blocked set from the surviving edges and the
|
||||
// edge delete is keyed by (taskRunId, edge ids) — never a blind decrement — so running the
|
||||
// read+delete TWICE must not double-count or strand the run: after the first clear there are zero
|
||||
// edges, and the second pass is a no-op.
|
||||
heteroRunOpsPostgresTest(
|
||||
"replaying the unblock clear is idempotent (no double-decrement, no strand)",
|
||||
async ({ prisma14, prisma17 }) => {
|
||||
const newStore = makeStore(prisma17, "dedicated");
|
||||
const legacyStore = makeStore(prisma14, "legacy");
|
||||
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
|
||||
|
||||
const env17 = await seedEnvironment(prisma17, "dedicated", "wf17");
|
||||
const runId = `run_${NEW_ID_26}`;
|
||||
await router.createRun(
|
||||
buildCreateRunInput({
|
||||
runId,
|
||||
friendlyId: "run_friendly_wf",
|
||||
taskIdentifier: "my-task",
|
||||
organizationId: env17.organization.id,
|
||||
projectId: env17.project.id,
|
||||
runtimeEnvironmentId: env17.environment.id,
|
||||
})
|
||||
);
|
||||
|
||||
const token = "x".repeat(25); // LEGACY id-shape, edge co-located on #new
|
||||
await seedPendingWaitpoint(prisma17, {
|
||||
id: token,
|
||||
friendlyId: "waitpoint_wf_new",
|
||||
projectId: env17.project.id,
|
||||
environmentId: env17.environment.id,
|
||||
});
|
||||
await newStore.blockRunWithWaitpointEdges({
|
||||
runId,
|
||||
waitpointIds: [token],
|
||||
projectId: env17.project.id,
|
||||
});
|
||||
await router.updateManyWaitpoints({ where: { id: token }, data: { status: "COMPLETED" } });
|
||||
|
||||
// Drive the continueRunIfUnblocked read+delete shape (by taskRunId) twice.
|
||||
const unblockPass = async () => {
|
||||
const edges = await router.findManyTaskRunWaitpoints({
|
||||
where: { taskRunId: runId },
|
||||
select: { id: true, waitpoint: { select: { status: true } } },
|
||||
});
|
||||
const stillBlocked = edges.some((e) => e.waitpoint.status !== "COMPLETED");
|
||||
if (!stillBlocked && edges.length > 0) {
|
||||
await router.deleteManyTaskRunWaitpoints({
|
||||
where: { taskRunId: runId, id: { in: edges.map((e) => e.id) } },
|
||||
});
|
||||
}
|
||||
return { edgeCount: edges.length, stillBlocked };
|
||||
};
|
||||
|
||||
const first = await unblockPass();
|
||||
const second = await unblockPass();
|
||||
|
||||
expect(first).toEqual({ edgeCount: 1, stillBlocked: false }); // found + cleared
|
||||
expect(second).toEqual({ edgeCount: 0, stillBlocked: false }); // replay is a no-op
|
||||
// Edge gone from both DBs; the run is unblocked exactly once, not double-processed.
|
||||
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(0);
|
||||
expect(await prisma14.taskRunWaitpoint.count({ where: { taskRunId: runId } })).toBe(0);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,756 @@
|
||||
import type {
|
||||
BatchTaskRun,
|
||||
BatchTaskRunItemStatus,
|
||||
Prisma,
|
||||
PrismaClientOrTransaction,
|
||||
PrismaReplicaClient,
|
||||
TaskRun,
|
||||
TaskRunStatus,
|
||||
TaskRunExecutionStatus,
|
||||
RuntimeEnvironmentType,
|
||||
Waitpoint,
|
||||
} from "@trigger.dev/database";
|
||||
import type { TaskRunError } from "@trigger.dev/core/v3/schemas";
|
||||
import type { Residency } from "@trigger.dev/core/v3/isomorphic";
|
||||
|
||||
/**
|
||||
* Client accepted by the read methods. Reads route through the replica by
|
||||
* default, so callers may pass either the writer/transaction client or the
|
||||
* read replica — both expose the `taskRun.findFirst`/`findMany` surface the
|
||||
* reads use. Write methods stay on `PrismaClientOrTransaction`.
|
||||
*/
|
||||
export type ReadClient = PrismaClientOrTransaction | PrismaReplicaClient;
|
||||
|
||||
export type CreateRunSnapshotInput = {
|
||||
engine: "V2";
|
||||
executionStatus: TaskRunExecutionStatus;
|
||||
description: string;
|
||||
runStatus: TaskRunStatus;
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
workerId?: string;
|
||||
runnerId?: string;
|
||||
};
|
||||
|
||||
export type CompletionSnapshotInput = {
|
||||
executionStatus: "FINISHED";
|
||||
description: string;
|
||||
runStatus: TaskRunStatus;
|
||||
attemptNumber: number | null;
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
workerId?: string;
|
||||
runnerId?: string;
|
||||
};
|
||||
|
||||
export type ExpireSnapshotInput = {
|
||||
engine: "V2";
|
||||
executionStatus: "FINISHED";
|
||||
description: string;
|
||||
runStatus: TaskRunStatus;
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
};
|
||||
|
||||
export type RescheduleSnapshotInput = {
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
};
|
||||
|
||||
export type LockSnapshotInput = {
|
||||
id: string;
|
||||
previousSnapshotId: string;
|
||||
attemptNumber?: number;
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
checkpointId?: string;
|
||||
batchId?: string;
|
||||
completedWaitpointIds: string[];
|
||||
completedWaitpointOrder: string[];
|
||||
workerId?: string;
|
||||
runnerId?: string;
|
||||
};
|
||||
|
||||
export type RunAssociatedWaitpointInput = {
|
||||
id: string;
|
||||
friendlyId: string;
|
||||
type: "RUN";
|
||||
status: "PENDING";
|
||||
idempotencyKey: string;
|
||||
userProvidedIdempotencyKey: boolean;
|
||||
projectId: string;
|
||||
environmentId: string;
|
||||
};
|
||||
|
||||
// The ~60 trigger columns (the existing Prisma create `data` minus the nested relation creates).
|
||||
export type CreateRunData = {
|
||||
id: string;
|
||||
engine: "V2";
|
||||
status: TaskRunStatus;
|
||||
friendlyId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
idempotencyKeyOptions?: Prisma.InputJsonValue;
|
||||
taskIdentifier: string;
|
||||
payload: string;
|
||||
payloadType: string;
|
||||
context?: Prisma.InputJsonValue;
|
||||
traceContext: Prisma.InputJsonValue;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
parentSpanId?: string;
|
||||
lockedToVersionId?: string;
|
||||
taskVersion?: string;
|
||||
sdkVersion?: string;
|
||||
cliVersion?: string;
|
||||
concurrencyKey?: string;
|
||||
queue: string;
|
||||
lockedQueueId?: string;
|
||||
workerQueue?: string;
|
||||
region?: string | null;
|
||||
isTest: boolean;
|
||||
delayUntil?: Date;
|
||||
queuedAt?: Date;
|
||||
maxAttempts?: number;
|
||||
taskEventStore?: string;
|
||||
priorityMs?: number;
|
||||
queueTimestamp?: Date;
|
||||
ttl?: string;
|
||||
runTags?: string[];
|
||||
oneTimeUseToken?: string;
|
||||
parentTaskRunId?: string;
|
||||
rootTaskRunId?: string;
|
||||
replayedFromTaskRunFriendlyId?: string;
|
||||
batchId?: string;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
depth?: number;
|
||||
metadata?: string;
|
||||
metadataType?: string;
|
||||
seedMetadata?: string;
|
||||
seedMetadataType?: string;
|
||||
maxDurationInSeconds?: number;
|
||||
machinePreset?: string;
|
||||
scheduleId?: string;
|
||||
scheduleInstanceId?: string;
|
||||
createdAt?: Date;
|
||||
bulkActionGroupIds?: string[];
|
||||
planType?: string;
|
||||
realtimeStreamsVersion?: string;
|
||||
streamBasinName?: string | null;
|
||||
debounce?: Prisma.InputJsonValue;
|
||||
annotations?: Prisma.InputJsonValue;
|
||||
};
|
||||
|
||||
export type CreateRunInput = {
|
||||
data: CreateRunData;
|
||||
snapshot: CreateRunSnapshotInput;
|
||||
associatedWaitpoint?: RunAssociatedWaitpointInput;
|
||||
};
|
||||
|
||||
export type CreateCancelledRunInput = {
|
||||
data: CreateRunData & {
|
||||
error: Prisma.InputJsonValue;
|
||||
completedAt: Date;
|
||||
updatedAt: Date;
|
||||
attemptNumber: 0;
|
||||
};
|
||||
snapshot: CreateRunSnapshotInput;
|
||||
};
|
||||
|
||||
export type CreateFailedRunData = {
|
||||
id: string;
|
||||
engine: "V2";
|
||||
status: "SYSTEM_FAILURE";
|
||||
friendlyId: string;
|
||||
runtimeEnvironmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
organizationId: string;
|
||||
projectId: string;
|
||||
taskIdentifier: string;
|
||||
payload: string;
|
||||
payloadType: string;
|
||||
context: Prisma.InputJsonValue;
|
||||
traceContext: Prisma.InputJsonValue;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
queue: string;
|
||||
lockedQueueId?: string;
|
||||
isTest: false;
|
||||
completedAt: Date;
|
||||
error: Prisma.InputJsonObject;
|
||||
parentTaskRunId?: string;
|
||||
rootTaskRunId?: string;
|
||||
depth: number;
|
||||
batchId?: string;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
taskEventStore?: string;
|
||||
};
|
||||
|
||||
export type CreateFailedRunInput = {
|
||||
data: CreateFailedRunData;
|
||||
associatedWaitpoint?: RunAssociatedWaitpointInput;
|
||||
};
|
||||
|
||||
export type LockRunData = {
|
||||
lockedAt: Date;
|
||||
lockedById: string;
|
||||
lockedToVersionId: string;
|
||||
lockedQueueId: string;
|
||||
lockedRetryConfig?: Prisma.InputJsonValue;
|
||||
startedAt: Date;
|
||||
baseCostInCents: number;
|
||||
machinePreset: string;
|
||||
taskVersion: string;
|
||||
sdkVersion: string | null;
|
||||
cliVersion: string | null;
|
||||
maxDurationInSeconds: number | null | undefined;
|
||||
maxAttempts?: number;
|
||||
snapshot: LockSnapshotInput;
|
||||
};
|
||||
|
||||
export type RewriteDebouncedRunData = {
|
||||
payload: string;
|
||||
payloadType: string;
|
||||
metadata?: string;
|
||||
metadataType?: string;
|
||||
maxAttempts?: number;
|
||||
maxDurationInSeconds?: number;
|
||||
machinePreset?: string;
|
||||
runTags?: string[];
|
||||
};
|
||||
|
||||
export type ClearIdempotencyKeyInput =
|
||||
| { byId: { runId: string; idempotencyKey: string }; byPredicate?: never; byFriendlyIds?: never }
|
||||
| {
|
||||
byPredicate: { idempotencyKey: string; taskIdentifier: string; runtimeEnvironmentId: string };
|
||||
byId?: never;
|
||||
byFriendlyIds?: never;
|
||||
}
|
||||
| { byFriendlyIds: string[]; byId?: never; byPredicate?: never };
|
||||
|
||||
export type TaskRunWithWaitpoint = TaskRun & { associatedWaitpoint: Waitpoint | null };
|
||||
|
||||
/**
|
||||
* Structured input for {@link RunStore.createExecutionSnapshot}. The store derives the
|
||||
* `completedWaitpoints.connect` / `completedWaitpointOrder` / `isValid` fields from this
|
||||
* input — callers pass the high-level shape, not a raw Prisma `data`/`include`.
|
||||
*/
|
||||
export type CreateExecutionSnapshotInput = {
|
||||
run: { id: string; status: TaskRunStatus; attemptNumber?: number | null };
|
||||
snapshot: {
|
||||
executionStatus: TaskRunExecutionStatus;
|
||||
description: string;
|
||||
metadata?: Prisma.JsonValue;
|
||||
};
|
||||
previousSnapshotId?: string;
|
||||
batchId?: string;
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
checkpointId?: string;
|
||||
workerId?: string;
|
||||
runnerId?: string;
|
||||
completedWaitpoints?: { id: string; index?: number }[];
|
||||
error?: string;
|
||||
};
|
||||
|
||||
// Create payload for `createBatchTaskRun`: scalar `runtimeEnvironmentId` (the FK is
|
||||
// dropped for cross-DB residency; env existence is validated app-side at create).
|
||||
export type CreateBatchTaskRunData = Prisma.BatchTaskRunUncheckedCreateInput;
|
||||
|
||||
/**
|
||||
* Mirror of the webapp's `UnblockRouteKind`. The engine/run-store cannot import the
|
||||
* webapp types, so this union is kept IDENTICAL (members + field names) to
|
||||
* `apps/webapp/app/v3/runOpsMigration/types.ts` so the two cannot drift conceptually.
|
||||
*/
|
||||
export type WaitpointUnblockRouteKind =
|
||||
| "MANUAL"
|
||||
| "DATETIME"
|
||||
| "RESUME_TOKEN"
|
||||
| "IDEMPOTENCY_REUSE"
|
||||
| "RUN";
|
||||
|
||||
/**
|
||||
* Pinning context for {@link RunStore.forWaitpointCompletion}. Mirrors the webapp's
|
||||
* waitpoint-completion pinning input shape.
|
||||
*/
|
||||
export interface ForWaitpointCompletionContext {
|
||||
routeKind: WaitpointUnblockRouteKind;
|
||||
treeOwnerResidency?: Residency;
|
||||
isCrossTreeIdempotency?: boolean;
|
||||
hasLegacyParent?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Co-location hint for the waitpoint write/lookup methods. A DATETIME/MANUAL wait waitpoint's
|
||||
* minted id is always a cuid, so id-shape routing always sends it to LEGACY; when `coLocateWithRunId`
|
||||
* is set the router routes by the OWNING RUN's id instead, landing the waitpoint on the run's DB so
|
||||
* the block edge's local `Waitpoint` join resolves. Single-store implementations ignore it.
|
||||
*/
|
||||
export interface WaitpointColocationOptions {
|
||||
coLocateWithRunId?: string;
|
||||
}
|
||||
|
||||
export interface RunStore {
|
||||
/**
|
||||
* Run a co-resident multi-write unit atomically on the store that OWNS `runId`. The callback gets
|
||||
* the owning `RunStore` plus a `tx` opened on THAT store's OWN client; passing `tx` to the inner
|
||||
* writes lands them all in ONE transaction on the owning DB (NEW for a run-ops run, LEGACY for a cuid
|
||||
* run), so a failure between two writes rolls BOTH back. NOT a cross-DB transaction: `tx` is the
|
||||
* owning store's own client (never the control-plane tx), and every write MUST target the same run /
|
||||
* its co-resident subgraph. Callers MUST use the supplied `store` + `tx`, not the outer router
|
||||
* (which would re-route and drop the tx). Single-store impls run `fn(this, tx)` in their own
|
||||
* `$transaction`.
|
||||
*/
|
||||
runInTransaction<R>(
|
||||
runId: string | undefined,
|
||||
fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise<R>
|
||||
): Promise<R>;
|
||||
|
||||
// Create
|
||||
createRun(params: CreateRunInput, tx?: PrismaClientOrTransaction): Promise<TaskRunWithWaitpoint>;
|
||||
createCancelledRun(
|
||||
params: CreateCancelledRunInput,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<TaskRun>;
|
||||
createFailedRun(
|
||||
params: CreateFailedRunInput,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<TaskRunWithWaitpoint>;
|
||||
|
||||
// Attempt lifecycle
|
||||
startAttempt<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: { attemptNumber: number; executedAt?: Date; isWarmStart: boolean },
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
completeAttemptSuccess<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: {
|
||||
completedAt: Date;
|
||||
output?: string;
|
||||
outputType: string;
|
||||
usageDurationMs: number;
|
||||
costInCents: number;
|
||||
snapshot: CompletionSnapshotInput;
|
||||
},
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
recordRetryOutcome<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: { machinePreset?: string; usageDurationMs: number; costInCents: number },
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
requeueRun<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
recordBulkActionMembership(
|
||||
runId: string,
|
||||
bulkActionId: string,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<void>;
|
||||
cancelRun<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: {
|
||||
completedAt?: Date;
|
||||
error: TaskRunError;
|
||||
bulkActionId?: string;
|
||||
usageDurationMs?: number;
|
||||
costInCents?: number;
|
||||
},
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
failRunPermanently<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: {
|
||||
status: TaskRunStatus;
|
||||
completedAt: Date;
|
||||
error: TaskRunError;
|
||||
usageDurationMs: number;
|
||||
costInCents: number;
|
||||
},
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
|
||||
// Expiry
|
||||
expireRun<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: {
|
||||
error: TaskRunError;
|
||||
completedAt: Date;
|
||||
expiredAt: Date;
|
||||
snapshot: ExpireSnapshotInput;
|
||||
},
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
expireRunsBatch(
|
||||
runIds: string[],
|
||||
data: { error: TaskRunError; now: Date },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<number>;
|
||||
|
||||
// Dequeue / version / checkpoint
|
||||
lockRunToWorker(
|
||||
runId: string,
|
||||
data: LockRunData,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{}>>;
|
||||
parkPendingVersion<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
data: { statusReason: string },
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
promotePendingVersionRuns(
|
||||
runId: string,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<{ count: number }>;
|
||||
suspendForCheckpoint<I extends Prisma.TaskRunInclude>(
|
||||
runId: string,
|
||||
args: { include: I },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }>>;
|
||||
resumeFromCheckpoint<S extends Prisma.TaskRunSelect>(
|
||||
runId: string,
|
||||
args: { select: S },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
|
||||
// Delayed / debounce
|
||||
rescheduleRun(
|
||||
runId: string,
|
||||
data: { delayUntil: Date; queueTimestamp?: Date; snapshot?: RescheduleSnapshotInput },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<TaskRun>;
|
||||
enqueueDelayedRun(
|
||||
runId: string,
|
||||
data: { queuedAt: Date },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<TaskRun>;
|
||||
rewriteDebouncedRun(
|
||||
runId: string,
|
||||
data: RewriteDebouncedRunData,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<TaskRunWithWaitpoint>;
|
||||
|
||||
// Field touches
|
||||
updateMetadata(
|
||||
runId: string,
|
||||
data: {
|
||||
metadata: string | null;
|
||||
metadataType?: string;
|
||||
metadataVersion: { increment: number };
|
||||
updatedAt: Date;
|
||||
},
|
||||
options: { expectedMetadataVersion?: number },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<{ count: number }>;
|
||||
clearIdempotencyKey(
|
||||
params: ClearIdempotencyKeyInput,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<{ count: number }>;
|
||||
pushTags(
|
||||
runId: string,
|
||||
tags: string[],
|
||||
where: { runtimeEnvironmentId: string },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<{ updatedAt: Date }>;
|
||||
pushRealtimeStream(
|
||||
runId: string,
|
||||
streamId: string,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<void>;
|
||||
|
||||
// Read
|
||||
|
||||
// This store's own PRIMARY (writer) handle in read-client form. The routing layer passes it as
|
||||
// the `client` for a routed read when the CALLER supplied one: the caller's client is bound to
|
||||
// the control-plane DB (the wrong database for a NEW-resident row), so read-your-writes is
|
||||
// honored by reading the OWNING store's own primary instead of its replica.
|
||||
readonly primaryReadClient: ReadClient;
|
||||
|
||||
findRun<S extends Prisma.TaskRunSelect>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { select: S },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }> | null>;
|
||||
findRun<I extends Prisma.TaskRunInclude>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { include: I },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }> | null>;
|
||||
findRun(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise<TaskRun | null>;
|
||||
|
||||
findRunOrThrow<S extends Prisma.TaskRunSelect>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { select: S },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
findRunOrThrow<I extends Prisma.TaskRunInclude>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { include: I },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }>>;
|
||||
findRunOrThrow(where: Prisma.TaskRunWhereInput, client?: ReadClient): Promise<TaskRun>;
|
||||
|
||||
// Read-after-write on the OWNING store's primary (writer), never the replica — for re-reading a
|
||||
// run just written in this request, where replica lag would cause a false miss (mirrors
|
||||
// findWaitpointOnPrimary). The routing store dispatches here per owning store so each reads its
|
||||
// own writer, never leaking a control-plane client into another DB.
|
||||
findRunOnPrimary<S extends Prisma.TaskRunSelect>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { select: S }
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }> | null>;
|
||||
findRunOnPrimary<I extends Prisma.TaskRunInclude>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { include: I }
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }> | null>;
|
||||
findRunOnPrimary(where: Prisma.TaskRunWhereInput): Promise<TaskRun | null>;
|
||||
|
||||
findRunOrThrowOnPrimary<S extends Prisma.TaskRunSelect>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { select: S }
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>>;
|
||||
findRunOrThrowOnPrimary<I extends Prisma.TaskRunInclude>(
|
||||
where: Prisma.TaskRunWhereInput,
|
||||
args: { include: I }
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }>>;
|
||||
findRunOrThrowOnPrimary(where: Prisma.TaskRunWhereInput): Promise<TaskRun>;
|
||||
|
||||
findRuns<S extends Prisma.TaskRunSelect>(
|
||||
args: {
|
||||
where: Prisma.TaskRunWhereInput;
|
||||
select: S;
|
||||
orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[];
|
||||
take?: number;
|
||||
skip?: number;
|
||||
cursor?: Prisma.TaskRunWhereUniqueInput;
|
||||
},
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ select: S }>[]>;
|
||||
findRuns<I extends Prisma.TaskRunInclude>(
|
||||
args: {
|
||||
where: Prisma.TaskRunWhereInput;
|
||||
include: I;
|
||||
orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[];
|
||||
take?: number;
|
||||
skip?: number;
|
||||
cursor?: Prisma.TaskRunWhereUniqueInput;
|
||||
},
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunGetPayload<{ include: I }>[]>;
|
||||
findRuns(
|
||||
args: {
|
||||
where: Prisma.TaskRunWhereInput;
|
||||
orderBy?: Prisma.TaskRunOrderByWithRelationInput | Prisma.TaskRunOrderByWithRelationInput[];
|
||||
take?: number;
|
||||
skip?: number;
|
||||
cursor?: Prisma.TaskRunWhereUniqueInput;
|
||||
},
|
||||
client?: ReadClient
|
||||
): Promise<TaskRun[]>;
|
||||
|
||||
// --- run-ops persistence ---
|
||||
// Snapshots, waitpoints, implicit M:N joins, dependents, attempts and checkpoints. The
|
||||
// generic model wrappers are thin generics over the Prisma `*Args` types so include/select
|
||||
// payload typing survives at the call site; the snapshot DTO builder and the two raw-SQL
|
||||
// waitpoint methods keep their hand-written shapes.
|
||||
|
||||
// Batch membership
|
||||
createBatchTaskRunItem(
|
||||
data: { batchTaskRunId: string; taskRunId: string; status: BatchTaskRunItemStatus },
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<void>;
|
||||
|
||||
// Snapshot group
|
||||
findLatestExecutionSnapshot(
|
||||
runId: string,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunExecutionSnapshotGetPayload<{
|
||||
include: { completedWaitpoints: true; checkpoint: true };
|
||||
}> | null>;
|
||||
findExecutionSnapshot<T extends Prisma.TaskRunExecutionSnapshotFindFirstArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.TaskRunExecutionSnapshotFindFirstArgs>,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunExecutionSnapshotGetPayload<T> | null>;
|
||||
findManyExecutionSnapshots<T extends Prisma.TaskRunExecutionSnapshotFindManyArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.TaskRunExecutionSnapshotFindManyArgs>,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunExecutionSnapshotGetPayload<T>[]>;
|
||||
createExecutionSnapshot(
|
||||
input: CreateExecutionSnapshotInput,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunExecutionSnapshotGetPayload<{ include: { checkpoint: true } }>>;
|
||||
|
||||
// Implicit-join group
|
||||
findSnapshotCompletedWaitpointIds(snapshotId: string, client?: ReadClient): Promise<string[]>;
|
||||
/** As above, but reports in the SAME read whether the snapshot is visible on the reader: `present=false`
|
||||
* means this reader lacks the snapshot, so its empty id list is not authoritative (repair from primary). */
|
||||
findSnapshotCompletedWaitpointIdsWithPresence(
|
||||
snapshotId: string,
|
||||
client?: ReadClient
|
||||
): Promise<{ present: boolean; ids: string[] }>;
|
||||
/** Run ids connected to a waitpoint (WaitpointRunConnection / `_WaitpointRunConnections`), this DB only. */
|
||||
findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise<string[]>;
|
||||
/** Snapshot ids that completed a waitpoint (CompletedWaitpoint / `_completedWaitpoints`), this DB only. */
|
||||
findWaitpointCompletedSnapshotIds(waitpointId: string, client?: ReadClient): Promise<string[]>;
|
||||
blockRunWithWaitpointEdges(params: {
|
||||
runId: string;
|
||||
waitpointIds: string[];
|
||||
projectId: string;
|
||||
spanIdToComplete?: string;
|
||||
batchId?: string;
|
||||
batchIndex?: number;
|
||||
tx?: PrismaClientOrTransaction;
|
||||
}): Promise<void>;
|
||||
countPendingWaitpoints(waitpointIds: string[], client?: ReadClient): Promise<number>;
|
||||
|
||||
// Waitpoint group
|
||||
createWaitpoint<T extends Prisma.WaitpointCreateArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointCreateArgs>,
|
||||
tx?: PrismaClientOrTransaction,
|
||||
opts?: WaitpointColocationOptions
|
||||
): Promise<Prisma.WaitpointGetPayload<T>>;
|
||||
upsertWaitpoint<T extends Prisma.WaitpointUpsertArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointUpsertArgs>,
|
||||
tx?: PrismaClientOrTransaction,
|
||||
opts?: WaitpointColocationOptions
|
||||
): Promise<Prisma.WaitpointGetPayload<T>>;
|
||||
findWaitpoint<T extends Prisma.WaitpointFindFirstArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointFindFirstArgs>,
|
||||
client?: ReadClient,
|
||||
opts?: WaitpointColocationOptions
|
||||
): Promise<Prisma.WaitpointGetPayload<T> | null>;
|
||||
// Read-after-write on the owning store's primary (never the replica) — for re-reading a
|
||||
// waitpoint just written on the unblock path, where replica lag would cause a false miss.
|
||||
findWaitpointOnPrimary<T extends Prisma.WaitpointFindFirstArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointFindFirstArgs>
|
||||
): Promise<Prisma.WaitpointGetPayload<T> | null>;
|
||||
findManyWaitpoints<T extends Prisma.WaitpointFindManyArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointFindManyArgs>,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.WaitpointGetPayload<T>[]>;
|
||||
updateWaitpoint<T extends Prisma.WaitpointUpdateArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.WaitpointUpdateArgs>,
|
||||
tx?: PrismaClientOrTransaction,
|
||||
opts?: WaitpointColocationOptions
|
||||
): Promise<Prisma.WaitpointGetPayload<T>>;
|
||||
updateManyWaitpoints(
|
||||
args: Prisma.WaitpointUpdateManyArgs,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.BatchPayload>;
|
||||
|
||||
/**
|
||||
* Select the run-ops store that OWNS a waitpoint completion, by waitpointId
|
||||
* residency. completeWaitpoint arrives with only (waitpointId, output) — no run
|
||||
* id — so selection is by the waitpoint's own residency, with the documented
|
||||
* pins to legacy. Returns the store HANDLE to apply the completion on.
|
||||
* Single-store implementations return `this`. Throws UnclassifiableRunId on an
|
||||
* ambiguous id in split mode (the engine rethrows it as UnclassifiableWaitpointId).
|
||||
*/
|
||||
forWaitpointCompletion(
|
||||
waitpointId: string,
|
||||
context: ForWaitpointCompletionContext
|
||||
): Promise<RunStore>;
|
||||
|
||||
// TaskRunWaitpoint group
|
||||
findManyTaskRunWaitpoints<T extends Prisma.TaskRunWaitpointFindManyArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.TaskRunWaitpointFindManyArgs>,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunWaitpointGetPayload<T>[]>;
|
||||
deleteManyTaskRunWaitpoints(
|
||||
args: Prisma.TaskRunWaitpointDeleteManyArgs,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.BatchPayload>;
|
||||
|
||||
// Attempt-model group (TaskRunAttempt, V1-residual)
|
||||
findTaskRunAttempt<T extends Prisma.TaskRunAttemptFindFirstArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.TaskRunAttemptFindFirstArgs>,
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.TaskRunAttemptGetPayload<T> | null>;
|
||||
|
||||
// Checkpoint family. `ownerRunId` is the run whose snapshot references this checkpoint via the
|
||||
// kept `TaskRunExecutionSnapshot.checkpointId` FK — the routing store co-locates the checkpoint
|
||||
// with that run so the snapshot insert can satisfy the FK on the same DB. The checkpoint
|
||||
// row itself carries no runId scalar, so the owning run id must be threaded explicitly.
|
||||
createTaskRunCheckpoint<T extends Prisma.TaskRunCheckpointCreateArgs>(
|
||||
args: Prisma.SelectSubset<T, Prisma.TaskRunCheckpointCreateArgs>,
|
||||
ownerRunId?: string,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.TaskRunCheckpointGetPayload<T>>;
|
||||
|
||||
// --- BatchTaskRun (run-ops) ---
|
||||
// Batch row is born on the run-ops store at create. `findBatchTaskRunById`
|
||||
// reads the primary by default (worker reads the just-written row; replica lag).
|
||||
createBatchTaskRun(
|
||||
data: CreateBatchTaskRunData,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<BatchTaskRun>;
|
||||
updateBatchTaskRun<S extends Prisma.BatchTaskRunSelect>(
|
||||
args: {
|
||||
where: Prisma.BatchTaskRunWhereUniqueInput;
|
||||
data: Prisma.BatchTaskRunUpdateInput;
|
||||
select: S;
|
||||
},
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.BatchTaskRunGetPayload<{ select: S }>>;
|
||||
findBatchTaskRunById<T extends Prisma.BatchTaskRunInclude = {}>(
|
||||
id: string,
|
||||
args?: { include?: T },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.BatchTaskRunGetPayload<{ include: T }> | null>;
|
||||
findBatchTaskRunByFriendlyId<T extends Prisma.BatchTaskRunInclude = {}>(
|
||||
friendlyId: string,
|
||||
environmentId: string,
|
||||
args?: { include?: T },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.BatchTaskRunGetPayload<{ include: T }> | null>;
|
||||
|
||||
// --- BatchTaskRun (run-ops) — batch residency additions ---
|
||||
// The idempotency probe is keyed by (environmentId, idempotencyKey) — no classifiable
|
||||
// batch id — so the router fans out NEW→LEGACY (mirrors `findBatchTaskRunByFriendlyId`).
|
||||
findBatchTaskRunByIdempotencyKey<T extends Prisma.BatchTaskRunInclude = {}>(
|
||||
environmentId: string,
|
||||
idempotencyKey: string,
|
||||
args?: { include?: T },
|
||||
client?: ReadClient
|
||||
): Promise<Prisma.BatchTaskRunGetPayload<{ include: T }> | null>;
|
||||
// updateMany of batch rows: route by `where.id` when scalar, else fan-out + sum counts.
|
||||
updateManyBatchTaskRun(
|
||||
args: Prisma.BatchTaskRunUpdateManyArgs,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.BatchPayload>;
|
||||
// Count batch items by `batchTaskRunId` (items co-reside with the batch).
|
||||
countBatchTaskRunItems(
|
||||
where: { batchTaskRunId: string; status?: BatchTaskRunItemStatus },
|
||||
client?: ReadClient
|
||||
): Promise<number>;
|
||||
// updateMany of batch items: route by `where.id`/`where.batchTaskRunId`, else fan-out + sum.
|
||||
updateManyBatchTaskRunItems(
|
||||
args: Prisma.BatchTaskRunItemUpdateManyArgs,
|
||||
tx?: PrismaClientOrTransaction
|
||||
): Promise<Prisma.BatchPayload>;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"outDir": "dist",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"declaration": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "Node16",
|
||||
"module": "Node16",
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "src/**/*.test.ts"],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"include": ["src/**/*.test.ts"],
|
||||
"references": [{ "path": "./tsconfig.src.json" }],
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"moduleDetection": "force",
|
||||
"verbatimModuleSyntax": false,
|
||||
"types": ["vitest/globals"],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"customConditions": ["@triggerdotdev/source"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["**/*.test.ts"],
|
||||
globals: true,
|
||||
isolate: true,
|
||||
fileParallelism: false,
|
||||
testTimeout: 120_000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user