chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,759 @@
|
||||
import {
|
||||
type BatchTriggerTaskV2RequestBody,
|
||||
type BatchTriggerTaskV3RequestBody,
|
||||
type BatchTriggerTaskV3Response,
|
||||
type IOPacket,
|
||||
packetRequiresOffloading,
|
||||
parsePacket,
|
||||
TaskRunErrorCodes,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { Evt } from "evt";
|
||||
import { z } from "zod";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
||||
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
||||
import {
|
||||
downloadPacketFromObjectStore,
|
||||
uploadPacketToObjectStore,
|
||||
} from "../../v3/objectStore.server";
|
||||
import type { RunEngine } from "../../v3/runEngine.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { TriggerTaskService } from "../../v3/services/triggerTask.server";
|
||||
import { startActiveSpan } from "../../v3/tracer.server";
|
||||
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
||||
|
||||
const PROCESSING_BATCH_SIZE = 50;
|
||||
const ASYNC_BATCH_PROCESS_SIZE_THRESHOLD = 20;
|
||||
const MAX_ATTEMPTS = 10;
|
||||
|
||||
export const BatchProcessingStrategy = z.enum(["sequential", "parallel"]);
|
||||
export type BatchProcessingStrategy = z.infer<typeof BatchProcessingStrategy>;
|
||||
|
||||
export const BatchProcessingOptions = z.object({
|
||||
batchId: z.string(),
|
||||
processingId: z.string(),
|
||||
range: z.object({ start: z.number().int(), count: z.number().int() }),
|
||||
attemptCount: z.number().int(),
|
||||
strategy: BatchProcessingStrategy,
|
||||
parentRunId: z.string().optional(),
|
||||
resumeParentOnCompletion: z.boolean().optional(),
|
||||
planType: z.string().optional(),
|
||||
});
|
||||
|
||||
export type BatchProcessingOptions = z.infer<typeof BatchProcessingOptions>;
|
||||
|
||||
export type BatchTriggerTaskServiceOptions = {
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
||||
spanParentAsLink?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
realtimeStreamsVersion?: "v1" | "v2";
|
||||
triggerSource?: string;
|
||||
triggerAction?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Larger batches, used in Run Engine v2
|
||||
*/
|
||||
export class RunEngineBatchTriggerService extends WithRunEngine {
|
||||
private _batchProcessingStrategy: BatchProcessingStrategy;
|
||||
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
||||
|
||||
constructor(
|
||||
batchProcessingStrategy?: BatchProcessingStrategy,
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
engine?: RunEngine
|
||||
) {
|
||||
super({ prisma: _prisma, engine });
|
||||
|
||||
// Eric note: We need to force sequential processing because when doing parallel, we end up with high-contention on the parent run lock
|
||||
// becuase we are triggering a lot of runs at once, and each one is trying to lock the parent run.
|
||||
// by forcing sequential, we are only ever locking the parent run for a single run at a time.
|
||||
this._batchProcessingStrategy = "sequential";
|
||||
}
|
||||
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV3RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
): Promise<BatchTriggerTaskV3Response> {
|
||||
try {
|
||||
return await this.traceWithEnv<BatchTriggerTaskV3Response>(
|
||||
"call()",
|
||||
environment,
|
||||
async (span) => {
|
||||
const { friendlyId } = await mintBatchFriendlyId({
|
||||
environment: {
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: body.parentRunId,
|
||||
});
|
||||
|
||||
span.setAttribute("batchId", friendlyId);
|
||||
|
||||
const payloadPacket = await this.#handlePayloadPacket(
|
||||
body.items,
|
||||
`batch/${friendlyId}`,
|
||||
environment
|
||||
);
|
||||
|
||||
const batch = await this.#createAndProcessBatchTaskRun(
|
||||
friendlyId,
|
||||
payloadPacket,
|
||||
environment,
|
||||
body,
|
||||
options
|
||||
);
|
||||
|
||||
if (!batch) {
|
||||
throw new Error("Failed to create batch");
|
||||
}
|
||||
|
||||
return {
|
||||
id: batch.friendlyId,
|
||||
isCached: false,
|
||||
idempotencyKey: batch.idempotencyKey ?? undefined,
|
||||
runCount: body.items.length,
|
||||
};
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
// Detect a prisma transaction Unique constraint violation
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
logger.debug("RunEngineBatchTrigger: Prisma transaction error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
meta: error.meta,
|
||||
});
|
||||
|
||||
if (error.code === "P2002") {
|
||||
const target = error.meta?.target;
|
||||
|
||||
if (
|
||||
Array.isArray(target) &&
|
||||
target.length > 0 &&
|
||||
typeof target[0] === "string" &&
|
||||
target[0].includes("oneTimeUseToken")
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger with a one-time use token as it has already been used."
|
||||
);
|
||||
} else {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger as it has already been triggered with the same idempotency key."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndProcessBatchTaskRun(
|
||||
batchId: string,
|
||||
payloadPacket: IOPacket,
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV2RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
) {
|
||||
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
||||
// validate env existence app-side (covers both create arms below).
|
||||
await controlPlaneResolver.assertEnvExists(environment.id);
|
||||
|
||||
if (body.items.length <= ASYNC_BATCH_PROCESS_SIZE_THRESHOLD) {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: 0,
|
||||
batchSize: PROCESSING_BATCH_SIZE,
|
||||
items: body.items,
|
||||
options,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing complete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: 0,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
});
|
||||
|
||||
// If processing inline does not finish for some reason, enqueue processing the rest of the batch
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][call] Batch inline processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
});
|
||||
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
switch (this._batchProcessingStrategy) {
|
||||
case "sequential": {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: batchId,
|
||||
range: { start: 0, count: PROCESSING_BATCH_SIZE },
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "parallel": {
|
||||
const ranges = Array.from({
|
||||
length: Math.ceil(body.items.length / PROCESSING_BATCH_SIZE),
|
||||
}).map((_, index) => ({
|
||||
start: index * PROCESSING_BATCH_SIZE,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
}));
|
||||
|
||||
await Promise.all(
|
||||
ranges.map((range, index) =>
|
||||
this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: `${index}`,
|
||||
range,
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
|
||||
async #enqueueBatchTaskRun(options: BatchProcessingOptions) {
|
||||
await batchTriggerWorker.enqueue({
|
||||
id: `RunEngineBatchTriggerService.process:${options.batchId}:${options.processingId}`,
|
||||
job: "runengine.processBatchTaskRun",
|
||||
payload: options,
|
||||
});
|
||||
}
|
||||
|
||||
// This is the function that the worker will call
|
||||
async processBatchTaskRun(options: BatchProcessingOptions) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch", {
|
||||
options,
|
||||
});
|
||||
|
||||
const $attemptCount = options.attemptCount + 1;
|
||||
|
||||
if ($attemptCount > MAX_ATTEMPTS) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Max attempts reached", {
|
||||
options,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
// You might want to update the batch status to failed here
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = await this._engine.runStore.findBatchTaskRunById(options.batchId);
|
||||
|
||||
if (!batch) {
|
||||
return;
|
||||
}
|
||||
|
||||
// BatchTaskRun -> RuntimeEnvironment FK is dropped; resolve the env from the scalar id.
|
||||
const environment = await findEnvironmentById(batch.runtimeEnvironmentId);
|
||||
if (!environment) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Environment not found", {
|
||||
batchId: batch.id,
|
||||
runtimeEnvironmentId: batch.runtimeEnvironmentId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.range.start >= batch.runCount) {
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] currentIndex is greater than runCount",
|
||||
{
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
runCount: batch.runCount,
|
||||
attemptCount: $attemptCount,
|
||||
}
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const payloadPacket = await downloadPacketFromObjectStore(
|
||||
{
|
||||
data: batch.payload ?? undefined,
|
||||
dataType: batch.payloadType,
|
||||
},
|
||||
environment
|
||||
);
|
||||
|
||||
const payload = await parsePacket(payloadPacket);
|
||||
|
||||
if (!payload) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Failed to parse payload", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
throw new Error("Failed to parse payload");
|
||||
}
|
||||
|
||||
// Skip zod parsing
|
||||
const $payload = payload as BatchTriggerTaskV2RequestBody["items"];
|
||||
const $options = batch.options as BatchTriggerTaskServiceOptions;
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: options.range.start,
|
||||
batchSize: options.range.count,
|
||||
items: $payload,
|
||||
options: $options,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing complete", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// Only enqueue the next batch task run if the strategy is sequential
|
||||
// if the strategy is parallel, we will already have enqueued the next batch task run
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// if the strategy is sequential, we will requeue processing with a count of the PROCESSING_BATCH_SIZE
|
||||
// if the strategy is parallel, we will requeue processing with a range starting at the workingIndex and a count that is the remainder of this "slice" of the batch
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count, // This will be the same as the original count
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
} else {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
// This will be the remainder of the slice
|
||||
// for example if the original range was 0-50 and the workingIndex is 25, the new range will be 25-25
|
||||
// if the original range was 51-100 and the workingIndex is 75, the new range will be 75-25
|
||||
count: options.range.count - result.workingIndex - options.range.start,
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex,
|
||||
batchSize,
|
||||
items,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
currentIndex: number;
|
||||
batchSize: number;
|
||||
items: BatchTriggerTaskV2RequestBody["items"];
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId?: string | undefined;
|
||||
resumeParentOnCompletion?: boolean | undefined;
|
||||
}): Promise<
|
||||
| { status: "COMPLETE" }
|
||||
| { status: "INCOMPLETE"; workingIndex: number }
|
||||
| { status: "ERROR"; error: string; workingIndex: number }
|
||||
> {
|
||||
// Grab the next PROCESSING_BATCH_SIZE items
|
||||
const itemsToProcess = items.slice(currentIndex, currentIndex + batchSize);
|
||||
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch items", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
runCount: batch.runCount,
|
||||
});
|
||||
|
||||
let workingIndex = currentIndex;
|
||||
|
||||
let runIds: string[] = [];
|
||||
|
||||
const triggerFailedTaskService = new TriggerFailedTaskService({
|
||||
prisma: this._prisma,
|
||||
engine: this._engine,
|
||||
replicaPrisma: this._replica,
|
||||
});
|
||||
|
||||
for (const item of itemsToProcess) {
|
||||
let runFriendlyId: string | null = null;
|
||||
|
||||
try {
|
||||
const run = await this.#processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex: workingIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
if (run) {
|
||||
runFriendlyId = run.friendlyId;
|
||||
}
|
||||
} catch (_error) {
|
||||
// Trigger failed - will try to create pre-failed run below
|
||||
runFriendlyId = null;
|
||||
}
|
||||
|
||||
if (!runFriendlyId) {
|
||||
const errorMessage =
|
||||
"Trigger failed for batch item (queue limit, entitlement, or validation error)";
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Item trigger failed, creating pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
task: item.task,
|
||||
}
|
||||
);
|
||||
|
||||
const failedRunId = await triggerFailedTaskService.call({
|
||||
taskId: item.task,
|
||||
environment,
|
||||
payload: item.payload,
|
||||
payloadType: item.options?.payloadType,
|
||||
errorMessage,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
batch: { id: batch.id, index: workingIndex },
|
||||
// Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
|
||||
// co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
|
||||
runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
|
||||
options: item.options as Record<string, unknown>,
|
||||
traceContext: options?.traceContext as Record<string, unknown> | undefined,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
|
||||
});
|
||||
|
||||
if (failedRunId) {
|
||||
runFriendlyId = failedRunId;
|
||||
} else {
|
||||
logger.error(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Failed to create pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
status: "ERROR",
|
||||
error: "Could not trigger item and could not create pre-failed run",
|
||||
workingIndex,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
runIds.push(runFriendlyId);
|
||||
workingIndex++;
|
||||
}
|
||||
|
||||
const updatedBatch = await this._engine.runStore.updateBatchTaskRun({
|
||||
where: { id: batch.id },
|
||||
data: {
|
||||
runIds: {
|
||||
push: runIds,
|
||||
},
|
||||
processingJobsCount: {
|
||||
increment: runIds.length,
|
||||
},
|
||||
},
|
||||
select: {
|
||||
processingJobsCount: true,
|
||||
runCount: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (updatedBatch.processingJobsCount >= updatedBatch.runCount) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] All runs created", {
|
||||
batchId: batch.friendlyId,
|
||||
processingJobsCount: updatedBatch.processingJobsCount,
|
||||
runCount: updatedBatch.runCount,
|
||||
workingIndex,
|
||||
});
|
||||
|
||||
//if all the runs were idempotent, it's possible the batch is already completed
|
||||
await this._engine.tryCompleteBatch({ batchId: batch.id });
|
||||
}
|
||||
|
||||
// if there are more items to process, requeue the batch
|
||||
if (workingIndex < batch.runCount) {
|
||||
return { status: "INCOMPLETE", workingIndex };
|
||||
}
|
||||
|
||||
return { status: "COMPLETE" };
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
item: BatchTriggerTaskV2RequestBody["items"][number];
|
||||
currentIndex: number;
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId: string | undefined;
|
||||
resumeParentOnCompletion: boolean | undefined;
|
||||
}) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRunItem] Processing item", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
});
|
||||
|
||||
const triggerTaskService = new TriggerTaskService();
|
||||
|
||||
// Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
|
||||
// an org's mint flag flipping between batch creation and this (possibly much later,
|
||||
// worker-processed) item never splits the item from its BatchTaskRun row. See
|
||||
// mintAnchoredRunFriendlyId.server.ts.
|
||||
const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
|
||||
|
||||
const result = await triggerTaskService.call(
|
||||
item.task,
|
||||
environment,
|
||||
{
|
||||
...item,
|
||||
options: {
|
||||
...item.options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
parentBatch: batch.id,
|
||||
},
|
||||
},
|
||||
{
|
||||
triggerVersion: options?.triggerVersion,
|
||||
traceContext: options?.traceContext,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
batchId: batch.id,
|
||||
batchIndex: currentIndex,
|
||||
runFriendlyId,
|
||||
realtimeStreamsVersion: options?.realtimeStreamsVersion,
|
||||
triggerSource: options?.triggerSource ?? "api",
|
||||
triggerAction: options?.triggerAction ?? "trigger",
|
||||
},
|
||||
"V2"
|
||||
);
|
||||
|
||||
return result
|
||||
? {
|
||||
friendlyId: result.run.friendlyId,
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async #handlePayloadPacket(
|
||||
payload: any,
|
||||
pathPrefix: string,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
||||
const packet = { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${pathPrefix}/payload.json`;
|
||||
|
||||
const uploadedFilename = await uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
environment
|
||||
);
|
||||
|
||||
return {
|
||||
data: uploadedFilename,
|
||||
dataType: "application/store",
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import type { InitializeBatchOptions } from "@internal/run-engine";
|
||||
import { type CreateBatchRequestBody, type CreateBatchResponse } from "@trigger.dev/core/v3";
|
||||
import { RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { Evt } from "evt";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { BatchRateLimitExceededError, getBatchLimits } from "../concerns/batchLimits.server";
|
||||
import { DefaultQueueManager } from "../concerns/queues.server";
|
||||
import { DefaultTriggerTaskValidator } from "../validators/triggerTaskValidator";
|
||||
|
||||
export type CreateBatchServiceOptions = {
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
||||
spanParentAsLink?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
realtimeStreamsVersion?: "v1" | "v2";
|
||||
triggerSource?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Create Batch Service (Phase 1 of 2-phase batch API).
|
||||
*
|
||||
* This service handles Phase 1 of the streaming batch API:
|
||||
* 1. Validates entitlement and queue limits
|
||||
* 2. Creates BatchTaskRun in Postgres with status=PENDING, expectedCount set
|
||||
* 3. For batchTriggerAndWait: blocks the parent run immediately
|
||||
* 4. Initializes batch metadata in Redis
|
||||
* 5. Returns batch ID - items are streamed separately via Phase 2
|
||||
*
|
||||
* The batch is NOT sealed until Phase 2 completes.
|
||||
*/
|
||||
export class CreateBatchService extends WithRunEngine {
|
||||
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
||||
private readonly queueConcern: DefaultQueueManager;
|
||||
private readonly validator: DefaultTriggerTaskValidator;
|
||||
|
||||
constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {
|
||||
super({ prisma: _prisma });
|
||||
|
||||
this.queueConcern = new DefaultQueueManager(this._prisma, this._engine, this._replica);
|
||||
this.validator = new DefaultTriggerTaskValidator();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a batch for 2-phase processing.
|
||||
* Items will be streamed separately via the StreamBatchItemsService.
|
||||
*/
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: CreateBatchRequestBody,
|
||||
options: CreateBatchServiceOptions = {}
|
||||
): Promise<CreateBatchResponse> {
|
||||
try {
|
||||
return await this.traceWithEnv<CreateBatchResponse>(
|
||||
"createBatch()",
|
||||
environment,
|
||||
async (span) => {
|
||||
const { id, friendlyId } = await mintBatchFriendlyId({
|
||||
environment: {
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: body.parentRunId,
|
||||
});
|
||||
|
||||
span.setAttribute("batchId", friendlyId);
|
||||
span.setAttribute("runCount", body.runCount);
|
||||
|
||||
const entitlementValidation = await this.validator.validateEntitlement({
|
||||
environment,
|
||||
});
|
||||
|
||||
if (!entitlementValidation.ok) {
|
||||
throw entitlementValidation.error;
|
||||
}
|
||||
|
||||
const planType = entitlementValidation.plan?.type;
|
||||
|
||||
const { config, rateLimiter } = await getBatchLimits(environment.organization);
|
||||
|
||||
// Rate-limit before creating the batch, to stop bursts exceeding the limit.
|
||||
const rateResult = await rateLimiter.limit(environment.id, body.runCount);
|
||||
|
||||
if (!rateResult.success) {
|
||||
throw new BatchRateLimitExceededError(
|
||||
rateResult.limit,
|
||||
rateResult.remaining,
|
||||
new Date(rateResult.reset),
|
||||
body.runCount
|
||||
);
|
||||
}
|
||||
|
||||
// Note: Queue size limits are validated per-queue when batch items are processed,
|
||||
// since we don't know which queues items will go to until they're streamed.
|
||||
|
||||
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
||||
// validate env existence app-side (passthrough when split is off).
|
||||
await controlPlaneResolver.assertEnvExists(environment.id);
|
||||
|
||||
// Created PENDING; sealed (status -> PROCESSING) once items are streamed.
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id,
|
||||
friendlyId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
status: "PENDING",
|
||||
runCount: body.runCount,
|
||||
expectedCount: body.runCount,
|
||||
runIds: [],
|
||||
batchVersion: "runengine:v2", // 2-phase streaming batch API
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
sealed: false,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
// Block parent run if this is a batchTriggerAndWait
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize batch metadata in Redis (without items)
|
||||
const initOptions: InitializeBatchOptions = {
|
||||
batchId: id,
|
||||
friendlyId,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
organizationId: environment.organizationId,
|
||||
projectId: environment.projectId,
|
||||
runCount: body.runCount,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
triggerVersion: options.triggerVersion,
|
||||
traceContext: options.traceContext as Record<string, unknown> | undefined,
|
||||
spanParentAsLink: options.spanParentAsLink,
|
||||
realtimeStreamsVersion: options.realtimeStreamsVersion,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
processingConcurrency: config.processingConcurrency,
|
||||
planType,
|
||||
triggerSource: options.triggerSource,
|
||||
};
|
||||
|
||||
await this._engine.initializeBatch(initOptions);
|
||||
|
||||
logger.info("Batch created", {
|
||||
batchId: friendlyId,
|
||||
runCount: body.runCount,
|
||||
envId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
processingConcurrency: config.processingConcurrency,
|
||||
});
|
||||
|
||||
return {
|
||||
id: friendlyId,
|
||||
runCount: body.runCount,
|
||||
isCached: false,
|
||||
idempotencyKey: body.idempotencyKey,
|
||||
};
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
// Handle Prisma unique constraint violations
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
logger.debug("CreateBatchService: Prisma error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
meta: error.meta,
|
||||
});
|
||||
|
||||
if (error.code === "P2002") {
|
||||
const target = error.meta?.target;
|
||||
|
||||
if (
|
||||
Array.isArray(target) &&
|
||||
target.length > 0 &&
|
||||
typeof target[0] === "string" &&
|
||||
target[0].includes("oneTimeUseToken")
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot create batch with a one-time use token as it has already been used."
|
||||
);
|
||||
} else {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot create batch as it has already been created with the same idempotency key."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,837 @@
|
||||
import {
|
||||
type StreamBatchItemsResponse,
|
||||
BatchItemNDJSON as BatchItemNDJSONSchema,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { BatchId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { BatchItem, RunEngine } from "@internal/run-engine";
|
||||
import pMap from "p-map";
|
||||
import type { BatchTaskRunStatus } from "@trigger.dev/database";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { BatchPayloadProcessor } from "../concerns/batchPayloads.server";
|
||||
|
||||
/**
|
||||
* Phase 2 retry idempotency check.
|
||||
*
|
||||
* Returns true when the batch is in a state that means the Phase 2 stream's
|
||||
* job has already been done — every item has a TaskRun record (real or
|
||||
* pre-failed) for the customer to monitor. A retry, or the original call
|
||||
* racing against a fast-completing BatchQueue, should return sealed:true
|
||||
* in these states so the SDK stops retrying.
|
||||
*
|
||||
* Three "work is done" shapes:
|
||||
* - status moved out of PENDING into PROCESSING/COMPLETED/PARTIAL_FAILED
|
||||
* (PROCESSING via our seal, COMPLETED via tryCompleteBatch, PARTIAL_FAILED
|
||||
* via the V2 batchCompletionCallback).
|
||||
* - status stuck at PENDING but `sealed=true`: another concurrent
|
||||
* streamBatchItems call sealed the batch and then the callback's
|
||||
* happy-path branch reset status to PENDING ("all runs created").
|
||||
* - status stuck at PENDING with `sealed=false` but `processingCompletedAt`
|
||||
* set: the cleanup-race. BatchQueue rushed through all items, callback
|
||||
* fired (setting processingCompletedAt), cleanup deleted the Redis
|
||||
* metadata — all before our service got the chance to seal. The work
|
||||
* is done; the discriminator is processingCompletedAt which is set
|
||||
* exclusively by the V2 completion callback.
|
||||
*
|
||||
* ABORTED is excluded — it means ZERO TaskRun records were created (every
|
||||
* per-item attempt failed AND the pre-failed-TaskRun fallback also failed,
|
||||
* or queue-overload on every item). The customer has nothing to monitor
|
||||
* at the run level, so the trigger call must throw to give their retry/
|
||||
* error handling a chance to create a fresh batch.
|
||||
*/
|
||||
export function isIdempotentRetrySuccess(
|
||||
status: BatchTaskRunStatus | null | undefined,
|
||||
sealed: boolean | null | undefined,
|
||||
processingCompletedAt: Date | null | undefined
|
||||
): boolean {
|
||||
return (
|
||||
status === "PROCESSING" ||
|
||||
status === "COMPLETED" ||
|
||||
status === "PARTIAL_FAILED" ||
|
||||
(status === "PENDING" && (sealed === true || processingCompletedAt != null))
|
||||
);
|
||||
}
|
||||
|
||||
export type StreamBatchItemsServiceOptions = {
|
||||
maxItemBytes: number;
|
||||
/** Max items processed concurrently. The route wires this to STREAMING_BATCH_INGEST_CONCURRENCY. */
|
||||
concurrency: number;
|
||||
};
|
||||
|
||||
export type OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED";
|
||||
index: number;
|
||||
task: string;
|
||||
actualSize: number;
|
||||
maxSize: number;
|
||||
};
|
||||
|
||||
export type StreamBatchItemsServiceConstructorOptions = {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
engine?: RunEngine;
|
||||
/** Override the payload processor (used in tests to observe ingest concurrency). */
|
||||
payloadProcessor?: BatchPayloadProcessor;
|
||||
};
|
||||
|
||||
/**
|
||||
* Stream Batch Items Service (Phase 2 of 2-phase batch API).
|
||||
*
|
||||
* This service handles Phase 2 of the streaming batch API:
|
||||
* 1. Validates batch exists and is in PENDING status
|
||||
* 2. Processes NDJSON stream item by item
|
||||
* 3. Calls engine.enqueueBatchItem() for each item
|
||||
* 4. Tracks accepted/deduplicated counts
|
||||
* 5. On completion: validates count, seals the batch
|
||||
*
|
||||
* The service is designed for streaming and processes items as they arrive,
|
||||
* providing backpressure through the async iterator pattern.
|
||||
*/
|
||||
export class StreamBatchItemsService extends WithRunEngine {
|
||||
private readonly payloadProcessor: BatchPayloadProcessor;
|
||||
|
||||
constructor(opts: StreamBatchItemsServiceConstructorOptions = {}) {
|
||||
super({ prisma: opts.prisma ?? prisma, engine: opts.engine });
|
||||
this.payloadProcessor = opts.payloadProcessor ?? new BatchPayloadProcessor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a batch friendly ID to its internal ID format.
|
||||
* Throws a ServiceValidationError with 400 status if the ID is malformed.
|
||||
*/
|
||||
private parseBatchFriendlyId(friendlyId: string): string {
|
||||
try {
|
||||
return BatchId.fromFriendlyId(friendlyId);
|
||||
} catch {
|
||||
throw new ServiceValidationError(`Invalid batchFriendlyId: ${friendlyId}`, 400);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a stream of batch items from an async iterator.
|
||||
* Each item is validated and enqueued to the BatchQueue.
|
||||
* The batch is sealed when the stream completes.
|
||||
*/
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
batchFriendlyId: string,
|
||||
itemsIterator: AsyncIterable<unknown>,
|
||||
options: StreamBatchItemsServiceOptions
|
||||
): Promise<StreamBatchItemsResponse> {
|
||||
return this.traceWithEnv<StreamBatchItemsResponse>(
|
||||
"streamBatchItems()",
|
||||
environment,
|
||||
async (span) => {
|
||||
span.setAttribute("batchId", batchFriendlyId);
|
||||
|
||||
// Convert friendly ID to internal ID
|
||||
const batchId = this.parseBatchFriendlyId(batchFriendlyId);
|
||||
|
||||
// Validate batch exists and belongs to this environment. Routed by batch id so a
|
||||
// run-ops id (NEW-resident) batch is found on the owning DB; the env-ownership check that
|
||||
// was in the where clause is enforced app-side below.
|
||||
const batch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (!batch || batch.runtimeEnvironmentId !== environment.id) {
|
||||
throw new ServiceValidationError(`Batch ${batchFriendlyId} not found`);
|
||||
}
|
||||
|
||||
if (isIdempotentRetrySuccess(batch.status, batch.sealed, batch.processingCompletedAt)) {
|
||||
logger.info("Batch already sealed/completed - treating Phase 2 retry as success", {
|
||||
batchId: batchFriendlyId,
|
||||
batchSealed: batch.sealed,
|
||||
batchStatus: batch.status,
|
||||
processingCompletedAt: batch.processingCompletedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted: 0,
|
||||
itemsDeduplicated: 0,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
if (batch.status !== "PENDING") {
|
||||
// ABORTED or any other unexpected non-PENDING state — surface as an error.
|
||||
// For ABORTED specifically, throwing is required so the customer's
|
||||
// batchTrigger() retries (a new batch) can recreate the runs.
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is not in PENDING status (current: ${batch.status})`
|
||||
);
|
||||
}
|
||||
|
||||
// Process items from the stream with bounded concurrency.
|
||||
//
|
||||
// Ordering and idempotency do NOT depend on processing order:
|
||||
// - The BatchQueue derives run order from each item's index
|
||||
// (enqueue timestamp = batch.createdAt + itemIndex), not enqueue order.
|
||||
// - enqueueBatchItem() dedups atomically per index.
|
||||
// We cap concurrency to bound peak in-flight memory (≈ concurrency ×
|
||||
// maxItemBytes) and to keep backpressure on the request body stream.
|
||||
// p-map pulls lazily from the async iterator — at most `concurrency`
|
||||
// items are read and in flight at once. stopOnError aborts ingestion on
|
||||
// the first failure (the batch is left unsealed; the SDK's retry
|
||||
// re-streams and dedups already-enqueued items).
|
||||
const outcomes = await pMap(
|
||||
itemsIterator,
|
||||
(rawItem) => this.#processItem(rawItem, batchId, environment, batch.runCount),
|
||||
{ concurrency: options.concurrency, stopOnError: true }
|
||||
);
|
||||
|
||||
let itemsAccepted = 0;
|
||||
let itemsDeduplicated = 0;
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome === "accepted") {
|
||||
itemsAccepted++;
|
||||
} else {
|
||||
itemsDeduplicated++;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the actual enqueued count from Redis
|
||||
const enqueuedCount = await this._engine.getBatchEnqueuedCount(batchId);
|
||||
|
||||
// Validate we received the expected number of items
|
||||
if (enqueuedCount !== batch.runCount) {
|
||||
// The batch queue consumers may have already processed all items and
|
||||
// cleaned up the Redis keys before we got here. This happens when all
|
||||
// runs complete fast enough that cleanup() deletes the enqueuedItemsKey
|
||||
// before we read it — typically when the last item executes in the
|
||||
// milliseconds between the loop ending and getBatchEnqueuedCount() being called.
|
||||
// Check both sealed (sealed by this endpoint on a concurrent request) and
|
||||
// COMPLETED (sealed by the BatchQueue completion path before we got here).
|
||||
const currentBatch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (
|
||||
isIdempotentRetrySuccess(
|
||||
currentBatch?.status,
|
||||
currentBatch?.sealed,
|
||||
currentBatch?.processingCompletedAt
|
||||
)
|
||||
) {
|
||||
logger.info("Batch already sealed before count check (fast completion)", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
enqueuedCount,
|
||||
expectedCount: batch.runCount,
|
||||
batchStatus: currentBatch?.status,
|
||||
processingCompletedAt: currentBatch?.processingCompletedAt,
|
||||
});
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
if (currentBatch?.status === "ABORTED") {
|
||||
// Zero TaskRuns exist — the count-mismatch sealed:false semantics
|
||||
// ("retry with missing items") would mislead the SDK. Throw so the
|
||||
// customer's batchTrigger() retry creates a fresh batch.
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is not in PENDING status (current: ABORTED)`
|
||||
);
|
||||
}
|
||||
|
||||
logger.warn("Batch item count mismatch", {
|
||||
batchId: batchFriendlyId,
|
||||
expected: batch.runCount,
|
||||
received: enqueuedCount,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
});
|
||||
|
||||
// Don't seal the batch if count doesn't match
|
||||
// Return sealed: false so client knows to retry with missing items
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: false,
|
||||
enqueuedCount,
|
||||
expectedCount: batch.runCount,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
// Seal the batch - use conditional update to prevent TOCTOU race
|
||||
// Another concurrent request may have already sealed this batch
|
||||
const now = new Date();
|
||||
const sealResult = await this._engine.runStore.updateManyBatchTaskRun({
|
||||
where: {
|
||||
id: batchId,
|
||||
sealed: false,
|
||||
status: "PENDING",
|
||||
},
|
||||
data: {
|
||||
sealed: true,
|
||||
sealedAt: now,
|
||||
status: "PROCESSING",
|
||||
processingStartedAt: now,
|
||||
},
|
||||
});
|
||||
|
||||
// Check if we won the race to seal the batch
|
||||
if (sealResult.count === 0) {
|
||||
// The conditional update failed because the batch was no longer in
|
||||
// PENDING status. Re-query to determine which path got there first:
|
||||
// - A concurrent streaming request already sealed and moved it to
|
||||
// PROCESSING.
|
||||
// - The BatchQueue completion path finished all runs and set it to
|
||||
// COMPLETED (without setting sealed=true — that's this endpoint's
|
||||
// job). This window exists between completionCallback (which calls
|
||||
// tryCompleteBatch) and cleanup() in BatchQueue — see
|
||||
// batch-queue/index.ts.
|
||||
// Either way the goal — a durable batch that the SDK stops retrying —
|
||||
// has been achieved, so we return sealed: true.
|
||||
const currentBatch = await this._engine.runStore.findBatchTaskRunById(batchId);
|
||||
|
||||
if (
|
||||
isIdempotentRetrySuccess(
|
||||
currentBatch?.status,
|
||||
currentBatch?.sealed,
|
||||
currentBatch?.processingCompletedAt
|
||||
)
|
||||
) {
|
||||
logger.info("Batch already sealed/completed by concurrent path", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
envId: environment.id,
|
||||
batchStatus: currentBatch?.status,
|
||||
batchSealed: currentBatch?.sealed,
|
||||
processingCompletedAt: currentBatch?.processingCompletedAt,
|
||||
});
|
||||
|
||||
span.setAttribute("itemsAccepted", itemsAccepted);
|
||||
span.setAttribute("itemsDeduplicated", itemsDeduplicated);
|
||||
span.setAttribute("sealedByConcurrentRequest", true);
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
|
||||
// Batch is in an unexpected state - fail with error
|
||||
const actualStatus = currentBatch?.status ?? "unknown";
|
||||
const actualSealed = currentBatch?.sealed ?? "unknown";
|
||||
logger.error("Batch seal race condition: unexpected state", {
|
||||
batchId: batchFriendlyId,
|
||||
expectedStatus: "PENDING",
|
||||
actualStatus,
|
||||
expectedSealed: false,
|
||||
actualSealed,
|
||||
envId: environment.id,
|
||||
});
|
||||
|
||||
throw new ServiceValidationError(
|
||||
`Batch ${batchFriendlyId} is in unexpected state (status: ${actualStatus}, sealed: ${actualSealed}). Cannot seal batch.`
|
||||
);
|
||||
}
|
||||
|
||||
logger.info("Batch sealed and ready for processing", {
|
||||
batchId: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
totalEnqueued: enqueuedCount,
|
||||
envId: environment.id,
|
||||
});
|
||||
|
||||
span.setAttribute("itemsAccepted", itemsAccepted);
|
||||
span.setAttribute("itemsDeduplicated", itemsDeduplicated);
|
||||
|
||||
return {
|
||||
id: batchFriendlyId,
|
||||
itemsAccepted,
|
||||
itemsDeduplicated,
|
||||
sealed: true,
|
||||
runCount: batch.runCount,
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single streamed batch item: validate it, offload its payload to
|
||||
* object storage if oversized, and enqueue it. Returns whether the item was
|
||||
* newly enqueued ("accepted") or was a duplicate ("deduplicated"). Throws
|
||||
* ServiceValidationError for invalid items, which aborts the stream.
|
||||
*
|
||||
* Safe to run concurrently: enqueueBatchItem() is atomic and order-independent
|
||||
* per item index, and each item carries its own index (real items from the
|
||||
* SDK; oversized markers are stamped by the NDJSON parser).
|
||||
*/
|
||||
async #processItem(
|
||||
rawItem: unknown,
|
||||
batchId: string,
|
||||
environment: AuthenticatedEnvironment,
|
||||
runCount: number
|
||||
): Promise<"accepted" | "deduplicated"> {
|
||||
// Oversized item marker emitted by the NDJSON parser
|
||||
if (rawItem && typeof rawItem === "object" && "__batchItemError" in rawItem) {
|
||||
const marker = rawItem as OversizedItemMarker;
|
||||
|
||||
// Same out-of-range guard as normal items: an oversized item with an
|
||||
// out-of-range index must 4xx rather than create a stray pre-failed run.
|
||||
if (marker.index >= runCount) {
|
||||
throw new ServiceValidationError(
|
||||
`Item index ${marker.index} exceeds batch runCount ${runCount}`
|
||||
);
|
||||
}
|
||||
|
||||
const errorMessage = `Batch item payload is too large (${(marker.actualSize / 1024).toFixed(
|
||||
1
|
||||
)} KB). Maximum allowed size is ${(marker.maxSize / 1024).toFixed(
|
||||
1
|
||||
)} KB. Reduce the payload size or offload large data to external storage.`;
|
||||
|
||||
// Enqueue with __error metadata - processItemCallback will detect this
|
||||
// and use TriggerFailedTaskService to create a pre-failed run
|
||||
const batchItem: BatchItem = {
|
||||
task: marker.task,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
options: {
|
||||
__error: errorMessage,
|
||||
__errorCode: "PAYLOAD_TOO_LARGE",
|
||||
},
|
||||
};
|
||||
|
||||
const result = await this._engine.enqueueBatchItem(
|
||||
batchId,
|
||||
environment.id,
|
||||
marker.index,
|
||||
batchItem
|
||||
);
|
||||
|
||||
return result.enqueued ? "accepted" : "deduplicated";
|
||||
}
|
||||
|
||||
// Parse and validate the item
|
||||
const parseResult = BatchItemNDJSONSchema.safeParse(rawItem);
|
||||
if (!parseResult.success) {
|
||||
const rawIndex = (rawItem as { index?: unknown } | null)?.index;
|
||||
const where = typeof rawIndex === "number" ? `index ${rawIndex}` : "unknown index";
|
||||
throw new ServiceValidationError(`Invalid item at ${where}: ${parseResult.error.message}`);
|
||||
}
|
||||
|
||||
const item = parseResult.data;
|
||||
|
||||
// Validate index is within expected range
|
||||
if (item.index >= runCount) {
|
||||
throw new ServiceValidationError(
|
||||
`Item index ${item.index} exceeds batch runCount ${runCount}`
|
||||
);
|
||||
}
|
||||
|
||||
// Get the original payload type
|
||||
const originalPayloadType = (item.options?.payloadType as string) ?? "application/json";
|
||||
|
||||
// Process payload - offload to object storage if it exceeds threshold
|
||||
const processedPayload = await this.payloadProcessor.process(
|
||||
item.payload,
|
||||
originalPayloadType,
|
||||
batchId,
|
||||
item.index,
|
||||
environment
|
||||
);
|
||||
|
||||
// Convert to BatchItem format with potentially offloaded payload
|
||||
const batchItem: BatchItem = {
|
||||
task: item.task,
|
||||
payload: processedPayload.payload,
|
||||
payloadType: processedPayload.payloadType,
|
||||
options: item.options,
|
||||
};
|
||||
|
||||
// Enqueue the item
|
||||
const result = await this._engine.enqueueBatchItem(
|
||||
batchId,
|
||||
environment.id,
|
||||
item.index,
|
||||
batchItem
|
||||
);
|
||||
|
||||
return result.enqueued ? "accepted" : "deduplicated";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `index` and `task` from raw JSON bytes without decoding the full line.
|
||||
* Scans at most 512 bytes, tracking JSON nesting depth to only match top-level keys.
|
||||
*/
|
||||
export function extractIndexAndTask(bytes: Uint8Array): { index: number; task: string } {
|
||||
let index = -1;
|
||||
let task = "unknown";
|
||||
let depth = 0;
|
||||
let foundIndex = false;
|
||||
let foundTask = false;
|
||||
const limit = Math.min(bytes.byteLength, 512);
|
||||
|
||||
const QUOTE = 0x22; // "
|
||||
const COLON = 0x3a; // :
|
||||
const LBRACE = 0x7b; // {
|
||||
const RBRACE = 0x7d; // }
|
||||
const LBRACKET = 0x5b; // [
|
||||
const RBRACKET = 0x5d; // ]
|
||||
const BACKSLASH = 0x5c; // \
|
||||
|
||||
// Byte patterns for "index" and "task" (without quotes)
|
||||
const INDEX_BYTES = [0x69, 0x6e, 0x64, 0x65, 0x78]; // index
|
||||
const TASK_BYTES = [0x74, 0x61, 0x73, 0x6b]; // task
|
||||
|
||||
let i = 0;
|
||||
while (i < limit && !(foundIndex && foundTask)) {
|
||||
const b = bytes[i];
|
||||
|
||||
if (b === LBRACE || b === LBRACKET) {
|
||||
depth++;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (b === RBRACE || b === RBRACKET) {
|
||||
depth--;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Only match keys at depth 1 (top-level object)
|
||||
if (b === QUOTE && depth === 1) {
|
||||
// Read the key inside quotes
|
||||
const keyStart = i + 1;
|
||||
let keyEnd = keyStart;
|
||||
while (keyEnd < limit && bytes[keyEnd] !== QUOTE) {
|
||||
if (bytes[keyEnd] === BACKSLASH) keyEnd++; // skip escaped char
|
||||
keyEnd++;
|
||||
}
|
||||
|
||||
const keyLen = keyEnd - keyStart;
|
||||
|
||||
// Check if this key matches "index" or "task"
|
||||
const isIndex =
|
||||
!foundIndex &&
|
||||
keyLen === INDEX_BYTES.length &&
|
||||
INDEX_BYTES.every((b, j) => bytes[keyStart + j] === b);
|
||||
const isTask =
|
||||
!foundTask &&
|
||||
keyLen === TASK_BYTES.length &&
|
||||
TASK_BYTES.every((b, j) => bytes[keyStart + j] === b);
|
||||
|
||||
if (isIndex || isTask) {
|
||||
// Skip past closing quote and find colon
|
||||
let pos = keyEnd + 1;
|
||||
while (pos < limit && bytes[pos] !== COLON) pos++;
|
||||
pos++; // skip colon
|
||||
// Skip whitespace
|
||||
while (pos < limit && (bytes[pos] === 0x20 || bytes[pos] === 0x09)) pos++;
|
||||
|
||||
if (isIndex) {
|
||||
// Parse digits
|
||||
let num = 0;
|
||||
let hasDigit = false;
|
||||
while (pos < limit && bytes[pos] >= 0x30 && bytes[pos] <= 0x39) {
|
||||
num = num * 10 + (bytes[pos] - 0x30);
|
||||
hasDigit = true;
|
||||
pos++;
|
||||
}
|
||||
if (hasDigit) {
|
||||
index = num;
|
||||
foundIndex = true;
|
||||
}
|
||||
} else {
|
||||
// Parse quoted string value
|
||||
if (pos < limit && bytes[pos] === QUOTE) {
|
||||
const valStart = pos + 1;
|
||||
let valEnd = valStart;
|
||||
while (valEnd < limit && bytes[valEnd] !== QUOTE) {
|
||||
if (bytes[valEnd] === BACKSLASH) valEnd++;
|
||||
valEnd++;
|
||||
}
|
||||
// Decode just this slice
|
||||
try {
|
||||
task = new TextDecoder("utf-8", { fatal: true }).decode(
|
||||
bytes.slice(valStart, valEnd)
|
||||
);
|
||||
foundTask = true;
|
||||
} catch {
|
||||
// Leave as "unknown"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip past the key's closing quote
|
||||
i = keyEnd + 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
return { index, task };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an NDJSON parser transform stream.
|
||||
*
|
||||
* Converts a stream of Uint8Array chunks into parsed JSON objects.
|
||||
* Each line in the NDJSON is parsed independently.
|
||||
*
|
||||
* Uses byte-buffer accumulation to:
|
||||
* - Prevent OOM from unbounded string buffers
|
||||
* - Properly handle multibyte UTF-8 characters across chunk boundaries
|
||||
* - Check size limits on raw bytes before decoding
|
||||
*
|
||||
* @param maxItemBytes - Maximum allowed bytes per line (item)
|
||||
* @returns TransformStream that outputs parsed JSON objects
|
||||
*/
|
||||
export function createNdjsonParserStream(
|
||||
maxItemBytes: number
|
||||
): TransformStream<Uint8Array, unknown> {
|
||||
// Single decoder instance, reused for all lines
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
|
||||
// Byte buffer: array of chunks with tracked total length
|
||||
let chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
let lineNumber = 0;
|
||||
// 0-based position of the next object we emit (parsed item or oversized
|
||||
// marker). The parser is the single sequential point in the pipeline, so this
|
||||
// is the authoritative source of item ordering — downstream consumers can
|
||||
// process items concurrently and must not rely on processing order to derive
|
||||
// an item's index. Used to back-fill an oversized marker's index when it
|
||||
// couldn't be extracted from the (truncated) raw bytes.
|
||||
let emittedCount = 0;
|
||||
// When an oversized incomplete line is detected (Case 2), we must discard
|
||||
// all remaining bytes of that line until the next newline delimiter.
|
||||
let skipUntilNewline = false;
|
||||
|
||||
const NEWLINE_BYTE = 0x0a; // '\n'
|
||||
|
||||
/**
|
||||
* Emit a parsed object or marker downstream and advance the emit position.
|
||||
* Every emitted object MUST go through here so `emittedCount` stays aligned
|
||||
* with item position (empty/skipped lines never emit, so they don't count).
|
||||
*/
|
||||
function emit(controller: TransformStreamDefaultController<unknown>, obj: unknown): void {
|
||||
controller.enqueue(obj);
|
||||
emittedCount++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate all chunks into a single Uint8Array
|
||||
*/
|
||||
function concatenateChunks(): Uint8Array {
|
||||
if (chunks.length === 0) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
if (chunks.length === 1) {
|
||||
return chunks[0];
|
||||
}
|
||||
const result = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the index of the first newline byte in the buffer.
|
||||
* Returns -1 if not found.
|
||||
*/
|
||||
function findNewlineIndex(): number {
|
||||
let globalIndex = 0;
|
||||
for (const chunk of chunks) {
|
||||
for (let i = 0; i < chunk.byteLength; i++) {
|
||||
if (chunk[i] === NEWLINE_BYTE) {
|
||||
return globalIndex + i;
|
||||
}
|
||||
}
|
||||
globalIndex += chunk.byteLength;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract bytes from the buffer up to (but not including) the given index,
|
||||
* and remove those bytes plus the delimiter from the buffer.
|
||||
*/
|
||||
function extractLine(newlineIndex: number): Uint8Array {
|
||||
const fullBuffer = concatenateChunks();
|
||||
const lineBytes = fullBuffer.slice(0, newlineIndex);
|
||||
const remaining = fullBuffer.slice(newlineIndex + 1); // Skip the newline
|
||||
|
||||
// Reset buffer with remaining bytes
|
||||
if (remaining.byteLength > 0) {
|
||||
chunks = [remaining];
|
||||
totalBytes = remaining.byteLength;
|
||||
} else {
|
||||
chunks = [];
|
||||
totalBytes = 0;
|
||||
}
|
||||
|
||||
return lineBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a line from bytes, handling whitespace trimming.
|
||||
* Returns the parsed object or null for empty lines.
|
||||
*/
|
||||
function parseLine(
|
||||
lineBytes: Uint8Array,
|
||||
controller: TransformStreamDefaultController<unknown>
|
||||
): void {
|
||||
lineNumber++;
|
||||
|
||||
// Decode the line bytes (stream: false since this is a complete line)
|
||||
let lineText: string;
|
||||
try {
|
||||
lineText = decoder.decode(lineBytes, { stream: false });
|
||||
} catch (err) {
|
||||
throw new Error(`Invalid UTF-8 at line ${lineNumber}: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
const trimmed = lineText.trim();
|
||||
if (!trimmed) {
|
||||
return; // Skip empty lines
|
||||
}
|
||||
|
||||
try {
|
||||
const obj = JSON.parse(trimmed);
|
||||
emit(controller, obj);
|
||||
} catch (err) {
|
||||
throw new Error(`Invalid JSON at line ${lineNumber}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return new TransformStream<Uint8Array, unknown>({
|
||||
transform(chunk, controller) {
|
||||
// If we're skipping the remainder of an oversized line, scan for the
|
||||
// next newline in this chunk and discard everything before it.
|
||||
if (skipUntilNewline) {
|
||||
const nlPos = chunk.indexOf(NEWLINE_BYTE);
|
||||
if (nlPos === -1) {
|
||||
// Entire chunk is still part of the oversized line — discard it
|
||||
return;
|
||||
}
|
||||
// Found the newline — keep everything after it
|
||||
skipUntilNewline = false;
|
||||
const remaining = chunk.slice(nlPos + 1);
|
||||
if (remaining.byteLength === 0) {
|
||||
return;
|
||||
}
|
||||
// Replace chunk with the remainder and fall through to normal processing
|
||||
chunk = remaining;
|
||||
}
|
||||
|
||||
// Append chunk to buffer
|
||||
chunks.push(chunk);
|
||||
totalBytes += chunk.byteLength;
|
||||
|
||||
// Process all complete lines in the buffer
|
||||
let newlineIndex: number;
|
||||
while ((newlineIndex = findNewlineIndex()) !== -1) {
|
||||
// Check size limit BEFORE extracting/decoding (bytes up to newline)
|
||||
if (newlineIndex > maxItemBytes) {
|
||||
// Case 1: Complete line exceeds limit - emit marker instead of throwing
|
||||
const lineBytes = extractLine(newlineIndex);
|
||||
const extracted = extractIndexAndTask(lineBytes);
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: newlineIndex,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
lineNumber++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const lineBytes = extractLine(newlineIndex);
|
||||
parseLine(lineBytes, controller);
|
||||
}
|
||||
|
||||
// Check if the remaining buffer (incomplete line) exceeds the limit
|
||||
// This prevents OOM from a single huge line without newlines
|
||||
if (totalBytes > maxItemBytes) {
|
||||
// Case 2: Incomplete line exceeds limit - emit marker instead of throwing
|
||||
const extracted = extractIndexAndTask(concatenateChunks());
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: totalBytes,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
lineNumber++;
|
||||
// Clear buffer and skip remaining bytes of this oversized line
|
||||
// until the next newline delimiter is found in a subsequent chunk
|
||||
chunks = [];
|
||||
totalBytes = 0;
|
||||
skipUntilNewline = true;
|
||||
return;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
// Flush any remaining bytes from the decoder's internal state
|
||||
// This handles multibyte characters that may have been split across chunks
|
||||
decoder.decode(new Uint8Array(0), { stream: false });
|
||||
|
||||
// Process any remaining buffered data (no trailing newline case)
|
||||
if (totalBytes === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check size limit before processing final line
|
||||
if (totalBytes > maxItemBytes) {
|
||||
// Case 3: Flush with oversized remaining - emit marker instead of throwing
|
||||
const extracted = extractIndexAndTask(concatenateChunks());
|
||||
const marker: OversizedItemMarker = {
|
||||
__batchItemError: "OVERSIZED",
|
||||
index: extracted.index >= 0 ? extracted.index : emittedCount,
|
||||
task: extracted.task,
|
||||
actualSize: totalBytes,
|
||||
maxSize: maxItemBytes,
|
||||
};
|
||||
emit(controller, marker);
|
||||
return;
|
||||
}
|
||||
|
||||
const finalBytes = concatenateChunks();
|
||||
parseLine(finalBytes, controller);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a ReadableStream into an AsyncIterable.
|
||||
* Useful for processing streams with for-await-of loops.
|
||||
*/
|
||||
export async function* streamToAsyncIterable<T>(stream: ReadableStream<T>): AsyncIterable<T> {
|
||||
const reader = stream.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
yield value;
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
// Empty singletons satisfy the module-level wiring imports; the mint method under test is driven
|
||||
// directly via (service as any) and never touches the DB (same boundary as triggerTask.server.test.ts).
|
||||
vi.mock("~/db.server", () => ({
|
||||
prisma: {},
|
||||
$replica: {},
|
||||
runOpsNewPrisma: {},
|
||||
runOpsLegacyPrisma: {},
|
||||
runOpsNewReplica: {},
|
||||
runOpsLegacyReplica: {},
|
||||
}));
|
||||
vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
|
||||
|
||||
import { classifyKind, generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
||||
|
||||
function buildService() {
|
||||
return new TriggerFailedTaskService({ prisma: {} as any, engine: {} as any });
|
||||
}
|
||||
|
||||
describe("TriggerFailedTaskService.mintFailedRunFriendlyId", () => {
|
||||
it("returns the caller-supplied runFriendlyId verbatim (override wins over any mint)", async () => {
|
||||
const override = RunId.toFriendlyId(generateRunOpsId());
|
||||
const minted = await (buildService() as any).mintFailedRunFriendlyId({
|
||||
organizationId: "org_1",
|
||||
environmentId: "env_1",
|
||||
runFriendlyId: override,
|
||||
});
|
||||
expect(minted).toBe(override);
|
||||
});
|
||||
|
||||
it("without an override, still inherits a run-ops (NEW) parent by id-shape", async () => {
|
||||
const parentRunFriendlyId = RunId.toFriendlyId(generateRunOpsId());
|
||||
const minted = await (buildService() as any).mintFailedRunFriendlyId({
|
||||
organizationId: "org_1",
|
||||
environmentId: "env_1",
|
||||
parentRunFriendlyId,
|
||||
});
|
||||
expect(classifyKind(minted)).toBe("runOpsId");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,435 @@
|
||||
import type { RunEngine } from "@internal/run-engine";
|
||||
import { TaskRunErrorCodes, type TaskRunError } from "@trigger.dev/core/v3";
|
||||
import { RunId, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type {
|
||||
PrismaClientOrTransaction,
|
||||
RuntimeEnvironmentType,
|
||||
TaskRun,
|
||||
} from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
|
||||
import { getEventRepository } from "~/v3/eventRepository/index.server";
|
||||
import { runStore as defaultRunStore } from "~/v3/runStore.server";
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import type { IEventRepository } from "~/v3/eventRepository/eventRepository.types";
|
||||
import { PerformTaskRunAlertsService } from "~/v3/services/alerts/performTaskRunAlerts.server";
|
||||
import { DefaultQueueManager } from "../concerns/queues.server";
|
||||
import type { TriggerTaskRequest } from "../types";
|
||||
|
||||
export type TriggerFailedTaskRequest = {
|
||||
/** The task identifier (e.g. "my-task") */
|
||||
taskId: string;
|
||||
/** The fully-resolved authenticated environment */
|
||||
environment: AuthenticatedEnvironment;
|
||||
/** Raw payload — string or object */
|
||||
payload: unknown;
|
||||
/** MIME type of the payload (defaults to "application/json") */
|
||||
payloadType?: string;
|
||||
/** Error message describing why the run failed */
|
||||
errorMessage: string;
|
||||
/** Parent run friendly ID (e.g. "run_xxxx") */
|
||||
parentRunId?: string;
|
||||
/** Whether completing this run should resume the parent */
|
||||
resumeParentOnCompletion?: boolean;
|
||||
/** Batch association */
|
||||
batch?: { id: string; index: number };
|
||||
/** Trigger options from the original request (queue config, etc.) */
|
||||
options?: Record<string, unknown>;
|
||||
/** Trace context for span correlation */
|
||||
traceContext?: Record<string, unknown>;
|
||||
/** Whether the span parent should be treated as a link rather than a parent */
|
||||
spanParentAsLink?: boolean;
|
||||
|
||||
errorCode?: TaskRunErrorCodes;
|
||||
|
||||
/** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
|
||||
runFriendlyId?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a pre-failed TaskRun with a trace event span.
|
||||
*
|
||||
* This is used when a task cannot be triggered (e.g. queue limit reached, validation
|
||||
* error, etc.) but we still need to record the failure so that:
|
||||
* - Batch completion can track the item
|
||||
* - Parent runs get unblocked
|
||||
* - The failed run shows up in the run logs view
|
||||
*
|
||||
* This service resolves the parent run (for rootTaskRunId/depth) and queue properties
|
||||
* the same way triggerTask does, so the run is correctly associated in the task tree
|
||||
* and the SpanPresenter can find the TaskQueue.
|
||||
*/
|
||||
export class TriggerFailedTaskService {
|
||||
private readonly prisma: PrismaClientOrTransaction;
|
||||
private readonly replicaPrisma: PrismaClientOrTransaction;
|
||||
private readonly engine: RunEngine;
|
||||
// Resolves the parent run for depth/root/parent linkage. Defaults to the shared
|
||||
// singleton (in production the same store the engine writes through). Injected in
|
||||
// tests so the read resolves on the same store the engine wrote to.
|
||||
private readonly runStore: RunStore;
|
||||
// Defaults to getEventRepository's org-flag resolution, which reads through the
|
||||
// global prisma client; tests inject a repository bound to their testcontainer DB.
|
||||
private readonly eventRepository?: { repository: IEventRepository; store: string };
|
||||
|
||||
constructor(opts: {
|
||||
prisma: PrismaClientOrTransaction;
|
||||
engine: RunEngine;
|
||||
replicaPrisma?: PrismaClientOrTransaction;
|
||||
runStore?: RunStore;
|
||||
eventRepository?: { repository: IEventRepository; store: string };
|
||||
}) {
|
||||
this.prisma = opts.prisma;
|
||||
this.replicaPrisma = opts.replicaPrisma ?? opts.prisma;
|
||||
this.engine = opts.engine;
|
||||
this.runStore = opts.runStore ?? defaultRunStore;
|
||||
this.eventRepository = opts.eventRepository;
|
||||
}
|
||||
|
||||
// Mint a failed run's friendlyId. The id-kind decides which store the run is
|
||||
// born in (cuid → legacy store, run-ops id → new store); the whole subgraph of a
|
||||
// run must agree. A caller-supplied runFriendlyId (batch-anchored id) wins verbatim;
|
||||
// otherwise root failed runs mint by the environment's setting and child failed runs
|
||||
// inherit the parent's current store so they never split.
|
||||
private async mintFailedRunFriendlyId(args: {
|
||||
organizationId: string;
|
||||
environmentId: string;
|
||||
orgFeatureFlags?: unknown;
|
||||
parentRunFriendlyId?: string;
|
||||
runFriendlyId?: string;
|
||||
}): Promise<string> {
|
||||
if (args.runFriendlyId) {
|
||||
return args.runFriendlyId;
|
||||
}
|
||||
|
||||
const mintKind = args.parentRunFriendlyId
|
||||
? resolveInheritedMintKind(args.parentRunFriendlyId)
|
||||
: await resolveRunIdMintKind({
|
||||
organizationId: args.organizationId,
|
||||
id: args.environmentId,
|
||||
orgFeatureFlags: args.orgFeatureFlags,
|
||||
});
|
||||
|
||||
return mintKind === "runOpsId"
|
||||
? RunId.toFriendlyId(generateRunOpsId())
|
||||
: RunId.generate().friendlyId;
|
||||
}
|
||||
|
||||
async call(request: TriggerFailedTaskRequest): Promise<string | null> {
|
||||
const taskRunError: TaskRunError = {
|
||||
type: "INTERNAL_ERROR" as const,
|
||||
code: request.errorCode ?? TaskRunErrorCodes.UNSPECIFIED_ERROR,
|
||||
message: request.errorMessage,
|
||||
};
|
||||
|
||||
// Held for the catch's log line; the in-try `const` is what consumers use.
|
||||
let mintedFriendlyId: string | undefined;
|
||||
|
||||
try {
|
||||
// Mint inside the try: classifying a user-supplied parentRunId throws on
|
||||
// an unclassifiable id, so keep it within the catch's null-return contract.
|
||||
const failedRunFriendlyId = await this.mintFailedRunFriendlyId({
|
||||
organizationId: request.environment.organizationId,
|
||||
environmentId: request.environment.id,
|
||||
orgFeatureFlags: request.environment.organization.featureFlags,
|
||||
parentRunFriendlyId: request.parentRunId,
|
||||
runFriendlyId: request.runFriendlyId,
|
||||
});
|
||||
mintedFriendlyId = failedRunFriendlyId;
|
||||
|
||||
const { repository, store } =
|
||||
this.eventRepository ??
|
||||
(await getEventRepository(
|
||||
request.environment.organization.id,
|
||||
request.environment.organization.featureFlags as Record<string, unknown>,
|
||||
undefined
|
||||
));
|
||||
|
||||
// Resolve parent run for rootTaskRunId and depth (same as triggerTask.server.ts)
|
||||
const parentRun = request.parentRunId
|
||||
? await this.runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(request.parentRunId),
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
},
|
||||
this.prisma
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const depth = parentRun ? parentRun.depth + 1 : 0;
|
||||
const rootTaskRunId = parentRun?.rootTaskRunId ?? parentRun?.id;
|
||||
|
||||
// Resolve queue properties (same as triggerTask) so span presenter can find TaskQueue.
|
||||
// Best-effort: if resolution throws (e.g. request shape, missing worker), we still create
|
||||
// the run without queue/lockedQueueId so run creation and trace events never regress.
|
||||
let queueName: string | undefined;
|
||||
let lockedQueueId: string | undefined;
|
||||
try {
|
||||
const queueConcern = new DefaultQueueManager(this.prisma, this.engine, this.replicaPrisma);
|
||||
const bodyOptions = request.options as TriggerTaskRequest["body"]["options"];
|
||||
const triggerRequest: TriggerTaskRequest = {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: request.environment,
|
||||
body: {
|
||||
payload:
|
||||
typeof request.payload === "string"
|
||||
? request.payload
|
||||
: JSON.stringify(request.payload ?? {}),
|
||||
options: bodyOptions,
|
||||
},
|
||||
};
|
||||
|
||||
// Resolve the locked background worker if lockToVersion is set (same as triggerTask).
|
||||
// resolveQueueProperties requires the worker to be passed when lockToVersion is present.
|
||||
const lockedToBackgroundWorker = bodyOptions?.lockToVersion
|
||||
? await this.prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
projectId: request.environment.projectId,
|
||||
runtimeEnvironmentId: request.environment.id,
|
||||
version: bodyOptions.lockToVersion,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
version: true,
|
||||
sdkVersion: true,
|
||||
cliVersion: true,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const resolved = await queueConcern.resolveQueueProperties(
|
||||
triggerRequest,
|
||||
lockedToBackgroundWorker ?? undefined
|
||||
);
|
||||
queueName = resolved.queueName;
|
||||
lockedQueueId = resolved.lockedQueueId;
|
||||
} catch (queueResolveError) {
|
||||
const err =
|
||||
queueResolveError instanceof Error
|
||||
? queueResolveError
|
||||
: new Error(String(queueResolveError));
|
||||
logger.warn("TriggerFailedTaskService: queue resolution failed, using defaults", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRunFriendlyId,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
|
||||
// Create the failed run inside a trace event span so it shows up in run logs
|
||||
const failedRun: TaskRun = await repository.traceEvent(
|
||||
request.taskId,
|
||||
{
|
||||
context: request.traceContext,
|
||||
spanParentAsLink: request.spanParentAsLink,
|
||||
kind: "SERVER",
|
||||
environment: {
|
||||
id: request.environment.id,
|
||||
type: request.environment.type,
|
||||
organizationId: request.environment.organizationId,
|
||||
projectId: request.environment.projectId,
|
||||
project: { externalRef: request.environment.project.externalRef },
|
||||
},
|
||||
taskSlug: request.taskId,
|
||||
attributes: {
|
||||
properties: {},
|
||||
style: { icon: "task" },
|
||||
},
|
||||
incomplete: false,
|
||||
isError: true,
|
||||
immediate: true,
|
||||
},
|
||||
async (event, traceContext) => {
|
||||
event.setAttribute("runId", failedRunFriendlyId);
|
||||
event.failWithError(taskRunError);
|
||||
|
||||
// `emitRunFailedEvent: false` because this call site owns the
|
||||
// trace-event lifecycle via the outer `traceEvent({
|
||||
// incomplete: false, isError: true })`. Letting the engine
|
||||
// emit `runFailed` here would race the
|
||||
// `completeFailedRunEvent` listener against the outer trace
|
||||
// event's own completion write for the same (traceId, spanId).
|
||||
// We re-trigger the alerts side directly after the trace
|
||||
// event closes, below.
|
||||
return await this.engine.createFailedTaskRun({
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: {
|
||||
id: request.environment.id,
|
||||
type: request.environment.type,
|
||||
project: { id: request.environment.project.id },
|
||||
organization: { id: request.environment.organization.id },
|
||||
},
|
||||
taskIdentifier: request.taskId,
|
||||
payload:
|
||||
typeof request.payload === "string"
|
||||
? request.payload
|
||||
: JSON.stringify(request.payload ?? ""),
|
||||
payloadType: request.payloadType ?? "application/json",
|
||||
error: taskRunError,
|
||||
parentTaskRunId: parentRun?.id,
|
||||
rootTaskRunId,
|
||||
depth,
|
||||
resumeParentOnCompletion: request.resumeParentOnCompletion,
|
||||
batch: request.batch,
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
traceContext: traceContext as Record<string, unknown>,
|
||||
taskEventStore: store,
|
||||
emitRunFailedEvent: false,
|
||||
...(queueName !== undefined && { queue: queueName }),
|
||||
...(lockedQueueId !== undefined && { lockedQueueId }),
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
// Alerts side of `runFailed` — the engine emit was suppressed
|
||||
// above so the trace-event completion isn't double-written; we
|
||||
// still need the alert pipeline to fire so customers' ERROR
|
||||
// channels see the failure. Best-effort: a failed enqueue logs
|
||||
// but doesn't block returning the friendlyId, mirroring the
|
||||
// engine handler's behaviour at runEngineHandlers.server.ts:81.
|
||||
try {
|
||||
await PerformTaskRunAlertsService.enqueue(failedRun.id);
|
||||
} catch (alertsError) {
|
||||
logger.warn("TriggerFailedTaskService: alert enqueue failed", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: failedRun.friendlyId,
|
||||
error: alertsError instanceof Error ? alertsError.message : String(alertsError),
|
||||
});
|
||||
}
|
||||
|
||||
return failedRun.friendlyId;
|
||||
} catch (createError) {
|
||||
const createErrorMsg =
|
||||
createError instanceof Error ? createError.message : String(createError);
|
||||
logger.error("TriggerFailedTaskService: failed to create pre-failed TaskRun", {
|
||||
taskId: request.taskId,
|
||||
friendlyId: mintedFriendlyId,
|
||||
originalError: request.errorMessage,
|
||||
createError: createErrorMsg,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a pre-failed run without trace events.
|
||||
* Used when the environment can't be fully resolved (e.g. environment not found)
|
||||
* and we can't create trace events or look up parent runs.
|
||||
*/
|
||||
async callWithoutTraceEvents(opts: {
|
||||
environmentId: string;
|
||||
environmentType: RuntimeEnvironmentType;
|
||||
projectId: string;
|
||||
organizationId: string;
|
||||
taskId: string;
|
||||
payload: unknown;
|
||||
payloadType?: string;
|
||||
errorMessage: string;
|
||||
parentRunId?: string;
|
||||
resumeParentOnCompletion?: boolean;
|
||||
batch?: { id: string; index: number };
|
||||
errorCode?: TaskRunErrorCodes;
|
||||
/** Pre-minted friendlyId; when set it wins over the mint. Batch callers pass a batch-anchored id. */
|
||||
runFriendlyId?: string;
|
||||
}): Promise<string | null> {
|
||||
// Held for the catch's log line; the in-try `const` is what consumers use.
|
||||
let mintedFriendlyId: string | undefined;
|
||||
|
||||
try {
|
||||
// Mint inside the try: classifying a user-supplied parentRunId throws on
|
||||
// an unclassifiable id, so keep it within the catch's null-return contract.
|
||||
const failedRunFriendlyId = await this.mintFailedRunFriendlyId({
|
||||
organizationId: opts.organizationId,
|
||||
environmentId: opts.environmentId,
|
||||
// No loaded org flags in this path; resolveRunIdMintKind falls back to a
|
||||
// single replica lookup by organizationId only when there is no parent.
|
||||
orgFeatureFlags: undefined,
|
||||
parentRunFriendlyId: opts.parentRunId,
|
||||
runFriendlyId: opts.runFriendlyId,
|
||||
});
|
||||
mintedFriendlyId = failedRunFriendlyId;
|
||||
|
||||
// Best-effort parent run lookup for rootTaskRunId/depth
|
||||
let parentTaskRunId: string | undefined;
|
||||
let rootTaskRunId: string | undefined;
|
||||
let depth = 0;
|
||||
|
||||
if (opts.parentRunId) {
|
||||
const parentRun = await this.runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(opts.parentRunId),
|
||||
runtimeEnvironmentId: opts.environmentId,
|
||||
},
|
||||
this.prisma
|
||||
);
|
||||
|
||||
if (parentRun) {
|
||||
parentTaskRunId = parentRun.id;
|
||||
rootTaskRunId = parentRun.rootTaskRunId ?? parentRun.id;
|
||||
depth = parentRun.depth + 1;
|
||||
} else {
|
||||
parentTaskRunId = RunId.fromFriendlyId(opts.parentRunId);
|
||||
}
|
||||
}
|
||||
|
||||
const failedRun = await this.engine.createFailedTaskRun({
|
||||
friendlyId: failedRunFriendlyId,
|
||||
environment: {
|
||||
id: opts.environmentId,
|
||||
type: opts.environmentType,
|
||||
project: { id: opts.projectId },
|
||||
organization: { id: opts.organizationId },
|
||||
},
|
||||
taskIdentifier: opts.taskId,
|
||||
payload:
|
||||
typeof opts.payload === "string" ? opts.payload : JSON.stringify(opts.payload ?? ""),
|
||||
payloadType: opts.payloadType ?? "application/json",
|
||||
error: {
|
||||
type: "INTERNAL_ERROR" as const,
|
||||
code: opts.errorCode ?? TaskRunErrorCodes.UNSPECIFIED_ERROR,
|
||||
message: opts.errorMessage,
|
||||
},
|
||||
parentTaskRunId,
|
||||
rootTaskRunId,
|
||||
depth,
|
||||
resumeParentOnCompletion: opts.resumeParentOnCompletion,
|
||||
batch: opts.batch,
|
||||
// Suppress the engine's `runFailed` bus emit — the listener
|
||||
// (`runEngineHandlers.server.ts` `runFailed`) calls
|
||||
// `completeFailedRunEvent`, which writes a ClickHouse trace event
|
||||
// row keyed on (traceId, spanId). This caller has no trace
|
||||
// context (the method name is literally `callWithoutTraceEvents`)
|
||||
// so the emit would write a row with empty traceId/spanId —
|
||||
// orphan event in the store. We still want alert coverage,
|
||||
// though, so enqueue directly below.
|
||||
emitRunFailedEvent: false,
|
||||
});
|
||||
|
||||
// Alerts side of `runFailed` — the engine emit was suppressed
|
||||
// above so we don't create an orphan trace event; enqueue the
|
||||
// alert directly so customers' ERROR channels still see the
|
||||
// failure. Best-effort, mirroring the `call()` path.
|
||||
try {
|
||||
await PerformTaskRunAlertsService.enqueue(failedRun.id);
|
||||
} catch (alertsError) {
|
||||
logger.warn("TriggerFailedTaskService.callWithoutTraceEvents: alert enqueue failed", {
|
||||
taskId: opts.taskId,
|
||||
friendlyId: failedRun.friendlyId,
|
||||
error: alertsError instanceof Error ? alertsError.message : String(alertsError),
|
||||
});
|
||||
}
|
||||
|
||||
return failedRunFriendlyId;
|
||||
} catch (createError) {
|
||||
logger.error("TriggerFailedTaskService: failed to create pre-failed TaskRun (no trace)", {
|
||||
taskId: opts.taskId,
|
||||
friendlyId: mintedFriendlyId,
|
||||
originalError: opts.errorMessage,
|
||||
createError: createError instanceof Error ? createError.message : String(createError),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,825 @@
|
||||
import { describe, expect, vi } from "vitest";
|
||||
|
||||
// Mock the db prisma client. The service is constructed against a real
|
||||
// testcontainer prisma instead — these empty singletons only satisfy the
|
||||
// module-level imports of the production wiring (infrastructure boundary).
|
||||
vi.mock("~/db.server", () => ({
|
||||
prisma: {},
|
||||
$replica: {},
|
||||
runOpsNewPrisma: {},
|
||||
runOpsLegacyPrisma: {},
|
||||
runOpsNewReplica: {},
|
||||
runOpsLegacyReplica: {},
|
||||
}));
|
||||
// Inherited harness boilerplate. The parent read under test takes the
|
||||
// findRun(where, client) overload with this.prisma, so it does not consult this
|
||||
// flag; the mock only satisfies other wiring imported transitively.
|
||||
vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
|
||||
|
||||
vi.mock("~/services/platform.v3.server", async (importOriginal) => {
|
||||
const actual = (await importOriginal()) as Record<string, unknown>;
|
||||
return {
|
||||
...actual,
|
||||
getEntitlement: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { RunEngine } from "@internal/run-engine";
|
||||
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests";
|
||||
import { assertNonNullable, containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import type { IOPacket } from "@trigger.dev/core/v3";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
|
||||
import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
|
||||
import type {
|
||||
EntitlementValidationParams,
|
||||
MaxAttemptsValidationParams,
|
||||
ParentRunValidationParams,
|
||||
PayloadProcessor,
|
||||
TagValidationParams,
|
||||
TracedEventSpan,
|
||||
TraceEventConcern,
|
||||
TriggerTaskRequest,
|
||||
TriggerTaskValidator,
|
||||
ValidationResult,
|
||||
} from "~/runEngine/types";
|
||||
import { RunEngineTriggerTaskService } from "./triggerTask.server";
|
||||
|
||||
vi.setConfig({ testTimeout: 60_000 }); // 60 seconds timeout
|
||||
|
||||
class MockPayloadProcessor implements PayloadProcessor {
|
||||
async process(request: TriggerTaskRequest): Promise<IOPacket> {
|
||||
return {
|
||||
data: JSON.stringify(request.body.payload),
|
||||
dataType: "application/json",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Captures the `parentRun` the service resolved (via runStore.findRun) and
|
||||
// passed into validation, so a test can assert on the resolved parent without
|
||||
// mocking the read itself. Returns ok so the child triggers regardless.
|
||||
class CapturingParentRunValidator implements TriggerTaskValidator {
|
||||
public capturedParentRun: ParentRunValidationParams["parentRun"] | "unset" = "unset";
|
||||
|
||||
validateTags(_params: TagValidationParams): ValidationResult {
|
||||
return { ok: true };
|
||||
}
|
||||
validateEntitlement(_params: EntitlementValidationParams): Promise<ValidationResult> {
|
||||
return Promise.resolve({ ok: true });
|
||||
}
|
||||
validateMaxAttempts(_params: MaxAttemptsValidationParams): ValidationResult {
|
||||
return { ok: true };
|
||||
}
|
||||
validateParentRun(params: ParentRunValidationParams): ValidationResult {
|
||||
this.capturedParentRun = params.parentRun;
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
|
||||
class MockTraceEventConcern implements TraceEventConcern {
|
||||
async traceRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
|
||||
async traceIdempotentRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
_options: {
|
||||
existingRun: TaskRun;
|
||||
idempotencyKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
|
||||
async traceDebouncedRun<T>(
|
||||
_request: TriggerTaskRequest,
|
||||
_parentStore: string | undefined,
|
||||
_options: {
|
||||
existingRun: TaskRun;
|
||||
debounceKey: string;
|
||||
incomplete: boolean;
|
||||
isError: boolean;
|
||||
},
|
||||
callback: (span: TracedEventSpan, store: string) => Promise<T>
|
||||
): Promise<T> {
|
||||
return await callback(
|
||||
{
|
||||
traceId: "test",
|
||||
spanId: "test",
|
||||
traceContext: {},
|
||||
traceparent: undefined,
|
||||
setAttribute: () => {},
|
||||
failWithError: () => {},
|
||||
stop: () => {},
|
||||
},
|
||||
"test"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function buildEngine(prisma: any, redisOptions: any) {
|
||||
return new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0005,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
}
|
||||
|
||||
describe("RunEngineTriggerTaskService parent + locked-worker reads", () => {
|
||||
containerTest(
|
||||
"resolves the parent run through the run-ops store by minted run id",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger a ROOT run first to create a real parent TaskRun.
|
||||
const parentResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// Trigger a CHILD pointing at the parent's friendlyId. The service must
|
||||
// resolve the parent via runStore.findRun (minted RunId, env-scoped).
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: { parentRunId: parentResult.run.friendlyId },
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
// The capturing validator observed the resolved parent — proving the
|
||||
// read ran (against the container DB) and returned the right row.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
const capturedParent = validator.capturedParentRun;
|
||||
assertNonNullable(capturedParent);
|
||||
expect(capturedParent.id).toBe(parentResult.run.id);
|
||||
expect(capturedParent.friendlyId).toBe(parentResult.run.friendlyId);
|
||||
|
||||
// depth and root carry through — proving parentRun.depth and the parent
|
||||
// id were read off the resolved row and threaded into the child.
|
||||
const parentRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: parentResult.run.id },
|
||||
});
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
|
||||
expect(childRow.depth).toBe(parentRow.depth + 1);
|
||||
expect(childRow.parentTaskRunId).toBe(parentRow.id);
|
||||
expect(childRow.rootTaskRunId).toBe(parentRow.id);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"scopes the parent lookup to the run's environment (cross-env parent is not resolved)",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
// Two independent authenticated environments. The setup helper hardcodes
|
||||
// several globally-unique fields (org/project slug, env apiKey/pkApiKey,
|
||||
// worker-group token hash), so rename envA's before the second call to
|
||||
// avoid unique-constraint collisions.
|
||||
const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
await prisma.organization.update({
|
||||
where: { id: envA.organizationId },
|
||||
data: { slug: `${envA.organization.slug}-a` },
|
||||
});
|
||||
await prisma.project.update({
|
||||
where: { id: envA.projectId },
|
||||
data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` },
|
||||
});
|
||||
await prisma.runtimeEnvironment.update({
|
||||
where: { id: envA.id },
|
||||
data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` },
|
||||
});
|
||||
await prisma.workerGroupToken.updateMany({
|
||||
where: { tokenHash: "token_hash" },
|
||||
data: { tokenHash: "token_hash_a" },
|
||||
});
|
||||
await prisma.workerInstanceGroup.updateMany({
|
||||
where: { masterQueue: "default" },
|
||||
data: { masterQueue: "default_a" },
|
||||
});
|
||||
const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
expect(envA.id).not.toBe(envB.id);
|
||||
expect(envA.organizationId).not.toBe(envB.organizationId);
|
||||
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, envA, taskIdentifier);
|
||||
await setupBackgroundWorker(engine, envB, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// A real parent run in envA.
|
||||
const parentResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envA,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// Trigger a child in envB pointing at the envA parent's friendlyId. The
|
||||
// env guard in runStore.findRun's `where` rejects the cross-env parent
|
||||
// in a single query, so the resolved parentRun is null.
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envB,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: { parentRunId: parentResult.run.friendlyId },
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
// validateParentRun was called with no resolved parent.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
expect(validator.capturedParentRun ?? null).toBeNull();
|
||||
|
||||
// The child still triggered, at the root depth with no parent linkage —
|
||||
// confirming the cross-env parent was dropped, not silently joined.
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
expect(childRow.depth).toBe(0);
|
||||
expect(childRow.parentTaskRunId).toBeNull();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"resolves the locked background worker on the control-plane client with no cross-DB join",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
// Read the seeded worker row to get its real version/id.
|
||||
const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: worker.id },
|
||||
});
|
||||
|
||||
// Counting proxy over the control-plane client. `this.prisma` is ALWAYS
|
||||
// the control-plane client; the locked-worker lookup is a DIRECT
|
||||
// backgroundWorker.findFirst on it. The parent read uses a DIFFERENT
|
||||
// call (runStore.findRun → taskRun), so a single call() issues two
|
||||
// separate single-table reads — never one cross-seam join. Here we count
|
||||
// the findFirst calls and capture their args to assert no include/join.
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
// The queue manager gets the real (unproxied) prisma so the counting
|
||||
// proxy only observes reads issued by the service itself.
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: workerRow.version },
|
||||
},
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
// Observable proof the locked worker was resolved on the control-plane
|
||||
// client: the created run records the worker id in lockedToVersionId.
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
expect(runRow.lockedToVersionId).toBe(workerRow.id);
|
||||
expect(runRow.taskVersion).toBe(workerRow.version);
|
||||
|
||||
// Exactly one backgroundWorker.findFirst fired for the locked-worker read.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
|
||||
// NO-JOIN assertion: the read referenced ONLY the backgroundWorker table.
|
||||
// No `include` (which would join into another table); the `select` lists
|
||||
// only backgroundWorker scalar columns.
|
||||
const args = findFirstArgs[0];
|
||||
assertNonNullable(args);
|
||||
expect(args.include).toBeUndefined();
|
||||
expect(Object.keys(args.select ?? {}).sort()).toEqual([
|
||||
"cliVersion",
|
||||
"id",
|
||||
"sdkVersion",
|
||||
"version",
|
||||
]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"issues two independent single-table reads when one call supplies both parentRunId and lockToVersion",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker } = await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const workerRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: worker.id },
|
||||
});
|
||||
|
||||
// Count BOTH reads issued by the service on the control-plane client:
|
||||
// the parent read (runStore.findRun → taskRun.findFirst) and the
|
||||
// locked-worker read (backgroundWorker.findFirst). Capture every
|
||||
// findFirst arg so we can assert no read carries a cross-seam include.
|
||||
let taskRunFindFirstCalls = 0;
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
if (prop === "taskRun") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(trTarget, trProp, trReceiver) {
|
||||
if (trProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
taskRunFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(trTarget, trProp, trReceiver);
|
||||
return typeof value === "function" ? value.bind(trTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
// queueConcern/idempotency get the real unproxied prisma so the
|
||||
// counting proxy only observes reads issued by the service itself.
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// ROOT parent first (uses the unproxied prisma via a separate service so
|
||||
// its internal reads don't pollute the child's counts).
|
||||
const parentService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
const parentResult = await parentService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "parent" } },
|
||||
});
|
||||
assertNonNullable(parentResult);
|
||||
|
||||
// CHILD supplying BOTH parentRunId AND lockToVersion in one call.
|
||||
const childResult = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "child" },
|
||||
options: {
|
||||
parentRunId: parentResult.run.friendlyId,
|
||||
lockToVersion: workerRow.version,
|
||||
},
|
||||
},
|
||||
});
|
||||
assertNonNullable(childResult);
|
||||
|
||||
const parentRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: parentResult.run.id },
|
||||
});
|
||||
const childRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: childResult.run.id },
|
||||
});
|
||||
|
||||
// Child resolved the parent (single-table parent read).
|
||||
expect(childRow.parentTaskRunId).toBe(parentRow.id);
|
||||
expect(childRow.depth).toBe(parentRow.depth + 1);
|
||||
|
||||
// Child locked to the worker (single-table worker read).
|
||||
expect(childRow.lockedToVersionId).toBe(workerRow.id);
|
||||
expect(childRow.taskVersion).toBe(workerRow.version);
|
||||
|
||||
// Exactly one backgroundWorker.findFirst fired for the locked-worker read,
|
||||
// and at least one taskRun.findFirst fired for the parent read.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
expect(taskRunFindFirstCalls).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// NO-JOIN proof: no captured read carried an `include` joining
|
||||
// taskRun <-> backgroundWorker. Every findFirst arg has include undefined.
|
||||
for (const args of findFirstArgs) {
|
||||
expect(args?.include).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"lockToVersion matching no worker rejects the trigger after a single scalar-only worker read",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
let backgroundWorkerFindFirstCalls = 0;
|
||||
const findFirstArgs: any[] = [];
|
||||
const countingPrisma = new Proxy(prisma, {
|
||||
get(target, prop, receiver) {
|
||||
if (prop === "backgroundWorker") {
|
||||
const delegate = Reflect.get(target, prop, receiver);
|
||||
return new Proxy(delegate, {
|
||||
get(bwTarget, bwProp, bwReceiver) {
|
||||
if (bwProp === "findFirst") {
|
||||
return async (args: any) => {
|
||||
backgroundWorkerFindFirstCalls += 1;
|
||||
findFirstArgs.push(args);
|
||||
return (delegate as any).findFirst(args);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(bwTarget, bwProp, bwReceiver);
|
||||
return typeof value === "function" ? value.bind(bwTarget) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
return typeof value === "function" ? value.bind(target) : value;
|
||||
},
|
||||
}) as typeof prisma;
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma: countingPrisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
const bogusVersion = "v-does-not-exist-0000";
|
||||
// The no-match worker read returns null; the queue concern then rejects
|
||||
// the trigger rather than silently locking the run to a phantom version.
|
||||
await expect(
|
||||
triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: bogusVersion },
|
||||
},
|
||||
})
|
||||
).rejects.toThrow(/no worker found with that version/);
|
||||
|
||||
// No run was locked to the bogus version (none was created).
|
||||
const lockedRuns = await prisma.taskRun.findMany({
|
||||
where: { runtimeEnvironmentId: environment.id, taskVersion: bogusVersion },
|
||||
});
|
||||
expect(lockedRuns).toEqual([]);
|
||||
|
||||
// The lone worker read fired exactly once with the scalar-only select and
|
||||
// no cross-seam include.
|
||||
expect(backgroundWorkerFindFirstCalls).toBe(1);
|
||||
const args = findFirstArgs[0];
|
||||
assertNonNullable(args);
|
||||
expect(args.include).toBeUndefined();
|
||||
expect(Object.keys(args.select ?? {}).sort()).toEqual([
|
||||
"cliVersion",
|
||||
"id",
|
||||
"sdkVersion",
|
||||
"version",
|
||||
]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"does not resolve a locked worker from a different environment",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
// Two independent authenticated environments. Rename envA's globally-unique
|
||||
// fields before the second setup call to avoid unique-constraint collisions.
|
||||
const envA = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
await prisma.organization.update({
|
||||
where: { id: envA.organizationId },
|
||||
data: { slug: `${envA.organization.slug}-a` },
|
||||
});
|
||||
await prisma.project.update({
|
||||
where: { id: envA.projectId },
|
||||
data: { slug: `${envA.project.slug}-a`, externalRef: `${envA.project.externalRef}-a` },
|
||||
});
|
||||
await prisma.runtimeEnvironment.update({
|
||||
where: { id: envA.id },
|
||||
data: { apiKey: `${envA.apiKey}-a`, pkApiKey: `${envA.pkApiKey}-a` },
|
||||
});
|
||||
await prisma.workerGroupToken.updateMany({
|
||||
where: { tokenHash: "token_hash" },
|
||||
data: { tokenHash: "token_hash_a" },
|
||||
});
|
||||
await prisma.workerInstanceGroup.updateMany({
|
||||
where: { masterQueue: "default" },
|
||||
data: { masterQueue: "default_a" },
|
||||
});
|
||||
const envB = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
expect(envA.id).not.toBe(envB.id);
|
||||
expect(envA.organizationId).not.toBe(envB.organizationId);
|
||||
|
||||
const taskIdentifier = "test-task";
|
||||
const { worker: workerA } = await setupBackgroundWorker(engine, envA, taskIdentifier);
|
||||
const { worker: workerB } = await setupBackgroundWorker(engine, envB, taskIdentifier);
|
||||
|
||||
const workerARow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: workerA.id },
|
||||
});
|
||||
const workerBRow = await prisma.backgroundWorker.findUniqueOrThrow({
|
||||
where: { id: workerB.id },
|
||||
});
|
||||
// Both seeded workers share the same version string.
|
||||
expect(workerARow.version).toBe(workerBRow.version);
|
||||
expect(workerARow.id).not.toBe(workerBRow.id);
|
||||
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator: new CapturingParentRunValidator(),
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger in envB locking to the shared version string.
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment: envB,
|
||||
body: {
|
||||
payload: { kind: "locked" },
|
||||
options: { lockToVersion: workerBRow.version },
|
||||
},
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
// The projectId + runtimeEnvironmentId guard in the single-table worker
|
||||
// read resolves envB's worker, never envA's same-version worker.
|
||||
expect(runRow.lockedToVersionId).toBe(workerBRow.id);
|
||||
expect(runRow.lockedToVersionId).not.toBe(workerARow.id);
|
||||
expect(runRow.taskVersion).toBe(workerBRow.version);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest("a root trigger issues no parent lookup", async ({ prisma, redisOptions }) => {
|
||||
const engine = buildEngine(prisma, redisOptions);
|
||||
|
||||
try {
|
||||
const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const validator = new CapturingParentRunValidator();
|
||||
const triggerTaskService = new RunEngineTriggerTaskService({
|
||||
engine,
|
||||
prisma,
|
||||
payloadProcessor: new MockPayloadProcessor(),
|
||||
queueConcern: new DefaultQueueManager(prisma, engine),
|
||||
idempotencyKeyConcern: new IdempotencyKeyConcern(
|
||||
prisma,
|
||||
engine,
|
||||
new MockTraceEventConcern()
|
||||
),
|
||||
validator,
|
||||
traceEventConcern: new MockTraceEventConcern(),
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
metadataMaximumSize: 1024 * 1024 * 1,
|
||||
});
|
||||
|
||||
// Trigger with NO parentRunId.
|
||||
const result = await triggerTaskService.call({
|
||||
taskId: taskIdentifier,
|
||||
environment,
|
||||
body: { payload: { kind: "root" } },
|
||||
});
|
||||
assertNonNullable(result);
|
||||
|
||||
// The validator ran but received no resolved parent: the parent read was
|
||||
// skipped because no parentRunId was supplied.
|
||||
expect(validator.capturedParentRun).not.toBe("unset");
|
||||
expect(validator.capturedParentRun).toBeUndefined();
|
||||
|
||||
const runRow = await prisma.taskRun.findUniqueOrThrow({
|
||||
where: { id: result.run.id },
|
||||
});
|
||||
expect(runRow.depth).toBe(0);
|
||||
expect(runRow.parentTaskRunId).toBeNull();
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,942 @@
|
||||
import {
|
||||
type RunEngine,
|
||||
RunDuplicateIdempotencyKeyError,
|
||||
RunOneTimeUseTokenError,
|
||||
} from "@internal/run-engine";
|
||||
import type { Tracer } from "@opentelemetry/api";
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import {
|
||||
type TriggerTaskRequestBody,
|
||||
RunAnnotations,
|
||||
TaskRunError,
|
||||
taskRunErrorEnhancer,
|
||||
taskRunErrorToString,
|
||||
TriggerTraceContext,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import {
|
||||
parseTraceparent,
|
||||
RunId,
|
||||
serializeTraceparent,
|
||||
stringifyDuration,
|
||||
} from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { parseDelay } from "~/utils/delays";
|
||||
import { handleMetadataPacket } from "~/utils/packets";
|
||||
import { startSpan } from "~/v3/tracing.server";
|
||||
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
|
||||
import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server";
|
||||
import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
||||
import type {
|
||||
TriggerTaskServiceOptions,
|
||||
TriggerTaskServiceResult,
|
||||
} from "../../v3/services/triggerTask.server";
|
||||
import { clampMaxDuration } from "../../v3/utils/maxDuration";
|
||||
import {
|
||||
type IdempotencyKeyConcern,
|
||||
type ClaimedIdempotency,
|
||||
} from "../concerns/idempotencyKeys.server";
|
||||
import {
|
||||
resolveScheduledQueueSplitEnabled,
|
||||
workerQueueForRun,
|
||||
} from "../concerns/workerQueueSplit.server";
|
||||
import { resolveComputeMigration } from "../concerns/computeMigration.server";
|
||||
import { workerRegionRegistry, backingForQueue, regionForQueue } from "~/v3/workerRegions.server";
|
||||
import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
|
||||
import {
|
||||
publishClaim as publishMollifierClaim,
|
||||
releaseClaim as releaseMollifierClaim,
|
||||
} from "~/v3/mollifier/idempotencyClaim.server";
|
||||
import type {
|
||||
PayloadProcessor,
|
||||
QueueManager,
|
||||
TraceEventConcern,
|
||||
TriggerRacepoints,
|
||||
TriggerRacepointSystem,
|
||||
TriggerTaskRequest,
|
||||
TriggerTaskValidator,
|
||||
} from "../types";
|
||||
import { env } from "~/env.server";
|
||||
import {
|
||||
evaluateGate as defaultEvaluateGate,
|
||||
type GateOutcome,
|
||||
type MollifierEvaluateGate,
|
||||
} from "~/v3/mollifier/mollifierGate.server";
|
||||
import {
|
||||
getMollifierBuffer as defaultGetMollifierBuffer,
|
||||
type MollifierGetBuffer,
|
||||
} from "~/v3/mollifier/mollifierBuffer.server";
|
||||
import { mollifyTrigger } from "~/v3/mollifier/mollifierMollify.server";
|
||||
import { QueueSizeLimitExceededError, ServiceValidationError } from "~/v3/services/common.server";
|
||||
import { runStore } from "~/v3/runStore.server";
|
||||
|
||||
class NoopTriggerRacepointSystem implements TriggerRacepointSystem {
|
||||
async waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void> {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export class RunEngineTriggerTaskService {
|
||||
private readonly queueConcern: QueueManager;
|
||||
private readonly validator: TriggerTaskValidator;
|
||||
private readonly payloadProcessor: PayloadProcessor;
|
||||
private readonly idempotencyKeyConcern: IdempotencyKeyConcern;
|
||||
private readonly prisma: PrismaClientOrTransaction;
|
||||
private readonly engine: RunEngine;
|
||||
private readonly tracer: Tracer;
|
||||
private readonly traceEventConcern: TraceEventConcern;
|
||||
private readonly triggerRacepointSystem: TriggerRacepointSystem;
|
||||
private readonly metadataMaximumSize: number;
|
||||
// Mollifier hooks are DI'd so tests can drive the call-site's mollify branch
|
||||
// deterministically (stub the gate to return mollify, inject a real or fake
|
||||
// buffer, force the global-enabled predicate to true so the call site
|
||||
// doesn't short-circuit on an unset env). In production all three default
|
||||
// to the live module-level singletons + env read.
|
||||
private readonly evaluateGate: MollifierEvaluateGate;
|
||||
private readonly getMollifierBuffer: MollifierGetBuffer;
|
||||
private readonly isMollifierGloballyEnabled: () => boolean;
|
||||
|
||||
constructor(opts: {
|
||||
prisma: PrismaClientOrTransaction;
|
||||
engine: RunEngine;
|
||||
queueConcern: QueueManager;
|
||||
validator: TriggerTaskValidator;
|
||||
payloadProcessor: PayloadProcessor;
|
||||
idempotencyKeyConcern: IdempotencyKeyConcern;
|
||||
traceEventConcern: TraceEventConcern;
|
||||
tracer: Tracer;
|
||||
metadataMaximumSize: number;
|
||||
triggerRacepointSystem?: TriggerRacepointSystem;
|
||||
evaluateGate?: MollifierEvaluateGate;
|
||||
getMollifierBuffer?: MollifierGetBuffer;
|
||||
isMollifierGloballyEnabled?: () => boolean;
|
||||
}) {
|
||||
this.prisma = opts.prisma;
|
||||
this.engine = opts.engine;
|
||||
this.queueConcern = opts.queueConcern;
|
||||
this.validator = opts.validator;
|
||||
this.payloadProcessor = opts.payloadProcessor;
|
||||
this.idempotencyKeyConcern = opts.idempotencyKeyConcern;
|
||||
this.tracer = opts.tracer;
|
||||
this.traceEventConcern = opts.traceEventConcern;
|
||||
this.metadataMaximumSize = opts.metadataMaximumSize;
|
||||
this.triggerRacepointSystem = opts.triggerRacepointSystem ?? new NoopTriggerRacepointSystem();
|
||||
this.evaluateGate = opts.evaluateGate ?? defaultEvaluateGate;
|
||||
this.getMollifierBuffer = opts.getMollifierBuffer ?? defaultGetMollifierBuffer;
|
||||
this.isMollifierGloballyEnabled =
|
||||
opts.isMollifierGloballyEnabled ?? (() => env.TRIGGER_MOLLIFIER_ENABLED === "1");
|
||||
}
|
||||
|
||||
// Mint a new run's friendlyId. The id-kind decides which store the run is born
|
||||
// in (cuid → legacy store, run-ops id → new store), so the whole subgraph of a run
|
||||
// must agree. Two cases:
|
||||
//
|
||||
// - ROOT run (no parent): mint by the environment's cutover setting.
|
||||
// - CHILD run (has a parent): inherit the parent's residency by id-shape, so a
|
||||
// parent and child never split across stores (run-ops parent → run-ops child,
|
||||
// cuid parent → cuid child).
|
||||
// `region` is the caller-requested region (body.options.region). The id is
|
||||
// minted before the worker queue is resolved (the idempotency concern needs
|
||||
// the friendlyId first), so the stamped region char reflects the requested
|
||||
// region — or the default char when the run targets the default region.
|
||||
private async mintRunFriendlyId(
|
||||
environment: AuthenticatedEnvironment,
|
||||
parentRunFriendlyId?: string,
|
||||
region?: string
|
||||
): Promise<string> {
|
||||
const mintKind = parentRunFriendlyId
|
||||
? resolveInheritedMintKind(parentRunFriendlyId)
|
||||
: await resolveRunIdMintKind({
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
});
|
||||
|
||||
return mintFriendlyIdForKind(mintKind, region);
|
||||
}
|
||||
|
||||
public async call({
|
||||
taskId,
|
||||
environment,
|
||||
body,
|
||||
options = {},
|
||||
attempt = 0,
|
||||
}: {
|
||||
taskId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
body: TriggerTaskRequestBody;
|
||||
options?: TriggerTaskServiceOptions;
|
||||
attempt?: number;
|
||||
}): Promise<TriggerTaskServiceResult | undefined> {
|
||||
// Pre-gate idempotency-claim ownership. Set inside the span when
|
||||
// `IdempotencyKeyConcern.handleTriggerRequest` returns `claim:
|
||||
// {...}`. The try/catch below resolves it once the span finishes.
|
||||
let idempotencyClaim: ClaimedIdempotency | undefined;
|
||||
try {
|
||||
const result = await startSpan(
|
||||
this.tracer,
|
||||
"RunEngineTriggerTaskService.call()",
|
||||
async (span) => {
|
||||
span.setAttribute("taskId", taskId);
|
||||
span.setAttribute("attempt", attempt);
|
||||
|
||||
// Mint the run id. A caller-supplied id (idempotent retry) wins;
|
||||
// otherwise mint by residency — inheriting the parent's store when a
|
||||
// parent is present, else the environment's setting.
|
||||
const runFriendlyId =
|
||||
options?.runFriendlyId ??
|
||||
(await this.mintRunFriendlyId(
|
||||
environment,
|
||||
body.options?.parentRunId,
|
||||
body.options?.region
|
||||
));
|
||||
const triggerRequest = {
|
||||
taskId,
|
||||
friendlyId: runFriendlyId,
|
||||
environment,
|
||||
body,
|
||||
options,
|
||||
} satisfies TriggerTaskRequest;
|
||||
|
||||
const maxAttemptsValidation = this.validator.validateMaxAttempts({
|
||||
taskId,
|
||||
attempt,
|
||||
});
|
||||
|
||||
if (!maxAttemptsValidation.ok) {
|
||||
throw maxAttemptsValidation.error;
|
||||
}
|
||||
|
||||
const tagValidation = this.validator.validateTags({
|
||||
tags: body.options?.tags,
|
||||
});
|
||||
|
||||
if (!tagValidation.ok) {
|
||||
throw tagValidation.error;
|
||||
}
|
||||
|
||||
let planType: string | undefined;
|
||||
|
||||
if (!options.skipChecks) {
|
||||
const entitlementValidation = await this.validator.validateEntitlement({
|
||||
environment,
|
||||
});
|
||||
|
||||
if (!entitlementValidation.ok) {
|
||||
throw entitlementValidation.error;
|
||||
}
|
||||
|
||||
planType = entitlementValidation.plan?.type;
|
||||
} else {
|
||||
// When skipChecks is enabled, planType should be passed via options
|
||||
planType = options.planType;
|
||||
|
||||
if (!planType) {
|
||||
logger.warn("Plan type not set but skipChecks is enabled", {
|
||||
taskId,
|
||||
environment: {
|
||||
id: environment.id,
|
||||
type: environment.type,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Parse delay from either explicit delay option or debounce.delay
|
||||
const delaySource = body.options?.delay ?? body.options?.debounce?.delay;
|
||||
const [parseDelayError, delayUntil] = await tryCatch(parseDelay(delaySource));
|
||||
|
||||
if (parseDelayError) {
|
||||
throw new ServiceValidationError(`Invalid delay ${delaySource}`);
|
||||
}
|
||||
|
||||
// Validate debounce options
|
||||
if (body.options?.debounce) {
|
||||
if (!delayUntil) {
|
||||
throw new ServiceValidationError(
|
||||
`Debounce requires a valid delay duration. Provided: ${body.options.debounce.delay}`
|
||||
);
|
||||
}
|
||||
|
||||
// Always validate debounce.delay separately since it's used for rescheduling
|
||||
// This catches the case where options.delay is valid but debounce.delay is invalid
|
||||
const [debounceDelayError, debounceDelayUntil] = await tryCatch(
|
||||
parseDelay(body.options.debounce.delay)
|
||||
);
|
||||
|
||||
if (debounceDelayError || !debounceDelayUntil) {
|
||||
throw new ServiceValidationError(
|
||||
`Invalid debounce delay: ${body.options.debounce.delay}. ` +
|
||||
`Supported formats: {number}s, {number}m, {number}h, {number}d, {number}w`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const parentRun = body.options?.parentRunId
|
||||
? await runStore.findRun(
|
||||
{
|
||||
id: RunId.fromFriendlyId(body.options.parentRunId),
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
this.prisma
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const parentRunValidation = this.validator.validateParentRun({
|
||||
taskId,
|
||||
parentRun: parentRun ?? undefined,
|
||||
resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
if (!parentRunValidation.ok) {
|
||||
throw parentRunValidation.error;
|
||||
}
|
||||
|
||||
const idempotencyKeyConcernResult = await this.idempotencyKeyConcern.handleTriggerRequest(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore
|
||||
);
|
||||
|
||||
if (idempotencyKeyConcernResult.isCached) {
|
||||
return idempotencyKeyConcernResult;
|
||||
}
|
||||
|
||||
const {
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
claim: claimResult,
|
||||
} = idempotencyKeyConcernResult;
|
||||
|
||||
// If we own an idempotency claim, the trigger pipeline below MUST
|
||||
// resolve it — publish on success so waiters see our runId,
|
||||
// release on error so the next claimant can retry. Stored in an
|
||||
// outer scope so the try/catch at the bottom of `callV2` can act
|
||||
// on whichever return path or throw the pipeline takes.
|
||||
idempotencyClaim = claimResult;
|
||||
|
||||
if (idempotencyKey) {
|
||||
await this.triggerRacepointSystem.waitForRacepoint({
|
||||
racepoint: "idempotencyKey",
|
||||
id: idempotencyKey,
|
||||
});
|
||||
}
|
||||
|
||||
const lockedToBackgroundWorker = body.options?.lockToVersion
|
||||
? await this.prisma.backgroundWorker.findFirst({
|
||||
where: {
|
||||
projectId: environment.projectId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
version: body.options?.lockToVersion,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
version: true,
|
||||
sdkVersion: true,
|
||||
cliVersion: true,
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const { queueName, lockedQueueId, taskTtl, taskKind } =
|
||||
await this.queueConcern.resolveQueueProperties(
|
||||
triggerRequest,
|
||||
lockedToBackgroundWorker ?? undefined
|
||||
);
|
||||
|
||||
// Resolve TTL with precedence: per-trigger > task-level > dev default
|
||||
let ttl: string | undefined;
|
||||
|
||||
if (body.options?.ttl !== undefined) {
|
||||
ttl =
|
||||
typeof body.options.ttl === "number"
|
||||
? stringifyDuration(body.options.ttl)
|
||||
: body.options.ttl;
|
||||
} else {
|
||||
ttl = taskTtl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined);
|
||||
}
|
||||
|
||||
if (!options.skipChecks) {
|
||||
const queueSizeGuard = await this.queueConcern.validateQueueLimits(
|
||||
environment,
|
||||
queueName
|
||||
);
|
||||
|
||||
if (!queueSizeGuard.ok) {
|
||||
throw new QueueSizeLimitExceededError(
|
||||
`Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}`,
|
||||
queueSizeGuard.maximumSize ?? 0,
|
||||
undefined,
|
||||
"warn"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataPacket = body.options?.metadata
|
||||
? handleMetadataPacket(
|
||||
body.options?.metadata,
|
||||
body.options?.metadataType ?? "application/json",
|
||||
this.metadataMaximumSize
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const tags = (
|
||||
body.options?.tags
|
||||
? typeof body.options.tags === "string"
|
||||
? [body.options.tags]
|
||||
: body.options.tags
|
||||
: []
|
||||
).filter((tag) => tag.trim().length > 0);
|
||||
|
||||
const depth = parentRun ? parentRun.depth + 1 : 0;
|
||||
|
||||
const workerQueueResult = await this.queueConcern.getWorkerQueue(
|
||||
environment,
|
||||
body.options?.region
|
||||
);
|
||||
const baseWorkerQueue = workerQueueResult?.masterQueue;
|
||||
const enableFastPath = workerQueueResult?.enableFastPath ?? false;
|
||||
|
||||
// Rewrite the region to its compute backing for migration-enrolled orgs,
|
||||
// from the in-memory snapshots (no DB query). A cold read (registry not yet
|
||||
// loaded) returns undefined/[] and the resolver falls back to not-migrated.
|
||||
const workerGroups = workerRegionRegistry.current() ?? [];
|
||||
const region = baseWorkerQueue
|
||||
? regionForQueue(baseWorkerQueue, workerGroups)
|
||||
: undefined;
|
||||
const backing = baseWorkerQueue
|
||||
? backingForQueue(baseWorkerQueue, workerGroups)
|
||||
: undefined;
|
||||
const migrated = resolveComputeMigration({
|
||||
baseWorkerQueue,
|
||||
baseEnableFastPath: enableFastPath,
|
||||
region,
|
||||
backing,
|
||||
planType,
|
||||
orgId: environment.organization.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null,
|
||||
flags: globalFlagsRegistry.current(),
|
||||
envType: environment.type,
|
||||
});
|
||||
|
||||
const triggerSource = options.triggerSource ?? "api";
|
||||
const triggerAction = options.triggerAction ?? "trigger";
|
||||
const parentAnnotations = RunAnnotations.safeParse(parentRun?.annotations).data;
|
||||
const annotations = {
|
||||
triggerSource,
|
||||
triggerAction,
|
||||
rootTriggerSource: parentAnnotations?.rootTriggerSource ?? triggerSource,
|
||||
rootScheduleId: parentAnnotations?.rootScheduleId || options.scheduleId || undefined,
|
||||
taskKind: taskKind ?? "STANDARD",
|
||||
};
|
||||
|
||||
// Route runs in a scheduled lineage (the scheduled run itself and every
|
||||
// descendant, via the propagated rootTriggerSource) to a dedicated
|
||||
// `<region>:scheduled` worker queue so a separate consumer fleet can
|
||||
// dequeue them independently of standard/agent runs. Gated per-org with
|
||||
// a global default, never applied to dev. Reads only the in-memory org
|
||||
// flags already on the environment — no DB query on the hot path.
|
||||
const scheduledQueueSplitEnabled =
|
||||
environment.type !== "DEVELOPMENT" &&
|
||||
resolveScheduledQueueSplitEnabled({
|
||||
orgFeatureFlags: environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null,
|
||||
globalDefault: env.TRIGGER_WORKER_QUEUE_SCHEDULED_SPLIT_ENABLED === "1",
|
||||
});
|
||||
const workerQueue =
|
||||
migrated.workerQueue !== undefined
|
||||
? workerQueueForRun({
|
||||
workerQueue: migrated.workerQueue,
|
||||
rootTriggerSource: annotations.rootTriggerSource,
|
||||
splitEnabled: scheduledQueueSplitEnabled,
|
||||
})
|
||||
: migrated.workerQueue;
|
||||
|
||||
try {
|
||||
return await this.traceEventConcern.traceRun(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore,
|
||||
async (event, store) => {
|
||||
event.setAttribute("queueName", queueName);
|
||||
span.setAttribute("queueName", queueName);
|
||||
event.setAttribute("runId", runFriendlyId);
|
||||
span.setAttribute("runId", runFriendlyId);
|
||||
|
||||
// Short-circuit when mollifier is globally off (the default
|
||||
// for every deployment that hasn't opted in). Avoids the
|
||||
// GateInputs allocation, the deps spread inside `evaluateGate`,
|
||||
// and the `mollifier.decisions{outcome=pass_through}` OTel
|
||||
// increment on every trigger — `triggerTask` is the
|
||||
// highest-throughput code path in the system. The check goes
|
||||
// through a DI'd predicate so unit tests that inject a custom
|
||||
// `evaluateGate` can also override the gate-on check (the
|
||||
// default reads `env.TRIGGER_MOLLIFIER_ENABLED`, which is "0"
|
||||
// in CI where no .env file is present).
|
||||
//
|
||||
// Batch items bypass the mollifier gate entirely.
|
||||
//
|
||||
// The mollify path returns a stripped run-shape `{ id,
|
||||
// friendlyId, spanId }` with no PG row written. Batch
|
||||
// tracking relies on `BatchTaskRunItem`, a join row whose
|
||||
// `taskRunId` column has a NOT NULL FK to `TaskRun.id` —
|
||||
// creating that join at trigger-time (in
|
||||
// `batchTriggerV3.server.ts:871`) fails with FK violation
|
||||
// for any mollified item, and skipping it at trigger-time
|
||||
// would silently drop the batch↔run link forever because
|
||||
// the drainer's materialise path doesn't (yet) create
|
||||
// `BatchTaskRunItem`. Either side alone is wrong:
|
||||
// - skip at trigger-time only → batch progress
|
||||
// under-reports forever, `batchTriggerAndWait` parent
|
||||
// stays parked
|
||||
// - mollify at trigger-time only → FK violation, 500
|
||||
//
|
||||
// The proper end state is a drainer-side
|
||||
// `BatchTaskRunItem` create-on-materialise (the snapshot
|
||||
// already carries `batch: { id, index }` so the drainer
|
||||
// has the info). That belongs in the drainer / replay PR,
|
||||
// not here. Until that lands, batch triggers pass-through
|
||||
// — they lose the burst-protection benefit, but the path
|
||||
// works end-to-end.
|
||||
const skipMollifierForBatch = !!options.batchId;
|
||||
const mollifierOutcome: GateOutcome | null =
|
||||
this.isMollifierGloballyEnabled() && !skipMollifierForBatch
|
||||
? await this.evaluateGate({
|
||||
envId: environment.id,
|
||||
orgId: environment.organizationId,
|
||||
taskId,
|
||||
orgFeatureFlags:
|
||||
(environment.organization.featureFlags as Record<
|
||||
string,
|
||||
unknown
|
||||
> | null) ?? null,
|
||||
options: {
|
||||
debounce: body.options?.debounce,
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
parentTaskRunId: body.options?.parentRunId,
|
||||
resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
|
||||
},
|
||||
})
|
||||
: null;
|
||||
|
||||
// When the gate says mollify, write the engine.trigger input
|
||||
// snapshot into the Redis buffer and return a synthesised
|
||||
// TriggerTaskServiceResult. The customer never waits on
|
||||
// Postgres; the drainer materialises the run later by replaying
|
||||
// engine.trigger against the snapshot. The run span has already
|
||||
// been opened by traceRun above (PARTIAL event in ClickHouse),
|
||||
// so its traceId/spanId live in the snapshot and the drainer's
|
||||
// `mollifier.drained` span parents on the same trace — buffered
|
||||
// runs become visible in the dashboard's trace view immediately,
|
||||
// not only after the drainer fires.
|
||||
if (mollifierOutcome?.action === "mollify") {
|
||||
const mollifierBuffer = this.getMollifierBuffer();
|
||||
if (mollifierBuffer && !body.options?.debounce) {
|
||||
event.setAttribute("mollifier.reason", mollifierOutcome.decision.reason);
|
||||
event.setAttribute("mollifier.count", String(mollifierOutcome.decision.count));
|
||||
event.setAttribute(
|
||||
"mollifier.threshold",
|
||||
String(mollifierOutcome.decision.threshold)
|
||||
);
|
||||
event.setAttribute("taskRunId", runFriendlyId);
|
||||
|
||||
const payloadPacket = await this.payloadProcessor.process(triggerRequest);
|
||||
|
||||
const engineTriggerInput = this.#buildEngineTriggerInput({
|
||||
runFriendlyId,
|
||||
environment,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
body,
|
||||
options,
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
workerQueue,
|
||||
region: migrated.region,
|
||||
enableFastPath: migrated.enableFastPath,
|
||||
lockedToBackgroundWorker: lockedToBackgroundWorker ?? undefined,
|
||||
delayUntil,
|
||||
ttl,
|
||||
metadataPacket,
|
||||
tags,
|
||||
depth,
|
||||
parentRun: parentRun ?? undefined,
|
||||
annotations,
|
||||
planType,
|
||||
taskId,
|
||||
payloadPacket,
|
||||
traceContext: this.#propagateExternalTraceContext(
|
||||
event.traceContext,
|
||||
parentRun?.traceContext,
|
||||
event.traceparent?.spanId
|
||||
),
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
parentSpanId:
|
||||
options.parentAsLinkType === "replay"
|
||||
? undefined
|
||||
: event.traceparent?.spanId,
|
||||
taskEventStore: store,
|
||||
});
|
||||
|
||||
const result = await mollifyTrigger({
|
||||
runFriendlyId,
|
||||
environmentId: environment.id,
|
||||
organizationId: environment.organizationId,
|
||||
engineTriggerInput,
|
||||
decision: mollifierOutcome.decision,
|
||||
buffer: mollifierBuffer,
|
||||
// Idempotency-key triple wires the buffer's SETNX into
|
||||
// the trigger-time dedup symmetric with PG.
|
||||
idempotencyKey,
|
||||
taskIdentifier: taskId,
|
||||
});
|
||||
|
||||
logger.debug("mollifier.buffered", {
|
||||
runId: runFriendlyId,
|
||||
envId: environment.id,
|
||||
orgId: environment.organizationId,
|
||||
taskId,
|
||||
reason: mollifierOutcome.decision.reason,
|
||||
});
|
||||
|
||||
// Synthetic result is structurally narrower than the full
|
||||
// TaskRun; the route handler only reads
|
||||
// `result.run.friendlyId`. traceRun flushes the PARTIAL
|
||||
// run-span event to ClickHouse on callback return.
|
||||
// `isMollified` flags the route to skip the request-
|
||||
// idempotency cache write — see the field's contract on
|
||||
// `TriggerTaskServiceResult`.
|
||||
return {
|
||||
...(result as unknown as TriggerTaskServiceResult),
|
||||
isMollified: true,
|
||||
};
|
||||
}
|
||||
if (!mollifierBuffer) {
|
||||
logger.warn(
|
||||
"mollifier gate said mollify but buffer is null — falling through to pass-through"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const payloadPacket = await this.payloadProcessor.process(triggerRequest);
|
||||
|
||||
const baseEngineInput = this.#buildEngineTriggerInput({
|
||||
runFriendlyId,
|
||||
environment,
|
||||
idempotencyKey,
|
||||
idempotencyKeyExpiresAt,
|
||||
body,
|
||||
options,
|
||||
queueName,
|
||||
lockedQueueId,
|
||||
workerQueue,
|
||||
region: migrated.region,
|
||||
enableFastPath: migrated.enableFastPath,
|
||||
lockedToBackgroundWorker: lockedToBackgroundWorker ?? undefined,
|
||||
delayUntil,
|
||||
ttl,
|
||||
metadataPacket,
|
||||
tags,
|
||||
depth,
|
||||
parentRun: parentRun ?? undefined,
|
||||
annotations,
|
||||
planType,
|
||||
taskId,
|
||||
payloadPacket,
|
||||
traceContext: this.#propagateExternalTraceContext(
|
||||
event.traceContext,
|
||||
parentRun?.traceContext,
|
||||
event.traceparent?.spanId
|
||||
),
|
||||
traceId: event.traceId,
|
||||
spanId: event.spanId,
|
||||
parentSpanId:
|
||||
options.parentAsLinkType === "replay" ? undefined : event.traceparent?.spanId,
|
||||
taskEventStore: store,
|
||||
});
|
||||
|
||||
const taskRun = await this.engine.trigger(
|
||||
{
|
||||
...baseEngineInput,
|
||||
// onDebounced is a closure over webapp state (triggerRequest +
|
||||
// traceEventConcern) and can't be serialised into the mollifier
|
||||
// snapshot. The pass-through path attaches it here; the drainer
|
||||
// path replays without it. The debounce and triggerAndWait gate
|
||||
// bypasses ensure neither reaches the mollify branch.
|
||||
onDebounced:
|
||||
body.options?.debounce && body.options?.resumeParentOnCompletion
|
||||
? async ({ existingRun, waitpoint, debounceKey }) => {
|
||||
return await this.traceEventConcern.traceDebouncedRun(
|
||||
triggerRequest,
|
||||
parentRun?.taskEventStore,
|
||||
{
|
||||
existingRun,
|
||||
debounceKey,
|
||||
incomplete: waitpoint.status === "PENDING",
|
||||
isError: waitpoint.outputIsError,
|
||||
},
|
||||
async (spanEvent) => {
|
||||
const spanId =
|
||||
options?.parentAsLinkType === "replay"
|
||||
? spanEvent.spanId
|
||||
: spanEvent.traceparent?.spanId
|
||||
? `${spanEvent.traceparent.spanId}:${spanEvent.spanId}`
|
||||
: spanEvent.spanId;
|
||||
return spanId;
|
||||
}
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
this.prisma
|
||||
);
|
||||
|
||||
// If the returned run has a different friendlyId, it was debounced.
|
||||
// For triggerAndWait: stop the outer span since a replacement debounced span was created via onDebounced.
|
||||
// For regular trigger: let the span complete normally - no replacement span needed since the
|
||||
// original run already has its span from when it was first created.
|
||||
if (
|
||||
taskRun.friendlyId !== runFriendlyId &&
|
||||
body.options?.debounce &&
|
||||
body.options?.resumeParentOnCompletion
|
||||
) {
|
||||
event.stop();
|
||||
}
|
||||
|
||||
const error = taskRun.error ? TaskRunError.parse(taskRun.error) : undefined;
|
||||
|
||||
if (error) {
|
||||
event.failWithError(error);
|
||||
}
|
||||
|
||||
const result = { run: taskRun, error, isCached: false };
|
||||
|
||||
if (result?.error) {
|
||||
throw new ServiceValidationError(
|
||||
taskRunErrorToString(taskRunErrorEnhancer(result.error))
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof RunDuplicateIdempotencyKeyError) {
|
||||
//retry calling this function, because this time it will return the idempotent run
|
||||
return await this.call({
|
||||
taskId,
|
||||
environment,
|
||||
body,
|
||||
options: { ...options, runFriendlyId },
|
||||
attempt: attempt + 1,
|
||||
});
|
||||
}
|
||||
|
||||
if (error instanceof RunOneTimeUseTokenError) {
|
||||
throw new ServiceValidationError(
|
||||
`Cannot trigger ${taskId} with a one-time use token as it has already been used.`
|
||||
);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
);
|
||||
// Pipeline returned successfully — publish the claim if we held
|
||||
// one. Waiters polling for our key resolve to this runId.
|
||||
if (idempotencyClaim && result?.run?.friendlyId) {
|
||||
await publishMollifierClaim({
|
||||
envId: idempotencyClaim.envId,
|
||||
taskIdentifier: idempotencyClaim.taskIdentifier,
|
||||
idempotencyKey: idempotencyClaim.idempotencyKey,
|
||||
token: idempotencyClaim.token,
|
||||
runId: result.run.friendlyId,
|
||||
ttlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
// Pipeline threw — release the claim so the next claimant can
|
||||
// retry. Re-throw so the caller sees the original error.
|
||||
if (idempotencyClaim) {
|
||||
await releaseMollifierClaim(idempotencyClaim);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Build the engine.trigger() input object from the values gathered during
|
||||
// this.call(). Extracted so the mollify path can construct the
|
||||
// same input shape without re-entering the trace-run span. The pass-through
|
||||
// path spreads this result and attaches `onDebounced` inline; the mollify
|
||||
// path serialises it into the buffer for drainer replay.
|
||||
#buildEngineTriggerInput(args: {
|
||||
runFriendlyId: string;
|
||||
environment: AuthenticatedEnvironment;
|
||||
idempotencyKey?: string;
|
||||
idempotencyKeyExpiresAt?: Date;
|
||||
body: TriggerTaskRequest["body"];
|
||||
options: TriggerTaskServiceOptions;
|
||||
queueName: string;
|
||||
lockedQueueId?: string;
|
||||
workerQueue?: string;
|
||||
region?: string;
|
||||
enableFastPath: boolean;
|
||||
lockedToBackgroundWorker?: {
|
||||
id: string;
|
||||
version: string;
|
||||
sdkVersion: string;
|
||||
cliVersion: string;
|
||||
};
|
||||
delayUntil?: Date;
|
||||
ttl?: string;
|
||||
metadataPacket?: { data?: string; dataType: string };
|
||||
tags: string[];
|
||||
depth: number;
|
||||
parentRun?: {
|
||||
id: string;
|
||||
rootTaskRunId?: string | null;
|
||||
queueTimestamp?: Date | null;
|
||||
taskEventStore?: string;
|
||||
};
|
||||
annotations: {
|
||||
triggerSource: string;
|
||||
triggerAction: string;
|
||||
rootTriggerSource: string;
|
||||
rootScheduleId?: string | undefined;
|
||||
};
|
||||
planType?: string;
|
||||
taskId: string;
|
||||
payloadPacket: { data?: string; dataType: string };
|
||||
traceContext: TriggerTraceContext;
|
||||
traceId: string;
|
||||
spanId: string;
|
||||
parentSpanId: string | undefined;
|
||||
taskEventStore: string;
|
||||
}) {
|
||||
return {
|
||||
friendlyId: args.runFriendlyId,
|
||||
environment: args.environment,
|
||||
idempotencyKey: args.idempotencyKey,
|
||||
idempotencyKeyExpiresAt: args.idempotencyKey ? args.idempotencyKeyExpiresAt : undefined,
|
||||
idempotencyKeyOptions: args.body.options?.idempotencyKeyOptions,
|
||||
taskIdentifier: args.taskId,
|
||||
payload: args.payloadPacket.data ?? "",
|
||||
payloadType: args.payloadPacket.dataType,
|
||||
context: args.body.context,
|
||||
traceContext: args.traceContext,
|
||||
traceId: args.traceId,
|
||||
spanId: args.spanId,
|
||||
parentSpanId: args.parentSpanId,
|
||||
replayedFromTaskRunFriendlyId: args.options.replayedFromTaskRunFriendlyId,
|
||||
lockedToVersionId: args.lockedToBackgroundWorker?.id,
|
||||
taskVersion: args.lockedToBackgroundWorker?.version,
|
||||
sdkVersion: args.lockedToBackgroundWorker?.sdkVersion,
|
||||
cliVersion: args.lockedToBackgroundWorker?.cliVersion,
|
||||
// Schema-level coercion now lands `body.options.concurrencyKey` as
|
||||
// `string` on the API path, but the BatchQueue worker rebuilds
|
||||
// body.options from Redis-stored items (Record<string, unknown>),
|
||||
// which can still carry the pre-fix shape from in-flight batches.
|
||||
concurrencyKey:
|
||||
typeof args.body.options?.concurrencyKey === "number"
|
||||
? String(args.body.options.concurrencyKey)
|
||||
: args.body.options?.concurrencyKey,
|
||||
queue: args.queueName,
|
||||
lockedQueueId: args.lockedQueueId,
|
||||
workerQueue: args.workerQueue,
|
||||
region: args.region,
|
||||
enableFastPath: args.enableFastPath,
|
||||
isTest: args.body.options?.test ?? false,
|
||||
delayUntil: args.delayUntil,
|
||||
queuedAt: args.delayUntil ? undefined : new Date(),
|
||||
maxAttempts: args.body.options?.maxAttempts,
|
||||
taskEventStore: args.taskEventStore,
|
||||
ttl: args.ttl,
|
||||
tags: args.tags,
|
||||
oneTimeUseToken: args.options.oneTimeUseToken,
|
||||
parentTaskRunId: args.parentRun?.id,
|
||||
rootTaskRunId: args.parentRun?.rootTaskRunId ?? args.parentRun?.id,
|
||||
batch: args.options?.batchId
|
||||
? { id: args.options.batchId, index: args.options.batchIndex ?? 0 }
|
||||
: undefined,
|
||||
resumeParentOnCompletion: args.body.options?.resumeParentOnCompletion,
|
||||
depth: args.depth,
|
||||
metadata: args.metadataPacket?.data,
|
||||
metadataType: args.metadataPacket?.dataType,
|
||||
seedMetadata: args.metadataPacket?.data,
|
||||
seedMetadataType: args.metadataPacket?.dataType,
|
||||
maxDurationInSeconds: args.body.options?.maxDuration
|
||||
? clampMaxDuration(args.body.options.maxDuration)
|
||||
: undefined,
|
||||
machine: args.body.options?.machine,
|
||||
priorityMs: args.body.options?.priority ? args.body.options.priority * 1_000 : undefined,
|
||||
queueTimestamp:
|
||||
args.options.queueTimestamp ??
|
||||
(args.parentRun && args.body.options?.resumeParentOnCompletion
|
||||
? (args.parentRun.queueTimestamp ?? undefined)
|
||||
: undefined),
|
||||
scheduleId: args.options.scheduleId,
|
||||
scheduleInstanceId: args.options.scheduleInstanceId,
|
||||
createdAt: args.options.overrideCreatedAt,
|
||||
bulkActionId: args.body.options?.bulkActionId,
|
||||
planType: args.planType,
|
||||
realtimeStreamsVersion: args.options.realtimeStreamsVersion,
|
||||
streamBasinName: args.environment.organization.streamBasinName,
|
||||
debounce: args.body.options?.debounce,
|
||||
annotations: args.annotations,
|
||||
};
|
||||
}
|
||||
|
||||
#propagateExternalTraceContext(
|
||||
traceContext: Record<string, unknown>,
|
||||
parentRunTraceContext: unknown,
|
||||
parentSpanId: string | undefined
|
||||
): TriggerTraceContext {
|
||||
if (!parentRunTraceContext) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const parsedParentRunTraceContext = TriggerTraceContext.safeParse(parentRunTraceContext);
|
||||
|
||||
if (!parsedParentRunTraceContext.success) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const { external } = parsedParentRunTraceContext.data;
|
||||
|
||||
if (!external) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
if (!external.traceparent) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const parsedTraceparent = parseTraceparent(external.traceparent);
|
||||
|
||||
if (!parsedTraceparent) {
|
||||
return traceContext;
|
||||
}
|
||||
|
||||
const newExternalTraceparent = serializeTraceparent(
|
||||
parsedTraceparent.traceId,
|
||||
parentSpanId ?? parsedTraceparent.spanId,
|
||||
parsedTraceparent.traceFlags
|
||||
);
|
||||
|
||||
return {
|
||||
...traceContext,
|
||||
external: {
|
||||
...external,
|
||||
traceparent: newExternalTraceparent,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user