chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,759 @@
|
||||
import {
|
||||
type BatchTriggerTaskV2RequestBody,
|
||||
type BatchTriggerTaskV3RequestBody,
|
||||
type BatchTriggerTaskV3Response,
|
||||
type IOPacket,
|
||||
packetRequiresOffloading,
|
||||
parsePacket,
|
||||
TaskRunErrorCodes,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
||||
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
||||
import { Evt } from "evt";
|
||||
import { z } from "zod";
|
||||
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
|
||||
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
||||
import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
||||
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
||||
import {
|
||||
downloadPacketFromObjectStore,
|
||||
uploadPacketToObjectStore,
|
||||
} from "../../v3/objectStore.server";
|
||||
import type { RunEngine } from "../../v3/runEngine.server";
|
||||
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
||||
import { TriggerTaskService } from "../../v3/services/triggerTask.server";
|
||||
import { startActiveSpan } from "../../v3/tracer.server";
|
||||
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
||||
|
||||
const PROCESSING_BATCH_SIZE = 50;
|
||||
const ASYNC_BATCH_PROCESS_SIZE_THRESHOLD = 20;
|
||||
const MAX_ATTEMPTS = 10;
|
||||
|
||||
export const BatchProcessingStrategy = z.enum(["sequential", "parallel"]);
|
||||
export type BatchProcessingStrategy = z.infer<typeof BatchProcessingStrategy>;
|
||||
|
||||
export const BatchProcessingOptions = z.object({
|
||||
batchId: z.string(),
|
||||
processingId: z.string(),
|
||||
range: z.object({ start: z.number().int(), count: z.number().int() }),
|
||||
attemptCount: z.number().int(),
|
||||
strategy: BatchProcessingStrategy,
|
||||
parentRunId: z.string().optional(),
|
||||
resumeParentOnCompletion: z.boolean().optional(),
|
||||
planType: z.string().optional(),
|
||||
});
|
||||
|
||||
export type BatchProcessingOptions = z.infer<typeof BatchProcessingOptions>;
|
||||
|
||||
export type BatchTriggerTaskServiceOptions = {
|
||||
triggerVersion?: string;
|
||||
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
||||
spanParentAsLink?: boolean;
|
||||
oneTimeUseToken?: string;
|
||||
realtimeStreamsVersion?: "v1" | "v2";
|
||||
triggerSource?: string;
|
||||
triggerAction?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Larger batches, used in Run Engine v2
|
||||
*/
|
||||
export class RunEngineBatchTriggerService extends WithRunEngine {
|
||||
private _batchProcessingStrategy: BatchProcessingStrategy;
|
||||
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
||||
|
||||
constructor(
|
||||
batchProcessingStrategy?: BatchProcessingStrategy,
|
||||
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
||||
engine?: RunEngine
|
||||
) {
|
||||
super({ prisma: _prisma, engine });
|
||||
|
||||
// Eric note: We need to force sequential processing because when doing parallel, we end up with high-contention on the parent run lock
|
||||
// becuase we are triggering a lot of runs at once, and each one is trying to lock the parent run.
|
||||
// by forcing sequential, we are only ever locking the parent run for a single run at a time.
|
||||
this._batchProcessingStrategy = "sequential";
|
||||
}
|
||||
|
||||
public async call(
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV3RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
): Promise<BatchTriggerTaskV3Response> {
|
||||
try {
|
||||
return await this.traceWithEnv<BatchTriggerTaskV3Response>(
|
||||
"call()",
|
||||
environment,
|
||||
async (span) => {
|
||||
const { friendlyId } = await mintBatchFriendlyId({
|
||||
environment: {
|
||||
organizationId: environment.organizationId,
|
||||
id: environment.id,
|
||||
orgFeatureFlags: environment.organization.featureFlags,
|
||||
},
|
||||
parentRunFriendlyId: body.parentRunId,
|
||||
});
|
||||
|
||||
span.setAttribute("batchId", friendlyId);
|
||||
|
||||
const payloadPacket = await this.#handlePayloadPacket(
|
||||
body.items,
|
||||
`batch/${friendlyId}`,
|
||||
environment
|
||||
);
|
||||
|
||||
const batch = await this.#createAndProcessBatchTaskRun(
|
||||
friendlyId,
|
||||
payloadPacket,
|
||||
environment,
|
||||
body,
|
||||
options
|
||||
);
|
||||
|
||||
if (!batch) {
|
||||
throw new Error("Failed to create batch");
|
||||
}
|
||||
|
||||
return {
|
||||
id: batch.friendlyId,
|
||||
isCached: false,
|
||||
idempotencyKey: batch.idempotencyKey ?? undefined,
|
||||
runCount: body.items.length,
|
||||
};
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
// Detect a prisma transaction Unique constraint violation
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
||||
logger.debug("RunEngineBatchTrigger: Prisma transaction error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
meta: error.meta,
|
||||
});
|
||||
|
||||
if (error.code === "P2002") {
|
||||
const target = error.meta?.target;
|
||||
|
||||
if (
|
||||
Array.isArray(target) &&
|
||||
target.length > 0 &&
|
||||
typeof target[0] === "string" &&
|
||||
target[0].includes("oneTimeUseToken")
|
||||
) {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger with a one-time use token as it has already been used."
|
||||
);
|
||||
} else {
|
||||
throw new ServiceValidationError(
|
||||
"Cannot batch trigger as it has already been triggered with the same idempotency key."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async #createAndProcessBatchTaskRun(
|
||||
batchId: string,
|
||||
payloadPacket: IOPacket,
|
||||
environment: AuthenticatedEnvironment,
|
||||
body: BatchTriggerTaskV2RequestBody,
|
||||
options: BatchTriggerTaskServiceOptions = {}
|
||||
) {
|
||||
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
||||
// validate env existence app-side (covers both create arms below).
|
||||
await controlPlaneResolver.assertEnvExists(environment.id);
|
||||
|
||||
if (body.items.length <= ASYNC_BATCH_PROCESS_SIZE_THRESHOLD) {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: 0,
|
||||
batchSize: PROCESSING_BATCH_SIZE,
|
||||
items: body.items,
|
||||
options,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing complete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: 0,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
});
|
||||
|
||||
// If processing inline does not finish for some reason, enqueue processing the rest of the batch
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][call] Batch inline processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
});
|
||||
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: "0",
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: "sequential",
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const batch = await this._engine.runStore.createBatchTaskRun({
|
||||
id: BatchId.fromFriendlyId(batchId),
|
||||
friendlyId: batchId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
runCount: body.items.length,
|
||||
runIds: [],
|
||||
payload: payloadPacket.data,
|
||||
payloadType: payloadPacket.dataType,
|
||||
options,
|
||||
batchVersion: "runengine:v1",
|
||||
oneTimeUseToken: options.oneTimeUseToken,
|
||||
});
|
||||
|
||||
this.onBatchTaskRunCreated.post(batch);
|
||||
|
||||
if (body.parentRunId && body.resumeParentOnCompletion) {
|
||||
await this._engine.blockRunWithCreatedBatch({
|
||||
runId: RunId.fromFriendlyId(body.parentRunId),
|
||||
batchId: batch.id,
|
||||
environmentId: environment.id,
|
||||
projectId: environment.projectId,
|
||||
organizationId: environment.organizationId,
|
||||
});
|
||||
}
|
||||
|
||||
switch (this._batchProcessingStrategy) {
|
||||
case "sequential": {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: batchId,
|
||||
range: { start: 0, count: PROCESSING_BATCH_SIZE },
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "parallel": {
|
||||
const ranges = Array.from({
|
||||
length: Math.ceil(body.items.length / PROCESSING_BATCH_SIZE),
|
||||
}).map((_, index) => ({
|
||||
start: index * PROCESSING_BATCH_SIZE,
|
||||
count: PROCESSING_BATCH_SIZE,
|
||||
}));
|
||||
|
||||
await Promise.all(
|
||||
ranges.map((range, index) =>
|
||||
this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: `${index}`,
|
||||
range,
|
||||
attemptCount: 0,
|
||||
strategy: this._batchProcessingStrategy,
|
||||
parentRunId: body.parentRunId,
|
||||
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
}
|
||||
|
||||
async #enqueueBatchTaskRun(options: BatchProcessingOptions) {
|
||||
await batchTriggerWorker.enqueue({
|
||||
id: `RunEngineBatchTriggerService.process:${options.batchId}:${options.processingId}`,
|
||||
job: "runengine.processBatchTaskRun",
|
||||
payload: options,
|
||||
});
|
||||
}
|
||||
|
||||
// This is the function that the worker will call
|
||||
async processBatchTaskRun(options: BatchProcessingOptions) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch", {
|
||||
options,
|
||||
});
|
||||
|
||||
const $attemptCount = options.attemptCount + 1;
|
||||
|
||||
if ($attemptCount > MAX_ATTEMPTS) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Max attempts reached", {
|
||||
options,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
// You might want to update the batch status to failed here
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = await this._engine.runStore.findBatchTaskRunById(options.batchId);
|
||||
|
||||
if (!batch) {
|
||||
return;
|
||||
}
|
||||
|
||||
// BatchTaskRun -> RuntimeEnvironment FK is dropped; resolve the env from the scalar id.
|
||||
const environment = await findEnvironmentById(batch.runtimeEnvironmentId);
|
||||
if (!environment) {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Environment not found", {
|
||||
batchId: batch.id,
|
||||
runtimeEnvironmentId: batch.runtimeEnvironmentId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.range.start >= batch.runCount) {
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] currentIndex is greater than runCount",
|
||||
{
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
runCount: batch.runCount,
|
||||
attemptCount: $attemptCount,
|
||||
}
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const payloadPacket = await downloadPacketFromObjectStore(
|
||||
{
|
||||
data: batch.payload ?? undefined,
|
||||
dataType: batch.payloadType,
|
||||
},
|
||||
environment
|
||||
);
|
||||
|
||||
const payload = await parsePacket(payloadPacket);
|
||||
|
||||
if (!payload) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Failed to parse payload", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
throw new Error("Failed to parse payload");
|
||||
}
|
||||
|
||||
// Skip zod parsing
|
||||
const $payload = payload as BatchTriggerTaskV2RequestBody["items"];
|
||||
const $options = batch.options as BatchTriggerTaskServiceOptions;
|
||||
|
||||
const result = await this.#processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex: options.range.start,
|
||||
batchSize: options.range.count,
|
||||
items: $payload,
|
||||
options: $options,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
switch (result.status) {
|
||||
case "COMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing complete", {
|
||||
options,
|
||||
batchId: batch.friendlyId,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
case "INCOMPLETE": {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing incomplete", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// Only enqueue the next batch task run if the strategy is sequential
|
||||
// if the strategy is parallel, we will already have enqueued the next batch task run
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count,
|
||||
},
|
||||
attemptCount: 0,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
case "ERROR": {
|
||||
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing error", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: result.workingIndex,
|
||||
error: result.error,
|
||||
attemptCount: $attemptCount,
|
||||
});
|
||||
|
||||
// if the strategy is sequential, we will requeue processing with a count of the PROCESSING_BATCH_SIZE
|
||||
// if the strategy is parallel, we will requeue processing with a range starting at the workingIndex and a count that is the remainder of this "slice" of the batch
|
||||
if (options.strategy === "sequential") {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
count: options.range.count, // This will be the same as the original count
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
} else {
|
||||
await this.#enqueueBatchTaskRun({
|
||||
batchId: batch.id,
|
||||
processingId: options.processingId,
|
||||
range: {
|
||||
start: result.workingIndex,
|
||||
// This will be the remainder of the slice
|
||||
// for example if the original range was 0-50 and the workingIndex is 25, the new range will be 25-25
|
||||
// if the original range was 51-100 and the workingIndex is 75, the new range will be 75-25
|
||||
count: options.range.count - result.workingIndex - options.range.start,
|
||||
},
|
||||
attemptCount: $attemptCount,
|
||||
strategy: options.strategy,
|
||||
parentRunId: options.parentRunId,
|
||||
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
||||
});
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItems({
|
||||
batch,
|
||||
environment,
|
||||
currentIndex,
|
||||
batchSize,
|
||||
items,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
currentIndex: number;
|
||||
batchSize: number;
|
||||
items: BatchTriggerTaskV2RequestBody["items"];
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId?: string | undefined;
|
||||
resumeParentOnCompletion?: boolean | undefined;
|
||||
}): Promise<
|
||||
| { status: "COMPLETE" }
|
||||
| { status: "INCOMPLETE"; workingIndex: number }
|
||||
| { status: "ERROR"; error: string; workingIndex: number }
|
||||
> {
|
||||
// Grab the next PROCESSING_BATCH_SIZE items
|
||||
const itemsToProcess = items.slice(currentIndex, currentIndex + batchSize);
|
||||
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch items", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
runCount: batch.runCount,
|
||||
});
|
||||
|
||||
let workingIndex = currentIndex;
|
||||
|
||||
let runIds: string[] = [];
|
||||
|
||||
const triggerFailedTaskService = new TriggerFailedTaskService({
|
||||
prisma: this._prisma,
|
||||
engine: this._engine,
|
||||
replicaPrisma: this._replica,
|
||||
});
|
||||
|
||||
for (const item of itemsToProcess) {
|
||||
let runFriendlyId: string | null = null;
|
||||
|
||||
try {
|
||||
const run = await this.#processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex: workingIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
});
|
||||
|
||||
if (run) {
|
||||
runFriendlyId = run.friendlyId;
|
||||
}
|
||||
} catch (_error) {
|
||||
// Trigger failed - will try to create pre-failed run below
|
||||
runFriendlyId = null;
|
||||
}
|
||||
|
||||
if (!runFriendlyId) {
|
||||
const errorMessage =
|
||||
"Trigger failed for batch item (queue limit, entitlement, or validation error)";
|
||||
logger.debug(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Item trigger failed, creating pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
task: item.task,
|
||||
}
|
||||
);
|
||||
|
||||
const failedRunId = await triggerFailedTaskService.call({
|
||||
taskId: item.task,
|
||||
environment,
|
||||
payload: item.payload,
|
||||
payloadType: item.options?.payloadType,
|
||||
errorMessage,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
batch: { id: batch.id, index: workingIndex },
|
||||
// Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
|
||||
// co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
|
||||
runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
|
||||
options: item.options as Record<string, unknown>,
|
||||
traceContext: options?.traceContext as Record<string, unknown> | undefined,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
|
||||
});
|
||||
|
||||
if (failedRunId) {
|
||||
runFriendlyId = failedRunId;
|
||||
} else {
|
||||
logger.error(
|
||||
"[RunEngineBatchTrigger][processBatchTaskRun] Failed to create pre-failed run",
|
||||
{
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex: workingIndex,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
status: "ERROR",
|
||||
error: "Could not trigger item and could not create pre-failed run",
|
||||
workingIndex,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
runIds.push(runFriendlyId);
|
||||
workingIndex++;
|
||||
}
|
||||
|
||||
const updatedBatch = await this._engine.runStore.updateBatchTaskRun({
|
||||
where: { id: batch.id },
|
||||
data: {
|
||||
runIds: {
|
||||
push: runIds,
|
||||
},
|
||||
processingJobsCount: {
|
||||
increment: runIds.length,
|
||||
},
|
||||
},
|
||||
select: {
|
||||
processingJobsCount: true,
|
||||
runCount: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (updatedBatch.processingJobsCount >= updatedBatch.runCount) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] All runs created", {
|
||||
batchId: batch.friendlyId,
|
||||
processingJobsCount: updatedBatch.processingJobsCount,
|
||||
runCount: updatedBatch.runCount,
|
||||
workingIndex,
|
||||
});
|
||||
|
||||
//if all the runs were idempotent, it's possible the batch is already completed
|
||||
await this._engine.tryCompleteBatch({ batchId: batch.id });
|
||||
}
|
||||
|
||||
// if there are more items to process, requeue the batch
|
||||
if (workingIndex < batch.runCount) {
|
||||
return { status: "INCOMPLETE", workingIndex };
|
||||
}
|
||||
|
||||
return { status: "COMPLETE" };
|
||||
}
|
||||
|
||||
async #processBatchTaskRunItem({
|
||||
batch,
|
||||
environment,
|
||||
item,
|
||||
currentIndex,
|
||||
options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
}: {
|
||||
batch: BatchTaskRun;
|
||||
environment: AuthenticatedEnvironment;
|
||||
item: BatchTriggerTaskV2RequestBody["items"][number];
|
||||
currentIndex: number;
|
||||
options?: BatchTriggerTaskServiceOptions;
|
||||
parentRunId: string | undefined;
|
||||
resumeParentOnCompletion: boolean | undefined;
|
||||
}) {
|
||||
logger.debug("[RunEngineBatchTrigger][processBatchTaskRunItem] Processing item", {
|
||||
batchId: batch.friendlyId,
|
||||
currentIndex,
|
||||
});
|
||||
|
||||
const triggerTaskService = new TriggerTaskService();
|
||||
|
||||
// Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
|
||||
// an org's mint flag flipping between batch creation and this (possibly much later,
|
||||
// worker-processed) item never splits the item from its BatchTaskRun row. See
|
||||
// mintAnchoredRunFriendlyId.server.ts.
|
||||
const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
|
||||
|
||||
const result = await triggerTaskService.call(
|
||||
item.task,
|
||||
environment,
|
||||
{
|
||||
...item,
|
||||
options: {
|
||||
...item.options,
|
||||
parentRunId,
|
||||
resumeParentOnCompletion,
|
||||
parentBatch: batch.id,
|
||||
},
|
||||
},
|
||||
{
|
||||
triggerVersion: options?.triggerVersion,
|
||||
traceContext: options?.traceContext,
|
||||
spanParentAsLink: options?.spanParentAsLink,
|
||||
batchId: batch.id,
|
||||
batchIndex: currentIndex,
|
||||
runFriendlyId,
|
||||
realtimeStreamsVersion: options?.realtimeStreamsVersion,
|
||||
triggerSource: options?.triggerSource ?? "api",
|
||||
triggerAction: options?.triggerAction ?? "trigger",
|
||||
},
|
||||
"V2"
|
||||
);
|
||||
|
||||
return result
|
||||
? {
|
||||
friendlyId: result.run.friendlyId,
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async #handlePayloadPacket(
|
||||
payload: any,
|
||||
pathPrefix: string,
|
||||
environment: AuthenticatedEnvironment
|
||||
) {
|
||||
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
||||
const packet = { data: JSON.stringify(payload), dataType: "application/json" };
|
||||
|
||||
if (!packet.data) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const { needsOffloading } = packetRequiresOffloading(
|
||||
packet,
|
||||
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
||||
);
|
||||
|
||||
if (!needsOffloading) {
|
||||
return packet;
|
||||
}
|
||||
|
||||
const filename = `${pathPrefix}/payload.json`;
|
||||
|
||||
const uploadedFilename = await uploadPacketToObjectStore(
|
||||
filename,
|
||||
packet.data,
|
||||
packet.dataType,
|
||||
environment
|
||||
);
|
||||
|
||||
return {
|
||||
data: uploadedFilename,
|
||||
dataType: "application/store",
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user