wehub-resource-sync
760 lines
24 KiB
TypeScript
760 lines
24 KiB
TypeScript
import {
|
|
type BatchTriggerTaskV2RequestBody,
|
|
type BatchTriggerTaskV3RequestBody,
|
|
type BatchTriggerTaskV3Response,
|
|
type IOPacket,
|
|
packetRequiresOffloading,
|
|
parsePacket,
|
|
TaskRunErrorCodes,
|
|
} from "@trigger.dev/core/v3";
|
|
import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic";
|
|
import { type BatchTaskRun, Prisma } from "@trigger.dev/database";
|
|
import { Evt } from "evt";
|
|
import { z } from "zod";
|
|
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
|
|
import { env } from "~/env.server";
|
|
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
|
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
|
import { logger } from "~/services/logger.server";
|
|
import { batchTriggerWorker } from "~/v3/batchTriggerWorker.server";
|
|
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
|
|
import { mintAnchoredRunFriendlyId } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server";
|
|
import { mintBatchFriendlyId } from "~/v3/runOpsMigration/mintBatchFriendlyId.server";
|
|
import {
|
|
downloadPacketFromObjectStore,
|
|
uploadPacketToObjectStore,
|
|
} from "../../v3/objectStore.server";
|
|
import type { RunEngine } from "../../v3/runEngine.server";
|
|
import { ServiceValidationError, WithRunEngine } from "../../v3/services/baseService.server";
|
|
import { TriggerTaskService } from "../../v3/services/triggerTask.server";
|
|
import { startActiveSpan } from "../../v3/tracer.server";
|
|
import { TriggerFailedTaskService } from "./triggerFailedTask.server";
|
|
|
|
const PROCESSING_BATCH_SIZE = 50;
|
|
const ASYNC_BATCH_PROCESS_SIZE_THRESHOLD = 20;
|
|
const MAX_ATTEMPTS = 10;
|
|
|
|
export const BatchProcessingStrategy = z.enum(["sequential", "parallel"]);
|
|
export type BatchProcessingStrategy = z.infer<typeof BatchProcessingStrategy>;
|
|
|
|
export const BatchProcessingOptions = z.object({
|
|
batchId: z.string(),
|
|
processingId: z.string(),
|
|
range: z.object({ start: z.number().int(), count: z.number().int() }),
|
|
attemptCount: z.number().int(),
|
|
strategy: BatchProcessingStrategy,
|
|
parentRunId: z.string().optional(),
|
|
resumeParentOnCompletion: z.boolean().optional(),
|
|
planType: z.string().optional(),
|
|
});
|
|
|
|
export type BatchProcessingOptions = z.infer<typeof BatchProcessingOptions>;
|
|
|
|
export type BatchTriggerTaskServiceOptions = {
|
|
triggerVersion?: string;
|
|
traceContext?: Record<string, string | undefined | Record<string, string | undefined>>;
|
|
spanParentAsLink?: boolean;
|
|
oneTimeUseToken?: string;
|
|
realtimeStreamsVersion?: "v1" | "v2";
|
|
triggerSource?: string;
|
|
triggerAction?: string;
|
|
};
|
|
|
|
/**
|
|
* Larger batches, used in Run Engine v2
|
|
*/
|
|
export class RunEngineBatchTriggerService extends WithRunEngine {
|
|
private _batchProcessingStrategy: BatchProcessingStrategy;
|
|
public onBatchTaskRunCreated: Evt<BatchTaskRun> = new Evt();
|
|
|
|
constructor(
|
|
batchProcessingStrategy?: BatchProcessingStrategy,
|
|
protected readonly _prisma: PrismaClientOrTransaction = prisma,
|
|
engine?: RunEngine
|
|
) {
|
|
super({ prisma: _prisma, engine });
|
|
|
|
// Eric note: We need to force sequential processing because when doing parallel, we end up with high-contention on the parent run lock
|
|
// becuase we are triggering a lot of runs at once, and each one is trying to lock the parent run.
|
|
// by forcing sequential, we are only ever locking the parent run for a single run at a time.
|
|
this._batchProcessingStrategy = "sequential";
|
|
}
|
|
|
|
public async call(
|
|
environment: AuthenticatedEnvironment,
|
|
body: BatchTriggerTaskV3RequestBody,
|
|
options: BatchTriggerTaskServiceOptions = {}
|
|
): Promise<BatchTriggerTaskV3Response> {
|
|
try {
|
|
return await this.traceWithEnv<BatchTriggerTaskV3Response>(
|
|
"call()",
|
|
environment,
|
|
async (span) => {
|
|
const { friendlyId } = await mintBatchFriendlyId({
|
|
environment: {
|
|
organizationId: environment.organizationId,
|
|
id: environment.id,
|
|
orgFeatureFlags: environment.organization.featureFlags,
|
|
},
|
|
parentRunFriendlyId: body.parentRunId,
|
|
});
|
|
|
|
span.setAttribute("batchId", friendlyId);
|
|
|
|
const payloadPacket = await this.#handlePayloadPacket(
|
|
body.items,
|
|
`batch/${friendlyId}`,
|
|
environment
|
|
);
|
|
|
|
const batch = await this.#createAndProcessBatchTaskRun(
|
|
friendlyId,
|
|
payloadPacket,
|
|
environment,
|
|
body,
|
|
options
|
|
);
|
|
|
|
if (!batch) {
|
|
throw new Error("Failed to create batch");
|
|
}
|
|
|
|
return {
|
|
id: batch.friendlyId,
|
|
isCached: false,
|
|
idempotencyKey: batch.idempotencyKey ?? undefined,
|
|
runCount: body.items.length,
|
|
};
|
|
}
|
|
);
|
|
} catch (error) {
|
|
// Detect a prisma transaction Unique constraint violation
|
|
if (error instanceof Prisma.PrismaClientKnownRequestError) {
|
|
logger.debug("RunEngineBatchTrigger: Prisma transaction error", {
|
|
code: error.code,
|
|
message: error.message,
|
|
meta: error.meta,
|
|
});
|
|
|
|
if (error.code === "P2002") {
|
|
const target = error.meta?.target;
|
|
|
|
if (
|
|
Array.isArray(target) &&
|
|
target.length > 0 &&
|
|
typeof target[0] === "string" &&
|
|
target[0].includes("oneTimeUseToken")
|
|
) {
|
|
throw new ServiceValidationError(
|
|
"Cannot batch trigger with a one-time use token as it has already been used."
|
|
);
|
|
} else {
|
|
throw new ServiceValidationError(
|
|
"Cannot batch trigger as it has already been triggered with the same idempotency key."
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async #createAndProcessBatchTaskRun(
|
|
batchId: string,
|
|
payloadPacket: IOPacket,
|
|
environment: AuthenticatedEnvironment,
|
|
body: BatchTriggerTaskV2RequestBody,
|
|
options: BatchTriggerTaskServiceOptions = {}
|
|
) {
|
|
// BatchTaskRun.runtimeEnvironmentId no longer has an FK into RuntimeEnvironment;
|
|
// validate env existence app-side (covers both create arms below).
|
|
await controlPlaneResolver.assertEnvExists(environment.id);
|
|
|
|
if (body.items.length <= ASYNC_BATCH_PROCESS_SIZE_THRESHOLD) {
|
|
const batch = await this._engine.runStore.createBatchTaskRun({
|
|
id: BatchId.fromFriendlyId(batchId),
|
|
friendlyId: batchId,
|
|
runtimeEnvironmentId: environment.id,
|
|
runCount: body.items.length,
|
|
runIds: [],
|
|
payload: payloadPacket.data,
|
|
payloadType: payloadPacket.dataType,
|
|
options,
|
|
batchVersion: "runengine:v1",
|
|
oneTimeUseToken: options.oneTimeUseToken,
|
|
});
|
|
|
|
this.onBatchTaskRunCreated.post(batch);
|
|
|
|
if (body.parentRunId && body.resumeParentOnCompletion) {
|
|
await this._engine.blockRunWithCreatedBatch({
|
|
runId: RunId.fromFriendlyId(body.parentRunId),
|
|
batchId: batch.id,
|
|
environmentId: environment.id,
|
|
projectId: environment.projectId,
|
|
organizationId: environment.organizationId,
|
|
});
|
|
}
|
|
|
|
const result = await this.#processBatchTaskRunItems({
|
|
batch,
|
|
environment,
|
|
currentIndex: 0,
|
|
batchSize: PROCESSING_BATCH_SIZE,
|
|
items: body.items,
|
|
options,
|
|
parentRunId: body.parentRunId,
|
|
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
|
});
|
|
|
|
switch (result.status) {
|
|
case "COMPLETE": {
|
|
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing complete", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex: 0,
|
|
});
|
|
|
|
return batch;
|
|
}
|
|
case "INCOMPLETE": {
|
|
logger.debug("[RunEngineBatchTrigger][call] Batch inline processing incomplete", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex: result.workingIndex,
|
|
});
|
|
|
|
// If processing inline does not finish for some reason, enqueue processing the rest of the batch
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: "0",
|
|
range: {
|
|
start: result.workingIndex,
|
|
count: PROCESSING_BATCH_SIZE,
|
|
},
|
|
attemptCount: 0,
|
|
strategy: "sequential",
|
|
parentRunId: body.parentRunId,
|
|
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
|
});
|
|
|
|
return batch;
|
|
}
|
|
case "ERROR": {
|
|
logger.error("[RunEngineBatchTrigger][call] Batch inline processing error", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex: result.workingIndex,
|
|
error: result.error,
|
|
});
|
|
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: "0",
|
|
range: {
|
|
start: result.workingIndex,
|
|
count: PROCESSING_BATCH_SIZE,
|
|
},
|
|
attemptCount: 0,
|
|
strategy: "sequential",
|
|
parentRunId: body.parentRunId,
|
|
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
|
});
|
|
|
|
return batch;
|
|
}
|
|
}
|
|
} else {
|
|
const batch = await this._engine.runStore.createBatchTaskRun({
|
|
id: BatchId.fromFriendlyId(batchId),
|
|
friendlyId: batchId,
|
|
runtimeEnvironmentId: environment.id,
|
|
runCount: body.items.length,
|
|
runIds: [],
|
|
payload: payloadPacket.data,
|
|
payloadType: payloadPacket.dataType,
|
|
options,
|
|
batchVersion: "runengine:v1",
|
|
oneTimeUseToken: options.oneTimeUseToken,
|
|
});
|
|
|
|
this.onBatchTaskRunCreated.post(batch);
|
|
|
|
if (body.parentRunId && body.resumeParentOnCompletion) {
|
|
await this._engine.blockRunWithCreatedBatch({
|
|
runId: RunId.fromFriendlyId(body.parentRunId),
|
|
batchId: batch.id,
|
|
environmentId: environment.id,
|
|
projectId: environment.projectId,
|
|
organizationId: environment.organizationId,
|
|
});
|
|
}
|
|
|
|
switch (this._batchProcessingStrategy) {
|
|
case "sequential": {
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: batchId,
|
|
range: { start: 0, count: PROCESSING_BATCH_SIZE },
|
|
attemptCount: 0,
|
|
strategy: this._batchProcessingStrategy,
|
|
parentRunId: body.parentRunId,
|
|
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
|
});
|
|
|
|
break;
|
|
}
|
|
case "parallel": {
|
|
const ranges = Array.from({
|
|
length: Math.ceil(body.items.length / PROCESSING_BATCH_SIZE),
|
|
}).map((_, index) => ({
|
|
start: index * PROCESSING_BATCH_SIZE,
|
|
count: PROCESSING_BATCH_SIZE,
|
|
}));
|
|
|
|
await Promise.all(
|
|
ranges.map((range, index) =>
|
|
this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: `${index}`,
|
|
range,
|
|
attemptCount: 0,
|
|
strategy: this._batchProcessingStrategy,
|
|
parentRunId: body.parentRunId,
|
|
resumeParentOnCompletion: body.resumeParentOnCompletion,
|
|
})
|
|
)
|
|
);
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
return batch;
|
|
}
|
|
}
|
|
|
|
async #enqueueBatchTaskRun(options: BatchProcessingOptions) {
|
|
await batchTriggerWorker.enqueue({
|
|
id: `RunEngineBatchTriggerService.process:${options.batchId}:${options.processingId}`,
|
|
job: "runengine.processBatchTaskRun",
|
|
payload: options,
|
|
});
|
|
}
|
|
|
|
// This is the function that the worker will call
|
|
async processBatchTaskRun(options: BatchProcessingOptions) {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch", {
|
|
options,
|
|
});
|
|
|
|
const $attemptCount = options.attemptCount + 1;
|
|
|
|
if ($attemptCount > MAX_ATTEMPTS) {
|
|
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Max attempts reached", {
|
|
options,
|
|
attemptCount: $attemptCount,
|
|
});
|
|
// You might want to update the batch status to failed here
|
|
return;
|
|
}
|
|
|
|
const batch = await this._engine.runStore.findBatchTaskRunById(options.batchId);
|
|
|
|
if (!batch) {
|
|
return;
|
|
}
|
|
|
|
// BatchTaskRun -> RuntimeEnvironment FK is dropped; resolve the env from the scalar id.
|
|
const environment = await findEnvironmentById(batch.runtimeEnvironmentId);
|
|
if (!environment) {
|
|
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Environment not found", {
|
|
batchId: batch.id,
|
|
runtimeEnvironmentId: batch.runtimeEnvironmentId,
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (options.range.start >= batch.runCount) {
|
|
logger.debug(
|
|
"[RunEngineBatchTrigger][processBatchTaskRun] currentIndex is greater than runCount",
|
|
{
|
|
options,
|
|
batchId: batch.friendlyId,
|
|
runCount: batch.runCount,
|
|
attemptCount: $attemptCount,
|
|
}
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
const payloadPacket = await downloadPacketFromObjectStore(
|
|
{
|
|
data: batch.payload ?? undefined,
|
|
dataType: batch.payloadType,
|
|
},
|
|
environment
|
|
);
|
|
|
|
const payload = await parsePacket(payloadPacket);
|
|
|
|
if (!payload) {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Failed to parse payload", {
|
|
options,
|
|
batchId: batch.friendlyId,
|
|
attemptCount: $attemptCount,
|
|
});
|
|
|
|
throw new Error("Failed to parse payload");
|
|
}
|
|
|
|
// Skip zod parsing
|
|
const $payload = payload as BatchTriggerTaskV2RequestBody["items"];
|
|
const $options = batch.options as BatchTriggerTaskServiceOptions;
|
|
|
|
const result = await this.#processBatchTaskRunItems({
|
|
batch,
|
|
environment,
|
|
currentIndex: options.range.start,
|
|
batchSize: options.range.count,
|
|
items: $payload,
|
|
options: $options,
|
|
parentRunId: options.parentRunId,
|
|
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
|
});
|
|
|
|
switch (result.status) {
|
|
case "COMPLETE": {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing complete", {
|
|
options,
|
|
batchId: batch.friendlyId,
|
|
attemptCount: $attemptCount,
|
|
});
|
|
|
|
return;
|
|
}
|
|
case "INCOMPLETE": {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing incomplete", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex: result.workingIndex,
|
|
attemptCount: $attemptCount,
|
|
});
|
|
|
|
// Only enqueue the next batch task run if the strategy is sequential
|
|
// if the strategy is parallel, we will already have enqueued the next batch task run
|
|
if (options.strategy === "sequential") {
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: options.processingId,
|
|
range: {
|
|
start: result.workingIndex,
|
|
count: options.range.count,
|
|
},
|
|
attemptCount: 0,
|
|
strategy: options.strategy,
|
|
parentRunId: options.parentRunId,
|
|
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
|
});
|
|
}
|
|
|
|
return;
|
|
}
|
|
case "ERROR": {
|
|
logger.error("[RunEngineBatchTrigger][processBatchTaskRun] Batch processing error", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex: result.workingIndex,
|
|
error: result.error,
|
|
attemptCount: $attemptCount,
|
|
});
|
|
|
|
// if the strategy is sequential, we will requeue processing with a count of the PROCESSING_BATCH_SIZE
|
|
// if the strategy is parallel, we will requeue processing with a range starting at the workingIndex and a count that is the remainder of this "slice" of the batch
|
|
if (options.strategy === "sequential") {
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: options.processingId,
|
|
range: {
|
|
start: result.workingIndex,
|
|
count: options.range.count, // This will be the same as the original count
|
|
},
|
|
attemptCount: $attemptCount,
|
|
strategy: options.strategy,
|
|
parentRunId: options.parentRunId,
|
|
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
|
});
|
|
} else {
|
|
await this.#enqueueBatchTaskRun({
|
|
batchId: batch.id,
|
|
processingId: options.processingId,
|
|
range: {
|
|
start: result.workingIndex,
|
|
// This will be the remainder of the slice
|
|
// for example if the original range was 0-50 and the workingIndex is 25, the new range will be 25-25
|
|
// if the original range was 51-100 and the workingIndex is 75, the new range will be 75-25
|
|
count: options.range.count - result.workingIndex - options.range.start,
|
|
},
|
|
attemptCount: $attemptCount,
|
|
strategy: options.strategy,
|
|
parentRunId: options.parentRunId,
|
|
resumeParentOnCompletion: options.resumeParentOnCompletion,
|
|
});
|
|
}
|
|
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
async #processBatchTaskRunItems({
|
|
batch,
|
|
environment,
|
|
currentIndex,
|
|
batchSize,
|
|
items,
|
|
options,
|
|
parentRunId,
|
|
resumeParentOnCompletion,
|
|
}: {
|
|
batch: BatchTaskRun;
|
|
environment: AuthenticatedEnvironment;
|
|
currentIndex: number;
|
|
batchSize: number;
|
|
items: BatchTriggerTaskV2RequestBody["items"];
|
|
options?: BatchTriggerTaskServiceOptions;
|
|
parentRunId?: string | undefined;
|
|
resumeParentOnCompletion?: boolean | undefined;
|
|
}): Promise<
|
|
| { status: "COMPLETE" }
|
|
| { status: "INCOMPLETE"; workingIndex: number }
|
|
| { status: "ERROR"; error: string; workingIndex: number }
|
|
> {
|
|
// Grab the next PROCESSING_BATCH_SIZE items
|
|
const itemsToProcess = items.slice(currentIndex, currentIndex + batchSize);
|
|
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] Processing batch items", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex,
|
|
runCount: batch.runCount,
|
|
});
|
|
|
|
let workingIndex = currentIndex;
|
|
|
|
let runIds: string[] = [];
|
|
|
|
const triggerFailedTaskService = new TriggerFailedTaskService({
|
|
prisma: this._prisma,
|
|
engine: this._engine,
|
|
replicaPrisma: this._replica,
|
|
});
|
|
|
|
for (const item of itemsToProcess) {
|
|
let runFriendlyId: string | null = null;
|
|
|
|
try {
|
|
const run = await this.#processBatchTaskRunItem({
|
|
batch,
|
|
environment,
|
|
item,
|
|
currentIndex: workingIndex,
|
|
options,
|
|
parentRunId,
|
|
resumeParentOnCompletion,
|
|
});
|
|
|
|
if (run) {
|
|
runFriendlyId = run.friendlyId;
|
|
}
|
|
} catch (_error) {
|
|
// Trigger failed - will try to create pre-failed run below
|
|
runFriendlyId = null;
|
|
}
|
|
|
|
if (!runFriendlyId) {
|
|
const errorMessage =
|
|
"Trigger failed for batch item (queue limit, entitlement, or validation error)";
|
|
logger.debug(
|
|
"[RunEngineBatchTrigger][processBatchTaskRun] Item trigger failed, creating pre-failed run",
|
|
{
|
|
batchId: batch.friendlyId,
|
|
currentIndex: workingIndex,
|
|
task: item.task,
|
|
}
|
|
);
|
|
|
|
const failedRunId = await triggerFailedTaskService.call({
|
|
taskId: item.task,
|
|
environment,
|
|
payload: item.payload,
|
|
payloadType: item.options?.payloadType,
|
|
errorMessage,
|
|
parentRunId,
|
|
resumeParentOnCompletion,
|
|
batch: { id: batch.id, index: workingIndex },
|
|
// Anchor the pre-failed run on the BATCH's residency (same as the happy path) so it
|
|
// co-resides with its BatchTaskRun row regardless of a mid-batch mint-flag flip.
|
|
runFriendlyId: mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region),
|
|
options: item.options as Record<string, unknown>,
|
|
traceContext: options?.traceContext as Record<string, unknown> | undefined,
|
|
spanParentAsLink: options?.spanParentAsLink,
|
|
errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER,
|
|
});
|
|
|
|
if (failedRunId) {
|
|
runFriendlyId = failedRunId;
|
|
} else {
|
|
logger.error(
|
|
"[RunEngineBatchTrigger][processBatchTaskRun] Failed to create pre-failed run",
|
|
{
|
|
batchId: batch.friendlyId,
|
|
currentIndex: workingIndex,
|
|
}
|
|
);
|
|
|
|
return {
|
|
status: "ERROR",
|
|
error: "Could not trigger item and could not create pre-failed run",
|
|
workingIndex,
|
|
};
|
|
}
|
|
}
|
|
|
|
runIds.push(runFriendlyId);
|
|
workingIndex++;
|
|
}
|
|
|
|
const updatedBatch = await this._engine.runStore.updateBatchTaskRun({
|
|
where: { id: batch.id },
|
|
data: {
|
|
runIds: {
|
|
push: runIds,
|
|
},
|
|
processingJobsCount: {
|
|
increment: runIds.length,
|
|
},
|
|
},
|
|
select: {
|
|
processingJobsCount: true,
|
|
runCount: true,
|
|
},
|
|
});
|
|
|
|
if (updatedBatch.processingJobsCount >= updatedBatch.runCount) {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRun] All runs created", {
|
|
batchId: batch.friendlyId,
|
|
processingJobsCount: updatedBatch.processingJobsCount,
|
|
runCount: updatedBatch.runCount,
|
|
workingIndex,
|
|
});
|
|
|
|
//if all the runs were idempotent, it's possible the batch is already completed
|
|
await this._engine.tryCompleteBatch({ batchId: batch.id });
|
|
}
|
|
|
|
// if there are more items to process, requeue the batch
|
|
if (workingIndex < batch.runCount) {
|
|
return { status: "INCOMPLETE", workingIndex };
|
|
}
|
|
|
|
return { status: "COMPLETE" };
|
|
}
|
|
|
|
async #processBatchTaskRunItem({
|
|
batch,
|
|
environment,
|
|
item,
|
|
currentIndex,
|
|
options,
|
|
parentRunId,
|
|
resumeParentOnCompletion,
|
|
}: {
|
|
batch: BatchTaskRun;
|
|
environment: AuthenticatedEnvironment;
|
|
item: BatchTriggerTaskV2RequestBody["items"][number];
|
|
currentIndex: number;
|
|
options?: BatchTriggerTaskServiceOptions;
|
|
parentRunId: string | undefined;
|
|
resumeParentOnCompletion: boolean | undefined;
|
|
}) {
|
|
logger.debug("[RunEngineBatchTrigger][processBatchTaskRunItem] Processing item", {
|
|
batchId: batch.friendlyId,
|
|
currentIndex,
|
|
});
|
|
|
|
const triggerTaskService = new TriggerTaskService();
|
|
|
|
// Anchor the item's mint on the BATCH's own friendlyId (not a fresh per-org flag read) so
|
|
// an org's mint flag flipping between batch creation and this (possibly much later,
|
|
// worker-processed) item never splits the item from its BatchTaskRun row. See
|
|
// mintAnchoredRunFriendlyId.server.ts.
|
|
const runFriendlyId = mintAnchoredRunFriendlyId(batch.friendlyId, item.options?.region);
|
|
|
|
const result = await triggerTaskService.call(
|
|
item.task,
|
|
environment,
|
|
{
|
|
...item,
|
|
options: {
|
|
...item.options,
|
|
parentRunId,
|
|
resumeParentOnCompletion,
|
|
parentBatch: batch.id,
|
|
},
|
|
},
|
|
{
|
|
triggerVersion: options?.triggerVersion,
|
|
traceContext: options?.traceContext,
|
|
spanParentAsLink: options?.spanParentAsLink,
|
|
batchId: batch.id,
|
|
batchIndex: currentIndex,
|
|
runFriendlyId,
|
|
realtimeStreamsVersion: options?.realtimeStreamsVersion,
|
|
triggerSource: options?.triggerSource ?? "api",
|
|
triggerAction: options?.triggerAction ?? "trigger",
|
|
},
|
|
"V2"
|
|
);
|
|
|
|
return result
|
|
? {
|
|
friendlyId: result.run.friendlyId,
|
|
}
|
|
: undefined;
|
|
}
|
|
|
|
async #handlePayloadPacket(
|
|
payload: any,
|
|
pathPrefix: string,
|
|
environment: AuthenticatedEnvironment
|
|
) {
|
|
return await startActiveSpan("handlePayloadPacket()", async (span) => {
|
|
const packet = { data: JSON.stringify(payload), dataType: "application/json" };
|
|
|
|
if (!packet.data) {
|
|
return packet;
|
|
}
|
|
|
|
const { needsOffloading } = packetRequiresOffloading(
|
|
packet,
|
|
env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
|
|
);
|
|
|
|
if (!needsOffloading) {
|
|
return packet;
|
|
}
|
|
|
|
const filename = `${pathPrefix}/payload.json`;
|
|
|
|
const uploadedFilename = await uploadPacketToObjectStore(
|
|
filename,
|
|
packet.data,
|
|
packet.dataType,
|
|
environment
|
|
);
|
|
|
|
return {
|
|
data: uploadedFilename,
|
|
dataType: "application/store",
|
|
};
|
|
});
|
|
}
|
|
}
|