chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,728 @@
|
||||
import { tryCatch } from "@trigger.dev/core/utils";
|
||||
import type {
|
||||
MachinePresetName,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionResult,
|
||||
TaskRunExecutionRetry,
|
||||
TaskRunFailedExecutionResult,
|
||||
TaskRunSuccessfulExecutionResult,
|
||||
V3TaskRunExecution,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import {
|
||||
TaskRunContext,
|
||||
TaskRunErrorCodes,
|
||||
flattenAttributes,
|
||||
isOOMRunError,
|
||||
sanitizeError,
|
||||
shouldRetryError,
|
||||
taskRunErrorEnhancer,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import type { TaskRun } from "@trigger.dev/database";
|
||||
import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
|
||||
import type { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { env } from "~/env.server";
|
||||
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { createExceptionPropertiesFromError } from "../eventRepository/common.server";
|
||||
import type { FAILED_RUN_STATUSES } from "../taskStatus";
|
||||
import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelAttemptService } from "./cancelAttempt.server";
|
||||
import { CreateCheckpointService } from "./createCheckpoint.server";
|
||||
import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
|
||||
import { RetryAttemptService } from "./retryAttempt.server";
|
||||
import { getEventRepositoryForStore } from "../eventRepository/index.server";
|
||||
|
||||
type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
|
||||
|
||||
type CheckpointData = {
|
||||
docker: boolean;
|
||||
location: string;
|
||||
};
|
||||
|
||||
type CompleteAttemptServiceOptions = {
|
||||
prisma?: PrismaClientOrTransaction;
|
||||
supportsRetryCheckpoints?: boolean;
|
||||
isSystemFailure?: boolean;
|
||||
isCrash?: boolean;
|
||||
};
|
||||
|
||||
export class CompleteAttemptService extends BaseService {
|
||||
constructor(private opts: CompleteAttemptServiceOptions = {}) {
|
||||
super(opts.prisma);
|
||||
}
|
||||
|
||||
public async call({
|
||||
completion,
|
||||
execution,
|
||||
env,
|
||||
checkpoint,
|
||||
}: {
|
||||
completion: TaskRunExecutionResult;
|
||||
execution: V3TaskRunExecution;
|
||||
env?: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
}): Promise<"COMPLETED" | "RETRIED"> {
|
||||
const taskRunAttempt = await findAttempt(this._prisma, execution.attempt.id);
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
logger.error("[CompleteAttemptService] Task run attempt not found", {
|
||||
id: execution.attempt.id,
|
||||
});
|
||||
|
||||
const run = await this.runStore.findRun(
|
||||
{
|
||||
friendlyId: execution.run.id,
|
||||
},
|
||||
{
|
||||
select: {
|
||||
id: true,
|
||||
},
|
||||
},
|
||||
this._prisma
|
||||
);
|
||||
|
||||
if (!run) {
|
||||
logger.error("[CompleteAttemptService] Task run not found", {
|
||||
friendlyId: execution.run.id,
|
||||
});
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: run.id,
|
||||
status: "SYSTEM_FAILURE",
|
||||
completedAt: new Date(),
|
||||
attemptStatus: "FAILED",
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
|
||||
message: "Tried to complete attempt but it doesn't exist",
|
||||
},
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
// No attempt, so there's no message to ACK
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
if (
|
||||
isFinalAttemptStatus(taskRunAttempt.status) ||
|
||||
isFinalRunStatus(taskRunAttempt.taskRun.status)
|
||||
) {
|
||||
// We don't want to retry a task run that has already been marked as failed, cancelled, or completed
|
||||
logger.debug("[CompleteAttemptService] Attempt or run is already in a final state", {
|
||||
taskRunAttempt,
|
||||
completion,
|
||||
});
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
if (completion.ok) {
|
||||
return await this.#completeAttemptSuccessfully(completion, taskRunAttempt, env);
|
||||
} else {
|
||||
return await this.#completeAttemptFailed({
|
||||
completion,
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
env,
|
||||
checkpoint,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async #completeAttemptSuccessfully(
|
||||
completion: TaskRunSuccessfulExecutionResult,
|
||||
taskRunAttempt: NonNullable<FoundAttempt>,
|
||||
env?: AuthenticatedEnvironment
|
||||
): Promise<"COMPLETED"> {
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: { id: taskRunAttempt.id },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
completedAt: new Date(),
|
||||
output: completion.output,
|
||||
outputType: completion.outputType,
|
||||
usageDurationMs: completion.usage?.durationMs,
|
||||
taskRun: {
|
||||
update: {
|
||||
output: completion.output,
|
||||
outputType: completion.outputType,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status: "COMPLETED_SUCCESSFULLY",
|
||||
completedAt: new Date(),
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
const [completeSuccessfulRunEventError] = await tryCatch(
|
||||
eventRepository.completeSuccessfulRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: new Date(),
|
||||
})
|
||||
);
|
||||
|
||||
if (completeSuccessfulRunEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to complete successful run event", {
|
||||
error: completeSuccessfulRunEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
async #completeAttemptFailed({
|
||||
completion,
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
env,
|
||||
checkpoint,
|
||||
}: {
|
||||
completion: TaskRunFailedExecutionResult;
|
||||
execution: V3TaskRunExecution;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
env?: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
}): Promise<"COMPLETED" | "RETRIED"> {
|
||||
if (
|
||||
completion.error.type === "INTERNAL_ERROR" &&
|
||||
completion.error.code === "TASK_RUN_CANCELLED"
|
||||
) {
|
||||
// We need to cancel the task run instead of fail it
|
||||
const cancelService = new CancelAttemptService();
|
||||
|
||||
// TODO: handle usages
|
||||
await cancelService.call(
|
||||
taskRunAttempt.friendlyId,
|
||||
taskRunAttempt.taskRunId,
|
||||
new Date(),
|
||||
"Canceled by user",
|
||||
env
|
||||
);
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
const failedAt = new Date();
|
||||
const sanitizedError = sanitizeError(completion.error);
|
||||
|
||||
await this._prisma.taskRunAttempt.update({
|
||||
where: { id: taskRunAttempt.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
completedAt: failedAt,
|
||||
error: sanitizedError,
|
||||
usageDurationMs: completion.usage?.durationMs,
|
||||
},
|
||||
});
|
||||
|
||||
const environment = env ?? (await this.#getEnvironment(execution.environment.id));
|
||||
|
||||
// This means that tasks won't know they are being retried
|
||||
let executionRetryInferred = false;
|
||||
let executionRetry = completion.retry;
|
||||
|
||||
const shouldInfer = this.opts.isCrash || this.opts.isSystemFailure;
|
||||
|
||||
if (!executionRetry && shouldInfer) {
|
||||
executionRetryInferred = true;
|
||||
executionRetry = FailedTaskRunRetryHelper.getExecutionRetry({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
}
|
||||
|
||||
let retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error));
|
||||
let isOOMRetry = false;
|
||||
let isOOMAttempt = isOOMRunError(completion.error);
|
||||
let isOnMaxOOMMachine = false;
|
||||
let oomMachine: MachinePresetName | undefined;
|
||||
|
||||
//OOM errors should retry (if an OOM machine is specified, and we're not already on it)
|
||||
if (isOOMAttempt) {
|
||||
const retryConfig = FailedTaskRunRetryHelper.getRetryConfig({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
|
||||
oomMachine = retryConfig?.outOfMemory?.machine;
|
||||
isOnMaxOOMMachine = oomMachine === taskRunAttempt.taskRun.machinePreset;
|
||||
|
||||
if (oomMachine && !isOnMaxOOMMachine) {
|
||||
//we will retry
|
||||
isOOMRetry = true;
|
||||
retriableError = true;
|
||||
executionRetry = FailedTaskRunRetryHelper.getExecutionRetry({
|
||||
run: {
|
||||
...taskRunAttempt.taskRun,
|
||||
lockedBy: taskRunAttempt.backgroundWorkerTask,
|
||||
lockedToVersion: taskRunAttempt.backgroundWorker,
|
||||
},
|
||||
execution,
|
||||
});
|
||||
|
||||
//update the machine on the run
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
machinePreset: oomMachine,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
retriableError &&
|
||||
executionRetry !== undefined &&
|
||||
taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS
|
||||
) {
|
||||
return await this.#retryAttempt({
|
||||
execution,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
taskRunAttempt,
|
||||
environment,
|
||||
checkpoint,
|
||||
forceRequeue: isOOMRetry,
|
||||
oomMachine,
|
||||
});
|
||||
}
|
||||
|
||||
// The attempt has failed and we won't retry
|
||||
|
||||
if (isOOMAttempt && isOnMaxOOMMachine && environment.type !== "DEVELOPMENT") {
|
||||
// The attempt failed due to an OOM error but we're already on the machine we should retry on
|
||||
exitRun(taskRunAttempt.taskRunId);
|
||||
}
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
const [completeFailedRunEventError] = await tryCatch(
|
||||
eventRepository.completeFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (completeFailedRunEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to complete failed run event", {
|
||||
error: completeFailedRunEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
error: sanitizedError,
|
||||
},
|
||||
});
|
||||
|
||||
let status: FAILED_RUN_STATUSES;
|
||||
|
||||
// Set the correct task run status
|
||||
if (this.opts.isSystemFailure) {
|
||||
status = "SYSTEM_FAILURE";
|
||||
} else if (this.opts.isCrash) {
|
||||
status = "CRASHED";
|
||||
} else if (
|
||||
sanitizedError.type === "INTERNAL_ERROR" &&
|
||||
sanitizedError.code === "MAX_DURATION_EXCEEDED"
|
||||
) {
|
||||
status = "TIMED_OUT";
|
||||
} else if (sanitizedError.type === "INTERNAL_ERROR") {
|
||||
status = "CRASHED";
|
||||
} else {
|
||||
status = "COMPLETED_WITH_ERRORS";
|
||||
}
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status,
|
||||
completedAt: failedAt,
|
||||
metadata: completion.metadata,
|
||||
env,
|
||||
});
|
||||
|
||||
if (status !== "CRASHED" && status !== "SYSTEM_FAILURE") {
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
// Handle in-progress events
|
||||
switch (status) {
|
||||
case "CRASHED": {
|
||||
const [createAttemptFailedEventError] = await tryCatch(
|
||||
eventRepository.createAttemptFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
attemptNumber: taskRunAttempt.number,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (createAttemptFailedEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
|
||||
error: createAttemptFailedEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case "SYSTEM_FAILURE": {
|
||||
const [createAttemptFailedEventError] = await tryCatch(
|
||||
eventRepository.createAttemptFailedRunEvent({
|
||||
run: taskRunAttempt.taskRun,
|
||||
endTime: failedAt,
|
||||
attemptNumber: taskRunAttempt.number,
|
||||
exception: createExceptionPropertiesFromError(sanitizedError),
|
||||
})
|
||||
);
|
||||
|
||||
if (createAttemptFailedEventError) {
|
||||
logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
|
||||
error: createAttemptFailedEventError,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
async #enqueueReattempt({
|
||||
run,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpointEventId,
|
||||
supportsLazyAttempts,
|
||||
forceRequeue = false,
|
||||
}: {
|
||||
run: TaskRun;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
checkpointEventId?: string;
|
||||
supportsLazyAttempts: boolean;
|
||||
forceRequeue?: boolean;
|
||||
}) {
|
||||
const retryViaQueue = () => {
|
||||
logger.debug("[CompleteAttemptService] Enqueuing retry attempt", { runId: run.id });
|
||||
|
||||
return marqs.requeueMessage(
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
checkpointEventId: this.opts.supportsRetryCheckpoints ? checkpointEventId : undefined,
|
||||
retryCheckpointsDisabled: !this.opts.supportsRetryCheckpoints,
|
||||
},
|
||||
executionRetry.timestamp,
|
||||
"retry"
|
||||
);
|
||||
};
|
||||
|
||||
const retryDirectly = () => {
|
||||
logger.debug("[CompleteAttemptService] Retrying attempt directly", { runId: run.id });
|
||||
return RetryAttemptService.enqueue(run.id, new Date(executionRetry.timestamp));
|
||||
};
|
||||
|
||||
// There's a checkpoint, so we need to go through the queue
|
||||
if (checkpointEventId) {
|
||||
if (!this.opts.supportsRetryCheckpoints) {
|
||||
logger.error(
|
||||
"[CompleteAttemptService] Worker does not support retry checkpoints, but a checkpoint was created",
|
||||
{
|
||||
runId: run.id,
|
||||
checkpointEventId,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
logger.debug("[CompleteAttemptService] Enqueuing retry attempt with checkpoint", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
|
||||
if (!supportsLazyAttempts) {
|
||||
logger.debug("[CompleteAttemptService] Worker does not support lazy attempts", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
if (forceRequeue) {
|
||||
logger.debug("[CompleteAttemptService] Forcing retry via queue", { runId: run.id });
|
||||
|
||||
// The run won't know it should shut down as we make the decision to force requeue here
|
||||
// This also ensures that this change is backwards compatible with older workers
|
||||
exitRun(run.id);
|
||||
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// Workers that never checkpoint between attempts will exit after completing their current attempt if the retry delay exceeds the threshold
|
||||
if (
|
||||
!this.opts.supportsRetryCheckpoints &&
|
||||
executionRetry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS
|
||||
) {
|
||||
logger.debug(
|
||||
"[CompleteAttemptService] Worker does not support retry checkpoints and the delay exceeds the threshold",
|
||||
{ runId: run.id }
|
||||
);
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
if (executionRetryInferred) {
|
||||
logger.debug("[CompleteAttemptService] Execution retry inferred, forcing retry via queue", {
|
||||
runId: run.id,
|
||||
});
|
||||
await retryViaQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
// The worker is still running and waiting for a retry message
|
||||
await retryDirectly();
|
||||
}
|
||||
|
||||
async #retryAttempt({
|
||||
execution,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
taskRunAttempt,
|
||||
environment,
|
||||
checkpoint,
|
||||
forceRequeue = false,
|
||||
oomMachine,
|
||||
}: {
|
||||
execution: V3TaskRunExecution;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
environment: AuthenticatedEnvironment;
|
||||
checkpoint?: CheckpointData;
|
||||
forceRequeue?: boolean;
|
||||
/** Setting this will also alter the retry span message */
|
||||
oomMachine?: MachinePresetName;
|
||||
}) {
|
||||
const retryAt = new Date(executionRetry.timestamp);
|
||||
|
||||
const eventRepository = await getEventRepositoryForStore(
|
||||
taskRunAttempt.taskRun.taskEventStore,
|
||||
taskRunAttempt.taskRun.organizationId ?? ""
|
||||
);
|
||||
|
||||
// Retry the task run
|
||||
await eventRepository.recordEvent(
|
||||
`Retry #${execution.attempt.number} delay${oomMachine ? " after OOM" : ""}`,
|
||||
{
|
||||
taskSlug: taskRunAttempt.taskRun.taskIdentifier,
|
||||
environment,
|
||||
attributes: {
|
||||
metadata: this.#generateMetadataAttributesForNextAttempt(execution),
|
||||
properties: {
|
||||
retryAt: retryAt.toISOString(),
|
||||
previousMachine: oomMachine
|
||||
? (taskRunAttempt.taskRun.machinePreset ?? undefined)
|
||||
: undefined,
|
||||
nextMachine: oomMachine,
|
||||
},
|
||||
runId: taskRunAttempt.taskRun.friendlyId,
|
||||
style: {
|
||||
icon: "schedule-attempt",
|
||||
},
|
||||
},
|
||||
context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
|
||||
spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
|
||||
endTime: retryAt,
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug("[CompleteAttemptService] Retrying", {
|
||||
taskRun: taskRunAttempt.taskRun.friendlyId,
|
||||
retry: executionRetry,
|
||||
});
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRunAttempt.taskRunId,
|
||||
},
|
||||
data: {
|
||||
status: "RETRYING_AFTER_FAILURE",
|
||||
},
|
||||
});
|
||||
|
||||
if (environment.type === "DEVELOPMENT") {
|
||||
await marqs.requeueMessage(taskRunAttempt.taskRunId, {}, executionRetry.timestamp, "retry");
|
||||
|
||||
return "RETRIED";
|
||||
}
|
||||
|
||||
if (checkpoint) {
|
||||
// This is only here for backwards compat - we don't checkpoint between attempts anymore
|
||||
return await this.#retryAttemptWithCheckpoint({
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpoint,
|
||||
});
|
||||
}
|
||||
|
||||
await this.#enqueueReattempt({
|
||||
run: taskRunAttempt.taskRun,
|
||||
executionRetry,
|
||||
supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
|
||||
executionRetryInferred,
|
||||
forceRequeue,
|
||||
});
|
||||
|
||||
return "RETRIED";
|
||||
}
|
||||
|
||||
async #retryAttemptWithCheckpoint({
|
||||
execution,
|
||||
taskRunAttempt,
|
||||
executionRetry,
|
||||
executionRetryInferred,
|
||||
checkpoint,
|
||||
}: {
|
||||
execution: V3TaskRunExecution;
|
||||
taskRunAttempt: NonNullable<FoundAttempt>;
|
||||
executionRetry: TaskRunExecutionRetry;
|
||||
executionRetryInferred: boolean;
|
||||
checkpoint: CheckpointData;
|
||||
}) {
|
||||
const createCheckpoint = new CreateCheckpointService(this._prisma);
|
||||
const checkpointCreateResult = await createCheckpoint.call({
|
||||
attemptFriendlyId: execution.attempt.id,
|
||||
docker: checkpoint.docker,
|
||||
location: checkpoint.location,
|
||||
reason: {
|
||||
type: "RETRYING_AFTER_FAILURE",
|
||||
attemptNumber: execution.attempt.number,
|
||||
},
|
||||
});
|
||||
|
||||
if (!checkpointCreateResult.success) {
|
||||
logger.error("[CompleteAttemptService] Failed to create reattempt checkpoint", {
|
||||
checkpoint,
|
||||
runId: execution.run.id,
|
||||
attemptId: execution.attempt.id,
|
||||
});
|
||||
|
||||
const finalizeService = new FinalizeTaskRunService();
|
||||
await finalizeService.call({
|
||||
id: taskRunAttempt.taskRunId,
|
||||
status: "SYSTEM_FAILURE",
|
||||
completedAt: new Date(),
|
||||
error: {
|
||||
type: "STRING_ERROR",
|
||||
raw: "Failed to create reattempt checkpoint",
|
||||
},
|
||||
});
|
||||
|
||||
return "COMPLETED" as const;
|
||||
}
|
||||
|
||||
await this.#enqueueReattempt({
|
||||
run: taskRunAttempt.taskRun,
|
||||
executionRetry,
|
||||
checkpointEventId: checkpointCreateResult.event.id,
|
||||
supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
|
||||
executionRetryInferred,
|
||||
});
|
||||
|
||||
return "RETRIED" as const;
|
||||
}
|
||||
|
||||
#generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {
|
||||
const context = TaskRunContext.parse(execution);
|
||||
|
||||
// @ts-ignore
|
||||
context.attempt = {
|
||||
number: context.attempt.number + 1,
|
||||
};
|
||||
|
||||
return flattenAttributes(context, "ctx");
|
||||
}
|
||||
|
||||
async #getEnvironment(id: string) {
|
||||
return await this._prisma.runtimeEnvironment.findFirstOrThrow({
|
||||
where: {
|
||||
id,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
organization: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId: string) {
|
||||
return prismaClient.taskRunAttempt.findFirst({
|
||||
where: { friendlyId },
|
||||
include: {
|
||||
taskRun: true,
|
||||
backgroundWorkerTask: true,
|
||||
backgroundWorker: {
|
||||
select: {
|
||||
id: true,
|
||||
supportsLazyAttempts: true,
|
||||
sdkVersion: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function exitRun(runId: string) {
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
|
||||
version: "v1",
|
||||
runId,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user