chore: import upstream snapshot with attribution

2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
@@ -0,0 +1,186 @@
+import { logger } from "~/services/logger.server";
+import { marqs } from "~/v3/marqs/index.server";
+
+import assertNever from "assert-never";
+import { FailedTaskRunService } from "./failedTaskRun.server";
+import { BaseService } from "./services/baseService.server";
+import type { PrismaClientOrTransaction } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+import { socketIo } from "./handleSocketIo.server";
+import { TaskRunErrorCodes } from "@trigger.dev/core/v3";
+import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { isV3Disabled } from "./engineDeprecation.server";
+
+export class TaskRunHeartbeatFailedService extends BaseService {
+  public async call(runId: string) {
+    const taskRun = await this.runStore.findRun(
+      {
+        id: runId,
+      },
+      {
+        select: {
+          id: true,
+          engine: true,
+          friendlyId: true,
+          status: true,
+          lockedAt: true,
+          runtimeEnvironmentId: true,
+          lockedToVersionId: true,
+          _count: {
+            select: {
+              attempts: true,
+            },
+          },
+        },
+      },
+      this._prisma
+    );
+
+    if (!taskRun) {
+      logger.error("[TaskRunHeartbeatFailedService] Task run not found", {
+        runId,
+      });
+
+      return;
+    }
+
+    // v3 (engine V1) shutdown: leave abandoned V1 runs as-is instead of doing
+    // MarQS/DB work to fail or requeue them. v4 (V2) is unaffected.
+    if (isV3Disabled() && taskRun.engine === "V1") {
+      logger.debug("[TaskRunHeartbeatFailedService] Skipping heartbeat for shut-down v3 run", {
+        runId,
+      });
+      return;
+    }
+
+    const env = await controlPlaneResolver.resolveEnv(taskRun.runtimeEnvironmentId);
+    const lockedWorker = await controlPlaneResolver.resolveRunLockedWorker({
+      lockedToVersionId: taskRun.lockedToVersionId,
+    });
+
+    if (!env) {
+      logger.debug("TaskRunHeartbeatFailedService: environment not found", { runId });
+      return;
+    }
+
+    const service = new FailedTaskRunService();
+
+    switch (taskRun.status) {
+      case "PENDING":
+      case "DEQUEUED":
+      case "WAITING_TO_RESUME":
+      case "PAUSED": {
+        const backInQueue = await marqs?.nackMessage(taskRun.id);
+
+        if (backInQueue) {
+          logger.debug(
+            `[TaskRunHeartbeatFailedService] ${taskRun.status} run is back in the queue run`,
+            {
+              taskRun,
+            }
+          );
+        } else {
+          logger.debug(
+            `[TaskRunHeartbeatFailedService] ${taskRun.status} run not back in the queue, failing`,
+            { taskRun }
+          );
+          await service.call(taskRun.friendlyId, {
+            ok: false,
+            id: taskRun.friendlyId,
+            retry: undefined,
+            error: {
+              type: "INTERNAL_ERROR",
+              code: TaskRunErrorCodes.TASK_RUN_HEARTBEAT_TIMEOUT,
+              message: "Did not receive a heartbeat from the worker in time",
+            },
+          });
+        }
+
+        break;
+      }
+      case "EXECUTING":
+      case "RETRYING_AFTER_FAILURE": {
+        logger.debug(`[RequeueTaskRunService] ${taskRun.status} failing task run`, { taskRun });
+
+        await service.call(taskRun.friendlyId, {
+          ok: false,
+          id: taskRun.friendlyId,
+          retry: undefined,
+          error: {
+            type: "INTERNAL_ERROR",
+            code: TaskRunErrorCodes.TASK_RUN_HEARTBEAT_TIMEOUT,
+            message: "Did not receive a heartbeat from the worker in time",
+          },
+        });
+
+        break;
+      }
+      case "DELAYED":
+      case "PENDING_VERSION":
+      case "WAITING_FOR_DEPLOY": {
+        logger.debug(
+          `[TaskRunHeartbeatFailedService] ${taskRun.status} Removing task run from queue`,
+          { taskRun }
+        );
+
+        await marqs?.acknowledgeMessage(
+          taskRun.id,
+          "Run is either DELAYED or WAITING_FOR_DEPLOY so we cannot requeue it in TaskRunHeartbeatFailedService"
+        );
+
+        break;
+      }
+      case "SYSTEM_FAILURE":
+      case "INTERRUPTED":
+      case "CRASHED":
+      case "COMPLETED_WITH_ERRORS":
+      case "COMPLETED_SUCCESSFULLY":
+      case "EXPIRED":
+      case "TIMED_OUT":
+      case "CANCELED": {
+        logger.debug("[TaskRunHeartbeatFailedService] Task run is completed", { taskRun });
+
+        await marqs?.acknowledgeMessage(
+          taskRun.id,
+          "Task run is already completed in TaskRunHeartbeatFailedService"
+        );
+
+        try {
+          if (env.type === "DEVELOPMENT") {
+            return;
+          }
+
+          // Signal to exit any leftover containers
+          socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
+            version: "v1",
+            runId: taskRun.id,
+            // Give the run a few seconds to exit to complete any flushing etc
+            delayInMs: lockedWorker?.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
+          });
+        } catch (error) {
+          logger.error("[TaskRunHeartbeatFailedService] Error signaling run cancellation", {
+            runId: taskRun.id,
+            error: error instanceof Error ? error.message : error,
+          });
+        }
+
+        break;
+      }
+      default: {
+        assertNever(taskRun.status);
+      }
+    }
+  }
+
+  public static async enqueue(runId: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
+    return await workerQueue.enqueue(
+      "v3.requeueTaskRun",
+      { runId },
+      { runAt, jobKey: `requeueTaskRun:${runId}` }
+    );
+  }
+
+  public static async dequeue(runId: string, tx?: PrismaClientOrTransaction) {
+    return await workerQueue.dequeue(`requeueTaskRun:${runId}`, { tx });
+  }
+}