triggerdotdev--trigger.dev/apps/webapp/app/v3/services/finalizeDeploymentV2.server.ts

import {
  ExternalBuildData,
  type FinalizeDeploymentRequestBody,
} from "@trigger.dev/core/v3/schemas";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { mkdtemp, writeFile } from "node:fs/promises";
import { env } from "~/env.server";
import { depot as execDepot } from "@depot/cli";
import { FinalizeDeploymentService } from "./finalizeDeployment.server";
import { remoteBuildsEnabled } from "../remoteImageBuilder.server";
import { getEcrAuthToken, isEcrRegistry } from "../getDeploymentImageRef.server";
import { tryCatch } from "@trigger.dev/core";
import { getRegistryConfig, type RegistryConfig } from "../registryConfig.server";
import { ComputeTemplateCreationService } from "./computeTemplateCreation.server";
import { ecrImageExists } from "./verifyDeploymentImage.server";

export class FinalizeDeploymentV2Service extends BaseService {
  public async call(
    authenticatedEnv: AuthenticatedEnvironment,
    id: string,
    body: FinalizeDeploymentRequestBody,
    writer?: WritableStreamDefaultWriter
  ) {
    const deployment = await this._prisma.workerDeployment.findFirst({
      where: {
        friendlyId: id,
        environmentId: authenticatedEnv.id,
      },
      select: {
        status: true,
        id: true,
        version: true,
        externalBuildData: true,
        environment: true,
        imageReference: true,
        type: true,
        worker: {
          select: {
            project: true,
          },
        },
      },
    });

    if (!deployment) {
      logger.error("Worker deployment not found", { id });
      return;
    }

    if (!deployment.worker) {
      logger.error("Worker deployment does not have a worker", { id });
      throw new ServiceValidationError("Worker deployment does not have a worker");
    }

    if (deployment.status === "DEPLOYED") {
      logger.debug("Worker deployment is already deployed", { id });
      return deployment;
    }

    if (deployment.status !== "DEPLOYING") {
      logger.error("Worker deployment is not in DEPLOYING status", { id });
      throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
    }

    const finalizeService = new FinalizeDeploymentService();

    // If remote builds are not enabled, skip image push and go straight to template + finalize
    if (!remoteBuildsEnabled() || body.skipPushToRegistry) {
      if (body.skipPushToRegistry) {
        logger.debug("Skipping push to registry during deployment finalization", {
          deployment,
        });
      }

      // The CLI claims the image is already in the registry (local build, or a
      // self-hosted setup). Verify before promoting so we never mark a
      // deployment DEPLOYED when nothing was actually pushed.
      await this.#assertImagePullable(deployment, body);

      await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer);
      return finalizeService.call(authenticatedEnv, id, body);
    }

    const externalBuildData = deployment.externalBuildData
      ? ExternalBuildData.safeParse(deployment.externalBuildData)
      : undefined;

    if (!externalBuildData) {
      throw new ServiceValidationError("External build data is missing");
    }

    if (!externalBuildData.success) {
      throw new ServiceValidationError("External build data is invalid");
    }

    const isV4Deployment = deployment.type === "MANAGED";
    const registryConfig = getRegistryConfig(isV4Deployment);

    // For non-ECR registries, username and password are required upfront
    if (
      !isEcrRegistry(registryConfig.host) &&
      (!registryConfig.username || !registryConfig.password)
    ) {
      throw new ServiceValidationError("Missing deployment registry credentials");
    }

    if (!env.DEPOT_TOKEN) {
      throw new ServiceValidationError("Missing depot token");
    }

    // All new deployments will set the image reference at creation time
    if (!deployment.imageReference) {
      throw new ServiceValidationError("Missing image reference");
    }

    logger.debug("Pushing image to registry", { id, deployment, body });

    const pushResult = await executePushToRegistry(
      {
        depot: {
          buildId: externalBuildData.data.buildId,
          orgToken: env.DEPOT_TOKEN,
          projectId: externalBuildData.data.projectId,
        },
        registry: registryConfig,
        deployment: {
          version: deployment.version,
          environmentSlug: deployment.environment.slug,
          projectExternalRef: deployment.worker.project.externalRef,
          imageReference: deployment.imageReference,
        },
      },
      writer
    );

    if (!pushResult.ok) {
      throw new ServiceValidationError(pushResult.error);
    }

    logger.debug("Image pushed to registry", {
      id,
      deployment,
      body,
      pushedImage: pushResult.image,
    });

    // Belt and suspenders: confirm the push actually landed before promoting.
    await this.#assertImagePullable(deployment, body);

    await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer);
    return finalizeService.call(authenticatedEnv, id, body);
  }

  async #assertImagePullable(
    deployment: { imageReference: string | null; type: string | null },
    body: FinalizeDeploymentRequestBody
  ): Promise<void> {
    if (!env.DEPLOY_IMAGE_VERIFICATION_ENABLED) {
      return;
    }

    if (!deployment.imageReference) {
      return;
    }

    const registryConfig = getRegistryConfig(deployment.type === "MANAGED");

    // ECR-only: non-ECR (self-hosted) registries can't be checked this way, so skip.
    if (!isEcrRegistry(registryConfig.host)) {
      return;
    }

    const result = await ecrImageExists({
      imageReference: deployment.imageReference,
      imageDigest: body.imageDigest,
      registryConfig,
    });

    if (result === "missing") {
      throw new ServiceValidationError(
        "Deployment image was not found in the registry. It may not have been pushed (for example a local build without a push, or a push to a different registry). Aborting the deploy to avoid promoting a version that cannot start."
      );
    }

    if (result === "nonconformant") {
      throw new ServiceValidationError(
        "Deployment image is not runnable: it contains zstd-compressed layers inside a Docker (v2s2) manifest, which the container runtime cannot pull. This typically comes from an outdated CLI version. Please upgrade to the latest trigger.dev CLI and re-deploy."
      );
    }

    // Fail closed: if we can't confirm the image is present, don't promote a version
    // that might not start. Set DEPLOY_IMAGE_VERIFICATION_ENABLED=0 for out-of-band pushes.
    if (result === "unknown") {
      throw new ServiceValidationError(
        "Could not verify the deployment image exists in the registry. Aborting the deploy."
      );
    }
  }

  async #createTemplateIfNeeded(
    deployment: { imageReference: string | null; worker: { project: { id: string } } | null },
    deploymentFriendlyId: string,
    authenticatedEnv: AuthenticatedEnvironment,
    writer?: WritableStreamDefaultWriter
  ): Promise<void> {
    if (!deployment.imageReference || !deployment.worker) {
      return;
    }

    const templateService = new ComputeTemplateCreationService();
    await templateService.handleDeployTemplate({
      projectId: deployment.worker.project.id,
      imageReference: deployment.imageReference,
      deploymentFriendlyId,
      authenticatedEnv,
      prisma: this._prisma,
      writer,
    });
  }
}

type ExecutePushToRegistryOptions = {
  depot: {
    buildId: string;
    orgToken: string;
    projectId: string;
  };
  registry: RegistryConfig;
  deployment: {
    version: string;
    environmentSlug: string;
    projectExternalRef: string;
    imageReference: string;
  };
};

type ExecutePushResult =
  | {
      ok: true;
      image: string;
      logs: string;
    }
  | {
      ok: false;
      error: string;
      logs: string;
    };

async function executePushToRegistry(
  { depot, registry, deployment }: ExecutePushToRegistryOptions,
  writer?: WritableStreamDefaultWriter
): Promise<ExecutePushResult> {
  // Step 1: We need to "login" to the registry
  const [loginError, configDir] = await tryCatch(ensureLoggedIntoDockerRegistry(registry));

  if (loginError) {
    logger.error("Failed to login to registry", {
      deployment,
      registryHost: registry.host,
      error: loginError.message,
    });

    return {
      ok: false as const,
      error: "Failed to login to registry",
      logs: "",
    };
  }

  const imageTag = deployment.imageReference;

  // Step 2: We need to run the depot push command
  const childProcess = execDepot(["push", depot.buildId, "-t", imageTag, "--progress", "plain"], {
    env: {
      NODE_ENV: process.env.NODE_ENV,
      DEPOT_TOKEN: depot.orgToken,
      DEPOT_PROJECT_ID: depot.projectId,
      DEPOT_NO_SUMMARY_LINK: "1",
      DEPOT_NO_UPDATE_NOTIFIER: "1",
      DOCKER_CONFIG: configDir,
    },
  });

  const errors: string[] = [];

  try {
    const processCode = await new Promise<number | null>((res, rej) => {
      // For some reason everything is output on stderr, not stdout
      childProcess.stderr?.on("data", async (data: Buffer) => {
        const text = data.toString();

        // Emitted data chunks can contain multiple lines. Remove empty lines.
        const lines = text.split("\n").filter(Boolean);

        errors.push(...lines);
        logger.debug(text, { deployment });

        // Now we can write strings directly
        if (writer) {
          for (const line of lines) {
            await writer.write(`event: log\ndata: ${JSON.stringify({ message: line })}\n\n`);
          }
        }
      });

      childProcess.on("error", (e) => rej(e));
      childProcess.on("close", (code) => res(code));
    });

    const logs = extractLogs(errors);

    if (processCode !== 0) {
      return {
        ok: false as const,
        error: `Error pushing image`,
        logs,
      };
    }

    return {
      ok: true as const,
      image: imageTag,
      logs,
    };
  } catch (e) {
    return {
      ok: false as const,
      error: e instanceof Error ? e.message : JSON.stringify(e),
      logs: extractLogs(errors),
    };
  }
}

async function ensureLoggedIntoDockerRegistry(registryConfig: RegistryConfig) {
  const tmpDir = await createTempDir();
  const dockerConfigPath = join(tmpDir, "config.json");

  let auth: { username: string; password: string };

  // If this is an ECR registry, get fresh credentials
  if (isEcrRegistry(registryConfig.host)) {
    auth = await getEcrAuthToken({
      registryHost: registryConfig.host,
      assumeRole: registryConfig.ecrAssumeRoleArn
        ? {
            roleArn: registryConfig.ecrAssumeRoleArn,
            externalId: registryConfig.ecrAssumeRoleExternalId,
          }
        : undefined,
    });
  } else if (!registryConfig.username || !registryConfig.password) {
    throw new Error("Authentication required for non-ECR registry");
  } else {
    auth = {
      username: registryConfig.username,
      password: registryConfig.password,
    };
  }

  await writeJSONFile(dockerConfigPath, {
    auths: {
      [registryConfig.host]: {
        auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
      },
    },
  });

  logger.debug(`Writing docker config to ${dockerConfigPath}`);

  return tmpDir;
}

// Create a temporary directory within the OS's temp directory
async function createTempDir(): Promise<string> {
  // Generate a unique temp directory path
  const tempDirPath: string = join(tmpdir(), "trigger-");

  // Create the temp directory synchronously and return the path
  const directory = await mkdtemp(tempDirPath);

  return directory;
}

async function writeJSONFile(path: string, json: any, pretty = false) {
  await writeFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined), "utf8");
}

function extractLogs(outputs: string[]) {
  // Remove empty lines
  const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== "");

  return cleanedOutputs.map((line) => line.trim()).join("\n");
}