import { ExternalBuildData, type FinalizeDeploymentRequestBody, } from "@trigger.dev/core/v3/schemas"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { BaseService, ServiceValidationError } from "./baseService.server"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { mkdtemp, writeFile } from "node:fs/promises"; import { env } from "~/env.server"; import { depot as execDepot } from "@depot/cli"; import { FinalizeDeploymentService } from "./finalizeDeployment.server"; import { remoteBuildsEnabled } from "../remoteImageBuilder.server"; import { getEcrAuthToken, isEcrRegistry } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig, type RegistryConfig } from "../registryConfig.server"; import { ComputeTemplateCreationService } from "./computeTemplateCreation.server"; import { ecrImageExists } from "./verifyDeploymentImage.server"; export class FinalizeDeploymentV2Service extends BaseService { public async call( authenticatedEnv: AuthenticatedEnvironment, id: string, body: FinalizeDeploymentRequestBody, writer?: WritableStreamDefaultWriter ) { const deployment = await this._prisma.workerDeployment.findFirst({ where: { friendlyId: id, environmentId: authenticatedEnv.id, }, select: { status: true, id: true, version: true, externalBuildData: true, environment: true, imageReference: true, type: true, worker: { select: { project: true, }, }, }, }); if (!deployment) { logger.error("Worker deployment not found", { id }); return; } if (!deployment.worker) { logger.error("Worker deployment does not have a worker", { id }); throw new ServiceValidationError("Worker deployment does not have a worker"); } if (deployment.status === "DEPLOYED") { logger.debug("Worker deployment is already deployed", { id }); return deployment; } if (deployment.status !== "DEPLOYING") { logger.error("Worker deployment is not in DEPLOYING status", { id }); throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); } const finalizeService = new FinalizeDeploymentService(); // If remote builds are not enabled, skip image push and go straight to template + finalize if (!remoteBuildsEnabled() || body.skipPushToRegistry) { if (body.skipPushToRegistry) { logger.debug("Skipping push to registry during deployment finalization", { deployment, }); } // The CLI claims the image is already in the registry (local build, or a // self-hosted setup). Verify before promoting so we never mark a // deployment DEPLOYED when nothing was actually pushed. await this.#assertImagePullable(deployment, body); await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer); return finalizeService.call(authenticatedEnv, id, body); } const externalBuildData = deployment.externalBuildData ? ExternalBuildData.safeParse(deployment.externalBuildData) : undefined; if (!externalBuildData) { throw new ServiceValidationError("External build data is missing"); } if (!externalBuildData.success) { throw new ServiceValidationError("External build data is invalid"); } const isV4Deployment = deployment.type === "MANAGED"; const registryConfig = getRegistryConfig(isV4Deployment); // For non-ECR registries, username and password are required upfront if ( !isEcrRegistry(registryConfig.host) && (!registryConfig.username || !registryConfig.password) ) { throw new ServiceValidationError("Missing deployment registry credentials"); } if (!env.DEPOT_TOKEN) { throw new ServiceValidationError("Missing depot token"); } // All new deployments will set the image reference at creation time if (!deployment.imageReference) { throw new ServiceValidationError("Missing image reference"); } logger.debug("Pushing image to registry", { id, deployment, body }); const pushResult = await executePushToRegistry( { depot: { buildId: externalBuildData.data.buildId, orgToken: env.DEPOT_TOKEN, projectId: externalBuildData.data.projectId, }, registry: registryConfig, deployment: { version: deployment.version, environmentSlug: deployment.environment.slug, projectExternalRef: deployment.worker.project.externalRef, imageReference: deployment.imageReference, }, }, writer ); if (!pushResult.ok) { throw new ServiceValidationError(pushResult.error); } logger.debug("Image pushed to registry", { id, deployment, body, pushedImage: pushResult.image, }); // Belt and suspenders: confirm the push actually landed before promoting. await this.#assertImagePullable(deployment, body); await this.#createTemplateIfNeeded(deployment, id, authenticatedEnv, writer); return finalizeService.call(authenticatedEnv, id, body); } async #assertImagePullable( deployment: { imageReference: string | null; type: string | null }, body: FinalizeDeploymentRequestBody ): Promise { if (!env.DEPLOY_IMAGE_VERIFICATION_ENABLED) { return; } if (!deployment.imageReference) { return; } const registryConfig = getRegistryConfig(deployment.type === "MANAGED"); // ECR-only: non-ECR (self-hosted) registries can't be checked this way, so skip. if (!isEcrRegistry(registryConfig.host)) { return; } const result = await ecrImageExists({ imageReference: deployment.imageReference, imageDigest: body.imageDigest, registryConfig, }); if (result === "missing") { throw new ServiceValidationError( "Deployment image was not found in the registry. It may not have been pushed (for example a local build without a push, or a push to a different registry). Aborting the deploy to avoid promoting a version that cannot start." ); } if (result === "nonconformant") { throw new ServiceValidationError( "Deployment image is not runnable: it contains zstd-compressed layers inside a Docker (v2s2) manifest, which the container runtime cannot pull. This typically comes from an outdated CLI version. Please upgrade to the latest trigger.dev CLI and re-deploy." ); } // Fail closed: if we can't confirm the image is present, don't promote a version // that might not start. Set DEPLOY_IMAGE_VERIFICATION_ENABLED=0 for out-of-band pushes. if (result === "unknown") { throw new ServiceValidationError( "Could not verify the deployment image exists in the registry. Aborting the deploy." ); } } async #createTemplateIfNeeded( deployment: { imageReference: string | null; worker: { project: { id: string } } | null }, deploymentFriendlyId: string, authenticatedEnv: AuthenticatedEnvironment, writer?: WritableStreamDefaultWriter ): Promise { if (!deployment.imageReference || !deployment.worker) { return; } const templateService = new ComputeTemplateCreationService(); await templateService.handleDeployTemplate({ projectId: deployment.worker.project.id, imageReference: deployment.imageReference, deploymentFriendlyId, authenticatedEnv, prisma: this._prisma, writer, }); } } type ExecutePushToRegistryOptions = { depot: { buildId: string; orgToken: string; projectId: string; }; registry: RegistryConfig; deployment: { version: string; environmentSlug: string; projectExternalRef: string; imageReference: string; }; }; type ExecutePushResult = | { ok: true; image: string; logs: string; } | { ok: false; error: string; logs: string; }; async function executePushToRegistry( { depot, registry, deployment }: ExecutePushToRegistryOptions, writer?: WritableStreamDefaultWriter ): Promise { // Step 1: We need to "login" to the registry const [loginError, configDir] = await tryCatch(ensureLoggedIntoDockerRegistry(registry)); if (loginError) { logger.error("Failed to login to registry", { deployment, registryHost: registry.host, error: loginError.message, }); return { ok: false as const, error: "Failed to login to registry", logs: "", }; } const imageTag = deployment.imageReference; // Step 2: We need to run the depot push command const childProcess = execDepot(["push", depot.buildId, "-t", imageTag, "--progress", "plain"], { env: { NODE_ENV: process.env.NODE_ENV, DEPOT_TOKEN: depot.orgToken, DEPOT_PROJECT_ID: depot.projectId, DEPOT_NO_SUMMARY_LINK: "1", DEPOT_NO_UPDATE_NOTIFIER: "1", DOCKER_CONFIG: configDir, }, }); const errors: string[] = []; try { const processCode = await new Promise((res, rej) => { // For some reason everything is output on stderr, not stdout childProcess.stderr?.on("data", async (data: Buffer) => { const text = data.toString(); // Emitted data chunks can contain multiple lines. Remove empty lines. const lines = text.split("\n").filter(Boolean); errors.push(...lines); logger.debug(text, { deployment }); // Now we can write strings directly if (writer) { for (const line of lines) { await writer.write(`event: log\ndata: ${JSON.stringify({ message: line })}\n\n`); } } }); childProcess.on("error", (e) => rej(e)); childProcess.on("close", (code) => res(code)); }); const logs = extractLogs(errors); if (processCode !== 0) { return { ok: false as const, error: `Error pushing image`, logs, }; } return { ok: true as const, image: imageTag, logs, }; } catch (e) { return { ok: false as const, error: e instanceof Error ? e.message : JSON.stringify(e), logs: extractLogs(errors), }; } } async function ensureLoggedIntoDockerRegistry(registryConfig: RegistryConfig) { const tmpDir = await createTempDir(); const dockerConfigPath = join(tmpDir, "config.json"); let auth: { username: string; password: string }; // If this is an ECR registry, get fresh credentials if (isEcrRegistry(registryConfig.host)) { auth = await getEcrAuthToken({ registryHost: registryConfig.host, assumeRole: registryConfig.ecrAssumeRoleArn ? { roleArn: registryConfig.ecrAssumeRoleArn, externalId: registryConfig.ecrAssumeRoleExternalId, } : undefined, }); } else if (!registryConfig.username || !registryConfig.password) { throw new Error("Authentication required for non-ECR registry"); } else { auth = { username: registryConfig.username, password: registryConfig.password, }; } await writeJSONFile(dockerConfigPath, { auths: { [registryConfig.host]: { auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"), }, }, }); logger.debug(`Writing docker config to ${dockerConfigPath}`); return tmpDir; } // Create a temporary directory within the OS's temp directory async function createTempDir(): Promise { // Generate a unique temp directory path const tempDirPath: string = join(tmpdir(), "trigger-"); // Create the temp directory synchronously and return the path const directory = await mkdtemp(tempDirPath); return directory; } async function writeJSONFile(path: string, json: any, pretty = false) { await writeFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined), "utf8"); } function extractLogs(outputs: string[]) { // Remove empty lines const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== ""); return cleanedOutputs.map((line) => line.trim()).join("\n"); }