chore: import upstream snapshot with attribution
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* In-memory `@google-cloud/storage` stand-in for the adapter's unit tests.
|
||||
*
|
||||
* Mimics just the surface the transport + deploySite use:
|
||||
* storage.bucket(name).file(key) → { createReadStream, exists, getMetadata }
|
||||
* storage.bucket(name).upload(localPath, { destination, contentType })
|
||||
*
|
||||
* Objects live in a Map keyed `bucket/key`. `upload` reads the real local
|
||||
* file from disk (the handler writes real tarballs), so round-trips through
|
||||
* tar/untar exercise the actual code path. Every op is recorded for
|
||||
* sequence assertions.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import type { Storage } from "@google-cloud/storage";
|
||||
|
||||
export interface FakeGcsOp {
|
||||
kind: "download" | "upload" | "exists" | "getMetadata";
|
||||
uri: string;
|
||||
bytes?: number;
|
||||
}
|
||||
|
||||
export class FakeGcs {
|
||||
ops: FakeGcsOp[] = [];
|
||||
objects = new Map<string, Buffer>();
|
||||
|
||||
// Accessed only through the `Storage` cast in tests, so fallow's static
|
||||
// analysis can't see the reference.
|
||||
// fallow-ignore-next-line unused-class-member
|
||||
bucket(bucketName: string): FakeBucket {
|
||||
return new FakeBucket(this, bucketName);
|
||||
}
|
||||
|
||||
/** Seed an object directly (e.g. a pre-built project tarball). */
|
||||
seed(uri: string, bytes: Buffer): void {
|
||||
this.objects.set(uri, bytes);
|
||||
}
|
||||
|
||||
/** Seed from a local file on disk. */
|
||||
seedFromFile(uri: string, localPath: string): void {
|
||||
this.objects.set(uri, readFileSync(localPath));
|
||||
}
|
||||
}
|
||||
|
||||
class FakeBucket {
|
||||
constructor(
|
||||
private readonly gcs: FakeGcs,
|
||||
private readonly bucketName: string,
|
||||
) {}
|
||||
|
||||
get name(): string {
|
||||
return this.bucketName;
|
||||
}
|
||||
|
||||
file(key: string): FakeFile {
|
||||
return new FakeFile(this.gcs, this.bucketName, key);
|
||||
}
|
||||
|
||||
async upload(
|
||||
localPath: string,
|
||||
opts: { destination: string; contentType?: string },
|
||||
): Promise<unknown> {
|
||||
const uri = `gs://${this.bucketName}/${opts.destination}`;
|
||||
const bytes = readFileSync(localPath);
|
||||
this.gcs.objects.set(uri, bytes);
|
||||
this.gcs.ops.push({ kind: "upload", uri, bytes: bytes.length });
|
||||
return [{}];
|
||||
}
|
||||
}
|
||||
|
||||
class FakeFile {
|
||||
private readonly uri: string;
|
||||
constructor(
|
||||
private readonly gcs: FakeGcs,
|
||||
bucketName: string,
|
||||
key: string,
|
||||
) {
|
||||
this.uri = `gs://${bucketName}/${key}`;
|
||||
}
|
||||
|
||||
createReadStream(): Readable {
|
||||
const bytes = this.gcs.objects.get(this.uri);
|
||||
if (!bytes) {
|
||||
const r = new Readable({ read() {} });
|
||||
r.destroy(new Error(`FakeGcs: object not found: ${this.uri}`));
|
||||
return r;
|
||||
}
|
||||
this.gcs.ops.push({ kind: "download", uri: this.uri, bytes: bytes.length });
|
||||
return Readable.from([bytes]);
|
||||
}
|
||||
|
||||
async exists(): Promise<[boolean]> {
|
||||
const has = this.gcs.objects.has(this.uri);
|
||||
this.gcs.ops.push({ kind: "exists", uri: this.uri });
|
||||
return [has];
|
||||
}
|
||||
|
||||
async getMetadata(): Promise<[{ size?: string | number; updated?: string }]> {
|
||||
const bytes = this.gcs.objects.get(this.uri);
|
||||
this.gcs.ops.push({ kind: "getMetadata", uri: this.uri });
|
||||
return [{ size: bytes?.length ?? 0, updated: "2026-06-06T00:00:00.000Z" }];
|
||||
}
|
||||
|
||||
/** Helper for tests that want to materialize an object to disk. */
|
||||
writeToDisk(destPath: string): void {
|
||||
const bytes = this.gcs.objects.get(this.uri);
|
||||
if (!bytes) throw new Error(`FakeGcs: object not found: ${this.uri}`);
|
||||
writeFileSync(destPath, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/** Cast so `deploySite({ storage: asStorage(fake) })` reads cleanly. */
|
||||
export function asStorage(fake: FakeGcs): Storage {
|
||||
return fake as unknown as Storage;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Cloud Run Chrome resolver.
|
||||
*
|
||||
* `renderChunk()` (the only primitive that needs a browser) launches Chrome
|
||||
* via the engine's `BrowserManager`. Because Cloud Run runs a container
|
||||
* image rather than a size-capped ZIP, the Chrome story is far simpler than
|
||||
* the Lambda adapter's: the `Dockerfile` installs `chrome-headless-shell`
|
||||
* (the same BeginFrame-capable build the K8s deploy uses) into the image at
|
||||
* a known path and exports `HYPERFRAMES_CHROME_PATH`. There is no runtime
|
||||
* decompression-into-/tmp step and no 250 MB packaging ceiling to fight.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. `PRODUCER_HEADLESS_SHELL_PATH` — the engine's own override. If a
|
||||
* caller (or the Docker image) already set it, honour it untouched.
|
||||
* 2. `HYPERFRAMES_CHROME_PATH` — set by the Dockerfile to the installed
|
||||
* `chrome-headless-shell` binary.
|
||||
* 3. A small list of conventional install paths, as a last resort for
|
||||
* images built outside our Dockerfile.
|
||||
*
|
||||
* Throws {@link ChromeBinaryUnavailableError} when nothing resolves, so a
|
||||
* misconfigured image fails loudly at the first chunk rather than emitting
|
||||
* a confusing puppeteer-core "executablePath must be specified" assertion.
|
||||
*/
|
||||
|
||||
import { existsSync } from "node:fs";
|
||||
|
||||
/**
|
||||
* Thrown when the Chrome binary resolver can't produce a usable path. The
|
||||
* class name is the workflow's non-retryable error discriminator.
|
||||
*/
|
||||
export class ChromeBinaryUnavailableError extends Error {
|
||||
// Read indirectly via the error envelope / Error.prototype.toString.
|
||||
// fallow-ignore-next-line unused-class-member
|
||||
override readonly name = "ChromeBinaryUnavailableError";
|
||||
readonly resolvedPath: string | null;
|
||||
constructor(resolvedPath: string | null, hint: string) {
|
||||
super(`[chromium] Chrome binary unavailable: ${hint}`);
|
||||
this.resolvedPath = resolvedPath;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Conventional locations a `chrome-headless-shell` (or full Chrome) binary
|
||||
* may live at in a Debian/Ubuntu-based container. Checked only after the
|
||||
* two env-var overrides miss.
|
||||
*/
|
||||
const FALLBACK_CHROME_PATHS = [
|
||||
"/opt/chrome/chrome-headless-shell",
|
||||
"/usr/bin/chrome-headless-shell",
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
];
|
||||
|
||||
/**
|
||||
* Resolve the absolute path to a Chrome binary suitable for BeginFrame.
|
||||
* Pure (no env mutation) so callers decide whether to export the result
|
||||
* into `PRODUCER_HEADLESS_SHELL_PATH`.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export function resolveChromeExecutablePath(): string {
|
||||
const fromEngineOverride = process.env.PRODUCER_HEADLESS_SHELL_PATH?.trim();
|
||||
if (fromEngineOverride) {
|
||||
if (!existsSync(fromEngineOverride)) {
|
||||
throw new ChromeBinaryUnavailableError(
|
||||
fromEngineOverride,
|
||||
`PRODUCER_HEADLESS_SHELL_PATH=${JSON.stringify(fromEngineOverride)} does not exist on disk.`,
|
||||
);
|
||||
}
|
||||
return fromEngineOverride;
|
||||
}
|
||||
|
||||
const fromImage = process.env.HYPERFRAMES_CHROME_PATH?.trim();
|
||||
if (fromImage) {
|
||||
if (!existsSync(fromImage)) {
|
||||
throw new ChromeBinaryUnavailableError(
|
||||
fromImage,
|
||||
`HYPERFRAMES_CHROME_PATH=${JSON.stringify(fromImage)} does not exist on disk.`,
|
||||
);
|
||||
}
|
||||
return fromImage;
|
||||
}
|
||||
|
||||
for (const candidate of FALLBACK_CHROME_PATHS) {
|
||||
if (existsSync(candidate)) return candidate;
|
||||
}
|
||||
|
||||
throw new ChromeBinaryUnavailableError(
|
||||
null,
|
||||
"no Chrome binary found. Set HYPERFRAMES_CHROME_PATH (the Dockerfile does this) or " +
|
||||
"PRODUCER_HEADLESS_SHELL_PATH to the absolute path of a chrome-headless-shell binary. " +
|
||||
`Searched: ${FALLBACK_CHROME_PATHS.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Request + result types for the HyperFrames distributed render handler
|
||||
* running on Cloud Run.
|
||||
*
|
||||
* The Cloud Workflows definition in `packages/gcp-cloud-run/terraform/workflow.yaml`
|
||||
* dispatches on the `Action` field of the JSON request body. Each action
|
||||
* maps 1:1 onto one of the three OSS distributed primitives:
|
||||
*
|
||||
* "plan" → `plan(projectDir, config, planDir)` (Activity A)
|
||||
* "renderChunk" → `renderChunk(planDir, chunkIndex, output)` (Activity B)
|
||||
* "assemble" → `assemble(planDir, chunkPaths, audio, out)` (Activity C)
|
||||
*
|
||||
* All file I/O is mediated by GCS — the handler downloads inputs into a
|
||||
* per-request workdir under the container's writable `/tmp`, invokes the
|
||||
* primitive, uploads outputs back to GCS, and returns a small JSON payload
|
||||
* that fits inside a Cloud Workflows step variable (Workflows caps a single
|
||||
* step's memory; chunk results stay well under 1 KB so the orchestration
|
||||
* can hold one per Map iteration).
|
||||
*
|
||||
* These shapes are intentionally identical to `@hyperframes/aws-lambda`'s
|
||||
* `events.ts` apart from the URI scheme (`gs://` vs `s3://`): the wire
|
||||
* contract is the adapter's, the primitives underneath are shared.
|
||||
*/
|
||||
|
||||
import type {
|
||||
DistributedFormat,
|
||||
SerializableDistributedRenderConfig,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
|
||||
export type { SerializableDistributedRenderConfig } from "@hyperframes/producer/distributed";
|
||||
|
||||
/** Discriminator for the three roles the one Cloud Run image fulfills. */
|
||||
export type CloudRunAction = "plan" | "renderChunk" | "assemble";
|
||||
|
||||
/**
|
||||
* Top-level shape of any request body the handler may receive.
|
||||
*
|
||||
* Cloud Workflows passes the step's `body` through verbatim, but a caller
|
||||
* driving the service directly (or a Workflows definition that wraps the
|
||||
* payload) may nest it under `Payload` / `Input`; the handler unwraps both
|
||||
* before dispatching, matching the Lambda adapter's envelope tolerance.
|
||||
*/
|
||||
export type CloudRunEvent =
|
||||
| PlanEvent
|
||||
| RenderChunkEvent
|
||||
| AssembleEvent
|
||||
| { Payload: CloudRunEvent }
|
||||
| { Input: CloudRunEvent };
|
||||
|
||||
/** Activity A: produce a planDir, upload to GCS. */
|
||||
export interface PlanEvent {
|
||||
Action: "plan";
|
||||
/** GCS URI pointing at a `tar -czf`-archived project directory (`gs://bucket/key.tar.gz`). */
|
||||
ProjectGcsUri: string;
|
||||
/** GCS URI prefix where the planDir tar should be uploaded (`gs://bucket/{prefix}/`). */
|
||||
PlanOutputGcsPrefix: string;
|
||||
/** `DistributedRenderConfig` minus runtime-only fields (logger, abortSignal). */
|
||||
Config: SerializableDistributedRenderConfig;
|
||||
}
|
||||
|
||||
/** Activity B: fetch planDir, render one chunk, upload result. */
|
||||
export interface RenderChunkEvent {
|
||||
Action: "renderChunk";
|
||||
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
|
||||
PlanGcsUri: string;
|
||||
/**
|
||||
* `PlanResult.planHash` from the Plan invocation. The handler verifies
|
||||
* this against the untarred planDir's `plan.json` before invoking the
|
||||
* producer, throwing a typed `PLAN_HASH_MISMATCH` on divergence so the
|
||||
* workflow routes it as non-retryable. Defense-in-depth — the producer
|
||||
* also re-checks internally.
|
||||
*/
|
||||
PlanHash: string;
|
||||
/** 0-based chunk index this invocation should render. */
|
||||
ChunkIndex: number;
|
||||
/** GCS URI prefix where the chunk output should be uploaded (`gs://bucket/{prefix}/`). */
|
||||
ChunkOutputGcsPrefix: string;
|
||||
/** Output container format from the plan's encoder.json; drives file vs frame-dir handling. */
|
||||
Format: DistributedFormat;
|
||||
}
|
||||
|
||||
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
|
||||
export interface AssembleEvent {
|
||||
Action: "assemble";
|
||||
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
|
||||
PlanGcsUri: string;
|
||||
/** GCS URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
|
||||
ChunkGcsUris: string[];
|
||||
/** GCS URI of the planDir's `audio.aac` if the composition has audio; `null` otherwise. */
|
||||
AudioGcsUri: string | null;
|
||||
/** Final output GCS URI (`gs://bucket/key.mp4`). */
|
||||
OutputGcsUri: string;
|
||||
/** Output container format; drives file vs frame-dir handling. */
|
||||
Format: DistributedFormat;
|
||||
/**
|
||||
* Optional exact-CFR re-encode at assemble time. When `true`, the final
|
||||
* assembled video is re-encoded with `-fps_mode cfr -r <fps>` so the
|
||||
* stream's `avg_frame_rate` matches the container's `r_frame_rate`
|
||||
* exactly (and the file's duration is exact, not PTS-derived). Trade-off
|
||||
* is ~2-5x the assemble wall-clock. mp4 only — webm / mov stream-copy
|
||||
* paths already produce exact avg_frame_rate. Default `false` /
|
||||
* unset preserves current `-c copy` behavior.
|
||||
*/
|
||||
Cfr?: boolean;
|
||||
}
|
||||
|
||||
// ── Result types — kept small to fit Cloud Workflows step budgets ────────────
|
||||
|
||||
/** Result of a `plan` invocation. Carries enough to size the Map(N) state. */
|
||||
export interface PlanResultBody {
|
||||
Action: "plan";
|
||||
PlanGcsUri: string;
|
||||
PlanHash: string;
|
||||
ChunkCount: number;
|
||||
TotalFrames: number;
|
||||
Fps: 24 | 30 | 60;
|
||||
Width: number;
|
||||
Height: number;
|
||||
Format: DistributedFormat;
|
||||
HasAudio: boolean;
|
||||
AudioGcsUri: string | null;
|
||||
FfmpegVersion: string;
|
||||
ProducerVersion: string;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
/** Result of a `renderChunk` invocation. Sized ≤200 bytes. */
|
||||
export interface RenderChunkResultBody {
|
||||
Action: "renderChunk";
|
||||
ChunkGcsUri: string;
|
||||
ChunkIndex: number;
|
||||
Sha256: string;
|
||||
FramesEncoded: number;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
/** Result of an `assemble` invocation. */
|
||||
export interface AssembleResultBody {
|
||||
Action: "assemble";
|
||||
OutputGcsUri: string;
|
||||
FramesEncoded: number;
|
||||
FileSize: number;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
export type CloudRunResult = PlanResultBody | RenderChunkResultBody | AssembleResultBody;
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Map a distributed `format` to the file extension the assembled output
|
||||
* should carry on disk + in GCS. Shared by `src/server.ts` (chunk +
|
||||
* assemble output paths) and `src/sdk/renderToCloudRun.ts` (final
|
||||
* output key construction) so the two sides agree on what an mp4
|
||||
* looks like vs a png-sequence.
|
||||
*/
|
||||
|
||||
import type { DistributedFormat } from "@hyperframes/producer/distributed";
|
||||
|
||||
export type { DistributedFormat } from "@hyperframes/producer/distributed";
|
||||
|
||||
// Closed-enum lookup table. TS enforces exhaustiveness via the
|
||||
// `Record<DistributedFormat, string>` annotation — adding a format to
|
||||
// `DistributedFormat` without adding the matching key here fails to
|
||||
// typecheck, which is the same exhaustiveness guarantee a switch +
|
||||
// `_exhaustive: never` arm provides but at lower complexity.
|
||||
const FORMAT_EXTENSIONS: Record<DistributedFormat, string> = {
|
||||
mp4: ".mp4",
|
||||
mov: ".mov",
|
||||
webm: ".webm",
|
||||
"png-sequence": "",
|
||||
};
|
||||
|
||||
export function formatExtension(format: DistributedFormat): string {
|
||||
return FORMAT_EXTENSIONS[format];
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* GCS transport unit tests — URI parsing, tar round-trip, and the
|
||||
* download/upload bridge over the `FakeGcs` double.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
|
||||
import {
|
||||
downloadGcsObjectToFile,
|
||||
formatGcsUri,
|
||||
parseGcsUri,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToGcs,
|
||||
} from "./gcsTransport.js";
|
||||
|
||||
const tmpDirs: string[] = [];
|
||||
function mkTmp(prefix: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix));
|
||||
tmpDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("parseGcsUri", () => {
|
||||
it("splits bucket and key", () => {
|
||||
expect(parseGcsUri("gs://my-bucket/path/to/object.tar.gz")).toEqual({
|
||||
bucket: "my-bucket",
|
||||
key: "path/to/object.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects non-gs URIs", () => {
|
||||
expect(() => parseGcsUri("s3://b/k")).toThrow(/expected gs:\/\//);
|
||||
});
|
||||
|
||||
it("rejects a bucket with no key", () => {
|
||||
expect(() => parseGcsUri("gs://just-a-bucket")).toThrow(/missing key/);
|
||||
});
|
||||
|
||||
it("rejects an empty bucket", () => {
|
||||
expect(() => parseGcsUri("gs:///key")).toThrow(/empty bucket or key/);
|
||||
});
|
||||
|
||||
it("round-trips through formatGcsUri", () => {
|
||||
const uri = "gs://b/some/key";
|
||||
expect(formatGcsUri(parseGcsUri(uri))).toBe(uri);
|
||||
});
|
||||
});
|
||||
|
||||
describe("tarDirectory / untarDirectory", () => {
|
||||
it("round-trips a directory tree", async () => {
|
||||
const src = mkTmp("hf-tar-src-");
|
||||
mkdirSync(join(src, "nested"), { recursive: true });
|
||||
writeFileSync(join(src, "index.html"), "<html>hi</html>");
|
||||
writeFileSync(join(src, "nested", "data.json"), '{"a":1}');
|
||||
|
||||
const work = mkTmp("hf-tar-work-");
|
||||
const tarball = join(work, "out.tar.gz");
|
||||
await tarDirectory(src, tarball);
|
||||
expect(existsSync(tarball)).toBe(true);
|
||||
|
||||
const dest = join(work, "extracted");
|
||||
await untarDirectory(tarball, dest);
|
||||
expect(readFileSync(join(dest, "index.html"), "utf8")).toBe("<html>hi</html>");
|
||||
expect(readFileSync(join(dest, "nested", "data.json"), "utf8")).toBe('{"a":1}');
|
||||
});
|
||||
|
||||
it("untar wipes a stale destination first", async () => {
|
||||
const src = mkTmp("hf-tar-src2-");
|
||||
writeFileSync(join(src, "keep.txt"), "new");
|
||||
const work = mkTmp("hf-tar-work2-");
|
||||
const tarball = join(work, "out.tar.gz");
|
||||
await tarDirectory(src, tarball);
|
||||
|
||||
const dest = join(work, "extracted");
|
||||
mkdirSync(dest, { recursive: true });
|
||||
writeFileSync(join(dest, "stale.txt"), "should be gone");
|
||||
|
||||
await untarDirectory(tarball, dest);
|
||||
expect(existsSync(join(dest, "stale.txt"))).toBe(false);
|
||||
expect(readFileSync(join(dest, "keep.txt"), "utf8")).toBe("new");
|
||||
});
|
||||
});
|
||||
|
||||
describe("download/upload bridge", () => {
|
||||
it("uploads a local file then downloads identical bytes", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const work = mkTmp("hf-dl-");
|
||||
const srcFile = join(work, "src.bin");
|
||||
writeFileSync(srcFile, Buffer.from("hello gcs"));
|
||||
|
||||
const uri = "gs://bucket/obj.bin";
|
||||
await uploadFileToGcs(asStorage(gcs), srcFile, uri, "application/octet-stream");
|
||||
|
||||
const dest = join(work, "dl.bin");
|
||||
await downloadGcsObjectToFile(asStorage(gcs), uri, dest);
|
||||
expect(readFileSync(dest, "utf8")).toBe("hello gcs");
|
||||
|
||||
expect(gcs.ops.map((o) => o.kind)).toEqual(["upload", "download"]);
|
||||
});
|
||||
|
||||
it("upload throws when the source file is missing", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await expect(uploadFileToGcs(asStorage(gcs), "/no/such/file", "gs://b/k")).rejects.toThrow(
|
||||
/upload source missing/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Thin GCS transport for the Cloud Run handler.
|
||||
*
|
||||
* The OSS distributed primitives are pure functions over local file paths;
|
||||
* the handler bridges GCS ↔ the container's writable `/tmp` filesystem on
|
||||
* each request. Functions here are intentionally narrow: parse a URI,
|
||||
* download an object to a local path, upload a path, tar-pack a planDir,
|
||||
* tar-extract a planDir back out.
|
||||
*
|
||||
* Tar (not zip) for planDir transit:
|
||||
* - planDirs contain symlinks (the extract stage materializes them but
|
||||
* the compiled/ subtree may include linked assets); tar preserves them,
|
||||
* zip does not.
|
||||
* - We use the `tar` npm package (pure JS over `node:zlib`) so the
|
||||
* archive format doesn't depend on a system `tar`/`unzip` being present
|
||||
* in the container image.
|
||||
*
|
||||
* Apart from the `gs://` scheme and the `@google-cloud/storage` client this
|
||||
* is the same shape as `@hyperframes/aws-lambda`'s `s3Transport.ts`.
|
||||
*/
|
||||
|
||||
import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import type { Storage } from "@google-cloud/storage";
|
||||
import * as tar from "tar";
|
||||
|
||||
/** Parsed `gs://bucket/key` URI. */
|
||||
export interface GcsLocation {
|
||||
bucket: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
/** Parse `gs://bucket/key/path` → `{ bucket, key }`. Throws on malformed input. */
|
||||
// fallow-ignore-next-line complexity
|
||||
export function parseGcsUri(uri: string): GcsLocation {
|
||||
if (!uri.startsWith("gs://")) {
|
||||
throw new Error(`[gcsTransport] expected gs:// URI, got: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
const rest = uri.slice("gs://".length);
|
||||
const slash = rest.indexOf("/");
|
||||
if (slash === -1) {
|
||||
throw new Error(`[gcsTransport] missing key in gs URI: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
const bucket = rest.slice(0, slash);
|
||||
const key = rest.slice(slash + 1);
|
||||
if (!bucket || !key) {
|
||||
throw new Error(`[gcsTransport] empty bucket or key in gs URI: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
return { bucket, key };
|
||||
}
|
||||
|
||||
/** Build `gs://bucket/key` from a location. */
|
||||
export function formatGcsUri(loc: GcsLocation): string {
|
||||
return `gs://${loc.bucket}/${loc.key}`;
|
||||
}
|
||||
|
||||
/** Stream a GCS object to a local file path. */
|
||||
export async function downloadGcsObjectToFile(
|
||||
storage: Storage,
|
||||
uri: string,
|
||||
destPath: string,
|
||||
): Promise<void> {
|
||||
const { bucket, key } = parseGcsUri(uri);
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
const file = storage.bucket(bucket).file(key);
|
||||
// `createReadStream` streams the object body; piping into a write stream
|
||||
// keeps memory flat for large plan tarballs / chunk files rather than
|
||||
// buffering the whole object the way `file.download()` would.
|
||||
await pipeline(file.createReadStream(), createWriteStream(destPath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a local file's contents to a GCS URI using a resumable upload.
|
||||
* GCS objects have no practical size ceiling for the artifacts this adapter
|
||||
* handles (plan tarballs ≤ 2 GB, chunks ≤ a few hundred MB), so a single
|
||||
* upload call works for every case.
|
||||
*/
|
||||
export async function uploadFileToGcs(
|
||||
storage: Storage,
|
||||
localPath: string,
|
||||
uri: string,
|
||||
contentType?: string,
|
||||
): Promise<void> {
|
||||
if (!existsSync(localPath)) {
|
||||
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
|
||||
}
|
||||
const { bucket, key } = parseGcsUri(uri);
|
||||
await storage.bucket(bucket).upload(localPath, {
|
||||
destination: key,
|
||||
// `resumable: false` (simple upload) is faster for the small-to-medium
|
||||
// objects this adapter moves and avoids the extra round-trip a resumable
|
||||
// session start costs; GCS recommends resumable only past ~8 MB but our
|
||||
// chunks are reliably above that, so let the client pick by default.
|
||||
contentType,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack a directory into a `.tar.gz` at `destTarball`. Uses the `tar` npm
|
||||
* package (pure JS over `node:zlib`) rather than spawning a system tar
|
||||
* binary so the archive format is independent of the container's userland.
|
||||
*/
|
||||
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
|
||||
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
||||
throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
|
||||
}
|
||||
mkdirSync(dirname(destTarball), { recursive: true });
|
||||
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a `.tar.gz` produced by {@link tarDirectory} into `destDir`.
|
||||
* The directory is created (or cleared) before extraction so a retried
|
||||
* request doesn't observe stale files from a prior run on the same warm
|
||||
* container instance.
|
||||
*/
|
||||
export async function untarDirectory(tarballPath: string, destDir: string): Promise<void> {
|
||||
if (!existsSync(tarballPath)) {
|
||||
throw new Error(`[gcsTransport] tarball missing: ${tarballPath}`);
|
||||
}
|
||||
// Wipe target so a warm container instance's prior planDir doesn't bleed
|
||||
// into the new request. Cloud Run re-uses the instance filesystem across
|
||||
// requests served by the same instance.
|
||||
if (existsSync(destDir)) {
|
||||
rmSync(destDir, { recursive: true, force: true });
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
await tar.extract({ file: tarballPath, cwd: destDir });
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* `@hyperframes/gcp-cloud-run` — Google Cloud Run + Workflows adapter for
|
||||
* the HyperFrames distributed render pipeline.
|
||||
*
|
||||
* Two surfaces, one package:
|
||||
*
|
||||
* - **Server-side handler.** `dispatch`, `createApp`, the request/result
|
||||
* types, Chrome resolution, and GCS transport. These power the Cloud Run
|
||||
* service built from the package `Dockerfile`.
|
||||
* - **Client-side SDK.** `renderToCloudRun`, `getRenderProgress`,
|
||||
* `deploySite`, `validateDistributedRenderConfig`, and `computeRenderCost`.
|
||||
* Adopters call these from their Node process (CI scripts, CLIs) to drive
|
||||
* a deployed stack without writing GCS / Workflows boilerplate.
|
||||
*
|
||||
* The Terraform module that provisions the bucket + service + workflow lives
|
||||
* under `terraform/` in the published package; see the README. The package
|
||||
* is NOT a dependency of `@hyperframes/producer`; consumers install it
|
||||
* separately.
|
||||
*/
|
||||
|
||||
export { createApp, dispatch, type HandlerDeps, startServer, unwrapEvent } from "./server.js";
|
||||
export {
|
||||
type AssembleEvent,
|
||||
type AssembleResultBody,
|
||||
type CloudRunAction,
|
||||
type CloudRunEvent,
|
||||
type CloudRunResult,
|
||||
type PlanEvent,
|
||||
type PlanResultBody,
|
||||
type RenderChunkEvent,
|
||||
type RenderChunkResultBody,
|
||||
type SerializableDistributedRenderConfig,
|
||||
} from "./events.js";
|
||||
export { ChromeBinaryUnavailableError, resolveChromeExecutablePath } from "./chromium.js";
|
||||
export {
|
||||
downloadGcsObjectToFile,
|
||||
formatGcsUri,
|
||||
type GcsLocation,
|
||||
parseGcsUri,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToGcs,
|
||||
} from "./gcsTransport.js";
|
||||
|
||||
// ── Client-side SDK ─────────────────────────────────────────────────────────
|
||||
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./sdk/deploySite.js";
|
||||
export {
|
||||
renderToCloudRun,
|
||||
type RenderHandle,
|
||||
type RenderToCloudRunOptions,
|
||||
} from "./sdk/renderToCloudRun.js";
|
||||
export {
|
||||
getRenderProgress,
|
||||
type GetRenderProgressOptions,
|
||||
type RenderError,
|
||||
type RenderProgress,
|
||||
type RenderStatus,
|
||||
} from "./sdk/getRenderProgress.js";
|
||||
export {
|
||||
type BilledCloudRunInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./sdk/costAccounting.js";
|
||||
export { InvalidConfigError, validateDistributedRenderConfig } from "./sdk/validateConfig.js";
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* `computeRenderCost` unit tests — Cloud Run vCPU/GiB-second + Workflows
|
||||
* step math.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { type BilledCloudRunInvocation, computeRenderCost } from "./costAccounting.js";
|
||||
|
||||
describe("computeRenderCost", () => {
|
||||
it("returns zero for no invocations", () => {
|
||||
const cost = computeRenderCost([], 0);
|
||||
expect(cost.accruedSoFarUsd).toBe(0);
|
||||
expect(cost.displayCost).toBe("$0.0000");
|
||||
});
|
||||
|
||||
it("sums vCPU + memory seconds plus per-request and step charges", () => {
|
||||
const invs: BilledCloudRunInvocation[] = [
|
||||
{ durationMs: 10_000, vcpu: 4, memoryGib: 16, estimated: false },
|
||||
{ durationMs: 10_000, vcpu: 4, memoryGib: 16, estimated: false },
|
||||
];
|
||||
const cost = computeRenderCost(invs, 6);
|
||||
// 2 × 10s: vCPU 80 vcpu-s × 0.000024 = 0.00192; mem 320 GiB-s × 0.0000025 = 0.0008;
|
||||
// requests 2 × 4e-7 ≈ 0 → raw 0.0027208, rounded to 4 dp = 0.0027.
|
||||
// workflows 6 × 1e-5 = 0.00006 → rounds up to 0.0001 at 4 dp.
|
||||
expect(cost.breakdown.cloudRunUsd).toBeCloseTo(0.0027, 4);
|
||||
expect(cost.breakdown.workflowsUsd).toBeCloseTo(0.0001, 4);
|
||||
expect(cost.accruedSoFarUsd).toBeGreaterThan(0);
|
||||
expect(cost.breakdown.gcsEstimate).toBe("not-included");
|
||||
});
|
||||
|
||||
it("flags estimated when any invocation was estimated", () => {
|
||||
const cost = computeRenderCost([{ durationMs: 0, vcpu: 4, memoryGib: 16, estimated: true }], 4);
|
||||
expect(cost.breakdown.estimated).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Per-render cost accounting for {@link getRenderProgress}.
|
||||
*
|
||||
* Google bills the render service two ways:
|
||||
*
|
||||
* - **Cloud Run** by **vCPU-seconds** and **GiB-seconds** of request
|
||||
* processing time, plus a flat per-request charge. Each handler
|
||||
* invocation returns its own `DurationMs` in the result body, so the
|
||||
* progress reader can recover billed time per step without a separate
|
||||
* Cloud Monitoring query — multiply by the service's configured vCPU /
|
||||
* memory to get the resource-seconds.
|
||||
* - **Cloud Workflows** by **steps executed**. The orchestration is a
|
||||
* fixed shape (Plan + N×RenderChunk + Assemble + a handful of control
|
||||
* steps), so the step count scales with chunk count.
|
||||
*
|
||||
* The math is documented inline so the constants stay close to the pricing
|
||||
* source they came from. Cost is **best-effort**: GCP pricing varies by
|
||||
* region + committed-use discounts; we use on-demand `us-central1` (Tier 1)
|
||||
* rates as of 2026-06 and label the result `displayCost` so callers see the
|
||||
* dollar value but downstream automation can also read the raw number.
|
||||
*/
|
||||
|
||||
/** Cloud Run request-based billing, us-central1 Tier 1: USD per vCPU-second. */
|
||||
const CLOUD_RUN_USD_PER_VCPU_SECOND = 0.000024;
|
||||
/** Cloud Run request-based billing, us-central1 Tier 1: USD per GiB-second. */
|
||||
const CLOUD_RUN_USD_PER_GIB_SECOND = 0.0000025;
|
||||
/** Cloud Run: USD per request ($0.40 per million). */
|
||||
const CLOUD_RUN_USD_PER_REQUEST = 0.0000004;
|
||||
/** Cloud Workflows: USD per internal step ($0.01 per 1,000, after a free tier). */
|
||||
const WORKFLOWS_USD_PER_STEP = 0.00001;
|
||||
|
||||
/** Per-invocation billed slice the cost calc cares about. */
|
||||
export interface BilledCloudRunInvocation {
|
||||
/** Wall-clock the handler reported via `DurationMs` in its result body. */
|
||||
durationMs: number;
|
||||
/** vCPU the Cloud Run service was configured with at invocation time. */
|
||||
vcpu: number;
|
||||
/** Memory in GiB the Cloud Run service was configured with. */
|
||||
memoryGib: number;
|
||||
/** `true` if the duration was inferred (step result missing) rather than read from the handler payload. */
|
||||
estimated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of {@link computeRenderCost}.
|
||||
*
|
||||
* NOTE: `displayCost` / `accruedSoFarUsd` cover Cloud Run compute + Cloud
|
||||
* Workflows steps only. They EXCLUDE GCS storage + network egress for the
|
||||
* plan tarball (which can be ~100 MB), chunk artifacts, and the final output
|
||||
* — see `breakdown.gcsEstimate`. Treat the figure as a compute-cost floor,
|
||||
* not the authoritative total bill.
|
||||
*/
|
||||
export interface RenderCost {
|
||||
/** USD accrued to date (Cloud Run + Workflows only; excludes GCS — see note above). */
|
||||
accruedSoFarUsd: number;
|
||||
/** Human-readable USD string, e.g. `"$0.0214"`. Excludes GCS storage/egress. */
|
||||
displayCost: string;
|
||||
breakdown: {
|
||||
cloudRunUsd: number;
|
||||
workflowsUsd: number;
|
||||
/** GCS transfer + storage cost varies by tier; we don't try to compute it here. */
|
||||
gcsEstimate: "not-included";
|
||||
/** `true` if any invocation fell back to estimated billing. */
|
||||
estimated: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sum Cloud Run vCPU-seconds + GiB-seconds + per-request charges and Cloud
|
||||
* Workflows steps into an aggregate USD figure.
|
||||
*
|
||||
* `workflowSteps` is the count of Workflows steps executed so far — Plan
|
||||
* (1) + RenderChunk (chunkCount) + Assemble (1) + the control steps
|
||||
* (BuildChunkList, AssertChunkCount, …). Pass the count the progress reader
|
||||
* derived from the execution; a rough constant overhead is fine since the
|
||||
* step charge is a rounding error next to Cloud Run compute.
|
||||
*/
|
||||
export function computeRenderCost(
|
||||
invocations: BilledCloudRunInvocation[],
|
||||
workflowSteps: number,
|
||||
): RenderCost {
|
||||
let cloudRunUsd = 0;
|
||||
let anyEstimated = false;
|
||||
for (const inv of invocations) {
|
||||
const seconds = inv.durationMs / 1000;
|
||||
cloudRunUsd += seconds * inv.vcpu * CLOUD_RUN_USD_PER_VCPU_SECOND;
|
||||
cloudRunUsd += seconds * inv.memoryGib * CLOUD_RUN_USD_PER_GIB_SECOND;
|
||||
cloudRunUsd += CLOUD_RUN_USD_PER_REQUEST;
|
||||
if (inv.estimated) anyEstimated = true;
|
||||
}
|
||||
const workflowsUsd = workflowSteps * WORKFLOWS_USD_PER_STEP;
|
||||
const accruedSoFarUsd = roundUsd(cloudRunUsd + workflowsUsd);
|
||||
return {
|
||||
accruedSoFarUsd,
|
||||
displayCost: formatUsd(accruedSoFarUsd),
|
||||
breakdown: {
|
||||
cloudRunUsd: roundUsd(cloudRunUsd),
|
||||
workflowsUsd: roundUsd(workflowsUsd),
|
||||
gcsEstimate: "not-included",
|
||||
estimated: anyEstimated,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function roundUsd(usd: number): number {
|
||||
// Four decimal places — enough resolution for per-chunk granularity.
|
||||
// Anything finer is noise vs GCP's own rounding.
|
||||
return Math.round(usd * 10_000) / 10_000;
|
||||
}
|
||||
|
||||
function formatUsd(usd: number): string {
|
||||
return `$${usd.toFixed(4)}`;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* `deploySite` unit tests — content-addressed siteId, existence
|
||||
* short-circuit, and the upload path over `FakeGcs`.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { asStorage, FakeGcs } from "../__fixtures__/fakeGcs.js";
|
||||
import { deploySite } from "./deploySite.js";
|
||||
|
||||
const tmpDirs: string[] = [];
|
||||
function mkProject(content: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-site-"));
|
||||
tmpDirs.push(dir);
|
||||
writeFileSync(join(dir, "index.html"), content);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("deploySite", () => {
|
||||
it("uploads and returns a content-addressed handle", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const dir = mkProject("<html>v1</html>");
|
||||
const handle = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
|
||||
expect(handle.uploaded).toBe(true);
|
||||
expect(handle.projectGcsUri).toBe(`gs://b/sites/${handle.siteId}/project.tar.gz`);
|
||||
expect(gcs.objects.has(handle.projectGcsUri)).toBe(true);
|
||||
});
|
||||
|
||||
it("produces a stable siteId for identical content", async () => {
|
||||
const a = await deploySite({
|
||||
projectDir: mkProject("<html>same</html>"),
|
||||
bucketName: "b",
|
||||
storage: asStorage(new FakeGcs()),
|
||||
});
|
||||
const b = await deploySite({
|
||||
projectDir: mkProject("<html>same</html>"),
|
||||
bucketName: "b",
|
||||
storage: asStorage(new FakeGcs()),
|
||||
});
|
||||
expect(a.siteId).toBe(b.siteId);
|
||||
});
|
||||
|
||||
it("produces different siteIds for different content", async () => {
|
||||
const a = await deploySite({
|
||||
projectDir: mkProject("<html>one</html>"),
|
||||
bucketName: "b",
|
||||
storage: asStorage(new FakeGcs()),
|
||||
});
|
||||
const b = await deploySite({
|
||||
projectDir: mkProject("<html>two</html>"),
|
||||
bucketName: "b",
|
||||
storage: asStorage(new FakeGcs()),
|
||||
});
|
||||
expect(a.siteId).not.toBe(b.siteId);
|
||||
});
|
||||
|
||||
it("short-circuits the upload when the object already exists", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const dir = mkProject("<html>cache</html>");
|
||||
const first = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
|
||||
expect(first.uploaded).toBe(true);
|
||||
|
||||
const second = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
|
||||
expect(second.uploaded).toBe(false);
|
||||
expect(second.siteId).toBe(first.siteId);
|
||||
// Only one upload op total.
|
||||
expect(gcs.ops.filter((o) => o.kind === "upload").length).toBe(1);
|
||||
});
|
||||
|
||||
it("honours an explicit siteId override", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const handle = await deploySite({
|
||||
projectDir: mkProject("<html></html>"),
|
||||
bucketName: "b",
|
||||
siteId: "my-git-sha",
|
||||
storage: asStorage(gcs),
|
||||
});
|
||||
expect(handle.siteId).toBe("my-git-sha");
|
||||
expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* `deploySite` — upload a project directory to GCS once per content hash
|
||||
* and return a reusable handle.
|
||||
*
|
||||
* `renderToCloudRun` calls this implicitly when no `siteHandle` is passed,
|
||||
* but exposing it as a standalone verb lets adopters bundle a project ahead
|
||||
* of time and reuse the handle across many renders without re-tarring the
|
||||
* project tree on every call.
|
||||
*
|
||||
* The handle is **content-addressed**: `siteId` is derived from a SHA-256
|
||||
* over the project files. Two `deploySite` calls on an unchanged tree
|
||||
* produce the same `siteId` and short-circuit the upload after a single
|
||||
* existence check.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Storage } from "@google-cloud/storage";
|
||||
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
||||
import { formatGcsUri, tarDirectory, uploadFileToGcs } from "../gcsTransport.js";
|
||||
|
||||
/** Options for {@link deploySite}. */
|
||||
export interface DeploySiteOptions {
|
||||
/** Local project directory containing `index.html` (and any composition assets). */
|
||||
projectDir: string;
|
||||
/** GCS bucket the Terraform module provisioned. */
|
||||
bucketName: string;
|
||||
/**
|
||||
* Override the content-addressed site id. Useful when the caller has a
|
||||
* stable external identifier they want to use (e.g. a git SHA); if unset,
|
||||
* the hash of the project tree picks it.
|
||||
*/
|
||||
siteId?: string;
|
||||
/** Injection seam for tests. Production callers leave unset. */
|
||||
storage?: Storage;
|
||||
}
|
||||
|
||||
/** Stable handle returned by {@link deploySite}. Pass back to {@link renderToCloudRun}. */
|
||||
export interface SiteHandle {
|
||||
/** Content-addressed (or caller-supplied) identifier; stable across re-uploads of the same tree. */
|
||||
siteId: string;
|
||||
/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectGcsUri`. */
|
||||
bucketName: string;
|
||||
/** Full `gs://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToCloudRun`. */
|
||||
projectGcsUri: string;
|
||||
/** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */
|
||||
bytes: number;
|
||||
/** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */
|
||||
uploadedAt: string;
|
||||
/** `false` if the object already existed and we skipped the upload. */
|
||||
uploaded: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload `projectDir` to `gs://bucketName/sites/<siteId>/project.tar.gz`.
|
||||
*
|
||||
* Short-circuits when an object with the same key already exists in the
|
||||
* bucket — `siteId` derives from the project's content hash, so the same
|
||||
* bytes produce the same key, and re-uploading would be redundant.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
|
||||
if (!statSync(opts.projectDir).isDirectory()) {
|
||||
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
||||
}
|
||||
|
||||
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
||||
const key = `sites/${siteId}/project.tar.gz`;
|
||||
const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key });
|
||||
const storage = opts.storage ?? new Storage();
|
||||
const file = storage.bucket(opts.bucketName).file(key);
|
||||
|
||||
// Existence short-circuit. Adopters re-rendering the same project on a
|
||||
// tight inner loop (CI smoke, demo flows) save the tar+gzip+upload pass
|
||||
// on every iteration.
|
||||
const existing = await headObject(file);
|
||||
if (existing) {
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectGcsUri,
|
||||
bytes: existing.bytes,
|
||||
uploadedAt: existing.lastModified,
|
||||
uploaded: false,
|
||||
};
|
||||
}
|
||||
|
||||
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
|
||||
try {
|
||||
const tarball = join(workdir, "project.tar.gz");
|
||||
await tarDirectory(opts.projectDir, tarball);
|
||||
const size = statSync(tarball).size;
|
||||
await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectGcsUri,
|
||||
bytes: size,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
uploaded: true,
|
||||
};
|
||||
} finally {
|
||||
rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Narrow surface of the `@google-cloud/storage` `File` this module uses —
|
||||
* lets the test double implement just `exists()` + `getMetadata()` without
|
||||
* pulling the full client type.
|
||||
*/
|
||||
interface FileLike {
|
||||
exists(): Promise<[boolean, ...unknown[]]>;
|
||||
getMetadata(): Promise<[{ size?: string | number; updated?: string }, ...unknown[]]>;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async function headObject(file: FileLike): Promise<{ bytes: number; lastModified: string } | null> {
|
||||
const [exists] = await file.exists();
|
||||
if (!exists) return null;
|
||||
const [meta] = await file.getMetadata();
|
||||
const sizeRaw = meta.size;
|
||||
const bytes =
|
||||
typeof sizeRaw === "string" ? Number(sizeRaw) : typeof sizeRaw === "number" ? sizeRaw : 0;
|
||||
return {
|
||||
bytes: Number.isFinite(bytes) ? bytes : 0,
|
||||
lastModified: meta.updated ?? new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* `getRenderProgress` unit tests — state mapping + parsing the accumulated
|
||||
* workflow result into frame totals, output file, and cost.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
type ExecutionRecord,
|
||||
type ExecutionsGetClientLike,
|
||||
getRenderProgress,
|
||||
} from "./getRenderProgress.js";
|
||||
|
||||
function fakeExecutions(record: ExecutionRecord): ExecutionsGetClientLike {
|
||||
return {
|
||||
async getExecution(_req: { name: string }) {
|
||||
return [record] as [ExecutionRecord];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const accumulated = JSON.stringify({
|
||||
Plan: { TotalFrames: 90, DurationMs: 4000 },
|
||||
Chunks: [
|
||||
{ FramesEncoded: 30, DurationMs: 8000 },
|
||||
{ FramesEncoded: 30, DurationMs: 8000 },
|
||||
{ FramesEncoded: 30, DurationMs: 8000 },
|
||||
],
|
||||
Assemble: { OutputGcsUri: "gs://b/renders/r1/output.mp4", FileSize: 123456, DurationMs: 3000 },
|
||||
});
|
||||
|
||||
describe("getRenderProgress", () => {
|
||||
it("reports running with no frame data while ACTIVE", async () => {
|
||||
const p = await getRenderProgress({
|
||||
executionName: "x",
|
||||
executions: fakeExecutions({ state: "ACTIVE", startTime: { seconds: 1700000000 } }),
|
||||
});
|
||||
expect(p.status).toBe("running");
|
||||
expect(p.overallProgress).toBe(0);
|
||||
expect(p.totalFrames).toBeNull();
|
||||
expect(p.fatalErrorEncountered).toBe(false);
|
||||
});
|
||||
|
||||
it("reports succeeded with parsed frames + cost", async () => {
|
||||
const p = await getRenderProgress({
|
||||
executionName: "x",
|
||||
vcpu: 4,
|
||||
memoryGib: 16,
|
||||
executions: fakeExecutions({
|
||||
state: "SUCCEEDED",
|
||||
result: accumulated,
|
||||
startTime: { seconds: 1700000000 },
|
||||
endTime: { seconds: 1700000031 },
|
||||
}),
|
||||
});
|
||||
expect(p.status).toBe("succeeded");
|
||||
expect(p.overallProgress).toBe(1);
|
||||
expect(p.totalFrames).toBe(90);
|
||||
expect(p.framesRendered).toBe(90);
|
||||
expect(p.invocationsObserved).toBe(5); // plan + 3 chunks + assemble
|
||||
expect(p.outputFile).toEqual({ gcsUri: "gs://b/renders/r1/output.mp4", bytes: 123456 });
|
||||
expect(p.costs.accruedSoFarUsd).toBeGreaterThan(0);
|
||||
expect(p.costs.breakdown.estimated).toBe(false);
|
||||
});
|
||||
|
||||
it("maps FAILED to a fatal error and surfaces the error payload", async () => {
|
||||
const p = await getRenderProgress({
|
||||
executionName: "x",
|
||||
executions: fakeExecutions({
|
||||
state: "FAILED",
|
||||
error: { payload: "boom", context: "renderChunk" },
|
||||
}),
|
||||
});
|
||||
expect(p.status).toBe("failed");
|
||||
expect(p.fatalErrorEncountered).toBe(true);
|
||||
expect(p.errors[0]?.cause).toBe("boom");
|
||||
expect(p.errors[0]?.state).toBe("renderChunk");
|
||||
});
|
||||
|
||||
it("extracts the handler error name from a wrapped http failure payload", async () => {
|
||||
// Workflows wraps an http step failure as { code, message, body }, where
|
||||
// body is the handler's JSON { error, message }.
|
||||
const payload = JSON.stringify({
|
||||
code: 400,
|
||||
message: "HTTP server responded with error code 400",
|
||||
body: JSON.stringify({ error: "PLAN_HASH_MISMATCH", message: "mismatch" }),
|
||||
});
|
||||
const p = await getRenderProgress({
|
||||
executionName: "x",
|
||||
executions: fakeExecutions({ state: "FAILED", error: { payload, context: "renderChunk" } }),
|
||||
});
|
||||
expect(p.errors[0]?.error).toBe("PLAN_HASH_MISMATCH");
|
||||
});
|
||||
|
||||
it("maps CANCELLED", async () => {
|
||||
const p = await getRenderProgress({
|
||||
executionName: "x",
|
||||
executions: fakeExecutions({ state: "CANCELLED" }),
|
||||
});
|
||||
expect(p.status).toBe("cancelled");
|
||||
expect(p.fatalErrorEncountered).toBe(true);
|
||||
});
|
||||
|
||||
it("requires an executionName", async () => {
|
||||
await expect(
|
||||
getRenderProgress({ executionName: "", executions: fakeExecutions({}) }),
|
||||
).rejects.toThrow(/executionName is required/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* `getRenderProgress` — read-only progress + cost snapshot for a single
|
||||
* render started by {@link renderToCloudRun}.
|
||||
*
|
||||
* Pulls one `GetExecution` per call. Cloud Workflows does not surface
|
||||
* per-step payloads through the basic Executions API the way Step Functions
|
||||
* exposes its history, so this reader takes a different tack than the AWS
|
||||
* adapter: the workflow definition **accumulates** each step's result body
|
||||
* (Plan + every RenderChunk + Assemble) and returns them as one structured
|
||||
* object. On success we parse that object for frame totals, the output
|
||||
* file, and per-step `DurationMs` (which the handler stamps into every
|
||||
* result), then compute cost against the service's configured vCPU/memory.
|
||||
*
|
||||
* Progress is therefore coarse while the execution is ACTIVE (we report
|
||||
* `running` with `overallProgress = 0`) and exact once it SUCCEEDS
|
||||
* (`overallProgress = 1`, real frame + cost numbers). Mid-flight per-chunk
|
||||
* progress would require the Workflows step-entries API; that's a tracked
|
||||
* follow-up, not part of the first version.
|
||||
*/
|
||||
|
||||
import {
|
||||
type BilledCloudRunInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./costAccounting.js";
|
||||
|
||||
/** Normalised render status. Maps from Cloud Workflows execution states. */
|
||||
export type RenderStatus = "running" | "succeeded" | "failed" | "cancelled" | "unknown";
|
||||
|
||||
/** One error surfaced by the execution. */
|
||||
export interface RenderError {
|
||||
/** Step the failure surfaced in, when recoverable from the error context; else `<execution>`. */
|
||||
state: string;
|
||||
/** Error class / type. */
|
||||
error: string;
|
||||
/** Cause string (often a stringified JSON payload from the handler). */
|
||||
cause: string;
|
||||
}
|
||||
|
||||
/** Snapshot of a single render's progress + cost + errors at one point in time. */
|
||||
export interface RenderProgress {
|
||||
status: RenderStatus;
|
||||
/** `[0, 1]`; coarse while running, exact on success. */
|
||||
overallProgress: number;
|
||||
framesRendered: number;
|
||||
/** `null` until the execution succeeds and the accumulated plan result is read. */
|
||||
totalFrames: number | null;
|
||||
/** Cloud Run invocations the workflow scheduled (Plan + chunks + Assemble), when known. */
|
||||
invocationsObserved: number;
|
||||
costs: RenderCost;
|
||||
/** Final output object if Assemble succeeded; `null` otherwise. */
|
||||
outputFile: { gcsUri: string; bytes: number | null } | null;
|
||||
errors: RenderError[];
|
||||
/** `true` once the execution has terminated in a non-success state. */
|
||||
fatalErrorEncountered: boolean;
|
||||
startedAt: string;
|
||||
endedAt: string | null;
|
||||
}
|
||||
|
||||
/** Protobuf Timestamp shape the gapic client returns for start/end times. */
|
||||
interface ProtoTimestamp {
|
||||
seconds?: number | string | null;
|
||||
nanos?: number | null;
|
||||
}
|
||||
|
||||
/** Subset of a Cloud Workflows Execution this reader consumes. */
|
||||
export interface ExecutionRecord {
|
||||
name?: string | null;
|
||||
state?: string | null;
|
||||
result?: string | null;
|
||||
error?: { payload?: string | null; context?: string | null } | null;
|
||||
startTime?: ProtoTimestamp | string | null;
|
||||
endTime?: ProtoTimestamp | string | null;
|
||||
}
|
||||
|
||||
/** Minimal surface of `@google-cloud/workflows`' `ExecutionsClient` for reads. */
|
||||
export interface ExecutionsGetClientLike {
|
||||
getExecution(req: { name: string }): Promise<[ExecutionRecord, ...unknown[]]>;
|
||||
}
|
||||
|
||||
/** Options for {@link getRenderProgress}. */
|
||||
export interface GetRenderProgressOptions {
|
||||
/** Server-assigned execution resource name from a {@link renderToCloudRun} call. */
|
||||
executionName: string;
|
||||
/** vCPU the Cloud Run service is configured with (for cost). Default 4. */
|
||||
vcpu?: number;
|
||||
/** Memory in GiB the Cloud Run service is configured with (for cost). Default 16. */
|
||||
memoryGib?: number;
|
||||
/** Test injection seam — production callers leave unset. */
|
||||
executions?: ExecutionsGetClientLike;
|
||||
}
|
||||
|
||||
const DEFAULT_VCPU = 4;
|
||||
const DEFAULT_MEMORY_GIB = 16;
|
||||
|
||||
/** Result body the handler returns for each action; the workflow accumulates these. */
|
||||
interface AccumulatedResult {
|
||||
Plan?: { TotalFrames?: number; DurationMs?: number } | null;
|
||||
Chunks?: Array<{ FramesEncoded?: number; DurationMs?: number } | null> | null;
|
||||
Assemble?: {
|
||||
OutputGcsUri?: string;
|
||||
FileSize?: number;
|
||||
FramesEncoded?: number;
|
||||
DurationMs?: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
/** Pull a current progress snapshot for one render. */
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function getRenderProgress(opts: GetRenderProgressOptions): Promise<RenderProgress> {
|
||||
if (!opts.executionName) {
|
||||
throw new Error("[getRenderProgress] executionName is required");
|
||||
}
|
||||
const executions = opts.executions ?? (await defaultExecutionsClient());
|
||||
const vcpu = opts.vcpu ?? DEFAULT_VCPU;
|
||||
const memoryGib = opts.memoryGib ?? DEFAULT_MEMORY_GIB;
|
||||
|
||||
const [execution] = await executions.getExecution({ name: opts.executionName });
|
||||
const status = mapState(execution.state);
|
||||
const startedAt = toIso(execution.startTime) ?? new Date(0).toISOString();
|
||||
const endedAt = toIso(execution.endTime);
|
||||
|
||||
const errors: RenderError[] = [];
|
||||
if (execution.error) {
|
||||
errors.push({
|
||||
state: execution.error.context ?? "<execution>",
|
||||
error: extractErrorName(execution.error.payload) ?? "ExecutionError",
|
||||
cause: execution.error.payload ?? "",
|
||||
});
|
||||
}
|
||||
|
||||
// Default snapshot: running / unknown — no frame or cost data until the
|
||||
// accumulated result is available on success.
|
||||
if (status !== "succeeded") {
|
||||
return {
|
||||
status,
|
||||
overallProgress: 0,
|
||||
framesRendered: 0,
|
||||
totalFrames: null,
|
||||
invocationsObserved: 0,
|
||||
costs: computeRenderCost([], 0),
|
||||
outputFile: null,
|
||||
errors,
|
||||
fatalErrorEncountered: status === "failed" || status === "cancelled",
|
||||
startedAt,
|
||||
endedAt,
|
||||
};
|
||||
}
|
||||
|
||||
const acc = parseAccumulated(execution.result);
|
||||
const chunks = acc.Chunks?.filter((c): c is NonNullable<typeof c> => c != null) ?? [];
|
||||
const framesRendered = chunks.reduce((sum, c) => sum + (c.FramesEncoded ?? 0), 0);
|
||||
const totalFrames = typeof acc.Plan?.TotalFrames === "number" ? acc.Plan.TotalFrames : null;
|
||||
|
||||
const invocations: BilledCloudRunInvocation[] = [];
|
||||
const pushInv = (durationMs: number | undefined): void => {
|
||||
invocations.push({
|
||||
durationMs: typeof durationMs === "number" ? durationMs : 0,
|
||||
vcpu,
|
||||
memoryGib,
|
||||
estimated: typeof durationMs !== "number",
|
||||
});
|
||||
};
|
||||
if (acc.Plan) pushInv(acc.Plan.DurationMs);
|
||||
for (const c of chunks) pushInv(c.DurationMs);
|
||||
if (acc.Assemble) pushInv(acc.Assemble.DurationMs);
|
||||
|
||||
// Workflow step count: Plan + N chunks + Assemble + a small constant of
|
||||
// control steps (BuildChunkList, AssertChunkCount, the map scaffold).
|
||||
const workflowSteps = invocations.length + 4;
|
||||
const costs = computeRenderCost(invocations, workflowSteps);
|
||||
|
||||
const outputGcsUri = acc.Assemble?.OutputGcsUri;
|
||||
const outputFile = outputGcsUri
|
||||
? {
|
||||
gcsUri: outputGcsUri,
|
||||
bytes: typeof acc.Assemble?.FileSize === "number" ? acc.Assemble.FileSize : null,
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
status,
|
||||
overallProgress: 1,
|
||||
framesRendered,
|
||||
totalFrames,
|
||||
invocationsObserved: invocations.length,
|
||||
costs,
|
||||
outputFile,
|
||||
errors,
|
||||
fatalErrorEncountered: false,
|
||||
startedAt,
|
||||
endedAt,
|
||||
};
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
function mapState(state: string | null | undefined): RenderStatus {
|
||||
switch (state) {
|
||||
case "ACTIVE":
|
||||
case "QUEUED":
|
||||
return "running";
|
||||
case "SUCCEEDED":
|
||||
return "succeeded";
|
||||
case "FAILED":
|
||||
case "UNAVAILABLE":
|
||||
return "failed";
|
||||
case "CANCELLED":
|
||||
return "cancelled";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
function parseAccumulated(result: string | null | undefined): AccumulatedResult {
|
||||
if (!result) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(result) as unknown;
|
||||
if (parsed && typeof parsed === "object") return parsed as AccumulatedResult;
|
||||
} catch {
|
||||
// Non-JSON result — treat as empty so cost/frames degrade to zero
|
||||
// rather than throwing on a snapshot read.
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort pull of the handler's error name out of a Workflows failure
|
||||
* payload. On an http step failure, Workflows wraps the response as
|
||||
* `{ code, message, body, ... }` where `body` is the handler's JSON
|
||||
* `{ error, message }`. We dig out `error` (the typed name like
|
||||
* `PLAN_HASH_MISMATCH`) so triage sees the real cause, not a generic label.
|
||||
* Returns undefined for any shape we don't recognise — never throws.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
function extractErrorName(payload: string | null | undefined): string | undefined {
|
||||
if (!payload) return undefined;
|
||||
try {
|
||||
const outer = JSON.parse(payload) as { error?: unknown; body?: unknown };
|
||||
if (typeof outer.error === "string") return outer.error;
|
||||
if (typeof outer.body === "string") {
|
||||
const inner = JSON.parse(outer.body) as { error?: unknown };
|
||||
if (typeof inner.error === "string") return inner.error;
|
||||
} else if (outer.body && typeof outer.body === "object") {
|
||||
const inner = outer.body as { error?: unknown };
|
||||
if (typeof inner.error === "string") return inner.error;
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON / unexpected shape — fall through to the generic label.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
function toIso(ts: ProtoTimestamp | string | null | undefined): string | null {
|
||||
if (ts == null) return null;
|
||||
if (typeof ts === "string") return ts;
|
||||
const seconds = ts.seconds == null ? null : Number(ts.seconds);
|
||||
if (seconds == null || !Number.isFinite(seconds)) return null;
|
||||
const ms = seconds * 1000 + (ts.nanos ?? 0) / 1e6;
|
||||
return new Date(ms).toISOString();
|
||||
}
|
||||
|
||||
async function defaultExecutionsClient(): Promise<ExecutionsGetClientLike> {
|
||||
const mod = await import("@google-cloud/workflows");
|
||||
const client = new mod.ExecutionsClient();
|
||||
return client as unknown as ExecutionsGetClientLike;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* SDK subpath export — `@hyperframes/gcp-cloud-run/sdk`.
|
||||
*
|
||||
* Pulled into its own subpath so consumers that only drive renders (CLI, CI
|
||||
* scripts, adopter tooling) don't pay the cost of importing `./server.js`,
|
||||
* which transitively pulls `puppeteer-core` into the module graph. The SDK
|
||||
* files here are GCS + Workflows clients only — safe to load in any Node
|
||||
* environment.
|
||||
*/
|
||||
|
||||
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";
|
||||
export {
|
||||
type ExecutionsClientLike,
|
||||
renderToCloudRun,
|
||||
type RenderHandle,
|
||||
type RenderToCloudRunOptions,
|
||||
} from "./renderToCloudRun.js";
|
||||
export {
|
||||
type ExecutionRecord,
|
||||
type ExecutionsGetClientLike,
|
||||
getRenderProgress,
|
||||
type GetRenderProgressOptions,
|
||||
type RenderError,
|
||||
type RenderProgress,
|
||||
type RenderStatus,
|
||||
} from "./getRenderProgress.js";
|
||||
export {
|
||||
type BilledCloudRunInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./costAccounting.js";
|
||||
export {
|
||||
InvalidConfigError,
|
||||
MAX_WORKFLOWS_INPUT_BYTES,
|
||||
validateDistributedRenderConfig,
|
||||
validateVariablesPayload,
|
||||
validateWorkflowsInputSize,
|
||||
} from "./validateConfig.js";
|
||||
export type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
export type { DistributedFormat } from "../formatExtension.js";
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* `renderToCloudRun` unit tests — argument assembly, required-field
|
||||
* validation, and the CreateExecution call over a fake ExecutionsClient.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import { type ExecutionsClientLike, renderToCloudRun } from "./renderToCloudRun.js";
|
||||
import type { SiteHandle } from "./deploySite.js";
|
||||
|
||||
const config = {
|
||||
fps: 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4",
|
||||
} as SerializableDistributedRenderConfig;
|
||||
|
||||
const site: SiteHandle = {
|
||||
siteId: "abc",
|
||||
bucketName: "b",
|
||||
projectGcsUri: "gs://b/sites/abc/project.tar.gz",
|
||||
bytes: 100,
|
||||
uploadedAt: "2026-06-06T00:00:00Z",
|
||||
uploaded: true,
|
||||
};
|
||||
|
||||
class FakeExecutions implements ExecutionsClientLike {
|
||||
lastArgument: string | null = null;
|
||||
lastParent: string | null = null;
|
||||
|
||||
workflowPath(project: string, location: string, workflow: string): string {
|
||||
return `projects/${project}/locations/${location}/workflows/${workflow}`;
|
||||
}
|
||||
|
||||
async createExecution(req: {
|
||||
parent: string;
|
||||
execution: { argument: string };
|
||||
}): Promise<[{ name?: string | null; state?: string | null }]> {
|
||||
this.lastParent = req.parent;
|
||||
this.lastArgument = req.execution.argument;
|
||||
return [{ name: `${req.parent}/executions/exec-123`, state: "ACTIVE" }];
|
||||
}
|
||||
}
|
||||
|
||||
function opts(executions: ExecutionsClientLike) {
|
||||
return {
|
||||
siteHandle: site,
|
||||
config,
|
||||
bucketName: "b",
|
||||
projectId: "proj",
|
||||
location: "us-central1",
|
||||
workflowId: "hyperframes-render",
|
||||
serviceUrl: "https://render-abc.run.app",
|
||||
renderId: "hf-render-fixed",
|
||||
executions,
|
||||
};
|
||||
}
|
||||
|
||||
describe("renderToCloudRun", () => {
|
||||
it("starts an execution and returns a handle", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
const handle = await renderToCloudRun(opts(fake));
|
||||
expect(handle.renderId).toBe("hf-render-fixed");
|
||||
expect(handle.executionName).toBe(
|
||||
"projects/proj/locations/us-central1/workflows/hyperframes-render/executions/exec-123",
|
||||
);
|
||||
expect(handle.outputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.mp4");
|
||||
expect(handle.projectGcsUri).toBe("gs://b/sites/abc/project.tar.gz");
|
||||
});
|
||||
|
||||
it("builds the workflow argument the YAML expects", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
await renderToCloudRun(opts(fake));
|
||||
const arg = JSON.parse(fake.lastArgument ?? "{}");
|
||||
expect(arg.RenderId).toBe("hf-render-fixed");
|
||||
expect(arg.ProjectGcsUri).toBe("gs://b/sites/abc/project.tar.gz");
|
||||
expect(arg.PlanOutputGcsPrefix).toBe("gs://b/renders/hf-render-fixed/");
|
||||
expect(arg.OutputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.mp4");
|
||||
expect(arg.ServiceUrl).toBe("https://render-abc.run.app");
|
||||
expect(arg.Config.format).toBe("mp4");
|
||||
expect(fake.lastParent).toBe(
|
||||
"projects/proj/locations/us-central1/workflows/hyperframes-render",
|
||||
);
|
||||
});
|
||||
|
||||
it("derives the output extension from the format", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
const handle = await renderToCloudRun({
|
||||
...opts(fake),
|
||||
config: { ...config, format: "webm" } as SerializableDistributedRenderConfig,
|
||||
});
|
||||
expect(handle.outputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.webm");
|
||||
});
|
||||
|
||||
it("requires serviceUrl", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
await expect(renderToCloudRun({ ...opts(fake), serviceUrl: "" })).rejects.toThrow(
|
||||
/serviceUrl is required/,
|
||||
);
|
||||
});
|
||||
|
||||
it("requires a siteHandle or projectDir", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
const { siteHandle, ...rest } = opts(fake);
|
||||
void siteHandle;
|
||||
await expect(renderToCloudRun(rest)).rejects.toThrow(/siteHandle or projectDir/);
|
||||
});
|
||||
|
||||
it("validates the config before any GCP call", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
await expect(
|
||||
renderToCloudRun({ ...opts(fake), config: { ...config, fps: 25 } as never }),
|
||||
).rejects.toThrow(/config\.fps/);
|
||||
expect(fake.lastArgument).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects a renderId that could escape the GCS key prefix", async () => {
|
||||
const fake = new FakeExecutions();
|
||||
await expect(renderToCloudRun({ ...opts(fake), renderId: "../escape" })).rejects.toThrow(
|
||||
/renderId must match/,
|
||||
);
|
||||
await expect(renderToCloudRun({ ...opts(fake), renderId: "has/slash" })).rejects.toThrow(
|
||||
/renderId must match/,
|
||||
);
|
||||
expect(fake.lastArgument).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* `renderToCloudRun` — start a distributed render against an already-deployed
|
||||
* Cloud Run service + Cloud Workflows definition and return a handle the
|
||||
* caller can poll with {@link getRenderProgress}.
|
||||
*
|
||||
* The function does *not* wait for the render to finish. Cloud Workflows
|
||||
* executions can run for hours; blocking the caller's process on the
|
||||
* execution is the wrong default. The returned `RenderHandle` carries
|
||||
* everything the progress / cost / download paths need.
|
||||
*
|
||||
* Wire order:
|
||||
* 1. Validate config (typed throw before any GCP call).
|
||||
* 2. `deploySite` if no `siteHandle` was provided.
|
||||
* 3. `CreateExecution` against the workflow with the argument shape the
|
||||
* `packages/gcp-cloud-run/terraform/workflow.yaml` definition expects.
|
||||
* 4. Return handle. The GCS `outputKey` is deterministic from the
|
||||
* client-generated `renderId` so the caller can predict the final
|
||||
* object URL before the (server-assigned) execution id exists.
|
||||
*
|
||||
* Unlike Step Functions, Cloud Workflows assigns the execution id
|
||||
* server-side, so we cannot use it as the GCS prefix. We mint a `renderId`
|
||||
* (uuid) client-side, use it for every GCS path, and pass it into the
|
||||
* workflow argument; the server-assigned execution resource name is tracked
|
||||
* separately for polling.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { Storage } from "@google-cloud/storage";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import { formatExtension } from "../formatExtension.js";
|
||||
import { formatGcsUri } from "../gcsTransport.js";
|
||||
import { deploySite, type SiteHandle } from "./deploySite.js";
|
||||
import { validateDistributedRenderConfig, validateWorkflowsInputSize } from "./validateConfig.js";
|
||||
|
||||
/**
|
||||
* Minimal surface of `@google-cloud/workflows`' `ExecutionsClient` that
|
||||
* this module needs. The real client satisfies this; tests inject a double.
|
||||
*/
|
||||
export interface ExecutionsClientLike {
|
||||
workflowPath(project: string, location: string, workflow: string): string;
|
||||
createExecution(req: {
|
||||
parent: string;
|
||||
execution: { argument: string };
|
||||
}): Promise<[{ name?: string | null; state?: string | null }, ...unknown[]]>;
|
||||
}
|
||||
|
||||
/** Options for {@link renderToCloudRun}. */
|
||||
export interface RenderToCloudRunOptions {
|
||||
/** Local project directory. Required when `siteHandle` is not supplied. */
|
||||
projectDir?: string;
|
||||
/** Re-use an existing `deploySite` upload (skips tar+GCS upload). */
|
||||
siteHandle?: SiteHandle;
|
||||
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
|
||||
config: SerializableDistributedRenderConfig;
|
||||
/** GCS bucket from the Terraform output (`render_bucket_name`). */
|
||||
bucketName: string;
|
||||
/** GCP project id hosting the workflow. */
|
||||
projectId: string;
|
||||
/** Workflow location, e.g. `us-central1`. */
|
||||
location: string;
|
||||
/** Workflow id from the Terraform output (`workflow_name`). */
|
||||
workflowId: string;
|
||||
/**
|
||||
* HTTPS URL of the deployed Cloud Run render service (Terraform output
|
||||
* `service_url`). The workflow POSTs every step (plan / renderChunk /
|
||||
* assemble) to this URL; passed as an execution argument so the workflow
|
||||
* definition stays free of hard-coded URLs.
|
||||
*/
|
||||
serviceUrl: string;
|
||||
/**
|
||||
* Final output GCS key. Defaults to `renders/<renderId>/output.<ext>`
|
||||
* where `<ext>` is derived from `config.format`.
|
||||
*/
|
||||
outputKey?: string;
|
||||
/**
|
||||
* Client-generated render id. Defaults to `hf-render-<uuid>`. Used as the
|
||||
* GCS key prefix and echoed into the workflow argument; not the same as
|
||||
* the server-assigned execution id.
|
||||
*/
|
||||
renderId?: string;
|
||||
/** Test injection seam — production callers leave unset. */
|
||||
executions?: ExecutionsClientLike;
|
||||
/** Test injection seam — propagated to `deploySite` when applicable. */
|
||||
storage?: Storage;
|
||||
}
|
||||
|
||||
/** Stable identifier + every URL/name the caller needs to follow the render. */
|
||||
export interface RenderHandle {
|
||||
/** Client-generated render id; the GCS prefix everything lands under. */
|
||||
renderId: string;
|
||||
/** Server-assigned execution resource name; pass to {@link getRenderProgress}. */
|
||||
executionName: string;
|
||||
bucketName: string;
|
||||
workflowId: string;
|
||||
outputGcsUri: string;
|
||||
projectGcsUri: string;
|
||||
startedAt: string;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise<RenderHandle> {
|
||||
validateDistributedRenderConfig(opts.config);
|
||||
|
||||
if (!opts.bucketName) throw new Error("[renderToCloudRun] bucketName is required");
|
||||
if (!opts.projectId) throw new Error("[renderToCloudRun] projectId is required");
|
||||
if (!opts.location) throw new Error("[renderToCloudRun] location is required");
|
||||
if (!opts.workflowId) throw new Error("[renderToCloudRun] workflowId is required");
|
||||
if (!opts.serviceUrl) throw new Error("[renderToCloudRun] serviceUrl is required");
|
||||
if (!opts.siteHandle && !opts.projectDir) {
|
||||
throw new Error("[renderToCloudRun] either siteHandle or projectDir must be supplied");
|
||||
}
|
||||
|
||||
const renderId = opts.renderId ?? `hf-render-${randomUUID()}`;
|
||||
// `renderId` is interpolated directly into GCS object keys
|
||||
// (`renders/<renderId>/…`). Reject anything that could escape that prefix
|
||||
// or build a malformed key — `..`, slashes, or other path metacharacters —
|
||||
// so a caller-supplied id can't collide with or overwrite another render's
|
||||
// artifacts elsewhere in the bucket.
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(renderId) || renderId.includes("..")) {
|
||||
throw new Error(
|
||||
`[renderToCloudRun] renderId must match [A-Za-z0-9._-]+ and not contain "..": ${JSON.stringify(renderId)}`,
|
||||
);
|
||||
}
|
||||
const ext = formatExtension(opts.config.format);
|
||||
const outputKey = opts.outputKey ?? `renders/${renderId}/output${ext}`;
|
||||
const planOutputGcsPrefix = formatGcsUri({
|
||||
bucket: opts.bucketName,
|
||||
key: `renders/${renderId}/`,
|
||||
});
|
||||
const outputGcsUri = formatGcsUri({ bucket: opts.bucketName, key: outputKey });
|
||||
|
||||
const site =
|
||||
opts.siteHandle ??
|
||||
(await deploySite({
|
||||
projectDir: opts.projectDir as string,
|
||||
bucketName: opts.bucketName,
|
||||
storage: opts.storage,
|
||||
}));
|
||||
|
||||
const argument = {
|
||||
RenderId: renderId,
|
||||
ProjectGcsUri: site.projectGcsUri,
|
||||
PlanOutputGcsPrefix: planOutputGcsPrefix,
|
||||
OutputGcsUri: outputGcsUri,
|
||||
ServiceUrl: opts.serviceUrl,
|
||||
Config: opts.config,
|
||||
};
|
||||
|
||||
// Reject oversize input client-side. Cloud Workflows caps the execution
|
||||
// argument at 512 KiB; without this check, input bloat (typically from
|
||||
// `config.variables` containing inlined media) surfaces as an opaque
|
||||
// server-side error after the execution starts, far from the caller's
|
||||
// stack frame.
|
||||
validateWorkflowsInputSize(argument);
|
||||
|
||||
const executions = opts.executions ?? (await defaultExecutionsClient());
|
||||
const parent = executions.workflowPath(opts.projectId, opts.location, opts.workflowId);
|
||||
const startedAt = new Date().toISOString();
|
||||
const [execution] = await executions.createExecution({
|
||||
parent,
|
||||
execution: { argument: JSON.stringify(argument) },
|
||||
});
|
||||
|
||||
if (!execution.name) {
|
||||
throw new Error("[renderToCloudRun] CreateExecution returned no execution name");
|
||||
}
|
||||
|
||||
return {
|
||||
renderId,
|
||||
executionName: execution.name,
|
||||
bucketName: opts.bucketName,
|
||||
workflowId: opts.workflowId,
|
||||
outputGcsUri,
|
||||
projectGcsUri: site.projectGcsUri,
|
||||
startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily import the real `@google-cloud/workflows` ExecutionsClient. Dynamic
|
||||
* so SDK consumers that only call `validateDistributedRenderConfig` (or
|
||||
* inject their own client) don't pay the import cost.
|
||||
*/
|
||||
async function defaultExecutionsClient(): Promise<ExecutionsClientLike> {
|
||||
const mod = await import("@google-cloud/workflows");
|
||||
const client = new mod.ExecutionsClient();
|
||||
return client as unknown as ExecutionsClientLike;
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* `validateDistributedRenderConfig` + `validateWorkflowsInputSize` unit
|
||||
* tests. Pins the shape rejections the SDK surfaces synchronously before a
|
||||
* Cloud Workflows execution starts.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import {
|
||||
InvalidConfigError,
|
||||
MAX_WORKFLOWS_INPUT_BYTES,
|
||||
validateDistributedRenderConfig,
|
||||
validateVariablesPayload,
|
||||
validateWorkflowsInputSize,
|
||||
} from "./validateConfig.js";
|
||||
|
||||
function base(): SerializableDistributedRenderConfig {
|
||||
return {
|
||||
fps: 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4",
|
||||
} as SerializableDistributedRenderConfig;
|
||||
}
|
||||
|
||||
describe("validateDistributedRenderConfig", () => {
|
||||
it("accepts a minimal valid config", () => {
|
||||
expect(validateDistributedRenderConfig(base())).toBeDefined();
|
||||
});
|
||||
|
||||
it("rejects a bad fps", () => {
|
||||
expect(() => validateDistributedRenderConfig({ ...base(), fps: 25 } as never)).toThrow(
|
||||
/config\.fps/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects odd dimensions (yuv420p)", () => {
|
||||
expect(() => validateDistributedRenderConfig({ ...base(), width: 1921 })).toThrow(/even/);
|
||||
});
|
||||
|
||||
it("rejects an out-of-range dimension", () => {
|
||||
expect(() => validateDistributedRenderConfig({ ...base(), height: 8 })).toThrow(/\[16, 7680\]/);
|
||||
});
|
||||
|
||||
it("rejects an unknown format", () => {
|
||||
expect(() => validateDistributedRenderConfig({ ...base(), format: "gif" as never })).toThrow(
|
||||
/config\.format/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects codec with a non-mp4 format", () => {
|
||||
expect(() =>
|
||||
validateDistributedRenderConfig({ ...base(), format: "webm", codec: "h264" } as never),
|
||||
).toThrow(/only valid with format="mp4"/);
|
||||
});
|
||||
|
||||
it("rejects crf + bitrate together", () => {
|
||||
expect(() =>
|
||||
validateDistributedRenderConfig({ ...base(), crf: 20, bitrate: "10M" } as never),
|
||||
).toThrow(/mutually exclusive/);
|
||||
});
|
||||
|
||||
it("rejects force-hdr", () => {
|
||||
expect(() =>
|
||||
validateDistributedRenderConfig({ ...base(), hdrMode: "force-hdr" as never }),
|
||||
).toThrow(/force-sdr/);
|
||||
});
|
||||
|
||||
it("rejects an over-cap chunkSize", () => {
|
||||
expect(() => validateDistributedRenderConfig({ ...base(), chunkSize: 5000 } as never)).toThrow(
|
||||
/<= 3600/,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws InvalidConfigError with a field pointer", () => {
|
||||
try {
|
||||
validateDistributedRenderConfig({ ...base(), fps: 1 } as never);
|
||||
throw new Error("should have thrown");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.fps");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateVariablesPayload", () => {
|
||||
it("accepts a plain JSON object", () => {
|
||||
expect(() =>
|
||||
validateVariablesPayload({ title: "Hi", count: 3, nested: { ok: true } }),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects undefined leaves", () => {
|
||||
expect(() => validateVariablesPayload({ a: undefined })).toThrow(/undefined leaves/);
|
||||
});
|
||||
|
||||
it("rejects a top-level array", () => {
|
||||
expect(() => validateVariablesPayload([1, 2])).toThrow(/plain JSON object/);
|
||||
});
|
||||
|
||||
it("rejects NaN", () => {
|
||||
expect(() => validateVariablesPayload({ x: NaN })).toThrow(/non-finite/);
|
||||
});
|
||||
|
||||
it("rejects a Date (non-plain object)", () => {
|
||||
expect(() => validateVariablesPayload({ when: new Date(0) })).toThrow(/non-plain objects/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateWorkflowsInputSize", () => {
|
||||
it("accepts a small payload", () => {
|
||||
expect(() => validateWorkflowsInputSize({ a: "b" })).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects a payload over the 512 KiB cap", () => {
|
||||
const big = { blob: "x".repeat(MAX_WORKFLOWS_INPUT_BYTES + 1) };
|
||||
expect(() => validateWorkflowsInputSize(big)).toThrow(/512 KiB/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Client-side validation for the Cloud Run adapter.
|
||||
*
|
||||
* The cloud-agnostic config-shape validation (`validateDistributedRenderConfig`,
|
||||
* `validateVariablesPayload`, `InvalidConfigError`) lives in
|
||||
* `@hyperframes/producer/distributed` and is shared with the other adapters.
|
||||
* This module re-exports those and adds the one piece that is specific to
|
||||
* Cloud Workflows: the 512 KiB execution-argument size cap.
|
||||
*/
|
||||
|
||||
import { InvalidConfigError } from "@hyperframes/producer/distributed";
|
||||
|
||||
export {
|
||||
InvalidConfigError,
|
||||
validateDistributedRenderConfig,
|
||||
validateVariablesPayload,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
|
||||
/**
|
||||
* Hard cap on Cloud Workflows execution arguments — 512 KiB per the Workflows
|
||||
* quotas page (maximum size of arguments passed when an execution starts).
|
||||
* The cap is on the entire serialized argument, not just the variables,
|
||||
* because users hit it at the wire boundary regardless of which field caused
|
||||
* the bloat.
|
||||
*
|
||||
* Specific to Cloud Workflows. Other runtimes (Lambda + Step Functions,
|
||||
* Temporal) have different caps; don't reuse this constant for those without
|
||||
* confirming the limit.
|
||||
*/
|
||||
export const MAX_WORKFLOWS_INPUT_BYTES = 512 * 1024;
|
||||
|
||||
/** Pointer to the docs section that explains the URL-your-assets convention. */
|
||||
const LARGE_VARIABLES_DOCS_URL =
|
||||
"https://hyperframes.heygen.com/deploy/templates-on-lambda#working-with-large-variables";
|
||||
|
||||
/**
|
||||
* Validate that the serialized Cloud Workflows execution argument fits inside
|
||||
* the 512 KiB cap. Measured in UTF-8 bytes (the format the API uses on the
|
||||
* wire) — JS strings count UTF-16 code units, which under-reports for any
|
||||
* multi-byte character.
|
||||
*
|
||||
* Throws {@link InvalidConfigError} with a clear message naming the actual
|
||||
* byte count, the cap, and a pointer to the "working with large variables"
|
||||
* docs section, so users hit the limit at the SDK boundary with actionable
|
||||
* guidance instead of as an opaque argument-too-large error after the
|
||||
* execution starts.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export function validateWorkflowsInputSize(input: unknown): void {
|
||||
let serialized: string | undefined;
|
||||
try {
|
||||
serialized = JSON.stringify(input);
|
||||
} catch (err) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
`Cloud Workflows execution argument is not JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
if (serialized === undefined) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
"Cloud Workflows execution argument is not JSON-serializable (JSON.stringify returned undefined). " +
|
||||
"Check that all fields, including config.variables, are plain JSON values.",
|
||||
);
|
||||
}
|
||||
const byteLength = Buffer.byteLength(serialized, "utf8");
|
||||
if (byteLength > MAX_WORKFLOWS_INPUT_BYTES) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
`Cloud Workflows execution argument is ${byteLength} bytes, which exceeds the ` +
|
||||
`${MAX_WORKFLOWS_INPUT_BYTES}-byte (512 KiB) limit. Variables are for typed data ` +
|
||||
`(strings, numbers, structured records); media assets (images, audio, video) should ` +
|
||||
`be passed as URL references the composition resolves at render time, not inlined as ` +
|
||||
`base64. See ${LARGE_VARIABLES_DOCS_URL} for the URL-your-assets convention.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Handler dispatch + HTTP-shell unit tests.
|
||||
*
|
||||
* Asserts that:
|
||||
* - `dispatch` routes Action="plan"/"renderChunk"/"assemble" to the
|
||||
* matching OSS primitive and plumbs GCS download/upload around it.
|
||||
* - It unwraps `{ Payload }` / `{ Input }` envelopes and rejects unknown
|
||||
* actions.
|
||||
* - The handler-boundary guards fire: plan-hash mismatch + bucket
|
||||
* allowlist throw the typed, non-retryable errors.
|
||||
* - `createApp` maps non-retryable errors → 400 and retryable → 500.
|
||||
*
|
||||
* The real OSS primitives are NOT exercised — they have their own coverage
|
||||
* in `packages/producer`. This file pins the adapter glue's contract.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AssembleResult, ChunkResult, PlanResult } from "@hyperframes/producer/distributed";
|
||||
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
|
||||
import type { AssembleEvent, CloudRunEvent, PlanEvent, RenderChunkEvent } from "./events.js";
|
||||
import { createApp, dispatch, type HandlerDeps, unwrapEvent } from "./server.js";
|
||||
import { tarDirectory } from "./gcsTransport.js";
|
||||
|
||||
const tmpDirs: string[] = [];
|
||||
function mkTmp(prefix: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), prefix));
|
||||
tmpDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterEach(() => {
|
||||
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const PLAN_HASH = "abc123planhash";
|
||||
|
||||
/** Build a real project tarball, seed it into the fake, return its URI. */
|
||||
async function seedProjectTar(gcs: FakeGcs, uri: string): Promise<void> {
|
||||
const src = mkTmp("hf-proj-");
|
||||
writeFileSync(join(src, "index.html"), "<html></html>");
|
||||
const tarPath = join(mkTmp("hf-proj-tar-"), "project.tar.gz");
|
||||
await tarDirectory(src, tarPath);
|
||||
gcs.seedFromFile(uri, tarPath);
|
||||
}
|
||||
|
||||
/** Build a real plan tarball containing plan.json, seed it, return its URI. */
|
||||
async function seedPlanTar(gcs: FakeGcs, uri: string, planHash: string): Promise<void> {
|
||||
const planDir = mkTmp("hf-plan-");
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash }));
|
||||
const tarPath = join(mkTmp("hf-plan-tar-"), "plan.tar.gz");
|
||||
await tarDirectory(planDir, tarPath);
|
||||
gcs.seedFromFile(uri, tarPath);
|
||||
}
|
||||
|
||||
const planResult: PlanResult = {
|
||||
planDir: "(set at call time)",
|
||||
planHash: PLAN_HASH,
|
||||
chunkCount: 3,
|
||||
totalFrames: 90,
|
||||
fps: 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4",
|
||||
ffmpegVersion: "ffmpeg version 6.1.1",
|
||||
producerVersion: "0.6.79",
|
||||
};
|
||||
|
||||
function depsWith(
|
||||
gcs: FakeGcs,
|
||||
overrides: Partial<NonNullable<HandlerDeps["primitives"]>> = {},
|
||||
): HandlerDeps {
|
||||
const plan = async (_projectDir: string, _config: unknown, planDir: string) => {
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: PLAN_HASH }));
|
||||
return planResult;
|
||||
};
|
||||
const renderChunk = async (_planDir: string, chunkIndex: number, outputBase: string) => {
|
||||
writeFileSync(outputBase, Buffer.from(`chunk-${chunkIndex}`));
|
||||
return {
|
||||
outputPath: outputBase,
|
||||
outputKind: "file",
|
||||
framesEncoded: 30,
|
||||
sha256: `sha-${chunkIndex}`,
|
||||
} satisfies ChunkResult;
|
||||
};
|
||||
const assemble = async (
|
||||
_planDir: string,
|
||||
_chunkPaths: string[],
|
||||
_audio: string | null,
|
||||
finalOutput: string,
|
||||
) => {
|
||||
writeFileSync(finalOutput, Buffer.from("final-output"));
|
||||
return { framesEncoded: 90, fileSize: 12 } satisfies AssembleResult;
|
||||
};
|
||||
return {
|
||||
storage: asStorage(gcs),
|
||||
skipChromeResolution: true,
|
||||
primitives: { plan, renderChunk, assemble, ...overrides } as NonNullable<
|
||||
HandlerDeps["primitives"]
|
||||
>,
|
||||
};
|
||||
}
|
||||
|
||||
describe("unwrapEvent", () => {
|
||||
const plan: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectGcsUri: "gs://b/p.tar.gz",
|
||||
PlanOutputGcsPrefix: "gs://b/out/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" } as PlanEvent["Config"],
|
||||
};
|
||||
|
||||
it("returns a bare event unchanged", () => {
|
||||
expect(unwrapEvent(plan).Action).toBe("plan");
|
||||
});
|
||||
|
||||
it("unwraps { Payload }", () => {
|
||||
expect(unwrapEvent({ Payload: plan } as CloudRunEvent).Action).toBe("plan");
|
||||
});
|
||||
|
||||
it("unwraps nested { Input: { Payload } }", () => {
|
||||
expect(unwrapEvent({ Input: { Payload: plan } } as CloudRunEvent).Action).toBe("plan");
|
||||
});
|
||||
|
||||
it("throws when no Action is found", () => {
|
||||
expect(() => unwrapEvent({ foo: "bar" } as unknown as CloudRunEvent)).toThrow(
|
||||
/no recognised Action/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("dispatch", () => {
|
||||
it("routes plan, uploads the plan tarball", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedProjectTar(gcs, "gs://b/sites/x/project.tar.gz");
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectGcsUri: "gs://b/sites/x/project.tar.gz",
|
||||
PlanOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" } as PlanEvent["Config"],
|
||||
};
|
||||
const res = await dispatch(event, depsWith(gcs));
|
||||
expect(res.Action).toBe("plan");
|
||||
if (res.Action !== "plan") throw new Error("unreachable");
|
||||
expect(res.PlanHash).toBe(PLAN_HASH);
|
||||
expect(res.ChunkCount).toBe(3);
|
||||
expect(res.PlanGcsUri).toBe("gs://b/renders/r1/plan.tar.gz");
|
||||
expect(gcs.objects.has("gs://b/renders/r1/plan.tar.gz")).toBe(true);
|
||||
});
|
||||
|
||||
it("routes renderChunk and uploads the chunk", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
|
||||
PlanHash: PLAN_HASH,
|
||||
ChunkIndex: 2,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Format: "mp4",
|
||||
};
|
||||
const res = await dispatch(event, depsWith(gcs));
|
||||
if (res.Action !== "renderChunk") throw new Error("unreachable");
|
||||
expect(res.ChunkIndex).toBe(2);
|
||||
expect(res.ChunkGcsUri).toBe("gs://b/renders/r1/chunks/0002.mp4");
|
||||
expect(gcs.objects.has("gs://b/renders/r1/chunks/0002.mp4")).toBe(true);
|
||||
});
|
||||
|
||||
it("throws PLAN_HASH_MISMATCH when the event hash disagrees", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
|
||||
PlanHash: "WRONG_HASH",
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Format: "mp4",
|
||||
};
|
||||
await expect(dispatch(event, depsWith(gcs))).rejects.toThrow(/PLAN_HASH_MISMATCH/);
|
||||
});
|
||||
|
||||
it("routes assemble and uploads the final output", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
gcs.seed("gs://b/renders/r1/chunks/0000.mp4", Buffer.from("c0"));
|
||||
gcs.seed("gs://b/renders/r1/chunks/0001.mp4", Buffer.from("c1"));
|
||||
const event: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
|
||||
ChunkGcsUris: ["gs://b/renders/r1/chunks/0000.mp4", "gs://b/renders/r1/chunks/0001.mp4"],
|
||||
AudioGcsUri: null,
|
||||
OutputGcsUri: "gs://b/renders/r1/output.mp4",
|
||||
Format: "mp4",
|
||||
};
|
||||
const res = await dispatch(event, depsWith(gcs));
|
||||
if (res.Action !== "assemble") throw new Error("unreachable");
|
||||
expect(res.FramesEncoded).toBe(90);
|
||||
expect(gcs.objects.has("gs://b/renders/r1/output.mp4")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects an unknown action", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await expect(
|
||||
dispatch({ Action: "nope" } as unknown as CloudRunEvent, depsWith(gcs)),
|
||||
).rejects.toThrow(/no recognised Action/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bucket allowlist guard", () => {
|
||||
it("throws GCS_URI_NOT_ALLOWED for an off-bucket URI", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const prev = process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
process.env.HYPERFRAMES_RENDER_BUCKET = "allowed-bucket";
|
||||
try {
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://evil-bucket/plan.tar.gz",
|
||||
PlanHash: PLAN_HASH,
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://allowed-bucket/renders/r1/",
|
||||
Format: "mp4",
|
||||
};
|
||||
await expect(dispatch(event, depsWith(gcs))).rejects.toThrow(/GCS_URI_NOT_ALLOWED/);
|
||||
} finally {
|
||||
if (prev === undefined) delete process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
else process.env.HYPERFRAMES_RENDER_BUCKET = prev;
|
||||
}
|
||||
});
|
||||
|
||||
it('treats HYPERFRAMES_RENDER_BUCKET="*" as an explicit opt-out (off-bucket allowed)', async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://any-bucket/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
const prev = process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
process.env.HYPERFRAMES_RENDER_BUCKET = "*";
|
||||
try {
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://any-bucket/renders/r1/plan.tar.gz",
|
||||
PlanHash: PLAN_HASH,
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://any-bucket/renders/r1/",
|
||||
Format: "mp4",
|
||||
};
|
||||
const res = await dispatch(event, depsWith(gcs));
|
||||
expect(res.Action).toBe("renderChunk");
|
||||
} finally {
|
||||
if (prev === undefined) delete process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
else process.env.HYPERFRAMES_RENDER_BUCKET = prev;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("createApp HTTP mapping", () => {
|
||||
it("returns 200 with the result body on success", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
const app = createApp(depsWith(gcs));
|
||||
const res = await app.request("/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
|
||||
PlanHash: PLAN_HASH,
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Format: "mp4",
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { Action: string };
|
||||
expect(body.Action).toBe("renderChunk");
|
||||
});
|
||||
|
||||
it("returns 400 for a non-retryable error (plan-hash mismatch)", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
|
||||
const app = createApp(depsWith(gcs));
|
||||
const res = await app.request("/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
|
||||
PlanHash: "WRONG",
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Format: "mp4",
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
const body = (await res.json()) as { error: string };
|
||||
expect(body.error).toBe("PLAN_HASH_MISMATCH");
|
||||
});
|
||||
|
||||
it("returns 500 for a retryable/unknown error", async () => {
|
||||
const gcs = new FakeGcs(); // plan tar NOT seeded → download fails (retryable)
|
||||
const app = createApp(depsWith(gcs));
|
||||
const res = await app.request("/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
Action: "renderChunk",
|
||||
PlanGcsUri: "gs://b/renders/r1/missing.tar.gz",
|
||||
PlanHash: PLAN_HASH,
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
|
||||
Format: "mp4",
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(500);
|
||||
});
|
||||
|
||||
it("returns 400 when the body is not JSON", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
const app = createApp(depsWith(gcs));
|
||||
const res = await app.request("/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: "not json{",
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("healthz returns ok", async () => {
|
||||
const app = createApp(depsWith(new FakeGcs()));
|
||||
const res = await app.request("/healthz");
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,647 @@
|
||||
/**
|
||||
* Cloud Run request handler for HyperFrames distributed rendering.
|
||||
*
|
||||
* One container image, three roles. Cloud Workflows POSTs a JSON body with
|
||||
* an `Action` field; the handler unwraps any `Payload`/`Input` envelope,
|
||||
* primes the runtime (Chrome path), and forwards to the matching OSS
|
||||
* primitive from `@hyperframes/producer/distributed`.
|
||||
*
|
||||
* Everything heavy — capture, encode, audio mix — happens inside the OSS
|
||||
* primitives. The handler is thin glue: parse body → GCS download → call
|
||||
* primitive → GCS upload → return small JSON result.
|
||||
*
|
||||
* `dispatch()` is the testable core (inject `storage` + `primitives`); the
|
||||
* Hono app at the bottom is the HTTP shell the Dockerfile runs. The shape
|
||||
* deliberately tracks `@hyperframes/aws-lambda`'s `handler.ts` so the two
|
||||
* adapters stay easy to diff.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, extname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { serve } from "@hono/node-server";
|
||||
import { Storage } from "@google-cloud/storage";
|
||||
import { Hono } from "hono";
|
||||
import {
|
||||
assemble,
|
||||
type AssembleResult,
|
||||
type ChunkResult,
|
||||
type DistributedRenderConfig,
|
||||
plan,
|
||||
type PlanResult,
|
||||
renderChunk,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
import { resolveChromeExecutablePath } from "./chromium.js";
|
||||
import type {
|
||||
AssembleEvent,
|
||||
AssembleResultBody,
|
||||
CloudRunAction,
|
||||
CloudRunEvent,
|
||||
CloudRunResult,
|
||||
PlanEvent,
|
||||
PlanResultBody,
|
||||
RenderChunkEvent,
|
||||
RenderChunkResultBody,
|
||||
} from "./events.js";
|
||||
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
|
||||
import {
|
||||
downloadGcsObjectToFile,
|
||||
parseGcsUri,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToGcs,
|
||||
} from "./gcsTransport.js";
|
||||
|
||||
/**
|
||||
* Lazily-constructed Storage client. Cached at module scope so warm
|
||||
* container instances reuse the underlying HTTP keep-alive pool across
|
||||
* requests.
|
||||
*/
|
||||
let cachedStorage: Storage | null = null;
|
||||
function getStorage(): Storage {
|
||||
if (cachedStorage) return cachedStorage;
|
||||
cachedStorage = new Storage();
|
||||
return cachedStorage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional injection points used by the handler's unit tests. Production
|
||||
* callers leave these unset; the real OSS primitives are used. Tests inject
|
||||
* `storage` and `primitives` directly rather than mutating module state.
|
||||
*/
|
||||
export interface HandlerDeps {
|
||||
storage?: Storage;
|
||||
primitives?: {
|
||||
plan: typeof plan;
|
||||
renderChunk: typeof renderChunk;
|
||||
assemble: typeof assemble;
|
||||
};
|
||||
/** Override the per-request workdir root (defaults to the OS tmpdir). */
|
||||
tmpRoot?: string;
|
||||
/** Skip Chrome resolution (used by dispatch tests that mock renderChunk). */
|
||||
skipChromeResolution?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch a single render request. Cloud Workflows (or a direct caller)
|
||||
* sometimes wraps the body in `{ Payload: ... }` or `{ Input: ... }`; unwrap
|
||||
* until we hit a discriminated event.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function dispatch(event: CloudRunEvent, deps?: HandlerDeps): Promise<CloudRunResult> {
|
||||
const unwrapped = unwrapEvent(event);
|
||||
validateEventGcsUris(unwrapped);
|
||||
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
|
||||
try {
|
||||
switch (unwrapped.Action) {
|
||||
case "plan":
|
||||
return await handlePlan(unwrapped, deps);
|
||||
case "renderChunk":
|
||||
return await handleRenderChunk(unwrapped, deps);
|
||||
case "assemble":
|
||||
return await handleAssemble(unwrapped, deps);
|
||||
default: {
|
||||
// Compile-time exhaustiveness: a new CloudRunAction member trips
|
||||
// the `never` assignment before the runtime error is reachable.
|
||||
const _exhaustive: never = unwrapped;
|
||||
throw new Error(
|
||||
`[handler] unknown Action: ${JSON.stringify(
|
||||
(_exhaustive as { Action?: string }).Action,
|
||||
)}. Expected one of "plan", "renderChunk", "assemble".`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logEvent({
|
||||
event: "handler_error",
|
||||
action: unwrapped.Action,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
name: err instanceof Error ? err.name : undefined,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// At most `{Payload: {Input: ...}}` is expected; 4 levels is 2× headroom
|
||||
// and prevents infinite loops on malformed input.
|
||||
const MAX_ENVELOPE_DEPTH = 4;
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export function unwrapEvent(event: CloudRunEvent): PlanEvent | RenderChunkEvent | AssembleEvent {
|
||||
let cursor: CloudRunEvent = event;
|
||||
for (let i = 0; i < MAX_ENVELOPE_DEPTH; i++) {
|
||||
if (cursor && typeof cursor === "object") {
|
||||
const obj = cursor as Record<string, unknown>;
|
||||
if (typeof obj.Action === "string" && isCloudRunAction(obj.Action)) {
|
||||
return cursor as PlanEvent | RenderChunkEvent | AssembleEvent;
|
||||
}
|
||||
if ("Payload" in obj) {
|
||||
cursor = obj.Payload as CloudRunEvent;
|
||||
continue;
|
||||
}
|
||||
if ("Input" in obj) {
|
||||
cursor = obj.Input as CloudRunEvent;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
throw new Error(
|
||||
`[handler] body has no recognised Action; unwrapped ${MAX_ENVELOPE_DEPTH} levels of Payload/Input without finding one.`,
|
||||
);
|
||||
}
|
||||
|
||||
function isCloudRunAction(value: string): value is CloudRunAction {
|
||||
return value === "plan" || value === "renderChunk" || value === "assemble";
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a single JSON line to stdout. Cloud Logging ingests each stdout line
|
||||
* as a structured `jsonPayload` entry, so Logs Explorer can filter on
|
||||
* `jsonPayload.event="handler_start"` and project specific fields when
|
||||
* triaging without attaching a debugger.
|
||||
*/
|
||||
function logEvent(payload: Record<string, unknown>): void {
|
||||
console.log(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact, non-PII summary of an event for logging. The full body can
|
||||
* include the entire project config; we only emit the routable fields
|
||||
* needed to triage a failure from Cloud Logging.
|
||||
*/
|
||||
function summarizeEvent(
|
||||
event: PlanEvent | RenderChunkEvent | AssembleEvent,
|
||||
): Record<string, unknown> {
|
||||
switch (event.Action) {
|
||||
case "plan":
|
||||
return {
|
||||
projectGcsUri: event.ProjectGcsUri,
|
||||
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
|
||||
format: event.Config.format,
|
||||
fps: event.Config.fps,
|
||||
};
|
||||
case "renderChunk":
|
||||
return {
|
||||
planGcsUri: event.PlanGcsUri,
|
||||
chunkIndex: event.ChunkIndex,
|
||||
format: event.Format,
|
||||
};
|
||||
case "assemble":
|
||||
return {
|
||||
planGcsUri: event.PlanGcsUri,
|
||||
chunkCount: event.ChunkGcsUris.length,
|
||||
hasAudio: event.AudioGcsUri !== null,
|
||||
outputGcsUri: event.OutputGcsUri,
|
||||
format: event.Format,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Point the engine at the in-image Chrome binary. The OSS engine resolves
|
||||
* Chrome via `PRODUCER_HEADLESS_SHELL_PATH` first; set it once per instance
|
||||
* before invoking any browser-touching primitive. ffmpeg is on the image's
|
||||
* PATH (apt-installed by the Dockerfile), so nothing to prime there.
|
||||
*/
|
||||
function primeChrome(deps?: HandlerDeps): void {
|
||||
if (deps?.skipChromeResolution) return;
|
||||
if (process.env.PRODUCER_HEADLESS_SHELL_PATH) return;
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = resolveChromeExecutablePath();
|
||||
}
|
||||
|
||||
// ── Plan ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanResultBody> {
|
||||
const started = Date.now();
|
||||
const storage = deps?.storage ?? getStorage();
|
||||
const primitive = deps?.primitives?.plan ?? plan;
|
||||
|
||||
// The producer's probe stage launches Chromium whenever the composition
|
||||
// needs a runtime duration probe or has unresolved sub-compositions, so
|
||||
// plan has to resolve Chrome the same way renderChunk does.
|
||||
primeChrome(deps);
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-plan-"));
|
||||
const projectArchive = join(work, "project.tar.gz");
|
||||
const projectDir = join(work, "project");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
|
||||
await untarDirectory(projectArchive, projectDir);
|
||||
|
||||
const config: DistributedRenderConfig = {
|
||||
...event.Config,
|
||||
};
|
||||
const result: PlanResult = await primitive(projectDir, config, planDir);
|
||||
|
||||
// Upload the planDir as a single tarball. The workflow cannot pass a
|
||||
// directory-shaped artifact between steps; we serialize and rely on the
|
||||
// consumer (renderChunk / assemble) to untar. `audio.aac` lives inside
|
||||
// planDir, so it already rides along in this tarball — every consumer
|
||||
// (including assemble) gets it from the untar. We deliberately do NOT
|
||||
// upload a separate audio object: it would duplicate the bytes on every
|
||||
// plan upload and be re-downloaded + overwritten by assemble. `AudioGcsUri`
|
||||
// stays in the result shape for wire compatibility but is null.
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
await tarDirectory(planDir, planTar);
|
||||
const planTarUri = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
|
||||
const audioPath = join(planDir, "audio.aac");
|
||||
const hasAudio = existsSync(audioPath) && statSync(audioPath).size > 0;
|
||||
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
|
||||
|
||||
return {
|
||||
Action: "plan",
|
||||
PlanGcsUri: planTarUri,
|
||||
PlanHash: result.planHash,
|
||||
ChunkCount: result.chunkCount,
|
||||
TotalFrames: result.totalFrames,
|
||||
Fps: result.fps,
|
||||
Width: result.width,
|
||||
Height: result.height,
|
||||
Format: result.format,
|
||||
HasAudio: hasAudio,
|
||||
AudioGcsUri: null,
|
||||
FfmpegVersion: result.ffmpegVersion,
|
||||
ProducerVersion: result.producerVersion,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
// ── RenderChunk ─────────────────────────────────────────────────────────────
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async function handleRenderChunk(
|
||||
event: RenderChunkEvent,
|
||||
deps?: HandlerDeps,
|
||||
): Promise<RenderChunkResultBody> {
|
||||
const started = Date.now();
|
||||
const storage = deps?.storage ?? getStorage();
|
||||
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
||||
|
||||
primeChrome(deps);
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-chunk-"));
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
||||
await untarDirectory(planTar, planDir);
|
||||
|
||||
// Verify the plan's hash matches what the workflow told us to render.
|
||||
// The producer's renderChunk re-checks internally (defense-in-depth),
|
||||
// but doing it here at the handler boundary lets us fail before paying
|
||||
// the Chrome-launch + render cost on a misrouted chunk. Throws a typed
|
||||
// PLAN_HASH_MISMATCH the workflow can route as non-retryable.
|
||||
verifyPlanHash(planDir, event.PlanHash);
|
||||
|
||||
const chunkOutputBase = join(
|
||||
work,
|
||||
event.Format === "png-sequence"
|
||||
? `chunk-${pad(event.ChunkIndex)}`
|
||||
: `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`,
|
||||
);
|
||||
|
||||
const result: ChunkResult = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
||||
|
||||
const chunkUri = await uploadChunkOutput(
|
||||
storage,
|
||||
result,
|
||||
event.ChunkOutputGcsPrefix,
|
||||
event.ChunkIndex,
|
||||
);
|
||||
|
||||
return {
|
||||
Action: "renderChunk",
|
||||
ChunkGcsUri: chunkUri,
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadChunkOutput(
|
||||
storage: Storage,
|
||||
result: ChunkResult,
|
||||
prefix: string,
|
||||
chunkIndex: number,
|
||||
): Promise<string> {
|
||||
const trimmed = trimTrailingSlash(prefix);
|
||||
if (result.outputKind === "file") {
|
||||
const ext = extname(result.outputPath);
|
||||
const uri = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
||||
await uploadFileToGcs(storage, result.outputPath, uri);
|
||||
return uri;
|
||||
}
|
||||
// frame-dir: upload as a tarball so a single GCS object represents the
|
||||
// chunk. Assemble's png-sequence path expects a directory per chunk; it
|
||||
// untars on its end.
|
||||
const tarball = `${result.outputPath}.tar.gz`;
|
||||
await tarDirectory(result.outputPath, tarball);
|
||||
const uri = `${trimmed}/chunks/${pad(chunkIndex)}.tar.gz`;
|
||||
await uploadFileToGcs(storage, tarball, uri, "application/gzip");
|
||||
return uri;
|
||||
}
|
||||
|
||||
// ── Assemble ────────────────────────────────────────────────────────────────
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
async function handleAssemble(
|
||||
event: AssembleEvent,
|
||||
deps?: HandlerDeps,
|
||||
): Promise<AssembleResultBody> {
|
||||
const started = Date.now();
|
||||
const storage = deps?.storage ?? getStorage();
|
||||
const primitive = deps?.primitives?.assemble ?? assemble;
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-assemble-"));
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
|
||||
await untarDirectory(planTar, planDir);
|
||||
|
||||
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
|
||||
|
||||
// Audio rides inside the plan tarball, so it's already on disk after the
|
||||
// untar above — no separate download. Fall back to a supplied AudioGcsUri
|
||||
// only for backward compatibility with an older Plan that uploaded it
|
||||
// standalone.
|
||||
let audioPath: string | null = null;
|
||||
const planAudio = join(planDir, "audio.aac");
|
||||
if (existsSync(planAudio) && statSync(planAudio).size > 0) {
|
||||
audioPath = planAudio;
|
||||
} else if (event.AudioGcsUri) {
|
||||
audioPath = planAudio;
|
||||
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
|
||||
}
|
||||
|
||||
const finalOutput =
|
||||
event.Format === "png-sequence"
|
||||
? join(work, "output-frames")
|
||||
: join(work, `output${formatExtension(event.Format)}`);
|
||||
|
||||
const result: AssembleResult = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
||||
cfr: event.Cfr === true,
|
||||
});
|
||||
|
||||
if (event.Format === "png-sequence") {
|
||||
const tarball = `${finalOutput}.tar.gz`;
|
||||
await tarDirectory(finalOutput, tarball);
|
||||
await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
|
||||
} else {
|
||||
await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
|
||||
}
|
||||
|
||||
return {
|
||||
Action: "assemble",
|
||||
OutputGcsUri: event.OutputGcsUri,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
FileSize: result.fileSize,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadChunkObjects(
|
||||
storage: Storage,
|
||||
uris: string[],
|
||||
workDir: string,
|
||||
format: DistributedFormat,
|
||||
): Promise<string[]> {
|
||||
const chunksDir = join(workDir, "chunks");
|
||||
mkdirSync(chunksDir, { recursive: true });
|
||||
// Each chunk is an independent GCS GET (+ untar for png-sequence). Run
|
||||
// them in parallel — assemble's wall-clock is otherwise dominated by
|
||||
// `Σ chunk-download-ms` instead of `max(chunk-download-ms)`. Preserve the
|
||||
// input order by writing into a pre-sized array rather than pushing as
|
||||
// each task settles.
|
||||
const local: string[] = new Array<string>(uris.length);
|
||||
await Promise.all(
|
||||
uris.map(async (uri, i) => {
|
||||
if (!uri) {
|
||||
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
||||
}
|
||||
const { key } = parseGcsUri(uri);
|
||||
const localPath = join(chunksDir, basename(key));
|
||||
await downloadGcsObjectToFile(storage, uri, localPath);
|
||||
if (format === "png-sequence") {
|
||||
const dirPath = join(chunksDir, `frames-${pad(i)}`);
|
||||
await untarDirectory(localPath, dirPath);
|
||||
local[i] = dirPath;
|
||||
} else {
|
||||
local[i] = localPath;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return local;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Collect every GCS URI that the handler will touch for a given event. */
|
||||
function getEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): string[] {
|
||||
switch (event.Action) {
|
||||
case "plan":
|
||||
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
|
||||
case "renderChunk":
|
||||
return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
|
||||
case "assemble":
|
||||
return [
|
||||
event.PlanGcsUri,
|
||||
...event.ChunkGcsUris,
|
||||
event.OutputGcsUri,
|
||||
event.AudioGcsUri,
|
||||
].filter((u): u is string => u != null);
|
||||
}
|
||||
}
|
||||
|
||||
/** Emit the "guard disabled" warning at most once per instance. */
|
||||
let warnedAllowlistDisabled = false;
|
||||
|
||||
/**
|
||||
* Verify every GCS URI in the event resolves to the configured render
|
||||
* bucket. Throws `GCS_URI_NOT_ALLOWED` (non-retryable) when a URI targets a
|
||||
* different bucket, preventing request injection from reading or writing
|
||||
* arbitrary GCS data.
|
||||
*
|
||||
* Opt-out is explicit: set `HYPERFRAMES_RENDER_BUCKET="*"` to disable the
|
||||
* guard intentionally. If the env var is simply unset (or empty), the guard
|
||||
* is disabled but a warning is logged once so the gap is visible in Cloud
|
||||
* Logging — it shouldn't silently fail open. The Terraform module always
|
||||
* wires the bucket name, so the prod path enforces.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
function validateEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): void {
|
||||
const allowedBucket = process.env.HYPERFRAMES_RENDER_BUCKET?.trim();
|
||||
if (allowedBucket === "*") return; // explicit, intentional opt-out
|
||||
if (!allowedBucket) {
|
||||
if (!warnedAllowlistDisabled) {
|
||||
warnedAllowlistDisabled = true;
|
||||
logEvent({
|
||||
event: "bucket_allowlist_disabled",
|
||||
level: "WARNING",
|
||||
message:
|
||||
"HYPERFRAMES_RENDER_BUCKET is unset — the GCS bucket-allowlist guard is DISABLED. " +
|
||||
'Set it to the render bucket name to enforce, or to "*" to opt out intentionally.',
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (const uri of getEventGcsUris(event)) {
|
||||
const { bucket } = parseGcsUri(uri);
|
||||
if (bucket !== allowedBucket) {
|
||||
const err = new Error(
|
||||
`[handler] GCS_URI_NOT_ALLOWED: URI ${JSON.stringify(uri)} targets bucket "${bucket}" but only "${allowedBucket}" is permitted`,
|
||||
);
|
||||
err.name = "GCS_URI_NOT_ALLOWED";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function pad(n: number): string {
|
||||
return n.toString().padStart(4, "0");
|
||||
}
|
||||
|
||||
function trimTrailingSlash(prefix: string): string {
|
||||
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
||||
}
|
||||
|
||||
function cleanupDir(dir: string): void {
|
||||
try {
|
||||
// Cloud Run re-uses an instance's filesystem across requests; clean up
|
||||
// aggressively so we don't leak a chunk-sized footprint between renders
|
||||
// (the writable filesystem counts against the instance's memory).
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort — leak is preferable to crashing on the success path.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the untarred planDir's `plan.json` and assert its `planHash` matches
|
||||
* what the workflow event claims. Throws on mismatch with a typed
|
||||
* `PLAN_HASH_MISMATCH` error name so the workflow's non-retryable list
|
||||
* routes it correctly. Defense-in-depth — the producer's `renderChunk` does
|
||||
* the same check internally — but performing it here lets us fail before
|
||||
* paying the Chrome-launch + per-frame capture cost on a misrouted chunk.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
function verifyPlanHash(planDir: string, expected: string): void {
|
||||
const planJsonPath = join(planDir, "plan.json");
|
||||
let parsed: { planHash?: unknown };
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8")) as { planHash?: unknown };
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const error = new Error(`PLAN_HASH_MISMATCH: failed to read ${planJsonPath}: ${msg}`);
|
||||
error.name = "PLAN_HASH_MISMATCH";
|
||||
throw error;
|
||||
}
|
||||
const actual = parsed.planHash;
|
||||
if (typeof actual !== "string" || actual !== expected) {
|
||||
const error = new Error(
|
||||
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match plan.json planHash=${String(actual)}`,
|
||||
);
|
||||
error.name = "PLAN_HASH_MISMATCH";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP shell ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Error names the workflow treats as non-retryable. A request that fails
|
||||
* with one of these is the caller's fault (bad input, misrouted chunk) and
|
||||
* retrying it just burns instance-seconds, so we map them to HTTP 400 while
|
||||
* any other failure maps to 500 (which the workflow retry policy backs off
|
||||
* and re-attempts). Keep this list in sync with the `retry` predicate in
|
||||
* `packages/gcp-cloud-run/terraform/workflow.yaml`.
|
||||
*/
|
||||
const NON_RETRYABLE_ERROR_NAMES = new Set([
|
||||
// Handler-boundary guards.
|
||||
"GCS_URI_NOT_ALLOWED",
|
||||
"PLAN_HASH_MISMATCH",
|
||||
// Producer error class names (`.name`) + their string code aliases — the
|
||||
// class sets `.name` to the class name but wraps a `code`; cover both so a
|
||||
// raw-code throw is caught too. Mirrors the AWS state machine's
|
||||
// non-retryable list.
|
||||
"FormatNotSupportedInDistributedError",
|
||||
"PlanTooLargeError",
|
||||
"RenderChunkValidationError",
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
||||
"PLAN_TOO_LARGE",
|
||||
"BROWSER_GPU_NOT_SOFTWARE",
|
||||
"FONT_FETCH_FAILED",
|
||||
"ChromeBinaryUnavailableError",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Build the Hono app. A single `POST /` endpoint dispatches on the body's
|
||||
* `Action` field — the workflow points every step (plan, each renderChunk,
|
||||
* assemble) at the same URL and varies only the body. `GET /healthz` backs
|
||||
* the Cloud Run startup/liveness probe.
|
||||
*
|
||||
* `deps` is threaded through so tests can drive the real HTTP surface with
|
||||
* an injected Storage double + mocked primitives.
|
||||
*/
|
||||
export function createApp(deps?: HandlerDeps): Hono {
|
||||
const app = new Hono();
|
||||
|
||||
app.get("/healthz", (c) => c.json({ status: "ok" }));
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
app.post("/", async (c) => {
|
||||
let body: CloudRunEvent;
|
||||
try {
|
||||
body = (await c.req.json()) as CloudRunEvent;
|
||||
} catch {
|
||||
return c.json({ error: "BAD_REQUEST", message: "request body must be JSON" }, 400);
|
||||
}
|
||||
try {
|
||||
const result = await dispatch(body, deps);
|
||||
return c.json(result, 200);
|
||||
} catch (err) {
|
||||
const name = err instanceof Error ? err.name : undefined;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const status = name && NON_RETRYABLE_ERROR_NAMES.has(name) ? 400 : 500;
|
||||
// Surface `error` (the name) as the discriminator the workflow's
|
||||
// retry predicate keys off, plus `message` for human triage.
|
||||
return c.json({ error: name ?? "RenderError", message }, status);
|
||||
}
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
/** Start the HTTP server. Cloud Run injects `PORT` (default 8080). */
|
||||
export function startServer(): void {
|
||||
const port = Number(process.env.PORT ?? 8080);
|
||||
const app = createApp();
|
||||
serve({ fetch: app.fetch, port }, (info) => {
|
||||
logEvent({ event: "server_listening", port: info.port });
|
||||
});
|
||||
}
|
||||
|
||||
// Boot when executed directly (the Dockerfile runs `node dist/server.js`),
|
||||
// but not when imported by tests or the SDK.
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
startServer();
|
||||
}
|
||||
Reference in New Issue
Block a user