chore: import upstream snapshot with attribution
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:58:35 +08:00
commit 85453da49f
4031 changed files with 710987 additions and 0 deletions
+118
View File
@@ -0,0 +1,118 @@
# HyperFrames distributed render — Cloud Run image.
#
# One container image, three roles (plan / renderChunk / assemble). Cloud
# Workflows POSTs a JSON body with an `Action` field; `dist/server.js`
# dispatches to the matching `@hyperframes/producer/distributed` primitive.
#
# Build context is the REPOSITORY ROOT (not this package dir) because the
# package depends on the `@hyperframes/producer` workspace and renders with
# the same chrome-headless-shell + fonts + ffmpeg the production renderer
# uses. The example smoke script and `hyperframes cloudrun deploy` both
# build with the repo root as context:
#
# docker build -f packages/gcp-cloud-run/Dockerfile -t <image> .
#
# NOTE: this Dockerfile only builds from a full hyperframes monorepo checkout
# — it COPYs sibling workspace packages (core/engine/producer). It is NOT
# buildable from the published npm tarball alone; install the repo (or use
# `hyperframes cloudrun deploy`, which builds from your checkout via Cloud
# Build) to produce the image.
#
# Unlike the AWS Lambda adapter there is no ZIP-size ceiling and no runtime
# Chrome decompression step — the binary lives in the image at a fixed path.
FROM node:22-bookworm-slim
# ── System dependencies (identical set to the producer's render image) ───────
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
unzip \
ffmpeg \
libgbm1 \
libnss3 \
libatk-bridge2.0-0 \
libdrm2 \
libxcomposite1 \
libxdamage1 \
libxrandr2 \
libcups2 \
libasound2 \
libpangocairo-1.0-0 \
libxshmfence1 \
libgtk-3-0 \
fonts-liberation \
fonts-noto-color-emoji \
fonts-noto-cjk \
fonts-noto-core \
fonts-noto-extra \
fonts-noto-ui-core \
fonts-freefont-ttf \
fonts-dejavu-core \
fontconfig \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean \
&& fc-cache -fv
# ── chrome-headless-shell (deterministic BeginFrame capture) ─────────────────
# Pinned to the SAME build the producer regression baselines were generated
# against so distributed renders are pixel-comparable. Bump together with the
# producer's Dockerfile.test pin and a baseline regen.
RUN npx --yes @puppeteer/browsers install chrome-headless-shell@148.0.7778.167 \
--path /opt/puppeteer \
&& CHS="$(find /opt/puppeteer/chrome-headless-shell -name chrome-headless-shell -type f | head -n1)" \
&& mkdir -p /opt/chrome \
&& ln -s "$CHS" /opt/chrome/chrome-headless-shell \
&& /opt/chrome/chrome-headless-shell --version
# The chromium.ts resolver reads this first; pointing the engine straight at
# the symlinked binary avoids the runtime cache scan.
ENV HYPERFRAMES_CHROME_PATH=/opt/chrome/chrome-headless-shell
ENV PRODUCER_HEADLESS_SHELL_PATH=/opt/chrome/chrome-headless-shell
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV CONTAINER=true
WORKDIR /app
# Install bun (the repo's package manager / build driver). Pin the version:
# the container runs `bun dist/server.js` and relies on bun's ESM `require`
# interop to load the producer bundle, so a future bun release changing that
# behaviour shouldn't silently break the next image rebuild. Bump deliberately.
RUN curl -fsSL https://bun.sh/install | bash -s "bun-v1.3.9"
ENV PATH="/root/.bun/bin:$PATH"
# Install workspace dependencies. Copy manifests first for layer caching.
COPY package.json bun.lock ./
COPY packages/core/package.json packages/core/package.json
COPY packages/engine/package.json packages/engine/package.json
COPY packages/player/package.json packages/player/package.json
COPY packages/producer/package.json packages/producer/package.json
COPY packages/cli/package.json packages/cli/package.json
COPY packages/studio/package.json packages/studio/package.json
COPY packages/shader-transitions/package.json packages/shader-transitions/package.json
COPY packages/aws-lambda/package.json packages/aws-lambda/package.json
COPY packages/gcp-cloud-run/package.json packages/gcp-cloud-run/package.json
RUN bun install --frozen-lockfile
# Copy source for the packages the render path needs.
COPY packages/core/ packages/core/
COPY packages/engine/ packages/engine/
COPY packages/producer/ packages/producer/
COPY packages/gcp-cloud-run/ packages/gcp-cloud-run/
# Build core runtime artifacts (needed by the renderer) + producer, then the
# adapter. Generate embedded font data so glyph layout matches production.
RUN bun run --filter @hyperframes/core build:hyperframes-runtime:modular \
&& (cd packages/producer && bunx tsx scripts/generate-font-data.ts) \
&& bun run --cwd packages/producer build \
&& bun run --cwd packages/gcp-cloud-run build
# Cloud Run injects PORT (default 8080). The server reads it.
ENV PORT=8080
EXPOSE 8080
WORKDIR /app/packages/gcp-cloud-run
# Run under bun, not node. The repo is bun-native and `@hyperframes/producer`'s
# bundled `distributed.js` relies on a `require` being available at runtime
# (its esbuild interop shim); bun provides one in ESM, bare `node` does not.
CMD ["bun", "dist/server.js"]
+128
View File
@@ -0,0 +1,128 @@
# @hyperframes/gcp-cloud-run
Google Cloud Run + Cloud Workflows adapter for HyperFrames distributed
rendering. The OSS render primitives (`plan``renderChunk` × N →
`assemble`) are pure functions over local file paths; this package is the
deployment, orchestration, and storage glue that runs them on Google Cloud —
the GCP counterpart to [`@hyperframes/aws-lambda`](../aws-lambda).
Two surfaces, one package:
- **Server-side handler** (`./server`) — a Cloud Run HTTP service that
dispatches `plan` / `renderChunk` / `assemble` on the request body's
`Action` field, bridging GCS ↔ the container's filesystem around each OSS
primitive. This is what the bundled `Dockerfile` runs.
- **Client-side SDK** (`./sdk`) — `renderToCloudRun`, `getRenderProgress`,
`deploySite`, `validateDistributedRenderConfig`, and `computeRenderCost`.
Call these from a Node process (CI, CLI, app backend) to drive a deployed
stack without writing GCS / Workflows boilerplate.
The package is **not** a dependency of `@hyperframes/producer`; install it
separately.
## Architecture
```
GCS bucket ←→ Cloud Run service (plan / renderChunk / assemble)
│ OIDC-authenticated http.post, one per step
Cloud Workflows (Plan → parallel RenderChunk → Assemble)
```
- **Plan** downloads the project tarball, runs `plan()`, uploads the planDir
tarball (+ audio) to GCS, and returns the chunk count.
- **RenderChunk** runs in a parallel `for` loop in the workflow, fanned out
up to the plan's chunk count. Each invocation renders one chunk and uploads
it.
- **Assemble** downloads every chunk + audio, stitches the final
deliverable, and uploads it.
Every step is a `POST` to the same Cloud Run URL with a different `Action`.
The workflow accumulates each step's small result body and returns
`{ Plan, Chunks, Assemble }` so `getRenderProgress` can read frame totals and
per-step durations on success.
## Chrome runtime
Unlike the Lambda adapter — which fights a 250 MB ZIP ceiling and
decompresses `@sparticuz/chromium` into `/tmp` at runtime — Cloud Run runs a
container image. The `Dockerfile` installs the same pinned
`chrome-headless-shell` build and font set the production renderer uses, at a
fixed path, and exports `HYPERFRAMES_CHROME_PATH`. CDP-level `BeginFrame`
works because the command lives in the protocol, not the binary. There is no
runtime decompression step and no packaging ceiling.
## Deploying
The `terraform/` module provisions everything: the GCS render bucket, the
Cloud Run service, the Cloud Workflows definition, two least-privilege
service accounts (the service reads/writes the bucket; the workflow invokes
the service), and a runaway-request alert.
```bash
# 1. Build + push the image (Cloud Build or local docker).
gcloud builds submit . \
--tag REGION-docker.pkg.dev/PROJECT/REPO/hyperframes-render:TAG
# 2. Apply the module.
terraform -chdir=node_modules/@hyperframes/gcp-cloud-run/terraform init
terraform -chdir=node_modules/@hyperframes/gcp-cloud-run/terraform apply \
-var project_id=PROJECT \
-var region=us-central1 \
-var image=REGION-docker.pkg.dev/PROJECT/REPO/hyperframes-render:TAG
```
Terraform outputs `render_bucket_name`, `service_url`, `workflow_name`, and
`region` — pass them straight into the SDK.
## Using the SDK
```ts
import { renderToCloudRun, getRenderProgress } from "@hyperframes/gcp-cloud-run/sdk";
const handle = await renderToCloudRun({
projectDir: "./my-composition",
config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
bucketName: "hyperframes-render-my-project", // from terraform output
projectId: "my-project",
location: "us-central1",
workflowId: "hyperframes-render",
serviceUrl: "https://hyperframes-render-abc.us-central1.run.app",
});
// Poll until done.
let progress = await getRenderProgress({ executionName: handle.executionName });
while (progress.status === "running") {
await new Promise((r) => setTimeout(r, 5000));
progress = await getRenderProgress({ executionName: handle.executionName });
}
console.log(progress.status, progress.outputFile, progress.costs.displayCost);
```
`deploySite` is called implicitly when you pass `projectDir`; call it
yourself to pre-upload once and reuse the `siteHandle` across many renders
(e.g. personalised template batches).
## Running tests
```bash
bun test # unit tests over an in-memory GCS double — no network
bun run typecheck
```
The live end-to-end smoke (build image → terraform apply → render a fixture
through the workflow → PSNR-compare → destroy) lives at
`examples/gcp-cloud-run/scripts/smoke.sh` and needs a GCP project with
billing enabled.
## What's still ahead
- **Mid-flight per-chunk progress.** `getRenderProgress` reports coarse
`running` progress and exact numbers on success. Reading the Cloud
Workflows step-entries API would give per-chunk progress while the render
is in flight; tracked as a follow-up.
- **Cloud Run Jobs / Firebase Functions variants.** This first version
targets Cloud Run services + Workflows (the closest analog to Lambda +
Step Functions). The same handler runs unchanged under Cloud Run Jobs;
only the orchestration trigger differs.
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env node
/**
* Build script for @hyperframes/gcp-cloud-run (public OSS package).
*
* Bundles each subpath barrel via esbuild → dist/, then emits .d.ts via tsc.
*
* Subpaths (each gets its own dist entry so adopters that import one path
* don't load the others' transitive graphs at module-load time):
*
* . (the umbrella barrel: server + sdk types re-exported)
* ./server (the Cloud Run runtime entry — what the Dockerfile runs)
* ./sdk (client-side helpers — GCS + Workflows clients only, no
* chromium/puppeteer)
*
* All production deps are kept external so consumers (and the container
* image) resolve them via their own node_modules.
*/
import { build } from "esbuild";
import { execSync } from "node:child_process";
import { mkdirSync, rmSync } from "node:fs";
rmSync("dist", { recursive: true, force: true });
mkdirSync("dist", { recursive: true });
const sharedOpts = {
bundle: true,
platform: "node",
target: "node22",
format: "esm",
minify: false,
sourcemap: true,
external: [
"@google-cloud/storage",
"@google-cloud/workflows",
"@hono/node-server",
"@hyperframes/producer",
"@hyperframes/producer/distributed",
"hono",
"puppeteer-core",
"tar",
],
};
await Promise.all([
build({ ...sharedOpts, entryPoints: ["src/index.ts"], outfile: "dist/index.js" }),
build({ ...sharedOpts, entryPoints: ["src/server.ts"], outfile: "dist/server.js" }),
build({ ...sharedOpts, entryPoints: ["src/sdk/index.ts"], outfile: "dist/sdk/index.js" }),
]);
// esbuild doesn't emit .d.ts. tsc does, with a build-only tsconfig that
// drops the workspace `paths` overrides so `@hyperframes/producer` resolves
// through node_modules to the sibling package's already-built `dist/`
// types instead of pulling its full source tree into emit (which would
// violate rootDir).
execSync("tsc -p tsconfig.build.json --emitDeclarationOnly", { stdio: "inherit" });
console.log("[Build] Complete: dist/{index,server,sdk/index}.js + .d.ts");
+62
View File
@@ -0,0 +1,62 @@
{
"name": "@hyperframes/gcp-cloud-run",
"version": "0.7.55",
"description": "Google Cloud Run + Workflows adapter for HyperFrames distributed rendering — request handler, client-side SDK, and Terraform module.",
"repository": {
"type": "git",
"url": "https://github.com/heygen-com/hyperframes",
"directory": "packages/gcp-cloud-run"
},
"files": [
"dist/",
"terraform/",
"Dockerfile",
"README.md"
],
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
},
"./server": {
"import": "./dist/server.js",
"types": "./dist/server.d.ts"
},
"./sdk": {
"import": "./dist/sdk/index.js",
"types": "./dist/sdk/index.d.ts"
}
},
"publishConfig": {
"access": "public",
"registry": "https://registry.npmjs.org/"
},
"scripts": {
"build": "node build.mjs",
"start": "bun dist/server.js",
"test": "bun test",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@google-cloud/storage": "^7.14.0",
"@google-cloud/workflows": "^4.2.0",
"@hono/node-server": "^1.13.0",
"@hyperframes/producer": "workspace:^",
"hono": "^4.6.0",
"puppeteer-core": "^25.2.1",
"tar": "^7.4.3"
},
"devDependencies": {
"@types/node": "^25.0.10",
"@types/tar": "^6.1.13",
"esbuild": "^0.25.12",
"tsx": "^4.21.0",
"typescript": "^5.7.2"
},
"engines": {
"node": ">=22"
}
}
@@ -0,0 +1,116 @@
/**
* In-memory `@google-cloud/storage` stand-in for the adapter's unit tests.
*
* Mimics just the surface the transport + deploySite use:
* storage.bucket(name).file(key) → { createReadStream, exists, getMetadata }
* storage.bucket(name).upload(localPath, { destination, contentType })
*
* Objects live in a Map keyed `bucket/key`. `upload` reads the real local
* file from disk (the handler writes real tarballs), so round-trips through
* tar/untar exercise the actual code path. Every op is recorded for
* sequence assertions.
*/
import { readFileSync, writeFileSync } from "node:fs";
import { Readable } from "node:stream";
import type { Storage } from "@google-cloud/storage";
export interface FakeGcsOp {
kind: "download" | "upload" | "exists" | "getMetadata";
uri: string;
bytes?: number;
}
export class FakeGcs {
ops: FakeGcsOp[] = [];
objects = new Map<string, Buffer>();
// Accessed only through the `Storage` cast in tests, so fallow's static
// analysis can't see the reference.
// fallow-ignore-next-line unused-class-member
bucket(bucketName: string): FakeBucket {
return new FakeBucket(this, bucketName);
}
/** Seed an object directly (e.g. a pre-built project tarball). */
seed(uri: string, bytes: Buffer): void {
this.objects.set(uri, bytes);
}
/** Seed from a local file on disk. */
seedFromFile(uri: string, localPath: string): void {
this.objects.set(uri, readFileSync(localPath));
}
}
class FakeBucket {
constructor(
private readonly gcs: FakeGcs,
private readonly bucketName: string,
) {}
get name(): string {
return this.bucketName;
}
file(key: string): FakeFile {
return new FakeFile(this.gcs, this.bucketName, key);
}
async upload(
localPath: string,
opts: { destination: string; contentType?: string },
): Promise<unknown> {
const uri = `gs://${this.bucketName}/${opts.destination}`;
const bytes = readFileSync(localPath);
this.gcs.objects.set(uri, bytes);
this.gcs.ops.push({ kind: "upload", uri, bytes: bytes.length });
return [{}];
}
}
class FakeFile {
private readonly uri: string;
constructor(
private readonly gcs: FakeGcs,
bucketName: string,
key: string,
) {
this.uri = `gs://${bucketName}/${key}`;
}
createReadStream(): Readable {
const bytes = this.gcs.objects.get(this.uri);
if (!bytes) {
const r = new Readable({ read() {} });
r.destroy(new Error(`FakeGcs: object not found: ${this.uri}`));
return r;
}
this.gcs.ops.push({ kind: "download", uri: this.uri, bytes: bytes.length });
return Readable.from([bytes]);
}
async exists(): Promise<[boolean]> {
const has = this.gcs.objects.has(this.uri);
this.gcs.ops.push({ kind: "exists", uri: this.uri });
return [has];
}
async getMetadata(): Promise<[{ size?: string | number; updated?: string }]> {
const bytes = this.gcs.objects.get(this.uri);
this.gcs.ops.push({ kind: "getMetadata", uri: this.uri });
return [{ size: bytes?.length ?? 0, updated: "2026-06-06T00:00:00.000Z" }];
}
/** Helper for tests that want to materialize an object to disk. */
writeToDisk(destPath: string): void {
const bytes = this.gcs.objects.get(this.uri);
if (!bytes) throw new Error(`FakeGcs: object not found: ${this.uri}`);
writeFileSync(destPath, bytes);
}
}
/** Cast so `deploySite({ storage: asStorage(fake) })` reads cleanly. */
export function asStorage(fake: FakeGcs): Storage {
return fake as unknown as Storage;
}
+95
View File
@@ -0,0 +1,95 @@
/**
* Cloud Run Chrome resolver.
*
* `renderChunk()` (the only primitive that needs a browser) launches Chrome
* via the engine's `BrowserManager`. Because Cloud Run runs a container
* image rather than a size-capped ZIP, the Chrome story is far simpler than
* the Lambda adapter's: the `Dockerfile` installs `chrome-headless-shell`
* (the same BeginFrame-capable build the K8s deploy uses) into the image at
* a known path and exports `HYPERFRAMES_CHROME_PATH`. There is no runtime
* decompression-into-/tmp step and no 250 MB packaging ceiling to fight.
*
* Resolution order:
* 1. `PRODUCER_HEADLESS_SHELL_PATH` — the engine's own override. If a
* caller (or the Docker image) already set it, honour it untouched.
* 2. `HYPERFRAMES_CHROME_PATH` — set by the Dockerfile to the installed
* `chrome-headless-shell` binary.
* 3. A small list of conventional install paths, as a last resort for
* images built outside our Dockerfile.
*
* Throws {@link ChromeBinaryUnavailableError} when nothing resolves, so a
* misconfigured image fails loudly at the first chunk rather than emitting
* a confusing puppeteer-core "executablePath must be specified" assertion.
*/
import { existsSync } from "node:fs";
/**
* Thrown when the Chrome binary resolver can't produce a usable path. The
* class name is the workflow's non-retryable error discriminator.
*/
export class ChromeBinaryUnavailableError extends Error {
// Read indirectly via the error envelope / Error.prototype.toString.
// fallow-ignore-next-line unused-class-member
override readonly name = "ChromeBinaryUnavailableError";
readonly resolvedPath: string | null;
constructor(resolvedPath: string | null, hint: string) {
super(`[chromium] Chrome binary unavailable: ${hint}`);
this.resolvedPath = resolvedPath;
}
}
/**
* Conventional locations a `chrome-headless-shell` (or full Chrome) binary
* may live at in a Debian/Ubuntu-based container. Checked only after the
* two env-var overrides miss.
*/
const FALLBACK_CHROME_PATHS = [
"/opt/chrome/chrome-headless-shell",
"/usr/bin/chrome-headless-shell",
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
];
/**
* Resolve the absolute path to a Chrome binary suitable for BeginFrame.
* Pure (no env mutation) so callers decide whether to export the result
* into `PRODUCER_HEADLESS_SHELL_PATH`.
*/
// fallow-ignore-next-line complexity
export function resolveChromeExecutablePath(): string {
const fromEngineOverride = process.env.PRODUCER_HEADLESS_SHELL_PATH?.trim();
if (fromEngineOverride) {
if (!existsSync(fromEngineOverride)) {
throw new ChromeBinaryUnavailableError(
fromEngineOverride,
`PRODUCER_HEADLESS_SHELL_PATH=${JSON.stringify(fromEngineOverride)} does not exist on disk.`,
);
}
return fromEngineOverride;
}
const fromImage = process.env.HYPERFRAMES_CHROME_PATH?.trim();
if (fromImage) {
if (!existsSync(fromImage)) {
throw new ChromeBinaryUnavailableError(
fromImage,
`HYPERFRAMES_CHROME_PATH=${JSON.stringify(fromImage)} does not exist on disk.`,
);
}
return fromImage;
}
for (const candidate of FALLBACK_CHROME_PATHS) {
if (existsSync(candidate)) return candidate;
}
throw new ChromeBinaryUnavailableError(
null,
"no Chrome binary found. Set HYPERFRAMES_CHROME_PATH (the Dockerfile does this) or " +
"PRODUCER_HEADLESS_SHELL_PATH to the absolute path of a chrome-headless-shell binary. " +
`Searched: ${FALLBACK_CHROME_PATHS.join(", ")}.`,
);
}
+146
View File
@@ -0,0 +1,146 @@
/**
* Request + result types for the HyperFrames distributed render handler
* running on Cloud Run.
*
* The Cloud Workflows definition in `packages/gcp-cloud-run/terraform/workflow.yaml`
* dispatches on the `Action` field of the JSON request body. Each action
* maps 1:1 onto one of the three OSS distributed primitives:
*
* "plan" → `plan(projectDir, config, planDir)` (Activity A)
* "renderChunk" → `renderChunk(planDir, chunkIndex, output)` (Activity B)
* "assemble" → `assemble(planDir, chunkPaths, audio, out)` (Activity C)
*
* All file I/O is mediated by GCS — the handler downloads inputs into a
* per-request workdir under the container's writable `/tmp`, invokes the
* primitive, uploads outputs back to GCS, and returns a small JSON payload
* that fits inside a Cloud Workflows step variable (Workflows caps a single
* step's memory; chunk results stay well under 1 KB so the orchestration
* can hold one per Map iteration).
*
* These shapes are intentionally identical to `@hyperframes/aws-lambda`'s
* `events.ts` apart from the URI scheme (`gs://` vs `s3://`): the wire
* contract is the adapter's, the primitives underneath are shared.
*/
import type {
DistributedFormat,
SerializableDistributedRenderConfig,
} from "@hyperframes/producer/distributed";
export type { SerializableDistributedRenderConfig } from "@hyperframes/producer/distributed";
/** Discriminator for the three roles the one Cloud Run image fulfills. */
export type CloudRunAction = "plan" | "renderChunk" | "assemble";
/**
* Top-level shape of any request body the handler may receive.
*
* Cloud Workflows passes the step's `body` through verbatim, but a caller
* driving the service directly (or a Workflows definition that wraps the
* payload) may nest it under `Payload` / `Input`; the handler unwraps both
* before dispatching, matching the Lambda adapter's envelope tolerance.
*/
export type CloudRunEvent =
| PlanEvent
| RenderChunkEvent
| AssembleEvent
| { Payload: CloudRunEvent }
| { Input: CloudRunEvent };
/** Activity A: produce a planDir, upload to GCS. */
export interface PlanEvent {
Action: "plan";
/** GCS URI pointing at a `tar -czf`-archived project directory (`gs://bucket/key.tar.gz`). */
ProjectGcsUri: string;
/** GCS URI prefix where the planDir tar should be uploaded (`gs://bucket/{prefix}/`). */
PlanOutputGcsPrefix: string;
/** `DistributedRenderConfig` minus runtime-only fields (logger, abortSignal). */
Config: SerializableDistributedRenderConfig;
}
/** Activity B: fetch planDir, render one chunk, upload result. */
export interface RenderChunkEvent {
Action: "renderChunk";
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
/**
* `PlanResult.planHash` from the Plan invocation. The handler verifies
* this against the untarred planDir's `plan.json` before invoking the
* producer, throwing a typed `PLAN_HASH_MISMATCH` on divergence so the
* workflow routes it as non-retryable. Defense-in-depth — the producer
* also re-checks internally.
*/
PlanHash: string;
/** 0-based chunk index this invocation should render. */
ChunkIndex: number;
/** GCS URI prefix where the chunk output should be uploaded (`gs://bucket/{prefix}/`). */
ChunkOutputGcsPrefix: string;
/** Output container format from the plan's encoder.json; drives file vs frame-dir handling. */
Format: DistributedFormat;
}
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
export interface AssembleEvent {
Action: "assemble";
/** GCS URI of the plan tar produced by a PlanEvent invocation. */
PlanGcsUri: string;
/** GCS URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
ChunkGcsUris: string[];
/** GCS URI of the planDir's `audio.aac` if the composition has audio; `null` otherwise. */
AudioGcsUri: string | null;
/** Final output GCS URI (`gs://bucket/key.mp4`). */
OutputGcsUri: string;
/** Output container format; drives file vs frame-dir handling. */
Format: DistributedFormat;
/**
* Optional exact-CFR re-encode at assemble time. When `true`, the final
* assembled video is re-encoded with `-fps_mode cfr -r <fps>` so the
* stream's `avg_frame_rate` matches the container's `r_frame_rate`
* exactly (and the file's duration is exact, not PTS-derived). Trade-off
* is ~2-5x the assemble wall-clock. mp4 only — webm / mov stream-copy
* paths already produce exact avg_frame_rate. Default `false` /
* unset preserves current `-c copy` behavior.
*/
Cfr?: boolean;
}
// ── Result types — kept small to fit Cloud Workflows step budgets ────────────
/** Result of a `plan` invocation. Carries enough to size the Map(N) state. */
export interface PlanResultBody {
Action: "plan";
PlanGcsUri: string;
PlanHash: string;
ChunkCount: number;
TotalFrames: number;
Fps: 24 | 30 | 60;
Width: number;
Height: number;
Format: DistributedFormat;
HasAudio: boolean;
AudioGcsUri: string | null;
FfmpegVersion: string;
ProducerVersion: string;
DurationMs: number;
}
/** Result of a `renderChunk` invocation. Sized ≤200 bytes. */
export interface RenderChunkResultBody {
Action: "renderChunk";
ChunkGcsUri: string;
ChunkIndex: number;
Sha256: string;
FramesEncoded: number;
DurationMs: number;
}
/** Result of an `assemble` invocation. */
export interface AssembleResultBody {
Action: "assemble";
OutputGcsUri: string;
FramesEncoded: number;
FileSize: number;
DurationMs: number;
}
export type CloudRunResult = PlanResultBody | RenderChunkResultBody | AssembleResultBody;
@@ -0,0 +1,27 @@
/**
* Map a distributed `format` to the file extension the assembled output
* should carry on disk + in GCS. Shared by `src/server.ts` (chunk +
* assemble output paths) and `src/sdk/renderToCloudRun.ts` (final
* output key construction) so the two sides agree on what an mp4
* looks like vs a png-sequence.
*/
import type { DistributedFormat } from "@hyperframes/producer/distributed";
export type { DistributedFormat } from "@hyperframes/producer/distributed";
// Closed-enum lookup table. TS enforces exhaustiveness via the
// `Record<DistributedFormat, string>` annotation — adding a format to
// `DistributedFormat` without adding the matching key here fails to
// typecheck, which is the same exhaustiveness guarantee a switch +
// `_exhaustive: never` arm provides but at lower complexity.
const FORMAT_EXTENSIONS: Record<DistributedFormat, string> = {
mp4: ".mp4",
mov: ".mov",
webm: ".webm",
"png-sequence": "",
};
export function formatExtension(format: DistributedFormat): string {
return FORMAT_EXTENSIONS[format];
}
@@ -0,0 +1,114 @@
/**
* GCS transport unit tests — URI parsing, tar round-trip, and the
* download/upload bridge over the `FakeGcs` double.
*/
import { afterEach, describe, expect, it } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
import {
downloadGcsObjectToFile,
formatGcsUri,
parseGcsUri,
tarDirectory,
untarDirectory,
uploadFileToGcs,
} from "./gcsTransport.js";
const tmpDirs: string[] = [];
function mkTmp(prefix: string): string {
const dir = mkdtempSync(join(tmpdir(), prefix));
tmpDirs.push(dir);
return dir;
}
afterEach(() => {
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
});
describe("parseGcsUri", () => {
it("splits bucket and key", () => {
expect(parseGcsUri("gs://my-bucket/path/to/object.tar.gz")).toEqual({
bucket: "my-bucket",
key: "path/to/object.tar.gz",
});
});
it("rejects non-gs URIs", () => {
expect(() => parseGcsUri("s3://b/k")).toThrow(/expected gs:\/\//);
});
it("rejects a bucket with no key", () => {
expect(() => parseGcsUri("gs://just-a-bucket")).toThrow(/missing key/);
});
it("rejects an empty bucket", () => {
expect(() => parseGcsUri("gs:///key")).toThrow(/empty bucket or key/);
});
it("round-trips through formatGcsUri", () => {
const uri = "gs://b/some/key";
expect(formatGcsUri(parseGcsUri(uri))).toBe(uri);
});
});
describe("tarDirectory / untarDirectory", () => {
it("round-trips a directory tree", async () => {
const src = mkTmp("hf-tar-src-");
mkdirSync(join(src, "nested"), { recursive: true });
writeFileSync(join(src, "index.html"), "<html>hi</html>");
writeFileSync(join(src, "nested", "data.json"), '{"a":1}');
const work = mkTmp("hf-tar-work-");
const tarball = join(work, "out.tar.gz");
await tarDirectory(src, tarball);
expect(existsSync(tarball)).toBe(true);
const dest = join(work, "extracted");
await untarDirectory(tarball, dest);
expect(readFileSync(join(dest, "index.html"), "utf8")).toBe("<html>hi</html>");
expect(readFileSync(join(dest, "nested", "data.json"), "utf8")).toBe('{"a":1}');
});
it("untar wipes a stale destination first", async () => {
const src = mkTmp("hf-tar-src2-");
writeFileSync(join(src, "keep.txt"), "new");
const work = mkTmp("hf-tar-work2-");
const tarball = join(work, "out.tar.gz");
await tarDirectory(src, tarball);
const dest = join(work, "extracted");
mkdirSync(dest, { recursive: true });
writeFileSync(join(dest, "stale.txt"), "should be gone");
await untarDirectory(tarball, dest);
expect(existsSync(join(dest, "stale.txt"))).toBe(false);
expect(readFileSync(join(dest, "keep.txt"), "utf8")).toBe("new");
});
});
describe("download/upload bridge", () => {
it("uploads a local file then downloads identical bytes", async () => {
const gcs = new FakeGcs();
const work = mkTmp("hf-dl-");
const srcFile = join(work, "src.bin");
writeFileSync(srcFile, Buffer.from("hello gcs"));
const uri = "gs://bucket/obj.bin";
await uploadFileToGcs(asStorage(gcs), srcFile, uri, "application/octet-stream");
const dest = join(work, "dl.bin");
await downloadGcsObjectToFile(asStorage(gcs), uri, dest);
expect(readFileSync(dest, "utf8")).toBe("hello gcs");
expect(gcs.ops.map((o) => o.kind)).toEqual(["upload", "download"]);
});
it("upload throws when the source file is missing", async () => {
const gcs = new FakeGcs();
await expect(uploadFileToGcs(asStorage(gcs), "/no/such/file", "gs://b/k")).rejects.toThrow(
/upload source missing/,
);
});
});
+130
View File
@@ -0,0 +1,130 @@
/**
* Thin GCS transport for the Cloud Run handler.
*
* The OSS distributed primitives are pure functions over local file paths;
* the handler bridges GCS ↔ the container's writable `/tmp` filesystem on
* each request. Functions here are intentionally narrow: parse a URI,
* download an object to a local path, upload a path, tar-pack a planDir,
* tar-extract a planDir back out.
*
* Tar (not zip) for planDir transit:
* - planDirs contain symlinks (the extract stage materializes them but
* the compiled/ subtree may include linked assets); tar preserves them,
* zip does not.
* - We use the `tar` npm package (pure JS over `node:zlib`) so the
* archive format doesn't depend on a system `tar`/`unzip` being present
* in the container image.
*
* Apart from the `gs://` scheme and the `@google-cloud/storage` client this
* is the same shape as `@hyperframes/aws-lambda`'s `s3Transport.ts`.
*/
import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
import { dirname } from "node:path";
import { pipeline } from "node:stream/promises";
import type { Storage } from "@google-cloud/storage";
import * as tar from "tar";
/** Parsed `gs://bucket/key` URI. */
export interface GcsLocation {
bucket: string;
key: string;
}
/** Parse `gs://bucket/key/path` → `{ bucket, key }`. Throws on malformed input. */
// fallow-ignore-next-line complexity
export function parseGcsUri(uri: string): GcsLocation {
if (!uri.startsWith("gs://")) {
throw new Error(`[gcsTransport] expected gs:// URI, got: ${JSON.stringify(uri)}`);
}
const rest = uri.slice("gs://".length);
const slash = rest.indexOf("/");
if (slash === -1) {
throw new Error(`[gcsTransport] missing key in gs URI: ${JSON.stringify(uri)}`);
}
const bucket = rest.slice(0, slash);
const key = rest.slice(slash + 1);
if (!bucket || !key) {
throw new Error(`[gcsTransport] empty bucket or key in gs URI: ${JSON.stringify(uri)}`);
}
return { bucket, key };
}
/** Build `gs://bucket/key` from a location. */
export function formatGcsUri(loc: GcsLocation): string {
return `gs://${loc.bucket}/${loc.key}`;
}
/** Stream a GCS object to a local file path. */
export async function downloadGcsObjectToFile(
storage: Storage,
uri: string,
destPath: string,
): Promise<void> {
const { bucket, key } = parseGcsUri(uri);
mkdirSync(dirname(destPath), { recursive: true });
const file = storage.bucket(bucket).file(key);
// `createReadStream` streams the object body; piping into a write stream
// keeps memory flat for large plan tarballs / chunk files rather than
// buffering the whole object the way `file.download()` would.
await pipeline(file.createReadStream(), createWriteStream(destPath));
}
/**
* Upload a local file's contents to a GCS URI using a resumable upload.
* GCS objects have no practical size ceiling for the artifacts this adapter
* handles (plan tarballs ≤ 2 GB, chunks ≤ a few hundred MB), so a single
* upload call works for every case.
*/
export async function uploadFileToGcs(
storage: Storage,
localPath: string,
uri: string,
contentType?: string,
): Promise<void> {
if (!existsSync(localPath)) {
throw new Error(`[gcsTransport] upload source missing: ${localPath}`);
}
const { bucket, key } = parseGcsUri(uri);
await storage.bucket(bucket).upload(localPath, {
destination: key,
// `resumable: false` (simple upload) is faster for the small-to-medium
// objects this adapter moves and avoids the extra round-trip a resumable
// session start costs; GCS recommends resumable only past ~8 MB but our
// chunks are reliably above that, so let the client pick by default.
contentType,
});
}
/**
* Pack a directory into a `.tar.gz` at `destTarball`. Uses the `tar` npm
* package (pure JS over `node:zlib`) rather than spawning a system tar
* binary so the archive format is independent of the container's userland.
*/
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
throw new Error(`[gcsTransport] tar source must be an existing directory: ${sourceDir}`);
}
mkdirSync(dirname(destTarball), { recursive: true });
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
}
/**
* Extract a `.tar.gz` produced by {@link tarDirectory} into `destDir`.
* The directory is created (or cleared) before extraction so a retried
* request doesn't observe stale files from a prior run on the same warm
* container instance.
*/
export async function untarDirectory(tarballPath: string, destDir: string): Promise<void> {
if (!existsSync(tarballPath)) {
throw new Error(`[gcsTransport] tarball missing: ${tarballPath}`);
}
// Wipe target so a warm container instance's prior planDir doesn't bleed
// into the new request. Cloud Run re-uses the instance filesystem across
// requests served by the same instance.
if (existsSync(destDir)) {
rmSync(destDir, { recursive: true, force: true });
}
mkdirSync(destDir, { recursive: true });
await tar.extract({ file: tarballPath, cwd: destDir });
}
+64
View File
@@ -0,0 +1,64 @@
/**
* `@hyperframes/gcp-cloud-run` — Google Cloud Run + Workflows adapter for
* the HyperFrames distributed render pipeline.
*
* Two surfaces, one package:
*
* - **Server-side handler.** `dispatch`, `createApp`, the request/result
* types, Chrome resolution, and GCS transport. These power the Cloud Run
* service built from the package `Dockerfile`.
* - **Client-side SDK.** `renderToCloudRun`, `getRenderProgress`,
* `deploySite`, `validateDistributedRenderConfig`, and `computeRenderCost`.
* Adopters call these from their Node process (CI scripts, CLIs) to drive
* a deployed stack without writing GCS / Workflows boilerplate.
*
* The Terraform module that provisions the bucket + service + workflow lives
* under `terraform/` in the published package; see the README. The package
* is NOT a dependency of `@hyperframes/producer`; consumers install it
* separately.
*/
export { createApp, dispatch, type HandlerDeps, startServer, unwrapEvent } from "./server.js";
export {
type AssembleEvent,
type AssembleResultBody,
type CloudRunAction,
type CloudRunEvent,
type CloudRunResult,
type PlanEvent,
type PlanResultBody,
type RenderChunkEvent,
type RenderChunkResultBody,
type SerializableDistributedRenderConfig,
} from "./events.js";
export { ChromeBinaryUnavailableError, resolveChromeExecutablePath } from "./chromium.js";
export {
downloadGcsObjectToFile,
formatGcsUri,
type GcsLocation,
parseGcsUri,
tarDirectory,
untarDirectory,
uploadFileToGcs,
} from "./gcsTransport.js";
// ── Client-side SDK ─────────────────────────────────────────────────────────
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./sdk/deploySite.js";
export {
renderToCloudRun,
type RenderHandle,
type RenderToCloudRunOptions,
} from "./sdk/renderToCloudRun.js";
export {
getRenderProgress,
type GetRenderProgressOptions,
type RenderError,
type RenderProgress,
type RenderStatus,
} from "./sdk/getRenderProgress.js";
export {
type BilledCloudRunInvocation,
computeRenderCost,
type RenderCost,
} from "./sdk/costAccounting.js";
export { InvalidConfigError, validateDistributedRenderConfig } from "./sdk/validateConfig.js";
@@ -0,0 +1,35 @@
/**
* `computeRenderCost` unit tests — Cloud Run vCPU/GiB-second + Workflows
* step math.
*/
import { describe, expect, it } from "bun:test";
import { type BilledCloudRunInvocation, computeRenderCost } from "./costAccounting.js";
describe("computeRenderCost", () => {
it("returns zero for no invocations", () => {
const cost = computeRenderCost([], 0);
expect(cost.accruedSoFarUsd).toBe(0);
expect(cost.displayCost).toBe("$0.0000");
});
it("sums vCPU + memory seconds plus per-request and step charges", () => {
const invs: BilledCloudRunInvocation[] = [
{ durationMs: 10_000, vcpu: 4, memoryGib: 16, estimated: false },
{ durationMs: 10_000, vcpu: 4, memoryGib: 16, estimated: false },
];
const cost = computeRenderCost(invs, 6);
// 2 × 10s: vCPU 80 vcpu-s × 0.000024 = 0.00192; mem 320 GiB-s × 0.0000025 = 0.0008;
// requests 2 × 4e-7 ≈ 0 → raw 0.0027208, rounded to 4 dp = 0.0027.
// workflows 6 × 1e-5 = 0.00006 → rounds up to 0.0001 at 4 dp.
expect(cost.breakdown.cloudRunUsd).toBeCloseTo(0.0027, 4);
expect(cost.breakdown.workflowsUsd).toBeCloseTo(0.0001, 4);
expect(cost.accruedSoFarUsd).toBeGreaterThan(0);
expect(cost.breakdown.gcsEstimate).toBe("not-included");
});
it("flags estimated when any invocation was estimated", () => {
const cost = computeRenderCost([{ durationMs: 0, vcpu: 4, memoryGib: 16, estimated: true }], 4);
expect(cost.breakdown.estimated).toBe(true);
});
});
@@ -0,0 +1,113 @@
/**
* Per-render cost accounting for {@link getRenderProgress}.
*
* Google bills the render service two ways:
*
* - **Cloud Run** by **vCPU-seconds** and **GiB-seconds** of request
* processing time, plus a flat per-request charge. Each handler
* invocation returns its own `DurationMs` in the result body, so the
* progress reader can recover billed time per step without a separate
* Cloud Monitoring query — multiply by the service's configured vCPU /
* memory to get the resource-seconds.
* - **Cloud Workflows** by **steps executed**. The orchestration is a
* fixed shape (Plan + N×RenderChunk + Assemble + a handful of control
* steps), so the step count scales with chunk count.
*
* The math is documented inline so the constants stay close to the pricing
* source they came from. Cost is **best-effort**: GCP pricing varies by
* region + committed-use discounts; we use on-demand `us-central1` (Tier 1)
* rates as of 2026-06 and label the result `displayCost` so callers see the
* dollar value but downstream automation can also read the raw number.
*/
/** Cloud Run request-based billing, us-central1 Tier 1: USD per vCPU-second. */
const CLOUD_RUN_USD_PER_VCPU_SECOND = 0.000024;
/** Cloud Run request-based billing, us-central1 Tier 1: USD per GiB-second. */
const CLOUD_RUN_USD_PER_GIB_SECOND = 0.0000025;
/** Cloud Run: USD per request ($0.40 per million). */
const CLOUD_RUN_USD_PER_REQUEST = 0.0000004;
/** Cloud Workflows: USD per internal step ($0.01 per 1,000, after a free tier). */
const WORKFLOWS_USD_PER_STEP = 0.00001;
/** Per-invocation billed slice the cost calc cares about. */
export interface BilledCloudRunInvocation {
/** Wall-clock the handler reported via `DurationMs` in its result body. */
durationMs: number;
/** vCPU the Cloud Run service was configured with at invocation time. */
vcpu: number;
/** Memory in GiB the Cloud Run service was configured with. */
memoryGib: number;
/** `true` if the duration was inferred (step result missing) rather than read from the handler payload. */
estimated: boolean;
}
/**
* Result of {@link computeRenderCost}.
*
* NOTE: `displayCost` / `accruedSoFarUsd` cover Cloud Run compute + Cloud
* Workflows steps only. They EXCLUDE GCS storage + network egress for the
* plan tarball (which can be ~100 MB), chunk artifacts, and the final output
* — see `breakdown.gcsEstimate`. Treat the figure as a compute-cost floor,
* not the authoritative total bill.
*/
export interface RenderCost {
/** USD accrued to date (Cloud Run + Workflows only; excludes GCS — see note above). */
accruedSoFarUsd: number;
/** Human-readable USD string, e.g. `"$0.0214"`. Excludes GCS storage/egress. */
displayCost: string;
breakdown: {
cloudRunUsd: number;
workflowsUsd: number;
/** GCS transfer + storage cost varies by tier; we don't try to compute it here. */
gcsEstimate: "not-included";
/** `true` if any invocation fell back to estimated billing. */
estimated: boolean;
};
}
/**
* Sum Cloud Run vCPU-seconds + GiB-seconds + per-request charges and Cloud
* Workflows steps into an aggregate USD figure.
*
* `workflowSteps` is the count of Workflows steps executed so far — Plan
* (1) + RenderChunk (chunkCount) + Assemble (1) + the control steps
* (BuildChunkList, AssertChunkCount, …). Pass the count the progress reader
* derived from the execution; a rough constant overhead is fine since the
* step charge is a rounding error next to Cloud Run compute.
*/
export function computeRenderCost(
invocations: BilledCloudRunInvocation[],
workflowSteps: number,
): RenderCost {
let cloudRunUsd = 0;
let anyEstimated = false;
for (const inv of invocations) {
const seconds = inv.durationMs / 1000;
cloudRunUsd += seconds * inv.vcpu * CLOUD_RUN_USD_PER_VCPU_SECOND;
cloudRunUsd += seconds * inv.memoryGib * CLOUD_RUN_USD_PER_GIB_SECOND;
cloudRunUsd += CLOUD_RUN_USD_PER_REQUEST;
if (inv.estimated) anyEstimated = true;
}
const workflowsUsd = workflowSteps * WORKFLOWS_USD_PER_STEP;
const accruedSoFarUsd = roundUsd(cloudRunUsd + workflowsUsd);
return {
accruedSoFarUsd,
displayCost: formatUsd(accruedSoFarUsd),
breakdown: {
cloudRunUsd: roundUsd(cloudRunUsd),
workflowsUsd: roundUsd(workflowsUsd),
gcsEstimate: "not-included",
estimated: anyEstimated,
},
};
}
function roundUsd(usd: number): number {
// Four decimal places — enough resolution for per-chunk granularity.
// Anything finer is noise vs GCP's own rounding.
return Math.round(usd * 10_000) / 10_000;
}
function formatUsd(usd: number): string {
return `$${usd.toFixed(4)}`;
}
@@ -0,0 +1,86 @@
/**
* `deploySite` unit tests — content-addressed siteId, existence
* short-circuit, and the upload path over `FakeGcs`.
*/
import { afterEach, describe, expect, it } from "bun:test";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { asStorage, FakeGcs } from "../__fixtures__/fakeGcs.js";
import { deploySite } from "./deploySite.js";
const tmpDirs: string[] = [];
function mkProject(content: string): string {
const dir = mkdtempSync(join(tmpdir(), "hf-site-"));
tmpDirs.push(dir);
writeFileSync(join(dir, "index.html"), content);
return dir;
}
afterEach(() => {
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
});
describe("deploySite", () => {
it("uploads and returns a content-addressed handle", async () => {
const gcs = new FakeGcs();
const dir = mkProject("<html>v1</html>");
const handle = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
expect(handle.uploaded).toBe(true);
expect(handle.projectGcsUri).toBe(`gs://b/sites/${handle.siteId}/project.tar.gz`);
expect(gcs.objects.has(handle.projectGcsUri)).toBe(true);
});
it("produces a stable siteId for identical content", async () => {
const a = await deploySite({
projectDir: mkProject("<html>same</html>"),
bucketName: "b",
storage: asStorage(new FakeGcs()),
});
const b = await deploySite({
projectDir: mkProject("<html>same</html>"),
bucketName: "b",
storage: asStorage(new FakeGcs()),
});
expect(a.siteId).toBe(b.siteId);
});
it("produces different siteIds for different content", async () => {
const a = await deploySite({
projectDir: mkProject("<html>one</html>"),
bucketName: "b",
storage: asStorage(new FakeGcs()),
});
const b = await deploySite({
projectDir: mkProject("<html>two</html>"),
bucketName: "b",
storage: asStorage(new FakeGcs()),
});
expect(a.siteId).not.toBe(b.siteId);
});
it("short-circuits the upload when the object already exists", async () => {
const gcs = new FakeGcs();
const dir = mkProject("<html>cache</html>");
const first = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
expect(first.uploaded).toBe(true);
const second = await deploySite({ projectDir: dir, bucketName: "b", storage: asStorage(gcs) });
expect(second.uploaded).toBe(false);
expect(second.siteId).toBe(first.siteId);
// Only one upload op total.
expect(gcs.ops.filter((o) => o.kind === "upload").length).toBe(1);
});
it("honours an explicit siteId override", async () => {
const gcs = new FakeGcs();
const handle = await deploySite({
projectDir: mkProject("<html></html>"),
bucketName: "b",
siteId: "my-git-sha",
storage: asStorage(gcs),
});
expect(handle.siteId).toBe("my-git-sha");
expect(handle.projectGcsUri).toBe("gs://b/sites/my-git-sha/project.tar.gz");
});
});
@@ -0,0 +1,130 @@
/**
* `deploySite` — upload a project directory to GCS once per content hash
* and return a reusable handle.
*
* `renderToCloudRun` calls this implicitly when no `siteHandle` is passed,
* but exposing it as a standalone verb lets adopters bundle a project ahead
* of time and reuse the handle across many renders without re-tarring the
* project tree on every call.
*
* The handle is **content-addressed**: `siteId` is derived from a SHA-256
* over the project files. Two `deploySite` calls on an unchanged tree
* produce the same `siteId` and short-circuit the upload after a single
* existence check.
*/
import { mkdtempSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Storage } from "@google-cloud/storage";
import { hashProjectDir } from "@hyperframes/producer/distributed";
import { formatGcsUri, tarDirectory, uploadFileToGcs } from "../gcsTransport.js";
/** Options for {@link deploySite}. */
export interface DeploySiteOptions {
/** Local project directory containing `index.html` (and any composition assets). */
projectDir: string;
/** GCS bucket the Terraform module provisioned. */
bucketName: string;
/**
* Override the content-addressed site id. Useful when the caller has a
* stable external identifier they want to use (e.g. a git SHA); if unset,
* the hash of the project tree picks it.
*/
siteId?: string;
/** Injection seam for tests. Production callers leave unset. */
storage?: Storage;
}
/** Stable handle returned by {@link deploySite}. Pass back to {@link renderToCloudRun}. */
export interface SiteHandle {
/** Content-addressed (or caller-supplied) identifier; stable across re-uploads of the same tree. */
siteId: string;
/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectGcsUri`. */
bucketName: string;
/** Full `gs://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToCloudRun`. */
projectGcsUri: string;
/** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */
bytes: number;
/** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */
uploadedAt: string;
/** `false` if the object already existed and we skipped the upload. */
uploaded: boolean;
}
/**
* Upload `projectDir` to `gs://bucketName/sites/<siteId>/project.tar.gz`.
*
* Short-circuits when an object with the same key already exists in the
* bucket — `siteId` derives from the project's content hash, so the same
* bytes produce the same key, and re-uploading would be redundant.
*/
// fallow-ignore-next-line complexity
export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
if (!statSync(opts.projectDir).isDirectory()) {
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
}
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
const key = `sites/${siteId}/project.tar.gz`;
const projectGcsUri = formatGcsUri({ bucket: opts.bucketName, key });
const storage = opts.storage ?? new Storage();
const file = storage.bucket(opts.bucketName).file(key);
// Existence short-circuit. Adopters re-rendering the same project on a
// tight inner loop (CI smoke, demo flows) save the tar+gzip+upload pass
// on every iteration.
const existing = await headObject(file);
if (existing) {
return {
siteId,
bucketName: opts.bucketName,
projectGcsUri,
bytes: existing.bytes,
uploadedAt: existing.lastModified,
uploaded: false,
};
}
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
try {
const tarball = join(workdir, "project.tar.gz");
await tarDirectory(opts.projectDir, tarball);
const size = statSync(tarball).size;
await uploadFileToGcs(storage, tarball, projectGcsUri, "application/gzip");
return {
siteId,
bucketName: opts.bucketName,
projectGcsUri,
bytes: size,
uploadedAt: new Date().toISOString(),
uploaded: true,
};
} finally {
rmSync(workdir, { recursive: true, force: true });
}
}
/**
* Narrow surface of the `@google-cloud/storage` `File` this module uses —
* lets the test double implement just `exists()` + `getMetadata()` without
* pulling the full client type.
*/
interface FileLike {
exists(): Promise<[boolean, ...unknown[]]>;
getMetadata(): Promise<[{ size?: string | number; updated?: string }, ...unknown[]]>;
}
// fallow-ignore-next-line complexity
async function headObject(file: FileLike): Promise<{ bytes: number; lastModified: string } | null> {
const [exists] = await file.exists();
if (!exists) return null;
const [meta] = await file.getMetadata();
const sizeRaw = meta.size;
const bytes =
typeof sizeRaw === "string" ? Number(sizeRaw) : typeof sizeRaw === "number" ? sizeRaw : 0;
return {
bytes: Number.isFinite(bytes) ? bytes : 0,
lastModified: meta.updated ?? new Date().toISOString(),
};
}
@@ -0,0 +1,108 @@
/**
* `getRenderProgress` unit tests — state mapping + parsing the accumulated
* workflow result into frame totals, output file, and cost.
*/
import { describe, expect, it } from "bun:test";
import {
type ExecutionRecord,
type ExecutionsGetClientLike,
getRenderProgress,
} from "./getRenderProgress.js";
function fakeExecutions(record: ExecutionRecord): ExecutionsGetClientLike {
return {
async getExecution(_req: { name: string }) {
return [record] as [ExecutionRecord];
},
};
}
const accumulated = JSON.stringify({
Plan: { TotalFrames: 90, DurationMs: 4000 },
Chunks: [
{ FramesEncoded: 30, DurationMs: 8000 },
{ FramesEncoded: 30, DurationMs: 8000 },
{ FramesEncoded: 30, DurationMs: 8000 },
],
Assemble: { OutputGcsUri: "gs://b/renders/r1/output.mp4", FileSize: 123456, DurationMs: 3000 },
});
describe("getRenderProgress", () => {
it("reports running with no frame data while ACTIVE", async () => {
const p = await getRenderProgress({
executionName: "x",
executions: fakeExecutions({ state: "ACTIVE", startTime: { seconds: 1700000000 } }),
});
expect(p.status).toBe("running");
expect(p.overallProgress).toBe(0);
expect(p.totalFrames).toBeNull();
expect(p.fatalErrorEncountered).toBe(false);
});
it("reports succeeded with parsed frames + cost", async () => {
const p = await getRenderProgress({
executionName: "x",
vcpu: 4,
memoryGib: 16,
executions: fakeExecutions({
state: "SUCCEEDED",
result: accumulated,
startTime: { seconds: 1700000000 },
endTime: { seconds: 1700000031 },
}),
});
expect(p.status).toBe("succeeded");
expect(p.overallProgress).toBe(1);
expect(p.totalFrames).toBe(90);
expect(p.framesRendered).toBe(90);
expect(p.invocationsObserved).toBe(5); // plan + 3 chunks + assemble
expect(p.outputFile).toEqual({ gcsUri: "gs://b/renders/r1/output.mp4", bytes: 123456 });
expect(p.costs.accruedSoFarUsd).toBeGreaterThan(0);
expect(p.costs.breakdown.estimated).toBe(false);
});
it("maps FAILED to a fatal error and surfaces the error payload", async () => {
const p = await getRenderProgress({
executionName: "x",
executions: fakeExecutions({
state: "FAILED",
error: { payload: "boom", context: "renderChunk" },
}),
});
expect(p.status).toBe("failed");
expect(p.fatalErrorEncountered).toBe(true);
expect(p.errors[0]?.cause).toBe("boom");
expect(p.errors[0]?.state).toBe("renderChunk");
});
it("extracts the handler error name from a wrapped http failure payload", async () => {
// Workflows wraps an http step failure as { code, message, body }, where
// body is the handler's JSON { error, message }.
const payload = JSON.stringify({
code: 400,
message: "HTTP server responded with error code 400",
body: JSON.stringify({ error: "PLAN_HASH_MISMATCH", message: "mismatch" }),
});
const p = await getRenderProgress({
executionName: "x",
executions: fakeExecutions({ state: "FAILED", error: { payload, context: "renderChunk" } }),
});
expect(p.errors[0]?.error).toBe("PLAN_HASH_MISMATCH");
});
it("maps CANCELLED", async () => {
const p = await getRenderProgress({
executionName: "x",
executions: fakeExecutions({ state: "CANCELLED" }),
});
expect(p.status).toBe("cancelled");
expect(p.fatalErrorEncountered).toBe(true);
});
it("requires an executionName", async () => {
await expect(
getRenderProgress({ executionName: "", executions: fakeExecutions({}) }),
).rejects.toThrow(/executionName is required/);
});
});
@@ -0,0 +1,268 @@
/**
* `getRenderProgress` — read-only progress + cost snapshot for a single
* render started by {@link renderToCloudRun}.
*
* Pulls one `GetExecution` per call. Cloud Workflows does not surface
* per-step payloads through the basic Executions API the way Step Functions
* exposes its history, so this reader takes a different tack than the AWS
* adapter: the workflow definition **accumulates** each step's result body
* (Plan + every RenderChunk + Assemble) and returns them as one structured
* object. On success we parse that object for frame totals, the output
* file, and per-step `DurationMs` (which the handler stamps into every
* result), then compute cost against the service's configured vCPU/memory.
*
* Progress is therefore coarse while the execution is ACTIVE (we report
* `running` with `overallProgress = 0`) and exact once it SUCCEEDS
* (`overallProgress = 1`, real frame + cost numbers). Mid-flight per-chunk
* progress would require the Workflows step-entries API; that's a tracked
* follow-up, not part of the first version.
*/
import {
type BilledCloudRunInvocation,
computeRenderCost,
type RenderCost,
} from "./costAccounting.js";
/** Normalised render status. Maps from Cloud Workflows execution states. */
export type RenderStatus = "running" | "succeeded" | "failed" | "cancelled" | "unknown";
/** One error surfaced by the execution. */
export interface RenderError {
/** Step the failure surfaced in, when recoverable from the error context; else `<execution>`. */
state: string;
/** Error class / type. */
error: string;
/** Cause string (often a stringified JSON payload from the handler). */
cause: string;
}
/** Snapshot of a single render's progress + cost + errors at one point in time. */
export interface RenderProgress {
status: RenderStatus;
/** `[0, 1]`; coarse while running, exact on success. */
overallProgress: number;
framesRendered: number;
/** `null` until the execution succeeds and the accumulated plan result is read. */
totalFrames: number | null;
/** Cloud Run invocations the workflow scheduled (Plan + chunks + Assemble), when known. */
invocationsObserved: number;
costs: RenderCost;
/** Final output object if Assemble succeeded; `null` otherwise. */
outputFile: { gcsUri: string; bytes: number | null } | null;
errors: RenderError[];
/** `true` once the execution has terminated in a non-success state. */
fatalErrorEncountered: boolean;
startedAt: string;
endedAt: string | null;
}
/** Protobuf Timestamp shape the gapic client returns for start/end times. */
interface ProtoTimestamp {
seconds?: number | string | null;
nanos?: number | null;
}
/** Subset of a Cloud Workflows Execution this reader consumes. */
export interface ExecutionRecord {
name?: string | null;
state?: string | null;
result?: string | null;
error?: { payload?: string | null; context?: string | null } | null;
startTime?: ProtoTimestamp | string | null;
endTime?: ProtoTimestamp | string | null;
}
/** Minimal surface of `@google-cloud/workflows`' `ExecutionsClient` for reads. */
export interface ExecutionsGetClientLike {
getExecution(req: { name: string }): Promise<[ExecutionRecord, ...unknown[]]>;
}
/** Options for {@link getRenderProgress}. */
export interface GetRenderProgressOptions {
/** Server-assigned execution resource name from a {@link renderToCloudRun} call. */
executionName: string;
/** vCPU the Cloud Run service is configured with (for cost). Default 4. */
vcpu?: number;
/** Memory in GiB the Cloud Run service is configured with (for cost). Default 16. */
memoryGib?: number;
/** Test injection seam — production callers leave unset. */
executions?: ExecutionsGetClientLike;
}
const DEFAULT_VCPU = 4;
const DEFAULT_MEMORY_GIB = 16;
/** Result body the handler returns for each action; the workflow accumulates these. */
interface AccumulatedResult {
Plan?: { TotalFrames?: number; DurationMs?: number } | null;
Chunks?: Array<{ FramesEncoded?: number; DurationMs?: number } | null> | null;
Assemble?: {
OutputGcsUri?: string;
FileSize?: number;
FramesEncoded?: number;
DurationMs?: number;
} | null;
}
/** Pull a current progress snapshot for one render. */
// fallow-ignore-next-line complexity
export async function getRenderProgress(opts: GetRenderProgressOptions): Promise<RenderProgress> {
if (!opts.executionName) {
throw new Error("[getRenderProgress] executionName is required");
}
const executions = opts.executions ?? (await defaultExecutionsClient());
const vcpu = opts.vcpu ?? DEFAULT_VCPU;
const memoryGib = opts.memoryGib ?? DEFAULT_MEMORY_GIB;
const [execution] = await executions.getExecution({ name: opts.executionName });
const status = mapState(execution.state);
const startedAt = toIso(execution.startTime) ?? new Date(0).toISOString();
const endedAt = toIso(execution.endTime);
const errors: RenderError[] = [];
if (execution.error) {
errors.push({
state: execution.error.context ?? "<execution>",
error: extractErrorName(execution.error.payload) ?? "ExecutionError",
cause: execution.error.payload ?? "",
});
}
// Default snapshot: running / unknown — no frame or cost data until the
// accumulated result is available on success.
if (status !== "succeeded") {
return {
status,
overallProgress: 0,
framesRendered: 0,
totalFrames: null,
invocationsObserved: 0,
costs: computeRenderCost([], 0),
outputFile: null,
errors,
fatalErrorEncountered: status === "failed" || status === "cancelled",
startedAt,
endedAt,
};
}
const acc = parseAccumulated(execution.result);
const chunks = acc.Chunks?.filter((c): c is NonNullable<typeof c> => c != null) ?? [];
const framesRendered = chunks.reduce((sum, c) => sum + (c.FramesEncoded ?? 0), 0);
const totalFrames = typeof acc.Plan?.TotalFrames === "number" ? acc.Plan.TotalFrames : null;
const invocations: BilledCloudRunInvocation[] = [];
const pushInv = (durationMs: number | undefined): void => {
invocations.push({
durationMs: typeof durationMs === "number" ? durationMs : 0,
vcpu,
memoryGib,
estimated: typeof durationMs !== "number",
});
};
if (acc.Plan) pushInv(acc.Plan.DurationMs);
for (const c of chunks) pushInv(c.DurationMs);
if (acc.Assemble) pushInv(acc.Assemble.DurationMs);
// Workflow step count: Plan + N chunks + Assemble + a small constant of
// control steps (BuildChunkList, AssertChunkCount, the map scaffold).
const workflowSteps = invocations.length + 4;
const costs = computeRenderCost(invocations, workflowSteps);
const outputGcsUri = acc.Assemble?.OutputGcsUri;
const outputFile = outputGcsUri
? {
gcsUri: outputGcsUri,
bytes: typeof acc.Assemble?.FileSize === "number" ? acc.Assemble.FileSize : null,
}
: null;
return {
status,
overallProgress: 1,
framesRendered,
totalFrames,
invocationsObserved: invocations.length,
costs,
outputFile,
errors,
fatalErrorEncountered: false,
startedAt,
endedAt,
};
}
// fallow-ignore-next-line complexity
function mapState(state: string | null | undefined): RenderStatus {
switch (state) {
case "ACTIVE":
case "QUEUED":
return "running";
case "SUCCEEDED":
return "succeeded";
case "FAILED":
case "UNAVAILABLE":
return "failed";
case "CANCELLED":
return "cancelled";
default:
return "unknown";
}
}
// fallow-ignore-next-line complexity
function parseAccumulated(result: string | null | undefined): AccumulatedResult {
if (!result) return {};
try {
const parsed = JSON.parse(result) as unknown;
if (parsed && typeof parsed === "object") return parsed as AccumulatedResult;
} catch {
// Non-JSON result — treat as empty so cost/frames degrade to zero
// rather than throwing on a snapshot read.
}
return {};
}
/**
* Best-effort pull of the handler's error name out of a Workflows failure
* payload. On an http step failure, Workflows wraps the response as
* `{ code, message, body, ... }` where `body` is the handler's JSON
* `{ error, message }`. We dig out `error` (the typed name like
* `PLAN_HASH_MISMATCH`) so triage sees the real cause, not a generic label.
* Returns undefined for any shape we don't recognise — never throws.
*/
// fallow-ignore-next-line complexity
function extractErrorName(payload: string | null | undefined): string | undefined {
if (!payload) return undefined;
try {
const outer = JSON.parse(payload) as { error?: unknown; body?: unknown };
if (typeof outer.error === "string") return outer.error;
if (typeof outer.body === "string") {
const inner = JSON.parse(outer.body) as { error?: unknown };
if (typeof inner.error === "string") return inner.error;
} else if (outer.body && typeof outer.body === "object") {
const inner = outer.body as { error?: unknown };
if (typeof inner.error === "string") return inner.error;
}
} catch {
// Non-JSON / unexpected shape — fall through to the generic label.
}
return undefined;
}
// fallow-ignore-next-line complexity
function toIso(ts: ProtoTimestamp | string | null | undefined): string | null {
if (ts == null) return null;
if (typeof ts === "string") return ts;
const seconds = ts.seconds == null ? null : Number(ts.seconds);
if (seconds == null || !Number.isFinite(seconds)) return null;
const ms = seconds * 1000 + (ts.nanos ?? 0) / 1e6;
return new Date(ms).toISOString();
}
async function defaultExecutionsClient(): Promise<ExecutionsGetClientLike> {
const mod = await import("@google-cloud/workflows");
const client = new mod.ExecutionsClient();
return client as unknown as ExecutionsGetClientLike;
}
+40
View File
@@ -0,0 +1,40 @@
/**
* SDK subpath export — `@hyperframes/gcp-cloud-run/sdk`.
*
* Pulled into its own subpath so consumers that only drive renders (CLI, CI
* scripts, adopter tooling) don't pay the cost of importing `./server.js`,
* which transitively pulls `puppeteer-core` into the module graph. The SDK
* files here are GCS + Workflows clients only — safe to load in any Node
* environment.
*/
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";
export {
type ExecutionsClientLike,
renderToCloudRun,
type RenderHandle,
type RenderToCloudRunOptions,
} from "./renderToCloudRun.js";
export {
type ExecutionRecord,
type ExecutionsGetClientLike,
getRenderProgress,
type GetRenderProgressOptions,
type RenderError,
type RenderProgress,
type RenderStatus,
} from "./getRenderProgress.js";
export {
type BilledCloudRunInvocation,
computeRenderCost,
type RenderCost,
} from "./costAccounting.js";
export {
InvalidConfigError,
MAX_WORKFLOWS_INPUT_BYTES,
validateDistributedRenderConfig,
validateVariablesPayload,
validateWorkflowsInputSize,
} from "./validateConfig.js";
export type { SerializableDistributedRenderConfig } from "../events.js";
export type { DistributedFormat } from "../formatExtension.js";
@@ -0,0 +1,127 @@
/**
* `renderToCloudRun` unit tests — argument assembly, required-field
* validation, and the CreateExecution call over a fake ExecutionsClient.
*/
import { describe, expect, it } from "bun:test";
import type { SerializableDistributedRenderConfig } from "../events.js";
import { type ExecutionsClientLike, renderToCloudRun } from "./renderToCloudRun.js";
import type { SiteHandle } from "./deploySite.js";
const config = {
fps: 30,
width: 1920,
height: 1080,
format: "mp4",
} as SerializableDistributedRenderConfig;
const site: SiteHandle = {
siteId: "abc",
bucketName: "b",
projectGcsUri: "gs://b/sites/abc/project.tar.gz",
bytes: 100,
uploadedAt: "2026-06-06T00:00:00Z",
uploaded: true,
};
class FakeExecutions implements ExecutionsClientLike {
lastArgument: string | null = null;
lastParent: string | null = null;
workflowPath(project: string, location: string, workflow: string): string {
return `projects/${project}/locations/${location}/workflows/${workflow}`;
}
async createExecution(req: {
parent: string;
execution: { argument: string };
}): Promise<[{ name?: string | null; state?: string | null }]> {
this.lastParent = req.parent;
this.lastArgument = req.execution.argument;
return [{ name: `${req.parent}/executions/exec-123`, state: "ACTIVE" }];
}
}
function opts(executions: ExecutionsClientLike) {
return {
siteHandle: site,
config,
bucketName: "b",
projectId: "proj",
location: "us-central1",
workflowId: "hyperframes-render",
serviceUrl: "https://render-abc.run.app",
renderId: "hf-render-fixed",
executions,
};
}
describe("renderToCloudRun", () => {
it("starts an execution and returns a handle", async () => {
const fake = new FakeExecutions();
const handle = await renderToCloudRun(opts(fake));
expect(handle.renderId).toBe("hf-render-fixed");
expect(handle.executionName).toBe(
"projects/proj/locations/us-central1/workflows/hyperframes-render/executions/exec-123",
);
expect(handle.outputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.mp4");
expect(handle.projectGcsUri).toBe("gs://b/sites/abc/project.tar.gz");
});
it("builds the workflow argument the YAML expects", async () => {
const fake = new FakeExecutions();
await renderToCloudRun(opts(fake));
const arg = JSON.parse(fake.lastArgument ?? "{}");
expect(arg.RenderId).toBe("hf-render-fixed");
expect(arg.ProjectGcsUri).toBe("gs://b/sites/abc/project.tar.gz");
expect(arg.PlanOutputGcsPrefix).toBe("gs://b/renders/hf-render-fixed/");
expect(arg.OutputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.mp4");
expect(arg.ServiceUrl).toBe("https://render-abc.run.app");
expect(arg.Config.format).toBe("mp4");
expect(fake.lastParent).toBe(
"projects/proj/locations/us-central1/workflows/hyperframes-render",
);
});
it("derives the output extension from the format", async () => {
const fake = new FakeExecutions();
const handle = await renderToCloudRun({
...opts(fake),
config: { ...config, format: "webm" } as SerializableDistributedRenderConfig,
});
expect(handle.outputGcsUri).toBe("gs://b/renders/hf-render-fixed/output.webm");
});
it("requires serviceUrl", async () => {
const fake = new FakeExecutions();
await expect(renderToCloudRun({ ...opts(fake), serviceUrl: "" })).rejects.toThrow(
/serviceUrl is required/,
);
});
it("requires a siteHandle or projectDir", async () => {
const fake = new FakeExecutions();
const { siteHandle, ...rest } = opts(fake);
void siteHandle;
await expect(renderToCloudRun(rest)).rejects.toThrow(/siteHandle or projectDir/);
});
it("validates the config before any GCP call", async () => {
const fake = new FakeExecutions();
await expect(
renderToCloudRun({ ...opts(fake), config: { ...config, fps: 25 } as never }),
).rejects.toThrow(/config\.fps/);
expect(fake.lastArgument).toBeNull();
});
it("rejects a renderId that could escape the GCS key prefix", async () => {
const fake = new FakeExecutions();
await expect(renderToCloudRun({ ...opts(fake), renderId: "../escape" })).rejects.toThrow(
/renderId must match/,
);
await expect(renderToCloudRun({ ...opts(fake), renderId: "has/slash" })).rejects.toThrow(
/renderId must match/,
);
expect(fake.lastArgument).toBeNull();
});
});
@@ -0,0 +1,188 @@
/**
* `renderToCloudRun` — start a distributed render against an already-deployed
* Cloud Run service + Cloud Workflows definition and return a handle the
* caller can poll with {@link getRenderProgress}.
*
* The function does *not* wait for the render to finish. Cloud Workflows
* executions can run for hours; blocking the caller's process on the
* execution is the wrong default. The returned `RenderHandle` carries
* everything the progress / cost / download paths need.
*
* Wire order:
* 1. Validate config (typed throw before any GCP call).
* 2. `deploySite` if no `siteHandle` was provided.
* 3. `CreateExecution` against the workflow with the argument shape the
* `packages/gcp-cloud-run/terraform/workflow.yaml` definition expects.
* 4. Return handle. The GCS `outputKey` is deterministic from the
* client-generated `renderId` so the caller can predict the final
* object URL before the (server-assigned) execution id exists.
*
* Unlike Step Functions, Cloud Workflows assigns the execution id
* server-side, so we cannot use it as the GCS prefix. We mint a `renderId`
* (uuid) client-side, use it for every GCS path, and pass it into the
* workflow argument; the server-assigned execution resource name is tracked
* separately for polling.
*/
import { randomUUID } from "node:crypto";
import type { Storage } from "@google-cloud/storage";
import type { SerializableDistributedRenderConfig } from "../events.js";
import { formatExtension } from "../formatExtension.js";
import { formatGcsUri } from "../gcsTransport.js";
import { deploySite, type SiteHandle } from "./deploySite.js";
import { validateDistributedRenderConfig, validateWorkflowsInputSize } from "./validateConfig.js";
/**
* Minimal surface of `@google-cloud/workflows`' `ExecutionsClient` that
* this module needs. The real client satisfies this; tests inject a double.
*/
export interface ExecutionsClientLike {
workflowPath(project: string, location: string, workflow: string): string;
createExecution(req: {
parent: string;
execution: { argument: string };
}): Promise<[{ name?: string | null; state?: string | null }, ...unknown[]]>;
}
/** Options for {@link renderToCloudRun}. */
export interface RenderToCloudRunOptions {
/** Local project directory. Required when `siteHandle` is not supplied. */
projectDir?: string;
/** Re-use an existing `deploySite` upload (skips tar+GCS upload). */
siteHandle?: SiteHandle;
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
config: SerializableDistributedRenderConfig;
/** GCS bucket from the Terraform output (`render_bucket_name`). */
bucketName: string;
/** GCP project id hosting the workflow. */
projectId: string;
/** Workflow location, e.g. `us-central1`. */
location: string;
/** Workflow id from the Terraform output (`workflow_name`). */
workflowId: string;
/**
* HTTPS URL of the deployed Cloud Run render service (Terraform output
* `service_url`). The workflow POSTs every step (plan / renderChunk /
* assemble) to this URL; passed as an execution argument so the workflow
* definition stays free of hard-coded URLs.
*/
serviceUrl: string;
/**
* Final output GCS key. Defaults to `renders/<renderId>/output.<ext>`
* where `<ext>` is derived from `config.format`.
*/
outputKey?: string;
/**
* Client-generated render id. Defaults to `hf-render-<uuid>`. Used as the
* GCS key prefix and echoed into the workflow argument; not the same as
* the server-assigned execution id.
*/
renderId?: string;
/** Test injection seam — production callers leave unset. */
executions?: ExecutionsClientLike;
/** Test injection seam — propagated to `deploySite` when applicable. */
storage?: Storage;
}
/** Stable identifier + every URL/name the caller needs to follow the render. */
export interface RenderHandle {
/** Client-generated render id; the GCS prefix everything lands under. */
renderId: string;
/** Server-assigned execution resource name; pass to {@link getRenderProgress}. */
executionName: string;
bucketName: string;
workflowId: string;
outputGcsUri: string;
projectGcsUri: string;
startedAt: string;
}
// fallow-ignore-next-line complexity
export async function renderToCloudRun(opts: RenderToCloudRunOptions): Promise<RenderHandle> {
validateDistributedRenderConfig(opts.config);
if (!opts.bucketName) throw new Error("[renderToCloudRun] bucketName is required");
if (!opts.projectId) throw new Error("[renderToCloudRun] projectId is required");
if (!opts.location) throw new Error("[renderToCloudRun] location is required");
if (!opts.workflowId) throw new Error("[renderToCloudRun] workflowId is required");
if (!opts.serviceUrl) throw new Error("[renderToCloudRun] serviceUrl is required");
if (!opts.siteHandle && !opts.projectDir) {
throw new Error("[renderToCloudRun] either siteHandle or projectDir must be supplied");
}
const renderId = opts.renderId ?? `hf-render-${randomUUID()}`;
// `renderId` is interpolated directly into GCS object keys
// (`renders/<renderId>/…`). Reject anything that could escape that prefix
// or build a malformed key — `..`, slashes, or other path metacharacters —
// so a caller-supplied id can't collide with or overwrite another render's
// artifacts elsewhere in the bucket.
if (!/^[A-Za-z0-9._-]+$/.test(renderId) || renderId.includes("..")) {
throw new Error(
`[renderToCloudRun] renderId must match [A-Za-z0-9._-]+ and not contain "..": ${JSON.stringify(renderId)}`,
);
}
const ext = formatExtension(opts.config.format);
const outputKey = opts.outputKey ?? `renders/${renderId}/output${ext}`;
const planOutputGcsPrefix = formatGcsUri({
bucket: opts.bucketName,
key: `renders/${renderId}/`,
});
const outputGcsUri = formatGcsUri({ bucket: opts.bucketName, key: outputKey });
const site =
opts.siteHandle ??
(await deploySite({
projectDir: opts.projectDir as string,
bucketName: opts.bucketName,
storage: opts.storage,
}));
const argument = {
RenderId: renderId,
ProjectGcsUri: site.projectGcsUri,
PlanOutputGcsPrefix: planOutputGcsPrefix,
OutputGcsUri: outputGcsUri,
ServiceUrl: opts.serviceUrl,
Config: opts.config,
};
// Reject oversize input client-side. Cloud Workflows caps the execution
// argument at 512 KiB; without this check, input bloat (typically from
// `config.variables` containing inlined media) surfaces as an opaque
// server-side error after the execution starts, far from the caller's
// stack frame.
validateWorkflowsInputSize(argument);
const executions = opts.executions ?? (await defaultExecutionsClient());
const parent = executions.workflowPath(opts.projectId, opts.location, opts.workflowId);
const startedAt = new Date().toISOString();
const [execution] = await executions.createExecution({
parent,
execution: { argument: JSON.stringify(argument) },
});
if (!execution.name) {
throw new Error("[renderToCloudRun] CreateExecution returned no execution name");
}
return {
renderId,
executionName: execution.name,
bucketName: opts.bucketName,
workflowId: opts.workflowId,
outputGcsUri,
projectGcsUri: site.projectGcsUri,
startedAt,
};
}
/**
* Lazily import the real `@google-cloud/workflows` ExecutionsClient. Dynamic
* so SDK consumers that only call `validateDistributedRenderConfig` (or
* inject their own client) don't pay the import cost.
*/
async function defaultExecutionsClient(): Promise<ExecutionsClientLike> {
const mod = await import("@google-cloud/workflows");
const client = new mod.ExecutionsClient();
return client as unknown as ExecutionsClientLike;
}
@@ -0,0 +1,119 @@
/**
* `validateDistributedRenderConfig` + `validateWorkflowsInputSize` unit
* tests. Pins the shape rejections the SDK surfaces synchronously before a
* Cloud Workflows execution starts.
*/
import { describe, expect, it } from "bun:test";
import type { SerializableDistributedRenderConfig } from "../events.js";
import {
InvalidConfigError,
MAX_WORKFLOWS_INPUT_BYTES,
validateDistributedRenderConfig,
validateVariablesPayload,
validateWorkflowsInputSize,
} from "./validateConfig.js";
function base(): SerializableDistributedRenderConfig {
return {
fps: 30,
width: 1920,
height: 1080,
format: "mp4",
} as SerializableDistributedRenderConfig;
}
describe("validateDistributedRenderConfig", () => {
it("accepts a minimal valid config", () => {
expect(validateDistributedRenderConfig(base())).toBeDefined();
});
it("rejects a bad fps", () => {
expect(() => validateDistributedRenderConfig({ ...base(), fps: 25 } as never)).toThrow(
/config\.fps/,
);
});
it("rejects odd dimensions (yuv420p)", () => {
expect(() => validateDistributedRenderConfig({ ...base(), width: 1921 })).toThrow(/even/);
});
it("rejects an out-of-range dimension", () => {
expect(() => validateDistributedRenderConfig({ ...base(), height: 8 })).toThrow(/\[16, 7680\]/);
});
it("rejects an unknown format", () => {
expect(() => validateDistributedRenderConfig({ ...base(), format: "gif" as never })).toThrow(
/config\.format/,
);
});
it("rejects codec with a non-mp4 format", () => {
expect(() =>
validateDistributedRenderConfig({ ...base(), format: "webm", codec: "h264" } as never),
).toThrow(/only valid with format="mp4"/);
});
it("rejects crf + bitrate together", () => {
expect(() =>
validateDistributedRenderConfig({ ...base(), crf: 20, bitrate: "10M" } as never),
).toThrow(/mutually exclusive/);
});
it("rejects force-hdr", () => {
expect(() =>
validateDistributedRenderConfig({ ...base(), hdrMode: "force-hdr" as never }),
).toThrow(/force-sdr/);
});
it("rejects an over-cap chunkSize", () => {
expect(() => validateDistributedRenderConfig({ ...base(), chunkSize: 5000 } as never)).toThrow(
/<= 3600/,
);
});
it("throws InvalidConfigError with a field pointer", () => {
try {
validateDistributedRenderConfig({ ...base(), fps: 1 } as never);
throw new Error("should have thrown");
} catch (err) {
expect(err).toBeInstanceOf(InvalidConfigError);
expect((err as InvalidConfigError).field).toBe("config.fps");
}
});
});
describe("validateVariablesPayload", () => {
it("accepts a plain JSON object", () => {
expect(() =>
validateVariablesPayload({ title: "Hi", count: 3, nested: { ok: true } }),
).not.toThrow();
});
it("rejects undefined leaves", () => {
expect(() => validateVariablesPayload({ a: undefined })).toThrow(/undefined leaves/);
});
it("rejects a top-level array", () => {
expect(() => validateVariablesPayload([1, 2])).toThrow(/plain JSON object/);
});
it("rejects NaN", () => {
expect(() => validateVariablesPayload({ x: NaN })).toThrow(/non-finite/);
});
it("rejects a Date (non-plain object)", () => {
expect(() => validateVariablesPayload({ when: new Date(0) })).toThrow(/non-plain objects/);
});
});
describe("validateWorkflowsInputSize", () => {
it("accepts a small payload", () => {
expect(() => validateWorkflowsInputSize({ a: "b" })).not.toThrow();
});
it("rejects a payload over the 512 KiB cap", () => {
const big = { blob: "x".repeat(MAX_WORKFLOWS_INPUT_BYTES + 1) };
expect(() => validateWorkflowsInputSize(big)).toThrow(/512 KiB/);
});
});
@@ -0,0 +1,77 @@
/**
* Client-side validation for the Cloud Run adapter.
*
* The cloud-agnostic config-shape validation (`validateDistributedRenderConfig`,
* `validateVariablesPayload`, `InvalidConfigError`) lives in
* `@hyperframes/producer/distributed` and is shared with the other adapters.
* This module re-exports those and adds the one piece that is specific to
* Cloud Workflows: the 512 KiB execution-argument size cap.
*/
import { InvalidConfigError } from "@hyperframes/producer/distributed";
export {
InvalidConfigError,
validateDistributedRenderConfig,
validateVariablesPayload,
} from "@hyperframes/producer/distributed";
/**
* Hard cap on Cloud Workflows execution arguments — 512 KiB per the Workflows
* quotas page (maximum size of arguments passed when an execution starts).
* The cap is on the entire serialized argument, not just the variables,
* because users hit it at the wire boundary regardless of which field caused
* the bloat.
*
* Specific to Cloud Workflows. Other runtimes (Lambda + Step Functions,
* Temporal) have different caps; don't reuse this constant for those without
* confirming the limit.
*/
export const MAX_WORKFLOWS_INPUT_BYTES = 512 * 1024;
/** Pointer to the docs section that explains the URL-your-assets convention. */
const LARGE_VARIABLES_DOCS_URL =
"https://hyperframes.heygen.com/deploy/templates-on-lambda#working-with-large-variables";
/**
* Validate that the serialized Cloud Workflows execution argument fits inside
* the 512 KiB cap. Measured in UTF-8 bytes (the format the API uses on the
* wire) — JS strings count UTF-16 code units, which under-reports for any
* multi-byte character.
*
* Throws {@link InvalidConfigError} with a clear message naming the actual
* byte count, the cap, and a pointer to the "working with large variables"
* docs section, so users hit the limit at the SDK boundary with actionable
* guidance instead of as an opaque argument-too-large error after the
* execution starts.
*/
// fallow-ignore-next-line complexity
export function validateWorkflowsInputSize(input: unknown): void {
let serialized: string | undefined;
try {
serialized = JSON.stringify(input);
} catch (err) {
throw new InvalidConfigError(
"config",
`Cloud Workflows execution argument is not JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,
);
}
if (serialized === undefined) {
throw new InvalidConfigError(
"config",
"Cloud Workflows execution argument is not JSON-serializable (JSON.stringify returned undefined). " +
"Check that all fields, including config.variables, are plain JSON values.",
);
}
const byteLength = Buffer.byteLength(serialized, "utf8");
if (byteLength > MAX_WORKFLOWS_INPUT_BYTES) {
throw new InvalidConfigError(
"config",
`Cloud Workflows execution argument is ${byteLength} bytes, which exceeds the ` +
`${MAX_WORKFLOWS_INPUT_BYTES}-byte (512 KiB) limit. Variables are for typed data ` +
`(strings, numbers, structured records); media assets (images, audio, video) should ` +
`be passed as URL references the composition resolves at render time, not inlined as ` +
`base64. See ${LARGE_VARIABLES_DOCS_URL} for the URL-your-assets convention.`,
);
}
}
+332
View File
@@ -0,0 +1,332 @@
/**
* Handler dispatch + HTTP-shell unit tests.
*
* Asserts that:
* - `dispatch` routes Action="plan"/"renderChunk"/"assemble" to the
* matching OSS primitive and plumbs GCS download/upload around it.
* - It unwraps `{ Payload }` / `{ Input }` envelopes and rejects unknown
* actions.
* - The handler-boundary guards fire: plan-hash mismatch + bucket
* allowlist throw the typed, non-retryable errors.
* - `createApp` maps non-retryable errors → 400 and retryable → 500.
*
* The real OSS primitives are NOT exercised — they have their own coverage
* in `packages/producer`. This file pins the adapter glue's contract.
*/
import { afterEach, describe, expect, it } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AssembleResult, ChunkResult, PlanResult } from "@hyperframes/producer/distributed";
import { asStorage, FakeGcs } from "./__fixtures__/fakeGcs.js";
import type { AssembleEvent, CloudRunEvent, PlanEvent, RenderChunkEvent } from "./events.js";
import { createApp, dispatch, type HandlerDeps, unwrapEvent } from "./server.js";
import { tarDirectory } from "./gcsTransport.js";
const tmpDirs: string[] = [];
function mkTmp(prefix: string): string {
const dir = mkdtempSync(join(tmpdir(), prefix));
tmpDirs.push(dir);
return dir;
}
afterEach(() => {
for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
});
const PLAN_HASH = "abc123planhash";
/** Build a real project tarball, seed it into the fake, return its URI. */
async function seedProjectTar(gcs: FakeGcs, uri: string): Promise<void> {
const src = mkTmp("hf-proj-");
writeFileSync(join(src, "index.html"), "<html></html>");
const tarPath = join(mkTmp("hf-proj-tar-"), "project.tar.gz");
await tarDirectory(src, tarPath);
gcs.seedFromFile(uri, tarPath);
}
/** Build a real plan tarball containing plan.json, seed it, return its URI. */
async function seedPlanTar(gcs: FakeGcs, uri: string, planHash: string): Promise<void> {
const planDir = mkTmp("hf-plan-");
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash }));
const tarPath = join(mkTmp("hf-plan-tar-"), "plan.tar.gz");
await tarDirectory(planDir, tarPath);
gcs.seedFromFile(uri, tarPath);
}
const planResult: PlanResult = {
planDir: "(set at call time)",
planHash: PLAN_HASH,
chunkCount: 3,
totalFrames: 90,
fps: 30,
width: 1920,
height: 1080,
format: "mp4",
ffmpegVersion: "ffmpeg version 6.1.1",
producerVersion: "0.6.79",
};
function depsWith(
gcs: FakeGcs,
overrides: Partial<NonNullable<HandlerDeps["primitives"]>> = {},
): HandlerDeps {
const plan = async (_projectDir: string, _config: unknown, planDir: string) => {
mkdirSync(planDir, { recursive: true });
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: PLAN_HASH }));
return planResult;
};
const renderChunk = async (_planDir: string, chunkIndex: number, outputBase: string) => {
writeFileSync(outputBase, Buffer.from(`chunk-${chunkIndex}`));
return {
outputPath: outputBase,
outputKind: "file",
framesEncoded: 30,
sha256: `sha-${chunkIndex}`,
} satisfies ChunkResult;
};
const assemble = async (
_planDir: string,
_chunkPaths: string[],
_audio: string | null,
finalOutput: string,
) => {
writeFileSync(finalOutput, Buffer.from("final-output"));
return { framesEncoded: 90, fileSize: 12 } satisfies AssembleResult;
};
return {
storage: asStorage(gcs),
skipChromeResolution: true,
primitives: { plan, renderChunk, assemble, ...overrides } as NonNullable<
HandlerDeps["primitives"]
>,
};
}
describe("unwrapEvent", () => {
const plan: PlanEvent = {
Action: "plan",
ProjectGcsUri: "gs://b/p.tar.gz",
PlanOutputGcsPrefix: "gs://b/out/",
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" } as PlanEvent["Config"],
};
it("returns a bare event unchanged", () => {
expect(unwrapEvent(plan).Action).toBe("plan");
});
it("unwraps { Payload }", () => {
expect(unwrapEvent({ Payload: plan } as CloudRunEvent).Action).toBe("plan");
});
it("unwraps nested { Input: { Payload } }", () => {
expect(unwrapEvent({ Input: { Payload: plan } } as CloudRunEvent).Action).toBe("plan");
});
it("throws when no Action is found", () => {
expect(() => unwrapEvent({ foo: "bar" } as unknown as CloudRunEvent)).toThrow(
/no recognised Action/,
);
});
});
describe("dispatch", () => {
it("routes plan, uploads the plan tarball", async () => {
const gcs = new FakeGcs();
await seedProjectTar(gcs, "gs://b/sites/x/project.tar.gz");
const event: PlanEvent = {
Action: "plan",
ProjectGcsUri: "gs://b/sites/x/project.tar.gz",
PlanOutputGcsPrefix: "gs://b/renders/r1/",
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" } as PlanEvent["Config"],
};
const res = await dispatch(event, depsWith(gcs));
expect(res.Action).toBe("plan");
if (res.Action !== "plan") throw new Error("unreachable");
expect(res.PlanHash).toBe(PLAN_HASH);
expect(res.ChunkCount).toBe(3);
expect(res.PlanGcsUri).toBe("gs://b/renders/r1/plan.tar.gz");
expect(gcs.objects.has("gs://b/renders/r1/plan.tar.gz")).toBe(true);
});
it("routes renderChunk and uploads the chunk", async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 2,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
};
const res = await dispatch(event, depsWith(gcs));
if (res.Action !== "renderChunk") throw new Error("unreachable");
expect(res.ChunkIndex).toBe(2);
expect(res.ChunkGcsUri).toBe("gs://b/renders/r1/chunks/0002.mp4");
expect(gcs.objects.has("gs://b/renders/r1/chunks/0002.mp4")).toBe(true);
});
it("throws PLAN_HASH_MISMATCH when the event hash disagrees", async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: "WRONG_HASH",
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
};
await expect(dispatch(event, depsWith(gcs))).rejects.toThrow(/PLAN_HASH_MISMATCH/);
});
it("routes assemble and uploads the final output", async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
gcs.seed("gs://b/renders/r1/chunks/0000.mp4", Buffer.from("c0"));
gcs.seed("gs://b/renders/r1/chunks/0001.mp4", Buffer.from("c1"));
const event: AssembleEvent = {
Action: "assemble",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
ChunkGcsUris: ["gs://b/renders/r1/chunks/0000.mp4", "gs://b/renders/r1/chunks/0001.mp4"],
AudioGcsUri: null,
OutputGcsUri: "gs://b/renders/r1/output.mp4",
Format: "mp4",
};
const res = await dispatch(event, depsWith(gcs));
if (res.Action !== "assemble") throw new Error("unreachable");
expect(res.FramesEncoded).toBe(90);
expect(gcs.objects.has("gs://b/renders/r1/output.mp4")).toBe(true);
});
it("rejects an unknown action", async () => {
const gcs = new FakeGcs();
await expect(
dispatch({ Action: "nope" } as unknown as CloudRunEvent, depsWith(gcs)),
).rejects.toThrow(/no recognised Action/);
});
});
describe("bucket allowlist guard", () => {
it("throws GCS_URI_NOT_ALLOWED for an off-bucket URI", async () => {
const gcs = new FakeGcs();
const prev = process.env.HYPERFRAMES_RENDER_BUCKET;
process.env.HYPERFRAMES_RENDER_BUCKET = "allowed-bucket";
try {
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanGcsUri: "gs://evil-bucket/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://allowed-bucket/renders/r1/",
Format: "mp4",
};
await expect(dispatch(event, depsWith(gcs))).rejects.toThrow(/GCS_URI_NOT_ALLOWED/);
} finally {
if (prev === undefined) delete process.env.HYPERFRAMES_RENDER_BUCKET;
else process.env.HYPERFRAMES_RENDER_BUCKET = prev;
}
});
it('treats HYPERFRAMES_RENDER_BUCKET="*" as an explicit opt-out (off-bucket allowed)', async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://any-bucket/renders/r1/plan.tar.gz", PLAN_HASH);
const prev = process.env.HYPERFRAMES_RENDER_BUCKET;
process.env.HYPERFRAMES_RENDER_BUCKET = "*";
try {
const event: RenderChunkEvent = {
Action: "renderChunk",
PlanGcsUri: "gs://any-bucket/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://any-bucket/renders/r1/",
Format: "mp4",
};
const res = await dispatch(event, depsWith(gcs));
expect(res.Action).toBe("renderChunk");
} finally {
if (prev === undefined) delete process.env.HYPERFRAMES_RENDER_BUCKET;
else process.env.HYPERFRAMES_RENDER_BUCKET = prev;
}
});
});
describe("createApp HTTP mapping", () => {
it("returns 200 with the result body on success", async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const app = createApp(depsWith(gcs));
const res = await app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
}),
});
expect(res.status).toBe(200);
const body = (await res.json()) as { Action: string };
expect(body.Action).toBe("renderChunk");
});
it("returns 400 for a non-retryable error (plan-hash mismatch)", async () => {
const gcs = new FakeGcs();
await seedPlanTar(gcs, "gs://b/renders/r1/plan.tar.gz", PLAN_HASH);
const app = createApp(depsWith(gcs));
const res = await app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/plan.tar.gz",
PlanHash: "WRONG",
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
}),
});
expect(res.status).toBe(400);
const body = (await res.json()) as { error: string };
expect(body.error).toBe("PLAN_HASH_MISMATCH");
});
it("returns 500 for a retryable/unknown error", async () => {
const gcs = new FakeGcs(); // plan tar NOT seeded → download fails (retryable)
const app = createApp(depsWith(gcs));
const res = await app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
Action: "renderChunk",
PlanGcsUri: "gs://b/renders/r1/missing.tar.gz",
PlanHash: PLAN_HASH,
ChunkIndex: 0,
ChunkOutputGcsPrefix: "gs://b/renders/r1/",
Format: "mp4",
}),
});
expect(res.status).toBe(500);
});
it("returns 400 when the body is not JSON", async () => {
const gcs = new FakeGcs();
const app = createApp(depsWith(gcs));
const res = await app.request("/", {
method: "POST",
headers: { "content-type": "application/json" },
body: "not json{",
});
expect(res.status).toBe(400);
});
it("healthz returns ok", async () => {
const app = createApp(depsWith(new FakeGcs()));
const res = await app.request("/healthz");
expect(res.status).toBe(200);
});
});
+647
View File
@@ -0,0 +1,647 @@
/**
* Cloud Run request handler for HyperFrames distributed rendering.
*
* One container image, three roles. Cloud Workflows POSTs a JSON body with
* an `Action` field; the handler unwraps any `Payload`/`Input` envelope,
* primes the runtime (Chrome path), and forwards to the matching OSS
* primitive from `@hyperframes/producer/distributed`.
*
* Everything heavy — capture, encode, audio mix — happens inside the OSS
* primitives. The handler is thin glue: parse body → GCS download → call
* primitive → GCS upload → return small JSON result.
*
* `dispatch()` is the testable core (inject `storage` + `primitives`); the
* Hono app at the bottom is the HTTP shell the Dockerfile runs. The shape
* deliberately tracks `@hyperframes/aws-lambda`'s `handler.ts` so the two
* adapters stay easy to diff.
*/
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, extname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { serve } from "@hono/node-server";
import { Storage } from "@google-cloud/storage";
import { Hono } from "hono";
import {
assemble,
type AssembleResult,
type ChunkResult,
type DistributedRenderConfig,
plan,
type PlanResult,
renderChunk,
} from "@hyperframes/producer/distributed";
import { resolveChromeExecutablePath } from "./chromium.js";
import type {
AssembleEvent,
AssembleResultBody,
CloudRunAction,
CloudRunEvent,
CloudRunResult,
PlanEvent,
PlanResultBody,
RenderChunkEvent,
RenderChunkResultBody,
} from "./events.js";
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
import {
downloadGcsObjectToFile,
parseGcsUri,
tarDirectory,
untarDirectory,
uploadFileToGcs,
} from "./gcsTransport.js";
/**
* Lazily-constructed Storage client. Cached at module scope so warm
* container instances reuse the underlying HTTP keep-alive pool across
* requests.
*/
let cachedStorage: Storage | null = null;
function getStorage(): Storage {
if (cachedStorage) return cachedStorage;
cachedStorage = new Storage();
return cachedStorage;
}
/**
* Optional injection points used by the handler's unit tests. Production
* callers leave these unset; the real OSS primitives are used. Tests inject
* `storage` and `primitives` directly rather than mutating module state.
*/
export interface HandlerDeps {
storage?: Storage;
primitives?: {
plan: typeof plan;
renderChunk: typeof renderChunk;
assemble: typeof assemble;
};
/** Override the per-request workdir root (defaults to the OS tmpdir). */
tmpRoot?: string;
/** Skip Chrome resolution (used by dispatch tests that mock renderChunk). */
skipChromeResolution?: boolean;
}
/**
* Dispatch a single render request. Cloud Workflows (or a direct caller)
* sometimes wraps the body in `{ Payload: ... }` or `{ Input: ... }`; unwrap
* until we hit a discriminated event.
*/
// fallow-ignore-next-line complexity
export async function dispatch(event: CloudRunEvent, deps?: HandlerDeps): Promise<CloudRunResult> {
const unwrapped = unwrapEvent(event);
validateEventGcsUris(unwrapped);
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
try {
switch (unwrapped.Action) {
case "plan":
return await handlePlan(unwrapped, deps);
case "renderChunk":
return await handleRenderChunk(unwrapped, deps);
case "assemble":
return await handleAssemble(unwrapped, deps);
default: {
// Compile-time exhaustiveness: a new CloudRunAction member trips
// the `never` assignment before the runtime error is reachable.
const _exhaustive: never = unwrapped;
throw new Error(
`[handler] unknown Action: ${JSON.stringify(
(_exhaustive as { Action?: string }).Action,
)}. Expected one of "plan", "renderChunk", "assemble".`,
);
}
}
} catch (err) {
logEvent({
event: "handler_error",
action: unwrapped.Action,
message: err instanceof Error ? err.message : String(err),
name: err instanceof Error ? err.name : undefined,
});
throw err;
}
}
// At most `{Payload: {Input: ...}}` is expected; 4 levels is 2× headroom
// and prevents infinite loops on malformed input.
const MAX_ENVELOPE_DEPTH = 4;
// fallow-ignore-next-line complexity
export function unwrapEvent(event: CloudRunEvent): PlanEvent | RenderChunkEvent | AssembleEvent {
let cursor: CloudRunEvent = event;
for (let i = 0; i < MAX_ENVELOPE_DEPTH; i++) {
if (cursor && typeof cursor === "object") {
const obj = cursor as Record<string, unknown>;
if (typeof obj.Action === "string" && isCloudRunAction(obj.Action)) {
return cursor as PlanEvent | RenderChunkEvent | AssembleEvent;
}
if ("Payload" in obj) {
cursor = obj.Payload as CloudRunEvent;
continue;
}
if ("Input" in obj) {
cursor = obj.Input as CloudRunEvent;
continue;
}
}
break;
}
throw new Error(
`[handler] body has no recognised Action; unwrapped ${MAX_ENVELOPE_DEPTH} levels of Payload/Input without finding one.`,
);
}
function isCloudRunAction(value: string): value is CloudRunAction {
return value === "plan" || value === "renderChunk" || value === "assemble";
}
/**
* Emit a single JSON line to stdout. Cloud Logging ingests each stdout line
* as a structured `jsonPayload` entry, so Logs Explorer can filter on
* `jsonPayload.event="handler_start"` and project specific fields when
* triaging without attaching a debugger.
*/
function logEvent(payload: Record<string, unknown>): void {
console.log(JSON.stringify(payload));
}
/**
* Compact, non-PII summary of an event for logging. The full body can
* include the entire project config; we only emit the routable fields
* needed to triage a failure from Cloud Logging.
*/
function summarizeEvent(
event: PlanEvent | RenderChunkEvent | AssembleEvent,
): Record<string, unknown> {
switch (event.Action) {
case "plan":
return {
projectGcsUri: event.ProjectGcsUri,
planOutputGcsPrefix: event.PlanOutputGcsPrefix,
format: event.Config.format,
fps: event.Config.fps,
};
case "renderChunk":
return {
planGcsUri: event.PlanGcsUri,
chunkIndex: event.ChunkIndex,
format: event.Format,
};
case "assemble":
return {
planGcsUri: event.PlanGcsUri,
chunkCount: event.ChunkGcsUris.length,
hasAudio: event.AudioGcsUri !== null,
outputGcsUri: event.OutputGcsUri,
format: event.Format,
};
}
}
/**
* Point the engine at the in-image Chrome binary. The OSS engine resolves
* Chrome via `PRODUCER_HEADLESS_SHELL_PATH` first; set it once per instance
* before invoking any browser-touching primitive. ffmpeg is on the image's
* PATH (apt-installed by the Dockerfile), so nothing to prime there.
*/
function primeChrome(deps?: HandlerDeps): void {
if (deps?.skipChromeResolution) return;
if (process.env.PRODUCER_HEADLESS_SHELL_PATH) return;
process.env.PRODUCER_HEADLESS_SHELL_PATH = resolveChromeExecutablePath();
}
// ── Plan ────────────────────────────────────────────────────────────────────
// fallow-ignore-next-line complexity
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanResultBody> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.plan ?? plan;
// The producer's probe stage launches Chromium whenever the composition
// needs a runtime duration probe or has unresolved sub-compositions, so
// plan has to resolve Chrome the same way renderChunk does.
primeChrome(deps);
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-plan-"));
const projectArchive = join(work, "project.tar.gz");
const projectDir = join(work, "project");
const planDir = join(work, "plan");
try {
await downloadGcsObjectToFile(storage, event.ProjectGcsUri, projectArchive);
await untarDirectory(projectArchive, projectDir);
const config: DistributedRenderConfig = {
...event.Config,
};
const result: PlanResult = await primitive(projectDir, config, planDir);
// Upload the planDir as a single tarball. The workflow cannot pass a
// directory-shaped artifact between steps; we serialize and rely on the
// consumer (renderChunk / assemble) to untar. `audio.aac` lives inside
// planDir, so it already rides along in this tarball — every consumer
// (including assemble) gets it from the untar. We deliberately do NOT
// upload a separate audio object: it would duplicate the bytes on every
// plan upload and be re-downloaded + overwritten by assemble. `AudioGcsUri`
// stays in the result shape for wire compatibility but is null.
const planTar = join(work, "plan.tar.gz");
await tarDirectory(planDir, planTar);
const planTarUri = `${trimTrailingSlash(event.PlanOutputGcsPrefix)}/plan.tar.gz`;
const audioPath = join(planDir, "audio.aac");
const hasAudio = existsSync(audioPath) && statSync(audioPath).size > 0;
await uploadFileToGcs(storage, planTar, planTarUri, "application/gzip");
return {
Action: "plan",
PlanGcsUri: planTarUri,
PlanHash: result.planHash,
ChunkCount: result.chunkCount,
TotalFrames: result.totalFrames,
Fps: result.fps,
Width: result.width,
Height: result.height,
Format: result.format,
HasAudio: hasAudio,
AudioGcsUri: null,
FfmpegVersion: result.ffmpegVersion,
ProducerVersion: result.producerVersion,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
// ── RenderChunk ─────────────────────────────────────────────────────────────
// fallow-ignore-next-line complexity
async function handleRenderChunk(
event: RenderChunkEvent,
deps?: HandlerDeps,
): Promise<RenderChunkResultBody> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
primeChrome(deps);
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-chunk-"));
const planTar = join(work, "plan.tar.gz");
const planDir = join(work, "plan");
try {
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
await untarDirectory(planTar, planDir);
// Verify the plan's hash matches what the workflow told us to render.
// The producer's renderChunk re-checks internally (defense-in-depth),
// but doing it here at the handler boundary lets us fail before paying
// the Chrome-launch + render cost on a misrouted chunk. Throws a typed
// PLAN_HASH_MISMATCH the workflow can route as non-retryable.
verifyPlanHash(planDir, event.PlanHash);
const chunkOutputBase = join(
work,
event.Format === "png-sequence"
? `chunk-${pad(event.ChunkIndex)}`
: `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`,
);
const result: ChunkResult = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
const chunkUri = await uploadChunkOutput(
storage,
result,
event.ChunkOutputGcsPrefix,
event.ChunkIndex,
);
return {
Action: "renderChunk",
ChunkGcsUri: chunkUri,
ChunkIndex: event.ChunkIndex,
Sha256: result.sha256,
FramesEncoded: result.framesEncoded,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
async function uploadChunkOutput(
storage: Storage,
result: ChunkResult,
prefix: string,
chunkIndex: number,
): Promise<string> {
const trimmed = trimTrailingSlash(prefix);
if (result.outputKind === "file") {
const ext = extname(result.outputPath);
const uri = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
await uploadFileToGcs(storage, result.outputPath, uri);
return uri;
}
// frame-dir: upload as a tarball so a single GCS object represents the
// chunk. Assemble's png-sequence path expects a directory per chunk; it
// untars on its end.
const tarball = `${result.outputPath}.tar.gz`;
await tarDirectory(result.outputPath, tarball);
const uri = `${trimmed}/chunks/${pad(chunkIndex)}.tar.gz`;
await uploadFileToGcs(storage, tarball, uri, "application/gzip");
return uri;
}
// ── Assemble ────────────────────────────────────────────────────────────────
// fallow-ignore-next-line complexity
async function handleAssemble(
event: AssembleEvent,
deps?: HandlerDeps,
): Promise<AssembleResultBody> {
const started = Date.now();
const storage = deps?.storage ?? getStorage();
const primitive = deps?.primitives?.assemble ?? assemble;
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-cr-assemble-"));
const planTar = join(work, "plan.tar.gz");
const planDir = join(work, "plan");
try {
await downloadGcsObjectToFile(storage, event.PlanGcsUri, planTar);
await untarDirectory(planTar, planDir);
const chunkPaths = await downloadChunkObjects(storage, event.ChunkGcsUris, work, event.Format);
// Audio rides inside the plan tarball, so it's already on disk after the
// untar above — no separate download. Fall back to a supplied AudioGcsUri
// only for backward compatibility with an older Plan that uploaded it
// standalone.
let audioPath: string | null = null;
const planAudio = join(planDir, "audio.aac");
if (existsSync(planAudio) && statSync(planAudio).size > 0) {
audioPath = planAudio;
} else if (event.AudioGcsUri) {
audioPath = planAudio;
await downloadGcsObjectToFile(storage, event.AudioGcsUri, audioPath);
}
const finalOutput =
event.Format === "png-sequence"
? join(work, "output-frames")
: join(work, `output${formatExtension(event.Format)}`);
const result: AssembleResult = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
cfr: event.Cfr === true,
});
if (event.Format === "png-sequence") {
const tarball = `${finalOutput}.tar.gz`;
await tarDirectory(finalOutput, tarball);
await uploadFileToGcs(storage, tarball, event.OutputGcsUri, "application/gzip");
} else {
await uploadFileToGcs(storage, finalOutput, event.OutputGcsUri);
}
return {
Action: "assemble",
OutputGcsUri: event.OutputGcsUri,
FramesEncoded: result.framesEncoded,
FileSize: result.fileSize,
DurationMs: Date.now() - started,
};
} finally {
cleanupDir(work);
}
}
async function downloadChunkObjects(
storage: Storage,
uris: string[],
workDir: string,
format: DistributedFormat,
): Promise<string[]> {
const chunksDir = join(workDir, "chunks");
mkdirSync(chunksDir, { recursive: true });
// Each chunk is an independent GCS GET (+ untar for png-sequence). Run
// them in parallel — assemble's wall-clock is otherwise dominated by
// `Σ chunk-download-ms` instead of `max(chunk-download-ms)`. Preserve the
// input order by writing into a pre-sized array rather than pushing as
// each task settles.
const local: string[] = new Array<string>(uris.length);
await Promise.all(
uris.map(async (uri, i) => {
if (!uri) {
throw new Error(`[handler] chunk URI at index ${i} is empty`);
}
const { key } = parseGcsUri(uri);
const localPath = join(chunksDir, basename(key));
await downloadGcsObjectToFile(storage, uri, localPath);
if (format === "png-sequence") {
const dirPath = join(chunksDir, `frames-${pad(i)}`);
await untarDirectory(localPath, dirPath);
local[i] = dirPath;
} else {
local[i] = localPath;
}
}),
);
return local;
}
// ── Helpers ─────────────────────────────────────────────────────────────────
/** Collect every GCS URI that the handler will touch for a given event. */
function getEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): string[] {
switch (event.Action) {
case "plan":
return [event.ProjectGcsUri, event.PlanOutputGcsPrefix];
case "renderChunk":
return [event.PlanGcsUri, event.ChunkOutputGcsPrefix];
case "assemble":
return [
event.PlanGcsUri,
...event.ChunkGcsUris,
event.OutputGcsUri,
event.AudioGcsUri,
].filter((u): u is string => u != null);
}
}
/** Emit the "guard disabled" warning at most once per instance. */
let warnedAllowlistDisabled = false;
/**
* Verify every GCS URI in the event resolves to the configured render
* bucket. Throws `GCS_URI_NOT_ALLOWED` (non-retryable) when a URI targets a
* different bucket, preventing request injection from reading or writing
* arbitrary GCS data.
*
* Opt-out is explicit: set `HYPERFRAMES_RENDER_BUCKET="*"` to disable the
* guard intentionally. If the env var is simply unset (or empty), the guard
* is disabled but a warning is logged once so the gap is visible in Cloud
* Logging — it shouldn't silently fail open. The Terraform module always
* wires the bucket name, so the prod path enforces.
*/
// fallow-ignore-next-line complexity
function validateEventGcsUris(event: PlanEvent | RenderChunkEvent | AssembleEvent): void {
const allowedBucket = process.env.HYPERFRAMES_RENDER_BUCKET?.trim();
if (allowedBucket === "*") return; // explicit, intentional opt-out
if (!allowedBucket) {
if (!warnedAllowlistDisabled) {
warnedAllowlistDisabled = true;
logEvent({
event: "bucket_allowlist_disabled",
level: "WARNING",
message:
"HYPERFRAMES_RENDER_BUCKET is unset — the GCS bucket-allowlist guard is DISABLED. " +
'Set it to the render bucket name to enforce, or to "*" to opt out intentionally.',
});
}
return;
}
for (const uri of getEventGcsUris(event)) {
const { bucket } = parseGcsUri(uri);
if (bucket !== allowedBucket) {
const err = new Error(
`[handler] GCS_URI_NOT_ALLOWED: URI ${JSON.stringify(uri)} targets bucket "${bucket}" but only "${allowedBucket}" is permitted`,
);
err.name = "GCS_URI_NOT_ALLOWED";
throw err;
}
}
}
function pad(n: number): string {
return n.toString().padStart(4, "0");
}
function trimTrailingSlash(prefix: string): string {
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
}
function cleanupDir(dir: string): void {
try {
// Cloud Run re-uses an instance's filesystem across requests; clean up
// aggressively so we don't leak a chunk-sized footprint between renders
// (the writable filesystem counts against the instance's memory).
rmSync(dir, { recursive: true, force: true });
} catch {
// Best-effort — leak is preferable to crashing on the success path.
}
}
/**
* Read the untarred planDir's `plan.json` and assert its `planHash` matches
* what the workflow event claims. Throws on mismatch with a typed
* `PLAN_HASH_MISMATCH` error name so the workflow's non-retryable list
* routes it correctly. Defense-in-depth — the producer's `renderChunk` does
* the same check internally — but performing it here lets us fail before
* paying the Chrome-launch + per-frame capture cost on a misrouted chunk.
*/
// fallow-ignore-next-line complexity
function verifyPlanHash(planDir: string, expected: string): void {
const planJsonPath = join(planDir, "plan.json");
let parsed: { planHash?: unknown };
try {
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8")) as { planHash?: unknown };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
const error = new Error(`PLAN_HASH_MISMATCH: failed to read ${planJsonPath}: ${msg}`);
error.name = "PLAN_HASH_MISMATCH";
throw error;
}
const actual = parsed.planHash;
if (typeof actual !== "string" || actual !== expected) {
const error = new Error(
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match plan.json planHash=${String(actual)}`,
);
error.name = "PLAN_HASH_MISMATCH";
throw error;
}
}
// ── HTTP shell ───────────────────────────────────────────────────────────────
/**
* Error names the workflow treats as non-retryable. A request that fails
* with one of these is the caller's fault (bad input, misrouted chunk) and
* retrying it just burns instance-seconds, so we map them to HTTP 400 while
* any other failure maps to 500 (which the workflow retry policy backs off
* and re-attempts). Keep this list in sync with the `retry` predicate in
* `packages/gcp-cloud-run/terraform/workflow.yaml`.
*/
const NON_RETRYABLE_ERROR_NAMES = new Set([
// Handler-boundary guards.
"GCS_URI_NOT_ALLOWED",
"PLAN_HASH_MISMATCH",
// Producer error class names (`.name`) + their string code aliases — the
// class sets `.name` to the class name but wraps a `code`; cover both so a
// raw-code throw is caught too. Mirrors the AWS state machine's
// non-retryable list.
"FormatNotSupportedInDistributedError",
"PlanTooLargeError",
"RenderChunkValidationError",
"FFMPEG_VERSION_MISMATCH",
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
"PLAN_TOO_LARGE",
"BROWSER_GPU_NOT_SOFTWARE",
"FONT_FETCH_FAILED",
"ChromeBinaryUnavailableError",
]);
/**
* Build the Hono app. A single `POST /` endpoint dispatches on the body's
* `Action` field — the workflow points every step (plan, each renderChunk,
* assemble) at the same URL and varies only the body. `GET /healthz` backs
* the Cloud Run startup/liveness probe.
*
* `deps` is threaded through so tests can drive the real HTTP surface with
* an injected Storage double + mocked primitives.
*/
export function createApp(deps?: HandlerDeps): Hono {
const app = new Hono();
app.get("/healthz", (c) => c.json({ status: "ok" }));
// fallow-ignore-next-line complexity
app.post("/", async (c) => {
let body: CloudRunEvent;
try {
body = (await c.req.json()) as CloudRunEvent;
} catch {
return c.json({ error: "BAD_REQUEST", message: "request body must be JSON" }, 400);
}
try {
const result = await dispatch(body, deps);
return c.json(result, 200);
} catch (err) {
const name = err instanceof Error ? err.name : undefined;
const message = err instanceof Error ? err.message : String(err);
const status = name && NON_RETRYABLE_ERROR_NAMES.has(name) ? 400 : 500;
// Surface `error` (the name) as the discriminator the workflow's
// retry predicate keys off, plus `message` for human triage.
return c.json({ error: name ?? "RenderError", message }, status);
}
});
return app;
}
/** Start the HTTP server. Cloud Run injects `PORT` (default 8080). */
export function startServer(): void {
const port = Number(process.env.PORT ?? 8080);
const app = createApp();
serve({ fetch: app.fetch, port }, (info) => {
logEvent({ event: "server_listening", port: info.port });
});
}
// Boot when executed directly (the Dockerfile runs `node dist/server.js`),
// but not when imported by tests or the SDK.
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
startServer();
}
@@ -0,0 +1,7 @@
# Local Terraform working files — never commit these.
.terraform/
.terraform.lock.hcl
*.tfstate
*.tfstate.*
*.tfvars
crash.log
+197
View File
@@ -0,0 +1,197 @@
# HyperFrames distributed render stack on Google Cloud.
#
# Topology (the GCP twin of the AWS Lambda adapter's SAM template):
#
# GCS bucket ←→ Cloud Run service (plan / renderChunk / assemble)
# ▲
# │ OIDC-authenticated http.post per step
# │
# Cloud Workflows (Plan → parallel RenderChunk → Assemble)
#
# Two service accounts keep least-privilege boundaries:
# - run_sa : the render service's identity; read/write the bucket only.
# - workflow_sa: the workflow's identity; invoke the render service only.
locals {
workflow_source = var.workflow_source_path != "" ? var.workflow_source_path : "${path.module}/workflow.yaml"
name = var.project_name
}
# ── Storage: plan tarballs, chunk outputs, final renders ─────────────────────
resource "google_storage_bucket" "render" {
name = "${local.name}-render-${var.project_id}"
project = var.project_id
location = var.region
uniform_bucket_level_access = true
force_destroy = var.bucket_force_destroy
# Render artifacts (plan tarballs, chunk files) are disposable scratch.
# Sweep them after 7 days so the bucket doesn't accumulate cost; final
# outputs that adopters want to keep should be copied elsewhere.
lifecycle_rule {
condition {
age = 7
}
action {
type = "Delete"
}
}
}
# ── Service accounts ─────────────────────────────────────────────────────────
resource "google_service_account" "run_sa" {
account_id = "${local.name}-run"
project = var.project_id
display_name = "HyperFrames render service (Cloud Run)"
}
resource "google_service_account" "workflow_sa" {
account_id = "${local.name}-wf"
project = var.project_id
display_name = "HyperFrames render orchestration (Workflows)"
}
# Render service reads inputs + writes outputs in the render bucket only.
resource "google_storage_bucket_iam_member" "run_sa_bucket" {
bucket = google_storage_bucket.render.name
role = "roles/storage.objectAdmin"
member = "serviceAccount:${google_service_account.run_sa.email}"
}
# ── Cloud Run render service ─────────────────────────────────────────────────
resource "google_cloud_run_v2_service" "render" {
name = "${local.name}-render"
project = var.project_id
location = var.region
# Authenticated only — no public invoker binding. Only the workflow SA can
# call it. Ingress stays "all" because Workflows reaches the service over
# Google's front door, not the VPC.
ingress = "INGRESS_TRAFFIC_ALL"
# Let `terraform destroy` (and replacement on image bumps) remove the
# service without a manual console step. The render service is stateless —
# all durable artifacts live in GCS.
deletion_protection = false
template {
service_account = google_service_account.run_sa.email
timeout = "${var.request_timeout_seconds}s"
# One render (chunk / plan / assemble) per instance — each uses the whole
# box's CPU + memory + /tmp. The workflow's concurrency_limit governs how
# many instances run at once.
max_instance_request_concurrency = 1
scaling {
min_instance_count = var.min_instances
max_instance_count = var.max_instances
}
containers {
image = var.image
resources {
limits = {
cpu = var.cpu
memory = var.memory
}
# Keep CPU allocated only during request processing (request-based
# billing). Renders are entirely request-scoped.
cpu_idle = true
}
env {
# Scopes every event's GCS URIs to this bucket (the handler's
# GCS_URI_NOT_ALLOWED guard). Defense against request injection.
name = "HYPERFRAMES_RENDER_BUCKET"
value = google_storage_bucket.render.name
}
startup_probe {
http_get {
path = "/healthz"
}
timeout_seconds = 5
period_seconds = 10
failure_threshold = 6
}
}
}
}
# Only the workflow's identity may invoke the render service.
resource "google_cloud_run_v2_service_iam_member" "workflow_invokes_run" {
name = google_cloud_run_v2_service.render.name
project = var.project_id
location = var.region
role = "roles/run.invoker"
member = "serviceAccount:${google_service_account.workflow_sa.email}"
}
# ── Cloud Workflows orchestration ────────────────────────────────────────────
resource "google_workflows_workflow" "render" {
name = "${local.name}-render"
project = var.project_id
region = var.region
service_account = google_service_account.workflow_sa.id
source_contents = file(local.workflow_source)
# Allow `terraform destroy` to remove the workflow without a manual step;
# the definition is reproducible from this module.
deletion_protection = false
}
# ── Runaway-request alert (backstop against a fan-out bug) ────────────────────
resource "google_monitoring_alert_policy" "runaway_requests" {
project = var.project_id
display_name = "${local.name}-render runaway request count"
combiner = "OR"
conditions {
display_name = "Render service request count > threshold (1h)"
condition_threshold {
filter = join(" AND ", [
"resource.type = \"cloud_run_revision\"",
"resource.labels.service_name = \"${google_cloud_run_v2_service.render.name}\"",
"metric.type = \"run.googleapis.com/request_count\"",
])
comparison = "COMPARISON_GT"
threshold_value = var.render_request_alarm_threshold
duration = "0s"
aggregations {
alignment_period = "3600s"
per_series_aligner = "ALIGN_SUM"
}
}
}
notification_channels = var.notification_channels
}
# ── Workflow-failure alert ───────────────────────────────────────────────────
# Request-count alone misses a render that fails 100% of the time at low
# volume. Alert on any FAILED workflow execution so a broken render path is
# visible even when traffic is light.
resource "google_monitoring_alert_policy" "workflow_failures" {
project = var.project_id
display_name = "${local.name}-render workflow execution failures"
combiner = "OR"
conditions {
display_name = "Failed workflow executions (5m)"
condition_threshold {
filter = join(" AND ", [
"resource.type = \"workflows.googleapis.com/Workflow\"",
"resource.labels.workflow_id = \"${google_workflows_workflow.render.name}\"",
"metric.type = \"workflows.googleapis.com/finished_execution_count\"",
"metric.labels.status = \"FAILED\"",
])
comparison = "COMPARISON_GT"
threshold_value = 0
duration = "0s"
aggregations {
alignment_period = "300s"
per_series_aligner = "ALIGN_SUM"
}
}
}
notification_channels = var.notification_channels
}
@@ -0,0 +1,34 @@
output "render_bucket_name" {
description = "GCS bucket holding plan tarballs, chunk outputs, and final renders. Pass as renderToCloudRun({ bucketName })."
value = google_storage_bucket.render.name
}
output "service_url" {
description = "HTTPS URL of the Cloud Run render service. Pass as renderToCloudRun({ serviceUrl })."
value = google_cloud_run_v2_service.render.uri
}
output "workflow_name" {
description = "Workflow id. Pass as renderToCloudRun({ workflowId })."
value = google_workflows_workflow.render.name
}
output "workflow_id_full" {
description = "Fully-qualified workflow resource name."
value = google_workflows_workflow.render.id
}
output "run_service_account_email" {
description = "Render service identity (read/write the render bucket)."
value = google_service_account.run_sa.email
}
output "workflow_service_account_email" {
description = "Workflow identity (invokes the render service)."
value = google_service_account.workflow_sa.email
}
output "region" {
description = "Region everything was deployed into. Pass as renderToCloudRun({ location })."
value = var.region
}
@@ -0,0 +1,12 @@
# This module is applied directly (by `examples/gcp-cloud-run/scripts/smoke.sh`
# and `hyperframes cloudrun deploy`), so it configures the google provider
# from its own variables. Credentials come from the environment — either
# Application Default Credentials (`gcloud auth application-default login`)
# or a `GOOGLE_OAUTH_ACCESS_TOKEN` env var.
#
# If you instead embed this as a CHILD module, delete this block and pass a
# configured provider from your root module.
provider "google" {
project = var.project_id
region = var.region
}
@@ -0,0 +1,75 @@
variable "project_id" {
type = string
description = "GCP project id to deploy the render stack into."
}
variable "region" {
type = string
description = "Region for the Cloud Run service, Workflow, and bucket."
default = "us-central1"
}
variable "project_name" {
type = string
description = "Name prefix applied to the service / workflow / bucket / service accounts."
default = "hyperframes"
}
variable "image" {
type = string
description = "Fully-qualified container image for the render service (e.g. us-central1-docker.pkg.dev/PROJECT/REPO/hyperframes-render:TAG), built from packages/gcp-cloud-run/Dockerfile."
}
variable "cpu" {
type = string
description = "vCPU per Cloud Run instance. Allowed: 1, 2, 4, 8. Renders are CPU-bound; 4 is a good default."
default = "4"
}
variable "memory" {
type = string
description = "Memory per Cloud Run instance. Must be ≥ 2Gi per vCPU at cpu=4. Headroom for Chrome + ffmpeg + the chunk's frames in /tmp."
default = "16Gi"
}
variable "request_timeout_seconds" {
type = number
description = "Per-request timeout. Cloud Run hard cap is 3600s; a single chunk should finish well inside this."
default = 3600
}
variable "min_instances" {
type = number
description = "Min Cloud Run instances. Default 0 (scale-to-zero) is cheapest but means the first render after idle pays a cold start (image pull + Chrome + bun boot, ~20-30s). Set to 1 to keep one warm if first-render latency matters."
default = 0
}
variable "max_instances" {
type = number
description = "Max Cloud Run instances. Each chunk pins one instance (request concurrency = 1), and the workflow fans out up to the plan's chunk count (which never exceeds Config.maxParallelChunks). Keep this >= the largest maxParallelChunks you render with, or excess chunks queue behind 429s + retry backoff. Also a runaway-cost backstop."
default = 100
}
variable "workflow_source_path" {
type = string
description = "Path to the Cloud Workflows YAML. Defaults to the copy shipped with this module."
default = ""
}
variable "render_request_alarm_threshold" {
type = number
description = "Cloud Monitoring alert fires when render-service request count exceeds this in a 1-hour window. Backstop against a runaway fan-out."
default = 1000
}
variable "notification_channels" {
type = list(string)
description = "Cloud Monitoring notification channel ids for the runaway-request alert. Empty disables notifications (the policy still records)."
default = []
}
variable "bucket_force_destroy" {
type = bool
description = "Allow `terraform destroy` to delete the render bucket even when it still holds objects. Off by default to match the AWS adapter's RETAIN policy."
default = false
}
@@ -0,0 +1,9 @@
terraform {
required_version = ">= 1.5.0"
required_providers {
google = {
source = "hashicorp/google"
version = ">= 5.0.0"
}
}
}
@@ -0,0 +1,179 @@
# HyperFrames distributed render orchestration on Cloud Workflows.
#
# Plan → BuildChunkList → AssertChunkCount → RenderChunks (parallel) → Assemble
#
# Mirrors the Step Functions state machine in
# `examples/aws-lambda/template.yaml`. Every step POSTs to the same Cloud Run
# service URL (passed in as `args.ServiceUrl`) and varies only the body's
# `Action`. The service returns the step's small result body on 2xx; on a
# non-retryable failure it returns HTTP 400, on a retryable failure HTTP 5xx —
# the `retryable` predicate below keys off exactly that split.
#
# The final returned object accumulates every step's result body so
# `getRenderProgress` can read frame totals + per-step durations on success:
# { Plan: {...}, Chunks: [{...}, ...], Assemble: {...} }
#
# Deploy with `gcloud workflows deploy` (the Terraform module / the
# `hyperframes cloudrun deploy` command do this for you).
main:
params: [args]
steps:
- init:
assign:
- serviceUrl: ${args.ServiceUrl}
- projectGcsUri: ${args.ProjectGcsUri}
- planOutputGcsPrefix: ${args.PlanOutputGcsPrefix}
- outputGcsUri: ${args.OutputGcsUri}
- config: ${args.Config}
# ── Plan (Activity A) ────────────────────────────────────────────────────
- plan:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
body:
Action: plan
ProjectGcsUri: ${projectGcsUri}
PlanOutputGcsPrefix: ${planOutputGcsPrefix}
Config: ${config}
result: planResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
- capturePlan:
assign:
- planResult: ${planResp.body}
- chunkCount: ${planResult.ChunkCount}
# ── BuildChunkList + AssertChunkCount ──────────────────────────────────────
- assertChunkCount:
switch:
- condition: ${chunkCount > 0}
next: buildChunkList
next: planProducedZeroChunks
- planProducedZeroChunks:
raise:
code: PLAN_PRODUCED_ZERO_CHUNKS
message: "Plan returned ChunkCount=0 — the composition produced no frames. Non-retryable producer-side invariant violation."
- buildChunkList:
# Pre-size the ordered chunk-URI + per-chunk result lists so the
# parallel branches below assign by index (distinct indices, no
# read-modify-write race on a shared accumulator).
assign:
- chunkIndexes: []
- chunkUris: []
- chunkResults: []
- fillLists:
for:
value: i
range: [0, ${chunkCount - 1}]
steps:
- appendSlots:
assign:
- chunkIndexes: ${list.concat(chunkIndexes, i)}
- chunkUris: ${list.concat(chunkUris, "")}
- chunkResults: ${list.concat(chunkResults, "")}
# ── RenderChunks (Activity B, fanned out) ──────────────────────────────────
- renderChunks:
parallel:
shared: [chunkUris, chunkResults]
# Run up to chunkCount chunks at once, clamped to 20 — Cloud
# Workflows hard-caps concurrent branches/iterations per execution
# at 20 (https://cloud.google.com/workflows/quotas). Above that,
# iterations queue regardless of concurrency_limit, so a config with
# maxParallelChunks > 20 still renders correctly; the extra chunks
# just wait. All chunkCount iterations always run.
concurrency_limit: ${math.min(chunkCount, 20)}
for:
value: idx
in: ${chunkIndexes}
steps:
- renderOneChunk:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
body:
Action: renderChunk
ChunkIndex: ${idx}
PlanGcsUri: ${planResult.PlanGcsUri}
PlanHash: ${planResult.PlanHash}
ChunkOutputGcsPrefix: ${planOutputGcsPrefix}
Format: ${planResult.Format}
result: chunkResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
- storeChunk:
assign:
- chunkUris[idx]: ${chunkResp.body.ChunkGcsUri}
- chunkResults[idx]: ${chunkResp.body}
# ── Assemble (Activity C) ──────────────────────────────────────────────────
- assemble:
try:
call: http.post
args:
url: ${serviceUrl}
timeout: 1800
auth:
type: OIDC
body:
Action: assemble
PlanGcsUri: ${planResult.PlanGcsUri}
ChunkGcsUris: ${chunkUris}
AudioGcsUri: ${planResult.AudioGcsUri}
OutputGcsUri: ${outputGcsUri}
Format: ${planResult.Format}
# Forward the caller's exact-CFR request (Config.cfr) to assemble.
# `"cfr" in config` guards the optional key; when unset this is
# false, which the handler reads as the default -c copy path.
Cfr: ${("cfr" in config) and config.cfr}
result: assembleResp
retry:
predicate: ${retryable}
max_retries: 4
backoff:
initial_delay: 2
max_delay: 60
multiplier: 2
- done:
return:
Plan: ${planResult}
Chunks: ${chunkResults}
Assemble: ${assembleResp.body}
# Retry predicate: retry transient/server failures (429 + 5xx), never the
# handler's non-retryable 400s (bad input, plan-hash mismatch, unsupported
# format, …). Connection / timeout errors carry no `.code`; retry those too.
retryable:
params: [e]
steps:
- classify:
switch:
- condition: ${not("code" in e)}
return: true
- condition: ${e.code == 429}
return: true
- condition: ${e.code >= 500 and e.code < 600}
return: true
- nonRetryable:
return: false
@@ -0,0 +1,12 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"paths": {},
"noEmit": false,
"outDir": "./dist",
"rootDir": "./src",
"declaration": true,
"declarationMap": true,
"sourceMap": true
}
}
+19
View File
@@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"esModuleInterop": true,
"strict": true,
"noUncheckedIndexedAccess": true,
"skipLibCheck": true,
"noEmit": true,
"baseUrl": ".",
"paths": {
"@hyperframes/producer": ["../producer/src/index.ts"],
"@hyperframes/producer/distributed": ["../producer/src/distributed.ts"]
}
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist", "src/**/*.test.ts", "src/**/__fixtures__/**", "scripts"]
}