chore: import upstream snapshot with attribution
CodeQL / Analyze (javascript-typescript) (push) Waiting to run
Docs / Validate docs (push) Waiting to run
CodeQL / Analyze (actions) (push) Waiting to run
CodeQL / Analyze (python) (push) Waiting to run
Sync skills to ClawHub / Publish changed skills (push) Waiting to run
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Waiting to run
Docs / Validate docs (push) Waiting to run
CodeQL / Analyze (actions) (push) Waiting to run
CodeQL / Analyze (python) (push) Waiting to run
Sync skills to ClawHub / Publish changed skills (push) Waiting to run
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
# @hyperframes/aws-lambda
|
||||
|
||||
AWS Lambda adapter for HyperFrames distributed rendering. Ships three
|
||||
things together:
|
||||
|
||||
1. The **Lambda handler** that wraps the OSS `plan` / `renderChunk` /
|
||||
`assemble` primitives behind a single dispatch boundary Step Functions
|
||||
can drive (`src/handler.ts`).
|
||||
2. A **client-side SDK** — `renderToLambda`, `getRenderProgress`,
|
||||
`deploySite`, plus `validateDistributedRenderConfig` and
|
||||
`computeRenderCost` (`src/sdk/`).
|
||||
3. An **`aws-cdk-lib` L2 construct** (`HyperframesRenderStack`) that
|
||||
provisions the same topology as `examples/aws-lambda/template.yaml`
|
||||
inside an adopter's own CDK app (`src/cdk/`).
|
||||
|
||||
The handler ZIP and the SAM template still drive a maintainer-run real-AWS
|
||||
smoke flow; the SDK + CDK are the supported public surface for adopters.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ Step Functions state machine │
|
||||
│ Plan → Map(N) RenderChunk → Assemble │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
│ dispatches by event.Action
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ One Lambda function (this package's `dist/handler.zip`) │
|
||||
│ handler.mjs │
|
||||
│ ├─ Action="plan" → @hyperframes/producer/distributed │
|
||||
│ ├─ Action="renderChunk" → @hyperframes/producer/distributed │
|
||||
│ └─ Action="assemble" → @hyperframes/producer/distributed │
|
||||
│ bin/ffmpeg — ffmpeg-static │
|
||||
│ node_modules/@sparticuz/chromium/ — Lambda-optimised Chromium │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
│ pure functions over local paths
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ S3 bucket — plan tarball + per-chunk outputs + final mp4 │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The handler downloads inputs from S3 into `/tmp`, calls the OSS primitive,
|
||||
uploads outputs back to S3, and returns a small JSON result that fits
|
||||
inside Step Functions' history budget (under 200 bytes per chunk).
|
||||
|
||||
## Chrome runtime
|
||||
|
||||
The package supports two Chromium sources:
|
||||
|
||||
| Source | Default | Size | When to pick it |
|
||||
| ------------------------------- | ------- | ------------------ | --------------------------------------------------------------------------------------------------------------------- |
|
||||
| `@sparticuz/chromium` | yes | ~70 MiB compressed | Lambda. Decompresses into `/tmp` at runtime; the rest of the ecosystem already uses it for headless-Chrome-in-Lambda. |
|
||||
| Bundled `chrome-headless-shell` | no | ~140 MiB | Fallback. Used if `@sparticuz/chromium` ever drops `HeadlessExperimental.beginFrame` support. |
|
||||
|
||||
Pick the source at build time:
|
||||
|
||||
```bash
|
||||
bun run --cwd packages/aws-lambda build:zip
|
||||
bun run --cwd packages/aws-lambda build:zip -- --source=chrome-headless-shell
|
||||
```
|
||||
|
||||
The handler reads `HYPERFRAMES_LAMBDA_CHROME_SOURCE` at boot. The build
|
||||
script sets that env var via Lambda function configuration in
|
||||
`examples/aws-lambda/template.yaml`.
|
||||
|
||||
## BeginFrame regression guard
|
||||
|
||||
HyperFrames' renderer drives Chrome via the CDP
|
||||
`HeadlessExperimental.beginFrame` command — same path the K8s deploy uses.
|
||||
The Lambda adapter assumes that `@sparticuz/chromium`'s
|
||||
chrome-headless-shell build honours BeginFrame. To prove it (and re-prove
|
||||
it on every release), the package ships a Docker probe:
|
||||
|
||||
```bash
|
||||
# Build the Lambda-like container and run the probe.
|
||||
bun run --cwd packages/aws-lambda probe:beginframe:docker
|
||||
```
|
||||
|
||||
The probe boots `@sparticuz/chromium` inside
|
||||
`public.ecr.aws/lambda/nodejs:22` and asserts CDP `beginFrame` with
|
||||
`screenshot: true` returns a PNG buffer. Exit code 0 = green; non-zero =
|
||||
fall back to bundling chrome-headless-shell directly via `--source=chrome-headless-shell`.
|
||||
|
||||
## Building the ZIP
|
||||
|
||||
```bash
|
||||
bun install # at the monorepo root
|
||||
bun run --cwd packages/aws-lambda build:zip # → packages/aws-lambda/dist/handler.zip
|
||||
bun run --cwd packages/aws-lambda verify:zip-size # CI gate
|
||||
```
|
||||
|
||||
The build script bundles `src/handler.ts` via esbuild, stages
|
||||
`@sparticuz/chromium` and `puppeteer-core` under `node_modules/`, copies
|
||||
ffmpeg-static into `bin/`, and zips the result. The unzipped layout is
|
||||
designed to extract cleanly into Lambda's `/var/task/`.
|
||||
|
||||
`verify:zip-size` enforces:
|
||||
|
||||
- Unzipped ≤ 248 MiB (in-house budget; Lambda hard ceiling is 250 MiB unzipped — AWS docs label this "250 MB" but use binary mebibytes)
|
||||
- Zipped ≤ 150 MiB (in-house budget; Lambda has no hard zipped cap for S3-deployed functions)
|
||||
|
||||
CI fails the PR if either is exceeded.
|
||||
|
||||
## Running tests
|
||||
|
||||
```bash
|
||||
bun run --cwd packages/aws-lambda test # unit tests (no Chrome)
|
||||
bun run --cwd packages/aws-lambda probe:beginframe # local probe (Linux only)
|
||||
```
|
||||
|
||||
## Using the SDK
|
||||
|
||||
After deploying the stack (via the SAM template, CDK construct below, or
|
||||
your own CFN of choice), drive renders from Node:
|
||||
|
||||
```ts
|
||||
import { deploySite, getRenderProgress, renderToLambda } from "@hyperframes/aws-lambda";
|
||||
|
||||
// One-time upload per project version.
|
||||
const site = await deploySite({
|
||||
projectDir: "./my-composition",
|
||||
bucketName: "hyperframes-render-bucket",
|
||||
});
|
||||
|
||||
// Start a render. Returns immediately — does NOT poll.
|
||||
const handle = await renderToLambda({
|
||||
siteHandle: site,
|
||||
bucketName: site.bucketName,
|
||||
stateMachineArn: "arn:aws:states:us-east-1:123:stateMachine:hyperframes-render",
|
||||
config: {
|
||||
fps: 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4",
|
||||
chunkSize: 240,
|
||||
maxParallelChunks: 16,
|
||||
runtimeCap: "lambda",
|
||||
},
|
||||
});
|
||||
|
||||
// Poll progress + cost on your own cadence.
|
||||
const progress = await getRenderProgress({ executionArn: handle.executionArn });
|
||||
console.log(progress.overallProgress, progress.costs.displayCost);
|
||||
if (progress.status === "SUCCEEDED" && progress.outputFile) {
|
||||
console.log("Render landed at", progress.outputFile.s3Uri);
|
||||
}
|
||||
```
|
||||
|
||||
`renderToLambda` validates the config client-side via
|
||||
`validateDistributedRenderConfig` and throws a typed `InvalidConfigError`
|
||||
before the Step Functions execution starts, so shape errors surface
|
||||
synchronously instead of as opaque `ExecutionFailed` results.
|
||||
|
||||
`getRenderProgress` reports an approximate per-render cost
|
||||
(`accruedSoFarUsd` plus a formatted `displayCost`) derived from Lambda
|
||||
billed-duration × memory × the us-east-1 on-demand rate plus the Step
|
||||
Functions transition price. The math is documented in
|
||||
`src/sdk/costAccounting.ts`; numbers are best-effort and exclude S3
|
||||
transfer.
|
||||
|
||||
## Using the CDK construct
|
||||
|
||||
```ts
|
||||
import { App, Stack } from "aws-cdk-lib";
|
||||
import { HyperframesRenderStack } from "@hyperframes/aws-lambda/cdk";
|
||||
|
||||
const app = new App();
|
||||
const stack = new Stack(app, "MyApp");
|
||||
const render = new HyperframesRenderStack(stack, "Render", {
|
||||
// optional: reservedConcurrency: 8,
|
||||
// optional: lambdaMemoryMb: 10240,
|
||||
// optional: chromeSource: "sparticuz",
|
||||
});
|
||||
|
||||
// Re-export so an adopter app can wire dashboards / SNS topics.
|
||||
new CfnOutput(stack, "RenderBucketName", { value: render.bucket.bucketName });
|
||||
new CfnOutput(stack, "StateMachineArn", { value: render.stateMachine.stateMachineArn });
|
||||
```
|
||||
|
||||
`aws-cdk-lib` and `constructs` are **optional peer dependencies**: SDK-only
|
||||
consumers don't pull them at runtime. The construct itself imports from
|
||||
`@hyperframes/aws-lambda/cdk`.
|
||||
|
||||
## What's still ahead
|
||||
|
||||
- `hyperframes lambda` CLI (deploy / sites create / render / progress / destroy) — PR 6.5.
|
||||
- IAM bootstrap subcommand (`policies role | user | validate`) — PR 6.9.
|
||||
- Lambda-local regression harness (`--mode=lambda-local`) — PR 6.6.
|
||||
- Adopter-facing migration guide — PR 6.8.
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build script for @hyperframes/aws-lambda (public OSS package).
|
||||
*
|
||||
* Bundles each subpath barrel via esbuild → dist/, then emits .d.ts via tsc.
|
||||
*
|
||||
* Subpaths (each gets its own dist entry so adopters that import one path
|
||||
* don't load the others' transitive graphs at module-load time):
|
||||
*
|
||||
* . (the umbrella barrel: handler + sdk + cdk types re-exported)
|
||||
* ./handler (the Lambda runtime entry — what `scripts/build-zip.ts` ZIPs)
|
||||
* ./sdk (client-side helpers — AWS-SDK only, no chromium/puppeteer)
|
||||
* ./cdk (CDK L2 construct — aws-cdk-lib is a peer dep)
|
||||
*
|
||||
* All production deps and peer deps are kept external so consumers resolve
|
||||
* them via their own node_modules.
|
||||
*/
|
||||
|
||||
import { build } from "esbuild";
|
||||
import { execSync } from "node:child_process";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
|
||||
rmSync("dist", { recursive: true, force: true });
|
||||
mkdirSync("dist", { recursive: true });
|
||||
|
||||
const sharedOpts = {
|
||||
bundle: true,
|
||||
platform: "node",
|
||||
target: "node22",
|
||||
format: "esm",
|
||||
minify: false,
|
||||
sourcemap: true,
|
||||
external: [
|
||||
"@aws-sdk/client-s3",
|
||||
"@aws-sdk/client-sfn",
|
||||
"@hyperframes/producer",
|
||||
"@hyperframes/producer/distributed",
|
||||
"@sparticuz/chromium",
|
||||
"aws-cdk-lib",
|
||||
"constructs",
|
||||
"ffmpeg-static",
|
||||
"ffprobe-static",
|
||||
"puppeteer-core",
|
||||
"tar",
|
||||
],
|
||||
};
|
||||
|
||||
await Promise.all([
|
||||
build({ ...sharedOpts, entryPoints: ["src/index.ts"], outfile: "dist/index.js" }),
|
||||
build({ ...sharedOpts, entryPoints: ["src/handler.ts"], outfile: "dist/handler.js" }),
|
||||
build({ ...sharedOpts, entryPoints: ["src/sdk/index.ts"], outfile: "dist/sdk/index.js" }),
|
||||
build({ ...sharedOpts, entryPoints: ["src/cdk/index.ts"], outfile: "dist/cdk/index.js" }),
|
||||
]);
|
||||
|
||||
// esbuild doesn't emit .d.ts. tsc does, with a build-only tsconfig that
|
||||
// drops the workspace `paths` overrides so `@hyperframes/producer` resolves
|
||||
// through node_modules to the sibling package's already-built `dist/`
|
||||
// types instead of pulling its full source tree into emit (which would
|
||||
// violate rootDir).
|
||||
execSync("tsc -p tsconfig.build.json --emitDeclarationOnly", { stdio: "inherit" });
|
||||
|
||||
console.log("[Build] Complete: dist/{index,handler,sdk/index,cdk/index}.js + .d.ts");
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"name": "@hyperframes/aws-lambda",
|
||||
"version": "0.7.55",
|
||||
"description": "AWS Lambda adapter for HyperFrames distributed rendering — handler, client-side SDK, and CDK construct.",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/heygen-com/hyperframes",
|
||||
"directory": "packages/aws-lambda"
|
||||
},
|
||||
"files": [
|
||||
"dist/",
|
||||
"scripts/",
|
||||
"README.md"
|
||||
],
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
},
|
||||
"./handler": {
|
||||
"import": "./dist/handler.js",
|
||||
"types": "./dist/handler.d.ts"
|
||||
},
|
||||
"./sdk": {
|
||||
"import": "./dist/sdk/index.js",
|
||||
"types": "./dist/sdk/index.d.ts"
|
||||
},
|
||||
"./cdk": {
|
||||
"import": "./dist/cdk/index.js",
|
||||
"types": "./dist/cdk/index.d.ts"
|
||||
}
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"registry": "https://registry.npmjs.org/"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "node build.mjs",
|
||||
"build:zip": "tsx scripts/build-zip.ts",
|
||||
"probe:beginframe": "tsx scripts/probe-beginframe.ts",
|
||||
"probe:beginframe:docker": "docker build -f scripts/probe-beginframe.dockerfile -t hyperframes-lambda-probe:local ../.. && docker run --rm hyperframes-lambda-probe:local",
|
||||
"test": "bun test",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"verify:zip-size": "tsx scripts/verify-zip-size.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.700.0",
|
||||
"@aws-sdk/client-sfn": "^3.700.0",
|
||||
"@hyperframes/producer": "workspace:^",
|
||||
"@sparticuz/chromium": "148.0.0",
|
||||
"ffmpeg-static": "^5.2.0",
|
||||
"ffprobe-static": "^3.1.0",
|
||||
"puppeteer-core": "^25.2.1",
|
||||
"tar": "^7.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/aws-lambda": "^8.10.146",
|
||||
"@types/node": "^25.0.10",
|
||||
"@types/tar": "^6.1.13",
|
||||
"aws-cdk-lib": "^2.130.0",
|
||||
"constructs": "^10.3.0",
|
||||
"esbuild": "^0.25.12",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5.7.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"aws-cdk-lib": "^2.130.0",
|
||||
"constructs": "^10.3.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"aws-cdk-lib": {
|
||||
"optional": true
|
||||
},
|
||||
"constructs": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Shared binary-unit byte formatter for the build/verify scripts.
|
||||
*
|
||||
* The Lambda ZIP-size budget is in mebibytes (Lambda's 250 MB / 248 MiB
|
||||
* gate is binary, not decimal), so logs and CI failure messages use
|
||||
* KiB / MiB / GiB. This is intentionally a different unit system from
|
||||
* `packages/cli/src/ui/format.ts`'s `formatBytes` (KB / MB, decimal) —
|
||||
* don't conflate them.
|
||||
*/
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GiB`;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* CJS-interop banner prepended to the ESM handler bundle.
|
||||
*
|
||||
* Lambda's Node 22 runtime treats `.mjs` as ESM, which has no `require`,
|
||||
* `__filename`, or `__dirname`. The handler bundle inlines CJS deps that
|
||||
* assume all three exist at module scope:
|
||||
* - postcss et al. call top-level `require(...)`
|
||||
* - wawoff2's emscripten build (pulled in unconditionally via
|
||||
* producer → fontCompression) reads `__dirname` at module scope
|
||||
* Without the shims the handler throws "Dynamic require of <X> is not
|
||||
* supported" / "__dirname is not defined in ES module scope" at import time,
|
||||
* before it can run — which is exactly how a freshly deployed stack crashed
|
||||
* on every render (#1932).
|
||||
*
|
||||
* This mirrors the producer's own CJS banner (packages/producer/build.mjs);
|
||||
* the handler bundle inlines producer source, so it needs the same shim.
|
||||
*
|
||||
* Kept in its own module (not inline in build-zip.ts, which self-executes on
|
||||
* import) so build-zip.test.ts can import the exact banner, bundle a fixture
|
||||
* with it, and assert the globals actually resolve.
|
||||
*/
|
||||
export const HANDLER_BANNER = [
|
||||
"// hyperframes-aws-lambda handler bundle",
|
||||
'import { createRequire as __hf_createRequire } from "module";',
|
||||
'import { fileURLToPath as __hf_fileURLToPath } from "url";',
|
||||
'import { dirname as __hf_dirname } from "path";',
|
||||
"const require = __hf_createRequire(import.meta.url);",
|
||||
"const __filename = __hf_fileURLToPath(import.meta.url);",
|
||||
"const __dirname = __hf_dirname(__filename);",
|
||||
].join("\n");
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import * as esbuild from "esbuild";
|
||||
import { HANDLER_BANNER } from "./_handlerBanner.js";
|
||||
|
||||
// The handler ships as ESM (.mjs) but inlines CJS deps that assume Node's CJS
|
||||
// globals exist at module scope: postcss et al. call top-level `require(...)`,
|
||||
// and wawoff2's emscripten build reads `__dirname`. A freshly deployed stack
|
||||
// crashed on every render (#1932) with "__dirname is not defined in ES module
|
||||
// scope"; the fix is the require/__filename/__dirname shim in HANDLER_BANNER.
|
||||
//
|
||||
// This bundles a fixture touching all three globals with the REAL banner and
|
||||
// imports the output, so it catches a dropped/renamed shim behaviourally
|
||||
// rather than by grepping for literals (which survives a broken refactor).
|
||||
//
|
||||
// The import MUST run under Node, not the `bun test` runtime: Bun defines
|
||||
// `__dirname`/`__filename` even in ESM, which would mask a missing shim and
|
||||
// green-light a broken bundle. Lambda runs Node, so we spawn `node` (guaranteed
|
||||
// present in CI alongside bun) to reproduce the deploy target faithfully.
|
||||
describe("build-zip handler banner", () => {
|
||||
it("shims require/__filename/__dirname so inlined CJS deps import under Node", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-banner-test-"));
|
||||
try {
|
||||
const entry = join(dir, "fixture.ts");
|
||||
const outfile = join(dir, "out.mjs");
|
||||
// Reference each CJS global at module top level, the way inlined deps do.
|
||||
// If any shim is missing, importing `out.mjs` throws at eval time.
|
||||
writeFileSync(
|
||||
entry,
|
||||
[
|
||||
"const cjsDir = __dirname;",
|
||||
"const cjsFile = __filename;",
|
||||
"const path = require('node:path');",
|
||||
"if (typeof cjsDir !== 'string') throw new Error('__dirname missing');",
|
||||
"if (typeof cjsFile !== 'string') throw new Error('__filename missing');",
|
||||
"if (typeof path.join !== 'function') throw new Error('require missing');",
|
||||
"console.log('BANNER_OK');",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
esbuild.buildSync({
|
||||
bundle: true,
|
||||
platform: "node",
|
||||
target: "node22",
|
||||
format: "esm",
|
||||
entryPoints: [entry],
|
||||
outfile,
|
||||
banner: { js: HANDLER_BANNER },
|
||||
});
|
||||
|
||||
// Import under real Node — Lambda's runtime — not the bun test runtime.
|
||||
const res = spawnSync(
|
||||
"node",
|
||||
[
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
`await import(${JSON.stringify(pathToFileURL(outfile).href)});`,
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
|
||||
// A missing shim surfaces as a non-zero exit + ReferenceError on stderr.
|
||||
// Guard against a silent skip if `node` isn't on PATH (it is in CI).
|
||||
expect(res.error).toBeUndefined();
|
||||
expect(res.stderr).not.toContain("is not defined in ES module scope");
|
||||
expect(res.stderr).not.toContain("Dynamic require");
|
||||
expect(res.status).toBe(0);
|
||||
expect(res.stdout).toContain("BANNER_OK");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,530 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Build the AWS Lambda deployment ZIP.
|
||||
*
|
||||
* Pack layout (paths inside the ZIP are relative to Lambda's
|
||||
* `/var/task/`):
|
||||
*
|
||||
* handler.mjs — bundled entry, set as Lambda's Handler
|
||||
* handler.mjs.map — sourcemap (debugging aid; small)
|
||||
* bin/ffmpeg — ffmpeg-static binary
|
||||
* bin/chrome-headless-shell — fallback Chrome (only when CHROME_SOURCE=shell)
|
||||
* node_modules/@sparticuz/chromium/
|
||||
* — primary Chrome (lives under node_modules so
|
||||
* runtime `import("@sparticuz/chromium")`
|
||||
* resolves; the package's own tarball stays
|
||||
* inside).
|
||||
*
|
||||
* The handler bundle (esbuild) externalises modules whose binary assets
|
||||
* must be present at runtime — `@sparticuz/chromium` for its bin tarball,
|
||||
* `puppeteer-core` because Lambda runtime resolves it via Node module
|
||||
* resolution from `node_modules/`. Everything else is inlined for cold
|
||||
* start speed.
|
||||
*
|
||||
* Run:
|
||||
* bun run --cwd packages/aws-lambda build:zip
|
||||
* bun run --cwd packages/aws-lambda build:zip -- --source=chrome-headless-shell
|
||||
*
|
||||
* Outputs the resolved ZIP path + size to stdout and writes a sidecar
|
||||
* JSON (`dist/handler.zip.manifest.json`) describing the contents.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
closeSync,
|
||||
cpSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import * as esbuild from "esbuild";
|
||||
import { formatBytes } from "./_formatBytes.js";
|
||||
import { HANDLER_BANNER } from "./_handlerBanner.js";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const packageRoot = resolve(scriptDir, "..");
|
||||
const monorepoRoot = resolve(packageRoot, "../..");
|
||||
const distDir = join(packageRoot, "dist");
|
||||
|
||||
interface BuildOptions {
|
||||
source: "sparticuz" | "chrome-headless-shell";
|
||||
/** Hard upper bound on the unzipped bundle size in bytes (Lambda limit is 250 MiB). */
|
||||
maxUnzippedBytes: number;
|
||||
/** Hard upper bound on the ZIP file size in bytes. */
|
||||
maxZippedBytes: number;
|
||||
}
|
||||
|
||||
const DEFAULT_OPTIONS: BuildOptions = {
|
||||
source: "sparticuz",
|
||||
// Lambda's hard ceiling for ZIP-deployed functions is 250 MiB unzipped
|
||||
// (AWS docs label it "250 MB" but the 262144000-byte value is 250
|
||||
// binary mebibytes). We gate at 248 MiB to keep ~2 MiB of headroom —
|
||||
// the sparticuz Chrome (~70 MiB) + ffmpeg (~80 MiB) + ffprobe (~62
|
||||
// MiB) + bundled Node deps put us close to the ceiling. Chrome itself
|
||||
// decompresses into Lambda's `/tmp` at cold start, which has its own
|
||||
// 10 GiB budget, so the unzipped /var/task footprint above is what
|
||||
// actually competes with Lambda's 250 MiB limit.
|
||||
maxUnzippedBytes: 248 * 1024 * 1024,
|
||||
// Lambda's only zipped-size cap is for direct console/CLI uploads (50
|
||||
// MiB); S3-deployed functions are bounded by the unzipped ceiling. We
|
||||
// gate at 150 MiB to flag a sudden bundle-size regression without
|
||||
// false-failing on the natural ~100 MiB sparticuz + ffmpeg payload.
|
||||
maxZippedBytes: 150 * 1024 * 1024,
|
||||
};
|
||||
|
||||
function parseArgs(argv: string[]): BuildOptions {
|
||||
const opts = { ...DEFAULT_OPTIONS };
|
||||
for (const arg of argv.slice(2)) {
|
||||
if (arg.startsWith("--source=")) {
|
||||
const v = arg.slice("--source=".length);
|
||||
if (v !== "sparticuz" && v !== "chrome-headless-shell") {
|
||||
throw new Error(`--source must be 'sparticuz' or 'chrome-headless-shell' (got ${v})`);
|
||||
}
|
||||
opts.source = v;
|
||||
} else if (arg.startsWith("--max-unzipped=")) {
|
||||
opts.maxUnzippedBytes = Number.parseInt(arg.slice("--max-unzipped=".length), 10);
|
||||
} else if (arg.startsWith("--max-zipped=")) {
|
||||
opts.maxZippedBytes = Number.parseInt(arg.slice("--max-zipped=".length), 10);
|
||||
} else if (arg === "--help") {
|
||||
console.log(
|
||||
"Usage: tsx build-zip.ts [--source=sparticuz|chrome-headless-shell]\n" +
|
||||
" [--max-unzipped=<bytes>] [--max-zipped=<bytes>]",
|
||||
);
|
||||
process.exit(0);
|
||||
} else {
|
||||
throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const opts = parseArgs(process.argv);
|
||||
const start = Date.now();
|
||||
|
||||
rmSync(distDir, { recursive: true, force: true });
|
||||
mkdirSync(distDir, { recursive: true });
|
||||
|
||||
const stagingDir = join(distDir, "staging");
|
||||
mkdirSync(stagingDir, { recursive: true });
|
||||
|
||||
console.log(`[build-zip] source=${opts.source}`);
|
||||
|
||||
// 1. Bundle the handler.
|
||||
await bundleHandler(stagingDir);
|
||||
|
||||
// 2. Stage runtime modules (puppeteer-core + @sparticuz/chromium or the
|
||||
// fallback chrome-headless-shell tar).
|
||||
stageRuntimeModules(stagingDir, opts.source);
|
||||
|
||||
// 3. Stage the ffmpeg binary.
|
||||
stageFfmpeg(stagingDir);
|
||||
|
||||
// 3b. Stage the hyperframe runtime manifest + IIFE as siblings of
|
||||
// handler.mjs. The producer's `hyperframeRuntimeLoader` checks
|
||||
// SIBLING_MANIFEST_PATH first, so dropping the manifest alongside
|
||||
// the bundled handler at /var/task/hyperframe.manifest.json lets
|
||||
// renderChunk find it without needing PRODUCER_HYPERFRAME_MANIFEST_PATH.
|
||||
stageHyperframeRuntime(stagingDir);
|
||||
|
||||
// 4. If we're on the chrome-headless-shell fallback, stage that binary.
|
||||
if (opts.source === "chrome-headless-shell") {
|
||||
stageChromeHeadlessShell(stagingDir);
|
||||
}
|
||||
|
||||
// 5. Compute the unzipped size BEFORE zipping so we fail loud when over budget.
|
||||
const unzippedBytes = directorySizeBytes(stagingDir);
|
||||
console.log(`[build-zip] unzipped staging size: ${formatBytes(unzippedBytes)}`);
|
||||
if (unzippedBytes > opts.maxUnzippedBytes) {
|
||||
throw new Error(
|
||||
`[build-zip] unzipped bundle ${formatBytes(unzippedBytes)} exceeds limit ${formatBytes(
|
||||
opts.maxUnzippedBytes,
|
||||
)} (Lambda ZIP ceiling: 250 MiB unzipped). ` +
|
||||
`Switch --source to the lighter option, or move Chrome to a Lambda Layer.`,
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Build the ZIP.
|
||||
const zipPath = join(distDir, "handler.zip");
|
||||
zipDirectory(stagingDir, zipPath);
|
||||
const zippedBytes = statSync(zipPath).size;
|
||||
console.log(`[build-zip] zip size: ${formatBytes(zippedBytes)} → ${zipPath}`);
|
||||
if (zippedBytes > opts.maxZippedBytes) {
|
||||
throw new Error(
|
||||
`[build-zip] zip ${formatBytes(zippedBytes)} exceeds ZIP size limit ${formatBytes(
|
||||
opts.maxZippedBytes,
|
||||
)}.`,
|
||||
);
|
||||
}
|
||||
|
||||
// 7. Sidecar manifest.
|
||||
const manifest = {
|
||||
builtAt: new Date().toISOString(),
|
||||
durationMs: Date.now() - start,
|
||||
source: opts.source,
|
||||
unzippedBytes,
|
||||
zippedBytes,
|
||||
maxUnzippedBytes: opts.maxUnzippedBytes,
|
||||
maxZippedBytes: opts.maxZippedBytes,
|
||||
};
|
||||
writeFileSync(join(distDir, "handler.zip.manifest.json"), JSON.stringify(manifest, null, 2));
|
||||
|
||||
// 8. Cleanup staging.
|
||||
rmSync(stagingDir, { recursive: true, force: true });
|
||||
console.log(`[build-zip] done in ${Date.now() - start}ms`);
|
||||
}
|
||||
|
||||
async function bundleHandler(stagingDir: string): Promise<void> {
|
||||
const entry = join(packageRoot, "src/handler.ts");
|
||||
const outfile = join(stagingDir, "handler.mjs");
|
||||
|
||||
const workspaceAliasPlugin: esbuild.Plugin = {
|
||||
name: "workspace-alias",
|
||||
setup(build) {
|
||||
build.onResolve({ filter: /^@hyperframes\/producer\/distributed$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/producer/src/distributed.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/producer$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/producer/src/index.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/engine$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/engine/src/index.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/engine\/alpha-blit$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/engine/src/utils/alphaBlit.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/engine\/shader-transitions$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/engine/src/utils/shaderTransitions.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/core$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/core/src/index.ts"),
|
||||
}));
|
||||
build.onResolve({ filter: /^@hyperframes\/core\/lint$/ }, () => ({
|
||||
path: resolve(monorepoRoot, "packages/core/src/lint/index.ts"),
|
||||
}));
|
||||
},
|
||||
};
|
||||
|
||||
await esbuild.build({
|
||||
bundle: true,
|
||||
platform: "node",
|
||||
target: "node22",
|
||||
format: "esm",
|
||||
// Externalise binary-shipped modules so node module resolution picks
|
||||
// them up at runtime. esbuild would otherwise try to inline their
|
||||
// postinstall-extracted binaries, which it cannot do.
|
||||
external: [
|
||||
"@sparticuz/chromium",
|
||||
"puppeteer-core",
|
||||
"puppeteer",
|
||||
// AWS SDK v3 is pre-installed in the Lambda Node 22 runtime; mark
|
||||
// external so we don't double-bundle 3+ MiB of SDK.
|
||||
"@aws-sdk/client-s3",
|
||||
],
|
||||
plugins: [workspaceAliasPlugin],
|
||||
minify: false,
|
||||
// sourcemap=false: the ZIP is tight on Lambda's 250 MiB unzipped cap
|
||||
// (Chrome ~70 MiB + ffmpeg ~80 MiB + ffprobe ~62 MiB + Node deps). A
|
||||
// 4-5 MiB sourcemap puts us over. Re-enable for local debugging by passing
|
||||
// --sourcemap; the bundle's stack traces stay readable enough without
|
||||
// it because we don't minify.
|
||||
sourcemap: false,
|
||||
entryPoints: [entry],
|
||||
outfile,
|
||||
// See HANDLER_BANNER (_handlerBanner.ts) for why the ESM bundle needs the
|
||||
// CJS require/__filename/__dirname shims.
|
||||
banner: {
|
||||
js: HANDLER_BANNER,
|
||||
},
|
||||
});
|
||||
console.log(`[build-zip] bundled handler → ${outfile}`);
|
||||
}
|
||||
|
||||
function stageRuntimeModules(stagingDir: string, source: BuildOptions["source"]): void {
|
||||
// Bun's isolated-install layout means cpSync(@sparticuz/chromium) only
|
||||
// copies the package's own files, missing transitive deps like `tar-fs`.
|
||||
// The clean cross-package-manager solution: write a tiny package.json
|
||||
// into staging/ that declares the production deps, then `npm install`
|
||||
// there. npm flattens transitive deps into staging/node_modules/.
|
||||
const pkg: Record<string, unknown> = {
|
||||
name: "hyperframes-aws-lambda-bundled",
|
||||
version: "0.0.0",
|
||||
private: true,
|
||||
dependencies: {
|
||||
"puppeteer-core": readDepVersion("puppeteer-core"),
|
||||
},
|
||||
};
|
||||
if (source === "sparticuz") {
|
||||
(pkg.dependencies as Record<string, string>)["@sparticuz/chromium"] =
|
||||
readDepVersion("@sparticuz/chromium");
|
||||
}
|
||||
writeFileSync(join(stagingDir, "package.json"), JSON.stringify(pkg, null, 2));
|
||||
|
||||
// --no-package-lock so we don't pollute staging with a lockfile we don't
|
||||
// ship; --no-audit/--no-fund just for log noise.
|
||||
const result = spawnSync(
|
||||
"npm",
|
||||
["install", "--no-package-lock", "--no-audit", "--no-fund", "--omit=dev", "--omit=optional"],
|
||||
{
|
||||
cwd: stagingDir,
|
||||
stdio: "inherit",
|
||||
},
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`[build-zip] npm install into staging failed (status ${result.status})`);
|
||||
}
|
||||
console.log(`[build-zip] staged node_modules via npm install`);
|
||||
}
|
||||
|
||||
function readDepVersion(moduleName: string): string {
|
||||
// Resolve the EXACT version bun installed into the workspace, not the
|
||||
// semver range declared in package.json. The staging-dir npm install
|
||||
// runs with `--no-package-lock`, so a caret range would float to the
|
||||
// latest registry version at build time — diverging from what the
|
||||
// workspace tests ran against and breaking ZIP-content determinism
|
||||
// across consecutive builds. The lockfile pin gives us reproducibility.
|
||||
const lockText = readFileSync(join(monorepoRoot, "bun.lock"), "utf-8");
|
||||
// bun.lock lines look like:
|
||||
// "puppeteer-core": ["puppeteer-core@24.43.1", "", { ... }, "sha512-..."],
|
||||
const re = new RegExp(
|
||||
`"${moduleName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}":\\s*\\["${moduleName.replace(
|
||||
/[.*+?^${}()|[\]\\]/g,
|
||||
"\\$&",
|
||||
)}@([^"]+)"`,
|
||||
);
|
||||
const match = re.exec(lockText);
|
||||
if (!match || !match[1]) {
|
||||
// Fall back to the manifest range — better than failing the build
|
||||
// entirely if bun.lock's format changes between bun versions.
|
||||
const manifest = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf-8")) as {
|
||||
dependencies?: Record<string, string>;
|
||||
};
|
||||
return manifest.dependencies?.[moduleName] ?? "latest";
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function resolveModuleDir(moduleName: string): string {
|
||||
// Walk up from packageRoot to find a matching node_modules entry.
|
||||
// Used by stageFfmpeg below; the @sparticuz/chromium + puppeteer-core
|
||||
// paths now go through npm install instead.
|
||||
let dir = packageRoot;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const candidate = join(dir, "node_modules", moduleName);
|
||||
if (existsSync(candidate)) return candidate;
|
||||
dir = dirname(dir);
|
||||
}
|
||||
throw new Error(
|
||||
`[build-zip] could not resolve ${moduleName} from ${packageRoot} — run 'bun install' first.`,
|
||||
);
|
||||
}
|
||||
|
||||
function stageHyperframeRuntime(stagingDir: string): void {
|
||||
const coreDist = resolve(monorepoRoot, "packages/core/dist");
|
||||
const manifestSrc = join(coreDist, "hyperframe.manifest.json");
|
||||
const iifeSrc = join(coreDist, "hyperframe.runtime.iife.js");
|
||||
if (!existsSync(manifestSrc) || !existsSync(iifeSrc)) {
|
||||
throw new Error(
|
||||
`[build-zip] hyperframe runtime artifacts missing under ${coreDist}. ` +
|
||||
`Run 'bun run --filter @hyperframes/core build:hyperframes-runtime:modular' first.`,
|
||||
);
|
||||
}
|
||||
cpSync(manifestSrc, join(stagingDir, "hyperframe.manifest.json"));
|
||||
cpSync(iifeSrc, join(stagingDir, "hyperframe.runtime.iife.js"));
|
||||
console.log(`[build-zip] staged hyperframe.manifest.json + hyperframe.runtime.iife.js`);
|
||||
}
|
||||
|
||||
// ELF header constants used by `assertLinuxX86_64Elf`. Header layout:
|
||||
// bytes 0..3 magic `0x7F 'E' 'L' 'F'`, byte 4 EI_CLASS (`2` = ELFCLASS64),
|
||||
// bytes 18..19 e_machine little-endian (`0x3E` = EM_X86_64).
|
||||
const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]);
|
||||
const ELF_CLASS_64 = 2;
|
||||
const ELF_MACHINE_X86_64 = 0x3e;
|
||||
const ELF_HEADER_BYTES = 20;
|
||||
|
||||
/**
|
||||
* Verify the binary at `path` is a Linux x86-64 ELF executable, the only
|
||||
* shape the Lambda runtime can exec. Throws with the canonical workaround
|
||||
* (Docker `--platform=linux/amd64` or `npm_config_platform` /
|
||||
* `npm_config_arch` overrides) so the build doesn't ship a silently
|
||||
* broken zip when a non-Linux host's postinstall fetched the wrong arch.
|
||||
*/
|
||||
function assertLinuxX86_64Elf(path: string, label: string): void {
|
||||
const head = readFileHead(path, ELF_HEADER_BYTES, label);
|
||||
const isElf = head.subarray(0, 4).equals(ELF_MAGIC);
|
||||
const isElf64 = head[4] === ELF_CLASS_64;
|
||||
const machine = head.readUInt16LE(18);
|
||||
if (isElf && isElf64 && machine === ELF_MACHINE_X86_64) return;
|
||||
|
||||
const magicHex = head.subarray(0, 4).toString("hex");
|
||||
throw new Error(
|
||||
`[build-zip] ${label} at ${path} is not a Linux x86-64 ELF executable ` +
|
||||
`(magic=0x${magicHex}, ei_class=${head[4]}, e_machine=0x${machine.toString(16)}). ` +
|
||||
`This usually means the deploy host's postinstall fetched a host-platform binary ` +
|
||||
`(e.g. macOS arm64 ffmpeg) instead of the linux/x64 binary Lambda needs. ` +
|
||||
`Re-run the build inside a linux/amd64 container, or pre-install with ` +
|
||||
`\`npm_config_platform=linux npm_config_arch=x64\` so the package fetches the right binary.`,
|
||||
);
|
||||
}
|
||||
|
||||
function readFileHead(path: string, byteCount: number, label: string): Buffer {
|
||||
const fd = openSync(path, "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(byteCount);
|
||||
const bytesRead = readSync(fd, buf, 0, byteCount, 0);
|
||||
if (bytesRead < byteCount) {
|
||||
throw new Error(
|
||||
`[build-zip] ${label} at ${path} is too short — read ${bytesRead} of ${byteCount} bytes.`,
|
||||
);
|
||||
}
|
||||
return buf;
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function stageFfmpeg(stagingDir: string): void {
|
||||
const binDir = join(stagingDir, "bin");
|
||||
mkdirSync(binDir, { recursive: true });
|
||||
|
||||
// ffmpeg from `ffmpeg-static`. The package only ships the encoder
|
||||
// binary; the audio pad/trim path also needs ffprobe, which comes
|
||||
// from `ffprobe-static`.
|
||||
//
|
||||
// `ffmpeg-static`'s postinstall fetches a binary for the host platform
|
||||
// (e.g. arm64 Mach-O on Apple Silicon macOS). Lambda runs Linux x86-64,
|
||||
// so a build from a non-Linux host silently produces a zip that boots
|
||||
// but fails at first ffmpeg invocation with `cannot execute binary file`.
|
||||
// Verify the ELF header up front and bail with a clear message instead.
|
||||
const ffmpegBinary = join(resolveModuleDir("ffmpeg-static"), "ffmpeg");
|
||||
if (!existsSync(ffmpegBinary)) {
|
||||
throw new Error(
|
||||
`[build-zip] ffmpeg-static binary missing at ${ffmpegBinary}. Did postinstall run?`,
|
||||
);
|
||||
}
|
||||
assertLinuxX86_64Elf(ffmpegBinary, "ffmpeg-static binary");
|
||||
const ffmpegDest = join(binDir, "ffmpeg");
|
||||
cpSync(ffmpegBinary, ffmpegDest);
|
||||
chmodSync(ffmpegDest, 0o755);
|
||||
|
||||
// ffprobe lives at `ffprobe-static/bin/<platform>/<arch>/ffprobe`.
|
||||
// The producer's `audioPadTrim` spawns `ffprobe` from PATH so we need
|
||||
// it alongside ffmpeg under /var/task/bin/.
|
||||
const ffprobeModule = resolveModuleDir("ffprobe-static");
|
||||
const ffprobeCandidates = [
|
||||
join(ffprobeModule, "bin", "linux", "x64", "ffprobe"),
|
||||
join(ffprobeModule, "bin", "linux", "arm64", "ffprobe"),
|
||||
];
|
||||
const ffprobeBinary = ffprobeCandidates.find((p) => existsSync(p));
|
||||
if (!ffprobeBinary) {
|
||||
throw new Error(
|
||||
`[build-zip] ffprobe-static binary not found under ${ffprobeModule}/bin/linux/. Did postinstall run?`,
|
||||
);
|
||||
}
|
||||
const ffprobeDest = join(binDir, "ffprobe");
|
||||
cpSync(ffprobeBinary, ffprobeDest);
|
||||
chmodSync(ffprobeDest, 0o755);
|
||||
|
||||
console.log(`[build-zip] staged ffmpeg + ffprobe → bin/`);
|
||||
}
|
||||
|
||||
function stageChromeHeadlessShell(stagingDir: string): void {
|
||||
// The fallback path bundles the same chrome-headless-shell binary the
|
||||
// K8s deploy uses. The binary is fetched via `@puppeteer/browsers` on
|
||||
// first build into the host's `~/.cache/puppeteer/`; the build script
|
||||
// re-uses that cache rather than redownloading.
|
||||
const home = process.env.HOME ?? "/root";
|
||||
const baseDir = join(home, ".cache", "puppeteer", "chrome-headless-shell");
|
||||
if (!existsSync(baseDir)) {
|
||||
throw new Error(
|
||||
`[build-zip] chrome-headless-shell cache missing at ${baseDir}. Run\n` +
|
||||
` npx --yes @puppeteer/browsers install chrome-headless-shell@stable --path ${home}/.cache/puppeteer\n` +
|
||||
`before --source=chrome-headless-shell.`,
|
||||
);
|
||||
}
|
||||
// Sort by numeric semver descending. `sort().reverse()` is lexicographic,
|
||||
// which silently picks "99.0.0" over "131.0.0" once Chrome ships
|
||||
// three-digit majors that aren't strictly width-aligned. `compareSemver`
|
||||
// returns negative/zero/positive on (a, b), so descending = `b - a`.
|
||||
const versions = readdirSync(baseDir).sort((a, b) => compareSemver(b, a));
|
||||
for (const v of versions) {
|
||||
const candidate = join(baseDir, v, "chrome-headless-shell-linux64", "chrome-headless-shell");
|
||||
if (existsSync(candidate)) {
|
||||
const dest = join(stagingDir, "bin", "chrome-headless-shell");
|
||||
mkdirSync(dirname(dest), { recursive: true });
|
||||
cpSync(candidate, dest);
|
||||
chmodSync(dest, 0o755);
|
||||
console.log(`[build-zip] staged chrome-headless-shell (${v}) → bin/chrome-headless-shell`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw new Error(`[build-zip] no linux64 chrome-headless-shell binary found under ${baseDir}.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two semver-shaped strings like "131.0.6778.108". Treats any
|
||||
* non-numeric directory name as `-Infinity` so it sorts to the bottom
|
||||
* (Puppeteer's cache layout sometimes includes `latest` or branch tags).
|
||||
* Used by `stageChromeHeadlessShell` to pick the newest cached Chrome
|
||||
* without tripping on the lexicographic "99 > 131" trap.
|
||||
*/
|
||||
function compareSemver(a: string, b: string): number {
|
||||
const partsA = a.split(".").map((s) => Number.parseInt(s, 10));
|
||||
const partsB = b.split(".").map((s) => Number.parseInt(s, 10));
|
||||
const len = Math.max(partsA.length, partsB.length);
|
||||
for (let i = 0; i < len; i++) {
|
||||
const ai = partsA[i] ?? 0;
|
||||
const bi = partsB[i] ?? 0;
|
||||
if (Number.isNaN(ai) && Number.isNaN(bi)) continue;
|
||||
if (Number.isNaN(ai)) return -1;
|
||||
if (Number.isNaN(bi)) return 1;
|
||||
if (ai !== bi) return ai - bi;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function zipDirectory(sourceDir: string, zipPath: string): void {
|
||||
const result = spawnSync("zip", ["-rq", zipPath, "."], { cwd: sourceDir, stdio: "inherit" });
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`[build-zip] zip exited with status ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function directorySizeBytes(dir: string): number {
|
||||
// Use spawnSync (no shell) instead of execSync so `dir` is passed as
|
||||
// an argv element rather than interpolated into a shell command —
|
||||
// CodeQL's `js/shell-command-injected-from-environment` rule fires
|
||||
// on the latter even with JSON-quoting. `du -sb` is Linux-only;
|
||||
// build-zip is CI-side where Linux coreutils is present.
|
||||
const result = spawnSync("du", ["-sb", dir], { encoding: "utf-8" });
|
||||
if (result.status === 0 && result.stdout) {
|
||||
const bytes = Number.parseInt(result.stdout.split(/\s+/)[0] ?? "0", 10);
|
||||
if (!Number.isNaN(bytes)) return bytes;
|
||||
}
|
||||
return walkSize(dir);
|
||||
}
|
||||
|
||||
function walkSize(dir: string): number {
|
||||
let total = 0;
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) total += walkSize(full);
|
||||
else if (entry.isFile()) total += statSync(full).size;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
void main().catch((err) => {
|
||||
console.error("[build-zip] failed:", err instanceof Error ? err.message : String(err));
|
||||
if (err instanceof Error && err.stack) console.error(err.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
# BeginFrame regression-guard container.
|
||||
#
|
||||
# Uses the official AWS Lambda Node 22 image as the base so the probe
|
||||
# exercises @sparticuz/chromium against the SAME glibc, kernel feature
|
||||
# set, and `/tmp` filesystem layout that real Lambda invocations see. If
|
||||
# this Dockerfile passes, the bundled handler is on solid footing for
|
||||
# real AWS.
|
||||
#
|
||||
# Build context: monorepo root (../../). Build + run:
|
||||
#
|
||||
# bun run --cwd packages/aws-lambda probe:beginframe:docker
|
||||
#
|
||||
# The default CMD runs `tsx scripts/probe-beginframe.ts` and exits 0 on
|
||||
# pass, 1 on BeginFrame failure, 2 on harness failure.
|
||||
|
||||
FROM public.ecr.aws/lambda/nodejs:22
|
||||
|
||||
# Shared libraries @sparticuz/chromium expects but the Lambda base image
|
||||
# does not bring in by default. Versions are pinned to whatever
|
||||
# `dnf install` resolves on the Lambda base image at build time; we just
|
||||
# need them present.
|
||||
RUN dnf install -y \
|
||||
alsa-lib \
|
||||
atk \
|
||||
cups-libs \
|
||||
gtk3 \
|
||||
libdrm \
|
||||
libxkbcommon \
|
||||
libXcomposite \
|
||||
libXdamage \
|
||||
libXrandr \
|
||||
mesa-libgbm \
|
||||
nss \
|
||||
pango \
|
||||
tar \
|
||||
gzip \
|
||||
unzip \
|
||||
&& dnf clean all
|
||||
|
||||
WORKDIR /var/task
|
||||
|
||||
# The probe is self-contained — we install the three deps it needs into a
|
||||
# fresh package directory rather than re-using the monorepo's
|
||||
# workspace-rooted manifests (which carry `workspace:` protocol deps npm
|
||||
# can't resolve).
|
||||
COPY packages/aws-lambda/scripts/ scripts/
|
||||
|
||||
RUN printf '{"name":"hf-lambda-probe","version":"1.0.0","type":"module"}\n' > package.json \
|
||||
&& npm install --no-audit --no-fund --omit=optional \
|
||||
@sparticuz/chromium@148.0.0 \
|
||||
puppeteer-core@^24.39.1 \
|
||||
tsx@^4.21.0
|
||||
|
||||
ENV NODE_PATH=/var/task/node_modules
|
||||
ENV PATH="/var/task/node_modules/.bin:${PATH}"
|
||||
|
||||
# Lambda's `tmpfs` is mounted at /tmp; sparticuz decompresses into /tmp
|
||||
# at runtime. The base image already has /tmp writable.
|
||||
|
||||
ENTRYPOINT []
|
||||
CMD ["node", "--experimental-strip-types", "scripts/probe-beginframe.ts"]
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* BeginFrame regression guard for `@sparticuz/chromium`.
|
||||
*
|
||||
* The load-bearing assumption of `@hyperframes/aws-lambda` is that the
|
||||
* Chromium build shipped by `@sparticuz/chromium` honours CDP
|
||||
* `HeadlessExperimental.beginFrame` with `screenshot: true`. This script
|
||||
* boots that Chromium build (decompressing into `/tmp` per the library's
|
||||
* runtime contract), navigates to a tiny static page, issues one
|
||||
* `beginFrame` with a screenshot request, and asserts the response
|
||||
* carries a PNG buffer.
|
||||
*
|
||||
* The script is the contract test, not a one-shot verification — every
|
||||
* release should run it inside the Docker container at
|
||||
* `scripts/probe-beginframe.dockerfile` to catch any future
|
||||
* `@sparticuz/chromium` rebuild that drops `HeadlessExperimental` support.
|
||||
*
|
||||
* Exits 0 on pass, 1 on fail. Run via:
|
||||
*
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe # host
|
||||
* bun run --cwd packages/aws-lambda probe:beginframe:docker # Lambda-like
|
||||
*/
|
||||
|
||||
import { mkdtempSync, promises as fs } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
interface ProbeResult {
|
||||
passed: boolean;
|
||||
durationMs: number;
|
||||
chromiumPath: string;
|
||||
screenshotBytes: number;
|
||||
hasDamage: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
const PROBE_HTML = `<!doctype html>
|
||||
<html><head><meta charset="utf-8"><title>hf-beginframe-probe</title>
|
||||
<style>html,body{margin:0;background:#173;color:#fff;font:48px/1 sans-serif;display:flex;align-items:center;justify-content:center;height:100vh}</style>
|
||||
</head><body><div id="x">hf-beginframe-probe</div></body></html>`;
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const start = Date.now();
|
||||
const result = await probe();
|
||||
result.durationMs = Date.now() - start;
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (!result.passed) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function probe(): Promise<ProbeResult> {
|
||||
let chromiumPath = "";
|
||||
try {
|
||||
const { default: chromium } = await import("@sparticuz/chromium");
|
||||
chromiumPath = await chromium.executablePath();
|
||||
const args = chromium.args;
|
||||
|
||||
const puppeteer = await import("puppeteer-core");
|
||||
|
||||
// Write probe HTML to /tmp + serve via file:// — no HTTP server in the
|
||||
// probe so we don't add a dependency surface that could mask a
|
||||
// Chrome-side issue. `mkdtempSync` (vs `tmpdir() + Date.now()`) gives
|
||||
// an unguessable directory name so two concurrent probes on the same
|
||||
// host don't collide and CodeQL's insecure-tempfile rule clears.
|
||||
const tmpHtmlDir = mkdtempSync(join(tmpdir(), "hf-beginframe-"));
|
||||
const htmlPath = join(tmpHtmlDir, "probe.html");
|
||||
await fs.writeFile(htmlPath, PROBE_HTML, "utf-8");
|
||||
|
||||
// BeginFrame requires the full compositor-driving flag set. These match
|
||||
// the args the engine's `browserManager` passes when `captureMode !==
|
||||
// "screenshot"`. Without the surface-synchronization + threaded-disable
|
||||
// flags, Chrome's compositor returns `hasDamage: false` and skips the
|
||||
// screenshot — the same observation pinned in the hyperframes memory
|
||||
// ("Chrome's beginFrame with `screenshot` param always reports
|
||||
// hasDamage=true").
|
||||
const beginFrameFlags = [
|
||||
"--deterministic-mode",
|
||||
"--enable-begin-frame-control",
|
||||
"--disable-new-content-rendering-timeout",
|
||||
"--run-all-compositor-stages-before-draw",
|
||||
"--disable-threaded-animation",
|
||||
"--disable-threaded-scrolling",
|
||||
"--disable-checker-imaging",
|
||||
"--disable-image-animation-resync",
|
||||
"--enable-surface-synchronization",
|
||||
// Software GL — Lambda has no GPU; matches the in-process renderer's
|
||||
// software-locked path.
|
||||
"--use-gl=angle",
|
||||
"--use-angle=swiftshader",
|
||||
"--enable-unsafe-swiftshader",
|
||||
];
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: chromiumPath,
|
||||
headless: "shell",
|
||||
args: [...args, ...beginFrameFlags],
|
||||
defaultViewport: { width: 800, height: 600 },
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(`file://${htmlPath}`, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
const session = await page.createCDPSession();
|
||||
await session.send("HeadlessExperimental.enable");
|
||||
// Warm-up beginFrame with noDisplayUpdates: true — drives the
|
||||
// compositor without producing a screenshot, matching how the engine
|
||||
// primes a capture loop.
|
||||
await session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 0,
|
||||
interval: 33,
|
||||
noDisplayUpdates: true,
|
||||
});
|
||||
const response = await session.send("HeadlessExperimental.beginFrame", {
|
||||
frameTimeTicks: 1000,
|
||||
interval: 33,
|
||||
screenshot: { format: "png" },
|
||||
});
|
||||
await fs.rm(tmpHtmlDir, { recursive: true, force: true }).catch(() => {});
|
||||
const screenshot = response.screenshotData ?? "";
|
||||
const bytes = screenshot ? Buffer.from(screenshot, "base64") : Buffer.alloc(0);
|
||||
const isPng =
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47;
|
||||
return {
|
||||
passed: isPng && bytes.length > 0,
|
||||
durationMs: 0,
|
||||
chromiumPath,
|
||||
screenshotBytes: bytes.length,
|
||||
hasDamage: response.hasDamage,
|
||||
detail: isPng
|
||||
? "OK — BeginFrame returned a PNG buffer."
|
||||
: `FAIL — BeginFrame returned ${bytes.length} bytes, PNG signature ${
|
||||
bytes.length >= 4 ? bytes.subarray(0, 4).toString("hex") : "<empty>"
|
||||
}`,
|
||||
};
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
passed: false,
|
||||
durationMs: 0,
|
||||
chromiumPath,
|
||||
screenshotBytes: 0,
|
||||
hasDamage: false,
|
||||
detail: `FAIL — ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
void main().catch((err) => {
|
||||
console.error("[probe-beginframe] unexpected:", err);
|
||||
process.exit(2);
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* CI gate on the Lambda ZIP size.
|
||||
*
|
||||
* Reads `dist/handler.zip.manifest.json` (written by `build-zip.ts`) and
|
||||
* exits non-zero if either the unzipped or zipped size exceeds the
|
||||
* declared limits. Lambda's hard ceiling for ZIP-deployed functions is
|
||||
* 250 MiB unzipped (262144000 bytes — AWS docs label it "250 MB" but use
|
||||
* binary mebibytes); the in-house budget is 248 MiB to keep headroom for
|
||||
* the Chrome tarball decompression that happens at cold start.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { formatBytes } from "./_formatBytes.js";
|
||||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const packageRoot = resolve(scriptDir, "..");
|
||||
const distDir = join(packageRoot, "dist");
|
||||
const zipPath = join(distDir, "handler.zip");
|
||||
const manifestPath = join(distDir, "handler.zip.manifest.json");
|
||||
|
||||
interface Manifest {
|
||||
unzippedBytes: number;
|
||||
zippedBytes: number;
|
||||
source: string;
|
||||
}
|
||||
|
||||
const IN_HOUSE_UNZIPPED_LIMIT = 248 * 1024 * 1024;
|
||||
const IN_HOUSE_ZIPPED_LIMIT = 150 * 1024 * 1024;
|
||||
|
||||
function main(): void {
|
||||
if (!existsSync(zipPath)) {
|
||||
console.error(`[verify-zip-size] ${zipPath} not found. Run 'bun run build:zip' first.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!existsSync(manifestPath)) {
|
||||
console.error(
|
||||
`[verify-zip-size] ${manifestPath} not found. The manifest is written by build-zip.ts; ` +
|
||||
`re-run the build.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as Manifest;
|
||||
const actualZipped = statSync(zipPath).size;
|
||||
if (actualZipped !== manifest.zippedBytes) {
|
||||
console.warn(
|
||||
`[verify-zip-size] note: zip file size on disk (${actualZipped}) differs from ` +
|
||||
`manifest (${manifest.zippedBytes}). Using on-disk value.`,
|
||||
);
|
||||
}
|
||||
|
||||
let failed = false;
|
||||
if (manifest.unzippedBytes > IN_HOUSE_UNZIPPED_LIMIT) {
|
||||
console.error(
|
||||
`[verify-zip-size] FAIL unzipped: ${formatBytes(manifest.unzippedBytes)} > ` +
|
||||
`${formatBytes(IN_HOUSE_UNZIPPED_LIMIT)} (in-house limit; Lambda hard ceiling is 250 MiB).`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
if (actualZipped > IN_HOUSE_ZIPPED_LIMIT) {
|
||||
console.error(
|
||||
`[verify-zip-size] FAIL zipped: ${formatBytes(actualZipped)} > ` +
|
||||
`${formatBytes(IN_HOUSE_ZIPPED_LIMIT)} (in-house limit; Lambda direct-upload ceiling is 50 MiB, ` +
|
||||
`S3-deploy ceiling is 250 MiB).`,
|
||||
);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
console.error("[verify-zip-size] FAILED — bundle is too large for Lambda ZIP deploy.");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(
|
||||
`[verify-zip-size] OK source=${manifest.source} unzipped=${formatBytes(
|
||||
manifest.unzippedBytes,
|
||||
)} zipped=${formatBytes(actualZipped)}`,
|
||||
);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* Contract tests for {@link HyperframesRenderStack}.
|
||||
*
|
||||
* The snapshot test (in this directory's sibling `.snapshot.test.ts`)
|
||||
* guards the full CloudFormation shape. The contract tests below pin
|
||||
* the few properties whose drift would cause a real production
|
||||
* regression — wrong Lambda runtime, lost reserved-concurrency knob,
|
||||
* missing alarms — so we get a high-signal failure independent of
|
||||
* the snapshot.
|
||||
*/
|
||||
|
||||
import { beforeAll, describe, it } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { App, Stack } from "aws-cdk-lib";
|
||||
import { Template } from "aws-cdk-lib/assertions";
|
||||
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
|
||||
|
||||
// CDK synth is slow on cold start (~5-8s on the slowest CI runner). The
|
||||
// default bun:test 5s timeout trips the first `it()` that calls it. Cache
|
||||
// the default-args synth in `beforeAll` so each test is pure assertions.
|
||||
// Tests that need non-default props still synth on demand and bump their
|
||||
// own per-test timeout.
|
||||
let DEFAULT_TEMPLATE: Template;
|
||||
|
||||
function synthFixture(): Template {
|
||||
const zipDir = mkdtempSync(join(tmpdir(), "hf-cdk-test-"));
|
||||
const zipPath = join(zipDir, "handler.zip");
|
||||
writeFileSync(zipPath, "fake zip bytes");
|
||||
const app = new App();
|
||||
const stack = new Stack(app, "TestStack");
|
||||
new HyperframesRenderStack(stack, "Render", { handlerZipPath: zipPath });
|
||||
return Template.fromStack(stack);
|
||||
}
|
||||
|
||||
describe("HyperframesRenderStack — contract", () => {
|
||||
beforeAll(() => {
|
||||
DEFAULT_TEMPLATE = synthFixture();
|
||||
}, 30000);
|
||||
|
||||
it("provisions exactly one Lambda function on the Node.js 22 runtime, x86_64, 10 GiB /tmp", () => {
|
||||
const t = DEFAULT_TEMPLATE;
|
||||
t.resourceCountIs("AWS::Lambda::Function", 1);
|
||||
t.hasResourceProperties("AWS::Lambda::Function", {
|
||||
Runtime: "nodejs22.x",
|
||||
Architectures: ["x86_64"],
|
||||
EphemeralStorage: { Size: 10240 },
|
||||
MemorySize: 10240,
|
||||
Handler: "handler.handler",
|
||||
});
|
||||
});
|
||||
|
||||
it("provisions exactly one Step Functions state machine of type STANDARD with tracing on", () => {
|
||||
const t = DEFAULT_TEMPLATE;
|
||||
t.resourceCountIs("AWS::StepFunctions::StateMachine", 1);
|
||||
t.hasResourceProperties("AWS::StepFunctions::StateMachine", {
|
||||
StateMachineType: "STANDARD",
|
||||
TracingConfiguration: { Enabled: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("provisions exactly one S3 bucket with PublicAccessBlockConfiguration and a 7-day intermediates lifecycle", () => {
|
||||
const t = DEFAULT_TEMPLATE;
|
||||
t.resourceCountIs("AWS::S3::Bucket", 1);
|
||||
t.hasResourceProperties("AWS::S3::Bucket", {
|
||||
PublicAccessBlockConfiguration: {
|
||||
BlockPublicAcls: true,
|
||||
BlockPublicPolicy: true,
|
||||
IgnorePublicAcls: true,
|
||||
RestrictPublicBuckets: true,
|
||||
},
|
||||
LifecycleConfiguration: {
|
||||
Rules: [
|
||||
{
|
||||
Id: "ExpireIntermediates",
|
||||
Status: "Enabled",
|
||||
Prefix: "renders/",
|
||||
ExpirationInDays: 7,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("provisions the three CloudWatch alarms (runaway invocations, Lambda Errors, SFN ExecutionsFailed)", () => {
|
||||
const t = DEFAULT_TEMPLATE;
|
||||
t.resourceCountIs("AWS::CloudWatch::Alarm", 3);
|
||||
t.hasResourceProperties("AWS::CloudWatch::Alarm", {
|
||||
MetricName: "Invocations",
|
||||
Period: 3600,
|
||||
Threshold: 1000,
|
||||
});
|
||||
t.hasResourceProperties("AWS::CloudWatch::Alarm", {
|
||||
MetricName: "Errors",
|
||||
Threshold: 1,
|
||||
});
|
||||
t.hasResourceProperties("AWS::CloudWatch::Alarm", {
|
||||
MetricName: "ExecutionsFailed",
|
||||
Threshold: 1,
|
||||
});
|
||||
});
|
||||
|
||||
// These two synth fresh stacks (non-default props), so they pay the
|
||||
// synth cost individually. Bump per-test timeout so a slow CI runner
|
||||
// doesn't trip the default 5s.
|
||||
it("honours reservedConcurrency when supplied", () => {
|
||||
const zipDir = mkdtempSync(join(tmpdir(), "hf-cdk-test-"));
|
||||
writeFileSync(join(zipDir, "handler.zip"), "fake");
|
||||
const app = new App();
|
||||
const stack = new Stack(app, "TestStack");
|
||||
new HyperframesRenderStack(stack, "Render", {
|
||||
handlerZipPath: join(zipDir, "handler.zip"),
|
||||
reservedConcurrency: 4,
|
||||
});
|
||||
const t = Template.fromStack(stack);
|
||||
t.hasResourceProperties("AWS::Lambda::Function", {
|
||||
ReservedConcurrentExecutions: 4,
|
||||
});
|
||||
}, 30000);
|
||||
|
||||
it("uses the projectName prefix on function + state-machine names", () => {
|
||||
const zipDir = mkdtempSync(join(tmpdir(), "hf-cdk-test-"));
|
||||
writeFileSync(join(zipDir, "handler.zip"), "fake");
|
||||
const app = new App();
|
||||
const stack = new Stack(app, "TestStack");
|
||||
new HyperframesRenderStack(stack, "Render", {
|
||||
handlerZipPath: join(zipDir, "handler.zip"),
|
||||
projectName: "demo",
|
||||
});
|
||||
const t = Template.fromStack(stack);
|
||||
t.hasResourceProperties("AWS::Lambda::Function", { FunctionName: "demo-render" });
|
||||
t.hasResourceProperties("AWS::StepFunctions::StateMachine", {
|
||||
StateMachineName: "demo-render",
|
||||
});
|
||||
}, 30000);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Structural snapshot of {@link HyperframesRenderStack}.
|
||||
*
|
||||
* `toMatchSnapshot` is intentionally avoided here: bun's snapshot format
|
||||
* is brittle against the CloudFormation tokens CDK emits (random suffixes
|
||||
* on log group + role logical ids, asset hashes that change with the
|
||||
* handler ZIP). Instead we freeze:
|
||||
*
|
||||
* - The count of each AWS::* resource type the synthed stack contains
|
||||
* (catches accidental new resources, deletions, type swaps).
|
||||
* - A frozen list of Step Functions state names in the parsed
|
||||
* `DefinitionString`, in declaration order (catches state-machine
|
||||
* topology drift).
|
||||
* - The full set of state-machine retry/catch error names (catches
|
||||
* accidental loss of typed non-retryable failure handling).
|
||||
*
|
||||
* Any intentional change to those properties should update this file in
|
||||
* the same commit — a reviewer reading the diff knows exactly what shifted
|
||||
* in the topology.
|
||||
*/
|
||||
|
||||
import { beforeAll, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { App, Stack } from "aws-cdk-lib";
|
||||
import { Template } from "aws-cdk-lib/assertions";
|
||||
import { HyperframesRenderStack } from "./HyperframesRenderStack.js";
|
||||
|
||||
// CDK synth + Template.fromStack is slow on cold start in CI (~5-8s on
|
||||
// the first call). The default bun:test 5s timeout trips it on the
|
||||
// first `it()` that calls `synth()`. Run synth once in `beforeAll`
|
||||
// and reuse the result — each test is a few µs of pure assertions
|
||||
// against the already-synthed template.
|
||||
let SYNTHED: ReturnType<typeof doSynth>;
|
||||
|
||||
const EXPECTED_RESOURCE_COUNTS: Record<string, number> = {
|
||||
"AWS::Lambda::Function": 1,
|
||||
"AWS::S3::Bucket": 1,
|
||||
"AWS::StepFunctions::StateMachine": 1,
|
||||
"AWS::CloudWatch::Alarm": 3,
|
||||
"AWS::Logs::LogGroup": 1,
|
||||
// CDK emits IAM roles for both the function and the state machine, plus
|
||||
// a managed policy for the bucket grant.
|
||||
"AWS::IAM::Role": 2,
|
||||
"AWS::IAM::Policy": 2,
|
||||
};
|
||||
|
||||
// Top-level state names emitted by ASL. The Map state's inner
|
||||
// `RenderChunk` task lives nested under `RenderChunks.Iterator.States`,
|
||||
// not at this level — we cover it separately in the contract test.
|
||||
const EXPECTED_STATE_NAMES = [
|
||||
"Plan",
|
||||
"BuildChunkList",
|
||||
"AssertChunkCount",
|
||||
"RenderChunks",
|
||||
"Assemble",
|
||||
"PlanProducedZeroChunks",
|
||||
];
|
||||
|
||||
const EXPECTED_NON_RETRYABLE_ERRORS = new Set([
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
"PLAN_HASH_MISMATCH",
|
||||
"BROWSER_GPU_NOT_SOFTWARE",
|
||||
"FONT_FETCH_FAILED",
|
||||
"PLAN_TOO_LARGE",
|
||||
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
||||
"ChromeBinaryUnavailableError",
|
||||
]);
|
||||
|
||||
function doSynth(): {
|
||||
template: Template;
|
||||
definition: { States: Record<string, unknown>; StartAt: string };
|
||||
} {
|
||||
const zipDir = mkdtempSync(join(tmpdir(), "hf-cdk-snap-"));
|
||||
writeFileSync(join(zipDir, "handler.zip"), "fake zip bytes");
|
||||
const app = new App();
|
||||
const stack = new Stack(app, "TestStack");
|
||||
new HyperframesRenderStack(stack, "Render", { handlerZipPath: join(zipDir, "handler.zip") });
|
||||
const template = Template.fromStack(stack);
|
||||
const stateMachine = Object.values(
|
||||
template.findResources("AWS::StepFunctions::StateMachine"),
|
||||
)[0] as {
|
||||
Properties: { DefinitionString: unknown };
|
||||
};
|
||||
const def = stateMachine.Properties.DefinitionString;
|
||||
// CDK emits a `Fn::Join` over interpolated ARN tokens; reduce it to
|
||||
// a definition string we can JSON.parse for inspection.
|
||||
let parsed: { States: Record<string, unknown>; StartAt: string };
|
||||
if (typeof def === "string") {
|
||||
parsed = JSON.parse(def);
|
||||
} else if (def && typeof def === "object" && "Fn::Join" in def) {
|
||||
const join = (def as { "Fn::Join": [string, unknown[]] })["Fn::Join"];
|
||||
const concatenated = join[1]
|
||||
.map((seg) => (typeof seg === "string" ? seg : "<<TOKEN>>"))
|
||||
.join("");
|
||||
parsed = JSON.parse(concatenated);
|
||||
} else {
|
||||
throw new Error(`Unexpected DefinitionString shape: ${JSON.stringify(def).slice(0, 200)}`);
|
||||
}
|
||||
return { template, definition: parsed };
|
||||
}
|
||||
|
||||
describe("HyperframesRenderStack — snapshot", () => {
|
||||
// 30s is plenty: cold synth on the slowest CI runner has measured ~8s.
|
||||
beforeAll(() => {
|
||||
SYNTHED = doSynth();
|
||||
}, 30000);
|
||||
|
||||
it("emits the expected set of AWS resource types in the expected counts", () => {
|
||||
const { template } = SYNTHED;
|
||||
const actual: Record<string, number> = {};
|
||||
const allResources = template.toJSON().Resources as Record<string, { Type: string }>;
|
||||
for (const res of Object.values(allResources)) {
|
||||
actual[res.Type] = (actual[res.Type] ?? 0) + 1;
|
||||
}
|
||||
// Only assert on the types we explicitly track so the assertion
|
||||
// failure highlights the drift, not the surrounding noise.
|
||||
for (const [type, expected] of Object.entries(EXPECTED_RESOURCE_COUNTS)) {
|
||||
expect({ type, count: actual[type] ?? 0 }).toEqual({ type, count: expected });
|
||||
}
|
||||
// And catch unexpected new resource types up front.
|
||||
const unexpected = Object.keys(actual).filter(
|
||||
(type) => EXPECTED_RESOURCE_COUNTS[type] === undefined,
|
||||
);
|
||||
expect(unexpected).toEqual([]);
|
||||
});
|
||||
|
||||
it("declares the state machine with the expected state names", () => {
|
||||
const { definition } = SYNTHED;
|
||||
expect(definition.StartAt).toBe("Plan");
|
||||
const actualStates = Object.keys(definition.States);
|
||||
expect(actualStates.sort()).toEqual([...EXPECTED_STATE_NAMES].sort());
|
||||
});
|
||||
|
||||
it("preserves every typed non-retryable error name across the three Lambda tasks", () => {
|
||||
const { definition } = SYNTHED;
|
||||
const collected = new Set<string>();
|
||||
// Plan + Assemble are top-level states; RenderChunk is nested inside
|
||||
// the Map's Iterator definition.
|
||||
const topLevelStates = ["Plan", "Assemble"] as const;
|
||||
for (const stateName of topLevelStates) {
|
||||
collectNonRetryableErrors(definition.States[stateName], collected);
|
||||
}
|
||||
const renderChunks = definition.States.RenderChunks as
|
||||
| {
|
||||
Iterator?: { States?: Record<string, unknown> };
|
||||
ItemProcessor?: { States?: Record<string, unknown> };
|
||||
}
|
||||
| undefined;
|
||||
const innerStates = renderChunks?.Iterator?.States ?? renderChunks?.ItemProcessor?.States ?? {};
|
||||
collectNonRetryableErrors(innerStates.RenderChunk, collected);
|
||||
|
||||
for (const expected of EXPECTED_NON_RETRYABLE_ERRORS) {
|
||||
expect({ error: expected, present: collected.has(expected) }).toEqual({
|
||||
error: expected,
|
||||
present: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function collectNonRetryableErrors(state: unknown, out: Set<string>): void {
|
||||
const retries =
|
||||
(state as { Retry?: { ErrorEquals: string[]; MaxAttempts?: number }[] })?.Retry ?? [];
|
||||
for (const retry of retries) {
|
||||
if (retry.MaxAttempts === 0) {
|
||||
for (const err of retry.ErrorEquals) out.add(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
/**
|
||||
* `HyperframesRenderStack` — aws-cdk-lib L2 construct that emits the same
|
||||
* topology as `examples/aws-lambda/template.yaml`.
|
||||
*
|
||||
* Adopters who embed HyperFrames inside their own CDK app can extend this
|
||||
* construct or compose alongside it; the construct exposes its `.bucket`,
|
||||
* `.renderFunction`, and `.stateMachine` properties so additional
|
||||
* resources (alarms, dashboards, SNS topics) can be wired without
|
||||
* re-deriving the ARNs from a stack export.
|
||||
*
|
||||
* `aws-cdk-lib` and `constructs` are **peerDependencies**. The package
|
||||
* still type-checks (and the snapshot test still runs) because they're
|
||||
* also `devDependencies`, but adopters who only consume the SDK side of
|
||||
* `@hyperframes/aws-lambda` don't pull the CDK tree at runtime.
|
||||
*
|
||||
* Drift from the SAM template is guarded by the snapshot test
|
||||
* (`HyperframesRenderStack.snapshot.test.ts`), which diffs the synthed
|
||||
* CloudFormation against the SAM-rendered CloudFormation modulo
|
||||
* normalisation.
|
||||
*/
|
||||
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Duration, RemovalPolicy, Size } from "aws-cdk-lib";
|
||||
import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
|
||||
import * as lambda from "aws-cdk-lib/aws-lambda";
|
||||
import * as logs from "aws-cdk-lib/aws-logs";
|
||||
import * as s3 from "aws-cdk-lib/aws-s3";
|
||||
import * as sfn from "aws-cdk-lib/aws-stepfunctions";
|
||||
import * as tasks from "aws-cdk-lib/aws-stepfunctions-tasks";
|
||||
import { Construct } from "constructs";
|
||||
|
||||
/** Construction-time props for {@link HyperframesRenderStack}. */
|
||||
export interface HyperframesRenderStackProps {
|
||||
/** Name prefix applied to function / state-machine / alarm names. Default `"hyperframes"`. */
|
||||
projectName?: string;
|
||||
/** Lambda memory in MB. Allowed: 2048..10240 in 1024 steps. Default 10240. */
|
||||
lambdaMemoryMb?: 2048 | 3072 | 4096 | 5120 | 6144 | 7168 | 8192 | 9216 | 10240;
|
||||
/** Per-invocation Lambda timeout. Default 900 (15 min, Lambda hard cap). */
|
||||
lambdaTimeoutSec?: number;
|
||||
/** Lambda reserved concurrency cap. `undefined` = unreserved (account default). */
|
||||
reservedConcurrency?: number;
|
||||
/** Which Chrome runtime was bundled into the handler ZIP. Default `"sparticuz"`. */
|
||||
chromeSource?: "sparticuz" | "chrome-headless-shell";
|
||||
/** Threshold for the runaway-invocations alarm. Default 1000 invocations/hour. */
|
||||
chunkInvocationAlarmThreshold?: number;
|
||||
/**
|
||||
* Absolute path to the handler ZIP produced by
|
||||
* `bun run --cwd packages/aws-lambda build:zip`. Defaults to the
|
||||
* package-relative path the build script writes to. Adopters who
|
||||
* deploy the published handler ZIP set this explicitly.
|
||||
*/
|
||||
handlerZipPath?: string;
|
||||
/** S3 bucket retention policy on stack delete. Default RETAIN. */
|
||||
bucketRemovalPolicy?: RemovalPolicy;
|
||||
}
|
||||
|
||||
const DEFAULT_MEMORY_MB = 10240;
|
||||
const DEFAULT_TIMEOUT_SEC = 900;
|
||||
const DEFAULT_CHROME_SOURCE = "sparticuz";
|
||||
const DEFAULT_ALARM_THRESHOLD = 1000;
|
||||
|
||||
export class HyperframesRenderStack extends Construct {
|
||||
/** S3 bucket for plan tarballs, chunk outputs, and final renders. */
|
||||
readonly bucket: s3.Bucket;
|
||||
/** The single Lambda function dispatching plan / renderChunk / assemble. */
|
||||
readonly renderFunction: lambda.Function;
|
||||
/** The Step Functions state machine orchestrating the render. */
|
||||
readonly stateMachine: sfn.StateMachine;
|
||||
|
||||
constructor(scope: Construct, id: string, props: HyperframesRenderStackProps = {}) {
|
||||
super(scope, id);
|
||||
|
||||
const projectName = props.projectName ?? "hyperframes";
|
||||
const memorySize = props.lambdaMemoryMb ?? DEFAULT_MEMORY_MB;
|
||||
const timeoutSec = props.lambdaTimeoutSec ?? DEFAULT_TIMEOUT_SEC;
|
||||
const chromeSource = props.chromeSource ?? DEFAULT_CHROME_SOURCE;
|
||||
const alarmThreshold = props.chunkInvocationAlarmThreshold ?? DEFAULT_ALARM_THRESHOLD;
|
||||
const handlerZipPath = props.handlerZipPath ?? defaultHandlerZipPath();
|
||||
|
||||
this.bucket = new s3.Bucket(this, "RenderBucket", {
|
||||
removalPolicy: props.bucketRemovalPolicy ?? RemovalPolicy.RETAIN,
|
||||
blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
|
||||
// `Suspended` is the cheapest mode that still satisfies KMS / replication
|
||||
// prerequisites callers can layer on later. Adopters who treat the final
|
||||
// mp4 as user-keepable can switch to `Enabled`.
|
||||
versioned: false,
|
||||
lifecycleRules: [
|
||||
{
|
||||
id: "ExpireIntermediates",
|
||||
enabled: true,
|
||||
prefix: "renders/",
|
||||
expiration: Duration.days(7),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
this.renderFunction = new lambda.Function(this, "RenderFunction", {
|
||||
functionName: `${projectName}-render`,
|
||||
runtime: lambda.Runtime.NODEJS_22_X,
|
||||
handler: "handler.handler",
|
||||
code: lambda.Code.fromAsset(handlerZipPath),
|
||||
memorySize,
|
||||
timeout: Duration.seconds(timeoutSec),
|
||||
ephemeralStorageSize: Size.gibibytes(10),
|
||||
architecture: lambda.Architecture.X86_64,
|
||||
reservedConcurrentExecutions: props.reservedConcurrency,
|
||||
tracing: lambda.Tracing.ACTIVE,
|
||||
environment: {
|
||||
NODE_OPTIONS: "--enable-source-maps",
|
||||
TMPDIR: "/tmp",
|
||||
HYPERFRAMES_LAMBDA_CHROME_SOURCE: chromeSource,
|
||||
HYPERFRAMES_RENDER_BUCKET: this.bucket.bucketName,
|
||||
},
|
||||
});
|
||||
|
||||
// Scoped S3 perms only — explicitly NOT `CloudWatchLogsFullAccess`,
|
||||
// which would grant `logs:*` on `*` and overscope adopter accounts.
|
||||
// SAM's AWSLambdaBasicExecutionRole equivalent is included by the
|
||||
// default `new lambda.Function` execution role.
|
||||
this.bucket.grantReadWrite(this.renderFunction);
|
||||
|
||||
const stateMachineLogGroup = new logs.LogGroup(this, "RenderStateMachineLogGroup", {
|
||||
logGroupName: `/aws/states/${projectName}-render`,
|
||||
retention: logs.RetentionDays.ONE_MONTH,
|
||||
removalPolicy: RemovalPolicy.DESTROY,
|
||||
});
|
||||
|
||||
const definition = this.buildStateMachineDefinition();
|
||||
|
||||
this.stateMachine = new sfn.StateMachine(this, "RenderStateMachine", {
|
||||
stateMachineName: `${projectName}-render`,
|
||||
stateMachineType: sfn.StateMachineType.STANDARD,
|
||||
definitionBody: sfn.DefinitionBody.fromChainable(definition),
|
||||
tracingEnabled: true,
|
||||
timeout: Duration.hours(1),
|
||||
logs: {
|
||||
destination: stateMachineLogGroup,
|
||||
level: sfn.LogLevel.ERROR,
|
||||
includeExecutionData: false,
|
||||
},
|
||||
});
|
||||
|
||||
this.renderFunction.grantInvoke(this.stateMachine);
|
||||
|
||||
new cloudwatch.Alarm(this, "RenderChunkInvocationAlarm", {
|
||||
alarmName: `${projectName}-runaway-chunk-invocations`,
|
||||
alarmDescription:
|
||||
"Fires if RenderChunk Lambda invocations exceed the configured threshold in a 1-hour window.",
|
||||
metric: this.renderFunction.metricInvocations({
|
||||
period: Duration.hours(1),
|
||||
statistic: cloudwatch.Stats.SUM,
|
||||
}),
|
||||
threshold: alarmThreshold,
|
||||
evaluationPeriods: 1,
|
||||
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
||||
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||
});
|
||||
|
||||
new cloudwatch.Alarm(this, "RenderFunctionErrorsAlarm", {
|
||||
alarmName: `${projectName}-render-function-errors`,
|
||||
alarmDescription: "Fires if the render Lambda reports any errors in a 5-minute window.",
|
||||
metric: this.renderFunction.metricErrors({
|
||||
period: Duration.minutes(5),
|
||||
statistic: cloudwatch.Stats.SUM,
|
||||
}),
|
||||
threshold: 1,
|
||||
evaluationPeriods: 1,
|
||||
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
||||
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||
});
|
||||
|
||||
new cloudwatch.Alarm(this, "RenderStateMachineFailedAlarm", {
|
||||
alarmName: `${projectName}-render-state-machine-failed`,
|
||||
alarmDescription: "Fires when the render state machine reports a failed execution.",
|
||||
metric: this.stateMachine.metricFailed({
|
||||
period: Duration.minutes(5),
|
||||
statistic: cloudwatch.Stats.SUM,
|
||||
}),
|
||||
threshold: 1,
|
||||
evaluationPeriods: 1,
|
||||
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
||||
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the state-machine chain. Kept in a single method so the SAM
|
||||
* template and this construct can be diffed shape-for-shape during
|
||||
* the snapshot test.
|
||||
*/
|
||||
private buildStateMachineDefinition(): sfn.IChainable {
|
||||
// `ChromeBinaryUnavailableError` is non-retryable: a wedged warm
|
||||
// instance keeps returning the same falsy executablePath until the
|
||||
// env recycles, so retries just burn the 4× 15-min budget.
|
||||
const NON_RETRYABLE_PLAN = [
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
"PLAN_HASH_MISMATCH",
|
||||
"S3_URI_NOT_ALLOWED",
|
||||
"BROWSER_GPU_NOT_SOFTWARE",
|
||||
"FONT_FETCH_FAILED",
|
||||
"PLAN_TOO_LARGE",
|
||||
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
||||
"ChromeBinaryUnavailableError",
|
||||
];
|
||||
const NON_RETRYABLE_CHUNK = [
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
"PLAN_HASH_MISMATCH",
|
||||
"S3_URI_NOT_ALLOWED",
|
||||
"BROWSER_GPU_NOT_SOFTWARE",
|
||||
"ChromeBinaryUnavailableError",
|
||||
];
|
||||
const NON_RETRYABLE_ASSEMBLE = [
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
"PLAN_HASH_MISMATCH",
|
||||
"S3_URI_NOT_ALLOWED",
|
||||
"FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED",
|
||||
"ChromeBinaryUnavailableError",
|
||||
];
|
||||
|
||||
const plan = new tasks.LambdaInvoke(this, "Plan", {
|
||||
lambdaFunction: this.renderFunction,
|
||||
payload: sfn.TaskInput.fromObject({
|
||||
Action: "plan",
|
||||
"ProjectS3Uri.$": "$.ProjectS3Uri",
|
||||
"PlanOutputS3Prefix.$": "$.PlanOutputS3Prefix",
|
||||
"Config.$": "$.Config",
|
||||
}),
|
||||
resultSelector: {
|
||||
"PlanS3Uri.$": "$.Payload.PlanS3Uri",
|
||||
"PlanHash.$": "$.Payload.PlanHash",
|
||||
"ChunkCount.$": "$.Payload.ChunkCount",
|
||||
"Format.$": "$.Payload.Format",
|
||||
"HasAudio.$": "$.Payload.HasAudio",
|
||||
"AudioS3Uri.$": "$.Payload.AudioS3Uri",
|
||||
},
|
||||
resultPath: "$.Plan",
|
||||
});
|
||||
plan.addRetry({
|
||||
errors: NON_RETRYABLE_PLAN,
|
||||
maxAttempts: 0,
|
||||
});
|
||||
plan.addRetry({
|
||||
errors: ["States.ALL"],
|
||||
interval: Duration.seconds(2),
|
||||
maxAttempts: 4,
|
||||
backoffRate: 2,
|
||||
maxDelay: Duration.seconds(60),
|
||||
});
|
||||
|
||||
const buildChunkList = new sfn.Pass(this, "BuildChunkList", {
|
||||
parameters: {
|
||||
"ChunkIndexes.$": "States.ArrayRange(0, States.MathAdd($.Plan.ChunkCount, -1), 1)",
|
||||
},
|
||||
resultPath: "$.Iterator",
|
||||
});
|
||||
|
||||
const planProducedZero = new sfn.Fail(this, "PlanProducedZeroChunks", {
|
||||
error: "PLAN_TOO_LARGE",
|
||||
cause: "Plan returned ChunkCount=0 — non-retryable producer-side invariant violation.",
|
||||
});
|
||||
|
||||
const renderChunkTask = new tasks.LambdaInvoke(this, "RenderChunk", {
|
||||
lambdaFunction: this.renderFunction,
|
||||
payload: sfn.TaskInput.fromObject({
|
||||
Action: "renderChunk",
|
||||
"ChunkIndex.$": "$.ChunkIndex",
|
||||
"PlanS3Uri.$": "$.PlanS3Uri",
|
||||
"PlanHash.$": "$.PlanHash",
|
||||
"ChunkOutputS3Prefix.$": "$.ChunkOutputS3Prefix",
|
||||
"Format.$": "$.Format",
|
||||
}),
|
||||
resultSelector: {
|
||||
"ChunkS3Uri.$": "$.Payload.ChunkS3Uri",
|
||||
"ChunkIndex.$": "$.Payload.ChunkIndex",
|
||||
"Sha256.$": "$.Payload.Sha256",
|
||||
},
|
||||
});
|
||||
renderChunkTask.addRetry({
|
||||
errors: NON_RETRYABLE_CHUNK,
|
||||
maxAttempts: 0,
|
||||
});
|
||||
renderChunkTask.addRetry({
|
||||
errors: ["States.ALL"],
|
||||
interval: Duration.seconds(2),
|
||||
maxAttempts: 4,
|
||||
backoffRate: 2,
|
||||
maxDelay: Duration.seconds(60),
|
||||
});
|
||||
|
||||
const renderChunks = new sfn.Map(this, "RenderChunks", {
|
||||
itemsPath: "$.Iterator.ChunkIndexes",
|
||||
itemSelector: {
|
||||
"ChunkIndex.$": "$$.Map.Item.Value",
|
||||
"PlanS3Uri.$": "$.Plan.PlanS3Uri",
|
||||
"PlanHash.$": "$.Plan.PlanHash",
|
||||
"ChunkOutputS3Prefix.$": "$.PlanOutputS3Prefix",
|
||||
"Format.$": "$.Plan.Format",
|
||||
},
|
||||
maxConcurrencyPath: "$.Plan.ChunkCount",
|
||||
resultPath: "$.Chunks",
|
||||
});
|
||||
renderChunks.itemProcessor(renderChunkTask);
|
||||
|
||||
const assemble = new tasks.LambdaInvoke(this, "Assemble", {
|
||||
lambdaFunction: this.renderFunction,
|
||||
payload: sfn.TaskInput.fromObject({
|
||||
Action: "assemble",
|
||||
"PlanS3Uri.$": "$.Plan.PlanS3Uri",
|
||||
"ChunkS3Uris.$": "$.Chunks[*].ChunkS3Uri",
|
||||
"AudioS3Uri.$": "$.Plan.AudioS3Uri",
|
||||
"OutputS3Uri.$": "$.OutputS3Uri",
|
||||
"Format.$": "$.Plan.Format",
|
||||
}),
|
||||
resultSelector: {
|
||||
"OutputS3Uri.$": "$.Payload.OutputS3Uri",
|
||||
"FramesEncoded.$": "$.Payload.FramesEncoded",
|
||||
"FileSize.$": "$.Payload.FileSize",
|
||||
},
|
||||
resultPath: "$.Output",
|
||||
});
|
||||
assemble.addRetry({
|
||||
errors: NON_RETRYABLE_ASSEMBLE,
|
||||
maxAttempts: 0,
|
||||
});
|
||||
assemble.addRetry({
|
||||
errors: ["States.ALL"],
|
||||
interval: Duration.seconds(2),
|
||||
maxAttempts: 4,
|
||||
backoffRate: 2,
|
||||
maxDelay: Duration.seconds(60),
|
||||
});
|
||||
|
||||
const assertChunkCount = new sfn.Choice(this, "AssertChunkCount")
|
||||
.when(sfn.Condition.numberGreaterThan("$.Plan.ChunkCount", 0), renderChunks.next(assemble))
|
||||
.otherwise(planProducedZero);
|
||||
|
||||
return plan.next(buildChunkList).next(assertChunkCount);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Default location of the handler ZIP relative to this source file. Two
|
||||
* parents up = `packages/aws-lambda/`; the build script writes the ZIP
|
||||
* to `packages/aws-lambda/dist/handler.zip`. The package is published with
|
||||
* `main: "./src/index.ts"`, so this path resolves correctly both in the
|
||||
* source tree (during `bun test` / local CDK synth) and in a consumer's
|
||||
* `node_modules/@hyperframes/aws-lambda/` install.
|
||||
*/
|
||||
function defaultHandlerZipPath(): string {
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
return resolve(here, "..", "..", "dist", "handler.zip");
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* CDK subpath export — `@hyperframes/aws-lambda/cdk`.
|
||||
*
|
||||
* Pulled into its own subpath so SDK-only consumers don't import
|
||||
* `aws-cdk-lib`. The construct itself depends on `aws-cdk-lib` and
|
||||
* `constructs` as peer dependencies; adopters using CDK already have
|
||||
* both installed.
|
||||
*/
|
||||
|
||||
export {
|
||||
HyperframesRenderStack,
|
||||
type HyperframesRenderStackProps,
|
||||
} from "./HyperframesRenderStack.js";
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Unit tests for the Chrome runtime resolver.
|
||||
*
|
||||
* The actual @sparticuz/chromium probe lives in
|
||||
* `scripts/probe-beginframe.ts` (run in a Lambda-like Docker container).
|
||||
* These tests pin the env-var → source-selection logic so a misconfigured
|
||||
* deploy fails loudly rather than silently picking the wrong binary.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
_setSparticuzChromiumForTests,
|
||||
ChromeBinaryUnavailableError,
|
||||
resolveChromeArgs,
|
||||
resolveChromeExecutablePath,
|
||||
resolveChromeSource,
|
||||
} from "./chromium.js";
|
||||
|
||||
const savedEnv: Record<string, string | undefined> = {};
|
||||
|
||||
beforeEach(() => {
|
||||
savedEnv.HYPERFRAMES_LAMBDA_CHROME_SOURCE = process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE;
|
||||
savedEnv.HYPERFRAMES_LAMBDA_CHROME_PATH = process.env.HYPERFRAMES_LAMBDA_CHROME_PATH;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = savedEnv.HYPERFRAMES_LAMBDA_CHROME_SOURCE;
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_PATH = savedEnv.HYPERFRAMES_LAMBDA_CHROME_PATH;
|
||||
_setSparticuzChromiumForTests(null);
|
||||
});
|
||||
|
||||
describe("resolveChromeSource", () => {
|
||||
it("defaults to sparticuz when no env var is set", () => {
|
||||
delete process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE;
|
||||
expect(resolveChromeSource()).toBe("sparticuz");
|
||||
});
|
||||
|
||||
it("returns chrome-headless-shell when env var requests it", () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell";
|
||||
expect(resolveChromeSource()).toBe("chrome-headless-shell");
|
||||
});
|
||||
|
||||
it("accepts the short alias 'shell'", () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "shell";
|
||||
expect(resolveChromeSource()).toBe("chrome-headless-shell");
|
||||
});
|
||||
|
||||
it("is case insensitive", () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "Chrome-Headless-Shell";
|
||||
expect(resolveChromeSource()).toBe("chrome-headless-shell");
|
||||
});
|
||||
|
||||
it("falls back to sparticuz on an unknown value", () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "wat";
|
||||
expect(resolveChromeSource()).toBe("sparticuz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveChromeExecutablePath", () => {
|
||||
it("returns the path from a stubbed sparticuz module", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz";
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-chrome-test-"));
|
||||
const binPath = join(dir, "chromium");
|
||||
writeFileSync(binPath, "fake binary");
|
||||
try {
|
||||
_setSparticuzChromiumForTests({
|
||||
args: ["--fake-arg"],
|
||||
executablePath: async () => binPath,
|
||||
});
|
||||
expect(await resolveChromeExecutablePath()).toBe(binPath);
|
||||
expect(await resolveChromeArgs()).toEqual(["--fake-arg"]);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Real-world wedge: a chunk hits `Sandbox.Timedout` mid-extraction,
|
||||
// leaving sparticuz's internal state inconsistent. On subsequent
|
||||
// invocations executablePath() returns empty/undefined and puppeteer-
|
||||
// core's downstream assertion buries the actionable signal. These
|
||||
// tests pin the typed-error contract so SFN can short-circuit retries.
|
||||
it("throws ChromeBinaryUnavailableError when sparticuz returns empty string", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz";
|
||||
_setSparticuzChromiumForTests({
|
||||
args: [],
|
||||
executablePath: async () => "",
|
||||
});
|
||||
await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf(
|
||||
ChromeBinaryUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws ChromeBinaryUnavailableError when sparticuz returns a non-existent path", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz";
|
||||
_setSparticuzChromiumForTests({
|
||||
args: [],
|
||||
executablePath: async () => "/nonexistent/sparticuz/chromium",
|
||||
});
|
||||
await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf(
|
||||
ChromeBinaryUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
it("ChromeBinaryUnavailableError carries the source + resolved path", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "sparticuz";
|
||||
_setSparticuzChromiumForTests({
|
||||
args: [],
|
||||
executablePath: async () => "/missing",
|
||||
});
|
||||
try {
|
||||
await resolveChromeExecutablePath();
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(ChromeBinaryUnavailableError);
|
||||
const typed = err as ChromeBinaryUnavailableError;
|
||||
expect(typed.source).toBe("sparticuz");
|
||||
expect(typed.resolvedPath).toBe("/missing");
|
||||
expect(typed.name).toBe("ChromeBinaryUnavailableError");
|
||||
}
|
||||
});
|
||||
|
||||
it("reads chrome-headless-shell path from HYPERFRAMES_LAMBDA_CHROME_PATH", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell";
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-chrome-test-"));
|
||||
const binPath = join(dir, "chrome-headless-shell");
|
||||
writeFileSync(binPath, "fake binary contents");
|
||||
try {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_PATH = binPath;
|
||||
expect(await resolveChromeExecutablePath()).toBe(binPath);
|
||||
expect(await resolveChromeArgs()).toEqual([]);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("throws ChromeBinaryUnavailableError if chrome-headless-shell path is missing", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell";
|
||||
delete process.env.HYPERFRAMES_LAMBDA_CHROME_PATH;
|
||||
await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf(
|
||||
ChromeBinaryUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
it("throws ChromeBinaryUnavailableError if chrome-headless-shell path doesn't exist on disk", async () => {
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE = "chrome-headless-shell";
|
||||
process.env.HYPERFRAMES_LAMBDA_CHROME_PATH = "/nonexistent/path/chrome-headless-shell";
|
||||
await expect(resolveChromeExecutablePath()).rejects.toBeInstanceOf(
|
||||
ChromeBinaryUnavailableError,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Lambda-runtime Chrome resolver.
|
||||
*
|
||||
* `renderChunk()` (the only primitive that needs a browser) launches Chrome
|
||||
* via the engine's `BrowserManager`. In Lambda we can't ship the full
|
||||
* Puppeteer-managed Chrome download — Puppeteer's Chrome binary is ~330 MB
|
||||
* unzipped, well over Lambda's 250 MB ZIP-deploy ceiling.
|
||||
*
|
||||
* Two valid runtime sources:
|
||||
*
|
||||
* 1. `@sparticuz/chromium` (primary). Decompresses a Lambda-optimised
|
||||
* `chrome-headless-shell` build into `/tmp` at runtime. ~70 MB
|
||||
* compressed; the same binary the rest of the ecosystem uses for
|
||||
* headless-Chrome-in-Lambda. CDP-level BeginFrame works because the
|
||||
* command lives in the protocol, not the binary; the
|
||||
* `scripts/probe-beginframe.ts` regression guard pins this.
|
||||
*
|
||||
* 2. A bundled `chrome-headless-shell` binary (fallback). If
|
||||
* `@sparticuz/chromium`'s build ever drops `HeadlessExperimental`
|
||||
* support, we fall back to the same `chrome-headless-shell` build
|
||||
* the K8s deploy uses. The fallback raises the ZIP from ~70 MB
|
||||
* Chrome to ~140 MB Chrome — still well under 250 MB.
|
||||
*
|
||||
* The runtime path is selected by the `HYPERFRAMES_LAMBDA_CHROME_SOURCE`
|
||||
* env var (set by `build-zip.ts`):
|
||||
*
|
||||
* "sparticuz" → use `@sparticuz/chromium.executablePath()`
|
||||
* "chrome-headless-shell" → use `process.env.HYPERFRAMES_LAMBDA_CHROME_PATH`
|
||||
*
|
||||
* Adapters that bundle this package can override
|
||||
* `HYPERFRAMES_LAMBDA_CHROME_PATH` directly when running outside Lambda
|
||||
* (e.g. the SAM-local RIE smoke).
|
||||
*/
|
||||
|
||||
import { existsSync } from "node:fs";
|
||||
|
||||
/** Discriminator for the two supported Chrome sources. */
|
||||
export type ChromeSource = "sparticuz" | "chrome-headless-shell";
|
||||
|
||||
/**
|
||||
* Thrown when the Chrome binary resolver can't produce a usable path.
|
||||
* The class name is the SFN `Retry: { ErrorEquals: [...] }` discriminator —
|
||||
* see {@link HyperframesRenderStack}'s NON_RETRYABLE_* lists.
|
||||
*/
|
||||
export class ChromeBinaryUnavailableError extends Error {
|
||||
// Lambda's runtime serializes the error envelope's `errorType` from
|
||||
// `err.name`; this class-field override sets it across the structured
|
||||
// clone. Read indirectly; fallow can't follow.
|
||||
// fallow-ignore-next-line unused-class-member
|
||||
override readonly name = "ChromeBinaryUnavailableError";
|
||||
readonly source: ChromeSource;
|
||||
readonly resolvedPath: string | null;
|
||||
constructor(source: ChromeSource, resolvedPath: string | null, hint: string) {
|
||||
super(`[chromium] Chrome binary unavailable (source=${source}): ${hint}`);
|
||||
this.source = source;
|
||||
this.resolvedPath = resolvedPath;
|
||||
}
|
||||
}
|
||||
|
||||
const SPARTICUZ_WEDGE_HINT =
|
||||
"@sparticuz/chromium.executablePath() returned a falsy value or a path that doesn't exist on disk. " +
|
||||
"This typically happens after a chunk hits `Sandbox.Timedout` mid-extraction and leaves /tmp in a " +
|
||||
"wedged state — subsequent invocations land on the same warm instance and never re-extract. " +
|
||||
"Recycle the function (e.g. `aws lambda update-function-configuration ... --environment ...` with a " +
|
||||
"bumped marker var, or redeploy via `hyperframes lambda deploy --skip-build`) to force fresh " +
|
||||
"execution environments. Tracking: investigate the upstream wedge so this auto-recovers.";
|
||||
|
||||
/**
|
||||
* Read which Chrome source the bundled ZIP was built against. Defaults to
|
||||
* `"sparticuz"` so a fresh build with no env override picks the primary
|
||||
* path.
|
||||
*/
|
||||
export function resolveChromeSource(): ChromeSource {
|
||||
const raw = process.env.HYPERFRAMES_LAMBDA_CHROME_SOURCE?.toLowerCase();
|
||||
if (raw === "chrome-headless-shell" || raw === "shell") return "chrome-headless-shell";
|
||||
return "sparticuz";
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the absolute path to a Chrome binary suitable for BeginFrame.
|
||||
*
|
||||
* For `"sparticuz"`: dynamically import `@sparticuz/chromium` and call
|
||||
* `chromium.executablePath()`. The module is dynamic so a build-zip that
|
||||
* never reaches the import (because the fallback Chrome is bundled) can
|
||||
* tree-shake it out.
|
||||
*
|
||||
* For `"chrome-headless-shell"`: read the path from
|
||||
* `HYPERFRAMES_LAMBDA_CHROME_PATH`. Throws if absent or non-existent so a
|
||||
* misconfigured deploy fails loudly at boot rather than at first frame.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function resolveChromeExecutablePath(): Promise<string> {
|
||||
const source = resolveChromeSource();
|
||||
if (source === "sparticuz") {
|
||||
const mod = await loadSparticuzChromium();
|
||||
const path = await mod.executablePath();
|
||||
// Guard against the wedge described in ChromeBinaryUnavailableError.
|
||||
// sparticuz's contract is "return the path to a usable binary" — when
|
||||
// it returns null/undefined/"" we can't hand that to puppeteer-core
|
||||
// (which will throw an unrelated-looking assertion). Same when the
|
||||
// returned path doesn't exist (extraction failed but the function
|
||||
// call returned).
|
||||
if (!path || typeof path !== "string") {
|
||||
throw new ChromeBinaryUnavailableError(source, null, SPARTICUZ_WEDGE_HINT);
|
||||
}
|
||||
if (!existsSync(path)) {
|
||||
throw new ChromeBinaryUnavailableError(source, path, SPARTICUZ_WEDGE_HINT);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
const explicit = process.env.HYPERFRAMES_LAMBDA_CHROME_PATH;
|
||||
if (!explicit) {
|
||||
throw new ChromeBinaryUnavailableError(
|
||||
source,
|
||||
null,
|
||||
"HYPERFRAMES_LAMBDA_CHROME_SOURCE=chrome-headless-shell requires " +
|
||||
"HYPERFRAMES_LAMBDA_CHROME_PATH to be set to the absolute path of the bundled binary.",
|
||||
);
|
||||
}
|
||||
if (!existsSync(explicit)) {
|
||||
throw new ChromeBinaryUnavailableError(
|
||||
source,
|
||||
explicit,
|
||||
`HYPERFRAMES_LAMBDA_CHROME_PATH=${JSON.stringify(explicit)} does not exist on disk.`,
|
||||
);
|
||||
}
|
||||
return explicit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the Chromium launch args for the selected source. For
|
||||
* `@sparticuz/chromium` we forward `chromium.args` (Lambda-tuned defaults
|
||||
* — single-process, no-sandbox, /tmp paths). For the shell fallback the
|
||||
* engine's own arg builder owns it; we return an empty array so the
|
||||
* engine's defaults apply.
|
||||
*/
|
||||
export async function resolveChromeArgs(): Promise<string[]> {
|
||||
if (resolveChromeSource() !== "sparticuz") return [];
|
||||
const mod = await loadSparticuzChromium();
|
||||
return mod.args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dynamic import wrapper isolated so unit tests can stub the module without
|
||||
* jest-style module mocking gymnastics. The narrow type here pins the
|
||||
* subset of `@sparticuz/chromium`'s surface this package depends on; if
|
||||
* the upstream module ever changes shape the type error here surfaces
|
||||
* before runtime.
|
||||
*/
|
||||
interface SparticuzChromiumModule {
|
||||
args: string[];
|
||||
executablePath(): Promise<string>;
|
||||
}
|
||||
|
||||
let cachedSparticuz: SparticuzChromiumModule | null = null;
|
||||
|
||||
async function loadSparticuzChromium(): Promise<SparticuzChromiumModule> {
|
||||
if (cachedSparticuz) return cachedSparticuz;
|
||||
const mod = (await import("@sparticuz/chromium")) as
|
||||
| SparticuzChromiumModule
|
||||
| { default: SparticuzChromiumModule };
|
||||
const resolved = "default" in mod ? mod.default : mod;
|
||||
cachedSparticuz = resolved;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Test-only seam: replace the cached `@sparticuz/chromium` module. */
|
||||
export function _setSparticuzChromiumForTests(mod: SparticuzChromiumModule | null): void {
|
||||
cachedSparticuz = mod;
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Lambda event + result types for the HyperFrames distributed render handler.
|
||||
*
|
||||
* The Step Functions state machine in `examples/aws-lambda/template.yaml`
|
||||
* dispatches on the `Action` field. Each action maps 1:1 onto one of the
|
||||
* three OSS distributed primitives:
|
||||
*
|
||||
* "plan" → `plan(projectDir, config, planDir)` (Activity A)
|
||||
* "renderChunk" → `renderChunk(planDir, chunkIndex, output)` (Activity B)
|
||||
* "assemble" → `assemble(planDir, chunkPaths, audio, out)` (Activity C)
|
||||
*
|
||||
* All file I/O is mediated by S3 — the handler downloads inputs into
|
||||
* `/tmp` (Lambda's only writable filesystem path), invokes the primitive,
|
||||
* uploads outputs back to S3, and returns a small JSON payload that fits
|
||||
* inside Step Functions' history budget (under 200 bytes for chunk
|
||||
* results per §2.4).
|
||||
*/
|
||||
|
||||
import type {
|
||||
DistributedFormat,
|
||||
SerializableDistributedRenderConfig,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
|
||||
export type { SerializableDistributedRenderConfig } from "@hyperframes/producer/distributed";
|
||||
|
||||
/** Discriminator for the three roles the one Lambda image fulfills. */
|
||||
export type LambdaAction = "plan" | "renderChunk" | "assemble";
|
||||
|
||||
/**
|
||||
* Top-level shape of any event the handler may receive.
|
||||
*
|
||||
* Step Functions can also invoke with a wrapped payload (e.g. when a Map
|
||||
* state's `ItemSelector` passes through `$$.Map.Item.Value`), so the
|
||||
* handler unwraps both `event.Payload` and `event.Input` before
|
||||
* dispatching.
|
||||
*/
|
||||
export type LambdaEvent =
|
||||
| PlanEvent
|
||||
| RenderChunkEvent
|
||||
| AssembleEvent
|
||||
| { Payload: LambdaEvent }
|
||||
| { Input: LambdaEvent };
|
||||
|
||||
/** Activity A: produce a planDir, upload to S3. */
|
||||
export interface PlanEvent {
|
||||
Action: "plan";
|
||||
/** S3 URI pointing at a `tar -czf`-archived project directory (`s3://bucket/key.tar.gz`). */
|
||||
ProjectS3Uri: string;
|
||||
/** S3 URI prefix where the planDir tar should be uploaded (`s3://bucket/{prefix}/`). */
|
||||
PlanOutputS3Prefix: string;
|
||||
/** `DistributedRenderConfig` minus runtime-only fields (logger, abortSignal). */
|
||||
Config: SerializableDistributedRenderConfig;
|
||||
}
|
||||
|
||||
/** Activity B: fetch planDir, render one chunk, upload result. */
|
||||
export interface RenderChunkEvent {
|
||||
Action: "renderChunk";
|
||||
/** S3 URI of the plan tar produced by a PlanEvent invocation. */
|
||||
PlanS3Uri: string;
|
||||
/**
|
||||
* `PlanResult.planHash` from the Plan invocation. The handler verifies
|
||||
* this against the untarred planDir's `plan.json` before invoking the
|
||||
* producer, throwing a typed `PLAN_HASH_MISMATCH` on divergence so the
|
||||
* state machine routes it as non-retryable. Defense-in-depth — the
|
||||
* producer also re-checks internally.
|
||||
*/
|
||||
PlanHash: string;
|
||||
/** 0-based chunk index this invocation should render. */
|
||||
ChunkIndex: number;
|
||||
/** S3 URI prefix where the chunk output should be uploaded (`s3://bucket/{prefix}/`). */
|
||||
ChunkOutputS3Prefix: string;
|
||||
/** Output container format from the plan's encoder.json; drives file vs frame-dir handling. */
|
||||
Format: DistributedFormat;
|
||||
}
|
||||
|
||||
/** Activity C: fetch planDir + all chunks + audio, assemble, upload final. */
|
||||
export interface AssembleEvent {
|
||||
Action: "assemble";
|
||||
/** S3 URI of the plan tar produced by a PlanEvent invocation. */
|
||||
PlanS3Uri: string;
|
||||
/** S3 URIs of every chunk, ordered by chunk index. Length must equal `chunkCount`. */
|
||||
ChunkS3Uris: string[];
|
||||
/** S3 URI of the planDir's `audio.aac` if the composition has audio; `null` otherwise. */
|
||||
AudioS3Uri: string | null;
|
||||
/** Final output S3 URI (`s3://bucket/key.mp4`). */
|
||||
OutputS3Uri: string;
|
||||
/** Output container format; drives file vs frame-dir handling. */
|
||||
Format: DistributedFormat;
|
||||
/**
|
||||
* Optional exact-CFR re-encode at assemble time. When `true`, the final
|
||||
* assembled video is re-encoded with `-fps_mode cfr -r <fps>` so the
|
||||
* stream's `avg_frame_rate` matches the container's `r_frame_rate`
|
||||
* exactly (and the file's duration is exact, not PTS-derived). Trade-off
|
||||
* is ~2-5x the assemble wall-clock. mp4 only — webm / mov stream-copy
|
||||
* paths already produce exact avg_frame_rate. Default `false` /
|
||||
* unset preserves current `-c copy` behavior.
|
||||
*/
|
||||
Cfr?: boolean;
|
||||
}
|
||||
|
||||
// ── Result types — kept small to fit Step Functions history budgets ─────────
|
||||
|
||||
/** Result of a `plan` invocation. Carries enough to size the Map(N) state. */
|
||||
export interface PlanLambdaResult {
|
||||
Action: "plan";
|
||||
PlanS3Uri: string;
|
||||
PlanHash: string;
|
||||
ChunkCount: number;
|
||||
TotalFrames: number;
|
||||
Fps: 24 | 30 | 60;
|
||||
Width: number;
|
||||
Height: number;
|
||||
Format: DistributedFormat;
|
||||
HasAudio: boolean;
|
||||
AudioS3Uri: string | null;
|
||||
FfmpegVersion: string;
|
||||
ProducerVersion: string;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
/** Result of a `renderChunk` invocation. Sized ≤200 bytes per §2.4. */
|
||||
export interface RenderChunkLambdaResult {
|
||||
Action: "renderChunk";
|
||||
ChunkS3Uri: string;
|
||||
ChunkIndex: number;
|
||||
Sha256: string;
|
||||
FramesEncoded: number;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
/** Result of an `assemble` invocation. */
|
||||
export interface AssembleLambdaResult {
|
||||
Action: "assemble";
|
||||
OutputS3Uri: string;
|
||||
FramesEncoded: number;
|
||||
FileSize: number;
|
||||
DurationMs: number;
|
||||
}
|
||||
|
||||
export type LambdaResult = PlanLambdaResult | RenderChunkLambdaResult | AssembleLambdaResult;
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Map a distributed `format` to the file extension the assembled output
|
||||
* should carry on disk + in S3. Shared by `src/handler.ts` (chunk +
|
||||
* assemble output paths) and `src/sdk/renderToLambda.ts` (final
|
||||
* output key construction) so the two sides agree on what an mp4
|
||||
* looks like vs a png-sequence.
|
||||
*/
|
||||
|
||||
import type { DistributedFormat } from "@hyperframes/producer/distributed";
|
||||
|
||||
export type { DistributedFormat } from "@hyperframes/producer/distributed";
|
||||
|
||||
// Closed-enum lookup table. TS enforces exhaustiveness via the
|
||||
// `Record<DistributedFormat, string>` annotation — adding a format to
|
||||
// `DistributedFormat` without adding the matching key here fails to
|
||||
// typecheck, which is the same exhaustiveness guarantee a switch +
|
||||
// `_exhaustive: never` arm provides but at lower complexity.
|
||||
const FORMAT_EXTENSIONS: Record<DistributedFormat, string> = {
|
||||
mp4: ".mp4",
|
||||
mov: ".mov",
|
||||
webm: ".webm",
|
||||
"png-sequence": "",
|
||||
};
|
||||
|
||||
export function formatExtension(format: DistributedFormat): string {
|
||||
return FORMAT_EXTENSIONS[format];
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
/**
|
||||
* Handler dispatch unit tests.
|
||||
*
|
||||
* Asserts that:
|
||||
* - The handler routes Action="plan" / "renderChunk" / "assemble" to the
|
||||
* matching OSS primitive.
|
||||
* - It unwraps Step Functions `{ Payload }` and `{ Input }` envelopes.
|
||||
* - It rejects unknown actions with a clear message.
|
||||
* - It plumbs S3 download/upload calls in the correct order.
|
||||
*
|
||||
* The real OSS primitives are NOT exercised here — they live in
|
||||
* `@hyperframes/producer/distributed` and have their own coverage in
|
||||
* `packages/producer`. The Lambda handler is thin glue; this file pins
|
||||
* the glue's contract.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AssembleResult, ChunkResult, PlanResult } from "@hyperframes/producer/distributed";
|
||||
import type { AssembleEvent, LambdaEvent, PlanEvent, RenderChunkEvent } from "./events.js";
|
||||
import { handler, unwrapEvent } from "./handler.js";
|
||||
|
||||
interface FakeS3Op {
|
||||
kind: "download" | "upload";
|
||||
uri: string;
|
||||
bytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory S3 stand-in. Records every operation so test assertions can
|
||||
* pin the exact sequence of downloads and uploads, plus fakes the GetObject
|
||||
* stream so {@link downloadS3ObjectToFile} writes the expected bytes.
|
||||
*/
|
||||
class FakeS3Client {
|
||||
ops: FakeS3Op[] = [];
|
||||
// Map S3 URIs → byte buffers the fake serves.
|
||||
objects = new Map<string, Buffer>();
|
||||
|
||||
// Methods called by the real S3 transport — minimal surface so the
|
||||
// handler's call sites don't need rewriting under test.
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const op = command as { input: { Bucket: string; Key: string } } & {
|
||||
constructor: { name: string };
|
||||
};
|
||||
const cmdName = op.constructor?.name ?? "";
|
||||
const uri = `s3://${op.input.Bucket}/${op.input.Key}`;
|
||||
if (cmdName === "GetObjectCommand") {
|
||||
const bytes = this.objects.get(uri) ?? Buffer.alloc(0);
|
||||
this.ops.push({ kind: "download", uri, bytes: bytes.length });
|
||||
// Mock the AWS SDK stream contract just enough for pipeline() to
|
||||
// pump bytes into a write stream.
|
||||
const { Readable } = await import("node:stream");
|
||||
return { Body: Readable.from([bytes]) };
|
||||
}
|
||||
if (cmdName === "PutObjectCommand") {
|
||||
// Buffer the body so we can record how many bytes were uploaded; the
|
||||
// handler's hot path streams from disk, but tests pin the count.
|
||||
const body = (command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body;
|
||||
let bytes = 0;
|
||||
if (Buffer.isBuffer(body)) {
|
||||
bytes = body.length;
|
||||
} else if (body && typeof (body as NodeJS.ReadableStream).pipe === "function") {
|
||||
for await (const chunk of body as NodeJS.ReadableStream) {
|
||||
bytes += (chunk as Buffer).length;
|
||||
}
|
||||
}
|
||||
this.ops.push({ kind: "upload", uri, bytes });
|
||||
this.objects.set(uri, Buffer.alloc(bytes));
|
||||
return {};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
const tmpDirs: string[] = [];
|
||||
|
||||
beforeEach(() => {
|
||||
// Each test gets its own tmp root so concurrent test runs don't share state.
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tmpDirs) {
|
||||
try {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
tmpDirs.length = 0;
|
||||
});
|
||||
|
||||
function makeTmpRoot(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-lambda-test-"));
|
||||
tmpDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe("unwrapEvent", () => {
|
||||
it("returns a bare event unchanged", () => {
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
expect(unwrapEvent(event).Action).toBe("plan");
|
||||
});
|
||||
|
||||
it("unwraps a Step Functions { Payload } envelope", () => {
|
||||
const inner: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "deadbeef",
|
||||
ChunkIndex: 3,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
};
|
||||
const wrapped: LambdaEvent = { Payload: inner };
|
||||
expect(unwrapEvent(wrapped).Action).toBe("renderChunk");
|
||||
});
|
||||
|
||||
it("unwraps multiple levels of envelopes", () => {
|
||||
const inner: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
ChunkS3Uris: ["s3://bucket/chunks/0001.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://bucket/output.mp4",
|
||||
Format: "mp4",
|
||||
};
|
||||
const doubly: LambdaEvent = { Payload: { Input: inner } };
|
||||
expect(unwrapEvent(doubly).Action).toBe("assemble");
|
||||
});
|
||||
|
||||
it("throws on unknown action", () => {
|
||||
expect(() => unwrapEvent({ Action: "doSomething" } as unknown as LambdaEvent)).toThrow(
|
||||
/no recognised Action/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handler dispatch", () => {
|
||||
it("routes Action='plan' to the plan primitive", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
// Seed a fake project tarball so the untar step has something to chew on.
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
|
||||
const planMock = mock(
|
||||
async (_projectDir: string, _config: unknown, planDir: string): Promise<PlanResult> => {
|
||||
// Simulate plan() writing a minimal planDir.
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), "[]");
|
||||
return {
|
||||
planDir,
|
||||
planHash: "fakehash",
|
||||
chunkCount: 4,
|
||||
totalFrames: 720,
|
||||
fps: 30 as const,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4" as const,
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "0.0.0-test",
|
||||
};
|
||||
},
|
||||
);
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("plan");
|
||||
if (result.Action !== "plan") throw new Error("unreachable");
|
||||
expect(result.PlanHash).toBe("fakehash");
|
||||
expect(result.ChunkCount).toBe(4);
|
||||
expect(planMock).toHaveBeenCalledTimes(1);
|
||||
expect(renderChunkMock).not.toHaveBeenCalled();
|
||||
expect(assembleMock).not.toHaveBeenCalled();
|
||||
// Plan should have downloaded the project zip and uploaded the plan tar.
|
||||
expect(
|
||||
s3.ops.some((o) => o.kind === "download" && o.uri === "s3://bucket/project.tar.gz"),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("plan honors a pre-set PRODUCER_HEADLESS_SHELL_PATH instead of re-resolving Chrome", async () => {
|
||||
// Mirrors the renderChunk env-var guard — when a caller (e.g. SAM-local
|
||||
// RIE smoke) seeds the path, handlePlan must not overwrite it.
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/project.tar.gz", await makeMinimalProjectTar());
|
||||
|
||||
const planMock = mock(
|
||||
async (_projectDir: string, _config: unknown, planDir: string): Promise<PlanResult> => {
|
||||
mkdirSync(planDir, { recursive: true });
|
||||
writeFileSync(join(planDir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
mkdirSync(join(planDir, "meta"), { recursive: true });
|
||||
writeFileSync(join(planDir, "meta", "chunks.json"), "[]");
|
||||
return {
|
||||
planDir,
|
||||
planHash: "fakehash",
|
||||
chunkCount: 1,
|
||||
totalFrames: 30,
|
||||
fps: 30 as const,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4" as const,
|
||||
ffmpegVersion: "6.0",
|
||||
producerVersion: "0.0.0-test",
|
||||
};
|
||||
},
|
||||
);
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
|
||||
const sentinel = "/tmp/test-chrome-sentinel";
|
||||
const prev = process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = sentinel;
|
||||
try {
|
||||
// Note: no skipChromeResolution flag — the guard must short-circuit
|
||||
// because PRODUCER_HEADLESS_SHELL_PATH is already set.
|
||||
await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
});
|
||||
expect(process.env.PRODUCER_HEADLESS_SHELL_PATH).toBe(sentinel);
|
||||
expect(planMock).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
if (prev === undefined) {
|
||||
delete process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
} else {
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = prev;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("routes Action='renderChunk' to the renderChunk primitive", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
// Seed a planDir tarball with a minimal structure renderChunk would
|
||||
// observe; the test mock doesn't read it, but the handler untar step does.
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
|
||||
const renderChunkMock = mock(
|
||||
async (
|
||||
_planDir: string,
|
||||
_chunkIndex: number,
|
||||
outputChunkPath: string,
|
||||
): Promise<ChunkResult> => {
|
||||
// Write a fake chunk file so the upload step has bytes to send.
|
||||
writeFileSync(outputChunkPath, Buffer.from("FAKE-MP4-CHUNK"));
|
||||
return {
|
||||
outputPath: outputChunkPath,
|
||||
outputKind: "file",
|
||||
framesEncoded: 240,
|
||||
sha256: "0".repeat(64),
|
||||
durationMs: 12345,
|
||||
perfPath: outputChunkPath + ".perf.json",
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const planMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "fakehash",
|
||||
ChunkIndex: 2,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("renderChunk");
|
||||
if (result.Action !== "renderChunk") throw new Error("unreachable");
|
||||
expect(result.ChunkIndex).toBe(2);
|
||||
expect(result.Sha256).toBe("0".repeat(64));
|
||||
expect(result.FramesEncoded).toBe(240);
|
||||
expect(renderChunkMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects renderChunk when event.PlanHash diverges from plan.json", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
// The fixture's plan.json has planHash="fakehash"; the event below
|
||||
// claims something else, so the handler must throw PLAN_HASH_MISMATCH
|
||||
// before invoking the primitive.
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
|
||||
const renderChunkMock = mock(async () => {
|
||||
throw new Error("primitive should not be called on a hash mismatch");
|
||||
});
|
||||
const planMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
const assembleMock = mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
});
|
||||
|
||||
const event: RenderChunkEvent = {
|
||||
Action: "renderChunk",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
PlanHash: "not-the-real-hash",
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputS3Prefix: "s3://bucket/renders/abc/",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: planMock as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk:
|
||||
renderChunkMock as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).name).toBe("PLAN_HASH_MISMATCH");
|
||||
expect((caught as Error).message).toMatch(/not-the-real-hash/);
|
||||
expect(renderChunkMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes Action='assemble' to the assemble primitive", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
s3.objects.set("s3://bucket/plan.tar.gz", await makeMinimalPlanTar());
|
||||
s3.objects.set("s3://bucket/chunks/0001.mp4", Buffer.from("CHUNK-1"));
|
||||
s3.objects.set("s3://bucket/chunks/0002.mp4", Buffer.from("CHUNK-2"));
|
||||
|
||||
const assembleMock = mock(
|
||||
async (
|
||||
_planDir: string,
|
||||
_chunkPaths: readonly string[],
|
||||
_audioPath: string | null,
|
||||
outputPath: string,
|
||||
): Promise<AssembleResult> => {
|
||||
writeFileSync(outputPath, Buffer.from("FAKE-FINAL-MP4"));
|
||||
return {
|
||||
outputPath,
|
||||
durationMs: 7777,
|
||||
framesEncoded: 480,
|
||||
fileSize: 14,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const event: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanS3Uri: "s3://bucket/plan.tar.gz",
|
||||
ChunkS3Uris: ["s3://bucket/chunks/0001.mp4", "s3://bucket/chunks/0002.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://bucket/renders/abc/output.mp4",
|
||||
Format: "mp4",
|
||||
};
|
||||
|
||||
const result = await handler(event, {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
primitives: {
|
||||
plan: mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").plan,
|
||||
renderChunk: mock(async () => {
|
||||
throw new Error("should not be called");
|
||||
}) as unknown as typeof import("@hyperframes/producer/distributed").renderChunk,
|
||||
assemble:
|
||||
assembleMock as unknown as typeof import("@hyperframes/producer/distributed").assemble,
|
||||
},
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
});
|
||||
|
||||
expect(result.Action).toBe("assemble");
|
||||
if (result.Action !== "assemble") throw new Error("unreachable");
|
||||
expect(result.OutputS3Uri).toBe("s3://bucket/renders/abc/output.mp4");
|
||||
expect(result.FramesEncoded).toBe(480);
|
||||
expect(assembleMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects unknown actions", async () => {
|
||||
const tmpRoot = makeTmpRoot();
|
||||
await expect(
|
||||
handler({ Action: "doSomething" } as unknown as LambdaEvent, {
|
||||
s3: new FakeS3Client() as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
}),
|
||||
).rejects.toThrow(/no recognised Action/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handler — S3 URI allowlist (security: F-004)", () => {
|
||||
let prevBucket: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
prevBucket = process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (prevBucket === undefined) {
|
||||
delete process.env.HYPERFRAMES_RENDER_BUCKET;
|
||||
} else {
|
||||
process.env.HYPERFRAMES_RENDER_BUCKET = prevBucket;
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a plan event whose ProjectS3Uri is outside the allowed bucket", async () => {
|
||||
process.env.HYPERFRAMES_RENDER_BUCKET = "good-bucket";
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
|
||||
const event: PlanEvent = {
|
||||
Action: "plan",
|
||||
ProjectS3Uri: "s3://evil-bucket/project.tar.gz",
|
||||
PlanOutputS3Prefix: "s3://good-bucket/renders/abc/",
|
||||
Config: { fps: 30, width: 1920, height: 1080, format: "mp4" },
|
||||
};
|
||||
const deps = {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
};
|
||||
|
||||
await expect(handler(event, deps)).rejects.toMatchObject({
|
||||
name: "S3_URI_NOT_ALLOWED",
|
||||
message: expect.stringContaining("evil-bucket"),
|
||||
});
|
||||
expect(s3.ops).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects an assemble event with a cross-bucket chunk URI", async () => {
|
||||
process.env.HYPERFRAMES_RENDER_BUCKET = "good-bucket";
|
||||
const tmpRoot = makeTmpRoot();
|
||||
const s3 = new FakeS3Client();
|
||||
|
||||
const event: AssembleEvent = {
|
||||
Action: "assemble",
|
||||
PlanS3Uri: "s3://good-bucket/plan.tar.gz",
|
||||
ChunkS3Uris: ["s3://good-bucket/chunks/0001.mp4", "s3://evil-bucket/chunks/0002.mp4"],
|
||||
AudioS3Uri: null,
|
||||
OutputS3Uri: "s3://good-bucket/renders/abc/output.mp4",
|
||||
Format: "mp4",
|
||||
};
|
||||
const deps = {
|
||||
s3: s3 as unknown as import("@aws-sdk/client-s3").S3Client,
|
||||
tmpRoot,
|
||||
skipChromeResolution: true,
|
||||
};
|
||||
|
||||
await expect(handler(event, deps)).rejects.toMatchObject({ name: "S3_URI_NOT_ALLOWED" });
|
||||
expect(s3.ops).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build the smallest valid `.tar.gz` the handler's untar step accepts: a
|
||||
* single file inside an archive. Uses the npm `tar` package (same as
|
||||
* `s3Transport.ts`) so the fixture builder runs cross-platform — Windows
|
||||
* doesn't ship GNU tar in `/usr/bin/tar`, and bare Alpine containers
|
||||
* don't ship `tar` at all. Keeps the test runnable everywhere the rest
|
||||
* of the suite runs.
|
||||
*/
|
||||
async function makeMinimalProjectTar(): Promise<Buffer> {
|
||||
const tar = await import("tar");
|
||||
const { mkdtempSync: mk, readFileSync, rmSync: rm, writeFileSync: wf } = await import("node:fs");
|
||||
const dir = mk(join(tmpdir(), "hf-lambda-mktar-"));
|
||||
try {
|
||||
wf(join(dir, "index.html"), "<!doctype html><title>test</title>");
|
||||
const tarPath = join(dir, "out.tar.gz");
|
||||
await tar.create({ gzip: true, file: tarPath, cwd: dir }, ["index.html"]);
|
||||
return readFileSync(tarPath);
|
||||
} finally {
|
||||
rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a minimal `.tar.gz` for a tiny planDir containing `plan.json` +
|
||||
* `meta/chunks.json`. Used by renderChunk/assemble tests where the handler
|
||||
* untars but the mock primitive doesn't inspect contents.
|
||||
*/
|
||||
async function makeMinimalPlanTar(): Promise<Buffer> {
|
||||
const tar = await import("tar");
|
||||
const {
|
||||
mkdtempSync: mk,
|
||||
mkdirSync: md,
|
||||
readFileSync: rf,
|
||||
writeFileSync: wf,
|
||||
} = await import("node:fs");
|
||||
const dir = mk(join(tmpdir(), "hf-lambda-test-plan-"));
|
||||
tmpDirs.push(dir);
|
||||
md(join(dir, "meta"), { recursive: true });
|
||||
wf(join(dir, "plan.json"), JSON.stringify({ planHash: "fakehash" }));
|
||||
wf(join(dir, "meta", "chunks.json"), "[]");
|
||||
const tarPath = join(dir, "out.tar.gz");
|
||||
await tar.create({ gzip: true, file: tarPath, cwd: dir }, ["plan.json", "meta"]);
|
||||
return rf(tarPath);
|
||||
}
|
||||
@@ -0,0 +1,566 @@
|
||||
/**
|
||||
* AWS Lambda handler for HyperFrames distributed rendering.
|
||||
*
|
||||
* One Lambda function, three roles. Step Functions dispatches by setting
|
||||
* `event.Action`; the handler unwraps Map-state envelopes, primes the
|
||||
* Lambda environment (Chrome path, ffmpeg path, tmpdir), and forwards to
|
||||
* the matching OSS primitive from `@hyperframes/producer/distributed`.
|
||||
*
|
||||
* Everything heavy — capture, encode, audio mix — happens inside the OSS
|
||||
* primitives. The handler is thin glue: parse event → S3 download → call
|
||||
* primitive → S3 upload → return small JSON result.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
assemble,
|
||||
type AssembleResult,
|
||||
type ChunkResult,
|
||||
type DistributedRenderConfig,
|
||||
plan,
|
||||
type PlanResult,
|
||||
renderChunk,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
import { resolveChromeExecutablePath } from "./chromium.js";
|
||||
import { type DistributedFormat, formatExtension } from "./formatExtension.js";
|
||||
import type {
|
||||
AssembleEvent,
|
||||
AssembleLambdaResult,
|
||||
LambdaAction,
|
||||
LambdaEvent,
|
||||
LambdaResult,
|
||||
PlanEvent,
|
||||
PlanLambdaResult,
|
||||
RenderChunkEvent,
|
||||
RenderChunkLambdaResult,
|
||||
} from "./events.js";
|
||||
import {
|
||||
downloadS3ObjectToFile,
|
||||
parseS3Uri,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToS3,
|
||||
} from "./s3Transport.js";
|
||||
|
||||
/**
|
||||
* Lazily-constructed S3 client. Cached at module scope so warm Lambda
|
||||
* containers reuse the underlying HTTP keep-alive pool across invocations.
|
||||
*/
|
||||
let cachedS3Client: S3Client | null = null;
|
||||
function getS3Client(): S3Client {
|
||||
if (cachedS3Client) return cachedS3Client;
|
||||
cachedS3Client = new S3Client({});
|
||||
return cachedS3Client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional injection points used by the handler's unit tests. Production
|
||||
* callers leave these unset; the real OSS primitives are used. Tests
|
||||
* inject `s3` and `primitives` directly rather than mutating module
|
||||
* state — the dependency-injection seam is sufficient and avoids a
|
||||
* second leak point for cross-test contamination.
|
||||
*/
|
||||
export interface HandlerDeps {
|
||||
s3?: S3Client;
|
||||
primitives?: {
|
||||
plan: typeof plan;
|
||||
renderChunk: typeof renderChunk;
|
||||
assemble: typeof assemble;
|
||||
};
|
||||
/** Override the per-invocation `/tmp` workdir root (defaults to Lambda's `/tmp`). */
|
||||
tmpRoot?: string;
|
||||
/** Skip Chrome resolution (used by handler dispatch tests that mock renderChunk). */
|
||||
skipChromeResolution?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lambda entry. Step Functions sometimes wraps the event in
|
||||
* `{ Payload: ... }` or `{ Input: ... }` depending on the state machine
|
||||
* shape; unwrap until we hit a discriminated event.
|
||||
*/
|
||||
export async function handler(event: LambdaEvent, deps?: HandlerDeps): Promise<LambdaResult> {
|
||||
const unwrapped = unwrapEvent(event);
|
||||
validateEventS3Uris(unwrapped);
|
||||
primeRuntimeEnv();
|
||||
// Single structured boot log line — CloudWatch Logs Insights queries
|
||||
// key off `event=handler_start` to grep for a specific Action / S3 URI
|
||||
// when triaging without attaching a debugger.
|
||||
logEvent({ event: "handler_start", action: unwrapped.Action, input: summarizeEvent(unwrapped) });
|
||||
try {
|
||||
switch (unwrapped.Action) {
|
||||
case "plan":
|
||||
return await handlePlan(unwrapped, deps);
|
||||
case "renderChunk":
|
||||
return await handleRenderChunk(unwrapped, deps);
|
||||
case "assemble":
|
||||
return await handleAssemble(unwrapped, deps);
|
||||
default: {
|
||||
// Compile-time exhaustiveness: a new LambdaAction member trips
|
||||
// the `never` assignment before the runtime error is reachable.
|
||||
const _exhaustive: never = unwrapped;
|
||||
throw new Error(
|
||||
`[handler] unknown Action: ${JSON.stringify(
|
||||
(_exhaustive as { Action?: string }).Action,
|
||||
)}. Expected one of "plan", "renderChunk", "assemble".`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Log before re-throwing so CloudWatch captures the structured
|
||||
// error context alongside Lambda's default stack trace. Otherwise
|
||||
// ops only sees the trace and has to correlate with execution
|
||||
// history to recover the action + input.
|
||||
logEvent({
|
||||
event: "handler_error",
|
||||
action: unwrapped.Action,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
name: err instanceof Error ? err.name : undefined,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk through Step Functions' Map-state and Task-state envelopes until
|
||||
* the discriminated event is found.
|
||||
*/
|
||||
// Step Functions wraps at most `{Payload: {Input: ...}}` in our state
|
||||
// machine; 4 levels is 2× headroom for unusual Map / Wait state
|
||||
// configurations and prevents infinite loops on malformed input.
|
||||
const MAX_ENVELOPE_DEPTH = 4;
|
||||
|
||||
export function unwrapEvent(event: LambdaEvent): PlanEvent | RenderChunkEvent | AssembleEvent {
|
||||
let cursor: LambdaEvent = event;
|
||||
for (let i = 0; i < MAX_ENVELOPE_DEPTH; i++) {
|
||||
if (cursor && typeof cursor === "object") {
|
||||
const obj = cursor as Record<string, unknown>;
|
||||
if (typeof obj.Action === "string" && isLambdaAction(obj.Action)) {
|
||||
return cursor as PlanEvent | RenderChunkEvent | AssembleEvent;
|
||||
}
|
||||
if ("Payload" in obj) {
|
||||
cursor = obj.Payload as LambdaEvent;
|
||||
continue;
|
||||
}
|
||||
if ("Input" in obj) {
|
||||
cursor = obj.Input as LambdaEvent;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
throw new Error(
|
||||
`[handler] event has no recognised Action; unwrapped ${MAX_ENVELOPE_DEPTH} levels of Payload/Input without finding one.`,
|
||||
);
|
||||
}
|
||||
|
||||
function isLambdaAction(value: string): value is LambdaAction {
|
||||
return value === "plan" || value === "renderChunk" || value === "assemble";
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a single JSON line to stdout. CloudWatch ingests each line as a
|
||||
* structured event; Logs Insights queries can `filter event="..."` and
|
||||
* project specific fields. We write to stdout (not stderr) because
|
||||
* Lambda's default destination for both is the same log group, and
|
||||
* Logs Insights' INFO/ERROR level parser keys off the JSON `level`
|
||||
* field, not the stream.
|
||||
*/
|
||||
function logEvent(payload: Record<string, unknown>): void {
|
||||
console.log(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact, non-PII summary of a Lambda event for logging. The full
|
||||
* event payload can include the entire project config; we only emit
|
||||
* the routable fields (S3 URIs, chunk index, format) needed to triage
|
||||
* a failure from CloudWatch.
|
||||
*/
|
||||
function summarizeEvent(
|
||||
event: PlanEvent | RenderChunkEvent | AssembleEvent,
|
||||
): Record<string, unknown> {
|
||||
switch (event.Action) {
|
||||
case "plan":
|
||||
return {
|
||||
projectS3Uri: event.ProjectS3Uri,
|
||||
planOutputS3Prefix: event.PlanOutputS3Prefix,
|
||||
format: event.Config.format,
|
||||
fps: event.Config.fps,
|
||||
};
|
||||
case "renderChunk":
|
||||
return {
|
||||
planS3Uri: event.PlanS3Uri,
|
||||
chunkIndex: event.ChunkIndex,
|
||||
format: event.Format,
|
||||
};
|
||||
case "assemble":
|
||||
return {
|
||||
planS3Uri: event.PlanS3Uri,
|
||||
chunkCount: event.ChunkS3Uris.length,
|
||||
hasAudio: event.AudioS3Uri !== null,
|
||||
outputS3Uri: event.OutputS3Uri,
|
||||
format: event.Format,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lambda sets `TMPDIR` to `/tmp` already, but the bundled binaries (Chrome
|
||||
* + ffmpeg) live alongside the handler at `/var/task/bin/`. Add that to
|
||||
* PATH the first time the handler runs so spawn("ffmpeg", …) inside the
|
||||
* OSS primitives resolves to the bundled binary.
|
||||
*/
|
||||
let runtimeEnvPrimed = false;
|
||||
function primeRuntimeEnv(): void {
|
||||
if (runtimeEnvPrimed) return;
|
||||
runtimeEnvPrimed = true;
|
||||
const taskRoot = process.env.LAMBDA_TASK_ROOT ?? "/var/task";
|
||||
const bin = join(taskRoot, "bin");
|
||||
if (existsSync(bin)) {
|
||||
process.env.PATH = `${bin}:${process.env.PATH ?? ""}`;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Plan ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function handlePlan(event: PlanEvent, deps?: HandlerDeps): Promise<PlanLambdaResult> {
|
||||
const started = Date.now();
|
||||
const s3 = deps?.s3 ?? getS3Client();
|
||||
const primitive = deps?.primitives?.plan ?? plan;
|
||||
|
||||
// The producer's probe stage launches Chromium whenever the composition
|
||||
// needs a runtime duration probe or has unresolved sub-compositions, so
|
||||
// plan has to resolve Chrome the same way renderChunk does. Without this
|
||||
// the probe throws "An `executablePath` or `channel` must be specified
|
||||
// for `puppeteer-core`" the moment runProbeStage calls puppeteer.launch.
|
||||
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
||||
const chromePath = await resolveChromeExecutablePath();
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
||||
}
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-plan-"));
|
||||
// We use `.tar.gz` (not `.zip`) as the project archive's on-the-wire
|
||||
// format because Lambda's Amazon Linux base image ships GNU `tar` but
|
||||
// not `unzip` in `/usr/bin`. The smoke script + future CLI both
|
||||
// produce tar.gz uploads.
|
||||
const projectArchive = join(work, "project.tar.gz");
|
||||
const projectDir = join(work, "project");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadS3ObjectToFile(s3, event.ProjectS3Uri, projectArchive);
|
||||
await untarDirectory(projectArchive, projectDir);
|
||||
|
||||
const config: DistributedRenderConfig = {
|
||||
...event.Config,
|
||||
};
|
||||
const result: PlanResult = await primitive(projectDir, config, planDir);
|
||||
|
||||
// Upload the planDir as a single tarball. Step Functions cannot pass
|
||||
// a directory-shaped artifact between states; we serialize and rely on
|
||||
// the consumer (renderChunk / assemble) to untar. Audio is co-located
|
||||
// alongside the plan so RenderChunk doesn't have to pull the whole
|
||||
// plan tarball when audio isn't relevant to the chunk.
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
await tarDirectory(planDir, planTar);
|
||||
const planTarUri = `${trimTrailingSlash(event.PlanOutputS3Prefix)}/plan.tar.gz`;
|
||||
const audioPath = join(planDir, "audio.aac");
|
||||
const hasAudio = existsSync(audioPath) && statSync(audioPath).size > 0;
|
||||
const audioUri = hasAudio ? `${trimTrailingSlash(event.PlanOutputS3Prefix)}/audio.aac` : null;
|
||||
// Plan and audio are independent S3 PUTs; run them in parallel so
|
||||
// the response returns as soon as the slower of the two completes.
|
||||
await Promise.all([
|
||||
uploadFileToS3(s3, planTar, planTarUri, "application/gzip"),
|
||||
hasAudio && audioUri ? uploadFileToS3(s3, audioPath, audioUri, "audio/aac") : null,
|
||||
]);
|
||||
|
||||
return {
|
||||
Action: "plan",
|
||||
PlanS3Uri: planTarUri,
|
||||
PlanHash: result.planHash,
|
||||
ChunkCount: result.chunkCount,
|
||||
TotalFrames: result.totalFrames,
|
||||
Fps: result.fps,
|
||||
Width: result.width,
|
||||
Height: result.height,
|
||||
Format: result.format,
|
||||
HasAudio: audioUri !== null,
|
||||
AudioS3Uri: audioUri,
|
||||
FfmpegVersion: result.ffmpegVersion,
|
||||
ProducerVersion: result.producerVersion,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
// ── RenderChunk ─────────────────────────────────────────────────────────────
|
||||
|
||||
async function handleRenderChunk(
|
||||
event: RenderChunkEvent,
|
||||
deps?: HandlerDeps,
|
||||
): Promise<RenderChunkLambdaResult> {
|
||||
const started = Date.now();
|
||||
const s3 = deps?.s3 ?? getS3Client();
|
||||
const primitive = deps?.primitives?.renderChunk ?? renderChunk;
|
||||
|
||||
// Sparticuz decompresses Chromium into /tmp on first call; warm starts
|
||||
// skip the work (path already cached). Guard the env-var mutation too so
|
||||
// a caller-supplied PRODUCER_HEADLESS_SHELL_PATH (e.g. the SAM-local
|
||||
// RIE smoke) wins over the auto-resolution.
|
||||
if (!deps?.skipChromeResolution && !process.env.PRODUCER_HEADLESS_SHELL_PATH) {
|
||||
const chromePath = await resolveChromeExecutablePath();
|
||||
// The OSS engine resolves Chrome via `PRODUCER_HEADLESS_SHELL_PATH`
|
||||
// first (see `browserManager.resolveHeadlessShellPath`); set it before
|
||||
// invoking the primitive so launch picks up the bundled binary.
|
||||
process.env.PRODUCER_HEADLESS_SHELL_PATH = chromePath;
|
||||
}
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-chunk-"));
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
||||
await untarDirectory(planTar, planDir);
|
||||
|
||||
// Verify the plan's hash matches what Step Functions told us to render.
|
||||
// The producer's renderChunk re-checks internally (defense-in-depth),
|
||||
// but doing it here at the handler boundary lets us fail before paying
|
||||
// the Chrome-launch + render cost on a misrouted chunk. Throws a
|
||||
// typed PLAN_HASH_MISMATCH that Step Functions can route as
|
||||
// non-retryable.
|
||||
verifyPlanHash(planDir, event.PlanHash);
|
||||
|
||||
const chunkOutputBase = join(
|
||||
work,
|
||||
event.Format === "png-sequence"
|
||||
? `chunk-${pad(event.ChunkIndex)}`
|
||||
: `chunk-${pad(event.ChunkIndex)}${formatExtension(event.Format)}`,
|
||||
);
|
||||
|
||||
const result: ChunkResult = await primitive(planDir, event.ChunkIndex, chunkOutputBase);
|
||||
|
||||
const chunkUri = await uploadChunkOutput(
|
||||
s3,
|
||||
result,
|
||||
event.ChunkOutputS3Prefix,
|
||||
event.ChunkIndex,
|
||||
);
|
||||
|
||||
return {
|
||||
Action: "renderChunk",
|
||||
ChunkS3Uri: chunkUri,
|
||||
ChunkIndex: event.ChunkIndex,
|
||||
Sha256: result.sha256,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadChunkOutput(
|
||||
s3: S3Client,
|
||||
result: ChunkResult,
|
||||
prefix: string,
|
||||
chunkIndex: number,
|
||||
): Promise<string> {
|
||||
const trimmed = trimTrailingSlash(prefix);
|
||||
if (result.outputKind === "file") {
|
||||
const ext = result.outputPath.slice(result.outputPath.lastIndexOf("."));
|
||||
const uri = `${trimmed}/chunks/${pad(chunkIndex)}${ext}`;
|
||||
await uploadFileToS3(s3, result.outputPath, uri);
|
||||
return uri;
|
||||
}
|
||||
// frame-dir: upload as a tarball so a single S3 object represents the chunk.
|
||||
// Assemble's png-sequence path expects a directory per chunk; it untars on
|
||||
// its end.
|
||||
const tarball = `${result.outputPath}.tar.gz`;
|
||||
await tarDirectory(result.outputPath, tarball);
|
||||
const uri = `${trimmed}/chunks/${pad(chunkIndex)}.tar.gz`;
|
||||
await uploadFileToS3(s3, tarball, uri, "application/gzip");
|
||||
return uri;
|
||||
}
|
||||
|
||||
// ── Assemble ────────────────────────────────────────────────────────────────
|
||||
|
||||
async function handleAssemble(
|
||||
event: AssembleEvent,
|
||||
deps?: HandlerDeps,
|
||||
): Promise<AssembleLambdaResult> {
|
||||
const started = Date.now();
|
||||
const s3 = deps?.s3 ?? getS3Client();
|
||||
const primitive = deps?.primitives?.assemble ?? assemble;
|
||||
|
||||
const work = mkdtempSync(join(deps?.tmpRoot ?? tmpdir(), "hf-lambda-assemble-"));
|
||||
const planTar = join(work, "plan.tar.gz");
|
||||
const planDir = join(work, "plan");
|
||||
|
||||
try {
|
||||
await downloadS3ObjectToFile(s3, event.PlanS3Uri, planTar);
|
||||
await untarDirectory(planTar, planDir);
|
||||
|
||||
const chunkPaths = await downloadChunkObjects(s3, event.ChunkS3Uris, work, event.Format);
|
||||
|
||||
let audioPath: string | null = null;
|
||||
if (event.AudioS3Uri) {
|
||||
audioPath = join(planDir, "audio.aac");
|
||||
await downloadS3ObjectToFile(s3, event.AudioS3Uri, audioPath);
|
||||
}
|
||||
|
||||
const finalOutput =
|
||||
event.Format === "png-sequence"
|
||||
? join(work, "output-frames")
|
||||
: join(work, `output${formatExtension(event.Format)}`);
|
||||
|
||||
const result: AssembleResult = await primitive(planDir, chunkPaths, audioPath, finalOutput, {
|
||||
cfr: event.Cfr === true,
|
||||
});
|
||||
|
||||
if (event.Format === "png-sequence") {
|
||||
const tarball = `${finalOutput}.tar.gz`;
|
||||
await tarDirectory(finalOutput, tarball);
|
||||
await uploadFileToS3(s3, tarball, event.OutputS3Uri, "application/gzip");
|
||||
} else {
|
||||
await uploadFileToS3(s3, finalOutput, event.OutputS3Uri);
|
||||
}
|
||||
|
||||
return {
|
||||
Action: "assemble",
|
||||
OutputS3Uri: event.OutputS3Uri,
|
||||
FramesEncoded: result.framesEncoded,
|
||||
FileSize: result.fileSize,
|
||||
DurationMs: Date.now() - started,
|
||||
};
|
||||
} finally {
|
||||
cleanupDir(work);
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadChunkObjects(
|
||||
s3: S3Client,
|
||||
uris: string[],
|
||||
workDir: string,
|
||||
format: DistributedFormat,
|
||||
): Promise<string[]> {
|
||||
const chunksDir = join(workDir, "chunks");
|
||||
mkdirSync(chunksDir, { recursive: true });
|
||||
// Each chunk is an independent S3 GET (+ untar for png-sequence). Run
|
||||
// them in parallel — assemble's wall-clock is otherwise dominated by
|
||||
// `Σ chunk-download-ms` instead of `max(chunk-download-ms)`. Preserve
|
||||
// the input order by writing into a pre-sized array rather than
|
||||
// pushing as each task settles.
|
||||
const local: string[] = new Array<string>(uris.length);
|
||||
await Promise.all(
|
||||
uris.map(async (uri, i) => {
|
||||
if (!uri) {
|
||||
throw new Error(`[handler] chunk URI at index ${i} is empty`);
|
||||
}
|
||||
const { key } = parseS3Uri(uri);
|
||||
const localPath = join(chunksDir, basename(key));
|
||||
await downloadS3ObjectToFile(s3, uri, localPath);
|
||||
if (format === "png-sequence") {
|
||||
const dirPath = join(chunksDir, `frames-${pad(i)}`);
|
||||
await untarDirectory(localPath, dirPath);
|
||||
local[i] = dirPath;
|
||||
} else {
|
||||
local[i] = localPath;
|
||||
}
|
||||
}),
|
||||
);
|
||||
return local;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Collect every S3 URI that the handler will touch for a given event. */
|
||||
function getEventS3Uris(event: PlanEvent | RenderChunkEvent | AssembleEvent): string[] {
|
||||
switch (event.Action) {
|
||||
case "plan":
|
||||
return [event.ProjectS3Uri, event.PlanOutputS3Prefix];
|
||||
case "renderChunk":
|
||||
return [event.PlanS3Uri, event.ChunkOutputS3Prefix];
|
||||
case "assemble":
|
||||
return [event.PlanS3Uri, ...event.ChunkS3Uris, event.OutputS3Uri, event.AudioS3Uri].filter(
|
||||
(u): u is string => u != null,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify every S3 URI in the event resolves to the configured render bucket.
|
||||
* Throws `S3_URI_NOT_ALLOWED` (non-retryable) when a URI targets a different
|
||||
* bucket, preventing event injection from reading or writing arbitrary S3 data.
|
||||
*
|
||||
* Skipped when `HYPERFRAMES_RENDER_BUCKET` is unset so existing deployments
|
||||
* without the env var continue to work.
|
||||
*/
|
||||
function validateEventS3Uris(event: PlanEvent | RenderChunkEvent | AssembleEvent): void {
|
||||
const allowedBucket = process.env.HYPERFRAMES_RENDER_BUCKET?.trim();
|
||||
if (!allowedBucket) return;
|
||||
|
||||
for (const uri of getEventS3Uris(event)) {
|
||||
const { bucket } = parseS3Uri(uri);
|
||||
if (bucket !== allowedBucket) {
|
||||
const err = new Error(
|
||||
`[handler] S3_URI_NOT_ALLOWED: URI ${JSON.stringify(uri)} targets bucket "${bucket}" but only "${allowedBucket}" is permitted`,
|
||||
);
|
||||
err.name = "S3_URI_NOT_ALLOWED";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function pad(n: number): string {
|
||||
return n.toString().padStart(4, "0");
|
||||
}
|
||||
|
||||
function trimTrailingSlash(prefix: string): string {
|
||||
return prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
||||
}
|
||||
|
||||
function cleanupDir(dir: string): void {
|
||||
try {
|
||||
// Lambda warm starts can reuse `/tmp` across invocations; clean up
|
||||
// aggressively so we don't leak a chunk-sized footprint between renders.
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort — leak is preferable to crashing on success path.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the untarred planDir's `plan.json` and assert its `planHash`
|
||||
* matches what the Step Functions event claims. Throws on mismatch with
|
||||
* a typed `PLAN_HASH_MISMATCH` error name so the state machine's typed
|
||||
* non-retryable list routes it correctly.
|
||||
*
|
||||
* This is defense-in-depth — the producer's `renderChunk` does the same
|
||||
* check internally — but performing it here lets us fail before paying
|
||||
* the Chrome-launch + per-frame capture cost on a misrouted chunk.
|
||||
*/
|
||||
function verifyPlanHash(planDir: string, expected: string): void {
|
||||
const planJsonPath = join(planDir, "plan.json");
|
||||
let parsed: { planHash?: unknown };
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(planJsonPath, "utf-8")) as { planHash?: unknown };
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const error = new Error(`PLAN_HASH_MISMATCH: failed to read ${planJsonPath}: ${msg}`);
|
||||
error.name = "PLAN_HASH_MISMATCH";
|
||||
throw error;
|
||||
}
|
||||
const actual = parsed.planHash;
|
||||
if (typeof actual !== "string" || actual !== expected) {
|
||||
const error = new Error(
|
||||
`PLAN_HASH_MISMATCH: event PlanHash=${expected} did not match plan.json planHash=${String(actual)}`,
|
||||
);
|
||||
error.name = "PLAN_HASH_MISMATCH";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* `@hyperframes/aws-lambda` — Lambda adapter for the HyperFrames
|
||||
* distributed render pipeline.
|
||||
*
|
||||
* Two surfaces, one package:
|
||||
*
|
||||
* - **Server-side handler.** `handler`, the Step Functions event/result
|
||||
* types, Chrome resolution, and S3 transport. These power the Lambda
|
||||
* function bundled into `dist/handler.zip`.
|
||||
* - **Client-side SDK.** `renderToLambda`, `getRenderProgress`,
|
||||
* `deploySite`, `validateDistributedRenderConfig`, and `computeRenderCost`.
|
||||
* Adopters call these from their Node process (CI scripts, CLIs, CDK
|
||||
* pipelines) to drive a deployed stack without writing AWS-SDK
|
||||
* boilerplate.
|
||||
*
|
||||
* The CDK L2 construct lives at the `./cdk` subpath export so SDK-only
|
||||
* consumers don't pull `aws-cdk-lib` into their runtime graph:
|
||||
*
|
||||
* import { HyperframesRenderStack } from "@hyperframes/aws-lambda/cdk";
|
||||
*
|
||||
* The package is NOT a dependency of `@hyperframes/producer`; consumers
|
||||
* install it separately.
|
||||
*/
|
||||
|
||||
export { handler, type HandlerDeps, unwrapEvent } from "./handler.js";
|
||||
export {
|
||||
type AssembleEvent,
|
||||
type AssembleLambdaResult,
|
||||
type LambdaAction,
|
||||
type LambdaEvent,
|
||||
type LambdaResult,
|
||||
type PlanEvent,
|
||||
type PlanLambdaResult,
|
||||
type RenderChunkEvent,
|
||||
type RenderChunkLambdaResult,
|
||||
type SerializableDistributedRenderConfig,
|
||||
} from "./events.js";
|
||||
// `_setSparticuzChromiumForTests` is intentionally NOT re-exported from
|
||||
// the package barrel — it's a test-only DI seam. Test files import it
|
||||
// directly from `./chromium.js`.
|
||||
export {
|
||||
ChromeBinaryUnavailableError,
|
||||
type ChromeSource,
|
||||
resolveChromeArgs,
|
||||
resolveChromeExecutablePath,
|
||||
resolveChromeSource,
|
||||
} from "./chromium.js";
|
||||
export {
|
||||
downloadS3ObjectToFile,
|
||||
formatS3Uri,
|
||||
parseS3Uri,
|
||||
type S3Location,
|
||||
tarDirectory,
|
||||
untarDirectory,
|
||||
uploadFileToS3,
|
||||
} from "./s3Transport.js";
|
||||
|
||||
// ── Client-side SDK ─────────────────────────────────────────────────────────
|
||||
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./sdk/deploySite.js";
|
||||
export {
|
||||
renderToLambda,
|
||||
type RenderHandle,
|
||||
type RenderToLambdaOptions,
|
||||
} from "./sdk/renderToLambda.js";
|
||||
export {
|
||||
getRenderProgress,
|
||||
type GetRenderProgressOptions,
|
||||
type RenderError,
|
||||
type RenderProgress,
|
||||
type RenderStatus,
|
||||
} from "./sdk/getRenderProgress.js";
|
||||
export {
|
||||
type BilledLambdaInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./sdk/costAccounting.js";
|
||||
export { InvalidConfigError, validateDistributedRenderConfig } from "./sdk/validateConfig.js";
|
||||
// The CDK construct is exported separately via the `./cdk` subpath so
|
||||
// SDK-only consumers don't pay the `aws-cdk-lib` import cost.
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Unit tests for the S3 URI parser + tar helpers. Real S3 network calls
|
||||
* are covered by the dispatch tests in `handler.test.ts` via a fake
|
||||
* S3Client; here we pin the lower-level helpers.
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { formatS3Uri, parseS3Uri, tarDirectory, untarDirectory } from "./s3Transport.js";
|
||||
|
||||
let scratchRoot: string;
|
||||
|
||||
beforeAll(() => {
|
||||
scratchRoot = mkdtempSync(join(tmpdir(), "hf-s3transport-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(scratchRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("parseS3Uri", () => {
|
||||
it("parses a simple bucket+key URI", () => {
|
||||
expect(parseS3Uri("s3://my-bucket/path/to/object.zip")).toEqual({
|
||||
bucket: "my-bucket",
|
||||
key: "path/to/object.zip",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves nested keys", () => {
|
||||
expect(parseS3Uri("s3://b/a/b/c/d.mp4").key).toBe("a/b/c/d.mp4");
|
||||
});
|
||||
|
||||
it("throws on non-s3 schemes", () => {
|
||||
expect(() => parseS3Uri("https://example.com/x")).toThrow(/expected s3:\/\//);
|
||||
});
|
||||
|
||||
it("throws on missing key", () => {
|
||||
expect(() => parseS3Uri("s3://bucket-only")).toThrow(/missing key/);
|
||||
});
|
||||
|
||||
it("throws on empty bucket", () => {
|
||||
expect(() => parseS3Uri("s3:///somekey")).toThrow(/empty bucket or key/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatS3Uri", () => {
|
||||
it("round-trips with parseS3Uri", () => {
|
||||
const uri = "s3://my-bucket/path/to/object.zip";
|
||||
expect(formatS3Uri(parseS3Uri(uri))).toBe(uri);
|
||||
});
|
||||
});
|
||||
|
||||
describe("tar round-trip", () => {
|
||||
it("tars a directory and untars to identical contents", async () => {
|
||||
const sourceDir = join(scratchRoot, "src");
|
||||
const destDir = join(scratchRoot, "dest");
|
||||
const tarPath = join(scratchRoot, "out.tar.gz");
|
||||
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(join(sourceDir, "nested"), { recursive: true });
|
||||
writeFileSync(join(sourceDir, "top.txt"), "hello-top");
|
||||
writeFileSync(join(sourceDir, "nested", "inner.txt"), "hello-inner");
|
||||
|
||||
await tarDirectory(sourceDir, tarPath);
|
||||
await untarDirectory(tarPath, destDir);
|
||||
|
||||
expect(readFileSync(join(destDir, "top.txt"), "utf-8")).toBe("hello-top");
|
||||
expect(readFileSync(join(destDir, "nested", "inner.txt"), "utf-8")).toBe("hello-inner");
|
||||
});
|
||||
|
||||
it("wipes the destination before extracting", async () => {
|
||||
const sourceDir = join(scratchRoot, "src2");
|
||||
const destDir = join(scratchRoot, "dest2");
|
||||
const tarPath = join(scratchRoot, "out2.tar.gz");
|
||||
|
||||
const { mkdirSync } = await import("node:fs");
|
||||
mkdirSync(sourceDir, { recursive: true });
|
||||
writeFileSync(join(sourceDir, "fresh.txt"), "new");
|
||||
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
writeFileSync(join(destDir, "stale.txt"), "leftover");
|
||||
|
||||
await tarDirectory(sourceDir, tarPath);
|
||||
await untarDirectory(tarPath, destDir);
|
||||
|
||||
// Stale file should be gone; fresh file should be present.
|
||||
expect(readFileSync(join(destDir, "fresh.txt"), "utf-8")).toBe("new");
|
||||
const { existsSync } = await import("node:fs");
|
||||
expect(existsSync(join(destDir, "stale.txt"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Thin S3 transport for the Lambda handler.
|
||||
*
|
||||
* The OSS distributed primitives are pure functions over local file paths;
|
||||
* the Lambda handler bridges S3 ↔ Lambda's `/tmp` filesystem on each
|
||||
* invocation. Functions here are intentionally narrow: parse a URI, download
|
||||
* an object to a local path, upload a path/directory, tar-extract a planDir,
|
||||
* tar-pack a planDir back out.
|
||||
*
|
||||
* Tar (not zip) for planDir transit:
|
||||
* - planDirs contain symlinks (extract stage materializes them but the
|
||||
* compiled/ subtree may include linked assets); tar preserves them, zip
|
||||
* does not.
|
||||
* - We use the `tar` npm package (pure JS over `node:zlib`) — AWS
|
||||
* Lambda's `nodejs:22` base image ships neither `tar` nor `unzip` in
|
||||
* `/usr/bin`, so a system-binary tar would ENOENT in the actual
|
||||
* deployment.
|
||||
*/
|
||||
|
||||
import {
|
||||
createReadStream,
|
||||
createWriteStream,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { GetObjectCommand, PutObjectCommand, type S3Client } from "@aws-sdk/client-s3";
|
||||
import * as tar from "tar";
|
||||
|
||||
/** Parsed `s3://bucket/key` URI. */
|
||||
export interface S3Location {
|
||||
bucket: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
/** Parse `s3://bucket/key/path` → `{ bucket, key }`. Throws on malformed input. */
|
||||
export function parseS3Uri(uri: string): S3Location {
|
||||
if (!uri.startsWith("s3://")) {
|
||||
throw new Error(`[s3Transport] expected s3:// URI, got: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
const rest = uri.slice("s3://".length);
|
||||
const slash = rest.indexOf("/");
|
||||
if (slash === -1) {
|
||||
throw new Error(`[s3Transport] missing key in s3 URI: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
const bucket = rest.slice(0, slash);
|
||||
const key = rest.slice(slash + 1);
|
||||
if (!bucket || !key) {
|
||||
throw new Error(`[s3Transport] empty bucket or key in s3 URI: ${JSON.stringify(uri)}`);
|
||||
}
|
||||
return { bucket, key };
|
||||
}
|
||||
|
||||
/** Build `s3://bucket/key` from a location. */
|
||||
export function formatS3Uri(loc: S3Location): string {
|
||||
return `s3://${loc.bucket}/${loc.key}`;
|
||||
}
|
||||
|
||||
/** Stream an S3 object to a local file path. Throws if the body is missing. */
|
||||
export async function downloadS3ObjectToFile(
|
||||
client: S3Client,
|
||||
uri: string,
|
||||
destPath: string,
|
||||
): Promise<void> {
|
||||
const { bucket, key } = parseS3Uri(uri);
|
||||
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
||||
const body = response.Body as NodeJS.ReadableStream | undefined;
|
||||
if (!body) {
|
||||
throw new Error(`[s3Transport] s3 GetObject returned empty body for ${uri}`);
|
||||
}
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
await pipeline(body, createWriteStream(destPath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a local file's contents to an S3 URI using a streaming
|
||||
* `PutObjectCommand`. PutObject's 5 GB cap comfortably exceeds the
|
||||
* distributed pipeline's 2 GB planDir limit and the typical
|
||||
* chunk size (≤ 200 MB), so a single PUT works for every artifact this
|
||||
* adapter handles.
|
||||
*/
|
||||
export async function uploadFileToS3(
|
||||
client: S3Client,
|
||||
localPath: string,
|
||||
uri: string,
|
||||
contentType?: string,
|
||||
): Promise<void> {
|
||||
if (!existsSync(localPath)) {
|
||||
throw new Error(`[s3Transport] upload source missing: ${localPath}`);
|
||||
}
|
||||
const { bucket, key } = parseS3Uri(uri);
|
||||
const size = statSync(localPath).size;
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: createReadStream(localPath),
|
||||
ContentType: contentType,
|
||||
ContentLength: size,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack a directory into a `.tar.gz` at `destTarball`. Uses the `tar` npm
|
||||
* package (pure JS over `node:zlib`) rather than spawning a system tar
|
||||
* binary — the AWS Lambda Node 22 base image ships a minimal set of
|
||||
* userland tools and does NOT include `tar` in `/usr/bin`.
|
||||
*/
|
||||
export async function tarDirectory(sourceDir: string, destTarball: string): Promise<void> {
|
||||
if (!existsSync(sourceDir) || !statSync(sourceDir).isDirectory()) {
|
||||
throw new Error(`[s3Transport] tar source must be an existing directory: ${sourceDir}`);
|
||||
}
|
||||
mkdirSync(dirname(destTarball), { recursive: true });
|
||||
await tar.create({ gzip: true, file: destTarball, cwd: sourceDir }, ["."]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a `.tar.gz` produced by {@link tarDirectory} into `destDir`.
|
||||
* The directory is created (or cleared) before extraction so a retried
|
||||
* invocation doesn't observe stale files from a prior run on the same
|
||||
* warm Lambda container.
|
||||
*/
|
||||
export async function untarDirectory(tarballPath: string, destDir: string): Promise<void> {
|
||||
if (!existsSync(tarballPath)) {
|
||||
throw new Error(`[s3Transport] tarball missing: ${tarballPath}`);
|
||||
}
|
||||
// Wipe target so the warm container's prior planDir doesn't bleed into
|
||||
// the new invocation. Lambda re-uses /tmp across invocations on the same
|
||||
// container.
|
||||
if (existsSync(destDir)) {
|
||||
rmSync(destDir, { recursive: true, force: true });
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
await tar.extract({ file: tarballPath, cwd: destDir });
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* Shared `FakeS3` for the SDK unit tests.
|
||||
*
|
||||
* Multiple SDK tests need to assert what `deploySite` / `renderToLambda`
|
||||
* sent to S3. Each fake here records `HeadObjectCommand` + `PutObjectCommand`
|
||||
* activity and drains the body stream so the lazy `createReadStream`
|
||||
* inside `uploadFileToS3` doesn't try to open the workdir tarball after
|
||||
* `deploySite`'s `finally` block has rmSync'd it.
|
||||
*/
|
||||
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
export interface FakeS3Op {
|
||||
kind: "head" | "put";
|
||||
bucket: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export class FakeS3 {
|
||||
ops: FakeS3Op[] = [];
|
||||
existing = new Set<string>();
|
||||
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const cmdName = (command as { constructor: { name: string } }).constructor.name;
|
||||
const input = (command as { input: { Bucket: string; Key: string } }).input;
|
||||
if (cmdName === "HeadObjectCommand") {
|
||||
this.ops.push({ kind: "head", bucket: input.Bucket, key: input.Key });
|
||||
if (this.existing.has(`${input.Bucket}/${input.Key}`)) {
|
||||
return { ContentLength: 1, LastModified: new Date("2026-05-16T00:00:00Z") };
|
||||
}
|
||||
const err = new Error("Not Found") as Error & {
|
||||
$metadata: { httpStatusCode: number };
|
||||
name: string;
|
||||
};
|
||||
err.name = "NotFound";
|
||||
err.$metadata = { httpStatusCode: 404 };
|
||||
throw err;
|
||||
}
|
||||
if (cmdName === "PutObjectCommand") {
|
||||
this.ops.push({ kind: "put", bucket: input.Bucket, key: input.Key });
|
||||
this.existing.add(`${input.Bucket}/${input.Key}`);
|
||||
await drainBody((command as { input: { Body: NodeJS.ReadableStream | Buffer } }).input.Body);
|
||||
return {};
|
||||
}
|
||||
throw new Error(`FakeS3: unexpected command ${cmdName}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Convenience cast so `deploySite({ s3: makeFakeS3() })` reads cleanly. */
|
||||
export function asS3Client(fake: FakeS3): S3Client {
|
||||
return fake as unknown as S3Client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume the body stream to completion. Without this, the lazy
|
||||
* `createReadStream` opened inside `uploadFileToS3` would attempt to
|
||||
* read the workdir tarball after deploySite's `finally` block removed
|
||||
* the workdir — surfacing as an "Unhandled error between tests"
|
||||
* teardown noise that bun's runner attributes to the next test.
|
||||
*/
|
||||
async function drainBody(body: NodeJS.ReadableStream | Buffer): Promise<void> {
|
||||
if (Buffer.isBuffer(body)) return;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
body.on("data", () => {});
|
||||
body.on("end", () => resolve());
|
||||
body.on("close", () => resolve());
|
||||
body.on("error", reject);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { type BilledLambdaInvocation, computeRenderCost } from "./costAccounting.js";
|
||||
|
||||
describe("computeRenderCost", () => {
|
||||
it("returns zero when nothing is billed", () => {
|
||||
const result = computeRenderCost([], 0);
|
||||
expect(result.accruedSoFarUsd).toBe(0);
|
||||
expect(result.displayCost).toBe("$0.0000");
|
||||
expect(result.breakdown.estimated).toBe(false);
|
||||
});
|
||||
|
||||
it("computes Lambda GB-seconds × USD per GB-s", () => {
|
||||
// 10 GiB × 6 s = 60 GB-s × $0.0000166667 = $0.001.
|
||||
const invs: BilledLambdaInvocation[] = [
|
||||
{ billedDurationMs: 6_000, memorySizeMb: 10_240, estimated: false },
|
||||
];
|
||||
const result = computeRenderCost(invs, 0);
|
||||
expect(result.breakdown.lambdaUsd).toBeCloseTo(0.001, 4);
|
||||
expect(result.breakdown.stepFunctionsUsd).toBe(0);
|
||||
});
|
||||
|
||||
it("sums multiple Lambda invocations", () => {
|
||||
const invs: BilledLambdaInvocation[] = [
|
||||
{ billedDurationMs: 1_000, memorySizeMb: 1_024, estimated: false }, // 1 GB-s
|
||||
{ billedDurationMs: 2_000, memorySizeMb: 2_048, estimated: false }, // 4 GB-s
|
||||
];
|
||||
const result = computeRenderCost(invs, 0);
|
||||
// (1 + 4) × $0.0000166667 ≈ $0.0000833335 → rounds to $0.0001 at 4 dp.
|
||||
expect(result.breakdown.lambdaUsd).toBe(0.0001);
|
||||
});
|
||||
|
||||
it("adds Step Functions transition costs", () => {
|
||||
const result = computeRenderCost([], 200);
|
||||
expect(result.breakdown.stepFunctionsUsd).toBeCloseTo(200 * 0.000025, 6);
|
||||
expect(result.breakdown.stepFunctionsUsd).toBeCloseTo(0.005, 4);
|
||||
});
|
||||
|
||||
it("flags estimated=true when any invocation was estimated", () => {
|
||||
const result = computeRenderCost(
|
||||
[
|
||||
{ billedDurationMs: 1_000, memorySizeMb: 1_024, estimated: true },
|
||||
{ billedDurationMs: 1_000, memorySizeMb: 1_024, estimated: false },
|
||||
],
|
||||
10,
|
||||
);
|
||||
expect(result.breakdown.estimated).toBe(true);
|
||||
});
|
||||
|
||||
it("formats USD to four decimal places", () => {
|
||||
const result = computeRenderCost([], 1);
|
||||
expect(result.displayCost).toBe("$0.0000");
|
||||
const result2 = computeRenderCost([], 4_000);
|
||||
expect(result2.displayCost).toBe("$0.1000");
|
||||
});
|
||||
|
||||
it("does not include S3 in the breakdown", () => {
|
||||
const result = computeRenderCost([], 0);
|
||||
expect(result.breakdown.s3Estimate).toBe("not-included");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Per-render cost accounting for {@link getRenderProgress}.
|
||||
*
|
||||
* AWS bills Lambda by **GB-seconds** (billed-duration × memory-in-GiB)
|
||||
* and Step Functions standard workflows by **state transitions**. Both
|
||||
* inputs are recoverable from the SFN execution history without an
|
||||
* extra CloudWatch query — the history events carry
|
||||
* `billedDurationInMillis` and `memorySizeInMB` on each Lambda
|
||||
* invocation, and the transition count is simply `history.length`
|
||||
* filtered to transition-worthy events.
|
||||
*
|
||||
* The math is documented inline so the constants stay close to the
|
||||
* pricing source they came from. Cost is **best-effort**: AWS pricing
|
||||
* varies by region + commitment plan; we use on-demand `us-east-1`
|
||||
* rates as of 2026-05 and label the result `displayCost` so callers
|
||||
* see the dollar value but downstream automation can also read the
|
||||
* raw number.
|
||||
*/
|
||||
|
||||
/** On-demand Lambda price, us-east-1, x86_64, on-demand: USD per GB-second. */
|
||||
const LAMBDA_USD_PER_GB_SECOND = 0.0000166667;
|
||||
/** Step Functions Standard Workflows, us-east-1: USD per state transition. */
|
||||
const SFN_USD_PER_TRANSITION = 0.000025;
|
||||
|
||||
/** Raw history event subset the cost calc cares about. Caller filters from `getExecutionHistory`. */
|
||||
export interface BilledLambdaInvocation {
|
||||
/** Millis of Lambda billed duration. Carried on `TaskSucceeded`/`TaskFailed` events. */
|
||||
billedDurationMs: number;
|
||||
/** Memory size in MB the function was configured with at invocation time. */
|
||||
memorySizeMb: number;
|
||||
/** `true` if the event payload did NOT carry a billed duration and we fell back to `Duration` or a constant. */
|
||||
estimated: boolean;
|
||||
}
|
||||
|
||||
/** Result of {@link computeRenderCost}. */
|
||||
export interface RenderCost {
|
||||
/** USD accrued to date. */
|
||||
accruedSoFarUsd: number;
|
||||
/** Human-readable USD string, e.g. `"$0.0214"`. */
|
||||
displayCost: string;
|
||||
breakdown: {
|
||||
lambdaUsd: number;
|
||||
stepFunctionsUsd: number;
|
||||
/** S3 transfer + storage cost varies by tier; we don't try to compute it here. */
|
||||
s3Estimate: "not-included";
|
||||
/** `true` if any Lambda invocation fell back to estimated billing. */
|
||||
estimated: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sum Lambda GB-seconds + SFN transitions into an aggregate USD figure.
|
||||
*
|
||||
* `stateTransitions` is the count of billable state-machine transitions
|
||||
* — every successful state entry transitions once for standard
|
||||
* workflows. Express workflows price differently and are out of scope.
|
||||
*/
|
||||
export function computeRenderCost(
|
||||
lambdaInvocations: BilledLambdaInvocation[],
|
||||
stateTransitions: number,
|
||||
): RenderCost {
|
||||
let lambdaUsd = 0;
|
||||
let anyEstimated = false;
|
||||
for (const inv of lambdaInvocations) {
|
||||
const gbSeconds = (inv.memorySizeMb / 1024) * (inv.billedDurationMs / 1000);
|
||||
lambdaUsd += gbSeconds * LAMBDA_USD_PER_GB_SECOND;
|
||||
if (inv.estimated) anyEstimated = true;
|
||||
}
|
||||
const stepFunctionsUsd = stateTransitions * SFN_USD_PER_TRANSITION;
|
||||
const accruedSoFarUsd = roundUsd(lambdaUsd + stepFunctionsUsd);
|
||||
return {
|
||||
accruedSoFarUsd,
|
||||
displayCost: formatUsd(accruedSoFarUsd),
|
||||
breakdown: {
|
||||
lambdaUsd: roundUsd(lambdaUsd),
|
||||
stepFunctionsUsd: roundUsd(stepFunctionsUsd),
|
||||
s3Estimate: "not-included",
|
||||
estimated: anyEstimated,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function roundUsd(usd: number): number {
|
||||
// Four decimal places — enough resolution for per-chunk granularity on
|
||||
// a 10 GB Lambda. Anything finer is noise vs AWS' own rounding.
|
||||
return Math.round(usd * 10_000) / 10_000;
|
||||
}
|
||||
|
||||
function formatUsd(usd: number): string {
|
||||
return `$${usd.toFixed(4)}`;
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
|
||||
import { deploySite } from "./deploySite.js";
|
||||
|
||||
let projectDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
projectDir = mkdtempSync(join(tmpdir(), "hf-deploy-site-test-"));
|
||||
mkdirSync(join(projectDir, "assets"));
|
||||
writeFileSync(join(projectDir, "index.html"), "<html><body>hi</body></html>");
|
||||
writeFileSync(join(projectDir, "assets", "style.css"), "body { color: red; }");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("deploySite", () => {
|
||||
it("uploads the tarball when no matching object exists", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const result = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
expect(result.uploaded).toBe(true);
|
||||
expect(result.siteId).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(result.bucketName).toBe("test-bucket");
|
||||
expect(result.projectS3Uri).toBe(`s3://test-bucket/sites/${result.siteId}/project.tar.gz`);
|
||||
expect(result.bytes).toBeGreaterThan(0);
|
||||
expect(s3.ops).toEqual([
|
||||
{ kind: "head", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
|
||||
{ kind: "put", bucket: "test-bucket", key: `sites/${result.siteId}/project.tar.gz` },
|
||||
]);
|
||||
});
|
||||
|
||||
it("yields a stable siteId across re-runs of the same tree", async () => {
|
||||
const s3a = new FakeS3();
|
||||
const a = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3a),
|
||||
});
|
||||
const s3b = new FakeS3();
|
||||
const b = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3b),
|
||||
});
|
||||
expect(a.siteId).toBe(b.siteId);
|
||||
});
|
||||
|
||||
it("changes siteId when a file's content changes", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const before = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
writeFileSync(join(projectDir, "index.html"), "<html><body>changed</body></html>");
|
||||
const s3b = new FakeS3();
|
||||
const after = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3b),
|
||||
});
|
||||
expect(after.siteId).not.toBe(before.siteId);
|
||||
});
|
||||
|
||||
it("short-circuits on HEAD 200 (skips PUT)", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const first = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
const second = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
expect(second.uploaded).toBe(false);
|
||||
expect(second.siteId).toBe(first.siteId);
|
||||
// Only one PUT total, plus two HEADs.
|
||||
expect(s3.ops.filter((op) => op.kind === "put")).toHaveLength(1);
|
||||
expect(s3.ops.filter((op) => op.kind === "head")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("honours a caller-supplied siteId", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const result = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
siteId: "release-v1.2.3",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(result.siteId).toBe("release-v1.2.3");
|
||||
expect(result.projectS3Uri).toBe("s3://test-bucket/sites/release-v1.2.3/project.tar.gz");
|
||||
});
|
||||
|
||||
it("propagates non-404 S3 errors", async () => {
|
||||
const errS3 = {
|
||||
async send(_cmd: unknown): Promise<unknown> {
|
||||
const err = new Error("Access Denied") as Error & {
|
||||
$metadata: { httpStatusCode: number };
|
||||
};
|
||||
err.$metadata = { httpStatusCode: 403 };
|
||||
throw err;
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: errS3 as unknown as S3Client,
|
||||
}),
|
||||
).rejects.toThrow(/Access Denied/);
|
||||
});
|
||||
|
||||
it("ignores SKIP_TOP_LEVEL dirs when hashing", async () => {
|
||||
const s3 = new FakeS3();
|
||||
const before = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
mkdirSync(join(projectDir, "node_modules"));
|
||||
writeFileSync(join(projectDir, "node_modules", "junk.bin"), "x".repeat(100));
|
||||
const s3b = new FakeS3();
|
||||
const after = await deploySite({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
s3: asS3Client(s3b),
|
||||
});
|
||||
|
||||
// node_modules contents shouldn't move the hash.
|
||||
expect(after.siteId).toBe(before.siteId);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* `deploySite` — upload a project directory to S3 once per content hash
|
||||
* and return a reusable handle.
|
||||
*
|
||||
* `renderToLambda` calls this implicitly when no `siteHandle` is passed,
|
||||
* but exposing it as a standalone verb lets adopters bundle a project
|
||||
* ahead of time and reuse the handle across many renders without
|
||||
* re-tarring the project tree on every call.
|
||||
*
|
||||
* The handle is **content-addressed**: `siteId` is derived from a SHA-256
|
||||
* over the project files. Two `deploySite` calls on an unchanged tree
|
||||
* produce the same `siteId` and `HeadObject`-short-circuit the upload.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { hashProjectDir } from "@hyperframes/producer/distributed";
|
||||
import { formatS3Uri, tarDirectory, uploadFileToS3 } from "../s3Transport.js";
|
||||
|
||||
/** Options for {@link deploySite}. */
|
||||
export interface DeploySiteOptions {
|
||||
/** Local project directory containing `index.html` (and any composition assets). */
|
||||
projectDir: string;
|
||||
/** S3 bucket the SAM stack / CDK construct provisioned. */
|
||||
bucketName: string;
|
||||
/** AWS region for the S3 client. Defaults to the SDK's default chain (env / config / IMDS). */
|
||||
region?: string;
|
||||
/**
|
||||
* Override the content-addressed site id. Useful when the caller has a
|
||||
* stable external identifier they want to use (e.g. a git SHA); if
|
||||
* unset, the hash of the project tree picks it.
|
||||
*/
|
||||
siteId?: string;
|
||||
/** Injection seam for tests. Production callers leave unset. */
|
||||
s3?: S3Client;
|
||||
}
|
||||
|
||||
/** Stable handle returned by {@link deploySite}. Pass back to {@link renderToLambda}. */
|
||||
export interface SiteHandle {
|
||||
/** Content-addressed (or caller-supplied) identifier; stable across re-uploads of the same tree. */
|
||||
siteId: string;
|
||||
/** Bucket the site landed in. Surfaced separately so callers don't have to re-parse `projectS3Uri`. */
|
||||
bucketName: string;
|
||||
/** Full `s3://bucket/sites/<siteId>/project.tar.gz` URI; pass through to `renderToLambda`. */
|
||||
projectS3Uri: string;
|
||||
/** Tarball size in bytes; useful for "did we actually skip the upload?" assertions. */
|
||||
bytes: number;
|
||||
/** ISO timestamp of the most recent upload OR the existing object the short-circuit found. */
|
||||
uploadedAt: string;
|
||||
/** `false` if the object already existed and we skipped the PUT. */
|
||||
uploaded: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload `projectDir` to `s3://bucketName/sites/<siteId>/project.tar.gz`.
|
||||
*
|
||||
* Short-circuits when an object with the same key already exists in the
|
||||
* bucket — `siteId` derives from the project's content hash, so the same
|
||||
* bytes produce the same key, and re-uploading would be redundant.
|
||||
*/
|
||||
export async function deploySite(opts: DeploySiteOptions): Promise<SiteHandle> {
|
||||
if (!statSync(opts.projectDir).isDirectory()) {
|
||||
throw new Error(`[deploySite] projectDir is not a directory: ${opts.projectDir}`);
|
||||
}
|
||||
|
||||
const siteId = opts.siteId ?? hashProjectDir(opts.projectDir);
|
||||
const key = `sites/${siteId}/project.tar.gz`;
|
||||
const projectS3Uri = formatS3Uri({ bucket: opts.bucketName, key });
|
||||
const s3 = opts.s3 ?? new S3Client({ region: opts.region });
|
||||
|
||||
// HeadObject short-circuit. Adopters re-rendering the same project on
|
||||
// a tight inner loop (CI smoke, demo flows) save the tar+gzip+PUT pass
|
||||
// on every iteration.
|
||||
const existing = await headObject(s3, opts.bucketName, key);
|
||||
if (existing) {
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectS3Uri,
|
||||
bytes: existing.bytes,
|
||||
uploadedAt: existing.lastModified,
|
||||
uploaded: false,
|
||||
};
|
||||
}
|
||||
|
||||
const workdir = mkdtempSync(join(tmpdir(), "hf-deploy-site-"));
|
||||
try {
|
||||
const tarball = join(workdir, "project.tar.gz");
|
||||
await tarDirectory(opts.projectDir, tarball);
|
||||
// Note: tarDirectory packs *everything* under `cwd`. We don't need to
|
||||
// re-implement the skip list inside the tar pack because the
|
||||
// producer's plan stage applies the same skip during its copy; the
|
||||
// archive is slightly bigger than the planDir's compiled/ subtree
|
||||
// but the cost is bounded by the project's user-authored content.
|
||||
const size = statSync(tarball).size;
|
||||
await uploadFileToS3(s3, tarball, projectS3Uri, "application/gzip");
|
||||
return {
|
||||
siteId,
|
||||
bucketName: opts.bucketName,
|
||||
projectS3Uri,
|
||||
bytes: size,
|
||||
uploadedAt: new Date().toISOString(),
|
||||
uploaded: true,
|
||||
};
|
||||
} finally {
|
||||
rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function headObject(
|
||||
s3: S3Client,
|
||||
bucket: string,
|
||||
key: string,
|
||||
): Promise<{ bytes: number; lastModified: string } | null> {
|
||||
try {
|
||||
const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
|
||||
return {
|
||||
bytes: typeof res.ContentLength === "number" ? res.ContentLength : 0,
|
||||
lastModified:
|
||||
res.LastModified instanceof Date
|
||||
? res.LastModified.toISOString()
|
||||
: new Date().toISOString(),
|
||||
};
|
||||
} catch (err) {
|
||||
// The SDK throws different error shapes for 404 vs 403 vs network;
|
||||
// a 404 means "needs upload" and is the most common case. Anything
|
||||
// else propagates so callers see auth / network failures.
|
||||
const status = (err as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
|
||||
if (status === 404) return null;
|
||||
const name = (err as { name?: string }).name;
|
||||
if (name === "NotFound" || name === "NoSuchKey") return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
DescribeExecutionCommand,
|
||||
GetExecutionHistoryCommand,
|
||||
type HistoryEvent,
|
||||
type SFNClient,
|
||||
} from "@aws-sdk/client-sfn";
|
||||
import { getRenderProgress } from "./getRenderProgress.js";
|
||||
|
||||
interface DescribeShape {
|
||||
status?: string;
|
||||
startDate?: Date;
|
||||
stopDate?: Date;
|
||||
}
|
||||
|
||||
class FakeSFN {
|
||||
describe: DescribeShape = {};
|
||||
// Pages of history events; FakeSFN walks them in order, paginating with `nextToken`.
|
||||
historyPages: HistoryEvent[][] = [];
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const cmdName = (command as { constructor: { name: string } }).constructor.name;
|
||||
if (cmdName === "DescribeExecutionCommand") {
|
||||
return {
|
||||
status: this.describe.status ?? "RUNNING",
|
||||
startDate: this.describe.startDate ?? new Date("2026-05-16T00:00:00Z"),
|
||||
stopDate: this.describe.stopDate,
|
||||
};
|
||||
}
|
||||
if (cmdName === "GetExecutionHistoryCommand") {
|
||||
const input = (command as { input: { nextToken?: string } }).input;
|
||||
const idx = input.nextToken ? Number.parseInt(input.nextToken, 10) : 0;
|
||||
const page = this.historyPages[idx] ?? [];
|
||||
const nextToken = idx + 1 < this.historyPages.length ? String(idx + 1) : undefined;
|
||||
return { events: page, nextToken };
|
||||
}
|
||||
throw new Error(`FakeSFN: unexpected command ${cmdName}`);
|
||||
}
|
||||
}
|
||||
|
||||
function lambdaSucceeded(payload: unknown): HistoryEvent {
|
||||
return {
|
||||
type: "LambdaFunctionSucceeded",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
lambdaFunctionSucceededEventDetails: { output: JSON.stringify(payload) },
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
function stateEntered(name: string): HistoryEvent {
|
||||
return {
|
||||
type: "TaskStateEntered",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
stateEnteredEventDetails: { name },
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
function stateExited(name: string, output?: unknown): HistoryEvent {
|
||||
return {
|
||||
type: "TaskStateExited",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
stateExitedEventDetails: {
|
||||
name,
|
||||
output: output === undefined ? undefined : JSON.stringify(output),
|
||||
},
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
// Optimized `lambda:invoke` integration's wire shape: Task* events with
|
||||
// the handler payload at `.Payload`. Helpers below fabricate these so
|
||||
// tests cover the same events a real CDK-deployed render produces.
|
||||
function taskScheduled(): HistoryEvent {
|
||||
return {
|
||||
type: "TaskScheduled",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
taskScheduledEventDetails: {
|
||||
resource: "invoke",
|
||||
resourceType: "lambda",
|
||||
region: "us-east-1",
|
||||
parameters: "{}",
|
||||
},
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
function taskSucceeded(payload: unknown): HistoryEvent {
|
||||
return {
|
||||
type: "TaskSucceeded",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
taskSucceededEventDetails: {
|
||||
resource: "invoke",
|
||||
resourceType: "lambda",
|
||||
output: JSON.stringify({
|
||||
ExecutedVersion: "$LATEST",
|
||||
Payload: payload,
|
||||
StatusCode: 200,
|
||||
}),
|
||||
},
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
function taskFailed(error: string, cause: string): HistoryEvent {
|
||||
return {
|
||||
type: "TaskFailed",
|
||||
id: 1,
|
||||
timestamp: new Date(),
|
||||
taskFailedEventDetails: {
|
||||
resource: "invoke",
|
||||
resourceType: "lambda",
|
||||
error,
|
||||
cause,
|
||||
},
|
||||
} as HistoryEvent;
|
||||
}
|
||||
|
||||
describe("getRenderProgress", () => {
|
||||
it("reports 0 progress before Plan completes", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [[]];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.status).toBe("RUNNING");
|
||||
expect(progress.overallProgress).toBe(0);
|
||||
expect(progress.totalFrames).toBeNull();
|
||||
expect(progress.framesRendered).toBe(0);
|
||||
});
|
||||
|
||||
it("reports 0.1 once Plan completes (totalFrames known, no chunks done)", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
lambdaSucceeded({
|
||||
Action: "plan",
|
||||
TotalFrames: 240,
|
||||
DurationMs: 1_000,
|
||||
}),
|
||||
],
|
||||
];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.totalFrames).toBe(240);
|
||||
expect(progress.overallProgress).toBeCloseTo(0.1, 6);
|
||||
expect(progress.framesRendered).toBe(0);
|
||||
});
|
||||
|
||||
it("advances chunk progress proportionally", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
lambdaSucceeded({ Action: "plan", TotalFrames: 100, DurationMs: 1_000 }),
|
||||
stateEntered("RenderChunk"),
|
||||
lambdaSucceeded({ Action: "renderChunk", FramesEncoded: 50, DurationMs: 2_000 }),
|
||||
],
|
||||
];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
// 0.1 + 0.8 × 0.5 = 0.5
|
||||
expect(progress.overallProgress).toBeCloseTo(0.5, 6);
|
||||
expect(progress.framesRendered).toBe(50);
|
||||
});
|
||||
|
||||
it("does not double-count Assemble's FramesEncoded toward framesRendered", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
lambdaSucceeded({ Action: "plan", TotalFrames: 100, DurationMs: 1_000 }),
|
||||
stateEntered("RenderChunk"),
|
||||
lambdaSucceeded({ Action: "renderChunk", FramesEncoded: 100, DurationMs: 2_000 }),
|
||||
stateEntered("Assemble"),
|
||||
lambdaSucceeded({
|
||||
Action: "assemble",
|
||||
FramesEncoded: 100,
|
||||
FileSize: 9_000_000,
|
||||
OutputS3Uri: "s3://b/k.mp4",
|
||||
DurationMs: 1_500,
|
||||
}),
|
||||
stateExited("Assemble", {
|
||||
Output: { OutputS3Uri: "s3://b/k.mp4", FileSize: 9_000_000, FramesEncoded: 100 },
|
||||
}),
|
||||
],
|
||||
];
|
||||
sfn.describe.status = "SUCCEEDED";
|
||||
sfn.describe.stopDate = new Date("2026-05-16T00:05:00Z");
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.framesRendered).toBe(100);
|
||||
expect(progress.overallProgress).toBe(1);
|
||||
expect(progress.outputFile).toEqual({ s3Uri: "s3://b/k.mp4", bytes: 9_000_000 });
|
||||
expect(progress.endedAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("computes cost from observed billed duration", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [[lambdaSucceeded({ Action: "plan", TotalFrames: 30, DurationMs: 6_000 })]];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
defaultMemorySizeMb: 10_240,
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
// 1 transition event + 1 invocation × 60 GB-s × $0.0000166667 ≈ $0.001
|
||||
expect(progress.costs.breakdown.lambdaUsd).toBeCloseTo(0.001, 4);
|
||||
});
|
||||
|
||||
it("captures Lambda failures with the enclosing state name", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const failed: HistoryEvent = {
|
||||
type: "LambdaFunctionFailed",
|
||||
id: 2,
|
||||
timestamp: new Date(),
|
||||
lambdaFunctionFailedEventDetails: {
|
||||
error: "PLAN_HASH_MISMATCH",
|
||||
cause: "bad plan",
|
||||
},
|
||||
} as HistoryEvent;
|
||||
sfn.historyPages = [[stateEntered("RenderChunk"), failed]];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.errors).toEqual([
|
||||
{ state: "RenderChunk", error: "PLAN_HASH_MISMATCH", cause: "bad plan" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks fatalErrorEncountered when execution ends FAILED", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [[]];
|
||||
sfn.describe.status = "FAILED";
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.fatalErrorEncountered).toBe(true);
|
||||
});
|
||||
|
||||
it("paginates the history", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
lambdaSucceeded({ Action: "plan", TotalFrames: 4, DurationMs: 1_000 }),
|
||||
],
|
||||
[
|
||||
stateEntered("RenderChunk"),
|
||||
lambdaSucceeded({ Action: "renderChunk", FramesEncoded: 4, DurationMs: 2_000 }),
|
||||
],
|
||||
];
|
||||
sfn.describe.status = "SUCCEEDED";
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.framesRendered).toBe(4);
|
||||
expect(progress.totalFrames).toBe(4);
|
||||
expect(progress.overallProgress).toBe(1);
|
||||
});
|
||||
|
||||
it("requires executionArn", async () => {
|
||||
await expect(getRenderProgress({ executionArn: "" })).rejects.toThrow(/executionArn/);
|
||||
});
|
||||
|
||||
describe("optimized lambda:invoke integration", () => {
|
||||
it("counts a single TaskSucceeded as one Lambda invocation", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({ Action: "plan", TotalFrames: 240, DurationMs: 1_000 }),
|
||||
],
|
||||
];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
defaultMemorySizeMb: 10_240,
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.lambdasInvoked).toBe(1);
|
||||
expect(progress.totalFrames).toBe(240);
|
||||
// computeRenderCost rounds to 4 decimals; precision=4 not 6.
|
||||
expect(progress.costs.breakdown.lambdaUsd).toBeCloseTo(0.0002, 4);
|
||||
expect(progress.costs.breakdown.lambdaUsd).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("attributes RenderChunk FramesEncoded but ignores Plan/Assemble FramesEncoded", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({ Action: "plan", TotalFrames: 100, DurationMs: 1_000 }),
|
||||
stateEntered("RenderChunk"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({ Action: "renderChunk", FramesEncoded: 50, DurationMs: 2_000 }),
|
||||
stateEntered("Assemble"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({
|
||||
Action: "assemble",
|
||||
FramesEncoded: 100, // would double-count if Assemble's count bled in
|
||||
FileSize: 9_000_000,
|
||||
OutputS3Uri: "s3://b/k.mp4",
|
||||
DurationMs: 1_500,
|
||||
}),
|
||||
],
|
||||
];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.framesRendered).toBe(50);
|
||||
expect(progress.lambdasInvoked).toBe(3);
|
||||
});
|
||||
|
||||
it("captures TaskFailed errors with the enclosing state name", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("RenderChunk"),
|
||||
taskScheduled(),
|
||||
taskFailed("Sandbox.Timedout", "Task timed out after 900.00 seconds"),
|
||||
],
|
||||
];
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
expect(progress.errors).toEqual([
|
||||
{
|
||||
state: "RenderChunk",
|
||||
error: "Sandbox.Timedout",
|
||||
cause: "Task timed out after 900.00 seconds",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("sums billed seconds across plan + chunks + assemble", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const renderChunkSucceeded = (frames: number, ms: number) => [
|
||||
stateEntered("RenderChunk"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({ Action: "renderChunk", FramesEncoded: frames, DurationMs: ms }),
|
||||
];
|
||||
// 1 plan + 16 chunks @ 217s + 1 assemble = 3492.7s billed.
|
||||
sfn.historyPages = [
|
||||
[
|
||||
stateEntered("Plan"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({ Action: "plan", TotalFrames: 1349, DurationMs: 13_000 }),
|
||||
...Array.from({ length: 16 }, () => renderChunkSucceeded(84, 217_000)).flat(),
|
||||
stateEntered("Assemble"),
|
||||
taskScheduled(),
|
||||
taskSucceeded({
|
||||
Action: "assemble",
|
||||
FileSize: 81_000_000,
|
||||
OutputS3Uri: "s3://b/k.mp4",
|
||||
DurationMs: 7_700,
|
||||
}),
|
||||
],
|
||||
];
|
||||
sfn.describe.status = "SUCCEEDED";
|
||||
const progress = await getRenderProgress({
|
||||
executionArn: "arn",
|
||||
defaultMemorySizeMb: 10_240,
|
||||
sfn: sfn as unknown as SFNClient,
|
||||
});
|
||||
// 3492.7s × 10GB × $0.0000166667/GB-s ≈ $0.582.
|
||||
expect(progress.costs.breakdown.lambdaUsd).toBeCloseTo(0.582, 2);
|
||||
expect(progress.framesRendered).toBe(84 * 16);
|
||||
expect(progress.lambdasInvoked).toBe(18);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
void DescribeExecutionCommand;
|
||||
void GetExecutionHistoryCommand;
|
||||
@@ -0,0 +1,396 @@
|
||||
/**
|
||||
* `getRenderProgress` — read-only progress + cost snapshot for a single
|
||||
* render started by {@link renderToLambda}.
|
||||
*
|
||||
* Pulls one `DescribeExecution` + one `GetExecutionHistory` per call. The
|
||||
* history is paginated server-side; the helper loops until exhausted so a
|
||||
* 1,000-event Step Functions execution still produces a single
|
||||
* `RenderProgress` snapshot.
|
||||
*
|
||||
* Progress math:
|
||||
* - 0 before Plan completes (no frame count is known yet)
|
||||
* - 0.1 once Plan completes (we know `totalFrames`)
|
||||
* - 0.1 + 0.8 × framesEncoded / totalFrames during chunk render
|
||||
* - 1.0 after Assemble completes
|
||||
*
|
||||
* Frame counts come from the parsed Lambda result payloads on each
|
||||
* `TaskSucceeded` event — Plan reports `TotalFrames`, RenderChunk reports
|
||||
* `FramesEncoded`. The shape mirrors what the handler produces in
|
||||
* `events.ts`, so the parser doesn't need to know anything beyond
|
||||
* "JSON.parse this string and grab two fields."
|
||||
*/
|
||||
|
||||
import {
|
||||
DescribeExecutionCommand,
|
||||
GetExecutionHistoryCommand,
|
||||
type HistoryEvent,
|
||||
SFNClient,
|
||||
} from "@aws-sdk/client-sfn";
|
||||
import {
|
||||
type BilledLambdaInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./costAccounting.js";
|
||||
|
||||
/** Options for {@link getRenderProgress}. */
|
||||
export interface GetRenderProgressOptions {
|
||||
/** Execution ARN from a {@link renderToLambda} call. */
|
||||
executionArn: string;
|
||||
/**
|
||||
* Default memory size in MB to assume for Lambda invocations when the
|
||||
* history event payload doesn't carry it explicitly. Matches the
|
||||
* `LambdaMemoryMb` parameter the stack was deployed with.
|
||||
*/
|
||||
defaultMemorySizeMb?: number;
|
||||
region?: string;
|
||||
/** Test injection seam. */
|
||||
sfn?: SFNClient;
|
||||
}
|
||||
|
||||
/** Render-status discriminant; mirrors Step Functions execution states. */
|
||||
export type RenderStatus =
|
||||
| "RUNNING"
|
||||
| "SUCCEEDED"
|
||||
| "FAILED"
|
||||
| "TIMED_OUT"
|
||||
| "ABORTED"
|
||||
| "PENDING_REDRIVE";
|
||||
|
||||
export interface RenderError {
|
||||
/** State name where the failure surfaced (`Plan`, `RenderChunk`, `Assemble`, or `<unknown>`). */
|
||||
state: string;
|
||||
/** Error class / type as Step Functions reports it. */
|
||||
error: string;
|
||||
/** Cause string Step Functions surfaces (often a stringified JSON payload from the handler). */
|
||||
cause: string;
|
||||
}
|
||||
|
||||
/** Snapshot of a single render's progress + cost + errors at one point in time. */
|
||||
export interface RenderProgress {
|
||||
status: RenderStatus;
|
||||
/** `[0, 1]`; see module doc for the math. */
|
||||
overallProgress: number;
|
||||
framesRendered: number;
|
||||
/** `null` until Plan completes. */
|
||||
totalFrames: number | null;
|
||||
/** Total Lambda invocations scheduled so far (both optimized + raw task integrations). */
|
||||
lambdasInvoked: number;
|
||||
costs: RenderCost;
|
||||
/** Final output object if Assemble succeeded; `null` otherwise. */
|
||||
outputFile: { s3Uri: string; bytes: number | null } | null;
|
||||
errors: RenderError[];
|
||||
/** `true` once the execution has terminated in a non-`SUCCEEDED` state. */
|
||||
fatalErrorEncountered: boolean;
|
||||
startedAt: string;
|
||||
endedAt: string | null;
|
||||
}
|
||||
|
||||
const DEFAULT_MEMORY_MB = 10240;
|
||||
|
||||
/** Pull a current progress snapshot for one render. */
|
||||
export async function getRenderProgress(opts: GetRenderProgressOptions): Promise<RenderProgress> {
|
||||
if (!opts.executionArn) {
|
||||
throw new Error("[getRenderProgress] executionArn is required");
|
||||
}
|
||||
const sfn = opts.sfn ?? new SFNClient({ region: opts.region });
|
||||
const memoryMb = opts.defaultMemorySizeMb ?? DEFAULT_MEMORY_MB;
|
||||
|
||||
const describe = await sfn.send(
|
||||
new DescribeExecutionCommand({ executionArn: opts.executionArn }),
|
||||
);
|
||||
const status = (describe.status ?? "RUNNING") as RenderStatus;
|
||||
const startedAt = describe.startDate?.toISOString() ?? new Date(0).toISOString();
|
||||
const endedAt = describe.stopDate?.toISOString() ?? null;
|
||||
|
||||
const history = await loadFullHistory(sfn, opts.executionArn);
|
||||
const summary = summarizeHistory(history, memoryMb);
|
||||
|
||||
const costs = computeRenderCost(summary.lambdaInvocations, summary.stateTransitions);
|
||||
const overallProgress = computeOverallProgress({
|
||||
status,
|
||||
totalFrames: summary.totalFrames,
|
||||
framesRendered: summary.framesRendered,
|
||||
assembleComplete: summary.assembleComplete,
|
||||
});
|
||||
|
||||
return {
|
||||
status,
|
||||
overallProgress,
|
||||
framesRendered: summary.framesRendered,
|
||||
totalFrames: summary.totalFrames,
|
||||
lambdasInvoked: summary.lambdasInvoked,
|
||||
costs,
|
||||
outputFile: summary.outputFile,
|
||||
errors: summary.errors,
|
||||
fatalErrorEncountered: isTerminalFailure(status),
|
||||
startedAt,
|
||||
endedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadFullHistory(sfn: SFNClient, executionArn: string): Promise<HistoryEvent[]> {
|
||||
const events: HistoryEvent[] = [];
|
||||
let nextToken: string | undefined;
|
||||
for (let page = 0; page < 50; page++) {
|
||||
const res = await sfn.send(
|
||||
new GetExecutionHistoryCommand({
|
||||
executionArn,
|
||||
maxResults: 1000,
|
||||
nextToken,
|
||||
reverseOrder: false,
|
||||
}),
|
||||
);
|
||||
if (res.events) events.push(...res.events);
|
||||
nextToken = res.nextToken;
|
||||
if (!nextToken) break;
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
interface HistorySummary {
|
||||
lambdaInvocations: BilledLambdaInvocation[];
|
||||
stateTransitions: number;
|
||||
framesRendered: number;
|
||||
totalFrames: number | null;
|
||||
lambdasInvoked: number;
|
||||
assembleComplete: boolean;
|
||||
outputFile: { s3Uri: string; bytes: number | null } | null;
|
||||
errors: RenderError[];
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass over the history events that pulls every number {@link getRenderProgress}
|
||||
* needs. State transitions = the count of events that advance the state
|
||||
* machine (entering/exiting states + map iteration completions). Lambda
|
||||
* invocations = `LambdaFunctionScheduled` count. Frame totals come from
|
||||
* the success-payload of each Lambda invocation.
|
||||
*/
|
||||
function summarizeHistory(events: HistoryEvent[], memoryMb: number): HistorySummary {
|
||||
let framesRendered = 0;
|
||||
let totalFrames: number | null = null;
|
||||
let lambdasInvoked = 0;
|
||||
let assembleComplete = false;
|
||||
let outputFile: HistorySummary["outputFile"] = null;
|
||||
let stateTransitions = 0;
|
||||
const errors: RenderError[] = [];
|
||||
const lambdaInvocations: BilledLambdaInvocation[] = [];
|
||||
|
||||
// Track the state name we most recently entered, so we can:
|
||||
// - attach the enclosing state to LambdaFunctionFailed errors, and
|
||||
// - identify when the Assemble state finished (StateExited.Assemble)
|
||||
// without relying on the inner Lambda payload's `Action` field.
|
||||
let currentLambdaState: string | null = null;
|
||||
|
||||
for (const ev of events) {
|
||||
switch (ev.type) {
|
||||
case "TaskStateEntered":
|
||||
case "MapStateEntered":
|
||||
case "PassStateEntered":
|
||||
case "ChoiceStateEntered":
|
||||
case "SucceedStateEntered":
|
||||
case "FailStateEntered":
|
||||
case "WaitStateEntered":
|
||||
case "ParallelStateEntered":
|
||||
// Step Functions Standard Workflows bill per *state entry*, not per
|
||||
// history event. Lambda invocations produce ~5-7 history events
|
||||
// each (Scheduled / Started / Succeeded / TaskStateExited / …);
|
||||
// counting every event as a transition over-reports cost by 3-5×.
|
||||
stateTransitions++;
|
||||
currentLambdaState = ev.stateEnteredEventDetails?.name ?? currentLambdaState;
|
||||
break;
|
||||
// Optimized `lambda:invoke` task emits Task* events; raw
|
||||
// `lambda:invokeFunction.sync` emits LambdaFunction*. Handle both.
|
||||
case "TaskScheduled":
|
||||
if (ev.taskScheduledEventDetails?.resourceType === "lambda") {
|
||||
lambdasInvoked++;
|
||||
}
|
||||
break;
|
||||
case "LambdaFunctionScheduled":
|
||||
lambdasInvoked++;
|
||||
break;
|
||||
case "TaskSucceeded": {
|
||||
if (ev.taskSucceededEventDetails?.resourceType !== "lambda") break;
|
||||
const wrapped = parseJson(ev.taskSucceededEventDetails?.output);
|
||||
const payload = unwrapLambdaPayload(wrapped);
|
||||
const billedDurationMs = inferBilledMs(payload);
|
||||
lambdaInvocations.push({
|
||||
billedDurationMs,
|
||||
memorySizeMb: memoryMb,
|
||||
estimated: billedDurationMs === 0,
|
||||
});
|
||||
applyPayloadFrameCounts(payload, currentLambdaState, (delta) => {
|
||||
framesRendered += delta;
|
||||
});
|
||||
if (payload && typeof payload === "object") {
|
||||
const obj = payload as Record<string, unknown>;
|
||||
if (typeof obj.TotalFrames === "number") totalFrames = obj.TotalFrames;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "LambdaFunctionSucceeded": {
|
||||
const payload = parseJson(ev.lambdaFunctionSucceededEventDetails?.output);
|
||||
const billedDurationMs = inferBilledMs(payload);
|
||||
lambdaInvocations.push({
|
||||
billedDurationMs,
|
||||
memorySizeMb: memoryMb,
|
||||
estimated: billedDurationMs === 0,
|
||||
});
|
||||
applyPayloadFrameCounts(payload, currentLambdaState, (delta) => {
|
||||
framesRendered += delta;
|
||||
});
|
||||
if (payload && typeof payload === "object") {
|
||||
const obj = payload as Record<string, unknown>;
|
||||
if (typeof obj.TotalFrames === "number") totalFrames = obj.TotalFrames;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "TaskStateExited":
|
||||
case "MapStateExited":
|
||||
// Mark the assemble step complete on its state-exit, independent
|
||||
// of the inner Lambda payload shape. The Assemble state's
|
||||
// ResultSelector pulls FileSize + OutputS3Uri from the Lambda
|
||||
// result, so we re-extract them here from the state exit's
|
||||
// own output rather than relying on the Lambda payload.
|
||||
if (ev.stateExitedEventDetails?.name === "Assemble") {
|
||||
assembleComplete = true;
|
||||
const exitPayload = parseJson(ev.stateExitedEventDetails?.output);
|
||||
if (exitPayload && typeof exitPayload === "object") {
|
||||
const obj = exitPayload as Record<string, unknown>;
|
||||
const out = obj.Output as Record<string, unknown> | undefined;
|
||||
const outputS3Uri = typeof out?.OutputS3Uri === "string" ? out.OutputS3Uri : null;
|
||||
const bytes = typeof out?.FileSize === "number" ? out.FileSize : null;
|
||||
outputFile = outputS3Uri ? { s3Uri: outputS3Uri, bytes } : outputFile;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "TaskFailed":
|
||||
if (ev.taskFailedEventDetails?.resourceType !== "lambda") break;
|
||||
errors.push({
|
||||
state: currentLambdaState ?? "<unknown>",
|
||||
error: ev.taskFailedEventDetails?.error ?? "UNKNOWN",
|
||||
cause: ev.taskFailedEventDetails?.cause ?? "",
|
||||
});
|
||||
break;
|
||||
case "LambdaFunctionFailed":
|
||||
errors.push({
|
||||
state: currentLambdaState ?? "<unknown>",
|
||||
error: ev.lambdaFunctionFailedEventDetails?.error ?? "UNKNOWN",
|
||||
cause: ev.lambdaFunctionFailedEventDetails?.cause ?? "",
|
||||
});
|
||||
break;
|
||||
case "ExecutionFailed":
|
||||
errors.push({
|
||||
state: "<execution>",
|
||||
error: ev.executionFailedEventDetails?.error ?? "UNKNOWN",
|
||||
cause: ev.executionFailedEventDetails?.cause ?? "",
|
||||
});
|
||||
break;
|
||||
case "ExecutionAborted":
|
||||
errors.push({
|
||||
state: "<execution>",
|
||||
error: ev.executionAbortedEventDetails?.error ?? "ABORTED",
|
||||
cause: ev.executionAbortedEventDetails?.cause ?? "",
|
||||
});
|
||||
break;
|
||||
case "ExecutionTimedOut":
|
||||
errors.push({
|
||||
state: "<execution>",
|
||||
error: "TIMEOUT",
|
||||
cause: ev.executionTimedOutEventDetails?.cause ?? "",
|
||||
});
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
lambdaInvocations,
|
||||
stateTransitions,
|
||||
framesRendered,
|
||||
totalFrames,
|
||||
lambdasInvoked,
|
||||
assembleComplete,
|
||||
outputFile,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
function parseJson(s: string | undefined): unknown {
|
||||
if (!s) return null;
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized `lambda:invoke` wraps the Lambda response as
|
||||
* `{ ExecutedVersion, Payload: {…handler payload…}, StatusCode }`. Raw
|
||||
* `lambda:invokeFunction.sync` puts the handler payload at the root.
|
||||
* Return the inner `Payload` when present so callers read the same fields
|
||||
* either way.
|
||||
*/
|
||||
function unwrapLambdaPayload(payload: unknown): unknown {
|
||||
if (payload && typeof payload === "object" && "Payload" in payload) {
|
||||
const inner = (payload as { Payload: unknown }).Payload;
|
||||
if (inner && typeof inner === "object") return inner;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bump `framesRendered` only inside the `RenderChunk` state. Plan and
|
||||
* Assemble also report `FramesEncoded`, so a state-blind add would
|
||||
* double-count once Assemble runs.
|
||||
*/
|
||||
function applyPayloadFrameCounts(
|
||||
payload: unknown,
|
||||
currentLambdaState: string | null,
|
||||
bump: (delta: number) => void,
|
||||
): void {
|
||||
if (currentLambdaState !== "RenderChunk") return;
|
||||
if (!payload || typeof payload !== "object") return;
|
||||
const obj = payload as Record<string, unknown>;
|
||||
if (typeof obj.FramesEncoded === "number") bump(obj.FramesEncoded);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lambda success payloads from our handler include `DurationMs` — the
|
||||
* wall-clock the handler observed. We use it as a best-effort proxy
|
||||
* for `BilledDuration` when SFN doesn't expose the latter directly
|
||||
* on `LambdaFunctionSucceeded` (the dedicated `BilledDuration` field
|
||||
* is in CloudWatch Metrics, not the SFN history payload).
|
||||
*/
|
||||
function inferBilledMs(payload: unknown): number {
|
||||
if (!payload || typeof payload !== "object") return 0;
|
||||
const obj = payload as Record<string, unknown>;
|
||||
if (typeof obj.DurationMs === "number") return obj.DurationMs;
|
||||
return 0;
|
||||
}
|
||||
|
||||
interface ComputeProgressArgs {
|
||||
status: RenderStatus;
|
||||
totalFrames: number | null;
|
||||
framesRendered: number;
|
||||
assembleComplete: boolean;
|
||||
}
|
||||
|
||||
function computeOverallProgress({
|
||||
status,
|
||||
totalFrames,
|
||||
framesRendered,
|
||||
assembleComplete,
|
||||
}: ComputeProgressArgs): number {
|
||||
if (status === "SUCCEEDED") return 1;
|
||||
if (assembleComplete) return 1;
|
||||
if (totalFrames === null) return 0;
|
||||
// 10 % Plan + 80 % chunk render + 10 % Assemble.
|
||||
const chunkProgress = Math.min(1, framesRendered / totalFrames);
|
||||
return 0.1 + 0.8 * chunkProgress;
|
||||
}
|
||||
|
||||
function isTerminalFailure(status: RenderStatus): boolean {
|
||||
return status === "FAILED" || status === "TIMED_OUT" || status === "ABORTED";
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* SDK subpath export — `@hyperframes/aws-lambda/sdk`.
|
||||
*
|
||||
* Pulled into its own subpath so consumers that only drive Lambda renders
|
||||
* (CLI, CI scripts, adopter tooling) don't pay the cost of importing
|
||||
* `./handler.js`, which transitively pulls `@sparticuz/chromium` +
|
||||
* `puppeteer-core` into the module graph. The SDK files here are
|
||||
* AWS-SDK only — safe to load in any Node environment.
|
||||
*/
|
||||
|
||||
export { deploySite, type DeploySiteOptions, type SiteHandle } from "./deploySite.js";
|
||||
export { renderToLambda, type RenderHandle, type RenderToLambdaOptions } from "./renderToLambda.js";
|
||||
export {
|
||||
getRenderProgress,
|
||||
type GetRenderProgressOptions,
|
||||
type RenderError,
|
||||
type RenderProgress,
|
||||
type RenderStatus,
|
||||
} from "./getRenderProgress.js";
|
||||
export {
|
||||
type BilledLambdaInvocation,
|
||||
computeRenderCost,
|
||||
type RenderCost,
|
||||
} from "./costAccounting.js";
|
||||
export {
|
||||
InvalidConfigError,
|
||||
MAX_STEP_FUNCTIONS_INPUT_BYTES,
|
||||
validateDistributedRenderConfig,
|
||||
validateStepFunctionsInputSize,
|
||||
validateVariablesPayload,
|
||||
} from "./validateConfig.js";
|
||||
export type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
export type { DistributedFormat } from "../formatExtension.js";
|
||||
@@ -0,0 +1,265 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { SFNClient } from "@aws-sdk/client-sfn";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import { asS3Client, FakeS3 } from "./__fixtures__/fakeS3.js";
|
||||
import type { SiteHandle } from "./deploySite.js";
|
||||
import { renderToLambda } from "./renderToLambda.js";
|
||||
import { InvalidConfigError } from "./validateConfig.js";
|
||||
|
||||
interface CapturedStart {
|
||||
stateMachineArn: string;
|
||||
name: string;
|
||||
input: unknown;
|
||||
}
|
||||
|
||||
class FakeSFN {
|
||||
starts: CapturedStart[] = [];
|
||||
async send(command: unknown): Promise<unknown> {
|
||||
const cmdName = (command as { constructor: { name: string } }).constructor.name;
|
||||
if (cmdName === "StartExecutionCommand") {
|
||||
const input = (command as { input: { stateMachineArn: string; name: string; input: string } })
|
||||
.input;
|
||||
this.starts.push({
|
||||
stateMachineArn: input.stateMachineArn,
|
||||
name: input.name,
|
||||
input: JSON.parse(input.input),
|
||||
});
|
||||
return {
|
||||
executionArn: `arn:aws:states:us-east-1:1234:execution:hf:${input.name}`,
|
||||
startDate: new Date(),
|
||||
};
|
||||
}
|
||||
throw new Error(`FakeSFN: unexpected command ${cmdName}`);
|
||||
}
|
||||
}
|
||||
|
||||
function asSFNClient(fake: { send(command: unknown): Promise<unknown> }): SFNClient {
|
||||
return fake as unknown as SFNClient;
|
||||
}
|
||||
|
||||
const baseConfig: SerializableDistributedRenderConfig = {
|
||||
fps: 30,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
format: "mp4",
|
||||
};
|
||||
|
||||
let projectDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
projectDir = mkdtempSync(join(tmpdir(), "hf-render-test-"));
|
||||
writeFileSync(join(projectDir, "index.html"), "<html></html>");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(projectDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("renderToLambda", () => {
|
||||
it("returns a handle and starts a state-machine execution with the right input", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const handle = await renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: baseConfig,
|
||||
executionName: "smoke-1",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
|
||||
expect(handle.renderId).toBe("smoke-1");
|
||||
expect(handle.executionArn).toContain("smoke-1");
|
||||
expect(handle.bucketName).toBe("test-bucket");
|
||||
expect(handle.outputS3Uri).toBe("s3://test-bucket/renders/smoke-1/output.mp4");
|
||||
expect(handle.projectS3Uri).toMatch(
|
||||
/^s3:\/\/test-bucket\/sites\/[0-9a-f]{16}\/project\.tar\.gz$/,
|
||||
);
|
||||
|
||||
expect(sfn.starts).toHaveLength(1);
|
||||
const start = sfn.starts[0]!;
|
||||
expect(start.name).toBe("smoke-1");
|
||||
expect(start.stateMachineArn).toBe("arn:aws:states:us-east-1:1234:stateMachine:hf");
|
||||
expect(start.input).toEqual({
|
||||
ProjectS3Uri: handle.projectS3Uri,
|
||||
PlanOutputS3Prefix: "s3://test-bucket/renders/smoke-1/",
|
||||
OutputS3Uri: "s3://test-bucket/renders/smoke-1/output.mp4",
|
||||
Config: baseConfig,
|
||||
});
|
||||
});
|
||||
|
||||
it("derives the file extension from config.format", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const handle = await renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: { ...baseConfig, format: "mov" },
|
||||
executionName: "smoke-mov",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(handle.outputS3Uri).toBe("s3://test-bucket/renders/smoke-mov/output.mov");
|
||||
});
|
||||
|
||||
it("reuses a supplied siteHandle (no deploy)", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const prebuilt: SiteHandle = {
|
||||
siteId: "prebaked",
|
||||
bucketName: "test-bucket",
|
||||
projectS3Uri: "s3://test-bucket/sites/prebaked/project.tar.gz",
|
||||
bytes: 4096,
|
||||
uploadedAt: "2026-05-16T00:00:00Z",
|
||||
uploaded: true,
|
||||
};
|
||||
const handle = await renderToLambda({
|
||||
siteHandle: prebuilt,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: baseConfig,
|
||||
executionName: "smoke-reuse",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(handle.projectS3Uri).toBe(prebuilt.projectS3Uri);
|
||||
// No HEAD/PUT means s3.existing stayed empty.
|
||||
expect(s3.existing.size).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects invalid configs synchronously before any AWS call", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
try {
|
||||
await renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: { ...baseConfig, fps: 25 as 24 | 30 | 60 },
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
}
|
||||
expect(sfn.starts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("requires either siteHandle or projectDir", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
await expect(
|
||||
renderToLambda({
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: baseConfig,
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
}),
|
||||
).rejects.toThrow(/either siteHandle or projectDir/);
|
||||
});
|
||||
|
||||
it("auto-generates an executionName when omitted", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const handle = await renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: baseConfig,
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(handle.renderId).toMatch(/^hf-render-[0-9a-f-]{36}$/);
|
||||
});
|
||||
|
||||
it("threads variables through the Step Functions execution input", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const variables = { title: "Hello Alice", accent: "#ff0000" };
|
||||
await renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: { ...baseConfig, variables },
|
||||
executionName: "smoke-variables",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
});
|
||||
expect(sfn.starts).toHaveLength(1);
|
||||
const start = sfn.starts[0]!;
|
||||
// The execution input carries the variables under Config.variables —
|
||||
// the Step Functions state machine forwards `Config` verbatim into the
|
||||
// PlanEvent's `Config` field, where the handler spreads it into the
|
||||
// producer's DistributedRenderConfig.
|
||||
const input = start.input as { Config: { variables?: Record<string, unknown> } };
|
||||
expect(input.Config.variables).toEqual(variables);
|
||||
});
|
||||
|
||||
it("rejects a config whose variables blob would push the execution input over 256 KiB", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
const huge = "x".repeat(260 * 1024);
|
||||
await expect(
|
||||
renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: { ...baseConfig, variables: { blob: huge } },
|
||||
executionName: "smoke-too-big",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
}),
|
||||
).rejects.toThrow(/256.*KiB|templates-on-lambda/);
|
||||
// The reject must happen BEFORE StartExecution — uncaught oversize input
|
||||
// surfaces as States.DataLimitExceeded 50ms in, far from this call site.
|
||||
expect(sfn.starts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects a config whose variables contain non-JSON-safe values", async () => {
|
||||
const sfn = new FakeSFN();
|
||||
const s3 = new FakeS3();
|
||||
await expect(
|
||||
renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: {
|
||||
...baseConfig,
|
||||
// BigInt would throw at JSON.stringify time; catch it at the validator
|
||||
// boundary with a typed error instead.
|
||||
variables: { count: 9_007_199_254_740_993n } as unknown as Record<string, unknown>,
|
||||
},
|
||||
executionName: "smoke-bigint",
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
}),
|
||||
).rejects.toThrow(InvalidConfigError);
|
||||
expect(sfn.starts).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("propagates a missing executionArn as an error", async () => {
|
||||
const sfn = {
|
||||
async send(_cmd: unknown): Promise<unknown> {
|
||||
return { executionArn: undefined };
|
||||
},
|
||||
};
|
||||
const s3 = new FakeS3();
|
||||
await expect(
|
||||
renderToLambda({
|
||||
projectDir,
|
||||
bucketName: "test-bucket",
|
||||
stateMachineArn: "arn:aws:states:us-east-1:1234:stateMachine:hf",
|
||||
config: baseConfig,
|
||||
sfn: asSFNClient(sfn),
|
||||
s3: asS3Client(s3),
|
||||
}),
|
||||
).rejects.toThrow(/no executionArn/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* `renderToLambda` — start a distributed render against an already-deployed
|
||||
* SAM/CDK stack and return a handle the caller can poll with
|
||||
* {@link getRenderProgress}.
|
||||
*
|
||||
* The function does *not* wait for the render to finish. Step Functions
|
||||
* standard workflows can run for hours; blocking the caller's process on
|
||||
* the SFN execution is the wrong default. The returned `RenderHandle`
|
||||
* carries everything the progress / cost / download paths need.
|
||||
*
|
||||
* Wire order:
|
||||
* 1. Validate config (typed throw before any AWS call).
|
||||
* 2. `deploySite` if no `siteHandle` was provided.
|
||||
* 3. `StartExecution` against the state machine with the same input
|
||||
* shape `examples/aws-lambda/scripts/smoke.sh` builds.
|
||||
* 4. Return handle. The S3 `outputKey` is deterministic from the
|
||||
* execution name so the caller can predict the final object URL.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { SFNClient, StartExecutionCommand } from "@aws-sdk/client-sfn";
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import { formatExtension } from "../formatExtension.js";
|
||||
import { formatS3Uri } from "../s3Transport.js";
|
||||
import { deploySite, type SiteHandle } from "./deploySite.js";
|
||||
import {
|
||||
validateDistributedRenderConfig,
|
||||
validateStepFunctionsInputSize,
|
||||
} from "./validateConfig.js";
|
||||
|
||||
/** Options for {@link renderToLambda}. */
|
||||
export interface RenderToLambdaOptions {
|
||||
/** Local project directory. Required when `siteHandle` is not supplied. */
|
||||
projectDir?: string;
|
||||
/** Re-use an existing `deploySite` upload (skips tar+S3 PUT). */
|
||||
siteHandle?: SiteHandle;
|
||||
/** Validated `SerializableDistributedRenderConfig` (no logger / abortSignal). */
|
||||
config: SerializableDistributedRenderConfig;
|
||||
/** S3 bucket from the SAM stack output (`RenderBucketName`). */
|
||||
bucketName: string;
|
||||
/** State machine ARN from the SAM stack output (`RenderStateMachineArn`). */
|
||||
stateMachineArn: string;
|
||||
/** AWS region; defaults to the SDK default chain. */
|
||||
region?: string;
|
||||
/**
|
||||
* Final output S3 key. Defaults to `renders/<executionName>/output.<ext>`
|
||||
* where `<ext>` is derived from `config.format`.
|
||||
*/
|
||||
outputKey?: string;
|
||||
/**
|
||||
* Step Functions execution name. Defaults to `hf-render-<uuid>`.
|
||||
* Used as `renderId` everywhere downstream (history queries, cost
|
||||
* accounting, predictable S3 key prefix).
|
||||
*/
|
||||
executionName?: string;
|
||||
/** Test injection seam — production callers leave unset. */
|
||||
sfn?: SFNClient;
|
||||
/** Test injection seam — propagated to `deploySite` when applicable. */
|
||||
s3?: S3Client;
|
||||
}
|
||||
|
||||
/** Stable identifier + every URL/ARN the caller needs to follow the render. */
|
||||
export interface RenderHandle {
|
||||
/** Same as the Step Functions execution name. */
|
||||
renderId: string;
|
||||
/** Full execution ARN; pass to {@link getRenderProgress}. */
|
||||
executionArn: string;
|
||||
bucketName: string;
|
||||
stateMachineArn: string;
|
||||
outputS3Uri: string;
|
||||
projectS3Uri: string;
|
||||
startedAt: string;
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line complexity
|
||||
export async function renderToLambda(opts: RenderToLambdaOptions): Promise<RenderHandle> {
|
||||
validateDistributedRenderConfig(opts.config);
|
||||
|
||||
if (!opts.bucketName) {
|
||||
throw new Error("[renderToLambda] bucketName is required");
|
||||
}
|
||||
if (!opts.stateMachineArn) {
|
||||
throw new Error("[renderToLambda] stateMachineArn is required");
|
||||
}
|
||||
if (!opts.siteHandle && !opts.projectDir) {
|
||||
throw new Error("[renderToLambda] either siteHandle or projectDir must be supplied");
|
||||
}
|
||||
|
||||
const executionName = opts.executionName ?? `hf-render-${randomUUID()}`;
|
||||
const ext = formatExtension(opts.config.format);
|
||||
const outputKey = opts.outputKey ?? `renders/${executionName}/output${ext}`;
|
||||
const planOutputS3Prefix = formatS3Uri({
|
||||
bucket: opts.bucketName,
|
||||
key: `renders/${executionName}/`,
|
||||
});
|
||||
const outputS3Uri = formatS3Uri({ bucket: opts.bucketName, key: outputKey });
|
||||
|
||||
const site =
|
||||
opts.siteHandle ??
|
||||
(await deploySite({
|
||||
projectDir: opts.projectDir as string,
|
||||
bucketName: opts.bucketName,
|
||||
region: opts.region,
|
||||
s3: opts.s3,
|
||||
}));
|
||||
|
||||
const input = {
|
||||
ProjectS3Uri: site.projectS3Uri,
|
||||
PlanOutputS3Prefix: planOutputS3Prefix,
|
||||
OutputS3Uri: outputS3Uri,
|
||||
Config: opts.config,
|
||||
};
|
||||
|
||||
// Reject oversize input client-side. Step Functions Standard caps the
|
||||
// execution input at 256 KiB; without this check, the input bloat
|
||||
// (typically from `config.variables` containing inlined media) surfaces
|
||||
// as `States.DataLimitExceeded` 50 ms into the execution, far from the
|
||||
// caller's stack frame. Measured AFTER `deploySite` so the synthesised
|
||||
// `ProjectS3Uri` is counted (a few hundred bytes either way, but the
|
||||
// check should be against the actual wire payload).
|
||||
validateStepFunctionsInputSize(input);
|
||||
|
||||
const sfn = opts.sfn ?? new SFNClient({ region: opts.region });
|
||||
const startedAt = new Date().toISOString();
|
||||
const response = await sfn.send(
|
||||
new StartExecutionCommand({
|
||||
stateMachineArn: opts.stateMachineArn,
|
||||
name: executionName,
|
||||
input: JSON.stringify(input),
|
||||
}),
|
||||
);
|
||||
|
||||
if (!response.executionArn) {
|
||||
throw new Error("[renderToLambda] StartExecution returned no executionArn");
|
||||
}
|
||||
|
||||
return {
|
||||
renderId: executionName,
|
||||
executionArn: response.executionArn,
|
||||
bucketName: opts.bucketName,
|
||||
stateMachineArn: opts.stateMachineArn,
|
||||
outputS3Uri,
|
||||
projectS3Uri: site.projectS3Uri,
|
||||
startedAt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { SerializableDistributedRenderConfig } from "../events.js";
|
||||
import {
|
||||
InvalidConfigError,
|
||||
MAX_STEP_FUNCTIONS_INPUT_BYTES,
|
||||
validateDistributedRenderConfig,
|
||||
validateStepFunctionsInputSize,
|
||||
validateVariablesPayload,
|
||||
} from "./validateConfig.js";
|
||||
|
||||
const VALID: SerializableDistributedRenderConfig = {
|
||||
fps: 30,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
format: "mp4",
|
||||
};
|
||||
|
||||
describe("validateDistributedRenderConfig", () => {
|
||||
it("returns the same reference on the happy path", () => {
|
||||
expect(validateDistributedRenderConfig(VALID)).toBe(VALID);
|
||||
});
|
||||
|
||||
it("accepts optional fields when valid", () => {
|
||||
const cfg: SerializableDistributedRenderConfig = {
|
||||
...VALID,
|
||||
codec: "h265",
|
||||
quality: "high",
|
||||
crf: 18,
|
||||
chunkSize: 240,
|
||||
maxParallelChunks: 16,
|
||||
runtimeCap: "lambda",
|
||||
hdrMode: "force-sdr",
|
||||
videoFrameFormat: "png",
|
||||
};
|
||||
expect(validateDistributedRenderConfig(cfg)).toBe(cfg);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["null config", null as unknown as SerializableDistributedRenderConfig, "config"],
|
||||
[
|
||||
"wrong fps",
|
||||
{ ...VALID, fps: 25 as 24 | 30 | 60 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.fps",
|
||||
],
|
||||
[
|
||||
"non-integer width",
|
||||
{ ...VALID, width: 1280.5 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.width",
|
||||
],
|
||||
[
|
||||
"odd width (yuv420p parity)",
|
||||
{ ...VALID, width: 1281 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.width",
|
||||
],
|
||||
[
|
||||
"out-of-range height",
|
||||
{ ...VALID, height: 8000 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.height",
|
||||
],
|
||||
[
|
||||
"unsupported format",
|
||||
{
|
||||
...VALID,
|
||||
format: "gif",
|
||||
} as unknown as SerializableDistributedRenderConfig,
|
||||
"config.format",
|
||||
],
|
||||
[
|
||||
"codec with non-mp4 format",
|
||||
{ ...VALID, format: "mov", codec: "h264" } satisfies SerializableDistributedRenderConfig,
|
||||
"config.codec",
|
||||
],
|
||||
[
|
||||
"unknown codec",
|
||||
{
|
||||
...VALID,
|
||||
codec: "av1",
|
||||
} as unknown as SerializableDistributedRenderConfig,
|
||||
"config.codec",
|
||||
],
|
||||
[
|
||||
"crf + bitrate together",
|
||||
{ ...VALID, crf: 18, bitrate: "10M" } satisfies SerializableDistributedRenderConfig,
|
||||
"config.crf",
|
||||
],
|
||||
[
|
||||
"crf out of range",
|
||||
{ ...VALID, crf: 60 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.crf",
|
||||
],
|
||||
[
|
||||
"malformed bitrate",
|
||||
{ ...VALID, bitrate: "fast" } satisfies SerializableDistributedRenderConfig,
|
||||
"config.bitrate",
|
||||
],
|
||||
[
|
||||
"unsupported videoFrameFormat",
|
||||
{
|
||||
...VALID,
|
||||
videoFrameFormat: "webp",
|
||||
} as unknown as SerializableDistributedRenderConfig,
|
||||
"config.videoFrameFormat",
|
||||
],
|
||||
[
|
||||
"non-positive chunkSize",
|
||||
{ ...VALID, chunkSize: 0 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.chunkSize",
|
||||
],
|
||||
[
|
||||
"chunkSize over Lambda ceiling",
|
||||
{ ...VALID, chunkSize: 9999 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.chunkSize",
|
||||
],
|
||||
[
|
||||
"maxParallelChunks 0",
|
||||
{ ...VALID, maxParallelChunks: 0 } satisfies SerializableDistributedRenderConfig,
|
||||
"config.maxParallelChunks",
|
||||
],
|
||||
[
|
||||
"unknown runtimeCap",
|
||||
{
|
||||
...VALID,
|
||||
runtimeCap: "azure",
|
||||
} as unknown as SerializableDistributedRenderConfig,
|
||||
"config.runtimeCap",
|
||||
],
|
||||
[
|
||||
"force-hdr rejected",
|
||||
{
|
||||
...VALID,
|
||||
hdrMode: "force-hdr",
|
||||
} as unknown as SerializableDistributedRenderConfig,
|
||||
"config.hdrMode",
|
||||
],
|
||||
])("rejects %s with field=%s", (_label, input, expectedField) => {
|
||||
try {
|
||||
validateDistributedRenderConfig(input);
|
||||
throw new Error("expected validateDistributedRenderConfig to throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe(expectedField);
|
||||
expect((err as InvalidConfigError).name).toBe("InvalidConfigError");
|
||||
}
|
||||
});
|
||||
|
||||
describe("variables", () => {
|
||||
it("accepts a plain JSON object", () => {
|
||||
const cfg: SerializableDistributedRenderConfig = {
|
||||
...VALID,
|
||||
variables: {
|
||||
title: "Hello",
|
||||
accent: "#ff0000",
|
||||
nested: { items: [1, 2, 3], visible: true, note: null },
|
||||
},
|
||||
};
|
||||
expect(validateDistributedRenderConfig(cfg)).toBe(cfg);
|
||||
});
|
||||
|
||||
it("rejects variables that's an array, not a plain object", () => {
|
||||
try {
|
||||
validateDistributedRenderConfig({
|
||||
...VALID,
|
||||
variables: [1, 2, 3] as unknown as Record<string, unknown>,
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects functions inside variables", () => {
|
||||
try {
|
||||
validateVariablesPayload({ greet: () => "hi" });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.greet");
|
||||
expect((err as Error).message).toMatch(/function/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects undefined leaves (silently dropped by JSON.stringify)", () => {
|
||||
try {
|
||||
validateVariablesPayload({ title: "x", maybe: undefined });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.maybe");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects BigInt values", () => {
|
||||
try {
|
||||
validateVariablesPayload({ count: 9_007_199_254_740_993n });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.count");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects NaN / Infinity numbers", () => {
|
||||
try {
|
||||
validateVariablesPayload({ ratio: Number.NaN });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.ratio");
|
||||
}
|
||||
try {
|
||||
validateVariablesPayload({ ratio: Number.POSITIVE_INFINITY });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.ratio");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects Symbols", () => {
|
||||
try {
|
||||
validateVariablesPayload({ id: Symbol("hi") });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.id");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects Date instances (non-plain objects)", () => {
|
||||
try {
|
||||
validateVariablesPayload({ when: new Date("2026-01-01") });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.when");
|
||||
expect((err as Error).message).toMatch(/Date|non-plain/);
|
||||
}
|
||||
});
|
||||
|
||||
it("walks into arrays and reports nested paths", () => {
|
||||
try {
|
||||
validateVariablesPayload({ items: ["a", { broken: () => 1 }] });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as InvalidConfigError).field).toBe("config.variables.items[1].broken");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects circular references with a typed error instead of stack-overflowing", () => {
|
||||
const cyclic: Record<string, unknown> = { title: "x" };
|
||||
cyclic.self = cyclic;
|
||||
try {
|
||||
validateVariablesPayload(cyclic);
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as Error).message).toMatch(/circular/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects cycles via arrays too", () => {
|
||||
const arr: unknown[] = ["a"];
|
||||
arr.push(arr);
|
||||
try {
|
||||
validateVariablesPayload({ items: arr });
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
expect((err as Error).message).toMatch(/circular/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("round-trips through JSON for the validated set", () => {
|
||||
const variables = {
|
||||
title: "Personalised render",
|
||||
scene: { intro: { lines: ["one", "two"], delay: 0.5 } },
|
||||
tags: ["alpha", "beta"],
|
||||
active: true,
|
||||
nothing: null,
|
||||
};
|
||||
validateVariablesPayload(variables);
|
||||
const round = JSON.parse(JSON.stringify(variables));
|
||||
expect(round).toEqual(variables);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateStepFunctionsInputSize", () => {
|
||||
it("accepts inputs under the 256 KiB cap", () => {
|
||||
const input = {
|
||||
ProjectS3Uri: "s3://bucket/sites/abc/project.tar.gz",
|
||||
Config: { fps: 30, width: 1280, height: 720, format: "mp4" },
|
||||
};
|
||||
expect(() => validateStepFunctionsInputSize(input)).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects inputs over the 256 KiB cap with a message that names the byte count", () => {
|
||||
// Build a variables blob that pushes the serialised input over the cap.
|
||||
// 256 KiB ÷ 2 bytes per char × 1 char per byte for ASCII; pad to 260 KiB
|
||||
// worth of payload so the serialiser overhead is dwarfed.
|
||||
const huge = "x".repeat(260 * 1024);
|
||||
const input = {
|
||||
ProjectS3Uri: "s3://bucket/sites/abc/project.tar.gz",
|
||||
Config: {
|
||||
fps: 30,
|
||||
width: 1280,
|
||||
height: 720,
|
||||
format: "mp4",
|
||||
variables: { blob: huge },
|
||||
},
|
||||
};
|
||||
try {
|
||||
validateStepFunctionsInputSize(input);
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(InvalidConfigError);
|
||||
const msg = (err as Error).message;
|
||||
expect(msg).toMatch(/256/);
|
||||
// Names the actual byte count so users see how far over the cap they are.
|
||||
const serialized = JSON.stringify(input);
|
||||
const expectedBytes = Buffer.byteLength(serialized, "utf8");
|
||||
expect(msg).toContain(String(expectedBytes));
|
||||
// Pointer to the docs section on URL'ing assets.
|
||||
expect(msg).toMatch(/templates-on-lambda/);
|
||||
}
|
||||
});
|
||||
|
||||
it("MAX_STEP_FUNCTIONS_INPUT_BYTES is 256 KiB", () => {
|
||||
expect(MAX_STEP_FUNCTIONS_INPUT_BYTES).toBe(256 * 1024);
|
||||
});
|
||||
|
||||
it("rejects non-JSON-serializable roots with a clear error", () => {
|
||||
// A top-level function reference makes JSON.stringify return undefined.
|
||||
expect(() => validateStepFunctionsInputSize(() => "boom")).toThrow(/not JSON-serializable/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Client-side validation for the AWS Lambda adapter.
|
||||
*
|
||||
* The cloud-agnostic config-shape validation (`validateDistributedRenderConfig`,
|
||||
* `validateVariablesPayload`, `InvalidConfigError`) lives in
|
||||
* `@hyperframes/producer/distributed` and is shared with the other adapters.
|
||||
* This module re-exports those and adds the one piece specific to Step
|
||||
* Functions: the 256 KiB Standard-workflow execution-input size cap.
|
||||
*/
|
||||
|
||||
import { InvalidConfigError } from "@hyperframes/producer/distributed";
|
||||
|
||||
export {
|
||||
InvalidConfigError,
|
||||
validateDistributedRenderConfig,
|
||||
validateVariablesPayload,
|
||||
} from "@hyperframes/producer/distributed";
|
||||
|
||||
/**
|
||||
* Hard cap on Step Functions Standard workflow execution input — 256 KiB per
|
||||
* the AWS limits page. Express workflows cap at 32 KiB; the render stack runs
|
||||
* Standard for execution-history visibility, so the larger limit applies. The
|
||||
* cap is on the entire serialized input, not just the variables, because
|
||||
* users hit it at the wire boundary regardless of which field caused the
|
||||
* bloat.
|
||||
*
|
||||
* Specific to Step Functions Standard. Other workflow runtimes (Temporal,
|
||||
* Express SFN, Cloud Workflows, raw Lambda invoke) have different caps; don't
|
||||
* reuse this constant for those without confirming the limit.
|
||||
*/
|
||||
export const MAX_STEP_FUNCTIONS_INPUT_BYTES = 256 * 1024;
|
||||
|
||||
/** Pointer to the docs section that explains the URL-your-assets convention. */
|
||||
const LARGE_VARIABLES_DOCS_URL =
|
||||
"https://hyperframes.heygen.com/deploy/templates-on-lambda#working-with-large-variables";
|
||||
|
||||
/**
|
||||
* Validate that the serialized Step Functions execution input fits inside the
|
||||
* 256 KiB Standard-workflow cap. Measured in UTF-8 bytes (the format Step
|
||||
* Functions uses on the wire) — JS strings count UTF-16 code units, which
|
||||
* under-reports for any multi-byte character.
|
||||
*
|
||||
* Throws {@link InvalidConfigError} with a clear message naming the actual
|
||||
* byte count, the cap, and a pointer to the "working with large variables"
|
||||
* docs section, so users hit the limit at the SDK boundary with actionable
|
||||
* guidance instead of as a `States.DataLimitExceeded` 50 ms into the
|
||||
* execution.
|
||||
*/
|
||||
// fallow-ignore-next-line complexity
|
||||
export function validateStepFunctionsInputSize(input: unknown): void {
|
||||
let serialized: string | undefined;
|
||||
try {
|
||||
serialized = JSON.stringify(input);
|
||||
} catch (err) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
`Step Functions execution input is not JSON-serializable: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
if (serialized === undefined) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
"Step Functions execution input is not JSON-serializable (JSON.stringify returned undefined). " +
|
||||
"Check that all fields, including config.variables, are plain JSON values.",
|
||||
);
|
||||
}
|
||||
const byteLength = Buffer.byteLength(serialized, "utf8");
|
||||
if (byteLength > MAX_STEP_FUNCTIONS_INPUT_BYTES) {
|
||||
throw new InvalidConfigError(
|
||||
"config",
|
||||
`Step Functions execution input is ${byteLength} bytes, which exceeds the ` +
|
||||
`${MAX_STEP_FUNCTIONS_INPUT_BYTES}-byte (256 KiB) limit for Standard workflows. ` +
|
||||
`Variables are for typed data (strings, numbers, structured records); media assets ` +
|
||||
`(images, audio, video) should be passed as URL references the composition resolves ` +
|
||||
`at render time, not inlined as base64. See ${LARGE_VARIABLES_DOCS_URL} for the ` +
|
||||
`URL-your-assets convention.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": {},
|
||||
"noEmit": false,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"esModuleInterop": true,
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@hyperframes/producer": ["../producer/src/index.ts"],
|
||||
"@hyperframes/producer/distributed": ["../producer/src/distributed.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "src/**/*.test.ts", "src/**/__fixtures__/**", "scripts"]
|
||||
}
|
||||
Reference in New Issue
Block a user