chore: import upstream snapshot with attribution

docker/certification/.gitignore

@@ -0,0 +1,3 @@
+# Runtime state of a certification run — never committed.
+queue/
+timings.json

docker/certification/Dockerfile.gpu

@@ -0,0 +1,99 @@
+# syntax=docker/dockerfile:1
+# GPU certification image for vast.ai runs (#14548, epic #14541).
+#
+# scripts/vast/run-certification.mjs rents a vast.ai RTX 4090 and boots this
+# image; the onstart script it injects only clones the repo at one sha and
+# runs the packages/evidence certification chain. vast caps onstart-cmd at
+# 16 KB, so EVERYTHING heavy is baked here instead: CUDA-enabled llama-server
+# (>= build b8525, the minimum scripts/gpu-vision accepts for the
+# DeepSeek-OCR mmproj), the sha256-pinned OCR + VLM GGUFs from
+# scripts/gpu-vision/models.lock.json, Node 24 + Bun (the workspace
+# toolchain), Playwright Chromium with OS deps, tesseract, and ffmpeg.
+#
+# Secrets NEVER bake into this image — the Ed25519 signing key and any
+# storage push command are injected as instance env at create time and die
+# with the instance. Build/publish: .github/workflows/certification-image.yml
+# pushes ghcr.io/elizaos/certification-gpu:{latest,sha-<short>}.
+
+ARG CUDA_VERSION=12.4.1
+ARG UBUNTU_VERSION=22.04
+
+# --- Stage 1: build llama-server with the CUDA backend --------------------
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS llama-builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      build-essential cmake git ca-certificates libcurl4-openssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# b8525 is the exact minimum scripts/gpu-vision/lib.mjs (MIN_LLAMA_BUILD)
+# demands; every llama.cpp CI build is tagged bNNNN so the ref is stable.
+ARG LLAMA_CPP_REF=b8525
+RUN git clone --depth 1 --branch "${LLAMA_CPP_REF}" \
+      https://github.com/ggml-org/llama.cpp /opt/llama.cpp
+
+# Static-ish build (no shared ggml libs to carry over) for the two GPU
+# generations the offer filter can realistically land on: sm_86 (RTX 3090)
+# and sm_89 (RTX 4090, the default --gpu-name).
+RUN cmake -S /opt/llama.cpp -B /opt/llama.cpp/build \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DGGML_CUDA=ON \
+      -DCMAKE_CUDA_ARCHITECTURES="86;89" \
+      -DBUILD_SHARED_LIBS=OFF \
+      -DLLAMA_CURL=ON \
+    && cmake --build /opt/llama.cpp/build --target llama-server -j "$(nproc)"
+
+# --- Stage 2: runtime ------------------------------------------------------
+FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
+
+LABEL org.opencontainers.image.source="https://github.com/elizaOS/eliza" \
+      org.opencontainers.image.description="elizaOS GPU certification runner (vast.ai) — CUDA llama-server + pinned vision models + node24/bun/playwright/tesseract/ffmpeg" \
+      org.opencontainers.image.licenses="MIT"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# openssh-server: vast's `runtype ssh` instances start sshd inside the
+# container; without it the instance never leaves the loading state.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates curl git unzip xz-utils jq \
+      openssh-server \
+      tesseract-ocr \
+      ffmpeg \
+      libcurl4 libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Node 24 — the engines.node pin of the workspace (plain-node scripts like
+# scripts/gpu-vision run under it directly).
+RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    && node --version
+
+# Bun, pinned to the same version CI pins (floating canary rewrites bun.lock).
+ARG BUN_VERSION=1.3.14
+ENV BUN_INSTALL=/root/.bun
+ENV PATH="${BUN_INSTALL}/bin:${PATH}"
+RUN curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}" \
+    && bun --version
+
+# Playwright Chromium + OS deps, pinned to the packages/evidence devDep line.
+ARG PLAYWRIGHT_VERSION=1.61.0
+ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers
+RUN npx --yes "playwright@${PLAYWRIGHT_VERSION}" install --with-deps chromium \
+    && rm -rf /var/lib/apt/lists/*
+
+# CUDA llama-server from the builder.
+COPY --from=llama-builder /opt/llama.cpp/build/bin/llama-server /usr/local/bin/llama-server
+# Build gate: a broken binary (missing lib, bad link) fails here, not on a
+# billed vast instance. --version needs no GPU.
+RUN llama-server --version
+
+# Pinned GPU-vision models, fetched + sha256-verified by the repo's own
+# setup script (lockfile resolves relative to lib.mjs, so the trio is copied
+# together). ELIZA_GPU_VISION_CACHE points the cloned repo's serve.mjs at
+# this baked cache at runtime — no model download on the billed instance.
+ENV ELIZA_GPU_VISION_CACHE=/opt/eliza/gpu-vision-cache
+COPY scripts/gpu-vision/setup.mjs scripts/gpu-vision/lib.mjs scripts/gpu-vision/models.lock.json /opt/eliza/gpu-vision/
+RUN node /opt/eliza/gpu-vision/setup.mjs --with-vlm
+
+WORKDIR /workspace
+CMD ["/bin/bash"]

docker/certification/README.md

@@ -0,0 +1,151 @@
+# Single-machine parallel certification
+
+Runs the entire certification stack (epic #14541, this piece #14549) on one
+beefy Linux box in reasonable wall-clock: container-parallel CPU test lanes
+plus pipelined GPU vision work. macOS certifiers keep the native path
+(`bun run test` + `scripts/gpu-vision/serve.mjs`) — compose is the Linux/vast
+lane; the orchestrator only cares that lanes report and the queue drains.
+
+## Quick start (Linux, docker + compose v2, NVIDIA container toolkit)
+
+```bash
+# cpu tier: all test lanes containerized, per-lane timings collected
+node docker/certification/certify-parallel.mjs
+
+# full tier: cpu lanes + resident GPU vision service + queue worker
+node docker/certification/certify-parallel.mjs --tier full
+
+# knobs (also accepted by the generator)
+node docker/certification/certify-parallel.mjs --cores 32 --unit-shards 6 --e2e-shards 3
+```
+
+`certify-parallel.mjs` writes `timings.json` beside this file: per-lane exit
+codes, ISO start/finish stamps, and a flat `timings` record
+(`Record<phase, milliseconds>`) that `packages/evidence` can merge verbatim
+into `BundleMeta.timings`.
+
+## Where the lane list comes from (there is no second registry)
+
+`generate-compose-lanes.mjs` derives the `cpu`-profile services from
+`node packages/scripts/run-all-tests.mjs --plan=json --all` — the test
+runner's own plan:
+
+- each distinct `scriptName` in `summary.byScript` becomes a lane
+  (`test` → `unit`, `test:e2e` → `e2e`, `test:ui` → `ui`, …); a package adding
+  a brand-new test script surfaces as a new lane on regeneration, with zero
+  edits here;
+- the big groups are split with the runner's own `TEST_SHARD=N/M`
+  (deterministic package-dir hashing), so shard membership is never listed in
+  the compose file;
+- the plan's `cloudStep` becomes the `cloud` lane (`bun run test:cloud`), which
+  is why every other lane can pass `--no-cloud` without losing coverage.
+
+`compose.yml` is generated-then-committed. The drift gate
+(`generate-compose-lanes.mjs --check`, enforced by
+`compose-lanes.test.mjs`) fails when the committed file no longer matches
+regeneration — regenerate and commit, never hand-edit.
+
+## Cores → lane mapping
+
+Defaults (`cores=16 unit-shards=4 e2e-shards=2`) produce 10 lanes; cpusets are
+contiguous, remainder cores go to the earliest (heaviest) lanes:
+
+| Lane          | cpuset | cpus | runs                                       |
+| ------------- | ------ | ---- | ------------------------------------------ |
+| unit-1of4     | 0-1    | 2    | `TEST_SCRIPT_FILTER=^test$ TEST_SHARD=1/4` |
+| unit-2of4     | 2-3    | 2    | shard 2/4                                  |
+| unit-3of4     | 4-5    | 2    | shard 3/4                                  |
+| unit-4of4     | 6-7    | 2    | shard 4/4                                  |
+| e2e-1of2      | 8-9    | 2    | `TEST_SCRIPT_FILTER=^test:e2e$ 1/2`        |
+| e2e-2of2      | 10-11  | 2    | shard 2/2                                  |
+| integration   | 12     | 1    | `^test:integration$`                       |
+| live          | 13     | 1    | `^test:live$` (pr lane → guarded skips)    |
+| ui            | 14     | 1    | `^test:ui$`                                |
+| cloud         | 15     | 1    | `bun run test:cloud`                       |
+
+Sizing guidance: ~1 lane per 1.5–2 physical cores. 32 cores → `--cores 32
+--unit-shards 6 --e2e-shards 3` is a good starting point; more than ~8 unit
+shards hits diminishing returns because `bun install` + `build:core` dominate
+short shards. Memory: budget ~4 GiB per unit/e2e lane (vitest + browsers),
+~8 GiB for `cloud`.
+
+Each lane gets the repo **read-only** at `/repo`, syncs it into its private
+`scratch-lane-*` volume (incremental after the first run), installs through
+the shared `build-cache` volume (`BUN_INSTALL_CACHE_DIR=/cache/bun`), and runs
+its lane command via `lane-entry.sh`.
+
+## GPU profile: one service owns the device
+
+```
+capture lanes ──(job files)──▶ queue/pending ──▶ gpu-queue-worker ──HTTP──▶ gpu-vision
+                                                       │                    (both llama-servers)
+                                                       └──▶ queue/results/<id>.json
+```
+
+- **Doctrine: containers do NOT share the GPU; they share the GPU *service*
+  over HTTP.** Exactly one container (`gpu-vision`) reserves the device and
+  runs both resident `llama-server` instances (`gpu-entry.sh`); lanes and the
+  worker talk to `http://gpu-vision:8090` (Unlimited-OCR) and `:8091`
+  (Qwen3-VL) with `--parallel N` slots each.
+- **VRAM budget ~6–10 GiB at Q4** (fits a single RTX 4090-class card with
+  headroom; both models resident, from `scripts/gpu-vision/models.lock.json`):
+
+  | Blob                          | Bytes      |
+  | ----------------------------- | ---------- |
+  | Unlimited-OCR Q4_K_M          | ~1.95 GiB  |
+  | Unlimited-OCR mmproj F16      | ~0.81 GiB  |
+  | Qwen3-VL-4B Q4_K_M            | ~2.50 GiB  |
+  | Qwen3-VL mmproj F16           | ~0.84 GiB  |
+  | KV cache + compute (2× serve) | ~1–4 GiB   |
+
+- **MPS is the escape hatch, not the default.** The two llama-server processes
+  in the one `gpu-vision` container time-slice the device, which is fine for a
+  drain-the-queue workload. If kernel-level concurrency between resident
+  processes ever becomes the bottleneck, enable NVIDIA MPS on the host —
+  do not hand the device to more containers. MIG is N/A on consumer cards.
+- Models mount read-only from `${ELIZA_GPU_MODELS_DIR:-~/.cache/eliza/gpu-vision}`
+  — populate with `node scripts/gpu-vision/setup.mjs --with-vlm` (hash-pinned).
+
+## Pipelined queue (filesystem, no redis)
+
+Producers drop `{ id?, model: "ocr"|"vlm", request, imagePath? }` JSON into
+`queue/pending/` (`enqueueJob()` from `queue-worker.mjs` does this atomically
+with max-pending backpressure). The worker claims by rename, POSTs
+`request` to the model's `/v1/chat/completions` (inlining `imagePath` as a
+data URI at the `queue:image` placeholder), and writes
+`queue/results/<id>.json` — `ok` (with response + duration), `failed` (with
+reason), or `skipped`. Analysis therefore finishes shortly after the last
+capture instead of serializing behind it.
+
+Degradation is explicit: while the service is unreachable jobs bounce back to
+pending and retry; past `--drain-after-ms` (default 120 s) the worker drains
+every pending job to an honest `skipped` record naming the outage. All state
+transitions are pure functions in `queue-lib.mjs` with unit tests.
+
+## Exit codes (`certify-parallel.mjs`)
+
+| Code | Meaning                                                             |
+| ---- | ------------------------------------------------------------------- |
+| 0    | all lanes exited 0 (and queue drained at `--tier full`)             |
+| 1    | one or more lanes failed (timings.json still written)               |
+| 4    | `EXIT_NO_DOCKER` — serial native fallback printed, nothing ran      |
+| 5    | `EXIT_DRIFT` — committed compose.yml stale vs the live plan         |
+| 6    | `EXIT_TIMEOUT` — `--timeout-min` elapsed with lanes still running   |
+
+## Image
+
+Services reference `ghcr.io/elizaos/certification-gpu` (override with
+`ELIZA_CERT_IMAGE` / `ELIZA_CERT_GPU_IMAGE`), the prebuilt CUDA + bun +
+node + playwright + llama-server image owned by the vast.ai runner work
+(#14548) — this directory consumes that image and never defines a competing
+Dockerfile. Lane containers additionally expect `rsync` and `git`;
+`lane-entry.sh` falls back to a slow full copy and says so if rsync is absent.
+
+## Tests
+
+`bun test docker/certification` (outside workspace discovery; CI runs it as an
+explicit step): generator drift, plan→lane derivation, cpuset math, YAML/
+`docker compose config` validation, queue state machine, real worker process
+against a stub HTTP service (success/failure/backpressure/drain-to-skip), and
+orchestrator degradation (no docker → exit 4). The full containerized run is
+the owner-gated on-Linux acceptance for #14549.

docker/certification/certify-parallel.mjs

@@ -0,0 +1,349 @@
+#!/usr/bin/env node
+/**
+ * Single-machine parallel certification orchestrator (#14549): regenerates the
+ * lane topology from the live test plan, brings the compose cpu profile (and,
+ * at --tier full, the gpu profile) up, collects per-lane exit codes and wall
+ * times from docker, and writes a timings.json whose `timings` record is
+ * BundleMeta.timings-compatible (packages/evidence: Record<phase, ms>) so a
+ * certification bundle can absorb it verbatim.
+ *
+ * Degradation is deterministic and loud, never silent: no docker → the serial
+ * native fallback is printed and the process exits with EXIT_NO_DOCKER;
+ * default-params drift between compose.yml and regeneration → EXIT_DRIFT with
+ * the regenerate command; lane failure → EXIT_LANE_FAILED after timings are
+ * still written; wall-clock timeout → EXIT_TIMEOUT. macOS certifiers keep the
+ * native path (bun run test + scripts/gpu-vision/serve.mjs) — compose is the
+ * Linux/vast lane.
+ *
+ * Usage:
+ *   node docker/certification/certify-parallel.mjs [--tier cpu|full]
+ *     [--cores N] [--unit-shards N] [--e2e-shards N] [--gpu-parallel N]
+ *     [--timeout-min N] [--out timings.json] [--keep-up] [--dry-run]
+ */
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { MODEL_SETS, parseArgs } from "../../scripts/gpu-vision/lib.mjs";
+import {
+  buildComposeModel,
+  paramsAreDefault,
+  renderCompose,
+  resolveParams,
+} from "./compose-lanes-lib.mjs";
+import { composePath, readPlan } from "./generate-compose-lanes.mjs";
+
+export const EXIT_LANE_FAILED = 1;
+export const EXIT_NO_DOCKER = 4;
+export const EXIT_DRIFT = 5;
+export const EXIT_TIMEOUT = 6;
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const POLL_MS = 5000;
+
+function log(message) {
+  process.stdout.write(`[certify-parallel] ${message}\n`);
+}
+
+function dockerAvailable() {
+  // Timed probe: a wedged docker daemon/desktop reads as unavailable (and the
+  // serial fallback prints) rather than hanging the orchestrator at step zero.
+  const probe = spawnSync("docker", ["compose", "version"], {
+    encoding: "utf8",
+    timeout: 15_000,
+  });
+  return !probe.error && probe.status === 0;
+}
+
+function compose(fileArg, args, { capture = false } = {}) {
+  const result = spawnSync(
+    "docker",
+    ["compose", "-f", fileArg, "--project-directory", here, ...args],
+    {
+      encoding: "utf8",
+      stdio: capture ? ["ignore", "pipe", "pipe"] : "inherit",
+      maxBuffer: 64 * 1024 * 1024,
+    },
+  );
+  if (result.error) throw result.error;
+  return result;
+}
+
+/** `docker compose ps -a --format json` emits one JSON object per line. */
+function composePs(fileArg, profiles) {
+  const result = compose(
+    fileArg,
+    [
+      ...profiles.flatMap((p) => ["--profile", p]),
+      "ps",
+      "-a",
+      "--format",
+      "json",
+    ],
+    {
+      capture: true,
+    },
+  );
+  if (result.status !== 0) {
+    throw new Error(
+      `[certify-parallel] docker compose ps failed:\n${result.stderr}`,
+    );
+  }
+  return result.stdout
+    .split("\n")
+    .filter((line) => line.trim().length > 0)
+    .map((line) => JSON.parse(line));
+}
+
+function inspectContainer(id) {
+  const result = spawnSync("docker", ["inspect", id], {
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) {
+    throw new Error(
+      `[certify-parallel] docker inspect ${id} failed:\n${result.stderr}`,
+    );
+  }
+  return JSON.parse(result.stdout)[0];
+}
+
+async function queueDrained(queueDir) {
+  for (const sub of ["pending", "processing"]) {
+    const entries = await fs.readdir(path.join(queueDir, sub)).catch((err) => {
+      if (err.code === "ENOENT") return [];
+      throw err;
+    });
+    if (entries.filter((name) => name.endsWith(".json")).length > 0)
+      return false;
+  }
+  return true;
+}
+
+/**
+ * Assemble the timings report. `timings` is deliberately a flat
+ * Record<string, number> of milliseconds so packages/evidence can merge it
+ * verbatim into BundleMeta.timings; everything richer (exit codes, ISO
+ * stamps, skips) lives beside it.
+ */
+export function buildTimingsReport({
+  tier,
+  params,
+  startedAt,
+  finishedAt,
+  lanes,
+  timedOut,
+}) {
+  const timings = { wall: finishedAt.getTime() - startedAt.getTime() };
+  for (const [lane, record] of Object.entries(lanes)) {
+    timings[`lane:${lane}`] = record.durationMs;
+  }
+  const skipped = [];
+  if (tier === "cpu") {
+    skipped.push({
+      lane: "gpu",
+      reason: "tier=cpu — gpu profile not started; queue jobs remain pending",
+    });
+  }
+  return {
+    schema: 1,
+    tier,
+    params,
+    startedAt: startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    wallClockMs: timings.wall,
+    timings,
+    lanes,
+    skipped,
+    timedOut,
+  };
+}
+
+function printSerialFallback() {
+  process.stdout.write(
+    [
+      "[certify-parallel] docker compose is not available on this host.",
+      "Falling back is manual and serial — run the native path instead:",
+      "  bun run test                                # all cpu lanes, serial",
+      "  node scripts/gpu-vision/setup.mjs --with-vlm  # pinned model blobs",
+      "  node scripts/gpu-vision/serve.mjs             # resident OCR llama-server",
+      "  node scripts/gpu-vision/serve.mjs --vlm       # resident Qwen3-VL llama-server",
+      "This is the supported macOS certifier path; compose is the Linux/vast lane.",
+      `Exiting with code ${EXIT_NO_DOCKER} (EXIT_NO_DOCKER) so callers can tell "degraded" from "failed".`,
+      "",
+    ].join("\n"),
+  );
+}
+
+async function main() {
+  const { flags } = parseArgs(process.argv.slice(2), {
+    booleans: ["keep-up", "dry-run"],
+  });
+  const params = resolveParams(flags);
+  const tier = flags.tier === undefined ? "cpu" : flags.tier;
+  if (tier !== "cpu" && tier !== "full") {
+    throw new Error(`[certify-parallel] --tier must be cpu|full, got: ${tier}`);
+  }
+  const timeoutMin =
+    flags["timeout-min"] === undefined ? 90 : Number(flags["timeout-min"]);
+  if (!Number.isFinite(timeoutMin) || timeoutMin <= 0) {
+    throw new Error(
+      `[certify-parallel] --timeout-min must be positive, got: ${flags["timeout-min"]}`,
+    );
+  }
+  const outPath = path.resolve(
+    flags.out === undefined ? path.join(here, "timings.json") : flags.out,
+  );
+
+  if (!dockerAvailable()) {
+    printSerialFallback();
+    process.exit(EXIT_NO_DOCKER);
+  }
+
+  log("deriving lanes from run-all-tests.mjs --plan=json --all");
+  const model = buildComposeModel(readPlan(), params, MODEL_SETS);
+  const text = renderCompose(model);
+
+  // The committed compose.yml is the reviewed artifact; with default params it
+  // must match regeneration or the board is being certified against a stale
+  // topology. Non-default params legitimately diverge and use a temp file.
+  if (paramsAreDefault(params)) {
+    const committed = await fs.readFile(composePath, "utf8").catch(() => null);
+    if (committed !== text) {
+      process.stderr.write(
+        "[certify-parallel] compose.yml drift: committed file does not match the current plan.\n" +
+          "  Regenerate and commit: node docker/certification/generate-compose-lanes.mjs\n",
+      );
+      process.exit(EXIT_DRIFT);
+    }
+  }
+  const composeFile = path.join(
+    await fs.mkdtemp(path.join(os.tmpdir(), "eliza-cert-")),
+    "compose.yml",
+  );
+  await fs.writeFile(composeFile, text, "utf8");
+
+  const profiles = tier === "full" ? ["cpu", "gpu"] : ["cpu"];
+  const laneServices = model.services.filter(
+    (service) => service.lane !== undefined,
+  );
+  log(
+    `tier=${tier} lanes=${laneServices.length} (${laneServices.map((s) => s.lane).join(", ")})`,
+  );
+
+  if (flags["dry-run"] === true) {
+    log(`dry run — compose file at ${composeFile}, not starting containers`);
+    return;
+  }
+
+  const queueDir = path.join(
+    here,
+    process.env.ELIZA_CERT_QUEUE_HOST ?? "queue",
+  );
+  await fs.mkdir(queueDir, { recursive: true });
+
+  const startedAt = new Date();
+  const up = compose(composeFile, [
+    ...profiles.flatMap((p) => ["--profile", p]),
+    "up",
+    "-d",
+  ]);
+  if (up.status !== 0) {
+    throw new Error(`[certify-parallel] docker compose up exited ${up.status}`);
+  }
+
+  // Poll until every cpu lane container exits (and, at full tier, the GPU
+  // queue drains — pipelining means the queue usually empties moments after
+  // the last capture lane finishes).
+  const deadline = Date.now() + timeoutMin * 60_000;
+  const lanes = {};
+  let timedOut = false;
+  for (;;) {
+    const ps = composePs(composeFile, profiles);
+    const byService = new Map(ps.map((entry) => [entry.Service, entry]));
+    let allExited = true;
+    for (const service of laneServices) {
+      const entry = byService.get(service.name);
+      if (entry?.State !== "exited") {
+        allExited = false;
+        continue;
+      }
+      if (lanes[service.lane]) continue;
+      const inspected = inspectContainer(entry.ID);
+      const started = new Date(inspected.State.StartedAt);
+      const finished = new Date(inspected.State.FinishedAt);
+      lanes[service.lane] = {
+        service: service.name,
+        exitCode: inspected.State.ExitCode,
+        startedAt: started.toISOString(),
+        finishedAt: finished.toISOString(),
+        durationMs: Math.max(0, finished.getTime() - started.getTime()),
+      };
+      log(
+        `lane ${service.lane}: exit ${inspected.State.ExitCode} in ${lanes[service.lane].durationMs}ms`,
+      );
+    }
+    const drained = tier === "full" ? await queueDrained(queueDir) : true;
+    if (allExited && drained) break;
+    if (Date.now() > deadline) {
+      timedOut = true;
+      break;
+    }
+    await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+  }
+  const finishedAt = new Date();
+
+  if (flags["keep-up"] !== true) {
+    compose(composeFile, [
+      ...profiles.flatMap((p) => ["--profile", p]),
+      "down",
+      "--remove-orphans",
+    ]);
+  }
+
+  const report = buildTimingsReport({
+    tier,
+    params,
+    startedAt,
+    finishedAt,
+    lanes,
+    timedOut,
+  });
+  await fs.writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+  log(`timings written to ${outPath}`);
+
+  if (timedOut) {
+    const missing = laneServices
+      .filter((s) => !lanes[s.lane])
+      .map((s) => s.lane);
+    process.stderr.write(
+      `[certify-parallel] TIMEOUT after ${timeoutMin}min — lanes still running: ${missing.join(", ") || "(none; queue not drained)"}\n`,
+    );
+    process.exit(EXIT_TIMEOUT);
+  }
+  const failed = Object.entries(lanes).filter(
+    ([, record]) => record.exitCode !== 0,
+  );
+  if (failed.length > 0) {
+    process.stderr.write(
+      `[certify-parallel] ${failed.length} lane(s) failed: ${failed
+        .map(([lane, record]) => `${lane} (exit ${record.exitCode})`)
+        .join(", ")}\n`,
+    );
+    process.exit(EXIT_LANE_FAILED);
+  }
+  log(`all ${laneServices.length} lanes passed in ${report.wallClockMs}ms`);
+}
+
+if (
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main().catch((err) => {
+    process.stderr.write(`${err.stack || err.message}\n`);
+    process.exit(1);
+  });
+}

docker/certification/certify-parallel.test.mjs

@@ -0,0 +1,145 @@
+/**
+ * Tests for certify-parallel.mjs (#14549) degradation behavior, run against
+ * the real process: a docker-less PATH must print the serial native fallback
+ * and exit with the distinct EXIT_NO_DOCKER code (callers can tell "degraded"
+ * from "failed"), and --dry-run with docker present must derive the full lane
+ * set from the live plan without starting containers. The containerized happy
+ * path is the owner-gated on-Linux acceptance run (issue #14549).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { buildTimingsReport } from "./certify-parallel.mjs";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(here, "../..");
+const orchestratorPath = path.join(here, "certify-parallel.mjs");
+
+function runOrchestrator(args, env) {
+  return spawnSync(process.execPath, [orchestratorPath, ...args], {
+    cwd: repoRoot,
+    encoding: "utf8",
+    env,
+  });
+}
+
+describe("certify-parallel degradation", () => {
+  test("no docker on PATH → serial fallback instructions + EXIT_NO_DOCKER(4)", () => {
+    // A bare PATH with only the node binary's dir: `docker` cannot resolve.
+    const nodeDir = path.dirname(process.execPath);
+    const result = runOrchestrator([], { ...process.env, PATH: nodeDir });
+    expect(result.status).toBe(4);
+    expect(result.stdout).toContain("docker compose is not available");
+    expect(result.stdout).toContain("bun run test");
+    expect(result.stdout).toContain("scripts/gpu-vision/serve.mjs");
+    expect(result.stdout).toContain("EXIT_NO_DOCKER");
+  });
+
+  test("rejects an unknown tier before touching docker", () => {
+    const result = runOrchestrator(["--tier", "warp"], process.env);
+    expect(result.status).toBe(1);
+    expect(result.stderr).toContain("--tier must be cpu|full");
+  });
+});
+
+describe("certify-parallel dry run (requires docker client)", () => {
+  // Timed probe: a wedged Docker Desktop must skip these legs, not hang them.
+  const probe = spawnSync("docker", ["compose", "version"], {
+    encoding: "utf8",
+    timeout: 15_000,
+  });
+  const dockerUsable = !probe.error && probe.status === 0;
+
+  test.if(dockerUsable)(
+    "derives the lane set from the live plan and stops before containers",
+    () => {
+      const result = runOrchestrator(["--dry-run"], process.env);
+      expect(result.stderr).toBe("");
+      expect(result.status).toBe(0);
+      expect(result.stdout).toMatch(/tier=cpu lanes=10/);
+      expect(result.stdout).toContain("unit-1of4");
+      expect(result.stdout).toContain("cloud");
+      expect(result.stdout).toContain("dry run");
+    },
+    120_000,
+  );
+
+  test.if(dockerUsable)(
+    "non-default params bypass the committed-file drift gate",
+    () => {
+      const result = runOrchestrator(
+        [
+          "--dry-run",
+          "--cores",
+          "9",
+          "--unit-shards",
+          "3",
+          "--e2e-shards",
+          "1",
+        ],
+        process.env,
+      );
+      expect(result.status).toBe(0);
+      expect(result.stdout).toMatch(/lanes=8/);
+    },
+    120_000,
+  );
+
+  test.if(!dockerUsable)(
+    "docker client unavailable — dry-run legs skipped (stated)",
+    () => {
+      // Not silent: this test names the skipped coverage. The drift gate and
+      // no-docker degradation above still ran.
+      expect(dockerUsable).toBe(false);
+    },
+  );
+});
+
+describe("buildTimingsReport", () => {
+  test("timings is a flat Record<phase, ms> (BundleMeta.timings-compatible)", () => {
+    const report = buildTimingsReport({
+      tier: "cpu",
+      params: { cores: 16, unitShards: 4, e2eShards: 2, gpuParallel: 4 },
+      startedAt: new Date("2026-07-06T10:00:00Z"),
+      finishedAt: new Date("2026-07-06T10:20:00Z"),
+      lanes: {
+        "unit-1of4": {
+          service: "lane-unit-1of4",
+          exitCode: 0,
+          startedAt: "2026-07-06T10:00:05Z",
+          finishedAt: "2026-07-06T10:15:00Z",
+          durationMs: 895_000,
+        },
+      },
+      timedOut: false,
+    });
+    expect(report.schema).toBe(1);
+    expect(report.wallClockMs).toBe(20 * 60 * 1000);
+    expect(report.timings).toEqual({
+      wall: 20 * 60 * 1000,
+      "lane:unit-1of4": 895_000,
+    });
+    for (const value of Object.values(report.timings)) {
+      expect(typeof value).toBe("number");
+    }
+    // cpu tier records the gpu profile as an honest skip, never as absent.
+    expect(report.skipped).toEqual([
+      {
+        lane: "gpu",
+        reason: "tier=cpu — gpu profile not started; queue jobs remain pending",
+      },
+    ]);
+    expect(
+      buildTimingsReport({
+        tier: "full",
+        params: {},
+        startedAt: new Date(0),
+        finishedAt: new Date(1),
+        lanes: {},
+        timedOut: true,
+      }),
+    ).toMatchObject({ skipped: [], timedOut: true, wallClockMs: 1 });
+  });
+});

docker/certification/compose-lanes-lib.mjs

@@ -0,0 +1,399 @@
+/**
+ * Pure derivation of the parallel-certification compose topology from the
+ * test-runner's own plan (#14549). The single source of lane truth is
+ * `packages/scripts/run-all-tests.mjs --plan=json --all`: each distinct
+ * `scriptName` in the plan's `summary.byScript` becomes one lane (optionally
+ * split into TEST_SHARD slices the runner itself understands), and the plan's
+ * `cloudStep` becomes the dedicated cloud lane — there is deliberately no
+ * second lane registry to drift from the runner.
+ *
+ * Everything here is side-effect-free (no fs, no spawn, no clock) so the shard
+ * math, cpuset allocation, and YAML rendering are unit-testable and the
+ * committed compose.yml is byte-reproducible: generate-compose-lanes.mjs feeds
+ * in the live plan and writes the rendered text; the drift test regenerates
+ * and compares. Rendering is a hand-rolled emitter over a fixed key order —
+ * a generic YAML library would not guarantee byte stability across versions.
+ */
+
+/** Generator parameters baked into the committed compose.yml. Fixed defaults
+ * (not `os.cpus()`) so regeneration is machine-independent. */
+export const DEFAULT_PARAMS = Object.freeze({
+  cores: 16,
+  unitShards: 4,
+  e2eShards: 2,
+  gpuParallel: 4,
+});
+
+/** Matches serve.mjs CONTEXT_SIZE — one context budget for native and compose. */
+export const GPU_CONTEXT_SIZE = 8192;
+
+const CPU_IMAGE =
+  "${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}";
+const GPU_IMAGE =
+  "${ELIZA_CERT_GPU_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}";
+const REPO_MOUNT = "${ELIZA_REPO_ROOT:-../..}:/repo:ro";
+const QUEUE_MOUNT = "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue";
+const MODELS_MOUNT =
+  "${ELIZA_GPU_MODELS_DIR:-~/.cache/eliza/gpu-vision}:/models:ro";
+
+/** Positive-integer guard for generator parameters; a zero/NaN shard count
+ * must fail here, not surface as a broken TEST_SHARD spec inside a container. */
+export function assertPositiveInteger(value, label) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(
+      `[compose-lanes] ${label} must be a positive integer, got: ${value}`,
+    );
+  }
+  return value;
+}
+
+/** Lane base name for a plan scriptName: `test` → `unit`, `test:e2e` → `e2e`. */
+export function laneBaseName(scriptName) {
+  if (scriptName === "test") return "unit";
+  return scriptName
+    .replace(/^test:/, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-");
+}
+
+/**
+ * Derive the lane list from a `--plan=json --all` document. Script lanes run
+ * `run-all-tests.mjs` filtered to one scriptName (sharded via the runner's own
+ * TEST_SHARD for the big groups); the plan's cloudStep command becomes the
+ * `cloud` lane so `--no-cloud` on the script lanes loses no coverage.
+ */
+export function deriveLanes(plan, params = DEFAULT_PARAMS) {
+  const byScript = plan?.summary?.byScript;
+  if (
+    !byScript ||
+    typeof byScript !== "object" ||
+    Object.keys(byScript).length === 0
+  ) {
+    throw new Error(
+      "[compose-lanes] plan has no summary.byScript — expected run-all-tests.mjs --plan=json --all output",
+    );
+  }
+  assertPositiveInteger(params.unitShards, "unitShards");
+  assertPositiveInteger(params.e2eShards, "e2eShards");
+
+  const lanes = [];
+  for (const scriptName of Object.keys(byScript).sort((a, b) =>
+    a.localeCompare(b),
+  )) {
+    const base = laneBaseName(scriptName);
+    const total =
+      scriptName === "test"
+        ? params.unitShards
+        : scriptName === "test:e2e"
+          ? params.e2eShards
+          : 1;
+    for (let index = 1; index <= total; index++) {
+      lanes.push({
+        name: total > 1 ? `${base}-${index}of${total}` : base,
+        kind: "run-all-tests",
+        script: scriptName,
+        shard: total > 1 ? `${index}/${total}` : null,
+      });
+    }
+  }
+  if (plan.cloudStep && typeof plan.cloudStep.command === "string") {
+    lanes.push({
+      name: "cloud",
+      kind: "command",
+      command: plan.cloudStep.command,
+      script: null,
+      shard: null,
+    });
+  }
+
+  const seen = new Set();
+  for (const lane of lanes) {
+    if (seen.has(lane.name)) {
+      throw new Error(
+        `[compose-lanes] duplicate lane name derived from plan: ${lane.name}`,
+      );
+    }
+    seen.add(lane.name);
+  }
+  return lanes;
+}
+
+/**
+ * Partition `totalCores` into contiguous cpuset ranges, one per lane, in lane
+ * order. The remainder cores go to the earliest lanes — derivation order puts
+ * the heavy `unit`/`e2e` shards first, so they absorb the extras. More lanes
+ * than cores is a sizing error the caller must resolve (fewer shards or more
+ * cores), never silent oversubscription of a pinned set.
+ */
+export function allocateCpusets(totalCores, laneCount) {
+  assertPositiveInteger(totalCores, "cores");
+  assertPositiveInteger(laneCount, "lane count");
+  if (laneCount > totalCores) {
+    throw new Error(
+      `[compose-lanes] ${laneCount} lanes need at least ${laneCount} cores but only ${totalCores} configured — ` +
+        "lower --unit-shards/--e2e-shards or raise --cores",
+    );
+  }
+  const base = Math.floor(totalCores / laneCount);
+  const remainder = totalCores % laneCount;
+  const allocations = [];
+  let next = 0;
+  for (let i = 0; i < laneCount; i++) {
+    const size = base + (i < remainder ? 1 : 0);
+    const first = next;
+    const last = next + size - 1;
+    next = last + 1;
+    allocations.push({
+      cpuset: size === 1 ? `${first}` : `${first}-${last}`,
+      cpus: size,
+    });
+  }
+  return allocations;
+}
+
+/** The shell command a script lane runs inside its container (via lane-entry.sh).
+ * `--no-cloud` is safe because the plan-derived `cloud` lane carries that step. */
+export function laneCommand(lane) {
+  if (lane.kind === "command") return lane.command;
+  return "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1";
+}
+
+/** Compose interpolation treats `$` specially; env values that must reach the
+ * container verbatim (anchored regexes like `^test$`) need `$$`. */
+export function escapeComposeValue(value) {
+  return value.replace(/\$/g, "$$$$");
+}
+
+/**
+ * Assemble the full compose document model: cpu-profile lane services with
+ * cpuset pinning + cpu quotas, the single GPU-owning vision service (both
+ * resident llama-servers live in that one container — lanes share the service
+ * over HTTP, never the device), and the queue worker.
+ */
+export function buildComposeModel(plan, params, modelSets) {
+  const lanes = deriveLanes(plan, params);
+  const allocations = allocateCpusets(params.cores, lanes.length);
+  assertPositiveInteger(params.gpuParallel, "gpuParallel");
+  const ocr = modelSets?.ocr;
+  const vlm = modelSets?.vlm;
+  if (!ocr?.files?.model?.name || !vlm?.files?.model?.name) {
+    throw new Error(
+      "[compose-lanes] MODEL_SETS missing ocr/vlm entries (scripts/gpu-vision/lib.mjs)",
+    );
+  }
+
+  const services = [];
+  lanes.forEach((lane, i) => {
+    const environment = {
+      BUN_INSTALL_CACHE_DIR: "/cache/bun",
+      ELIZA_CERT_QUEUE_DIR: "/queue",
+      LANE_COMMAND: laneCommand(lane),
+      LANE_NAME: lane.name,
+    };
+    if (lane.kind === "run-all-tests") {
+      environment.TEST_LANE = "pr";
+      environment.TEST_SCRIPT_FILTER = `^${lane.script}$`;
+      if (lane.shard) environment.TEST_SHARD = lane.shard;
+    }
+    services.push({
+      name: `lane-${lane.name}`,
+      lane: lane.name,
+      image: CPU_IMAGE,
+      profiles: ["cpu"],
+      command: ["bash", "/repo/docker/certification/lane-entry.sh"],
+      cpuset: allocations[i].cpuset,
+      cpus: allocations[i].cpus,
+      environment,
+      volumes: [
+        REPO_MOUNT,
+        `scratch-lane-${lane.name}:/work`,
+        "build-cache:/cache",
+        QUEUE_MOUNT,
+      ],
+    });
+  });
+
+  services.push({
+    name: "gpu-vision",
+    image: GPU_IMAGE,
+    profiles: ["gpu"],
+    command: ["bash", "/repo/docker/certification/gpu-entry.sh"],
+    environment: {
+      LLAMA_CONTEXT: `${GPU_CONTEXT_SIZE}`,
+      LLAMA_PARALLEL: `${params.gpuParallel}`,
+      OCR_MMPROJ: `/models/ocr/${ocr.files.mmproj.name}`,
+      OCR_MODEL: `/models/ocr/${ocr.files.model.name}`,
+      OCR_PORT: "8090",
+      VLM_MMPROJ: `/models/vlm/${vlm.files.mmproj.name}`,
+      VLM_MODEL: `/models/vlm/${vlm.files.model.name}`,
+      VLM_PORT: "8091",
+    },
+    volumes: [REPO_MOUNT, MODELS_MOUNT],
+    gpu: true,
+    healthcheck: {
+      test: [
+        "CMD-SHELL",
+        "curl -fsS http://127.0.0.1:8090/health && curl -fsS http://127.0.0.1:8091/health",
+      ],
+      interval: "10s",
+      timeout: "5s",
+      retries: 60,
+      start_period: "120s",
+    },
+  });
+
+  services.push({
+    name: "gpu-queue-worker",
+    image: GPU_IMAGE,
+    profiles: ["gpu"],
+    command: [
+      "node",
+      "/repo/docker/certification/queue-worker.mjs",
+      "--jobs",
+      "/queue",
+      "--service",
+      "ocr=http://gpu-vision:8090",
+      "--service",
+      "vlm=http://gpu-vision:8091",
+    ],
+    dependsOnHealthy: ["gpu-vision"],
+    volumes: [REPO_MOUNT, QUEUE_MOUNT],
+  });
+
+  return {
+    lanes,
+    services,
+    params,
+    scripts: Object.keys(plan.summary.byScript).sort(),
+  };
+}
+
+function yamlValue(value) {
+  if (typeof value === "number") return `${value}`;
+  // Quote every string: compose values include `${…}` defaults, `^…$$` regexes,
+  // and `no` / `0-3` scalars YAML would otherwise coerce; uniform quoting keeps
+  // the emitter trivially deterministic.
+  return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+}
+
+function renderList(lines, indent, items) {
+  for (const item of items) lines.push(`${indent}- ${yamlValue(item)}`);
+}
+
+/** Render the compose model to the exact committed YAML text. */
+export function renderCompose(model) {
+  const { params, scripts } = model;
+  const lines = [];
+  lines.push(
+    "# Single-machine parallel certification stack (#14549, epic #14541).",
+    "# GENERATED by docker/certification/generate-compose-lanes.mjs from the test",
+    "# runner's own plan (`run-all-tests.mjs --plan=json --all`) — the lane",
+    "# services mirror that plan's script groups; do not edit by hand.",
+    "#   Regenerate: node docker/certification/generate-compose-lanes.mjs",
+    "#   Drift check: node docker/certification/generate-compose-lanes.mjs --check",
+    `# Params: cores=${params.cores} unit-shards=${params.unitShards} e2e-shards=${params.e2eShards} gpu-parallel=${params.gpuParallel}`,
+    `# Plan scripts: ${scripts.join(", ")} (+ cloud step)`,
+    "",
+    "name: eliza-certification",
+    "",
+    "services:",
+  );
+
+  for (const service of model.services) {
+    lines.push(`  ${service.name}:`);
+    lines.push(`    image: ${service.image}`);
+    if (service.profiles) {
+      lines.push("    profiles:");
+      renderList(lines, "      ", service.profiles);
+    }
+    if (service.dependsOnHealthy) {
+      lines.push("    depends_on:");
+      for (const dep of service.dependsOnHealthy) {
+        lines.push(`      ${dep}:`);
+        lines.push("        condition: service_healthy");
+      }
+    }
+    lines.push("    command:");
+    renderList(lines, "      ", service.command);
+    if (service.cpuset !== undefined) {
+      lines.push(`    cpuset: ${yamlValue(service.cpuset)}`);
+      lines.push(`    cpus: ${service.cpus}`);
+    }
+    if (service.environment && Object.keys(service.environment).length > 0) {
+      lines.push("    environment:");
+      for (const key of Object.keys(service.environment).sort((a, b) =>
+        a.localeCompare(b),
+      )) {
+        lines.push(
+          `      ${key}: ${yamlValue(escapeComposeValue(service.environment[key]))}`,
+        );
+      }
+    }
+    if (service.volumes) {
+      lines.push("    volumes:");
+      renderList(lines, "      ", service.volumes);
+    }
+    if (service.gpu) {
+      lines.push(
+        "    deploy:",
+        "      resources:",
+        "        reservations:",
+        "          devices:",
+        '            - driver: "nvidia"',
+        "              count: 1",
+        "              capabilities:",
+        '                - "gpu"',
+      );
+    }
+    if (service.healthcheck) {
+      lines.push("    healthcheck:");
+      lines.push("      test:");
+      renderList(lines, "        ", service.healthcheck.test);
+      lines.push(`      interval: ${service.healthcheck.interval}`);
+      lines.push(`      timeout: ${service.healthcheck.timeout}`);
+      lines.push(`      retries: ${service.healthcheck.retries}`);
+      lines.push(`      start_period: ${service.healthcheck.start_period}`);
+    }
+    lines.push('    restart: "no"');
+    lines.push("");
+  }
+
+  lines.push("volumes:");
+  lines.push("  build-cache: {}");
+  for (const lane of model.lanes) {
+    lines.push(`  scratch-lane-${lane.name}: {}`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+/** Parse generator/orchestrator shared CLI params on top of DEFAULT_PARAMS. */
+export function resolveParams(flags) {
+  const read = (key, fallback, label) => {
+    const raw = flags[key];
+    if (raw === undefined) return fallback;
+    const value = Number(raw);
+    return assertPositiveInteger(value, label);
+  };
+  return {
+    cores: read("cores", DEFAULT_PARAMS.cores, "--cores"),
+    unitShards: read("unit-shards", DEFAULT_PARAMS.unitShards, "--unit-shards"),
+    e2eShards: read("e2e-shards", DEFAULT_PARAMS.e2eShards, "--e2e-shards"),
+    gpuParallel: read(
+      "gpu-parallel",
+      DEFAULT_PARAMS.gpuParallel,
+      "--gpu-parallel",
+    ),
+  };
+}
+
+/** True when the resolved params match the committed defaults — the only case
+ * where the orchestrator holds the generated text against compose.yml. */
+export function paramsAreDefault(params) {
+  return (
+    params.cores === DEFAULT_PARAMS.cores &&
+    params.unitShards === DEFAULT_PARAMS.unitShards &&
+    params.e2eShards === DEFAULT_PARAMS.e2eShards &&
+    params.gpuParallel === DEFAULT_PARAMS.gpuParallel
+  );
+}

docker/certification/compose-lanes.test.mjs

@@ -0,0 +1,291 @@
+/**
+ * Tests for the compose-lane generator (#14549): pure plan→lane derivation and
+ * cpuset math against a fixture plan, byte-determinism of the renderer, the
+ * committed-compose drift gate (spawns the real generator, which runs the real
+ * run-all-tests.mjs --plan=json), and structural validation of the committed
+ * file — via a real `docker compose config` when docker is on PATH, else a
+ * YAML-parse sanity check (Bun.YAML), stated explicitly in the test output.
+ * Run with `bun test docker/certification` (outside workspace discovery, so
+ * CI carries an explicit step).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  allocateCpusets,
+  buildComposeModel,
+  DEFAULT_PARAMS,
+  deriveLanes,
+  escapeComposeValue,
+  laneBaseName,
+  paramsAreDefault,
+  renderCompose,
+  resolveParams,
+} from "./compose-lanes-lib.mjs";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(here, "../..");
+const composePath = path.join(here, "compose.yml");
+
+/** Minimal but shape-faithful --plan=json --all document. */
+const fixturePlan = {
+  summary: {
+    byScript: {
+      test: 241,
+      "test:e2e": 17,
+      "test:live": 1,
+      "test:ui": 1,
+      "test:integration": 4,
+    },
+  },
+  tasks: [],
+  skipped: [],
+  cloudStep: { label: "cloud#test", command: "bun run test:cloud" },
+};
+
+const fixtureModelSets = {
+  ocr: {
+    files: { model: { name: "ocr.gguf" }, mmproj: { name: "ocr-mmproj.gguf" } },
+  },
+  vlm: {
+    files: { model: { name: "vlm.gguf" }, mmproj: { name: "vlm-mmproj.gguf" } },
+  },
+};
+
+describe("laneBaseName", () => {
+  test("maps plan scriptNames to lane names", () => {
+    expect(laneBaseName("test")).toBe("unit");
+    expect(laneBaseName("test:e2e")).toBe("e2e");
+    expect(laneBaseName("test:integration")).toBe("integration");
+    expect(laneBaseName("test:some odd/Name")).toBe("some-odd-name");
+  });
+});
+
+describe("deriveLanes", () => {
+  test("derives sharded unit/e2e lanes, singleton lanes, and the cloud lane", () => {
+    const lanes = deriveLanes(fixturePlan, DEFAULT_PARAMS);
+    expect(lanes.map((lane) => lane.name)).toEqual([
+      "unit-1of4",
+      "unit-2of4",
+      "unit-3of4",
+      "unit-4of4",
+      "e2e-1of2",
+      "e2e-2of2",
+      "integration",
+      "live",
+      "ui",
+      "cloud",
+    ]);
+    expect(lanes[0]).toMatchObject({
+      script: "test",
+      shard: "1/4",
+      kind: "run-all-tests",
+    });
+    expect(lanes.at(-1)).toMatchObject({
+      kind: "command",
+      command: "bun run test:cloud",
+    });
+  });
+
+  test("omits the cloud lane when the plan has no cloud step", () => {
+    const lanes = deriveLanes(
+      { ...fixturePlan, cloudStep: null },
+      DEFAULT_PARAMS,
+    );
+    expect(lanes.some((lane) => lane.name === "cloud")).toBe(false);
+  });
+
+  test("a new scriptName in the plan appears as a new lane without code changes", () => {
+    const plan = {
+      ...fixturePlan,
+      summary: {
+        byScript: { ...fixturePlan.summary.byScript, "test:visual": 3 },
+      },
+    };
+    const lanes = deriveLanes(plan, DEFAULT_PARAMS);
+    expect(
+      lanes.some(
+        (lane) => lane.name === "visual" && lane.script === "test:visual",
+      ),
+    ).toBe(true);
+  });
+
+  test("rejects a plan without byScript", () => {
+    expect(() => deriveLanes({ summary: {} }, DEFAULT_PARAMS)).toThrow(
+      /byScript/,
+    );
+    expect(() => deriveLanes({}, DEFAULT_PARAMS)).toThrow(/byScript/);
+  });
+
+  test("rejects non-positive shard params", () => {
+    expect(() =>
+      deriveLanes(fixturePlan, { ...DEFAULT_PARAMS, unitShards: 0 }),
+    ).toThrow(/unitShards/);
+    expect(() =>
+      deriveLanes(fixturePlan, { ...DEFAULT_PARAMS, e2eShards: 1.5 }),
+    ).toThrow(/e2eShards/);
+  });
+});
+
+describe("allocateCpusets", () => {
+  test("partitions cores contiguously with the remainder front-loaded", () => {
+    expect(allocateCpusets(16, 10)).toEqual([
+      { cpuset: "0-1", cpus: 2 },
+      { cpuset: "2-3", cpus: 2 },
+      { cpuset: "4-5", cpus: 2 },
+      { cpuset: "6-7", cpus: 2 },
+      { cpuset: "8-9", cpus: 2 },
+      { cpuset: "10-11", cpus: 2 },
+      { cpuset: "12", cpus: 1 },
+      { cpuset: "13", cpus: 1 },
+      { cpuset: "14", cpus: 1 },
+      { cpuset: "15", cpus: 1 },
+    ]);
+  });
+
+  test("covers every core exactly once", () => {
+    for (const [cores, lanes] of [
+      [16, 10],
+      [32, 10],
+      [9, 9],
+      [64, 3],
+    ]) {
+      const allocations = allocateCpusets(cores, lanes);
+      const covered = allocations.flatMap(({ cpuset }) => {
+        const [first, last] = cpuset.split("-").map(Number);
+        const end = last ?? first;
+        return Array.from({ length: end - first + 1 }, (_, i) => first + i);
+      });
+      expect(covered).toEqual(Array.from({ length: cores }, (_, i) => i));
+      expect(allocations.reduce((sum, a) => sum + a.cpus, 0)).toBe(cores);
+    }
+  });
+
+  test("refuses more lanes than cores instead of oversubscribing pinned sets", () => {
+    expect(() => allocateCpusets(8, 10)).toThrow(/10 lanes/);
+  });
+});
+
+describe("escapeComposeValue", () => {
+  test("doubles $ so anchored regexes survive compose interpolation", () => {
+    expect(escapeComposeValue("^test$")).toBe("^test$$");
+    expect(escapeComposeValue("^test:e2e$")).toBe("^test:e2e$$");
+    expect(escapeComposeValue("no dollars")).toBe("no dollars");
+  });
+});
+
+describe("resolveParams / paramsAreDefault", () => {
+  test("defaults, overrides, and validation", () => {
+    expect(resolveParams({})).toEqual(DEFAULT_PARAMS);
+    expect(paramsAreDefault(resolveParams({}))).toBe(true);
+    const custom = resolveParams({ cores: "32", "unit-shards": "6" });
+    expect(custom.cores).toBe(32);
+    expect(custom.unitShards).toBe(6);
+    expect(paramsAreDefault(custom)).toBe(false);
+    expect(() => resolveParams({ cores: "0" })).toThrow(/--cores/);
+    expect(() => resolveParams({ "e2e-shards": "abc" })).toThrow(
+      /--e2e-shards/,
+    );
+  });
+});
+
+describe("renderCompose", () => {
+  test("is deterministic and carries pinning, mounts, and both profiles", () => {
+    const render = () =>
+      renderCompose(
+        buildComposeModel(fixturePlan, DEFAULT_PARAMS, fixtureModelSets),
+      );
+    const text = render();
+    expect(render()).toBe(text);
+    expect(text).toContain('TEST_SCRIPT_FILTER: "^test$$"');
+    expect(text).toContain('TEST_SHARD: "1/4"');
+    expect(text).toContain('cpuset: "0-1"');
+    expect(text).toContain("${ELIZA_REPO_ROOT:-../..}:/repo:ro");
+    expect(text).toContain("scratch-lane-unit-1of4:/work");
+    expect(text).toContain("build-cache:/cache");
+    expect(text).toContain("gpu-vision:");
+    expect(text).toContain("gpu-queue-worker:");
+    expect(text).toContain('OCR_MODEL: "/models/ocr/ocr.gguf"');
+    // Referenced image is owned by #14548 — this stack consumes, never builds.
+    expect(text).toContain("ghcr.io/elizaos/certification-gpu");
+  });
+
+  test("rejects missing model sets", () => {
+    expect(() => buildComposeModel(fixturePlan, DEFAULT_PARAMS, {})).toThrow(
+      /MODEL_SETS/,
+    );
+  });
+});
+
+describe("committed compose.yml", () => {
+  test("drift gate: committed file matches regeneration from the live plan", () => {
+    const result = spawnSync(
+      process.execPath,
+      [path.join(here, "generate-compose-lanes.mjs"), "--check"],
+      { cwd: repoRoot, encoding: "utf8" },
+    );
+    expect(result.stderr).toBe("");
+    expect(result.status).toBe(0);
+    expect(result.stdout).toContain("matches regeneration");
+  }, 120_000);
+
+  test("validates: real `docker compose config` when available, else YAML parse", () => {
+    const text = readFileSync(composePath, "utf8");
+    // A wedged Docker Desktop can hang even `docker compose version`; a
+    // timed-out probe degrades to the YAML branch instead of failing here.
+    const probe = spawnSync("docker", ["compose", "version"], {
+      encoding: "utf8",
+      timeout: 15_000,
+    });
+    const dockerUsable = !probe.error && probe.status === 0;
+    if (dockerUsable) {
+      const result = spawnSync(
+        "docker",
+        [
+          "compose",
+          "-f",
+          composePath,
+          "--project-directory",
+          here,
+          "--profile",
+          "cpu",
+          "--profile",
+          "gpu",
+          "config",
+          "--quiet",
+        ],
+        { encoding: "utf8", timeout: 60_000 },
+      );
+      if (result.status !== 0) {
+        throw new Error(`docker compose config failed:\n${result.stderr}`);
+      }
+      console.log(
+        "[compose-lanes.test] validated with real `docker compose config`",
+      );
+      return;
+    }
+    // Docker-less fallback (stated, not silent): structural YAML sanity only.
+    console.log(
+      "[compose-lanes.test] docker unavailable — YAML-parse sanity check instead",
+    );
+    const doc = Bun.YAML.parse(text);
+    expect(doc.name).toBe("eliza-certification");
+    expect(Object.keys(doc.services).length).toBeGreaterThanOrEqual(12);
+    expect(doc.services["gpu-vision"].profiles).toEqual(["gpu"]);
+    expect(doc.volumes["build-cache"]).toBeDefined();
+  }, 120_000);
+
+  test("committed lane env survives YAML + compose interpolation round-trip", () => {
+    const doc = Bun.YAML.parse(readFileSync(composePath, "utf8"));
+    const lane = doc.services["lane-unit-1of4"];
+    // In-file value is the $$-escaped form; compose unescapes it at runtime
+    // (verified against a live daemon: container env receives `^test$`).
+    expect(lane.environment.TEST_SCRIPT_FILTER).toBe("^test$$");
+    expect(lane.environment.TEST_SHARD).toBe("1/4");
+    expect(lane.cpus).toBe(2);
+    expect(lane.restart).toBe("no");
+  });
+});

docker/certification/compose.yml

@@ -0,0 +1,316 @@
+# Single-machine parallel certification stack (#14549, epic #14541).
+# GENERATED by docker/certification/generate-compose-lanes.mjs from the test
+# runner's own plan (`run-all-tests.mjs --plan=json --all`) — the lane
+# services mirror that plan's script groups; do not edit by hand.
+#   Regenerate: node docker/certification/generate-compose-lanes.mjs
+#   Drift check: node docker/certification/generate-compose-lanes.mjs --check
+# Params: cores=16 unit-shards=4 e2e-shards=2 gpu-parallel=4
+# Plan scripts: test, test:e2e, test:integration, test:live, test:ui (+ cloud step)
+
+name: eliza-certification
+
+services:
+  lane-unit-1of4:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "0-1"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "unit-1of4"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test$$"
+      TEST_SHARD: "1/4"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-unit-1of4:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-unit-2of4:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "2-3"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "unit-2of4"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test$$"
+      TEST_SHARD: "2/4"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-unit-2of4:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-unit-3of4:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "4-5"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "unit-3of4"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test$$"
+      TEST_SHARD: "3/4"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-unit-3of4:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-unit-4of4:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "6-7"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "unit-4of4"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test$$"
+      TEST_SHARD: "4/4"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-unit-4of4:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-e2e-1of2:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "8-9"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "e2e-1of2"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test:e2e$$"
+      TEST_SHARD: "1/2"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-e2e-1of2:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-e2e-2of2:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "10-11"
+    cpus: 2
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "e2e-2of2"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test:e2e$$"
+      TEST_SHARD: "2/2"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-e2e-2of2:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-integration:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "12"
+    cpus: 1
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "integration"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test:integration$$"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-integration:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-live:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "13"
+    cpus: 1
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "live"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test:live$$"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-live:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-ui:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "14"
+    cpus: 1
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "node packages/scripts/run-all-tests.mjs --no-cloud --min-tasks=1"
+      LANE_NAME: "ui"
+      TEST_LANE: "pr"
+      TEST_SCRIPT_FILTER: "^test:ui$$"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-ui:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  lane-cloud:
+    image: ${ELIZA_CERT_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "cpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/lane-entry.sh"
+    cpuset: "15"
+    cpus: 1
+    environment:
+      BUN_INSTALL_CACHE_DIR: "/cache/bun"
+      ELIZA_CERT_QUEUE_DIR: "/queue"
+      LANE_COMMAND: "bun run test:cloud"
+      LANE_NAME: "cloud"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "scratch-lane-cloud:/work"
+      - "build-cache:/cache"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+  gpu-vision:
+    image: ${ELIZA_CERT_GPU_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "gpu"
+    command:
+      - "bash"
+      - "/repo/docker/certification/gpu-entry.sh"
+    environment:
+      LLAMA_CONTEXT: "8192"
+      LLAMA_PARALLEL: "4"
+      OCR_MMPROJ: "/models/ocr/mmproj-Unlimited-OCR-F16.gguf"
+      OCR_MODEL: "/models/ocr/Unlimited-OCR-Q4_K_M.gguf"
+      OCR_PORT: "8090"
+      VLM_MMPROJ: "/models/vlm/mmproj-Qwen3VL-4B-Instruct-F16.gguf"
+      VLM_MODEL: "/models/vlm/Qwen3VL-4B-Instruct-Q4_K_M.gguf"
+      VLM_PORT: "8091"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "${ELIZA_GPU_MODELS_DIR:-~/.cache/eliza/gpu-vision}:/models:ro"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: "nvidia"
+              count: 1
+              capabilities:
+                - "gpu"
+    healthcheck:
+      test:
+        - "CMD-SHELL"
+        - "curl -fsS http://127.0.0.1:8090/health && curl -fsS http://127.0.0.1:8091/health"
+      interval: 10s
+      timeout: 5s
+      retries: 60
+      start_period: 120s
+    restart: "no"
+
+  gpu-queue-worker:
+    image: ${ELIZA_CERT_GPU_IMAGE:-ghcr.io/elizaos/certification-gpu:latest}
+    profiles:
+      - "gpu"
+    depends_on:
+      gpu-vision:
+        condition: service_healthy
+    command:
+      - "node"
+      - "/repo/docker/certification/queue-worker.mjs"
+      - "--jobs"
+      - "/queue"
+      - "--service"
+      - "ocr=http://gpu-vision:8090"
+      - "--service"
+      - "vlm=http://gpu-vision:8091"
+    volumes:
+      - "${ELIZA_REPO_ROOT:-../..}:/repo:ro"
+      - "${ELIZA_CERT_QUEUE_HOST:-./queue}:/queue"
+    restart: "no"
+
+volumes:
+  build-cache: {}
+  scratch-lane-unit-1of4: {}
+  scratch-lane-unit-2of4: {}
+  scratch-lane-unit-3of4: {}
+  scratch-lane-unit-4of4: {}
+  scratch-lane-e2e-1of2: {}
+  scratch-lane-e2e-2of2: {}
+  scratch-lane-integration: {}
+  scratch-lane-live: {}
+  scratch-lane-ui: {}
+  scratch-lane-cloud: {}

docker/certification/generate-compose-lanes.mjs

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * Generates docker/certification/compose.yml from the test runner's live plan
+ * (#14549). Lane services derive from `run-all-tests.mjs --plan=json --all` —
+ * the runner's own lane/shard model — plus the pinned GPU model set from
+ * scripts/gpu-vision; there is no lane list in this directory to fall out of
+ * sync. The output is committed, and `--check` fails when the committed file
+ * no longer matches a fresh regeneration (the drift gate the tests run).
+ *
+ * Usage:
+ *   node docker/certification/generate-compose-lanes.mjs [--check] [--print]
+ *     [--cores N] [--unit-shards N] [--e2e-shards N] [--gpu-parallel N]
+ */
+
+import { spawnSync } from "node:child_process";
+import {
+  closeSync,
+  mkdtempSync,
+  openSync,
+  readFileSync,
+  rmSync,
+} from "node:fs";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { MODEL_SETS, parseArgs } from "../../scripts/gpu-vision/lib.mjs";
+import {
+  buildComposeModel,
+  renderCompose,
+  resolveParams,
+} from "./compose-lanes-lib.mjs";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+export const repoRoot = path.resolve(here, "../..");
+export const composePath = path.join(here, "compose.yml");
+
+/**
+ * Run the planner with lane/shard/filter env stripped so the derived topology
+ * never depends on what happens to be exported in the invoking shell.
+ */
+export function readPlan() {
+  const env = { ...process.env };
+  for (const key of [
+    "TEST_SHARD",
+    "TEST_PACKAGE_FILTER",
+    "TEST_SCRIPT_FILTER",
+    "TEST_START_AT",
+  ]) {
+    delete env[key];
+  }
+  env.TEST_LANE = "pr";
+  // The planner exits before an OS pipe drains, truncating large plans at one
+  // pipe buffer — a file fd is flushed synchronously, so stdout goes to a temp
+  // file instead of being captured over a pipe.
+  const outDir = mkdtempSync(path.join(os.tmpdir(), "eliza-cert-plan-"));
+  const outPath = path.join(outDir, "plan.json");
+  const outFd = openSync(outPath, "w");
+  try {
+    const result = spawnSync(
+      process.execPath,
+      [
+        path.join(repoRoot, "packages/scripts/run-all-tests.mjs"),
+        "--plan=json",
+        "--all",
+      ],
+      {
+        cwd: repoRoot,
+        env,
+        encoding: "utf8",
+        stdio: ["ignore", outFd, "pipe"],
+      },
+    );
+    if (result.error) throw result.error;
+    if (result.status !== 0) {
+      throw new Error(
+        `[compose-lanes] run-all-tests.mjs --plan=json exited ${result.status}:\n${result.stderr}`,
+      );
+    }
+    return JSON.parse(readFileSync(outPath, "utf8"));
+  } finally {
+    closeSync(outFd);
+    rmSync(outDir, { recursive: true, force: true });
+  }
+}
+
+/** Full generation pipeline: plan → model → YAML text. */
+export function generateComposeText(params) {
+  return renderCompose(buildComposeModel(readPlan(), params, MODEL_SETS));
+}
+
+async function main() {
+  const { flags } = parseArgs(process.argv.slice(2), {
+    booleans: ["check", "print"],
+  });
+  const params = resolveParams(flags);
+  const text = generateComposeText(params);
+
+  if (flags.print === true) {
+    process.stdout.write(text);
+    return;
+  }
+
+  if (flags.check === true) {
+    const committed = await fs.readFile(composePath, "utf8").catch((err) => {
+      if (err.code === "ENOENT") return null;
+      throw err;
+    });
+    if (committed !== text) {
+      process.stderr.write(
+        "[compose-lanes] DRIFT: docker/certification/compose.yml does not match regeneration.\n" +
+          "  Regenerate and commit: node docker/certification/generate-compose-lanes.mjs\n",
+      );
+      process.exit(1);
+    }
+    process.stdout.write("[compose-lanes] compose.yml matches regeneration\n");
+    return;
+  }
+
+  await fs.writeFile(composePath, text, "utf8");
+  process.stdout.write(
+    `[compose-lanes] wrote ${path.relative(repoRoot, composePath)}\n`,
+  );
+}
+
+// Library consumers (certify-parallel.mjs, tests) import the exports above
+// without triggering a generation run.
+if (
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main().catch((err) => {
+    process.stderr.write(`${err.stack || err.message}\n`);
+    process.exit(1);
+  });
+}

docker/certification/gpu-entry.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Entrypoint for the single GPU-owning gpu-vision container: both resident
+# llama-server instances (Unlimited-OCR + Qwen3-VL) run here, in one container,
+# so lanes share the GPU *service* over HTTP rather than the device itself.
+# Either server dying takes the container down (wait -n under set -e) so
+# compose surfaces the failure instead of half the queue silently stalling.
+# Model paths/ports arrive as environment baked by generate-compose-lanes.mjs
+# from scripts/gpu-vision (models.lock.json pins the exact blobs).
+set -euo pipefail
+
+: "${OCR_MODEL:?OCR_MODEL is required}"
+: "${OCR_MMPROJ:?OCR_MMPROJ is required}"
+: "${VLM_MODEL:?VLM_MODEL is required}"
+: "${VLM_MMPROJ:?VLM_MMPROJ is required}"
+
+PARALLEL="${LLAMA_PARALLEL:-4}"
+CONTEXT="${LLAMA_CONTEXT:-8192}"
+
+for blob in "$OCR_MODEL" "$OCR_MMPROJ" "$VLM_MODEL" "$VLM_MMPROJ"; do
+  if [ ! -f "$blob" ]; then
+    echo "[gpu-entry] missing model blob: $blob — populate the models mount with scripts/gpu-vision/setup.mjs --with-vlm" >&2
+    exit 1
+  fi
+done
+
+echo "[gpu-entry] launching Unlimited-OCR llama-server on :${OCR_PORT:-8090} (parallel=${PARALLEL}, ctx=${CONTEXT})"
+llama-server -m "$OCR_MODEL" --mmproj "$OCR_MMPROJ" -c "$CONTEXT" \
+  --parallel "$PARALLEL" --host 0.0.0.0 --port "${OCR_PORT:-8090}" &
+
+echo "[gpu-entry] launching Qwen3-VL llama-server on :${VLM_PORT:-8091} (parallel=${PARALLEL}, ctx=${CONTEXT})"
+llama-server -m "$VLM_MODEL" --mmproj "$VLM_MMPROJ" -c "$CONTEXT" \
+  --parallel "$PARALLEL" --host 0.0.0.0 --port "${VLM_PORT:-8091}" &
+
+# First server to exit ends the container; a healthy stack never reaches here.
+wait -n
+echo "[gpu-entry] a llama-server exited — tearing the gpu-vision service down" >&2
+exit 1

docker/certification/lane-entry.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Entrypoint for the cpu-profile lane containers: sync the read-only /repo
+# mount into this lane's writable /work scratch (tests write node_modules,
+# dist, .turbo, tmp files), install through the shared /cache bun cache, then
+# exec the lane command the generator baked into LANE_COMMAND. The scratch
+# volume persists between runs, so the rsync is incremental after the first.
+set -euo pipefail
+
+: "${LANE_NAME:?LANE_NAME is required}"
+: "${LANE_COMMAND:?LANE_COMMAND is required}"
+
+mkdir -p /work/repo
+if command -v rsync >/dev/null 2>&1; then
+  # node_modules is reproduced by bun install below; syncing it from the host
+  # would both be slow and leak host-platform binaries into the container.
+  rsync -a --delete --exclude=node_modules/ --exclude=.turbo/ /repo/ /work/repo/
+else
+  echo "[lane-entry] ${LANE_NAME}: rsync missing from image — falling back to a full copy (add rsync to ghcr.io/elizaos/certification-gpu)" >&2
+  rm -rf /work/repo
+  mkdir -p /work/repo
+  cp -a /repo/. /work/repo/
+fi
+
+cd /work/repo
+export BUN_INSTALL_CACHE_DIR="${BUN_INSTALL_CACHE_DIR:-/cache/bun}"
+
+echo "[lane-entry] ${LANE_NAME}: bun install (cache: ${BUN_INSTALL_CACHE_DIR})"
+bun install --frozen-lockfile
+
+echo "[lane-entry] ${LANE_NAME}: ${LANE_COMMAND}"
+exec bash -c "${LANE_COMMAND}"

docker/certification/queue-lib.d.mts

@@ -0,0 +1,55 @@
+/**
+ * Type surface for the plain-node queue kernel in `queue-lib.mjs` (#14549).
+ * Exists so TS consumers — notably the cross-layer drift guard
+ * `packages/evidence/src/queue/parity.test.ts` — see a typed contract instead of
+ * `any`; keep it in lockstep with the module's exports.
+ */
+
+export const QUEUE_DIRS: readonly string[];
+
+export interface QueueLimits {
+  maxPending: number;
+  drainAfterMs: number;
+  pollMs: number;
+  requestTimeoutMs: number;
+}
+export const DEFAULT_LIMITS: Readonly<QueueLimits>;
+
+export interface WorkerState {
+  unreachableSince: number | null;
+  draining: boolean;
+}
+
+export class QueueJobInvalidError extends Error {
+  reason: string;
+  constructor(reason: string);
+}
+
+export function parseJob(
+  raw: string,
+  knownModels: readonly string[],
+): Record<string, unknown>;
+export function claimOrder(fileNames: readonly string[]): string[];
+export function decideEnqueue(
+  pendingCount: number,
+  maxPending: number,
+): { accept: true } | { accept: false; reason: string };
+export function createWorkerState(): WorkerState;
+export function onServiceUnreachable(
+  state: WorkerState,
+  nowMs: number,
+  drainAfterMs: number,
+): WorkerState;
+export function onServiceOk(): WorkerState;
+export function shouldSkipJob(state: WorkerState): boolean;
+export function resultRecord(
+  job: Record<string, unknown>,
+  outcome: { status: string; [key: string]: unknown },
+  completedAtIso: string,
+): Record<string, unknown>;
+export const IMAGE_PLACEHOLDER: string;
+export function resolveImagePlaceholders(
+  request: unknown,
+  dataUri: string,
+): unknown;
+export function makeJobId(nowMs: number, entropy: string): string;

docker/certification/queue-lib.mjs

@@ -0,0 +1,194 @@
+/**
+ * Pure state machine for the pipelined GPU job queue (#14549). Capture lanes
+ * enqueue JSON job files as images/videos are produced; queue-worker.mjs (the
+ * only fs/network shell) consumes them concurrently and calls the resident
+ * gpu-vision service, so analysis finishes shortly after the last capture
+ * instead of serializing behind it. Every decision the worker makes — job
+ * validity, claim order, backpressure, unreachable-service draining, result
+ * record shape — is a pure function here so the transitions are unit-testable
+ * without a GPU, a filesystem, or a clock.
+ *
+ * Queue layout under the jobs root (one directory, no redis/sqlite):
+ *   pending/<id>.json     enqueued job, claimed by rename into processing/
+ *   processing/<id>.json  claimed job (rename is the atomic claim)
+ *   done/<id>.json        consumed job file, kept for provenance
+ *   results/<id>.json     result record: ok | failed | skipped — a skipped
+ *                         record is an honest degradation marker, never a
+ *                         fabricated success.
+ */
+
+export const QUEUE_DIRS = Object.freeze([
+  "pending",
+  "processing",
+  "done",
+  "results",
+]);
+
+export const DEFAULT_LIMITS = Object.freeze({
+  // Enqueue backpressure: producers block/fail loudly instead of burying an
+  // unreachable worker under thousands of jobs.
+  maxPending: 256,
+  // Consecutive unreachability past this window flips the worker into drain
+  // mode: pending jobs become skip records instead of waiting forever.
+  drainAfterMs: 120_000,
+  pollMs: 500,
+  requestTimeoutMs: 300_000,
+});
+
+export class QueueJobInvalidError extends Error {
+  constructor(reason) {
+    super(`[queue] invalid job: ${reason}`);
+    this.name = "QueueJobInvalidError";
+    this.reason = reason;
+  }
+}
+
+/**
+ * Parse an untrusted job file into a typed job. Throws QueueJobInvalidError
+ * with the concrete defect — the worker records a `failed` result for the id
+ * (when recoverable) rather than guessing at a fake-valid job (J3).
+ */
+export function parseJob(raw, knownModels) {
+  let job;
+  try {
+    job = JSON.parse(raw);
+  } catch {
+    // error-policy:J3 malformed JSON in an untrusted job file becomes a typed
+    // invalid signal the worker turns into a failed result record.
+    throw new QueueJobInvalidError("not valid JSON");
+  }
+  if (typeof job !== "object" || job === null)
+    throw new QueueJobInvalidError("not an object");
+  if (typeof job.id !== "string" || job.id.length === 0) {
+    throw new QueueJobInvalidError("missing id");
+  }
+  if (!/^[a-zA-Z0-9._-]+$/.test(job.id)) {
+    throw new QueueJobInvalidError(`id has unsafe characters: ${job.id}`);
+  }
+  if (typeof job.model !== "string" || !knownModels.includes(job.model)) {
+    throw new QueueJobInvalidError(
+      `model must be one of ${knownModels.join("|")}, got: ${job.model}`,
+    );
+  }
+  if (typeof job.request !== "object" || job.request === null) {
+    throw new QueueJobInvalidError(
+      "missing request body (OpenAI chat.completions shape)",
+    );
+  }
+  if (job.imagePath !== undefined) {
+    if (typeof job.imagePath !== "string" || job.imagePath.length === 0) {
+      throw new QueueJobInvalidError(
+        "imagePath must be a non-empty string when present",
+      );
+    }
+    if (job.imagePath.includes("..") || job.imagePath.startsWith("/")) {
+      throw new QueueJobInvalidError(
+        "imagePath must be relative to the jobs root, without ..",
+      );
+    }
+  }
+  return job;
+}
+
+/** Oldest-first claim order. Ids are enqueue-timestamp-prefixed, so a plain
+ * lexicographic sort of filenames is the arrival order. */
+export function claimOrder(fileNames) {
+  return [...fileNames]
+    .filter((name) => name.endsWith(".json"))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/** Backpressure decision for enqueue: accept or refuse-with-reason. Refusal is
+ * an explicit failure for the producer to handle — never a dropped job. */
+export function decideEnqueue(pendingCount, maxPending) {
+  if (pendingCount >= maxPending) {
+    return {
+      accept: false,
+      reason: `backpressure: ${pendingCount} pending >= max ${maxPending}`,
+    };
+  }
+  return { accept: true };
+}
+
+/** Initial worker connectivity state. */
+export function createWorkerState() {
+  return { unreachableSince: null, draining: false };
+}
+
+/**
+ * Transition on a failed service call (network error / timeout). The first
+ * failure stamps `unreachableSince`; once the outage outlasts `drainAfterMs`
+ * the worker enters drain mode and starts writing skip records.
+ */
+export function onServiceUnreachable(state, nowMs, drainAfterMs) {
+  const since = state.unreachableSince ?? nowMs;
+  return {
+    unreachableSince: since,
+    draining: state.draining || nowMs - since >= drainAfterMs,
+  };
+}
+
+/** Transition on a successful service contact: full reset out of drain mode. */
+export function onServiceOk() {
+  return createWorkerState();
+}
+
+/** In drain mode every job is skipped (honest degradation) instead of retried. */
+export function shouldSkipJob(state) {
+  return state.draining;
+}
+
+/**
+ * Shape of the result record written beside the consumed job. Exactly one of
+ * ok/failed/skipped; `skipped` carries the drain reason so downstream readers
+ * can distinguish "GPU never analyzed this" from "analysis said nothing".
+ */
+export function resultRecord(job, outcome, completedAtIso) {
+  if (!["ok", "failed", "skipped"].includes(outcome.status)) {
+    throw new Error(`[queue] unknown outcome status: ${outcome.status}`);
+  }
+  const record = {
+    schema: 1,
+    id: job.id,
+    model: job.model,
+    status: outcome.status,
+    completedAt: completedAtIso,
+  };
+  if (outcome.status === "ok") {
+    record.durationMs = outcome.durationMs;
+    record.response = outcome.response;
+  } else {
+    record.reason = outcome.reason;
+  }
+  return record;
+}
+
+/**
+ * Substitute the `queue:image` placeholder URL in an OpenAI chat.completions
+ * body with the data URI the worker built from the job's imagePath. Pure deep
+ * walk; the original request is not mutated. Jobs may instead embed the data
+ * URI directly and omit imagePath.
+ */
+export const IMAGE_PLACEHOLDER = "queue:image";
+
+export function resolveImagePlaceholders(request, dataUri) {
+  const walk = (node) => {
+    if (Array.isArray(node)) return node.map(walk);
+    if (node && typeof node === "object") {
+      const out = {};
+      for (const [key, value] of Object.entries(node)) {
+        out[key] =
+          key === "url" && value === IMAGE_PLACEHOLDER ? dataUri : walk(value);
+      }
+      return out;
+    }
+    return node;
+  };
+  return walk(request);
+}
+
+/** Enqueue-side id: sortable timestamp prefix + entropy, filename-safe. */
+export function makeJobId(nowMs, entropy) {
+  const stamp = new Date(nowMs).toISOString().replace(/[-:.TZ]/g, "");
+  return `${stamp}-${entropy}`;
+}

docker/certification/queue-worker.mjs

@@ -0,0 +1,391 @@
+#!/usr/bin/env node
+/**
+ * Filesystem GPU job-queue worker (#14549): the one long-running consumer in
+ * the compose `gpu` profile. Watches `<jobs>/pending/`, claims jobs by rename
+ * (atomic on one filesystem, so multiple producers and one worker never race),
+ * POSTs each job's OpenAI-shaped request to the resident gpu-vision service,
+ * and writes a result record beside the consumed job. All decision logic is
+ * pure in queue-lib.mjs; this file is fs + fetch + clock.
+ *
+ * Degradation is explicit: while the service is unreachable, jobs bounce back
+ * to pending and are retried; once the outage outlasts --drain-after-ms the
+ * worker drains — every pending job becomes a `skipped` result record naming
+ * the outage — so a run on a GPU-less/half-up machine finishes with honest
+ * skip records instead of hanging or fabricating analyses.
+ *
+ * Usage:
+ *   node docker/certification/queue-worker.mjs --jobs <dir>
+ *     --service ocr=http://gpu-vision:8090 --service vlm=http://gpu-vision:8091
+ *     [--poll-ms N] [--drain-after-ms N] [--request-timeout-ms N] [--once]
+ *
+ * Producers enqueue via the exported `enqueueJob(jobsRoot, job, limits)`.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  claimOrder,
+  createWorkerState,
+  DEFAULT_LIMITS,
+  decideEnqueue,
+  makeJobId,
+  onServiceOk,
+  onServiceUnreachable,
+  parseJob,
+  QUEUE_DIRS,
+  QueueJobInvalidError,
+  resolveImagePlaceholders,
+  resultRecord,
+  shouldSkipJob,
+} from "./queue-lib.mjs";
+
+const MIME_BY_EXT = {
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".webp": "image/webp",
+  ".gif": "image/gif",
+};
+
+export class QueueBackpressureError extends Error {
+  constructor(reason) {
+    super(`[queue] enqueue refused — ${reason}`);
+    this.name = "QueueBackpressureError";
+  }
+}
+
+export async function ensureQueueDirs(jobsRoot) {
+  for (const dir of QUEUE_DIRS) {
+    await fs.mkdir(path.join(jobsRoot, dir), { recursive: true });
+  }
+}
+
+/**
+ * Producer-side enqueue with max-pending backpressure. Writes the job file
+ * atomically (tmp + rename) so the worker can never claim a half-written job.
+ * Throws QueueBackpressureError instead of dropping work.
+ */
+export async function enqueueJob(jobsRoot, job, limits = DEFAULT_LIMITS) {
+  await ensureQueueDirs(jobsRoot);
+  const pendingDir = path.join(jobsRoot, "pending");
+  const pending = claimOrder(await fs.readdir(pendingDir));
+  const decision = decideEnqueue(pending.length, limits.maxPending);
+  if (!decision.accept) throw new QueueBackpressureError(decision.reason);
+
+  const id =
+    typeof job.id === "string" && job.id.length > 0
+      ? job.id
+      : makeJobId(Date.now(), Math.random().toString(36).slice(2, 8));
+  const record = { ...job, id, enqueuedAt: new Date().toISOString() };
+  const tmpPath = path.join(jobsRoot, `.${id}.tmp`);
+  await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, "utf8");
+  await fs.rename(tmpPath, path.join(pendingDir, `${id}.json`));
+  return id;
+}
+
+function parseServices(argv) {
+  const services = {};
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] !== "--service") continue;
+    const spec = argv[i + 1];
+    const eq = spec?.indexOf("=") ?? -1;
+    if (eq === -1) {
+      throw new Error(`[queue] --service expects model=baseUrl, got: ${spec}`);
+    }
+    services[spec.slice(0, eq)] = spec.slice(eq + 1).replace(/\/$/, "");
+  }
+  if (Object.keys(services).length === 0) {
+    throw new Error("[queue] at least one --service model=baseUrl is required");
+  }
+  return services;
+}
+
+function flagValue(argv, name, fallback) {
+  const i = argv.indexOf(name);
+  if (i === -1) return fallback;
+  const value = Number(argv[i + 1]);
+  if (!Number.isFinite(value) || value <= 0) {
+    throw new Error(
+      `[queue] ${name} must be a positive number, got: ${argv[i + 1]}`,
+    );
+  }
+  return value;
+}
+
+async function buildRequestBody(jobsRoot, job) {
+  if (job.imagePath === undefined) return job.request;
+  const imageFile = path.join(jobsRoot, job.imagePath);
+  const ext = path.extname(imageFile).toLowerCase();
+  const mime = MIME_BY_EXT[ext];
+  if (!mime) {
+    throw new QueueJobInvalidError(
+      `unsupported image extension: ${ext || "(none)"}`,
+    );
+  }
+  const bytes = await fs.readFile(imageFile);
+  return resolveImagePlaceholders(
+    job.request,
+    `data:${mime};base64,${bytes.toString("base64")}`,
+  );
+}
+
+async function writeResult(jobsRoot, record) {
+  const tmpPath = path.join(jobsRoot, `.${record.id}.result.tmp`);
+  await fs.writeFile(tmpPath, `${JSON.stringify(record, null, 2)}\n`, "utf8");
+  await fs.rename(tmpPath, path.join(jobsRoot, "results", `${record.id}.json`));
+}
+
+async function finishJob(jobsRoot, name, record) {
+  await writeResult(jobsRoot, record);
+  await fs.rename(
+    path.join(jobsRoot, "processing", name),
+    path.join(jobsRoot, "done", name),
+  );
+}
+
+async function probeServices(services, timeoutMs) {
+  for (const baseUrl of Object.values(services)) {
+    const response = await fetch(`${baseUrl}/health`, {
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!response.ok)
+      throw new Error(`health ${response.status} from ${baseUrl}`);
+  }
+}
+
+/**
+ * Process one claimed job file. Returns "ok" | "failed" | "skipped" |
+ * "unreachable"; only "unreachable" leaves the job in pending for retry.
+ */
+async function processClaimed(jobsRoot, name, raw, services, state, opts) {
+  const idFromName = name.replace(/\.json$/, "");
+  const nowIso = () => new Date().toISOString();
+
+  let job;
+  try {
+    job = parseJob(raw, Object.keys(services));
+  } catch (err) {
+    if (!(err instanceof QueueJobInvalidError)) throw err;
+    // An unparseable job can never succeed on retry; record the defect.
+    await finishJob(
+      jobsRoot,
+      name,
+      resultRecord(
+        { id: idFromName, model: "unknown" },
+        { status: "failed", reason: err.reason },
+        nowIso(),
+      ),
+    );
+    return "failed";
+  }
+
+  if (shouldSkipJob(state)) {
+    await finishJob(
+      jobsRoot,
+      name,
+      resultRecord(
+        job,
+        {
+          status: "skipped",
+          reason: `gpu service unreachable past ${opts.drainAfterMs}ms — drained`,
+        },
+        nowIso(),
+      ),
+    );
+    return "skipped";
+  }
+
+  let body;
+  try {
+    body = await buildRequestBody(jobsRoot, job);
+  } catch (err) {
+    if (!(err instanceof QueueJobInvalidError) && err.code !== "ENOENT")
+      throw err;
+    await finishJob(
+      jobsRoot,
+      name,
+      resultRecord(
+        job,
+        { status: "failed", reason: `image unreadable: ${err.message}` },
+        nowIso(),
+      ),
+    );
+    return "failed";
+  }
+
+  const startedAt = Date.now();
+  let response;
+  try {
+    response = await fetch(`${services[job.model]}/v1/chat/completions`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(opts.requestTimeoutMs),
+    });
+  } catch (err) {
+    // error-policy:J1 the service boundary: a network failure here is the
+    // queue's designed retry/drain signal, observed via worker state + the
+    // eventual skipped records — never a fabricated analysis.
+    process.stderr.write(
+      `[queue] ${job.id}: service unreachable (${err.message}) — will retry\n`,
+    );
+    await fs.rename(
+      path.join(jobsRoot, "processing", name),
+      path.join(jobsRoot, "pending", name),
+    );
+    return "unreachable";
+  }
+
+  const text = await response.text();
+  if (!response.ok) {
+    await finishJob(
+      jobsRoot,
+      name,
+      resultRecord(
+        job,
+        {
+          status: "failed",
+          reason: `http ${response.status}: ${text.slice(0, 2000)}`,
+        },
+        nowIso(),
+      ),
+    );
+    return "failed";
+  }
+
+  await finishJob(
+    jobsRoot,
+    name,
+    resultRecord(
+      job,
+      {
+        status: "ok",
+        durationMs: Date.now() - startedAt,
+        response: JSON.parse(text),
+      },
+      nowIso(),
+    ),
+  );
+  return "ok";
+}
+
+export async function runWorker({ jobsRoot, services, opts, stopSignal }) {
+  await ensureQueueDirs(jobsRoot);
+  let state = createWorkerState();
+  process.stdout.write(
+    `[queue] worker up — jobs: ${jobsRoot}, services: ${Object.entries(services)
+      .map(([model, url]) => `${model}=${url}`)
+      .join(", ")}\n`,
+  );
+
+  while (!stopSignal.stopped) {
+    const pendingDir = path.join(jobsRoot, "pending");
+    const pending = claimOrder(await fs.readdir(pendingDir));
+
+    if (pending.length === 0) {
+      if (opts.once) return state;
+      await new Promise((resolve) => setTimeout(resolve, opts.pollMs));
+      continue;
+    }
+
+    // While unreachable (draining or not), probe health first so recovery is
+    // detected without burning a job attempt, and drain time keeps accruing
+    // even when no fetch is attempted.
+    if (state.unreachableSince !== null) {
+      try {
+        await probeServices(services, Math.min(opts.requestTimeoutMs, 5000));
+        state = onServiceOk();
+        process.stdout.write(
+          "[queue] gpu service reachable again — resuming\n",
+        );
+      } catch {
+        // error-policy:J1 probe failure is the state machine's input, not an
+        // error to escalate: it advances unreachableSince toward drain mode.
+        state = onServiceUnreachable(state, Date.now(), opts.drainAfterMs);
+      }
+    }
+
+    for (const name of pending) {
+      if (stopSignal.stopped) return state;
+      const processingPath = path.join(jobsRoot, "processing", name);
+      let raw;
+      try {
+        await fs.rename(path.join(pendingDir, name), processingPath);
+        raw = await fs.readFile(processingPath, "utf8");
+      } catch (err) {
+        if (err.code === "ENOENT") continue; // claimed by a concurrent worker
+        throw err;
+      }
+      const outcome = await processClaimed(
+        jobsRoot,
+        name,
+        raw,
+        services,
+        state,
+        opts,
+      );
+      if (outcome === "unreachable") {
+        state = onServiceUnreachable(state, Date.now(), opts.drainAfterMs);
+        break; // stop hammering; retry after the poll interval / drain instead
+      }
+      if (outcome === "ok" || outcome === "failed") state = onServiceOk();
+    }
+
+    if (opts.once) {
+      // --once runs until the queue is empty; jobs bounced back by an outage
+      // keep the loop alive until they either succeed or drain to skips.
+      const left = claimOrder(await fs.readdir(pendingDir));
+      if (left.length === 0) return state;
+    }
+    await new Promise((resolve) => setTimeout(resolve, opts.pollMs));
+  }
+  return state;
+}
+
+async function main() {
+  const argv = process.argv.slice(2);
+  const jobsIdx = argv.indexOf("--jobs");
+  if (jobsIdx === -1 || !argv[jobsIdx + 1]) {
+    throw new Error("[queue] --jobs <dir> is required");
+  }
+  const jobsRoot = path.resolve(argv[jobsIdx + 1]);
+  const services = parseServices(argv);
+  const opts = {
+    pollMs: flagValue(argv, "--poll-ms", DEFAULT_LIMITS.pollMs),
+    drainAfterMs: flagValue(
+      argv,
+      "--drain-after-ms",
+      DEFAULT_LIMITS.drainAfterMs,
+    ),
+    requestTimeoutMs: flagValue(
+      argv,
+      "--request-timeout-ms",
+      DEFAULT_LIMITS.requestTimeoutMs,
+    ),
+    once: argv.includes("--once"),
+  };
+
+  const stopSignal = { stopped: false };
+  for (const signal of ["SIGINT", "SIGTERM"]) {
+    process.on(signal, () => {
+      process.stdout.write(
+        `[queue] ${signal} — finishing current job then exiting\n`,
+      );
+      stopSignal.stopped = true;
+    });
+  }
+
+  await runWorker({ jobsRoot, services, opts, stopSignal });
+  process.stdout.write("[queue] worker stopped\n");
+}
+
+if (
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main().catch((err) => {
+    process.stderr.write(`${err.stack || err.message}\n`);
+    process.exit(1);
+  });
+}

docker/certification/queue.test.mjs

@@ -0,0 +1,419 @@
+/**
+ * Tests for the GPU job queue (#14549): the pure state machine in
+ * queue-lib.mjs (job validation, claim order, backpressure, unreachable→drain
+ * transitions, result records), and the real worker process end-to-end — a
+ * spawned queue-worker.mjs consuming a temp jobs dir against a live in-test
+ * HTTP stub standing in for the llama-server endpoint (the GPU model itself is
+ * exercised on the owner-gated Linux acceptance run), including the
+ * drain-to-skip path against an unreachable port. Run with
+ * `bun test docker/certification`.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import http from "node:http";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  claimOrder,
+  createWorkerState,
+  decideEnqueue,
+  IMAGE_PLACEHOLDER,
+  makeJobId,
+  onServiceOk,
+  onServiceUnreachable,
+  parseJob,
+  QueueJobInvalidError,
+  resolveImagePlaceholders,
+  resultRecord,
+  shouldSkipJob,
+} from "./queue-lib.mjs";
+import { enqueueJob, QueueBackpressureError } from "./queue-worker.mjs";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const workerPath = path.join(here, "queue-worker.mjs");
+
+describe("parseJob", () => {
+  const models = ["ocr", "vlm"];
+  test("accepts a well-formed job", () => {
+    const job = parseJob(
+      JSON.stringify({ id: "a-1", model: "ocr", request: { messages: [] } }),
+      models,
+    );
+    expect(job.id).toBe("a-1");
+  });
+
+  test("rejects malformed JSON, missing fields, unknown models, unsafe paths", () => {
+    expect(() => parseJob("{nope", models)).toThrow(QueueJobInvalidError);
+    expect(() => parseJob('"str"', models)).toThrow(/not an object/);
+    expect(() => parseJob("{}", models)).toThrow(/missing id/);
+    expect(() =>
+      parseJob(
+        JSON.stringify({ id: "a/../b", model: "ocr", request: {} }),
+        models,
+      ),
+    ).toThrow(/unsafe characters/);
+    expect(() =>
+      parseJob(JSON.stringify({ id: "a", model: "nope", request: {} }), models),
+    ).toThrow(/model must be one of ocr\|vlm/);
+    expect(() =>
+      parseJob(JSON.stringify({ id: "a", model: "ocr" }), models),
+    ).toThrow(/missing request/);
+    expect(() =>
+      parseJob(
+        JSON.stringify({
+          id: "a",
+          model: "ocr",
+          request: {},
+          imagePath: "../etc/passwd",
+        }),
+        models,
+      ),
+    ).toThrow(/without \.\./);
+    expect(() =>
+      parseJob(
+        JSON.stringify({
+          id: "a",
+          model: "ocr",
+          request: {},
+          imagePath: "/abs",
+        }),
+        models,
+      ),
+    ).toThrow(/relative/);
+  });
+});
+
+describe("claim order and ids", () => {
+  test("claims oldest-first by timestamp-prefixed id, json files only", () => {
+    const names = [
+      "20260706T2-b.json",
+      "20260706T1-a.json",
+      "junk.tmp",
+      "20260706T3-c.json",
+    ];
+    expect(claimOrder(names)).toEqual([
+      "20260706T1-a.json",
+      "20260706T2-b.json",
+      "20260706T3-c.json",
+    ]);
+  });
+
+  test("makeJobId sorts by enqueue time", () => {
+    const early = makeJobId(Date.UTC(2026, 6, 6, 10, 0, 0), "aaa");
+    const late = makeJobId(Date.UTC(2026, 6, 6, 10, 0, 1), "aaa");
+    expect(early < late).toBe(true);
+    expect(early).toMatch(/^[0-9]+-aaa$/);
+  });
+});
+
+describe("backpressure", () => {
+  test("accepts under the cap, refuses at the cap with a reason", () => {
+    expect(decideEnqueue(0, 2)).toEqual({ accept: true });
+    expect(decideEnqueue(1, 2)).toEqual({ accept: true });
+    expect(decideEnqueue(2, 2).accept).toBe(false);
+    expect(decideEnqueue(2, 2).reason).toMatch(/2 pending >= max 2/);
+  });
+});
+
+describe("unreachable → drain state machine", () => {
+  test("stamps the outage start, drains only past the window, resets on contact", () => {
+    const drainAfterMs = 1000;
+    let state = createWorkerState();
+    expect(shouldSkipJob(state)).toBe(false);
+
+    state = onServiceUnreachable(state, 10_000, drainAfterMs);
+    expect(state.unreachableSince).toBe(10_000);
+    expect(state.draining).toBe(false);
+
+    state = onServiceUnreachable(state, 10_500, drainAfterMs);
+    expect(state.unreachableSince).toBe(10_000); // outage start is preserved
+    expect(state.draining).toBe(false);
+
+    state = onServiceUnreachable(state, 11_000, drainAfterMs);
+    expect(state.draining).toBe(true);
+    expect(shouldSkipJob(state)).toBe(true);
+
+    // Draining latches until the service actually answers again.
+    state = onServiceUnreachable(state, 11_001, drainAfterMs);
+    expect(state.draining).toBe(true);
+
+    state = onServiceOk();
+    expect(state).toEqual({ unreachableSince: null, draining: false });
+  });
+});
+
+describe("resultRecord", () => {
+  const job = { id: "j1", model: "ocr" };
+  test("ok carries duration + response; failed/skipped carry a reason", () => {
+    const ok = resultRecord(
+      job,
+      { status: "ok", durationMs: 42, response: { x: 1 } },
+      "t",
+    );
+    expect(ok).toEqual({
+      schema: 1,
+      id: "j1",
+      model: "ocr",
+      status: "ok",
+      completedAt: "t",
+      durationMs: 42,
+      response: { x: 1 },
+    });
+    const skipped = resultRecord(
+      job,
+      { status: "skipped", reason: "drained" },
+      "t",
+    );
+    expect(skipped.reason).toBe("drained");
+    expect(skipped.response).toBeUndefined();
+    expect(() => resultRecord(job, { status: "wat" }, "t")).toThrow(
+      /unknown outcome/,
+    );
+  });
+});
+
+describe("resolveImagePlaceholders", () => {
+  test("replaces only the placeholder url, without mutating the input", () => {
+    const request = {
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "image_url", image_url: { url: IMAGE_PLACEHOLDER } },
+            { type: "text", text: `keep ${IMAGE_PLACEHOLDER} in text` },
+            { type: "image_url", image_url: { url: "data:already/inline" } },
+          ],
+        },
+      ],
+    };
+    const resolved = resolveImagePlaceholders(
+      request,
+      "data:image/png;base64,AA==",
+    );
+    expect(resolved.messages[0].content[0].image_url.url).toBe(
+      "data:image/png;base64,AA==",
+    );
+    expect(resolved.messages[0].content[1].text).toContain(IMAGE_PLACEHOLDER);
+    expect(resolved.messages[0].content[2].image_url.url).toBe(
+      "data:already/inline",
+    );
+    expect(request.messages[0].content[0].image_url.url).toBe(
+      IMAGE_PLACEHOLDER,
+    );
+  });
+});
+
+/** Spawn the real worker in --once mode and wait for exit. */
+function runWorkerOnce(jobsRoot, serviceUrl, extraArgs = []) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(
+      process.execPath,
+      [
+        workerPath,
+        "--jobs",
+        jobsRoot,
+        "--service",
+        `ocr=${serviceUrl}`,
+        "--service",
+        `vlm=${serviceUrl}`,
+        "--poll-ms",
+        "50",
+        "--once",
+        ...extraArgs,
+      ],
+      { stdio: ["ignore", "pipe", "pipe"] },
+    );
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("exit", (code) => resolve({ code, stdout, stderr }));
+  });
+}
+
+describe("worker end-to-end (real process, stub llama-server)", () => {
+  let server;
+  let baseUrl;
+  const seen = [];
+
+  beforeAll(async () => {
+    server = http.createServer((req, res) => {
+      if (req.url === "/health") {
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end('{"status":"ok"}');
+        return;
+      }
+      let body = "";
+      req.on("data", (chunk) => {
+        body += chunk;
+      });
+      req.on("end", () => {
+        const parsed = JSON.parse(body);
+        seen.push({ url: req.url, body: parsed });
+        if (parsed.messages?.[0]?.content === "explode") {
+          res.writeHead(500, { "content-type": "text/plain" });
+          res.end("kaboom");
+          return;
+        }
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end(
+          JSON.stringify({
+            choices: [
+              { message: { role: "assistant", content: "stub-answer" } },
+            ],
+            usage: { prompt_tokens: 10, completion_tokens: 2 },
+          }),
+        );
+      });
+    });
+    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+    baseUrl = `http://127.0.0.1:${server.address().port}`;
+  });
+
+  afterAll(() => {
+    server.close();
+  });
+
+  test("consumes enqueued jobs, calls the service, writes ok/failed results beside them", async () => {
+    const jobsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "eliza-queue-"));
+    const okId = await enqueueJob(jobsRoot, {
+      model: "ocr",
+      request: { messages: [{ role: "user", content: "read this" }] },
+    });
+    const failId = await enqueueJob(jobsRoot, {
+      model: "vlm",
+      request: { messages: [{ role: "user", content: "explode" }] },
+    });
+    // Invalid job file dropped straight into pending/ (bypasses enqueueJob).
+    await fs.writeFile(
+      path.join(jobsRoot, "pending", "zzz-invalid.json"),
+      "{nope",
+      "utf8",
+    );
+
+    const { code, stderr } = await runWorkerOnce(jobsRoot, baseUrl);
+    expect(stderr).toBe("");
+    expect(code).toBe(0);
+
+    const okResult = JSON.parse(
+      await fs.readFile(path.join(jobsRoot, "results", `${okId}.json`), "utf8"),
+    );
+    expect(okResult.status).toBe("ok");
+    expect(okResult.response.choices[0].message.content).toBe("stub-answer");
+    expect(okResult.durationMs).toBeGreaterThanOrEqual(0);
+
+    const failResult = JSON.parse(
+      await fs.readFile(
+        path.join(jobsRoot, "results", `${failId}.json`),
+        "utf8",
+      ),
+    );
+    expect(failResult.status).toBe("failed");
+    expect(failResult.reason).toMatch(/http 500: kaboom/);
+
+    const invalidResult = JSON.parse(
+      await fs.readFile(
+        path.join(jobsRoot, "results", "zzz-invalid.json"),
+        "utf8",
+      ),
+    );
+    expect(invalidResult.status).toBe("failed");
+    expect(invalidResult.reason).toMatch(/not valid JSON/);
+
+    // Consumed jobs moved to done/, pending drained.
+    expect((await fs.readdir(path.join(jobsRoot, "pending"))).length).toBe(0);
+    expect((await fs.readdir(path.join(jobsRoot, "done"))).sort().length).toBe(
+      3,
+    );
+    await fs.rm(jobsRoot, { recursive: true, force: true });
+  }, 30_000);
+
+  test("inlines imagePath as a data URI at the queue:image placeholder", async () => {
+    const jobsRoot = await fs.mkdtemp(
+      path.join(os.tmpdir(), "eliza-queue-img-"),
+    );
+    await fs.mkdir(path.join(jobsRoot, "images"), { recursive: true });
+    // 1x1 PNG.
+    const png = Buffer.from(
+      "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
+      "base64",
+    );
+    await fs.writeFile(path.join(jobsRoot, "images", "shot.png"), png);
+    seen.length = 0;
+    const id = await enqueueJob(jobsRoot, {
+      model: "ocr",
+      imagePath: "images/shot.png",
+      request: {
+        messages: [
+          {
+            role: "user",
+            content: [
+              { type: "image_url", image_url: { url: IMAGE_PLACEHOLDER } },
+            ],
+          },
+        ],
+      },
+    });
+    const { code } = await runWorkerOnce(jobsRoot, baseUrl);
+    expect(code).toBe(0);
+    const result = JSON.parse(
+      await fs.readFile(path.join(jobsRoot, "results", `${id}.json`), "utf8"),
+    );
+    expect(result.status).toBe("ok");
+    const sent = seen[0].body.messages[0].content[0].image_url.url;
+    expect(sent).toBe(`data:image/png;base64,${png.toString("base64")}`);
+    await fs.rm(jobsRoot, { recursive: true, force: true });
+  }, 30_000);
+
+  test("enqueueJob applies max-pending backpressure", async () => {
+    const jobsRoot = await fs.mkdtemp(
+      path.join(os.tmpdir(), "eliza-queue-bp-"),
+    );
+    const limits = { maxPending: 2 };
+    await enqueueJob(jobsRoot, { model: "ocr", request: {} }, limits);
+    await enqueueJob(jobsRoot, { model: "ocr", request: {} }, limits);
+    await expect(
+      enqueueJob(jobsRoot, { model: "ocr", request: {} }, limits),
+    ).rejects.toThrow(QueueBackpressureError);
+    await fs.rm(jobsRoot, { recursive: true, force: true });
+  });
+});
+
+describe("worker drain-to-skip (unreachable service, real process)", () => {
+  test("past the drain window every pending job becomes an honest skipped record", async () => {
+    const jobsRoot = await fs.mkdtemp(
+      path.join(os.tmpdir(), "eliza-queue-drain-"),
+    );
+    const ids = [];
+    for (let i = 0; i < 3; i++) {
+      ids.push(
+        await enqueueJob(jobsRoot, { model: "ocr", request: { messages: [] } }),
+      );
+    }
+    // Port 1 on localhost: connection refused immediately, nobody listens.
+    const { code } = await runWorkerOnce(jobsRoot, "http://127.0.0.1:1", [
+      "--drain-after-ms",
+      "300",
+      "--request-timeout-ms",
+      "500",
+    ]);
+    expect(code).toBe(0);
+    for (const id of ids) {
+      const result = JSON.parse(
+        await fs.readFile(path.join(jobsRoot, "results", `${id}.json`), "utf8"),
+      );
+      expect(result.status).toBe("skipped");
+      expect(result.reason).toMatch(/unreachable past 300ms — drained/);
+    }
+    expect((await fs.readdir(path.join(jobsRoot, "pending"))).length).toBe(0);
+    await fs.rm(jobsRoot, { recursive: true, force: true });
+  }, 30_000);
+});