chore: import upstream snapshot with attribution
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:58:35 +08:00
commit 85453da49f
4031 changed files with 710987 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
{
"compLoadColdP95Ms": 2000,
"compLoadWarmP95Ms": 1000,
"compositionTimeAdvancementRatioMin": 0.95,
"scrubLatencyP95IsolatedMs": 80,
"scrubLatencyP95InlineMs": 33,
"driftMaxMs": 500,
"driftP95Ms": 100,
"paritySsimMin": 0.93,
"allowedRegressionRatio": 0.1
}
@@ -0,0 +1,126 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>perf fixture: 10-video-grid</title>
<style>
:root {
color-scheme: dark;
}
html,
body {
margin: 0;
padding: 0;
background: #050714;
color: #e6e6f0;
font-family:
system-ui,
-apple-system,
sans-serif;
overflow: hidden;
}
#root {
position: relative;
width: 1920px;
height: 1080px;
display: grid;
grid-template-columns: repeat(5, 1fr);
grid-template-rows: repeat(2, 1fr);
gap: 8px;
padding: 8px;
box-sizing: border-box;
}
.tile {
position: relative;
background: #111827;
border-radius: 12px;
overflow: hidden;
box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.05);
will-change: transform;
}
.tile video {
position: absolute;
inset: 0;
width: 100%;
height: 100%;
object-fit: cover;
}
.tile .label {
position: absolute;
top: 8px;
left: 8px;
z-index: 2;
font:
600 14px/1 system-ui,
sans-serif;
color: #fff;
background: rgba(0, 0, 0, 0.6);
padding: 4px 8px;
border-radius: 6px;
pointer-events: none;
}
</style>
<script src="/vendor/gsap.min.js"></script>
<script data-hyperframes-runtime="1" src="/vendor/hyperframe.runtime.iife.js"></script>
</head>
<body>
<div
id="root"
data-composition-id="main"
data-width="1920"
data-height="1080"
data-duration="10"
data-fps="30"
></div>
<script>
(function () {
var TILE_COUNT = 10;
var DURATION_SEC = 10;
var root = document.getElementById("root");
var tiles = [];
for (var i = 0; i < TILE_COUNT; i++) {
var tile = document.createElement("div");
tile.className = "tile";
tile.id = "tile-" + i;
var label = document.createElement("div");
label.className = "label";
label.textContent = "video " + (i + 1);
tile.appendChild(label);
var video = document.createElement("video");
video.id = "video-" + i;
video.setAttribute("data-start", "0");
video.setAttribute("data-duration", String(DURATION_SEC));
video.setAttribute("data-track-index", String(i));
video.setAttribute("src", "sample.mp4");
video.setAttribute("preload", "auto");
video.setAttribute("playsinline", "");
video.muted = true;
tile.appendChild(video);
root.appendChild(tile);
tiles.push(tile);
}
// Lightweight parent timeline so the player has a non-empty composition
// to drive. Each tile gets a subtle scale "breath" over the full
// duration — enough to keep GSAP scrubbing real properties without
// dominating the rAF budget that the video decoder needs.
var tl = gsap.timeline({ paused: true });
for (var j = 0; j < tiles.length; j++) {
tl.fromTo(
tiles[j],
{ scale: 0.985 },
{ scale: 1, duration: DURATION_SEC, ease: "sine.inOut" },
0,
);
}
window.__timelines = window.__timelines || {};
window.__timelines["main"] = tl;
})();
</script>
</body>
</html>
@@ -0,0 +1,115 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>perf fixture: gsap-heavy</title>
<style>
:root {
color-scheme: dark;
}
html,
body {
margin: 0;
padding: 0;
background: #0b0b12;
color: #e6e6f0;
font-family:
system-ui,
-apple-system,
sans-serif;
overflow: hidden;
}
#root {
position: relative;
width: 1920px;
height: 1080px;
overflow: hidden;
}
.tile {
position: absolute;
width: 96px;
height: 96px;
border-radius: 12px;
background: linear-gradient(135deg, #4f46e5, #ec4899);
box-shadow: 0 6px 16px rgba(0, 0, 0, 0.4);
transform: translate3d(0, 0, 0);
will-change: transform, opacity;
}
</style>
<script src="/vendor/gsap.min.js"></script>
<script data-hyperframes-runtime="1" src="/vendor/hyperframe.runtime.iife.js"></script>
</head>
<body>
<div
id="root"
data-composition-id="main"
data-width="1920"
data-height="1080"
data-duration="10"
data-fps="60"
></div>
<script>
(function () {
var TILE_COUNT = 60;
var DURATION_SEC = 10;
var COLS = 12;
var ROWS = 5;
var TILE = 96;
var GAP_X = 1920 / COLS;
var GAP_Y = 1080 / ROWS;
var root = document.getElementById("root");
var tiles = [];
for (var i = 0; i < TILE_COUNT; i++) {
var col = i % COLS;
var row = Math.floor(i / COLS);
var el = document.createElement("div");
el.className = "tile";
el.style.left = col * GAP_X + (GAP_X - TILE) / 2 + "px";
el.style.top = row * GAP_Y + (GAP_Y - TILE) / 2 + "px";
el.setAttribute("data-tile-index", String(i));
root.appendChild(el);
tiles.push(el);
}
var tl = gsap.timeline({ paused: true });
for (var j = 0; j < tiles.length; j++) {
var t = tiles[j];
var phase = j / tiles.length;
var start = phase * (DURATION_SEC - 4);
tl.to(
t,
{
x: 200 * Math.cos(phase * Math.PI * 2),
y: 120 * Math.sin(phase * Math.PI * 2),
rotation: 360,
scale: 1.4,
opacity: 0.6,
borderRadius: "48px",
duration: 2,
ease: "power2.inOut",
},
start,
);
tl.to(
t,
{
x: 0,
y: 0,
rotation: 0,
scale: 1,
opacity: 1,
borderRadius: "12px",
duration: 2,
ease: "power2.inOut",
},
start + 2,
);
}
window.__timelines = window.__timelines || {};
window.__timelines["main"] = tl;
})();
</script>
</body>
</html>
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env bun
/**
* Player Performance Test Runner
*
* Boots a static server, launches puppeteer-core against locally-served fixtures,
* runs the configured scenarios, then evaluates the collected metrics against
* baseline.json via perf-gate.
*
* Usage:
* bun run packages/player/tests/perf/index.ts
* bun run packages/player/tests/perf/index.ts --mode enforce
* bun run packages/player/tests/perf/index.ts --scenarios load
* bun run packages/player/tests/perf/index.ts --runs 5 --headful
*
* Flags:
* --mode <measure|enforce> default: PLAYER_PERF_MODE env or "measure"
* --scenarios <list> comma-separated scenario ids; default: all enabled
* --runs <n> override per-scenario run count
* --fixture <name> single fixture (default: every fixture in fixtures/)
* --headful show the browser; default: headless
*
* Exit codes:
* 0 all pass (or measure mode)
* 1 scenario crashed
* 2 perf gate failed in enforce mode
*/
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { runFps } from "./scenarios/02-fps.ts";
import { runLoad } from "./scenarios/03-load.ts";
import { runScrub } from "./scenarios/04-scrub.ts";
import { runDrift } from "./scenarios/05-drift.ts";
import { runParity } from "./scenarios/06-parity.ts";
import { reportAndGate, type GateMode, type GateResult, type Metric } from "./perf-gate.ts";
import { launchBrowser } from "./runner.ts";
import { startServer } from "./server.ts";
const HERE = dirname(fileURLToPath(import.meta.url));
const RESULTS_DIR = resolve(HERE, "results");
const RESULTS_FILE = resolve(RESULTS_DIR, "metrics.json");
type ScenarioId = "load" | "fps" | "scrub" | "drift" | "parity";
/**
* Per-scenario default `runs` value when the caller didn't pass `--runs`.
*
* Why `load` gets 5 runs and the others get 3:
*
* - `load` reports a single p95 over `runs` measurements, so each `run` is
* one sample. p95 over n=3 is mostly noise (the 95th percentile of three
* numbers is just `max`), so we bump it to 5. We considered 10 — but cold
* load is the slowest scenario in the shard (~2s × 5 runs × 2 fixtures =
* ~20s with disk cache cleared), and going to 10 would push the load shard
* past 30s of pure-measurement wall time per CI invocation.
* - `fps` aggregates as `min(ratio)` over runs — 3 runs gives us a worst-
* of-three signal, which is what we want for a floor metric. Adding more
* runs would only make the ratio strictly smaller (more chances to catch
* a stall) and shift the threshold toward false positives from runner
* contention rather than real regressions.
* - `scrub` and `drift` *pool* their per-run samples (10 seeks/run for
* scrub, ~1500 RVFC frames/run for drift) and compute the percentile over
* the pooled set. Their effective sample count for the percentile is
* `runs × samples_per_run`, not `runs`, so 3 runs already gives 30+ scrub
* samples and 4500+ drift samples per shard — well above the n≈30 rule of
* thumb for a stable p95.
*
* TODO(player-perf): revisit `fps: 3` once we have ~2 weeks of CI baseline
* data — if `min(ratio)` shows >5% inter-run variance attributable to runner
* jitter (not real player regressions), bump to 5 and tighten the
* `compositionTimeAdvancementRatioMin` baseline accordingly.
*/
const DEFAULT_RUNS: Record<ScenarioId, number> = {
load: 5,
fps: 3,
scrub: 3,
drift: 3,
parity: 3,
};
type ResultsFile = {
schemaVersion: 1;
timestamp: string;
gitSha: string | null;
mode: GateMode;
scenarios: ScenarioId[];
runs: number | null;
fixture: string | null;
crashed: boolean;
passed: boolean;
metrics: Metric[];
gate: GateResult[];
};
function readGitSha(): string | null {
try {
return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf-8" }).trim();
} catch {
return null;
}
}
function writeResults(file: ResultsFile): void {
if (!existsSync(RESULTS_DIR)) {
mkdirSync(RESULTS_DIR, { recursive: true });
}
writeFileSync(RESULTS_FILE, JSON.stringify(file, null, 2) + "\n");
console.log(`[player-perf] wrote results to ${RESULTS_FILE}`);
}
type ParsedArgs = {
mode: GateMode;
scenarios: ScenarioId[];
runs: number | null;
fixture: string | null;
headful: boolean;
};
function parseArgs(argv: string[]): ParsedArgs {
const result: ParsedArgs = {
// TODO(player-perf): once baselines have settled on CI for ~12 weeks and we
// are confident there are no false positives from runner jitter, flip this
// default from "measure" to "enforce" — that single line + bumping the
// workflow's `--mode=measure` flag in .github/workflows/player-perf.yml is
// the entire opt-in. See packages/player/tests/perf/perf-gate.ts for how
// `mode` is consumed (measure logs regressions but never fails; enforce
// exits non-zero on regression).
mode: (process.env.PLAYER_PERF_MODE as GateMode) === "enforce" ? "enforce" : "measure",
scenarios: ["load", "fps", "scrub", "drift", "parity"],
runs: null,
fixture: null,
headful: false,
};
// Normalize `--key=value` into `[--key, value]` so the rest of the loop
// only has to handle the space-separated form.
const tokens: string[] = [];
for (const raw of argv.slice(2)) {
if (raw.startsWith("--") && raw.includes("=")) {
const eq = raw.indexOf("=");
tokens.push(raw.slice(0, eq), raw.slice(eq + 1));
} else {
tokens.push(raw);
}
}
for (let i = 0; i < tokens.length; i++) {
const arg = tokens[i];
const next = tokens[i + 1];
if (arg === "--mode" && next) {
if (next !== "measure" && next !== "enforce") {
throw new Error(`--mode must be measure|enforce, got ${next}`);
}
result.mode = next;
i++;
} else if (arg === "--scenarios" && next) {
result.scenarios = next.split(",").map((s) => s.trim()) as ScenarioId[];
i++;
} else if (arg === "--runs" && next) {
result.runs = parseInt(next, 10);
i++;
} else if (arg === "--fixture" && next) {
result.fixture = next;
i++;
} else if (arg === "--headful") {
result.headful = true;
}
}
return result;
}
async function main(): Promise<void> {
const args = parseArgs(process.argv);
console.log(
`[player-perf] starting: mode=${args.mode} scenarios=${args.scenarios.join(",")} runs=${args.runs ?? "default"} fixture=${args.fixture ?? "all"}`,
);
const server = startServer();
console.log(`[player-perf] server listening at ${server.origin}`);
const browser = await launchBrowser({ headless: !args.headful });
console.log("[player-perf] browser launched");
const metrics: Metric[] = [];
let crashed = false;
try {
for (const scenario of args.scenarios) {
if (scenario === "load") {
const m = await runLoad({
browser,
origin: server.origin,
runs: args.runs ?? DEFAULT_RUNS.load,
fixture: args.fixture,
});
metrics.push(...m);
} else if (scenario === "fps") {
const m = await runFps({
browser,
origin: server.origin,
runs: args.runs ?? DEFAULT_RUNS.fps,
fixture: args.fixture,
});
metrics.push(...m);
} else if (scenario === "scrub") {
const m = await runScrub({
browser,
origin: server.origin,
runs: args.runs ?? DEFAULT_RUNS.scrub,
fixture: args.fixture,
});
metrics.push(...m);
} else if (scenario === "drift") {
const m = await runDrift({
browser,
origin: server.origin,
runs: args.runs ?? DEFAULT_RUNS.drift,
fixture: args.fixture,
});
metrics.push(...m);
} else if (scenario === "parity") {
const m = await runParity({
browser,
origin: server.origin,
runs: args.runs ?? DEFAULT_RUNS.parity,
fixture: args.fixture,
});
metrics.push(...m);
} else {
console.warn(`[player-perf] unknown scenario: ${scenario}`);
}
}
} catch (err) {
crashed = true;
console.error("[player-perf] scenario crashed:", err);
} finally {
await browser.close();
await server.stop();
}
let report: { passed: boolean; rows: GateResult[] } = { passed: !crashed, rows: [] };
if (!crashed && metrics.length > 0) {
report = reportAndGate(metrics, args.mode);
}
writeResults({
schemaVersion: 1,
timestamp: new Date().toISOString(),
gitSha: readGitSha(),
mode: args.mode,
scenarios: args.scenarios,
runs: args.runs,
fixture: args.fixture,
crashed,
passed: report.passed && !crashed,
metrics,
gate: report.rows,
});
if (crashed) {
process.exit(1);
}
if (!report.passed) {
process.exit(2);
}
process.exit(0);
}
main().catch((err) => {
console.error("[player-perf] fatal:", err);
process.exit(1);
});
+116
View File
@@ -0,0 +1,116 @@
import { readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
/**
* Compares measured perf metrics against baseline.json with an allowed regression ratio.
*
* Mirrors packages/producer/src/perf-gate.ts: each metric has a baseline value, the
* gate computes `max = baseline * (1 + allowedRegressionRatio)`, and any measured
* value above max counts as a regression. In "measure" mode the script logs but
* never exits non-zero — useful for the first runs while we collect realistic
* baselines on the CI runner. Flip to "enforce" once baselines are committed.
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const DEFAULT_BASELINE_PATH = resolve(HERE, "baseline.json");
export type Direction = "lower-is-better" | "higher-is-better";
export type Metric = {
/** Display name, e.g. "comp_load_cold_p95_ms" */
name: string;
/** Key into baseline.json, e.g. "compLoadColdP95Ms" */
baselineKey: keyof PerfBaseline;
value: number;
unit: string;
direction: Direction;
samples?: number[];
};
export type PerfBaseline = {
compLoadColdP95Ms: number;
compLoadWarmP95Ms: number;
/**
* Floor on `(compositionTime advanced) / (wallClock elapsed)` over a sustained
* playback window — see packages/player/tests/perf/scenarios/02-fps.ts. A
* healthy player keeps up with its intended speed and reads ~1.0; values
* below 1.0 mean the composition clock fell behind real time, which is the
* actual user-visible jank we want to gate against. Refresh-rate independent
* by construction, so it does not saturate to display refresh on high-Hz
* runners the way the previous `fpsMin` did. Direction: higher-is-better.
*/
compositionTimeAdvancementRatioMin: number;
scrubLatencyP95IsolatedMs: number;
scrubLatencyP95InlineMs: number;
driftMaxMs: number;
driftP95Ms: number;
paritySsimMin: number;
allowedRegressionRatio: number;
};
export type GateMode = "measure" | "enforce";
export type GateResult = {
metric: Metric;
baseline: number;
threshold: number;
passed: boolean;
ratio: number;
};
export function loadBaseline(path?: string): PerfBaseline {
const baselinePath = path ?? process.env.PLAYER_PERF_BASELINE_PATH ?? DEFAULT_BASELINE_PATH;
const raw = readFileSync(baselinePath, "utf-8");
return JSON.parse(raw) as PerfBaseline;
}
export function evaluateMetric(metric: Metric, baseline: PerfBaseline): GateResult {
const baselineValue = baseline[metric.baselineKey];
if (typeof baselineValue !== "number") {
throw new Error(`[player-perf] baseline missing numeric key: ${String(metric.baselineKey)}`);
}
const allowed = baseline.allowedRegressionRatio;
const threshold =
metric.direction === "lower-is-better"
? baselineValue * (1 + allowed)
: baselineValue * (1 - allowed);
const passed =
metric.direction === "lower-is-better" ? metric.value <= threshold : metric.value >= threshold;
const ratio = baselineValue === 0 ? 0 : metric.value / baselineValue;
return { metric, baseline: baselineValue, threshold, passed, ratio };
}
export type GateReport = {
passed: boolean;
rows: GateResult[];
};
export function reportAndGate(
metrics: Metric[],
// `mode` is resolved upstream in packages/player/tests/perf/index.ts
// (`parseArgs`): the default comes from PLAYER_PERF_MODE env or "measure", and
// the CLI flag `--mode=measure|enforce` overrides it. The "flip to enforce"
// TODO lives at that call site so it is a one-line change.
mode: GateMode,
baselinePath?: string,
): GateReport {
const baseline = loadBaseline(baselinePath);
const rows = metrics.map((m) => evaluateMetric(m, baseline));
console.log("[PerfGate] mode=" + mode);
for (const row of rows) {
const status = row.passed ? "PASS" : "FAIL";
const dir = row.metric.direction === "lower-is-better" ? "≤" : "≥";
console.log(
`[PerfGate] ${status} ${row.metric.name} = ${row.metric.value.toFixed(2)}${row.metric.unit} (baseline=${row.baseline}${row.metric.unit}, threshold ${dir} ${row.threshold.toFixed(2)}${row.metric.unit}, ratio=${row.ratio.toFixed(3)})`,
);
}
const failed = rows.filter((r) => !r.passed);
if (failed.length === 0) return { passed: true, rows };
if (mode === "measure") {
console.log(`[PerfGate] ${failed.length} regression(s) detected — measure mode, not failing`);
return { passed: true, rows };
}
console.error(`[PerfGate] ${failed.length} regression(s) detected — enforce mode, failing`);
return { passed: false, rows };
}
+137
View File
@@ -0,0 +1,137 @@
import { existsSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import puppeteer, { type Browser, type LaunchOptions, type Page } from "puppeteer-core";
/**
* Puppeteer browser + page helpers shared across all perf scenarios.
*
* Browser launch args mirror packages/producer/src/parity-harness.ts so we get
* the same SwiftShader-backed WebGL output and font hinting between perf runs
* and visual parity runs. That parity matters for P0-1c (live-playback parity)
* and is harmless for the load/scrub/drift scenarios.
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const PLAYER_PKG = resolve(HERE, "../..");
export type LaunchOpts = {
width?: number;
height?: number;
headless?: boolean;
};
export type LoadOpts = {
/** Fixture name (must match a directory under tests/perf/fixtures/). */
fixture: string;
width?: number;
height?: number;
/** Override timeout in ms for the player `ready` event. Default 30s. */
readyTimeoutMs?: number;
};
export type LoadResult = {
/** Wall-clock ms from page navigation start to player `ready` event. */
loadMs: number;
/** Composition duration as reported by the player (seconds). */
duration: number;
};
declare global {
interface Window {
__playerReady?: boolean;
__playerReadyAt?: number;
__playerNavStart?: number;
__playerDuration?: number;
__playerError?: string;
}
}
function findChromeExecutable(): string | undefined {
if (process.env.CHROME_PATH) return process.env.CHROME_PATH;
if (process.env.PUPPETEER_EXECUTABLE_PATH) return process.env.PUPPETEER_EXECUTABLE_PATH;
const candidates = [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/usr/bin/google-chrome",
"/usr/bin/chromium-browser",
"/usr/bin/chromium",
];
for (const path of candidates) {
if (existsSync(path)) return path;
}
return undefined;
}
export async function launchBrowser(options: LaunchOpts = {}): Promise<Browser> {
const width = options.width ?? 1920;
const height = options.height ?? 1080;
const executablePath = findChromeExecutable();
if (!executablePath) {
throw new Error(
`[player-perf] no chrome executable found. Set CHROME_PATH or install Google Chrome. (looked in: $CHROME_PATH, $PUPPETEER_EXECUTABLE_PATH, /Applications/Google Chrome.app, /usr/bin/google-chrome)`,
);
}
const launchOptions: LaunchOptions = {
executablePath,
headless: options.headless ?? true,
defaultViewport: {
width,
height,
deviceScaleFactor: 1,
},
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-accelerated-2d-canvas",
"--enable-webgl",
"--ignore-gpu-blocklist",
"--use-gl=angle",
"--use-angle=swiftshader",
"--font-render-hinting=none",
"--force-color-profile=srgb",
"--autoplay-policy=no-user-gesture-required",
`--window-size=${width},${height}`,
],
};
return puppeteer.launch(launchOptions);
}
/**
* Navigate a page to the host shell and wait for the player's `ready` event.
* Returns the wall-clock ms between `Page.goto` start and the `ready` event,
* along with the composition duration the player reported.
*/
export async function loadHostPage(
page: Page,
origin: string,
options: LoadOpts,
): Promise<LoadResult> {
const width = options.width ?? 1920;
const height = options.height ?? 1080;
const readyTimeoutMs = options.readyTimeoutMs ?? 30_000;
const url = `${origin}/host.html?fixture=${encodeURIComponent(options.fixture)}&width=${width}&height=${height}`;
const t0 = performance.now();
await page.goto(url, { waitUntil: "domcontentloaded", timeout: readyTimeoutMs });
await page.waitForFunction(() => window.__playerReady === true || !!window.__playerError, {
timeout: readyTimeoutMs,
});
const error = await page.evaluate(() => window.__playerError ?? null);
if (error) throw new Error(`[player-perf] player reported error during load: ${error}`);
const loadMs = performance.now() - t0;
const duration = (await page.evaluate(() => window.__playerDuration ?? 0)) ?? 0;
return { loadMs, duration };
}
export function percentile(samples: number[], pct: number): number {
if (samples.length === 0) return 0;
const sorted = [...samples].sort((a, b) => a - b);
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((pct / 100) * sorted.length) - 1));
return sorted[idx] ?? 0;
}
export function repoPlayerDir(): string {
return PLAYER_PKG;
}
@@ -0,0 +1,236 @@
/**
* Scenario 02: sustained playback against the composition clock.
*
* Loads the 10-video-grid fixture, calls `player.play()`, then samples
* `__player.getTime()` at fixed wall-clock intervals for ~5 seconds. The
* emitted metric is the ratio of composition-time advanced to wall-clock
* elapsed:
*
* composition_time_advancement_ratio = (getTime(end) - getTime(start)) / wallSeconds
*
* This reads ~1.0 when the runtime is keeping up with its intended playback
* speed and falls below 1.0 when the player stalls — a slow video decoder, a
* blocked main thread, a GC pause, anything that prevents the composition
* clock from advancing at real-time. The metric is independent of the host
* display refresh rate by construction: both numerator and denominator are
* wall-clock timestamps, neither is a frame count, so a 60Hz, 120Hz, or 240Hz
* runner sees the same value for a healthy player.
*
* Why we replaced the previous rAF-based FPS metric:
* The original implementation counted `requestAnimationFrame` ticks per
* wall-clock second and asserted `fps >= 55`. On a 120Hz CI runner that
* reads ~120 fps regardless of whether the composition is actually
* advancing, so the gate passed even when the player was silently stalling.
* See PR #400 review (jrusso1020 + miguel-heygen) for the full discussion;
* this implementation follows jrusso1020's "first choice" recommendation.
*
* Per the proposal:
* Test 1: Playback frame rate (player-perf-fps)
* Load 10-video composition → play 5s → measure how well the player kept
* up with the composition clock.
*
* Methodology details:
* - We install the wall-clock sampler before calling `play()` so the very
* first post-play tick is captured. We then wait for `__player.isPlaying()`
* to flip true (the parent→iframe `play` message is async via postMessage)
* and *reset* the sample buffer, so the measurement window only contains
* samples taken while the runtime was actively playing the timeline.
* - Sampling cadence is 100ms (10 samples/sec). That's fine-grained enough
* to spot a half-second stall but coarse enough that the sampler itself
* has negligible overhead. With a 5s window we collect ~50 samples; the
* ratio is computed from the first and last sample's `getTime()` values.
* - We use `setInterval` (not rAF) on purpose: rAF cadence is the metric we
* are trying to *avoid* depending on. `setInterval` is wall-clock-driven.
*
* Outputs one metric:
* - composition_time_advancement_ratio_min
* (higher-is-better, baseline key compositionTimeAdvancementRatioMin)
*
* Aggregation: `min(ratio)` across runs because the proposal asserts a floor
* — the worst run is the one that gates against regressions.
*/
import type { Browser, Frame, Page } from "puppeteer-core";
import { loadHostPage } from "../runner.ts";
import type { Metric } from "../perf-gate.ts";
export type FpsScenarioOpts = {
browser: Browser;
origin: string;
/** Number of measurement runs. */
runs: number;
/** If null, runs the default fixture (10-video-grid). */
fixture: string | null;
};
const DEFAULT_FIXTURE = "10-video-grid";
const PLAYBACK_DURATION_MS = 5_000;
const SAMPLE_INTERVAL_MS = 100;
const PLAY_CONFIRM_TIMEOUT_MS = 5_000;
const FRAME_LOOKUP_TIMEOUT_MS = 5_000;
declare global {
interface Window {
/** (wallClockMs, compositionTimeSec) pairs collected by the sampler. */
__perfPlaySamples?: Array<{ wall: number; comp: number }>;
/** setInterval handle used by the sampler; cleared at the end of the window. */
__perfPlaySamplerHandle?: number;
/** Hyperframes runtime player API exposed inside the composition iframe. */
__player?: {
play: () => void;
pause: () => void;
seek: (timeSeconds: number) => void;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
};
}
}
type RunResult = {
ratio: number;
compElapsedSec: number;
wallElapsedSec: number;
samples: number;
};
/**
* Find the iframe Puppeteer Frame that hosts the fixture composition. The
* `<hyperframes-player>` shell wraps an iframe whose URL is derived from the
* player's `src` attribute, so we match by path substring rather than full URL.
*/
async function getFixtureFrame(page: Page, fixture: string): Promise<Frame> {
const expected = `/fixtures/${fixture}/`;
const deadline = Date.now() + FRAME_LOOKUP_TIMEOUT_MS;
while (Date.now() < deadline) {
const frame = page.frames().find((f) => f.url().includes(expected));
if (frame) return frame;
await new Promise((r) => setTimeout(r, 50));
}
throw new Error(`[scenario:fps] fixture frame not found for "${fixture}" within timeout`);
}
async function runOnce(
opts: FpsScenarioOpts,
fixture: string,
idx: number,
total: number,
): Promise<RunResult> {
const ctx = await opts.browser.createBrowserContext();
try {
const page = await ctx.newPage();
const { duration } = await loadHostPage(page, opts.origin, { fixture });
const frame = await getFixtureFrame(page, fixture);
// Install the wall-clock sampler in the iframe context. We use setInterval
// because rAF cadence is exactly the host-display-dependent signal we are
// trying NOT to depend on; setInterval is driven by the event loop and
// gives us samples at fixed wall-clock cadence regardless of refresh rate.
await frame.evaluate((sampleIntervalMs: number) => {
window.__perfPlaySamples = [];
window.__perfPlaySamplerHandle = window.setInterval(() => {
const comp = window.__player?.getTime?.();
if (typeof comp !== "number" || !Number.isFinite(comp)) return;
window.__perfPlaySamples!.push({
wall: performance.timeOrigin + performance.now(),
comp,
});
}, sampleIntervalMs);
}, SAMPLE_INTERVAL_MS);
// Issue play from the host page (parent of the iframe). The player's
// public `play()` posts a control message into the iframe.
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { play: () => void }) | null;
if (!el) throw new Error("[scenario:fps] player element missing on host page");
el.play();
});
// Wait for the runtime to actually transition to playing — this is the
// signal that the postMessage round trip + timeline.play() finished.
await frame.waitForFunction(() => window.__player?.isPlaying?.() === true, {
timeout: PLAY_CONFIRM_TIMEOUT_MS,
});
// Reset samples now that playback is confirmed running. Anything captured
// before this point belongs to the ramp-up window (composition clock at
// 0, wall clock advancing) and would skew the ratio toward 0.
await frame.evaluate(() => {
window.__perfPlaySamples = [];
});
// Sustain playback for the measurement window.
await new Promise((r) => setTimeout(r, PLAYBACK_DURATION_MS));
// Stop the sampler and harvest the samples before pausing the runtime,
// so the pause command can't perturb the tail of the sample window.
const samples = (await frame.evaluate(() => {
if (window.__perfPlaySamplerHandle !== undefined) {
clearInterval(window.__perfPlaySamplerHandle);
window.__perfPlaySamplerHandle = undefined;
}
return window.__perfPlaySamples ?? [];
})) as Array<{ wall: number; comp: number }>;
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { pause: () => void }) | null;
el?.pause();
});
if (samples.length < 2) {
throw new Error(
`[scenario:fps] run ${idx + 1}/${total}: only ${samples.length} composition-clock samples captured (composition duration ${duration}s)`,
);
}
const first = samples[0]!;
const last = samples[samples.length - 1]!;
const wallElapsedSec = (last.wall - first.wall) / 1000;
const compElapsedSec = last.comp - first.comp;
const ratio = wallElapsedSec > 0 ? compElapsedSec / wallElapsedSec : 0;
console.log(
`[scenario:fps] run[${idx + 1}/${total}] ratio=${ratio.toFixed(4)} compElapsed=${compElapsedSec.toFixed(3)}s wallElapsed=${wallElapsedSec.toFixed(3)}s samples=${samples.length}`,
);
await page.close();
return {
ratio,
compElapsedSec,
wallElapsedSec,
samples: samples.length,
};
} finally {
await ctx.close();
}
}
export async function runFps(opts: FpsScenarioOpts): Promise<Metric[]> {
const fixture = opts.fixture ?? DEFAULT_FIXTURE;
const runs = Math.max(1, opts.runs);
console.log(
`[scenario:fps] fixture=${fixture} runs=${runs} window=${PLAYBACK_DURATION_MS}ms sampleInterval=${SAMPLE_INTERVAL_MS}ms`,
);
const ratios: number[] = [];
for (let i = 0; i < runs; i++) {
const result = await runOnce(opts, fixture, i, runs);
ratios.push(result.ratio);
}
// Worst run wins: the proposal asserts a floor on this ratio, so a single
// bad run (slow decoder, GC pause, host contention) is the one that gates.
const ratioMin = Math.min(...ratios);
console.log(`[scenario:fps] aggregate min ratio=${ratioMin.toFixed(4)} runs=${runs}`);
return [
{
name: "composition_time_advancement_ratio_min",
baselineKey: "compositionTimeAdvancementRatioMin",
value: ratioMin,
unit: "ratio",
direction: "higher-is-better",
samples: ratios,
},
];
}
@@ -0,0 +1,98 @@
/**
* Scenario 03: composition load (cold + warm).
*
* Cold: a fresh BrowserContext per run so the network cache is empty. Measures
* the wall-clock time from `page.goto` until the player fires its `ready`
* event (host shell sets `window.__playerReady`). This stresses html parse +
* runtime IIFE eval + GSAP eval + the player's first composition init.
*
* Warm: same BrowserContext is reused across runs so the static assets
* (player bundle, runtime, GSAP, fixture HTML) are served from disk cache.
* This isolates the player's per-composition init cost from network I/O.
*
* Both metrics report p95 over `runs` samples and feed into perf-gate.ts:
* - compLoadColdP95Ms (lower is better)
* - compLoadWarmP95Ms (lower is better)
*/
import type { Browser } from "puppeteer-core";
import { loadHostPage, percentile } from "../runner.ts";
import type { Metric } from "../perf-gate.ts";
export type LoadScenarioOpts = {
browser: Browser;
origin: string;
/** Number of cold and warm runs each. */
runs: number;
/** If null, runs the default fixture (gsap-heavy). */
fixture: string | null;
};
const DEFAULT_FIXTURE = "gsap-heavy";
export async function runLoad(opts: LoadScenarioOpts): Promise<Metric[]> {
const fixture = opts.fixture ?? DEFAULT_FIXTURE;
const runs = Math.max(1, opts.runs);
console.log(`[scenario:load] fixture=${fixture} runs=${runs}`);
const cold: number[] = [];
for (let i = 0; i < runs; i++) {
const ctx = await opts.browser.createBrowserContext();
try {
const page = await ctx.newPage();
const { loadMs, duration } = await loadHostPage(page, opts.origin, { fixture });
cold.push(loadMs);
console.log(
`[scenario:load] cold[${i + 1}/${runs}] loadMs=${loadMs.toFixed(1)} duration=${duration}s`,
);
await page.close();
} finally {
await ctx.close();
}
}
const warm: number[] = [];
const warmCtx = await opts.browser.createBrowserContext();
try {
const warmupPage = await warmCtx.newPage();
await loadHostPage(warmupPage, opts.origin, { fixture });
await warmupPage.close();
for (let i = 0; i < runs; i++) {
const page = await warmCtx.newPage();
const { loadMs, duration } = await loadHostPage(page, opts.origin, { fixture });
warm.push(loadMs);
console.log(
`[scenario:load] warm[${i + 1}/${runs}] loadMs=${loadMs.toFixed(1)} duration=${duration}s`,
);
await page.close();
}
} finally {
await warmCtx.close();
}
const coldP95 = percentile(cold, 95);
const warmP95 = percentile(warm, 95);
console.log(
`[scenario:load] cold p95=${coldP95.toFixed(1)}ms (samples=${cold.length}) warm p95=${warmP95.toFixed(1)}ms (samples=${warm.length})`,
);
return [
{
name: "comp_load_cold_p95_ms",
baselineKey: "compLoadColdP95Ms",
value: coldP95,
unit: "ms",
direction: "lower-is-better",
samples: cold,
},
{
name: "comp_load_warm_p95_ms",
baselineKey: "compLoadWarmP95Ms",
value: warmP95,
unit: "ms",
direction: "lower-is-better",
samples: warm,
},
];
}
@@ -0,0 +1,307 @@
/**
* Scenario 04: scrub latency.
*
* Loads the 10-video-grid fixture, pauses the player, then issues 10 seek
* calls in sequence — first through the synchronous "inline" path, then
* through the postMessage-driven "isolated" path — and measures the wall-clock
* latency from each `seek()` call to the first paint where the iframe's
* timeline reports the new time.
*
* Per the proposal:
* Test 2: Scrub latency (player-perf-scrub)
* Load composition → seek to 10 positions in sequence → measure time
* from seek() call to state update callback
* Assert: p95 < 80ms (isolated), p95 < 33ms (inline, Phase 4+)
*
* Methodology details:
* - Both modes are measured in the same page load. Inline runs first so
* the isolated mode's monkey-patch (forcing `_trySyncSeek` to return
* false) doesn't bleed into the inline samples.
* - "Inline" mode is the default behavior of `<hyperframes-player>` when the
* iframe is same-origin and exposes `__player.seek()` synchronously.
* `seek()` lands the new frame in the same task as the input event.
* - "Isolated" mode is forced by replacing the player element's
* `_trySyncSeek` method with `() => false`, which sends the player
* element through the postMessage bridge — exactly what cross-origin
* embeds and Phase 1 (pre-sync) builds did.
* - Detection is via a `requestAnimationFrame` watcher inside the iframe
* that polls `__player.getTime()` until it is within `MATCH_TOLERANCE_S`
* of the requested target. We use a tolerance because the postMessage
* bridge converts seconds → frame number → seconds, which can introduce
* sub-frame quantization drift even for targets on the canonical fps grid.
* - Timing uses `performance.timeOrigin + performance.now()` in both the
* host and iframe contexts. `timeOrigin` is consistent across same-process
* frames, so the difference is a true wall-clock measurement of latency.
* - Seek targets alternate forward/backward across the 10s composition so
* no two consecutive seeks land near each other; this avoids the rAF
* watcher matching against a stale `getTime()` value before the seek
* command is processed.
*
* Outputs two metrics:
* - scrub_latency_p95_inline_ms (lower-is-better, baseline scrubLatencyP95InlineMs)
* - scrub_latency_p95_isolated_ms (lower-is-better, baseline scrubLatencyP95IsolatedMs)
*
* Aggregation: percentile(95) is computed across the pooled per-seek
* latencies from every run. With 10 seeks per mode per run × 3 runs we get
* 30 samples per mode per CI shard, which is enough for a stable p95.
*/
import type { Browser, Frame, Page } from "puppeteer-core";
import { loadHostPage, percentile } from "../runner.ts";
import type { Metric } from "../perf-gate.ts";
export type ScrubScenarioOpts = {
browser: Browser;
origin: string;
/** Number of measurement runs. */
runs: number;
/** If null, runs the default fixture (10-video-grid). */
fixture: string | null;
};
const DEFAULT_FIXTURE = "10-video-grid";
/** Targets are seconds within the composition (10s duration). */
const SEEK_TARGETS: readonly number[] = [1.0, 7.0, 2.0, 8.0, 3.0, 9.0, 4.0, 6.0, 5.0, 0.5];
/**
* Tolerance window the rAF watcher uses to decide that the iframe's reported
* `__player.getTime()` matches the requested seek target. 50ms = 1.5 frames at
* 30fps, which absorbs three sources of expected slippage:
*
* 1. **Frame quantization on the postMessage path.** `_sendControl("seek")`
* converts seconds → integer frame number → seconds inside the runtime,
* so e.g. a target of 1.0s on a 30fps composition lands at frame 30 →
* 1.000s exactly, but a target of 1.005s lands at frame 30 → still
* 1.000s, a 5ms quantization error baked into the API itself.
* 2. **Sub-frame intra-clip clock advance.** Even with the iframe paused,
* between the `seek()` call landing and the next rAF tick, the runtime
* may have already nudged time by a fraction of a frame as part of
* finalizing the seek; `getTime()` reports the post-finalize value.
* 3. **Variable host load + browser jitter on CI.** GitHub runners share
* cores, so a noisy neighbor can delay the rAF tick that would otherwise
* register the match by tens of ms. Picking a tolerance much tighter
* than this would gate against runner contention rather than player
* regressions.
*
* The metric this scenario asserts is *latency to user-visible match*, not
* *exact equality of the reported time*, so a 50ms acceptance window is the
* intended behavior — but if we ever want to tighten this (e.g. to assert
* sub-frame precision on the inline path now that PR #397 documented it),
* this is the knob to turn. Configurability is deliberately deferred until
* we have a concrete second use case; YAGNI.
*
* TODO(player-perf): revisit this constant after P0-1b lands and we have ~2
* weeks of CI baseline data — if the inline-mode samples consistently cluster
* well below 50ms, drop this to e.g. 16ms (1 frame @ 60fps) and split the
* tolerance per mode (tighter for inline, current for isolated).
*/
const MATCH_TOLERANCE_S = 0.05;
/** Per-seek timeout; isolated p95 in the proposal is 80ms, so 1s is huge headroom. */
const SEEK_TIMEOUT_MS = 1_000;
const PAUSE_CONFIRM_TIMEOUT_MS = 5_000;
const FRAME_LOOKUP_TIMEOUT_MS = 5_000;
declare global {
interface Window {
/** Promise resolved by the iframe rAF watcher with the wall-clock t1 of the matching paint. */
__perfScrubAwait?: Promise<number>;
__player?: {
play: () => void;
pause: () => void;
seek: (timeSeconds: number) => void;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
};
}
}
type Mode = "inline" | "isolated";
type RunResult = {
inlineLatencies: number[];
isolatedLatencies: number[];
};
/**
* Find the iframe Puppeteer Frame that hosts the fixture composition. Same
* helper as 02-fps.ts; duplicated locally so each scenario file is
* self-contained.
*/
async function getFixtureFrame(page: Page, fixture: string): Promise<Frame> {
const expected = `/fixtures/${fixture}/`;
const deadline = Date.now() + FRAME_LOOKUP_TIMEOUT_MS;
while (Date.now() < deadline) {
const frame = page.frames().find((f) => f.url().includes(expected));
if (frame) return frame;
await new Promise((r) => setTimeout(r, 50));
}
throw new Error(`[scenario:scrub] fixture frame not found for "${fixture}" within timeout`);
}
/**
* Measure a single seek's latency.
*
* Sequence:
* 1. Install a rAF watcher in the iframe that resolves with the wall-clock
* timestamp of the first paint where `__player.getTime()` is within
* tolerance of `target`. Promise is stashed on `window.__perfScrubAwait`.
* 2. Capture host wall-clock t0 and call `el.seek(target)` in the same task.
* 3. Await the iframe's resolved Promise (returns t1).
* 4. Latency = t1 - t0 (ms).
*/
async function measureSingleSeek(page: Page, frame: Frame, target: number): Promise<number> {
await frame.evaluate(
(target: number, tolerance: number, timeoutMs: number) => {
window.__perfScrubAwait = new Promise<number>((resolve, reject) => {
const deadlineWall = performance.timeOrigin + performance.now() + timeoutMs;
const tick = () => {
const wall = performance.timeOrigin + performance.now();
const time = window.__player?.getTime?.() ?? Number.NaN;
if (Number.isFinite(time) && Math.abs(time - target) < tolerance) {
resolve(wall);
return;
}
if (wall > deadlineWall) {
reject(new Error(`[scrub] timeout target=${target} last=${time}`));
return;
}
requestAnimationFrame(tick);
};
requestAnimationFrame(tick);
});
},
target,
MATCH_TOLERANCE_S,
SEEK_TIMEOUT_MS,
);
const t0Wall = await page.evaluate((targetSeconds: number) => {
const el = document.getElementById("player") as
| (HTMLElement & { seek: (t: number) => void })
| null;
if (!el) throw new Error("[scenario:scrub] player element missing on host page");
const wall = performance.timeOrigin + performance.now();
el.seek(targetSeconds);
return wall;
}, target);
// Puppeteer awaits the Promise we stashed on window and returns its resolved value.
const t1Wall = (await frame.evaluate(() => window.__perfScrubAwait as Promise<number>)) as number;
return t1Wall - t0Wall;
}
async function runScrubBatch(
page: Page,
frame: Frame,
mode: Mode,
idx: number,
total: number,
): Promise<number[]> {
const latencies: number[] = [];
for (const target of SEEK_TARGETS) {
const latency = await measureSingleSeek(page, frame, target);
latencies.push(latency);
}
const p95 = percentile(latencies, 95);
console.log(
`[scenario:scrub] run[${idx + 1}/${total}] mode=${mode} p95=${p95.toFixed(2)}ms n=${latencies.length}`,
);
return latencies;
}
async function runOnce(
opts: ScrubScenarioOpts,
fixture: string,
idx: number,
total: number,
): Promise<RunResult> {
const ctx = await opts.browser.createBrowserContext();
try {
const page = await ctx.newPage();
const { duration } = await loadHostPage(page, opts.origin, { fixture });
const requiredDuration = Math.max(...SEEK_TARGETS);
if (duration < requiredDuration) {
throw new Error(
`[scenario:scrub] fixture composition is ${duration.toFixed(2)}s but scrub targets require >= ${requiredDuration}s`,
);
}
const frame = await getFixtureFrame(page, fixture);
// Defensively pause: the host shell doesn't autoplay, but `pause()` also
// cancels any pending autoplay-on-ready behavior and guarantees the
// timeline isn't ticking under our seek measurements.
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { pause?: () => void }) | null;
el?.pause?.();
});
await frame.waitForFunction(() => window.__player?.isPlaying?.() === false, {
timeout: PAUSE_CONFIRM_TIMEOUT_MS,
});
// Inline mode first — the player's default `_trySyncSeek` path lands the
// seek synchronously when the iframe is same-origin (which it is here).
const inlineLatencies = await runScrubBatch(page, frame, "inline", idx, total);
// Force isolated mode by shadowing `_trySyncSeek` on the instance with
// a function that always reports failure. The fallback in `seek()` then
// sends the seek through `_sendControl("seek", { frame })`, which is the
// same path a cross-origin embed (or a Phase 1 build without sync seek)
// would take.
await page.evaluate(() => {
const el = document.getElementById("player") as
| (HTMLElement & { _trySyncSeek?: (t: number) => boolean })
| null;
if (!el) throw new Error("[scenario:scrub] player element missing on host page");
el._trySyncSeek = () => false;
});
const isolatedLatencies = await runScrubBatch(page, frame, "isolated", idx, total);
await page.close();
return { inlineLatencies, isolatedLatencies };
} finally {
await ctx.close();
}
}
export async function runScrub(opts: ScrubScenarioOpts): Promise<Metric[]> {
const fixture = opts.fixture ?? DEFAULT_FIXTURE;
const runs = Math.max(1, opts.runs);
console.log(
`[scenario:scrub] fixture=${fixture} runs=${runs} seeks_per_mode=${SEEK_TARGETS.length} tolerance=${(MATCH_TOLERANCE_S * 1000).toFixed(0)}ms`,
);
const allInline: number[] = [];
const allIsolated: number[] = [];
for (let i = 0; i < runs; i++) {
const result = await runOnce(opts, fixture, i, runs);
allInline.push(...result.inlineLatencies);
allIsolated.push(...result.isolatedLatencies);
}
const inlineP95 = percentile(allInline, 95);
const isolatedP95 = percentile(allIsolated, 95);
console.log(
`[scenario:scrub] aggregate inline_p95=${inlineP95.toFixed(2)}ms isolated_p95=${isolatedP95.toFixed(2)}ms (runs=${runs} samples_per_mode=${allInline.length})`,
);
return [
{
name: "scrub_latency_p95_inline_ms",
baselineKey: "scrubLatencyP95InlineMs",
value: inlineP95,
unit: "ms",
direction: "lower-is-better",
samples: allInline,
},
{
name: "scrub_latency_p95_isolated_ms",
baselineKey: "scrubLatencyP95IsolatedMs",
value: isolatedP95,
unit: "ms",
direction: "lower-is-better",
samples: allIsolated,
},
];
}
@@ -0,0 +1,307 @@
/**
* Scenario 05: media sync drift.
*
* Loads the 10-video-grid fixture, starts playback, and uses
* `requestVideoFrameCallback` on every video element to record
* (compositionTime, actualMediaTime) pairs for each decoded frame. Drift is
* the absolute difference between the *expected* media time (derived from the
* composition time using the runtime's clip transform) and the actual media
* time the decoder presented to the compositor.
*
* Per the proposal:
* Test 4: Media sync drift (player-perf-drift)
* Load 5-video composition → play for 10 seconds → on each RVFC callback,
* record drift between expected and actual media time
* Assert: max drift < 500ms, p95 drift < 100ms
*
* Methodology details:
* - We instrument *every* `video[data-start]` element in the fixture. The
* proposal called for 5 videos; the 10-video-grid gives us 10 streams in
* the same composition, which is a more conservative regression signal.
* - The expected media time uses the same transform the runtime applies in
* packages/core/src/runtime/media.ts:
*
* expectedMediaTime = (compositionTime - clip.start) * clip.playbackRate
* + clip.mediaStart
*
* We snapshot `clip.start` / `clip.mediaStart` / `clip.playbackRate` from
* each element's dataset + `defaultPlaybackRate` once when the sampler is
* installed, so the per-frame work is just a subtract + multiply + abs.
* - The runtime's media sync runs on a 50ms `setInterval`. Between syncs the
* video element's clock free-runs. The drift we measure here is the
* residual after that 50ms loop catches up — i.e. the user-visible glitch
* budget. The runtime hard-resyncs when |currentTime - relTime| > 0.5s
* (see media.ts), which is exactly the proposal's max-drift ceiling: a
* regression past 500ms means the corrective resync kicked in and the
* viewer saw a jump.
* - We install RVFC *before* calling play(), then reset the sample buffer
* once `__player.isPlaying()` flips true. Frames captured during the
* postMessage round-trip would compare a non-zero mediaTime against
* `getTime() === 0` and inflate drift to several hundred ms — same gotcha
* as 02-fps.ts.
* - Sustain window is 6s instead of the proposal's 10s because the fixture
* composition is exactly 10s long, and we want headroom before the
* end-of-timeline pause/clamp behavior. With 10 videos × ~25fps × 6s we
* still pool ~1500 samples per run, more than enough for a stable p95.
*
* Outputs two metrics:
* - media_drift_max_ms (lower-is-better, baseline driftMaxMs)
* - media_drift_p95_ms (lower-is-better, baseline driftP95Ms)
*
* Aggregation: max() and percentile(95) across the pooled per-frame drifts
* from every video in every run.
*/
import type { Browser, Frame, Page } from "puppeteer-core";
import { loadHostPage, percentile } from "../runner.ts";
import type { Metric } from "../perf-gate.ts";
export type DriftScenarioOpts = {
browser: Browser;
origin: string;
/** Number of measurement runs. */
runs: number;
/** If null, runs the default fixture (10-video-grid). */
fixture: string | null;
};
const DEFAULT_FIXTURE = "10-video-grid";
const PLAYBACK_DURATION_MS = 6_000;
const PLAY_CONFIRM_TIMEOUT_MS = 5_000;
const FRAME_LOOKUP_TIMEOUT_MS = 5_000;
type DriftSample = {
compTime: number;
actualMediaTime: number;
clipStart: number;
clipMediaStart: number;
clipPlaybackRate: number;
};
declare global {
interface Window {
/** RVFC samples collected by the iframe-side observer. */
__perfDriftSamples?: DriftSample[];
/** Set to false to stop sampling at the end of the measurement window. */
__perfDriftActive?: boolean;
__player?: {
play: () => void;
pause: () => void;
seek: (timeSeconds: number) => void;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
};
}
}
type RunResult = {
drifts: number[];
videoCount: number;
};
/**
* Find the iframe Puppeteer Frame that hosts the fixture composition. Same
* helper as the other scenarios; duplicated locally so each scenario file is
* self-contained.
*/
async function getFixtureFrame(page: Page, fixture: string): Promise<Frame> {
const expected = `/fixtures/${fixture}/`;
const deadline = Date.now() + FRAME_LOOKUP_TIMEOUT_MS;
while (Date.now() < deadline) {
const frame = page.frames().find((f) => f.url().includes(expected));
if (frame) return frame;
await new Promise((r) => setTimeout(r, 50));
}
throw new Error(`[scenario:drift] fixture frame not found for "${fixture}" within timeout`);
}
async function runOnce(
opts: DriftScenarioOpts,
fixture: string,
idx: number,
total: number,
): Promise<RunResult> {
const ctx = await opts.browser.createBrowserContext();
try {
const page = await ctx.newPage();
const { duration } = await loadHostPage(page, opts.origin, { fixture });
const requiredDurationSec = PLAYBACK_DURATION_MS / 1000;
if (duration < requiredDurationSec) {
throw new Error(
`[scenario:drift] fixture composition is ${duration.toFixed(2)}s but drift sample window needs >= ${requiredDurationSec.toFixed(0)}s`,
);
}
const frame = await getFixtureFrame(page, fixture);
// Install RVFC on every `video[data-start]` element in the iframe. Each
// callback records the wall-clock-aligned (compositionTime, mediaTime)
// pair plus a snapshot of the clip transform so we can compute drift in
// node without re-querying the dataset on every frame.
const videoCount = (await frame.evaluate(() => {
window.__perfDriftSamples = [];
window.__perfDriftActive = true;
const videos = Array.from(document.querySelectorAll<HTMLVideoElement>("video[data-start]"));
type RvfcMetadata = { mediaTime: number; presentationTime: number };
type RvfcVideo = HTMLVideoElement & {
requestVideoFrameCallback?: (
cb: (now: DOMHighResTimeStamp, metadata: RvfcMetadata) => void,
) => number;
};
let installed = 0;
for (const video of videos) {
const rvfcVideo = video as RvfcVideo;
const rvfc = rvfcVideo.requestVideoFrameCallback;
// Headless Chrome supports RVFC; bail quietly on browsers that don't.
if (!rvfc) continue;
const clipStart = Number.parseFloat(video.dataset.start ?? "0") || 0;
const clipMediaStart =
Number.parseFloat(video.dataset.playbackStart ?? video.dataset.mediaStart ?? "0") || 0;
const rawRate = video.defaultPlaybackRate;
const clipPlaybackRate =
Number.isFinite(rawRate) && rawRate > 0 ? Math.max(0.1, Math.min(5, rawRate)) : 1;
const tick = (_now: DOMHighResTimeStamp, metadata: RvfcMetadata) => {
if (!window.__perfDriftActive) return;
const compTime = window.__player?.getTime?.() ?? Number.NaN;
if (Number.isFinite(compTime)) {
window.__perfDriftSamples!.push({
compTime,
actualMediaTime: metadata.mediaTime,
clipStart,
clipMediaStart,
clipPlaybackRate,
});
}
rvfc.call(video, tick);
};
rvfc.call(video, tick);
installed++;
}
return installed;
})) as number;
if (videoCount === 0) {
throw new Error(`[scenario:drift] fixture ${fixture} contains no video[data-start] elements`);
}
// Issue play from the host page; the player posts a control message into
// the iframe and the runtime starts the 50ms media sync poll.
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { play: () => void }) | null;
if (!el) throw new Error("[scenario:drift] player element missing on host page");
el.play();
});
// Wait for the runtime to confirm playing before we trust the samples.
await frame.waitForFunction(() => window.__player?.isPlaying?.() === true, {
timeout: PLAY_CONFIRM_TIMEOUT_MS,
});
// Reset the buffer now that playback is live. Anything captured during
// the postMessage round-trip would compare a non-zero mediaTime against
// `getTime() === 0` and bias drift up by hundreds of ms.
await frame.evaluate(() => {
window.__perfDriftSamples = [];
});
await new Promise((r) => setTimeout(r, PLAYBACK_DURATION_MS));
// Stop sampling first, then pause. Same ordering as 02-fps.ts so the
// pause command can't perturb the tail of the measurement window.
const samples = (await frame.evaluate(() => {
window.__perfDriftActive = false;
return window.__perfDriftSamples ?? [];
})) as DriftSample[];
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { pause: () => void }) | null;
el?.pause();
});
if (samples.length === 0) {
throw new Error(
`[scenario:drift] run ${idx + 1}/${total}: zero RVFC samples captured (videos=${videoCount}, duration=${duration.toFixed(2)}s)`,
);
}
// Apply the runtime's transform to derive the expected media time, then
// compare against the actual media time the decoder presented. Convert
// to ms here so the gate threshold (driftMaxMs / driftP95Ms) compares
// apples-to-apples.
const drifts: number[] = [];
for (const s of samples) {
const expectedMediaTime = (s.compTime - s.clipStart) * s.clipPlaybackRate + s.clipMediaStart;
const driftMs = Math.abs(s.actualMediaTime - expectedMediaTime) * 1000;
drifts.push(driftMs);
}
const max = Math.max(...drifts);
const p95 = percentile(drifts, 95);
console.log(
`[scenario:drift] run[${idx + 1}/${total}] max=${max.toFixed(2)}ms p95=${p95.toFixed(2)}ms videos=${videoCount} samples=${samples.length}`,
);
await page.close();
return { drifts, videoCount };
} finally {
await ctx.close();
}
}
export async function runDrift(opts: DriftScenarioOpts): Promise<Metric[]> {
const fixture = opts.fixture ?? DEFAULT_FIXTURE;
const runs = Math.max(1, opts.runs);
console.log(`[scenario:drift] fixture=${fixture} runs=${runs} window=${PLAYBACK_DURATION_MS}ms`);
const allDrifts: number[] = [];
let lastVideoCount = 0;
for (let i = 0; i < runs; i++) {
const result = await runOnce(opts, fixture, i, runs);
allDrifts.push(...result.drifts);
lastVideoCount = result.videoCount;
}
// Worst case wins for max; p95 is computed across the pooled per-frame
// drifts from every video in every run. The proposal asserts max < 500ms
// and p95 < 100ms, so a single bad sample legitimately gates the build.
const maxDrift = Math.max(...allDrifts);
const p95Drift = percentile(allDrifts, 95);
// Coefficient of variation (stddev / mean) is logged here as a soft signal
// we can eyeball in CI output. We deliberately do NOT gate on it — the
// baseline asserts absolute thresholds (max, p95), and the underlying
// distribution is heavy-tailed (most frames are sub-50ms, occasional ones
// spike during the 50ms media-sync interval). But CV is a useful early
// warning: if it climbs significantly across CI runs while max + p95 stay
// green, our jitter assumptions about the runtime's resync loop have
// shifted (e.g. if media.ts changes its 50ms `setInterval` cadence) and
// we should revisit the baselines before they start producing flakes.
// TODO(player-perf): once we have ~2 weeks of CI baseline data, decide
// whether to publish CV as a tracked-but-ungated metric in baseline.json
// alongside max + p95, or wire it into the Slack regression report.
const meanDrift = allDrifts.reduce((a, b) => a + b, 0) / allDrifts.length;
const variance = allDrifts.reduce((acc, d) => acc + (d - meanDrift) ** 2, 0) / allDrifts.length;
const stddev = Math.sqrt(variance);
const cv = meanDrift > 0 ? stddev / meanDrift : 0;
console.log(
`[scenario:drift] aggregate max=${maxDrift.toFixed(2)}ms p95=${p95Drift.toFixed(2)}ms mean=${meanDrift.toFixed(2)}ms cv=${cv.toFixed(3)} videos=${lastVideoCount} samples=${allDrifts.length} runs=${runs}`,
);
return [
{
name: "media_drift_max_ms",
baselineKey: "driftMaxMs",
value: maxDrift,
unit: "ms",
direction: "lower-is-better",
samples: allDrifts,
},
{
name: "media_drift_p95_ms",
baselineKey: "driftP95Ms",
value: p95Drift,
unit: "ms",
direction: "lower-is-better",
samples: allDrifts,
},
];
}
@@ -0,0 +1,418 @@
/**
* Scenario 06: live-playback parity vs synchronous seek.
*
* Loads the gsap-heavy fixture, plays it from t=0, then captures the rendered
* frame at a known timestamp (t≈5.0s, mid-animation). Without releasing the
* page, we then synchronously seek the same player back to that exact captured
* timestamp and capture a *reference* frame. The two PNGs are diffed with
* `ffmpeg -lavfi ssim` and the resulting average SSIM is the parity metric.
*
* Per the proposal:
* Test 5: Live-playback parity (player-perf-parity)
* Play composition → freeze at known t → screenshot → seek to same t →
* screenshot → compare via SSIM
* Assert: SSIM > 0.95 (effectively perfect with deterministic rendering)
*
* Baseline note (paritySsimMin=0.93, set deliberately wider than the proposal's
* 0.95): the host runner is headless Chromium with all the determinism flags
* we can practically apply, but the gsap-heavy fixture still has a small
* sub-pixel rasterization wobble between "paint immediately after pause()"
* and "paint after sync seek." Empirically the worst run sits around 0.960.98,
* but a 2-point cushion keeps us from chasing flakes on slower CI hardware
* while still catching real parity drift (anything < 0.93 means the two
* paths produced visibly different pixels, not just sub-pixel jitter).
* If we tighten determinism further (e.g. fixed device pixel ratio + forced
* software raster) we should ratchet this baseline back up to 0.95.
*
* Why this matters:
* `<hyperframes-player>`'s sync-seek path goes through `_trySyncSeek`, which
* for same-origin embeds calls into the iframe runtime's `seek()` directly.
* Live playback advances frames via the runtime's animation loop. If those
* two paths drift out of agreement — different rounding, different sub-frame
* sampling, different state ordering — scrubbing a paused composition will
* show different pixels than a paused-during-playback frame at the same time.
* This test pins them together visually.
*
* Methodology details:
* - Capture point is t=5.0s. The gsap-heavy fixture is a 10s composition
* with 60 tiles each running a staggered 4s out-and-back tween. At 5.0s
* a large fraction of those tiles are mid-flight, so the rendered frame
* has many distinct, position-sensitive pixels — the worst case for any
* sub-frame disagreement between the two paths.
* - Live capture uses an iframe-side rAF watcher that polls
* `__player.getTime()` every animation frame. When `getTime() >= 5.0`,
* the watcher calls `__player.pause()` *from inside the same rAF tick*.
* `pause()` is synchronous (it calls `timeline.pause()`), so the timeline
* freezes at exactly that getTime() value with no postMessage round-trip.
* We then read `getTime()` one more time to capture the canonical frozen
* timestamp `T_actual` — that's the ground truth both screenshots target.
* - Both screenshots wait for two `requestAnimationFrame` ticks on the host
* page before capture. The first rAF flushes any pending style/layout
* work; the second rAF guarantees the compositor has painted. This is
* the same paint-settlement pattern as packages/producer/src/parity-harness.ts.
* - Reference capture issues `el.seek(T_actual)` from the host page. The
* player's public `seek()` calls `_trySyncSeek` which (same-origin) calls
* `__player.seek()` synchronously, so we don't need a postMessage await.
* - SSIM is computed by `ffmpeg -lavfi ssim`, which emits per-channel and
* overall scores to stderr. We parse the `All:` value (clamped at 1.0
* because ffmpeg occasionally reports 1.000001 for identical inputs).
* - Both PNGs and the captured T_actual value are written under
* `tests/perf/results/parity/run-N/` for CI artifact upload and local
* debugging. The directory is gitignored via the existing
* `packages/player/tests/perf/results/` rule.
*
* Output metric:
* - parity_ssim_min (higher-is-better, baseline paritySsimMin = 0.93)
*
* Aggregation: min() across runs. We want the *worst* observed parity to
* pass the gate, so that one bad run can't get masked by averaging.
*/
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import type { Browser, Frame, Page } from "puppeteer-core";
import { loadHostPage } from "../runner.ts";
import type { Metric } from "../perf-gate.ts";
export type ParityScenarioOpts = {
browser: Browser;
origin: string;
/** Number of measurement runs. */
runs: number;
/** If null, runs the default fixture (gsap-heavy). */
fixture: string | null;
};
const DEFAULT_FIXTURE = "gsap-heavy";
/** Mid-composition; gsap-heavy is 10s and has many tiles in motion at this point. */
const TARGET_TIME_S = 5.0;
/** rAF watcher will resolve as soon as getTime() crosses TARGET_TIME_S. */
const TARGET_TIMEOUT_MS = 15_000;
const PLAY_CONFIRM_TIMEOUT_MS = 5_000;
const FRAME_LOOKUP_TIMEOUT_MS = 5_000;
/** ffmpeg occasionally reports 1.000001 on identical inputs; clamp to keep
* baseline math sane. */
const SSIM_CLAMP_MAX = 1.0;
const HERE = dirname(fileURLToPath(import.meta.url));
const RESULTS_DIR = resolve(HERE, "../results/parity");
declare global {
interface Window {
/** Promise resolved by the iframe rAF watcher with the frozen player time (s). */
__perfParityPauseAwait?: Promise<number>;
__player?: {
play: () => void;
pause: () => void;
seek: (timeSeconds: number) => void;
getTime: () => number;
getDuration: () => number;
isPlaying: () => boolean;
};
}
}
type RunResult = {
ssim: number;
capturedTime: number;
};
/**
* Find the iframe Puppeteer Frame that hosts the fixture composition. Same
* helper as the other scenarios; duplicated locally so each scenario file is
* self-contained.
*/
async function getFixtureFrame(page: Page, fixture: string): Promise<Frame> {
const expected = `/fixtures/${fixture}/`;
const deadline = Date.now() + FRAME_LOOKUP_TIMEOUT_MS;
while (Date.now() < deadline) {
const frame = page.frames().find((f) => f.url().includes(expected));
if (frame) return frame;
await new Promise((r) => setTimeout(r, 50));
}
throw new Error(`[scenario:parity] fixture frame not found for "${fixture}" within timeout`);
}
/**
* Wait for two animation frames on the host page so the compositor has had a
* chance to paint the latest player state before we screenshot. First rAF
* flushes pending style/layout, second rAF guarantees a painted commit.
*/
async function waitForPaint(page: Page): Promise<void> {
await page.evaluate(
() =>
new Promise<void>((resolve) =>
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
),
);
}
function ensureDir(path: string): void {
if (!existsSync(path)) {
mkdirSync(path, { recursive: true });
}
}
/**
* Run `ffmpeg -lavfi ssim` against two PNGs and return the overall SSIM
* score. ffmpeg writes the score to stderr in the form:
*
* [Parsed_ssim_0 @ 0x...] SSIM Y:0.998... U:0.999... V:0.999... All:0.998... (28.3)
*
* We grab the `All:` value, parse it as a float, and clamp to SSIM_CLAMP_MAX.
*
* Three failure modes, kept distinct so CI is debuggable without re-running:
* - `result.error` (e.g. ENOENT) — ffmpeg never started; the binary is
* missing or unexecutable. We surface the OS error so the operator
* immediately knows to install ffmpeg on the runner instead of chasing
* an "exit=undefined" red herring.
* - `result.status !== 0` — ffmpeg started but exited non-zero. Usually a
* decode/argument error; stderr has the real message.
* - parse failure — ffmpeg ran successfully but its output didn't contain
* the expected `All:` token. Indicates a version skew or a no-op input.
*
* On the second and third failure modes we additionally re-run ffmpeg with
* `stats_file` pointed at `<runDir>/ssim-stats.log` so the next CI artifact
* upload contains a per-frame SSIM dump alongside the two PNGs. That log is
* the cheapest possible bridge between "the assert tripped" and "this pixel
* region drifted" — without it, debugging a parity regression means pulling
* the PNGs locally and eyeballing them.
*/
function computeSsim(referencePath: string, actualPath: string, runDir: string): number {
const result = spawnSync(
"ffmpeg",
["-hide_banner", "-i", referencePath, "-i", actualPath, "-lavfi", "ssim", "-f", "null", "-"],
{ stdio: "pipe" },
);
if (result.error) {
// spawnSync surfaces ENOENT / EACCES / etc. on `result.error`. status is
// null in this case — ffmpeg never actually ran. Calling toString() on
// result.status would print "null", which is exactly what produced the
// confusing "exit=undefined" line that masked the real ENOENT in CI.
throw new Error(
`[scenario:parity] ffmpeg could not be started (${(result.error as NodeJS.ErrnoException).code ?? "unknown"}): ${result.error.message}. ` +
"Install ffmpeg on the runner (apt-get install -y ffmpeg) — the parity scenario " +
"requires it for SSIM scoring.",
);
}
if (result.status !== 0) {
const stderr = (result.stderr || Buffer.from("")).toString("utf-8");
writeSsimStatsOnFailure(referencePath, actualPath, runDir);
throw new Error(`[scenario:parity] ffmpeg ssim failed (exit=${result.status}): ${stderr}`);
}
const stderr = (result.stderr || Buffer.from("")).toString("utf-8");
const match = stderr.match(/All:\s*([0-9.]+)/);
if (!match) {
writeSsimStatsOnFailure(referencePath, actualPath, runDir);
throw new Error(`[scenario:parity] could not parse SSIM from ffmpeg stderr: ${stderr}`);
}
const raw = Number.parseFloat(match[1]);
if (!Number.isFinite(raw)) {
writeSsimStatsOnFailure(referencePath, actualPath, runDir);
throw new Error(`[scenario:parity] parsed SSIM is not finite: "${match[1]}"`);
}
return Math.min(SSIM_CLAMP_MAX, raw);
}
/**
* Best-effort: re-invoke ffmpeg with `stats_file=<runDir>/ssim-stats.log`
* so the per-frame SSIM dump lands in the artifact directory. This runs
* only on the failure paths in `computeSsim` — a successful parity check
* doesn't need the dump. We swallow any error from this helper because
* the caller is already on its way to throwing the original failure;
* losing the diagnostic dump shouldn't change the surfaced error.
*/
function writeSsimStatsOnFailure(referencePath: string, actualPath: string, runDir: string): void {
try {
const statsPath = resolve(runDir, "ssim-stats.log");
spawnSync(
"ffmpeg",
[
"-hide_banner",
"-i",
referencePath,
"-i",
actualPath,
"-lavfi",
// ffmpeg's lavfi parser uses '\:' to escape the path separator inside
// a filter argument. We don't expect ':' in `statsPath` but escape
// defensively to keep this robust on weird mounts.
`ssim=stats_file=${statsPath.replace(/:/g, "\\:")}`,
"-f",
"null",
"-",
],
{ stdio: "pipe" },
);
} catch {
// Best-effort: never let stats-dump failure mask the real error.
}
}
async function runOnce(
opts: ParityScenarioOpts,
fixture: string,
idx: number,
total: number,
): Promise<RunResult> {
const ctx = await opts.browser.createBrowserContext();
try {
const page = await ctx.newPage();
const { duration } = await loadHostPage(page, opts.origin, { fixture });
if (duration < TARGET_TIME_S + 0.1) {
throw new Error(
`[scenario:parity] fixture composition is ${duration.toFixed(2)}s but parity target needs >= ${(TARGET_TIME_S + 0.1).toFixed(2)}s`,
);
}
const frame = await getFixtureFrame(page, fixture);
// Install the iframe-side rAF watcher *before* we issue play(). The
// watcher polls __player.getTime() every animation frame and, the first
// time getTime() >= TARGET_TIME_S, calls __player.pause() in the same
// tick. pause() is synchronous (it calls timeline.pause()), so the
// timeline freezes at exactly that getTime() value with no postMessage
// round-trip. The Promise resolves with that frozen value as the
// canonical T_actual we'll use for both screenshots.
await frame.evaluate(
(target: number, timeoutMs: number) => {
window.__perfParityPauseAwait = new Promise<number>((resolve, reject) => {
const deadlineWall = performance.timeOrigin + performance.now() + timeoutMs;
const tick = () => {
const player = window.__player;
if (!player) {
reject(new Error("[parity] __player missing during rAF watcher"));
return;
}
const wall = performance.timeOrigin + performance.now();
const time = player.getTime();
if (Number.isFinite(time) && time >= target) {
// Pause from inside the rAF tick — synchronous in the runtime,
// so the timeline can't advance any further before we read
// getTime() back out as the canonical frozen value.
player.pause();
resolve(player.getTime());
return;
}
if (wall > deadlineWall) {
reject(new Error(`[parity] timeout waiting for getTime >= ${target} (last=${time})`));
return;
}
requestAnimationFrame(tick);
};
requestAnimationFrame(tick);
});
},
TARGET_TIME_S,
TARGET_TIMEOUT_MS,
);
// Start playback from the host page.
await page.evaluate(() => {
const el = document.getElementById("player") as (HTMLElement & { play: () => void }) | null;
if (!el) throw new Error("[scenario:parity] player element missing on host page");
el.play();
});
// Confirm the runtime is actually playing before we wait on the rAF
// watcher. Without this we can hang waiting for getTime() to advance
// when play() hasn't kicked the timeline yet.
await frame.waitForFunction(() => window.__player?.isPlaying?.() === true, {
timeout: PLAY_CONFIRM_TIMEOUT_MS,
});
// Block until the iframe watcher pauses the timeline and resolves with
// the frozen player time. This is the canonical T_actual for the run.
const capturedTime = (await frame.evaluate(
() => window.__perfParityPauseAwait as Promise<number>,
)) as number;
if (!Number.isFinite(capturedTime) || capturedTime < TARGET_TIME_S) {
throw new Error(
`[scenario:parity] watcher resolved with invalid time: ${capturedTime} (target=${TARGET_TIME_S})`,
);
}
// Capture frame #1: the live-playback frame frozen by pause().
await waitForPaint(page);
const actualImage = (await page.screenshot({ type: "png" })) as Buffer | Uint8Array;
// Capture frame #2: the same time, reached via synchronous seek. The
// player is already paused, so seek() lands the timeline directly on
// capturedTime via _trySyncSeek -> __player.seek().
await page.evaluate((targetSeconds: number) => {
const el = document.getElementById("player") as
| (HTMLElement & { seek: (t: number) => void })
| null;
if (!el) throw new Error("[scenario:parity] player element missing on host page");
el.seek(targetSeconds);
}, capturedTime);
await waitForPaint(page);
const referenceImage = (await page.screenshot({ type: "png" })) as Buffer | Uint8Array;
// Persist artifacts under results/parity/run-N/ for CI upload and local
// inspection. Captured time is written alongside so we can reproduce
// a specific run's seek target later.
const runDir = resolve(RESULTS_DIR, `run-${idx + 1}`);
ensureDir(runDir);
const actualPath = resolve(runDir, "actual.png");
const referencePath = resolve(runDir, "reference.png");
writeFileSync(actualPath, actualImage);
writeFileSync(referencePath, referenceImage);
writeFileSync(
resolve(runDir, "captured-time.txt"),
`${capturedTime}\n${TARGET_TIME_S}\n`,
"utf-8",
);
const ssim = computeSsim(referencePath, actualPath, runDir);
console.log(
`[scenario:parity] run[${idx + 1}/${total}] ssim=${ssim.toFixed(6)} captured_time=${capturedTime.toFixed(6)}s artifacts=${runDir}`,
);
await page.close();
return { ssim, capturedTime };
} finally {
await ctx.close();
}
}
export async function runParity(opts: ParityScenarioOpts): Promise<Metric[]> {
const fixture = opts.fixture ?? DEFAULT_FIXTURE;
const runs = Math.max(1, opts.runs);
console.log(`[scenario:parity] fixture=${fixture} runs=${runs} target=${TARGET_TIME_S}s`);
// Wipe stale per-run dirs from previous invocations so artifact upload
// only contains this run's PNGs. We don't rm -rf the parent dir to avoid
// surprising anyone debugging a previous failure.
ensureDir(RESULTS_DIR);
const ssims: number[] = [];
for (let i = 0; i < runs; i++) {
const result = await runOnce(opts, fixture, i, runs);
ssims.push(result.ssim);
}
// Worst case wins. A min < 0.93 means at least one run produced visibly
// different pixels between live playback and sync seek at the same time —
// which is the regression we're guarding against (see file-level JSDoc
// for why the gate is 0.93 rather than the proposal's 0.95).
const minSsim = Math.min(...ssims);
const meanSsim = ssims.reduce((a, b) => a + b, 0) / ssims.length;
console.log(
`[scenario:parity] aggregate min=${minSsim.toFixed(6)} mean=${meanSsim.toFixed(6)} runs=${runs}`,
);
return [
{
name: "parity_ssim_min",
baselineKey: "paritySsimMin",
value: minSsim,
unit: "ssim",
direction: "higher-is-better",
samples: ssims,
},
];
}
+202
View File
@@ -0,0 +1,202 @@
import { existsSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
/**
* Static file server for player perf tests.
*
* Serves all bundles, vendor scripts, fixtures, and the embed host page from
* a single origin so the player iframe stays same-origin. Without same-origin
* the runtime probe in `_onIframeLoad` falls into the cross-origin catch path
* and the `ready` event fires later (or not at all) — which would be measured
* as a player-side regression instead of an environment artifact.
*
* URL routes:
* / → host.html (default fixture: gsap-heavy)
* /host.html?fixture=<name> → embed page hosting <hyperframes-player>
* /player/hyperframes-player.global.js
* /vendor/gsap.min.js
* /vendor/hyperframe.runtime.iife.js
* /fixtures/<name>/<file> → fixture HTML + assets
*/
const HERE = dirname(fileURLToPath(import.meta.url));
const PLAYER_PKG = resolve(HERE, "../..");
const REPO_ROOT = resolve(PLAYER_PKG, "../..");
function firstExisting(candidates: string[]): string {
for (const p of candidates) {
if (existsSync(p)) return p;
}
return candidates[0] ?? "";
}
const PATHS = {
player: join(PLAYER_PKG, "dist/hyperframes-player.global.js"),
runtime: join(REPO_ROOT, "packages/core/dist/hyperframe.runtime.iife.js"),
// bun installs gsap into the package's node_modules in workspace mode, but
// hoists it to the repo root if multiple packages share the same version.
// Probe both locations so the server works regardless of layout.
gsap: firstExisting([
join(PLAYER_PKG, "node_modules/gsap/dist/gsap.min.js"),
join(REPO_ROOT, "node_modules/gsap/dist/gsap.min.js"),
]),
fixturesDir: join(HERE, "fixtures"),
} as const;
export type ServeOptions = {
port?: number;
/** Disables HTTP cache so every request is a "cold" fetch. Used for cold-load scenarios. */
noCache?: boolean;
};
export type RunningServer = {
port: number;
origin: string;
stop(): Promise<void>;
};
const MIME_TYPES: Record<string, string> = {
".html": "text/html; charset=utf-8",
".js": "application/javascript; charset=utf-8",
".mjs": "application/javascript; charset=utf-8",
".css": "text/css; charset=utf-8",
".json": "application/json; charset=utf-8",
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".webp": "image/webp",
".mp4": "video/mp4",
".webm": "video/webm",
".mp3": "audio/mpeg",
};
function mimeFor(path: string): string {
const dot = path.lastIndexOf(".");
if (dot < 0) return "application/octet-stream";
return MIME_TYPES[path.slice(dot).toLowerCase()] ?? "application/octet-stream";
}
function buildHostHtml(fixtureName: string, width: number, height: number): string {
const playerSrc = "/player/hyperframes-player.global.js";
const fixtureSrc = `/fixtures/${fixtureName}/index.html`;
return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>player perf host: ${fixtureName}</title>
<style>
html, body { margin: 0; padding: 0; background: #000; }
hyperframes-player { display: block; }
</style>
</head>
<body>
<hyperframes-player
id="player"
src="${fixtureSrc}"
width="${width}"
height="${height}"
muted
></hyperframes-player>
<script>
window.__playerReady = false;
window.__playerReadyAt = null;
window.__playerNavStart = performance.timeOrigin + performance.now();
const player = document.getElementById("player");
player.addEventListener("ready", function (event) {
window.__playerReady = true;
window.__playerReadyAt = performance.timeOrigin + performance.now();
window.__playerDuration = (event.detail && event.detail.duration) || 0;
});
player.addEventListener("error", function (event) {
window.__playerError = (event.detail && event.detail.message) || "unknown";
});
</script>
<script src="${playerSrc}"></script>
</body>
</html>`;
}
async function readBunFile(path: string): Promise<Response> {
if (!existsSync(path)) {
return new Response(`Not found: ${path}`, { status: 404 });
}
const file = Bun.file(path);
return new Response(file, {
headers: {
"Content-Type": mimeFor(path),
},
});
}
function applyCacheHeaders(res: Response, noCache: boolean): Response {
if (noCache) {
res.headers.set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0");
res.headers.set("Pragma", "no-cache");
res.headers.set("Expires", "0");
} else {
res.headers.set("Cache-Control", "public, max-age=3600");
}
return res;
}
export function startServer(options: ServeOptions = {}): RunningServer {
const noCache = options.noCache ?? false;
const server = Bun.serve({
port: options.port ?? 0,
async fetch(req) {
const url = new URL(req.url);
const path = url.pathname;
if (path === "/" || path === "/host.html") {
const fixture = url.searchParams.get("fixture") || "gsap-heavy";
const width = Number(url.searchParams.get("width") || "1920");
const height = Number(url.searchParams.get("height") || "1080");
const html = buildHostHtml(fixture, width, height);
return applyCacheHeaders(
new Response(html, { headers: { "Content-Type": "text/html; charset=utf-8" } }),
noCache,
);
}
if (path === "/player/hyperframes-player.global.js") {
return applyCacheHeaders(await readBunFile(PATHS.player), noCache);
}
if (path === "/vendor/hyperframe.runtime.iife.js") {
return applyCacheHeaders(await readBunFile(PATHS.runtime), noCache);
}
if (path === "/vendor/gsap.min.js") {
return applyCacheHeaders(await readBunFile(PATHS.gsap), noCache);
}
if (path.startsWith("/fixtures/")) {
const rel = path.replace(/^\/fixtures\//, "");
const filePath = join(PATHS.fixturesDir, rel);
if (!filePath.startsWith(PATHS.fixturesDir)) {
return new Response("Forbidden", { status: 403 });
}
return applyCacheHeaders(await readBunFile(filePath), noCache);
}
return new Response("Not found", { status: 404 });
},
});
// server.port is `number | undefined` in Bun's types (undefined only for unix-socket
// servers, which we never use). Narrow it once at startup so the rest of the perf
// harness can rely on a numeric origin.
const port = server.port;
if (port === undefined) {
throw new Error("[player-perf] Bun.serve did not assign a TCP port");
}
return {
port,
origin: `http://127.0.0.1:${port}`,
async stop() {
server.stop(true);
},
};
}
+16
View File
@@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"noEmit": true,
"types": ["bun"],
"allowImportingTsExtensions": true,
"resolveJsonModule": true
},
"include": ["**/*.ts"]
}