#!/usr/bin/env node
/**
* media-use eval — compare baseline (no media-use) vs. with media-use
* on real registry blocks. Produces an HTML report.
*/
import {
mkdtempSync,
cpSync,
rmSync,
readFileSync,
readdirSync,
existsSync,
writeFileSync,
} from "node:fs";
import { join, basename, resolve, dirname } from "node:path";
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = resolve(SCRIPT_DIR, "..", "..", "..");
const RESOLVE_SCRIPT = join(SCRIPT_DIR, "resolve.mjs");
const TEST_BLOCKS = [
"registry/blocks/nyc-paris-flight",
"registry/blocks/macos-tahoe-liquid-glass",
"registry/blocks/blue-sweater-intro-video",
"registry/blocks/vpn-youtube-spot",
"registry/blocks/apple-money-count",
"registry/blocks/liquid-glass-notification",
"registry/blocks/instagram-follow",
];
// Run resolve.mjs with args as a literal argv array (no shell), so values
// interpolated from manifest metadata (--intent prompt, --type) can't inject
// shell. Mirrors the execFileSync fix in probe.mjs / heygen-search.mjs.
function run(args, opts = {}) {
try {
return {
ok: true,
output: execFileSync(process.execPath, [RESOLVE_SCRIPT, ...args], {
encoding: "utf8",
timeout: 15000,
stdio: "pipe",
...opts,
}).trim(),
};
} catch (err) {
return { ok: false, output: (err.stdout || "") + (err.stderr || ""), code: err.status };
}
}
function countAssetFiles(dir) {
const assetsDir = join(dir, "assets");
if (!existsSync(assetsDir)) return { count: 0, files: [] };
const files = [];
function walk(d, base = "") {
for (const e of readdirSync(d, { withFileTypes: true })) {
const rel = base ? `${base}/${e.name}` : e.name;
if (e.isDirectory()) walk(join(d, e.name), rel);
else files.push(rel);
}
}
walk(assetsDir);
return { count: files.length, files };
}
function evalBlock(blockPath) {
const fullPath = join(REPO_ROOT, blockPath);
if (!existsSync(fullPath)) return null;
const name = basename(blockPath);
const tmp = mkdtempSync(join(tmpdir(), `mu-eval-${name}-`));
try {
cpSync(fullPath, tmp, { recursive: true });
// baseline: what the agent sees WITHOUT media-use
const baseline = countAssetFiles(tmp);
const htmlFiles = readdirSync(tmp).filter((f) => f.endsWith(".html"));
// parse compositions for asset references
const assetRefs = [];
for (const hf of htmlFiles) {
const html = readFileSync(join(tmp, hf), "utf8");
const srcMatches = html.matchAll(/src=["']([^"']+?)["']/g);
for (const m of srcMatches) {
const ref = m[1];
if (ref.startsWith("data:") || ref.startsWith("http")) continue;
assetRefs.push({ composition: hf, ref });
}
const urlMatches = html.matchAll(/url\(["']?([^"')]+?)["']?\)/g);
for (const m of urlMatches) {
const ref = m[1];
if (ref.startsWith("data:") || ref.startsWith("http") || ref.startsWith("#")) continue;
assetRefs.push({ composition: hf, ref });
}
}
// with media-use: run --adopt
const adoptResult = run(["--adopt", "--project", tmp, "--json"]);
let adopted = { ok: false, adopted: 0, assets: [] };
if (adoptResult.ok) {
try {
adopted = JSON.parse(adoptResult.output);
} catch {
/* */
}
}
// read the generated index
const indexPath = join(tmp, ".media", "index.md");
const indexContent = existsSync(indexPath)
? readFileSync(indexPath, "utf8")
: "(no index generated)";
// read manifest for detail
const manifestPath = join(tmp, ".media", "manifest.jsonl");
const manifest = existsSync(manifestPath)
? readFileSync(manifestPath, "utf8")
.trim()
.split("\n")
.map((l) => {
try {
return JSON.parse(l);
} catch {
return null;
}
})
.filter(Boolean)
: [];
// test resolve cache hit: try resolving something that was adopted
let resolveTest = null;
if (manifest.length > 0) {
const first = manifest[0];
const prompt = first.provenance?.prompt || first.description;
const r = run(["--type", first.type, "--intent", prompt, "--project", tmp, "--json"]);
if (r.ok) {
try {
resolveTest = JSON.parse(r.output);
} catch {
/* */
}
}
}
// test resolve miss: try resolving something that doesn't exist
const missResult = run([
"--type",
"bgm",
"--intent",
"nonexistent query xyz",
"--project",
tmp,
"--json",
]);
let resolveMiss = null;
if (!missResult.ok) {
try {
resolveMiss = JSON.parse(missResult.output);
} catch {
/* */
}
}
// coverage: which composition refs are covered by the manifest
const manifestPaths = new Set(manifest.map((m) => m.path));
const coverage = assetRefs.map((r) => ({
...r,
covered: manifestPaths.has(r.ref),
}));
return {
name,
baseline: { fileCount: baseline.count, files: baseline.files, htmlCount: htmlFiles.length },
compositions: htmlFiles,
assetRefs: coverage,
adopted: { count: adopted.adopted, assets: adopted.assets || [] },
index: indexContent,
manifest,
resolveTest,
resolveMiss,
};
} finally {
rmSync(tmp, { recursive: true, force: true });
}
}
function generateReport(results) {
const all = results.filter(Boolean);
const passed = all.filter((r) => r.adopted.count > 0);
const rows = results
.filter(Boolean)
.map((r) => {
const hasMetadata = r.manifest.some((m) => m.duration || m.width);
const cacheHit = r.resolveTest?._source === "cached";
const missHandled = r.resolveMiss?.ok === false;
return `
| ${r.name} |
${r.baseline.fileCount} files, ${r.baseline.htmlCount} comp${r.baseline.htmlCount === 1 ? "" : "s"} |
${r.adopted.count} adopted |
${hasMetadata ? "with metadata" : "no metadata"} |
${cacheHit ? "cache hit" : "no hit"} |
${missHandled ? "handled" : "unexpected"} |
`;
})
.join("\n");
const details = results
.filter(Boolean)
.filter((r) => r.adopted.count > 0)
.map((r) => {
const assetRows = r.manifest
.map((m) => {
const dur = m.duration != null ? `${m.duration}s` : "—";
const dims = m.width && m.height ? `${m.width}×${m.height}` : "—";
return `| ${m.id} | ${m.type} | ${dur} | ${dims} | ${m.path} | ${m.description || ""} |
`;
})
.join("\n");
const coveredCount = r.assetRefs.filter((c) => c.covered).length;
const totalRefs = r.assetRefs.length;
const coveragePct = totalRefs > 0 ? Math.round((coveredCount / totalRefs) * 100) : 100;
const refRows = r.assetRefs
.map(
(c) =>
`| ${c.composition} | ${c.ref} | ${c.covered ? "covered" : "not in manifest"} |
`,
)
.join("\n");
return `
${r.name}
${r.compositions.length} composition${r.compositions.length === 1 ? "" : "s"}: ${r.compositions.join(", ")}
Baseline (no media-use)
Agent sees: ${r.baseline.fileCount} raw files in assets/
No metadata, no type info, no relationship to compositions.
${r.baseline.files.join("\n") || "(no assets)"}
With media-use (after --adopt)
Agent reads index.md — structured, typed, with metadata:
${escapeHtml(r.index)}
${
totalRefs > 0
? `
Composition → asset coverage ${coveragePct}% (${coveredCount}/${totalRefs} refs)
| composition | asset reference | in manifest? |
|---|
${refRows}
`
: ""
}
Manifest records
| id | type | dur | dims | path | description |
|---|
${assetRows}
`;
})
.join("\n");
return `media-use eval report
media-use eval report
${new Date().toISOString().slice(0, 10)} · ${all.length} blocks evaluated · baseline vs. media-use --adopt
${all.length}
blocks tested
${passed.length}
with assets
${all.reduce((s, r) => s + r.adopted.count, 0)}
assets adopted
${all.filter((r) => r.manifest.some((m) => m.duration || m.width)).length}
with ffprobe metadata
${(() => {
const refs = all.flatMap((r) => r.assetRefs);
const covered = refs.filter((c) => c.covered).length;
return refs.length > 0 ? Math.round((covered / refs.length) * 100) + "%" : "—";
})()}
composition coverage
Results matrix
| Block | Baseline | Adopted | Metadata | Cache hit | Miss handling |
|---|
${rows}
Before / after comparisons
${details}
${
passed.length >= 3
? `Ship it. ${passed.length}/${all.length} blocks adopted successfully with metadata. Resolve cache hits work. Miss handling is clean.`
: `Needs work. Only ${passed.length} blocks adopted. Check the failures above.`
}
`;
}
function escapeHtml(str) {
return str.replace(/&/g, "&").replace(//g, ">");
}
console.log("media-use eval · running against registry blocks...\n");
const results = [];
for (const block of TEST_BLOCKS) {
const fullPath = join(REPO_ROOT, block);
if (!existsSync(fullPath)) {
console.log(` skip ${basename(block)} (not found)`);
results.push(null);
continue;
}
process.stdout.write(` ${basename(block)}...`);
const result = evalBlock(block);
if (result) {
console.log(
` ${result.adopted.count} adopted, ${result.manifest.filter((m) => m.duration || m.width).length} with metadata`,
);
} else {
console.log(" failed");
}
results.push(result);
}
const report = generateReport(results);
const outPath = join(SCRIPT_DIR, "..", "eval-report.html");
writeFileSync(outPath, report);
console.log(`\nReport: ${outPath}`);