chore: import upstream snapshot with attribution
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled
regression / regression-shards (style-16-prod style-9-prod style-17-prod iframe-render-compat variables-prod mp4-h265-sdr, shard-4) (push) Has been cancelled
regression / regression-shards (style-4-prod style-11-prod style-2-prod animejs-adapter typegpu-adapter parallel-capture-regression, shard-5) (push) Has been cancelled
regression / regression-shards (style-7-prod style-8-prod style-10-prod css-spinner-render-compat webm-transparency mp4-h264-sdr webm-vp9, shard-3) (push) Has been cancelled
regression / regression-shards (sub-composition-video style-18-prod raf-ball-render-compat font-variant-numeric sub-comp-t0 sub-comp-id-selector, shard-7) (push) Has been cancelled
Windows render verification / Detect changes (push) Has been cancelled
Windows render verification / Preflight (lint + format) (push) Has been cancelled
Windows render verification / Render on windows-latest (push) Has been cancelled
Windows render verification / Tests on windows-latest (push) Has been cancelled
CI / Detect changes (push) Has been cancelled
CI / Build (push) Has been cancelled
CI / Lint (push) Has been cancelled
CI / Fallow audit (push) Has been cancelled
CI / Format (push) Has been cancelled
CI / Typecheck (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Producer: integration tests (push) Has been cancelled
CI / Producer: unit tests (push) Has been cancelled
CI / File size check (push) Has been cancelled
CI / Test: skills (push) Has been cancelled
CI / Skills: manifest in sync (push) Has been cancelled
CI / CLI: npx shim (macos-latest) (push) Has been cancelled
CI / CLI: npx shim (ubuntu-latest) (push) Has been cancelled
CI / CLI: npx shim (windows-latest) (push) Has been cancelled
CI / SDK: unit + contract + smoke (push) Has been cancelled
CI / Test: runtime contract (push) Has been cancelled
CI / Studio: load smoke (push) Has been cancelled
CI / Smoke: global install (push) Has been cancelled
CI / CLI smoke (required) (push) Has been cancelled
CI / Semantic PR title (push) Has been cancelled
Player perf / Detect changes (push) Has been cancelled
Player perf / Preflight (lint + format) (push) Has been cancelled
Player perf / player-perf (push) Has been cancelled
Player perf / Perf: drift (push) Has been cancelled
Player perf / Perf: fps (push) Has been cancelled
Player perf / Perf: parity (push) Has been cancelled
Player perf / Perf: scrub (push) Has been cancelled
Player perf / Perf: load (push) Has been cancelled
preview-regression / Detect changes (push) Has been cancelled
preview-regression / Preflight (lint + format) (push) Has been cancelled
preview-regression / Preview parity (push) Has been cancelled
preview-regression / preview-regression (push) Has been cancelled
regression / regression (push) Has been cancelled
regression / Detect changes (push) Has been cancelled
regression / Preflight (lint + format) (push) Has been cancelled
regression / regression-shards (hdr-regression style-5-prod style-3-prod mov-prores, shard-1) (push) Has been cancelled
regression / regression-shards (overlay-montage-prod style-12-prod chat missing-host-comp-id png-sequence portrait-edge-bleed, shard-6) (push) Has been cancelled
regression / regression-shards (style-13-prod style-6-prod vignelli-stacking gsap-letters-render-compat audio-mux-parity, shard-8) (push) Has been cancelled
regression / regression-shards (style-15-prod hdr-hlg-regression style-1-prod many-cuts vfr-screen-recording render-symlinked-assets, shard-2) (push) Has been cancelled
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Docs / Validate docs (push) Has been cancelled
Sync skills to ClawHub / Publish changed skills (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env node
|
||||
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
||||
// BGM + SFX for every video workflow (product-launch, general-video, pr-to-video,
|
||||
// …). Workflows do NOT vendor a copy: they write a neutral `audio_request.json`
|
||||
// (a tiny per-workflow adapter maps their storyboard/scenes into it) and call:
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// The three capabilities degrade on ONE switch — whether HeyGen is configured
|
||||
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
// ],
|
||||
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
||||
// "query": "calm cinematic underscore", // mood for retrieval
|
||||
// "prompt": null, // full prompt for generation (else inferred)
|
||||
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
||||
// }
|
||||
//
|
||||
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
||||
// { tts_provider, voice_id,
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
// workflow can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
||||
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
||||
// before assembling.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import {
|
||||
ffprobeDuration,
|
||||
pickProvider,
|
||||
resolveVoiceId,
|
||||
synthesizeOne,
|
||||
transcribeWav,
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
const has = (name) => argv.includes(`--${name}`);
|
||||
const die = (m) => {
|
||||
console.error(`✗ audio engine: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Two independent reports of an unbounded Promise.all over TTS lines
|
||||
// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
|
||||
// whisper-transcribe lines on a resource-constrained laptop, the other saw
|
||||
// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
|
||||
// retry once the model was cached. Kokoro/Whisper each load their own local
|
||||
// model per subprocess, so firing every line at once multiplies that cost by
|
||||
// the line count. mapWithConcurrency caps how many run at once — still
|
||||
// parallel, just bounded.
|
||||
const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const providerOverride = flag("provider", null);
|
||||
const bgmModeOverride = flag("bgm-mode", null);
|
||||
const noBgm = has("no-bgm");
|
||||
const voiceOverride = flag("voice", null);
|
||||
const speedOverride = flag("speed", null);
|
||||
const langOverride = flag("lang", null);
|
||||
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
||||
|
||||
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
||||
} catch (e) {
|
||||
die(`audio_request.json parse: ${e.message}`);
|
||||
}
|
||||
const lines = Array.isArray(request.lines) ? request.lines : [];
|
||||
const lang = langOverride || request.lang || "en";
|
||||
const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
|
||||
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
const headers = heygenOK ? heygenAuthHeaders() : null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const prev = existsSync(outPath) ? JSON.parse(readFileSync(outPath, "utf8")) : {};
|
||||
const anomalies = [];
|
||||
|
||||
// ── TTS ───────────────────────────────────────────────────────────────────────
|
||||
let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = `assets/voice/${id}.wav`;
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
||||
const dur = ffprobeDuration(abs);
|
||||
if (!isFinite(dur) || dur <= 0) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
for (const v of voices)
|
||||
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
||||
}
|
||||
const hasVoice = voices.length > 0;
|
||||
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
||||
|
||||
// ── BGM ─────────────────────────────────────────────────────────────────────
|
||||
let bgm = prev.bgm ?? null;
|
||||
const bgmFields = {
|
||||
bgm_pending: prev.bgm_pending ?? false,
|
||||
bgm_provider: prev.bgm_provider ?? null,
|
||||
bgm_pid: prev.bgm_pid ?? null,
|
||||
bgm_log: prev.bgm_log ?? null,
|
||||
bgm_mode: prev.bgm_mode ?? null,
|
||||
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
||||
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
||||
bgm_loop_count: prev.bgm_loop_count ?? null,
|
||||
};
|
||||
if (only.has("bgm")) {
|
||||
bgm = null;
|
||||
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
||||
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
||||
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
||||
// generate (a caller with no wait-bgm step, e.g. product-launch, must not get
|
||||
// a pending job it can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
);
|
||||
mode = "none";
|
||||
}
|
||||
|
||||
if (mode === "none") {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({ query: request.bgm?.query, headers, hyperframesDir, hasVoice });
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
||||
} else {
|
||||
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
||||
}
|
||||
} catch (e) {
|
||||
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
||||
}
|
||||
} else {
|
||||
// generate
|
||||
const prompt = inferBgmPrompt({
|
||||
userPrompt: request.bgm?.prompt,
|
||||
blob: request.bgm?.blob || request.bgm?.query,
|
||||
archetype: request.bgm?.archetype,
|
||||
arc: request.bgm?.arc,
|
||||
});
|
||||
const gen = generateBgmDetached({
|
||||
prompt,
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
} else {
|
||||
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
||||
bgmFields.bgm_pending = true;
|
||||
bgmFields.bgm_provider = gen.provider;
|
||||
bgmFields.bgm_pid = gen.pid;
|
||||
bgmFields.bgm_log = gen.log;
|
||||
bgmFields.bgm_mode = gen.mode;
|
||||
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
||||
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
||||
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
||||
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SFX ─────────────────────────────────────────────────────────────────────
|
||||
let sfx = prev.sfx ?? [];
|
||||
if (only.has("sfx")) {
|
||||
const cues = lines.flatMap((l) =>
|
||||
(Array.isArray(l.sfx) ? l.sfx : [])
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
||||
const meta = {
|
||||
tts_provider: ttsProvider,
|
||||
voice_id: voiceId,
|
||||
bgm,
|
||||
...bgmFields,
|
||||
voices,
|
||||
sfx,
|
||||
total_duration_s: totalDuration,
|
||||
};
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
writeFileSync(outPath, JSON.stringify(meta, null, 2));
|
||||
|
||||
console.log(`✓ audio engine → ${outPath}`);
|
||||
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
||||
console.log(
|
||||
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
||||
);
|
||||
console.log(` total voice duration: ${totalDuration}s`);
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, rmSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Offline (heygenOK:false), no network.
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
|
||||
test("bundled SFX library resolves from the relocated path", async () => {
|
||||
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.match(sfx[0].file, /assets\/sfx\//);
|
||||
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an unknown cue is reported, not fatal", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0);
|
||||
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
||||
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
||||
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
||||
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
||||
// the published build is Kokoro-only.
|
||||
//
|
||||
// Usage:
|
||||
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
||||
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
||||
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
||||
// node heygen-tts.mjs --list # list starfish voices and exit
|
||||
//
|
||||
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
||||
// --voice (starfish id), --speed, --lang, --list.
|
||||
// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function flag(name, def) {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i < 0) return def;
|
||||
if (i + 1 >= argv.length) return true;
|
||||
const v = argv[i + 1];
|
||||
return v.startsWith("--") ? true : v;
|
||||
}
|
||||
const die = (m) => {
|
||||
console.error(`✗ heygen-tts: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
// First arg that isn't a flag or the -o value is the text / .txt path.
|
||||
const positional = (() => {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a.startsWith("--")) {
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith("--")) i++;
|
||||
continue;
|
||||
}
|
||||
if (a === "-o") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const output = resolve(
|
||||
(typeof flag("output") === "string" && flag("output")) ||
|
||||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
||||
"narration.wav",
|
||||
);
|
||||
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
||||
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
||||
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
||||
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
||||
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
||||
const listOnly = flag("list") === true;
|
||||
|
||||
loadEnvFromDir(process.cwd());
|
||||
let authHeaders;
|
||||
try {
|
||||
authHeaders = heygenAuthHeaders();
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
|
||||
// ---------- --list ----------
|
||||
if (listOnly) {
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: authHeaders,
|
||||
});
|
||||
for (const v of payload.data ?? payload.voices ?? []) {
|
||||
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ---------- resolve text + voice ----------
|
||||
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
||||
const text =
|
||||
positional.endsWith(".txt") && existsSync(resolve(positional))
|
||||
? readFileSync(resolve(positional), "utf8").trim()
|
||||
: positional;
|
||||
if (!text) die("input text is empty");
|
||||
|
||||
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
||||
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
||||
|
||||
// ---------- synthesize (shared engine code) ----------
|
||||
const { ok, words } = await synthesizeOne({
|
||||
provider: "heygen",
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: output,
|
||||
hyperframesDir: process.cwd(),
|
||||
});
|
||||
if (!ok) die("synthesis failed (HeyGen request/transcode error)");
|
||||
|
||||
let wordCount = 0;
|
||||
if (wordsPath) {
|
||||
if (words && words.length) {
|
||||
mkdirSync(dirname(wordsPath), { recursive: true });
|
||||
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
||||
wordCount = words.length;
|
||||
} else {
|
||||
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
||||
}
|
||||
}
|
||||
|
||||
const dur = ffprobeDuration(output);
|
||||
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
||||
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
||||
@@ -0,0 +1,257 @@
|
||||
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
||||
// same way as TTS/SFX:
|
||||
//
|
||||
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
||||
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
||||
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
||||
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
||||
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
||||
// DETACHED so the engine can return while audio renders; the caller marks
|
||||
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
||||
//
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
||||
|
||||
// Default BGM level. Under narration music is a bed that must stay under the
|
||||
// voice — 0.12 linear ≈ -18 dB. A silent film (no voice) has no voice to duck
|
||||
// beneath, so BGM sits forward at 0.9. Callers may override per composition.
|
||||
export const BGM_BED_VOLUME = 0.12;
|
||||
export const BGM_SILENT_VOLUME = 0.9;
|
||||
export const bgmDefaultVolume = (hasVoice) => (hasVoice ? BGM_BED_VOLUME : BGM_SILENT_VOLUME);
|
||||
|
||||
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
||||
const BGM_PY_PROBE =
|
||||
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
||||
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
||||
const LYRIA_PY_PROBE = "import google.genai";
|
||||
|
||||
function pyOk(probe) {
|
||||
const { cmd, args } = pythonInvocation(["-c", probe]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
|
||||
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
|
||||
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
|
||||
// actually installs. `-m pip` also guarantees the packages land in the SAME
|
||||
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
|
||||
// different Python installation than `python3` if more than one is on PATH.
|
||||
function pipInstall(deps) {
|
||||
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = "assets/bgm/track.mp3";
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
volume: bgmDefaultVolume(hasVoice),
|
||||
query: q,
|
||||
mode: "retrieve",
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
||||
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
||||
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
||||
// engine also calls it when generate has only a plain mood query.
|
||||
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
||||
if (userPrompt) return userPrompt;
|
||||
const b = String(blob).toLowerCase();
|
||||
let base;
|
||||
let bpm;
|
||||
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
||||
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
||||
bpm = 100;
|
||||
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
||||
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
||||
bpm = 92;
|
||||
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
||||
base = "playful electronic, warm pads, light percussion";
|
||||
bpm = 115;
|
||||
} else {
|
||||
base = "uplifting corporate tech, bright modern piano with synth pads";
|
||||
bpm = 108;
|
||||
}
|
||||
const at = String(archetype).toLowerCase();
|
||||
const ar = String(arc).toLowerCase();
|
||||
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
||||
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
||||
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
||||
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
||||
if (/cascade|feature.benefit/.test(at))
|
||||
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
||||
if (/demo.loop|question.+answer/.test(at))
|
||||
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
||||
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
||||
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
||||
if (/excit|awe|power|triumph/.test(ar))
|
||||
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
||||
if (/trust|ease|clarity|reassur/.test(ar))
|
||||
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
||||
return `${base}, BPM ${bpm}, MAJOR`;
|
||||
}
|
||||
|
||||
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
||||
// Returns a bgmMeta the caller folds into audio_meta:
|
||||
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
||||
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
||||
export function generateBgmDetached({
|
||||
prompt,
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
}) {
|
||||
const rel = "assets/bgm/track.wav";
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
const targetS = Math.max(1, durationS);
|
||||
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
// generation itself is detached, so the engine still returns promptly.
|
||||
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
"--output",
|
||||
abs,
|
||||
"--duration",
|
||||
String(targetS),
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "lyria",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
if (pyOk(BGM_PY_PROBE)) {
|
||||
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
||||
provider: "musicgen",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
seed_duration_s: seedS,
|
||||
loop_count: loops,
|
||||
};
|
||||
}
|
||||
|
||||
closeSync(fd);
|
||||
return {
|
||||
disabled: true,
|
||||
reason: lyriaConfigured
|
||||
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
||||
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
||||
};
|
||||
}
|
||||
|
||||
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
||||
// positional limit), then trim it down or crossfade-loop it up to the target.
|
||||
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
||||
return `
|
||||
import math, os, sys, traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
||||
|
||||
prompt = ${JSON.stringify(prompt)}
|
||||
out_path = ${JSON.stringify(abs)}
|
||||
target_s = float(${targetS.toFixed(3)})
|
||||
seed_s = float(${seedS.toFixed(3)})
|
||||
token_rate = 50
|
||||
crossfade_s = 0.3
|
||||
|
||||
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
||||
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
||||
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
||||
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
||||
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
||||
return arr
|
||||
|
||||
def loop_crossfade(seed, target_len, xf):
|
||||
if seed.shape[0] >= target_len: return seed[:target_len]
|
||||
xf = min(xf, seed.shape[0] // 2)
|
||||
if xf < 1:
|
||||
reps = int(math.ceil(target_len / seed.shape[0]))
|
||||
return np.tile(seed, reps)[:target_len]
|
||||
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
||||
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
||||
out = seed.copy()
|
||||
while out.shape[0] < target_len:
|
||||
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
||||
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
||||
return out[:target_len]
|
||||
|
||||
try:
|
||||
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
||||
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
||||
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
||||
model.eval()
|
||||
sr = int(model.config.audio_encoder.sampling_rate)
|
||||
gen_s = min(seed_s, target_s)
|
||||
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
||||
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
||||
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
||||
audio = model.generate(**inputs, max_new_tokens=tokens)
|
||||
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
||||
peak = float(np.max(np.abs(seed)))
|
||||
if peak > 1e-6: seed = seed * (0.89 / peak)
|
||||
want = max(1, int(round(target_s * sr)))
|
||||
if seed.shape[0] >= want:
|
||||
final = seed[:want].copy()
|
||||
else:
|
||||
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
||||
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
||||
else: final = final[:want]
|
||||
final = apply_fade(final, sr)
|
||||
peak = float(np.max(np.abs(final)))
|
||||
if peak > 1.0: final = final / peak
|
||||
sf.write(out_path, final, sr)
|
||||
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
||||
except Exception:
|
||||
traceback.print_exc(); sys.exit(1)
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm.mjs";
|
||||
|
||||
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
|
||||
// hotter than a music bed under a voice should be. The default under narration
|
||||
// must be a proper bed (≈ -18 dB); a silent film keeps the louder default.
|
||||
|
||||
const dbfs = (linear) => 20 * Math.log10(linear);
|
||||
|
||||
test("BGM under narration is a bed near -18 dB", () => {
|
||||
assert.equal(bgmDefaultVolume(true), BGM_BED_VOLUME);
|
||||
assert.equal(BGM_BED_VOLUME, 0.12);
|
||||
const db = dbfs(BGM_BED_VOLUME);
|
||||
assert.ok(db < -17 && db > -19, `bed should be ≈ -18 dB, got ${db.toFixed(1)} dB`);
|
||||
});
|
||||
|
||||
test("a silent film (no voice) keeps BGM forward", () => {
|
||||
assert.equal(bgmDefaultVolume(false), BGM_SILENT_VOLUME);
|
||||
assert.equal(BGM_SILENT_VOLUME, 0.9);
|
||||
});
|
||||
|
||||
test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
|
||||
// Voice sits at data-volume="1" (0 dBFS); the bed must be ~16+ dB under it.
|
||||
const separation = dbfs(1) - dbfs(bgmDefaultVolume(true));
|
||||
assert.ok(
|
||||
separation >= 16,
|
||||
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
// mapWithConcurrency — run `fn` over `items` with at most `limit` in flight at
|
||||
// once. Preserves input order in the result array regardless of completion order.
|
||||
export async function mapWithConcurrency(items, limit, fn) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapWithConcurrency } from "./concurrency.mjs";
|
||||
|
||||
// Regression: audio.mjs used a bare Promise.all(lines.map(synthLine)) to
|
||||
// synthesize every TTS line at once, spawning one Kokoro/whisper model load
|
||||
// per line concurrently. Two independent reports of this overwhelming a
|
||||
// machine (OOM, and cold-start contention causing spurious failures).
|
||||
// mapWithConcurrency is the extracted cap; test it in isolation since
|
||||
// audio.mjs itself is a script (runs CLI/exit side effects on import).
|
||||
test("processes every item and preserves input order regardless of completion order", async () => {
|
||||
const order = [5, 1, 3, 2, 4];
|
||||
const results = await mapWithConcurrency(order, 2, async (n) => {
|
||||
await new Promise((r) => setTimeout(r, n));
|
||||
return n * 10;
|
||||
});
|
||||
assert.deepEqual(results, [50, 10, 30, 20, 40]);
|
||||
});
|
||||
|
||||
test("never runs more than `limit` at once", async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const items = Array.from({ length: 10 }, (_, i) => i);
|
||||
await mapWithConcurrency(items, 3, async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
});
|
||||
assert.equal(maxInFlight, 3);
|
||||
});
|
||||
|
||||
test("limit larger than the item count runs everything without hanging", async () => {
|
||||
const results = await mapWithConcurrency([1, 2], 10, async (n) => n * 2);
|
||||
assert.deepEqual(results, [2, 4]);
|
||||
});
|
||||
|
||||
test("empty input resolves to an empty array", async () => {
|
||||
const results = await mapWithConcurrency([], 4, async (n) => n);
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
||||
// pipeline. The credential resolver matches the hyperframes CLI auth: first
|
||||
// usable source wins — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
||||
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
||||
// overrides the dir). Vendored so the skill ships standalone. Pure node.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
||||
export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" };
|
||||
|
||||
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
||||
export function loadEnvFromDir(startDir) {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const envPath = join(dir, ".env");
|
||||
if (existsSync(envPath)) {
|
||||
for (const raw of readFileSync(envPath, "utf8").split("\n")) {
|
||||
let line = raw.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq < 1) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
let val = line.slice(eq + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
const q = val[0];
|
||||
const end = val.indexOf(q, 1);
|
||||
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
||||
}
|
||||
if (!(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// → { headers } | { expired: true } | null. Never throws.
|
||||
export function heygenCredential() {
|
||||
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
||||
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
||||
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
if (!existsSync(file)) return null;
|
||||
const raw = readFileSync(file, "utf8").trim();
|
||||
if (!raw) return null;
|
||||
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
||||
|
||||
// A malformed credentials file (partial write / wrong shape) must degrade to
|
||||
// "no credential", not crash the engine at startup — this function never throws.
|
||||
let cred;
|
||||
try {
|
||||
cred = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const oauth = cred.oauth;
|
||||
if (oauth?.access_token) {
|
||||
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
||||
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
||||
if (!cred.api_key) return { expired: true };
|
||||
}
|
||||
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
||||
return null;
|
||||
}
|
||||
|
||||
// → "oauth" | "api_key" | null. Same oauth-vs-api-key check heygenAuthHeaders()
|
||||
// makes internally, exposed on its own so callers that only need to *tag* the
|
||||
// auth path (telemetry) don't have to parse headers back apart. Never throws:
|
||||
// no credential (or an expired one) is just `null`, same as a fresh resolve
|
||||
// with nothing to tag.
|
||||
export function heygenAuthMethod() {
|
||||
const cred = heygenCredential();
|
||||
if (!cred?.headers) return null;
|
||||
return "Authorization" in cred.headers ? "oauth" : "api_key";
|
||||
}
|
||||
|
||||
// → auth headers object, or throw with a fix hint.
|
||||
export function heygenAuthHeaders() {
|
||||
const cred = heygenCredential();
|
||||
if (cred?.headers) {
|
||||
// Only tag OAuth (Bearer) traffic as cli-source — the backend uses it to
|
||||
// grant the free allowance for OAuth requests and ignores it for API-key
|
||||
// (X-Api-Key) traffic, where it's dead metadata.
|
||||
const isOauth = "Authorization" in cred.headers;
|
||||
return isOauth ? { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS } : { ...cred.headers };
|
||||
}
|
||||
if (cred?.expired)
|
||||
throw new Error(
|
||||
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
||||
);
|
||||
throw new Error(
|
||||
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
||||
);
|
||||
}
|
||||
|
||||
// Authed JSON request against the v3 API; throws on a non-OK status.
|
||||
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
||||
const opts = { method, headers: { ...headers } };
|
||||
if (body !== undefined) {
|
||||
opts.headers["Content-Type"] = "application/json";
|
||||
opts.body = JSON.stringify(body);
|
||||
}
|
||||
const res = await fetch(`${HEYGEN_BASE}${path}`, opts);
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => "");
|
||||
throw new Error(
|
||||
`HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`,
|
||||
);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// Download a (presigned) URL to destPath; returns byte length.
|
||||
export async function downloadTo(url, destPath) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
||||
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
||||
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
||||
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
||||
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
||||
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
||||
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
||||
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
||||
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
||||
if (minScore != null) params.set("min_score", String(minScore));
|
||||
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
||||
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
||||
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
||||
const data = payload?.data ?? payload;
|
||||
if (Array.isArray(data)) return data;
|
||||
if (data && typeof data === "object") return Object.values(data);
|
||||
throw new Error(
|
||||
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenAuthMethod } from "./heygen.mjs";
|
||||
|
||||
function withCleanHeygenEnv(fn) {
|
||||
const previousApiKey = process.env.HEYGEN_API_KEY;
|
||||
const previousHyperframesApiKey = process.env.HYPERFRAMES_API_KEY;
|
||||
const previousConfigDir = process.env.HEYGEN_CONFIG_DIR;
|
||||
try {
|
||||
delete process.env.HEYGEN_API_KEY;
|
||||
delete process.env.HYPERFRAMES_API_KEY;
|
||||
delete process.env.HEYGEN_CONFIG_DIR;
|
||||
return fn();
|
||||
} finally {
|
||||
if (previousApiKey === undefined) delete process.env.HEYGEN_API_KEY;
|
||||
else process.env.HEYGEN_API_KEY = previousApiKey;
|
||||
if (previousHyperframesApiKey === undefined) delete process.env.HYPERFRAMES_API_KEY;
|
||||
else process.env.HYPERFRAMES_API_KEY = previousHyperframesApiKey;
|
||||
if (previousConfigDir === undefined) delete process.env.HEYGEN_CONFIG_DIR;
|
||||
else process.env.HEYGEN_CONFIG_DIR = previousConfigDir;
|
||||
}
|
||||
}
|
||||
|
||||
test("heygenAuthHeaders does not tag API-key requests as CLI traffic", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
// API-key requests use normal billing; the backend ignores the cli-source
|
||||
// header for them, so it's not sent.
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "hg_test",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders tags OAuth requests as CLI traffic", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_test",
|
||||
"X-HeyGen-Source": "cli",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns api_key for an env API key, without tagging headers", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns oauth for a live OAuth credential", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null with no credential at all", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir; // no credentials file written
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
// python.mjs — resolve which Python 3 executable to spawn, per platform.
|
||||
//
|
||||
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
|
||||
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
|
||||
// macOS/Linux, but on Windows the python.org installer only creates
|
||||
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
|
||||
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
|
||||
// standard Windows Python install, silently disabling every Python-backed audio
|
||||
// feature until the user hand-creates a `python3.exe` shim (reported twice).
|
||||
//
|
||||
// Resolve once, per process: probe the platform's candidates in order and take
|
||||
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
|
||||
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
function defaultProbe(cmd, args) {
|
||||
try {
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the argv prefix that launches Python 3 on this platform.
|
||||
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
|
||||
*
|
||||
* Pure except for `probe` (which runs `<cmd> … --version`); both `platform`
|
||||
* and `probe` are injectable so every branch is unit-testable without spawning.
|
||||
* If nothing probes OK, falls back to the canonical name for the platform so
|
||||
* the eventual spawn fails loudly exactly as it did before — never worse.
|
||||
*/
|
||||
export function resolvePythonCommand(platform = process.platform, probe = defaultProbe) {
|
||||
const candidates =
|
||||
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
|
||||
for (const prefix of candidates) {
|
||||
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
let cached = null;
|
||||
|
||||
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
|
||||
export function pythonCommand() {
|
||||
if (!cached) cached = resolvePythonCommand();
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
|
||||
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
|
||||
* prefix args) ahead of the caller's own arguments.
|
||||
*/
|
||||
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
|
||||
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
|
||||
}
|
||||
|
||||
/** Test-only: clear the cached resolution so a test can re-probe. */
|
||||
export function _resetPythonCommandCacheForTests() {
|
||||
cached = null;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { resolvePythonCommand, pythonInvocation } from "./python.mjs";
|
||||
|
||||
// Regression: on Windows a standard python.org install has no `python3.exe`
|
||||
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
|
||||
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
|
||||
// injectable platform/probe params so all branches are testable without
|
||||
// spawning a real interpreter.
|
||||
|
||||
// probeFor(names): a probe that reports success only for the given argv-0 names.
|
||||
function probeFor(...names) {
|
||||
const ok = new Set(names);
|
||||
return (cmd) => ok.has(cmd);
|
||||
}
|
||||
|
||||
test("non-win32 uses python3 when it runs", () => {
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
|
||||
// The exact reported scenario: no python3, but `python` exists.
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to the py launcher with -3 when only py exists", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
|
||||
});
|
||||
|
||||
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd, args) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return cmd === "py";
|
||||
};
|
||||
resolvePythonCommand("win32", probe);
|
||||
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
|
||||
});
|
||||
|
||||
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
|
||||
// No interpreter anywhere — must not throw, and must return python3 so the
|
||||
// eventual spawn fails exactly as it did before this fix, never worse.
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("win32", () => false),
|
||||
["python3"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("linux", () => false),
|
||||
["python3"],
|
||||
);
|
||||
});
|
||||
|
||||
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
|
||||
cmd: "python",
|
||||
args: ["-c", "import x"],
|
||||
});
|
||||
// The py launcher's -3 must stay ahead of the caller's own arguments.
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
|
||||
cmd: "py",
|
||||
args: ["-3", "-c", "import x"],
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
|
||||
// per-cue merge): the decision is made once, by whether HeyGen is configured —
|
||||
// mirroring how TTS and BGM degrade.
|
||||
//
|
||||
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
|
||||
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
|
||||
// library is NOT consulted.
|
||||
// HeyGen credential absent → resolve cues against the bundled 21-file
|
||||
// library (assets/sfx/manifest.json), copying matched files into the
|
||||
// project. Offline, deterministic, free.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
|
||||
export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
const seen = new Set();
|
||||
const uniq = cues.filter((c) => {
|
||||
const k = `${c.id}:${c.name}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (heygenOK) {
|
||||
for (const { id, name } of uniq) {
|
||||
try {
|
||||
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
|
||||
// silently drops most named cues — floor to 0.4. (BGM/music score high
|
||||
// and keep the default.)
|
||||
const results = await searchSounds(name, "sound_effects", headers, {
|
||||
limit: 3,
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results.length) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
|
||||
continue;
|
||||
}
|
||||
const top = results[0];
|
||||
const file = `assets/sfx/${slug(name)}.mp3`;
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file,
|
||||
source: "heygen",
|
||||
offset_s: 0,
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
|
||||
}
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
// ── offline: bundled library ──
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) {
|
||||
if (uniq.length)
|
||||
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
// Build lookups: by manifest key, by file basename, and by slug of either, so
|
||||
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
for (const { id, name } of uniq) {
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const destRel = `assets/sfx/${hit.file}`;
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// The bundled library may be incomplete: some installs of the skill ship
|
||||
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
|
||||
// a file we never copied produces a dangling reference that silently drops
|
||||
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
|
||||
// instead, so the audio_meta never references a missing file.
|
||||
if (!existsSync(dest)) {
|
||||
if (!existsSync(src)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
copyFileSync(src, dest);
|
||||
}
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
// behavior: a present file is copied + referenced; a missing file yields an
|
||||
// anomaly and NO dangling entry.
|
||||
|
||||
async function withDirs(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
|
||||
const libDir = join(root, "lib");
|
||||
const projDir = join(root, "proj");
|
||||
mkdirSync(libDir, { recursive: true });
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
try {
|
||||
// `await` is load-bearing: without it the finally cleanup runs before the
|
||||
// async test body resolves, deleting the temp dir mid-assertion.
|
||||
return await fn({ libDir, projDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("offline: copies and references a present bundled file", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1);
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.equal(anomalies.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,378 @@
|
||||
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
||||
// auto-detected from env, is the one documented in ../SKILL.md:
|
||||
//
|
||||
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
||||
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
||||
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
||||
// in the same call, so no separate transcribe pass.
|
||||
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
||||
// word timings → caller chains transcribeWav().
|
||||
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
||||
// CLI. No word timings → caller chains transcribeWav().
|
||||
//
|
||||
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
||||
// by the CLI — see the note above.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
return heygenCredential() !== null;
|
||||
}
|
||||
export function elevenlabsAvailable() {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return false;
|
||||
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
|
||||
const r = spawnSync(cmd, args, {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
|
||||
if (userProvider === "heygen" && !heygenAvailable())
|
||||
throw new Error(
|
||||
"provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx hyperframes auth login`)",
|
||||
);
|
||||
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
||||
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
||||
return userProvider;
|
||||
}
|
||||
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
||||
}
|
||||
|
||||
// ── voice resolution ──────────────────────────────────────────────────────────
|
||||
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
||||
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
||||
}
|
||||
// heygen — pin a fixed English default so the choice is deterministic. The old
|
||||
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
||||
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
||||
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
||||
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
const voices = payload.data ?? payload.voices ?? [];
|
||||
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
||||
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
||||
return pick.voice_id;
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
export function withWordIds(words) {
|
||||
return (words ?? []).map((w, i) => ({
|
||||
id: `w${i}`,
|
||||
text: w.text,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
}));
|
||||
}
|
||||
|
||||
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
|
||||
// though it exits non-zero with no output requested. Parsing pulled out as
|
||||
// a pure function so the ENOENT fallback below can be tested without
|
||||
// depending on whether ffprobe/ffmpeg are actually installed on the
|
||||
// machine running the tests.
|
||||
export function parseFfmpegDurationBanner(stderrText) {
|
||||
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
|
||||
if (!match) return NaN;
|
||||
const [, hours, minutes, seconds] = match;
|
||||
return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
|
||||
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
|
||||
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
|
||||
// already-successfully-synthesized TTS line, rather than "the tool for
|
||||
// measuring it is missing".
|
||||
function ffmpegDurationFallback(absPath) {
|
||||
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
|
||||
return parseFfmpegDurationBanner(r.stderr);
|
||||
}
|
||||
|
||||
export function ffprobeDuration(absPath) {
|
||||
const r = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", absPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
|
||||
if (r.status !== 0) return NaN;
|
||||
return parseFloat(String(r.stdout).trim());
|
||||
}
|
||||
|
||||
export function resolveNpxCliFromNpmExecPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (!npmExecPath) return null;
|
||||
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
||||
const npxCliPath =
|
||||
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
|
||||
return pathExists(npxCliPath) ? npxCliPath : null;
|
||||
}
|
||||
|
||||
export function resolveNpxCliPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
nodeExecPath = process.env.npm_node_execpath || process.execPath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
|
||||
if (fromNpm) return fromNpm;
|
||||
const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
|
||||
return pathExists(besideNode) ? besideNode : null;
|
||||
}
|
||||
|
||||
export function resolveSpawnCommand(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (cmd !== "npx" || platform !== "win32") {
|
||||
return { cmd, args, opts: { stdio: "ignore", ...opts } };
|
||||
}
|
||||
|
||||
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
|
||||
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
|
||||
// node, preserving request-provided values as argv data instead of shell text.
|
||||
const nodeExecPath = env.npm_node_execpath || process.execPath;
|
||||
const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
|
||||
if (!npxCliPath) return null;
|
||||
return {
|
||||
cmd: nodeExecPath,
|
||||
args: [npxCliPath, ...args.map((arg) => String(arg))],
|
||||
opts: { stdio: "ignore", windowsHide: true, ...opts },
|
||||
};
|
||||
}
|
||||
|
||||
// `platform`/`spawnFn` params (default process.platform / the real spawn)
|
||||
// exist so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable, so mock.method can't patch it).
|
||||
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
|
||||
let _warnedNpxResolution = false;
|
||||
/** Test-only: reset the one-shot npx-resolution warning latch. */
|
||||
export function _resetNpxResolutionWarnForTests() {
|
||||
_warnedNpxResolution = false;
|
||||
}
|
||||
|
||||
export function spawnP(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
spawnFn = spawn,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// resolveSpawnCommand only returns null for the npx-on-win32 case where
|
||||
// neither npm's configured CLI nor the beside-node fallback exists. Without
|
||||
// this, every call silently returns status:-1 and stdio:"ignore" hides why.
|
||||
if (!_warnedNpxResolution) {
|
||||
_warnedNpxResolution = true;
|
||||
const reason = env.npm_execpath
|
||||
? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
|
||||
: "npm_execpath is unset and the beside-node npm fallback could not be found";
|
||||
console.error(
|
||||
`[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
|
||||
`Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
|
||||
`\`npx\`/\`npm run\` with a valid npm_execpath.`,
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ status: -1 });
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
|
||||
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
||||
p.on("error", () => resolve({ status: -1 }));
|
||||
});
|
||||
}
|
||||
|
||||
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
||||
function transcodeToWav(bytes, destWav) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
||||
const tmp = join(td, "a.mp3");
|
||||
writeFileSync(tmp, bytes);
|
||||
mkdirSync(dirname(destWav), { recursive: true });
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return ff.status === 0 && existsSync(destWav);
|
||||
}
|
||||
|
||||
const ELEVENLABS_PY = `
|
||||
import os, sys
|
||||
from elevenlabs.client import ElevenLabs
|
||||
from elevenlabs import save
|
||||
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
||||
text = open(sys.argv[1]).read()
|
||||
audio = client.text_to_speech.convert(
|
||||
text=text, voice_id=sys.argv[2],
|
||||
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
||||
)
|
||||
save(audio, sys.argv[3])
|
||||
`;
|
||||
|
||||
// ── synthesize one line ───────────────────────────────────────────────────────
|
||||
// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
|
||||
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
||||
// (caller must transcribeWav). Never throws; failures return { ok:false, error }
|
||||
// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
|
||||
export async function synthesizeOne({
|
||||
provider,
|
||||
text,
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
|
||||
// and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
|
||||
// so on a fresh project (no assets/voice/ yet) the save fails and the line is
|
||||
// silently dropped as "TTS failed - omitted". Create it first, like the other
|
||||
// providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
|
||||
// { ok:false } like the rest of this branch rather than throwing (the
|
||||
// function's contract is "never throws; failures return { ok:false }").
|
||||
try {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
} catch {
|
||||
return { ok: false, words: null };
|
||||
}
|
||||
const { cmd, args } = pythonInvocation([
|
||||
"-c",
|
||||
ELEVENLABS_PY,
|
||||
writeTmpText(text),
|
||||
voiceId,
|
||||
wavAbs,
|
||||
]);
|
||||
const r = await spawnP(cmd, args, {});
|
||||
return synthResult(r, wavAbs, "elevenlabs (python)");
|
||||
}
|
||||
// kokoro — via the published CLI; --output is relative to the project dir.
|
||||
const wavRel = relTo(hyperframesDir, wavAbs);
|
||||
const args = ["hyperframes", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel];
|
||||
if (lang !== "en") args.push("--lang", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
return synthResult(r, wavAbs, "kokoro (npx hyperframes tts)");
|
||||
}
|
||||
|
||||
// Shape a spawn result into { ok, words, error }, naming why on failure so the
|
||||
// caller surfaces it instead of a bare "TTS failed".
|
||||
export function synthResult(r, wavAbs, label) {
|
||||
if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
|
||||
const why =
|
||||
r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
|
||||
return { ok: false, words: null, error: why };
|
||||
}
|
||||
|
||||
// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
|
||||
// Every failure path returns an `error` string so the caller can surface WHY a
|
||||
// line was dropped instead of the bare "TTS failed" that hid the real cause
|
||||
// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
|
||||
export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
|
||||
const requestJSON = deps.heygenJSON ?? heygenJSON;
|
||||
const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
|
||||
const fetchImpl = deps.fetch ?? fetch;
|
||||
const transcode = deps.transcodeToWav ?? transcodeToWav;
|
||||
try {
|
||||
const body = { text, voice_id: voiceId, speed };
|
||||
if (lang !== "en") body.language = lang;
|
||||
const payload = await requestJSON(`/voices/speech`, {
|
||||
method: "POST",
|
||||
headers: authHeaders(),
|
||||
body,
|
||||
});
|
||||
const inner = payload.data ?? payload;
|
||||
if (!inner.audio_url) {
|
||||
return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
|
||||
}
|
||||
const res = await fetchImpl(inner.audio_url);
|
||||
if (!res.ok) {
|
||||
return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
|
||||
}
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
||||
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
||||
if (wavAbs.endsWith(".wav")) {
|
||||
if (!transcode(bytes, wavAbs)) {
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: "wav transcode failed (ffmpeg)",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
}
|
||||
const words = Array.isArray(inner.word_timestamps)
|
||||
? inner.word_timestamps
|
||||
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
||||
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
||||
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
||||
: [];
|
||||
return { ok: true, words };
|
||||
} catch (e) {
|
||||
return { ok: false, words: null, error: e?.message ? String(e.message) : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
||||
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
||||
// --dir so parallel scenes don't collide on transcript.json.
|
||||
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
||||
const model = lang === "en" ? "small.en" : "small";
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
||||
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
||||
if (lang !== "en") args.push("--language", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
let words = null;
|
||||
if (r.status === 0) {
|
||||
const src = join(td, "transcript.json");
|
||||
if (existsSync(src)) {
|
||||
try {
|
||||
const arr = JSON.parse(readFileSync(src, "utf8"));
|
||||
if (Array.isArray(arr) && arr.length) words = arr;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return words;
|
||||
}
|
||||
|
||||
// ── tiny local utils ──────────────────────────────────────────────────────────
|
||||
function writeTmpText(text) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
||||
const p = join(td, "line.txt");
|
||||
writeFileSync(p, text);
|
||||
return p;
|
||||
}
|
||||
function relTo(base, abs) {
|
||||
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
resolveNpxCliFromNpmExecPath,
|
||||
resolveNpxCliPath,
|
||||
resolveSpawnCommand,
|
||||
spawnP,
|
||||
_resetNpxResolutionWarnForTests,
|
||||
} from "./tts.mjs";
|
||||
|
||||
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
|
||||
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
|
||||
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
|
||||
// need to touch the real process.platform or mock node:child_process (whose
|
||||
// ESM exports are non-configurable).
|
||||
function fakeSpawn(captured) {
|
||||
return (cmd, args, opts) => {
|
||||
captured.push({ cmd, args, opts });
|
||||
const p = new EventEmitter();
|
||||
setImmediate(() => p.emit("exit", 0));
|
||||
return p;
|
||||
};
|
||||
}
|
||||
|
||||
const envWithNpxCli = {
|
||||
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
|
||||
npm_node_execpath: "/opt/node/bin/node",
|
||||
};
|
||||
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
|
||||
const pathExists = (path) => path === npxCliPath;
|
||||
|
||||
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
|
||||
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
|
||||
});
|
||||
|
||||
test("resolveNpxCliPath finds npx-cli beside node when npm_execpath is unset", () => {
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const expected = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
assert.equal(
|
||||
resolveNpxCliPath(undefined, node, (path) => path === expected),
|
||||
expected,
|
||||
);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(resolved.args, [
|
||||
npxCliPath,
|
||||
"hyperframes",
|
||||
"tts",
|
||||
"C:\\Users\\Test User\\line.txt",
|
||||
"--voice",
|
||||
"am_michael",
|
||||
]);
|
||||
assert.equal(resolved.opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "hello & calc"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.deepEqual(resolved.args, [npxCliPath, "hyperframes", "tts", "hello & calc"]);
|
||||
});
|
||||
|
||||
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(captured[0].args, [npxCliPath, "hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for npx on darwin/linux", async () => {
|
||||
const captured = [];
|
||||
await spawnP("npx", ["hyperframes", "tts"], {}, "darwin", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "npx");
|
||||
assert.deepEqual(captured[0].args, ["hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "python3");
|
||||
assert.deepEqual(captured[0].args, ["-c", "pass"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP resolves npx beside node when npm_execpath is unset on win32", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const captured = [];
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const npxCli = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
const result = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
{ npm_node_execpath: node },
|
||||
(path) => path === npxCli,
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, node);
|
||||
assert.deepEqual(captured[0].args, [npxCli, "hyperframes", "tts"]);
|
||||
});
|
||||
|
||||
test("spawnP warns once with an accurate diagnostic when neither npx path exists", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const errors = [];
|
||||
const originalError = console.error;
|
||||
console.error = (message) => errors.push(String(message));
|
||||
try {
|
||||
const env = { npm_execpath: "C:/missing/npm-cli.js", npm_node_execpath: "C:/node/node.exe" };
|
||||
const missing = () => false;
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
} finally {
|
||||
console.error = originalError;
|
||||
}
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /npm_execpath \(C:\/missing\/npm-cli\.js\)/);
|
||||
assert.doesNotMatch(errors[0], /npm_execpath is not set/);
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync, chmodSync, rmSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
parseFfmpegDurationBanner,
|
||||
ffprobeDuration,
|
||||
synthesizeOne,
|
||||
synthesizeHeygen,
|
||||
synthResult,
|
||||
} from "./tts.mjs";
|
||||
|
||||
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
|
||||
const stderr = [
|
||||
"ffmpeg version 6.0",
|
||||
"Input #0, wav, from 'a.wav':",
|
||||
" Duration: 00:00:03.42, bitrate: 705 kb/s",
|
||||
"At least one output file must be specified",
|
||||
].join("\n");
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner handles an hours component", () => {
|
||||
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
|
||||
});
|
||||
|
||||
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
|
||||
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
|
||||
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
|
||||
// giving audio.mjs no way to tell "measure differently" from "give up".
|
||||
//
|
||||
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
|
||||
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
|
||||
// real, then verifies it recovers the duration via the ffmpeg fallback
|
||||
// instead of returning NaN.
|
||||
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
|
||||
const fakeFfmpeg = join(dir, "ffmpeg");
|
||||
writeFileSync(
|
||||
fakeFfmpeg,
|
||||
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(fakeFfmpeg, 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
|
||||
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // empty directory — nothing resolves
|
||||
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeOne(elevenlabs) creates the output dir before writing", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-el-mkdir-"));
|
||||
const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
|
||||
const savedKey = process.env.ELEVENLABS_API_KEY;
|
||||
try {
|
||||
// Unset the key so the Python side fails fast — the mkdir must run before
|
||||
// the spawn regardless, which is what this guards.
|
||||
delete process.env.ELEVENLABS_API_KEY;
|
||||
await synthesizeOne({
|
||||
provider: "elevenlabs",
|
||||
text: "hi",
|
||||
voiceId: "v",
|
||||
wavAbs,
|
||||
hyperframesDir: dir,
|
||||
});
|
||||
assert.ok(existsSync(dirname(wavAbs)), "output directory should be created");
|
||||
} finally {
|
||||
if (savedKey === undefined) delete process.env.ELEVENLABS_API_KEY;
|
||||
else process.env.ELEVENLABS_API_KEY = savedKey;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a thrown HTTP error (e.g. 402) instead of swallowing it", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => {
|
||||
throw new Error("HeyGen POST /voices/speech → HTTP 402\nplan_upgrade_required");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /402/);
|
||||
assert.match(res.error, /plan_upgrade_required/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a failed audio_url fetch with its status", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: false, status: 403 }),
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /HTTP 403/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports a missing audio_url", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{ heygenAuthHeaders: () => ({}), heygenJSON: async () => ({}) },
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /no audio_url/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports wav transcode failures", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-tts-test-"));
|
||||
try {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: join(dir, "voice.wav") },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }),
|
||||
transcodeToWav: () => false,
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.equal(res.error, "wav transcode failed (ffmpeg)");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthResult names a non-zero subprocess exit", () => {
|
||||
const res = synthResult({ status: 2 }, "/tmp/none.wav", "kokoro (npx hyperframes tts)");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /kokoro .* exited with status 2/);
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate BGM using Google Lyria RealTime API.
|
||||
|
||||
Usage:
|
||||
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
||||
|
||||
Requires:
|
||||
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
||||
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
||||
demand when a key is set but google.genai is not importable; if that install
|
||||
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
||||
SAMPLE_RATE = 48000
|
||||
CHANNELS = 2
|
||||
SAMPLE_WIDTH = 2 # 16-bit
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
||||
p.add_argument("--output", required=True, help="Output WAV path.")
|
||||
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
||||
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
||||
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
||||
p.add_argument("--bpm", type=int, default=110)
|
||||
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
||||
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
||||
p.add_argument(
|
||||
"--scale",
|
||||
default="MAJOR",
|
||||
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def generate_bgm(args: argparse.Namespace) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
||||
if not api_key:
|
||||
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
||||
|
||||
client = genai.Client(
|
||||
api_key=api_key,
|
||||
http_options={"api_version": "v1alpha"},
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
|
||||
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
||||
if args.density is not None:
|
||||
cfg["density"] = args.density
|
||||
if args.brightness is not None:
|
||||
cfg["brightness"] = args.brightness
|
||||
if args.scale:
|
||||
scale_enum = getattr(types.Scale, args.scale, None)
|
||||
if scale_enum:
|
||||
cfg["scale"] = scale_enum
|
||||
|
||||
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
||||
if args.negative_prompt:
|
||||
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
||||
|
||||
buf = bytearray()
|
||||
timeout = args.duration + 8
|
||||
|
||||
async with client.aio.live.music.connect(
|
||||
model="models/lyria-realtime-exp",
|
||||
) as session:
|
||||
await session.set_weighted_prompts(prompts=prompts)
|
||||
await session.set_music_generation_config(
|
||||
config=types.LiveMusicGenerationConfig(**cfg),
|
||||
)
|
||||
await session.play()
|
||||
|
||||
async def collect():
|
||||
while len(buf) < target_bytes:
|
||||
async for msg in session.receive():
|
||||
sc = msg.server_content
|
||||
if sc and sc.audio_chunks:
|
||||
for chunk in sc.audio_chunks:
|
||||
buf.extend(chunk.data)
|
||||
if len(buf) >= target_bytes:
|
||||
return
|
||||
await asyncio.sleep(1e-6)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(collect(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
||||
|
||||
audio = bytes(buf[:target_bytes])
|
||||
with wave.open(str(out_path), "wb") as wf:
|
||||
wf.setnchannels(CHANNELS)
|
||||
wf.setsampwidth(SAMPLE_WIDTH)
|
||||
wf.setframerate(SAMPLE_RATE)
|
||||
wf.writeframes(audio)
|
||||
|
||||
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
||||
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(generate_bgm(args))
|
||||
except RuntimeError as exc:
|
||||
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
|
||||
//
|
||||
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
|
||||
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
|
||||
// track, this script gives the background renderer a bounded chance to finish
|
||||
// and converts log/process state into a small bgm_status.json file.
|
||||
//
|
||||
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
|
||||
// voice/captions/SFX render. Structural invocation errors still exit 1.
|
||||
//
|
||||
// Usage:
|
||||
// node wait-bgm.mjs --audio-meta ./audio_meta.json --hyperframes . \
|
||||
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
|
||||
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
|
||||
function die(msg) {
|
||||
console.error(`✗ wait-bgm.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "bgm_status.json")));
|
||||
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
|
||||
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!pid || !Number.isFinite(Number(pid))) return false;
|
||||
try {
|
||||
process.kill(Number(pid), 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readTail(path, maxChars = 6000) {
|
||||
if (!path || !existsSync(path)) return "";
|
||||
const s = statSync(path);
|
||||
const txt = readFileSync(path, "utf8");
|
||||
return txt.slice(Math.max(0, txt.length - Math.min(maxChars, s.size)));
|
||||
}
|
||||
|
||||
function detectFailure(logTail) {
|
||||
if (!logTail) return "";
|
||||
const lines = logTail.split("\n");
|
||||
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
|
||||
// out of range, resampling" notice), mislabelling a healthy track as failed and
|
||||
// silently dropping the music. Anchor to the actual crash strings instead:
|
||||
// Python "(list) index out of range" and torch "index … out of bounds".
|
||||
const idx = lines.findIndex((line) =>
|
||||
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
|
||||
line,
|
||||
),
|
||||
);
|
||||
if (idx < 0) return "";
|
||||
return lines.slice(idx).join("\n").trim();
|
||||
}
|
||||
|
||||
function writeStatus(status) {
|
||||
const payload = {
|
||||
generated_at: new Date().toISOString(),
|
||||
...status,
|
||||
};
|
||||
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
|
||||
|
||||
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
const bgmPath = audioMeta.bgm?.path || "";
|
||||
const bgmAbsPath = bgmPath ? join(hyperframesDir, bgmPath) : "";
|
||||
const logPath = audioMeta.bgm_log || "";
|
||||
const pid = audioMeta.bgm_pid || null;
|
||||
|
||||
const base = {
|
||||
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
|
||||
provider: audioMeta.bgm_provider || null,
|
||||
mode: audioMeta.bgm_mode || null,
|
||||
path: bgmPath || null,
|
||||
log: logPath || null,
|
||||
pid,
|
||||
target_duration_s: audioMeta.bgm_target_duration_s || null,
|
||||
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
|
||||
loop_count: audioMeta.bgm_loop_count || null,
|
||||
timeout_ms: timeoutMs,
|
||||
};
|
||||
|
||||
if (!base.enabled) {
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "disabled",
|
||||
ready: false,
|
||||
waited_ms: 0,
|
||||
message: "BGM not requested or disabled in audio_meta.json.",
|
||||
});
|
||||
console.log(`✓ bgm: ${status.status} (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastFailure = "";
|
||||
let lastTail = "";
|
||||
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (existsSync(bgmAbsPath)) {
|
||||
const size = statSync(bgmAbsPath).size;
|
||||
writeStatus({
|
||||
...base,
|
||||
status: "ready",
|
||||
ready: true,
|
||||
waited_ms: Date.now() - started,
|
||||
size_bytes: size,
|
||||
message: `BGM ready at ${bgmPath}.`,
|
||||
});
|
||||
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
lastTail = readTail(logPath);
|
||||
lastFailure = detectFailure(lastTail);
|
||||
const alive = isProcessAlive(pid);
|
||||
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
|
||||
const message = lastFailure
|
||||
? `BGM renderer failed; see ${logPath}.`
|
||||
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "failed",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: alive,
|
||||
message,
|
||||
error_tail: lastFailure || lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: failed (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (timeoutMs === 0) break;
|
||||
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
|
||||
}
|
||||
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "timeout",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: isProcessAlive(pid),
|
||||
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
|
||||
log_tail: lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
|
||||
Reference in New Issue
Block a user