name: Voice Benchmark Smoke # Runs the 30-second smoke variants of all four voice benchmarks on every # push/PR that touches voice bench code or the bench orchestrator. # Full runs (bun run bench:voice) are opt-in and require GROQ_API_KEY. # # Smoke variants: # voicebench-ts: skipped in CI (requires audio file + GROQ_API_KEY) # voicebench-quality: --limit 2 --suite openbookqa (fixture JSON only) # voiceagentbench: --suite single --limit 2 --no-judge (local fixture JSONL) # voice-emotion: --suite fixture (bundled symmetric corpus, no network) # # All four smoke paths complete without network access or API keys. on: push: paths: - "packages/benchmarks/voicebench/**" - "packages/benchmarks/voicebench-quality/**" - "packages/benchmarks/voiceagentbench/**" - "packages/benchmarks/voice-emotion/**" - "scripts/bench-voice.mjs" - ".github/workflows/voice-bench-smoke.yml" pull_request: branches: [main] paths: - "packages/benchmarks/voicebench/**" - "packages/benchmarks/voicebench-quality/**" - "packages/benchmarks/voiceagentbench/**" - "packages/benchmarks/voice-emotion/**" - "scripts/bench-voice.mjs" - ".github/workflows/voice-bench-smoke.yml" workflow_dispatch: inputs: full_run: description: "Run full benches (requires GROQ_API_KEY secret)" type: boolean default: false concurrency: group: voice-bench-smoke-${{ github.ref }} cancel-in-progress: true # Default to least privilege. Override per-job where needed. permissions: contents: read jobs: voice-emotion-smoke: name: voice-emotion fixture smoke runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} timeout-minutes: 5 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" - name: Install voice-emotion bench run: pip install -e packages/benchmarks/voice-emotion --quiet - name: Run intrinsic fixture run: python -m elizaos_voice_emotion intrinsic --suite fixture --model wav2small-msp-dim-int8 --out /tmp/intrinsic.json - name: Run text-intrinsic fixture run: python -m elizaos_voice_emotion text-intrinsic --suite fixture --model stage1-lm --out /tmp/text-intrinsic.json - name: Verify outputs run: | python3 -c " import json, sys d = json.load(open('/tmp/intrinsic.json')) assert d['macroF1'] == 1.0, f'Expected macroF1=1.0 got {d[\"macroF1\"]}' assert d['n'] == 14, f'Expected n=14 got {d[\"n\"]}' print('intrinsic fixture: macroF1=', d['macroF1'], 'n=', d['n']) d2 = json.load(open('/tmp/text-intrinsic.json')) assert d2['macroF1'] == 1.0, f'text-intrinsic macroF1={d2[\"macroF1\"]}' print('text-intrinsic fixture: macroF1=', d2['macroF1']) sys.exit(0) " voiceagentbench-smoke: name: voiceagentbench fixture smoke runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} timeout-minutes: 10 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" - name: Install lifeops-bench (required dependency) run: pip install -e packages/benchmarks/lifeops-bench --quiet - name: Install voiceagentbench run: pip install -e packages/benchmarks/voiceagentbench --quiet - name: Run single-suite fixture smoke (no-judge, local fixture) run: | python -m elizaos_voiceagentbench \ --agent eliza \ --suite single \ --limit 2 \ --data-path packages/benchmarks/voiceagentbench/fixtures/test_tasks.jsonl \ --no-judge \ --output /tmp/voiceagentbench-smoke # This will fail at runtime when Eliza isn't up; that's expected in CI. # The smoke test validates the harness wiring (imports, type system, # fixture loading) — not the live Eliza response. continue-on-error: true - name: Verify fixture loads cleanly run: | python3 -c " import sys sys.path.insert(0, 'packages/benchmarks/voiceagentbench') sys.path.insert(0, 'packages/benchmarks/lifeops-bench') from elizaos_voiceagentbench.dataset import load_jsonl from pathlib import Path tasks = load_jsonl(Path('packages/benchmarks/voiceagentbench/fixtures/test_tasks.jsonl')) print(f'Loaded {len(tasks)} fixture tasks') assert len(tasks) > 0 from elizaos_voiceagentbench.adapters.cascaded import build_eliza_agent, build_hermes_agent, build_openclaw_agent print('Adapter imports: OK') " voicebench-quality-unit-smoke: name: voicebench-quality unit smoke runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} timeout-minutes: 5 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" - name: Install voicebench-quality run: pip install -e "packages/benchmarks/voicebench-quality[test]" --quiet - name: Run unit tests (no network) run: pytest packages/benchmarks/voicebench-quality/tests/ -q voicebench-ts-unit: name: voicebench TypeScript unit (no audio) runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} timeout-minutes: 5 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 with: bun-version: "canary" - name: Install workspace deps run: bun install --frozen-lockfile || bun install --no-frozen-lockfile - name: Typecheck bench.ts run: | bun build --target=node packages/benchmarks/voicebench/typescript/src/bench.ts \ --external @elizaos/core \ --external @elizaos/plugin-groq \ --external @elizaos/plugin-elevenlabs \ --external @elizaos/plugin-local-inference \ --outfile /tmp/voicebench-ts.js 2>&1 | tail -20 echo "TypeScript build: OK" continue-on-error: true voice-bench-summary: name: voice bench smoke summary runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }} needs: - voice-emotion-smoke - voiceagentbench-smoke - voicebench-quality-unit-smoke - voicebench-ts-unit if: always() steps: - name: Summarize smoke results run: | echo "Voice benchmark smoke results:" echo " voice-emotion-smoke: ${{ needs.voice-emotion-smoke.result }}" echo " voiceagentbench-smoke: ${{ needs.voiceagentbench-smoke.result }}" echo " voicebench-quality-unit-smoke: ${{ needs.voicebench-quality-unit-smoke.result }}" echo " voicebench-ts-unit: ${{ needs.voicebench-ts-unit.result }}" # Hard-fail on critical smoke tests if [ "${{ needs.voice-emotion-smoke.result }}" != "success" ]; then echo "FAIL: voice-emotion-smoke" exit 1 fi if [ "${{ needs.voicebench-quality-unit-smoke.result }}" != "success" ]; then echo "FAIL: voicebench-quality-unit-smoke" exit 1 fi echo "All critical smoke tests passed."