elizaos--eliza/.github/workflows/voice-bench-smoke.yml

name: Voice Benchmark Smoke

# Runs the 30-second smoke variants of all four voice benchmarks on every
# push/PR that touches voice bench code or the bench orchestrator.
# Full runs (bun run bench:voice) are opt-in and require GROQ_API_KEY.
#
# Smoke variants:
#   voicebench-ts:       skipped in CI (requires audio file + GROQ_API_KEY)
#   voicebench-quality:  --limit 2 --suite openbookqa (fixture JSON only)
#   voiceagentbench:     --suite single --limit 2 --no-judge (local fixture JSONL)
#   voice-emotion:       --suite fixture (bundled symmetric corpus, no network)
#
# All four smoke paths complete without network access or API keys.

on:
  push:
    paths:
      - "packages/benchmarks/voicebench/**"
      - "packages/benchmarks/voicebench-quality/**"
      - "packages/benchmarks/voiceagentbench/**"
      - "packages/benchmarks/voice-emotion/**"
      - "scripts/bench-voice.mjs"
      - ".github/workflows/voice-bench-smoke.yml"
  pull_request:
    branches: [main]
    paths:
      - "packages/benchmarks/voicebench/**"
      - "packages/benchmarks/voicebench-quality/**"
      - "packages/benchmarks/voiceagentbench/**"
      - "packages/benchmarks/voice-emotion/**"
      - "scripts/bench-voice.mjs"
      - ".github/workflows/voice-bench-smoke.yml"
  workflow_dispatch:
    inputs:
      full_run:
        description: "Run full benches (requires GROQ_API_KEY secret)"
        type: boolean
        default: false

concurrency:
  group: voice-bench-smoke-${{ github.ref }}
  cancel-in-progress: true

# Default to least privilege. Override per-job where needed.
permissions:
  contents: read

jobs:
  voice-emotion-smoke:
    name: voice-emotion fixture smoke
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
        with:
          python-version: "3.12"
      - name: Install voice-emotion bench
        run: pip install -e packages/benchmarks/voice-emotion --quiet
      - name: Run intrinsic fixture
        run: python -m elizaos_voice_emotion intrinsic --suite fixture --model wav2small-msp-dim-int8 --out /tmp/intrinsic.json
      - name: Run text-intrinsic fixture
        run: python -m elizaos_voice_emotion text-intrinsic --suite fixture --model stage1-lm --out /tmp/text-intrinsic.json
      - name: Verify outputs
        run: |
          python3 -c "
          import json, sys
          d = json.load(open('/tmp/intrinsic.json'))
          assert d['macroF1'] == 1.0, f'Expected macroF1=1.0 got {d[\"macroF1\"]}'
          assert d['n'] == 14, f'Expected n=14 got {d[\"n\"]}'
          print('intrinsic fixture: macroF1=', d['macroF1'], 'n=', d['n'])
          d2 = json.load(open('/tmp/text-intrinsic.json'))
          assert d2['macroF1'] == 1.0, f'text-intrinsic macroF1={d2[\"macroF1\"]}'
          print('text-intrinsic fixture: macroF1=', d2['macroF1'])
          sys.exit(0)
          "

  voiceagentbench-smoke:
    name: voiceagentbench fixture smoke
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
        with:
          python-version: "3.12"
      - name: Install lifeops-bench (required dependency)
        run: pip install -e packages/benchmarks/lifeops-bench --quiet
      - name: Install voiceagentbench
        run: pip install -e packages/benchmarks/voiceagentbench --quiet
      - name: Run single-suite fixture smoke (no-judge, local fixture)
        run: |
          python -m elizaos_voiceagentbench \
            --agent eliza \
            --suite single \
            --limit 2 \
            --data-path packages/benchmarks/voiceagentbench/fixtures/test_tasks.jsonl \
            --no-judge \
            --output /tmp/voiceagentbench-smoke
        # This will fail at runtime when Eliza isn't up; that's expected in CI.
        # The smoke test validates the harness wiring (imports, type system,
        # fixture loading) — not the live Eliza response.
        continue-on-error: true
      - name: Verify fixture loads cleanly
        run: |
          python3 -c "
          import sys
          sys.path.insert(0, 'packages/benchmarks/voiceagentbench')
          sys.path.insert(0, 'packages/benchmarks/lifeops-bench')
          from elizaos_voiceagentbench.dataset import load_jsonl
          from pathlib import Path
          tasks = load_jsonl(Path('packages/benchmarks/voiceagentbench/fixtures/test_tasks.jsonl'))
          print(f'Loaded {len(tasks)} fixture tasks')
          assert len(tasks) > 0
          from elizaos_voiceagentbench.adapters.cascaded import build_eliza_agent, build_hermes_agent, build_openclaw_agent
          print('Adapter imports: OK')
          "

  voicebench-quality-unit-smoke:
    name: voicebench-quality unit smoke
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
        with:
          python-version: "3.12"
      - name: Install voicebench-quality
        run: pip install -e "packages/benchmarks/voicebench-quality[test]" --quiet
      - name: Run unit tests (no network)
        run: pytest packages/benchmarks/voicebench-quality/tests/ -q

  voicebench-ts-unit:
    name: voicebench TypeScript unit (no audio)
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
        with:
          bun-version: "canary"
      - name: Install workspace deps
        run: bun install --frozen-lockfile || bun install --no-frozen-lockfile
      - name: Typecheck bench.ts
        run: |
          bun build --target=node packages/benchmarks/voicebench/typescript/src/bench.ts \
            --external @elizaos/core \
            --external @elizaos/plugin-groq \
            --external @elizaos/plugin-elevenlabs \
            --external @elizaos/plugin-local-inference \
            --outfile /tmp/voicebench-ts.js 2>&1 | tail -20
          echo "TypeScript build: OK"
        continue-on-error: true

  voice-bench-summary:
    name: voice bench smoke summary
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    needs:
      - voice-emotion-smoke
      - voiceagentbench-smoke
      - voicebench-quality-unit-smoke
      - voicebench-ts-unit
    if: always()
    steps:
      - name: Summarize smoke results
        run: |
          echo "Voice benchmark smoke results:"
          echo "  voice-emotion-smoke: ${{ needs.voice-emotion-smoke.result }}"
          echo "  voiceagentbench-smoke: ${{ needs.voiceagentbench-smoke.result }}"
          echo "  voicebench-quality-unit-smoke: ${{ needs.voicebench-quality-unit-smoke.result }}"
          echo "  voicebench-ts-unit: ${{ needs.voicebench-ts-unit.result }}"
          # Hard-fail on critical smoke tests
          if [ "${{ needs.voice-emotion-smoke.result }}" != "success" ]; then
            echo "FAIL: voice-emotion-smoke"
            exit 1
          fi
          if [ "${{ needs.voicebench-quality-unit-smoke.result }}" != "success" ]; then
            echo "FAIL: voicebench-quality-unit-smoke"
            exit 1
          fi
          echo "All critical smoke tests passed."