wehub-resource-sync
84 lines
3.0 KiB
YAML
84 lines
3.0 KiB
YAML
name: evals
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
|
|
jobs:
|
|
evals:
|
|
# don't run tests for PRs on forks
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
example: [frontdesk, drive-thru]
|
|
|
|
runs-on: ubuntu-latest
|
|
name: ${{ matrix.example }}
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
|
|
with:
|
|
enable-cache: true
|
|
cache-dependency-glob: "uv.lock"
|
|
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Install the project
|
|
run: uv sync --all-extras --dev
|
|
|
|
- name: Setup Google credentials
|
|
if: matrix.example == 'google'
|
|
shell: bash
|
|
run: |
|
|
printf '%s' '${{ secrets.GOOGLE_CREDENTIALS_JSON }}' > tests/google.json
|
|
|
|
- name: Run tests
|
|
shell: bash
|
|
env:
|
|
LIVEKIT_EVALS_VERBOSE: 1
|
|
PLUGIN: ${{ matrix.example }}
|
|
LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
|
|
LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
|
|
LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
|
|
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
ELEVEN_API_KEY: ${{ secrets.ELEVEN_API_KEY }}
|
|
CARTESIA_API_KEY: ${{ secrets.CARTESIA_API_KEY }}
|
|
AZURE_SPEECH_KEY: ${{ secrets.AZURE_SPEECH_KEY }}
|
|
AZURE_SPEECH_REGION: ${{ secrets.AZURE_SPEECH_REGION }} # nit: doesn't have to be secret
|
|
GOOGLE_CREDENTIALS_JSON: ${{ secrets.GOOGLE_CREDENTIALS_JSON }}
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
|
ASSEMBLYAI_API_KEY: ${{ secrets.ASSEMBLYAI_API_KEY }}
|
|
FAL_KEY: ${{ secrets.FAL_KEY }}
|
|
PLAYHT_API_KEY: ${{ secrets.PLAYHT_API_KEY }}
|
|
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
|
PLAYHT_USER_ID: ${{ secrets.PLAYHT_USER_ID }}
|
|
RIME_API_KEY: ${{ secrets.RIME_API_KEY }}
|
|
SPEECHMATICS_API_KEY: ${{ secrets.SPEECHMATICS_API_KEY }}
|
|
GOOGLE_APPLICATION_CREDENTIALS: tests/google.json
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
NEUPHONIC_API_KEY: ${{ secrets.NEUPHONIC_API_KEY }}
|
|
RESEMBLE_API_KEY: ${{ secrets.RESEMBLE_API_KEY }}
|
|
SPEECHIFY_API_KEY: ${{ secrets.SPEECHIFY_API_KEY }}
|
|
HUME_API_KEY: ${{ secrets.HUME_API_KEY }}
|
|
SPITCH_API_KEY: ${{ secrets.SPITCH_API_KEY }}
|
|
LMNT_API_KEY: ${{ secrets.LMNT_API_KEY }}
|
|
INWORLD_API_KEY: ${{ secrets.INWORLD_API_KEY }}
|
|
run: uv run pytest examples/${{matrix.example}}/test_agent.py -s
|