promptfoo--promptfoo/.github/workflows/main.yml

name: CI
on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
  VITE_TELEMETRY_DISABLED: 1
jobs:
  ci-config:
    name: CI Config
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      test-matrix: ${{ steps.set-matrix.outputs.test-matrix }}
      build-matrix: ${{ steps.set-matrix.outputs.build-matrix }}
    steps:
      - name: Determine CI matrix
        id: set-matrix
        env:
          EVENT_NAME: ${{ github.event_name }}
        run: |
          # Shared entries: all Linux versions + one Windows Node version (sharded)
          base_entries='
            {"node":"20.20","os":"ubuntu-latest","shard":""},
            {"node":"22.22","os":"ubuntu-latest","shard":""},
            {"node":"24.x","os":"ubuntu-latest","shard":""},
            {"node":"26.x","os":"ubuntu-latest","shard":""},
            {"node":"22.22","os":"windows-2025-vs2026","shard":1},
            {"node":"22.22","os":"windows-2025-vs2026","shard":2},
            {"node":"22.22","os":"windows-2025-vs2026","shard":3}
          '

          if [[ "$EVENT_NAME" == "pull_request" ]]; then
            # PRs: add 1 macOS version only
            extra_entries='
              ,{"node":"22.22","os":"macOS-latest","shard":""}
            '
            build_matrix='{"node":["20.20","24.x","26.x"]}'
          else
            # Main/dispatch: add all macOS versions + remaining Windows shards
            extra_entries='
              ,{"node":"20.20","os":"macOS-latest","shard":""},
              {"node":"22.22","os":"macOS-latest","shard":""},
              {"node":"20.20","os":"windows-2025-vs2026","shard":1},
              {"node":"20.20","os":"windows-2025-vs2026","shard":2},
              {"node":"20.20","os":"windows-2025-vs2026","shard":3},
              {"node":"24.x","os":"windows-2025-vs2026","shard":1},
              {"node":"24.x","os":"windows-2025-vs2026","shard":2},
              {"node":"24.x","os":"windows-2025-vs2026","shard":3},
              {"node":"26.x","os":"windows-2025-vs2026","shard":1},
              {"node":"26.x","os":"windows-2025-vs2026","shard":2},
              {"node":"26.x","os":"windows-2025-vs2026","shard":3}
            '
            build_matrix='{"node":["20.20","22.22","24.x","26.x"]}'
          fi

          # Build the matrix JSON (shard:"" is falsy in GHA expressions, used to skip shard flags)
          test_matrix=$(echo "{\"include\":[${base_entries}${extra_entries}]}" | jq -c .)
          echo "test-matrix=$test_matrix" >> "$GITHUB_OUTPUT"
          echo "build-matrix=$build_matrix" >> "$GITHUB_OUTPUT"

  test:
    needs: ci-config
    name: Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }}
    # Cold-cache Node 26 installs can approach 20m before the test phase starts, so leave room for the test body.
    timeout-minutes: 40
    runs-on: ${{ matrix.os }}
    permissions:
      contents: read
      checks: write
      id-token: write # Required for OIDC authentication with Codecov
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.ci-config.outputs.test-matrix) }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          fetch-depth: 2

      - name: Use Node ${{ matrix.node }}
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          # Keep matrix label at 20.20 for stable check names, but install 20.20.1 to satisfy engines.
          node-version: ${{ matrix.node == '20.20' && '20.20.1' || matrix.node }}
          cache: 'npm'

      - name: Use Python 3.14
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: 3.14.4

      # Ruby 4.0.1 is not yet available on Windows via setup-ruby, use 4.0.0 on Windows
      - name: Use Ruby
        uses: ruby/setup-ruby@4c56a21280b36d862b5fc31348f463d60bdc55d5 # v1
        with:
          ruby-version: ${{ startsWith(matrix.os, 'windows-') && '4.0.0' || '4.0.1' }}

      # Cache node_modules to speed up installs (especially on Windows where npm ci is slow)
      # Key includes OS and Node version since native modules are platform/version specific
      - name: Cache node_modules
        id: cache-node-modules
        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
        with:
          path: node_modules
          key: node-modules-${{ runner.os }}-node${{ matrix.node }}-${{ hashFiles('package-lock.json') }}

      - name: Install Dependencies (Node 26)
        if: steps.cache-node-modules.outputs.cache-hit != 'true' && matrix.node == '26.x'
        # Node 26 hosted installs stall when lifecycle scripts run during npm ci.
        # Install the tree without scripts, then rebuild the native addon tests need.
        run: |
          npm ci --ignore-scripts
          npm rebuild better-sqlite3

      - name: Install Dependencies
        if: steps.cache-node-modules.outputs.cache-hit != 'true' && matrix.node != '26.x'
        run: npx -y npm@11.11.0 ci

      - name: Test
        # Treat a post-run forks-worker crash (all tests already passed, then the
        # worker dies on teardown -> "Worker exited unexpectedly") as non-fatal in
        # CI. Real test/assertion failures still fail; see vitest.config.ts.
        env:
          PROMPTFOO_IGNORE_UNHANDLED_TEST_ERRORS: 'true'
        run: npm run test${{ matrix.shard && format(' -- --shard={0}/3', matrix.shard) || '' }}${{ matrix.os == 'ubuntu-latest' && matrix.node == '20.20' && !matrix.shard && ' -- --coverage' || '' }}

      - name: Check Backend Coverage Ratchets
        if: matrix.os == 'ubuntu-latest' && matrix.node == '20.20' && !matrix.shard
        run: npm run test:coverage:ratchet -- --report backend

      - name: Upload Backend Coverage to Codecov
        if: matrix.os == 'ubuntu-latest' && matrix.node == '20.20' && !matrix.shard
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          files: ./coverage/coverage-final.json
          flags: backend
          name: backend-coverage
          # Codecov upload is informational (coverage is gated in-repo, not by
          # Codecov); keep it non-blocking so Codecov infra outages can't fail main.
          fail_ci_if_error: false
          use_oidc: true
          # Pin the CLI version too: the action SHA pins the wrapper, but the CLI
          # binary it downloads is otherwise unpinned (action default is latest).
          version: v11.2.8

  build:
    needs: ci-config
    name: Build on Node ${{ matrix.node }}
    env:
      PROMPTFOO_POSTHOG_KEY: ${{ secrets.PROMPTFOO_POSTHOG_KEY }}
    # Cold-cache Node 26 installs can approach 20m before the build phase starts on GitHub-hosted runners.
    timeout-minutes: 30
    runs-on: ubuntu-latest
    strategy:
      matrix: ${{ fromJSON(needs.ci-config.outputs.build-matrix) }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node ${{ matrix.node }}
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          # Keep matrix label at 20.20 for stable check names, but install 20.20.1 to satisfy engines.
          node-version: ${{ matrix.node == '20.20' && '20.20.1' || matrix.node }}
          cache: 'npm'

      - name: Install Dependencies
        # The build does not need native dependency lifecycle scripts. Skipping them
        # keeps the Node 26 build lane out of the install stall seen on hosted runners.
        run: |
          if [ "${{ matrix.node }}" = "26.x" ]; then
            npm ci --ignore-scripts
          else
            npx -y npm@11.11.0 ci
          fi

      - name: Build
        run: npm run build

      - name: Check Telemetry
        env:
          EVENT_NAME: ${{ github.event_name }}
          HEAD_REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }}
          REPOSITORY: ${{ github.repository }}
        run: |
          # Skip PostHog key check for forks as they don't have access to secrets
          if [[ "$EVENT_NAME" == "pull_request" ]] && [[ "$HEAD_REPO_FULL_NAME" != "$REPOSITORY" ]]; then
            echo "Skipping PostHog key check for fork PR"
          elif [[ -z "$PROMPTFOO_POSTHOG_KEY" ]]; then
            echo "PostHog key not available (running without secret), skipping check"
          else
            # PostHog key is injected at build time via tsup's define option
            # Check that it's present in the built output (it will be inlined as a string)
            if ! grep -rq '"phc_' dist/src/*.js; then
              echo "Error: PostHog key not found in built output"
              echo "Checking for POSTHOG_KEY patterns in built files:"
              grep -r "POSTHOG_KEY" dist/src/*.js | head -10 || echo "No POSTHOG_KEY found"
              exit 1
            fi
            echo "PostHog key replacement verified successfully"
          fi

      - name: Test Package Artifact
        if: matrix.node == '20.20' || matrix.node == '26.x'
        env:
          PROMPTFOO_DISABLE_TELEMETRY: 1
        run: npm run test:package-artifact

  style-check:
    name: Style Check
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: |
          npm ci

      - name: Run Biome CI
        run: |
          npm run lint:ci

      - name: Run Prettier Check
        run: |
          npm run format:check:prettier

      - name: Check Dependency Versions
        run: |
          npx check-dependency-version-consistency

      - name: Check for circular dependencies
        run: |
          # shellcheck disable=SC2046
          npx madge $(git ls-files '*.ts') --circular

      - name: Check architecture boundaries
        run: |
          npm run architecture:check

      - name: Check for missing dependencies
        run: |
          npm run depcheck

      - name: Check for orphaned files
        run: |
          npm run knip -- --include files

      - name: Validate lockfile integrity
        run: |
          npx lockfile-lint --path package-lock.json --allowed-hosts npm --validate-https

  shell-format:
    name: Shell Format Check
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
      - uses: luizm/action-sh-checker@883217215b11c1fabbf00eb1a9a041f62d74c744
        env:
          SHFMT_OPTS: '-i 2' # 2 space indent
        with:
          sh_checker_shellcheck_disable: true

  assets:
    name: Generate Assets
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Use Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: '3.13'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Install ModelAudit schema dependencies
        run: python3 -m pip install --disable-pip-version-check --no-deps -r scripts/modelaudit_schema_requirements.txt

      - name: Generate JSON Schemas
        run: |
          npm run jsonSchema:generate
          npm run modelAuditSchema:generate

      - name: Check for changes
        run: |
          if [[ -n $(git status --porcelain) ]]; then
            echo "Changes detected after generating assets:"
            git status --porcelain
            exit 1
          else
            echo "No changes detected."
          fi

  python:
    name: Check Python
    timeout-minutes: 5
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.9, 3.14]
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Python ${{ matrix.python-version }}
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Dependencies
        run: |
          pip install ruff==0.15.1

      - name: Check Formatting
        run: |
          ruff check --select F401,F841,I --fix
          ruff format
          git diff --exit-code || (echo "Files were modified by ruff. Please commit these changes." && exit 1)

      - name: Run Tests
        run: |
          python -m unittest discover -s src/python -p '*_test.py'

  docs:
    name: Build Docs
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Type Check
        working-directory: site
        run: npm run typecheck

      - name: Build Documentation
        working-directory: site
        run: npm run build
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  code-scan-action:
    name: Code Scan Action
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      # Install root dependencies first (code-scan-action imports from ../../src/types/codeScan.ts which needs zod)
      - name: Install Root Dependencies
        run: npm ci

      - name: Install Code Scan Action Dependencies
        working-directory: code-scan-action
        run: npm install

      - name: Type Check
        working-directory: code-scan-action
        run: npm run tsc

      - name: Build Action
        working-directory: code-scan-action
        run: npm run build

  site-tests:
    name: Site tests
    timeout-minutes: 5
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
      id-token: write # Required for OIDC authentication with Codecov
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Run Site Tests with Coverage
        working-directory: site
        run: npm run test:coverage

      # Site has no coverage ratchet, so the Codecov upload is the only consumer of
      # this report. Since that upload is now non-blocking, verify the artifact was
      # produced here — otherwise a missing/misnamed report would fail silently.
      - name: Verify site coverage report exists
        run: test -s ./site/coverage/coverage-final.json

      - name: Upload Site Coverage to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          files: ./site/coverage/coverage-final.json
          flags: site
          name: site-coverage
          # Codecov upload is informational (coverage is gated in-repo, not by
          # Codecov); keep it non-blocking so Codecov infra outages can't fail main.
          fail_ci_if_error: false
          use_oidc: true
          # Pin the CLI version too: the action SHA pins the wrapper, but the CLI
          # binary it downloads is otherwise unpinned (action default is latest).
          version: v11.2.8

  webui:
    name: webui tests
    timeout-minutes: 10
    runs-on: ubuntu-latest
    permissions:
      contents: read
      checks: write
      id-token: write # Required for OIDC authentication with Codecov
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
        with:
          fetch-depth: 2

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Run App Browser Tests
        run: npm run test:app:browser

      - name: Run App Tests with Coverage
        run: npm run test:coverage --prefix src/app

      - name: Check Frontend Coverage Ratchets
        run: npm run test:coverage:ratchet -- --report frontend

      - name: Upload Frontend Coverage to Codecov
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          files: ./src/app/coverage/coverage-final.json
          flags: frontend
          name: frontend-coverage
          # Codecov upload is informational (coverage is gated in-repo, not by
          # Codecov); keep it non-blocking so Codecov infra outages can't fail main.
          fail_ci_if_error: false
          use_oidc: true
          # Pin the CLI version too: the action SHA pins the wrapper, but the CLI
          # binary it downloads is otherwise unpinned (action default is latest).
          version: v11.2.8

  integration-tests:
    name: Run Integration Tests
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Use Python 3.14
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: 3.14.4

      - name: Use Ruby
        uses: ruby/setup-ruby@4c56a21280b36d862b5fc31348f463d60bdc55d5 # v1
        with:
          ruby-version: 4.0.1

      - name: Install Dependencies
        run: |
          npm ci

      - name: Run Integration Tests
        run: npm run test:integration -- --coverage

  smoke-tests:
    name: Run Smoke Tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Use Python 3.14
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
        with:
          python-version: 3.14.4

      - name: Use Ruby
        uses: ruby/setup-ruby@4c56a21280b36d862b5fc31348f463d60bdc55d5 # v1
        with:
          ruby-version: 4.0.1

      - name: Install Dependencies
        run: npm ci

      - name: Build
        run: npm run build

      - name: Run Smoke Tests
        run: npm run test:smoke

  share-test:
    name: Share Test
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Run local server
        run: |
          mkdir -p "$RUNNER_TEMP/promptfoo-share-test"
          npm run build
          server_log="$RUNNER_TEMP/promptfoo-share-test/server.log"
          PROMPTFOO_CONFIG_DIR="$HOME/tmp" LOG_LEVEL=DEBUG API_PORT=8500 node dist/src/server/index.js >"$server_log" 2>&1 &
          echo "SERVER_PID=$!" >> "$GITHUB_ENV"
          echo "SERVER_LOG=$server_log" >> "$GITHUB_ENV"

      - name: Wait for server to be ready
        run: |
          for i in $(seq 1 45); do
            if ! kill -0 "$SERVER_PID" 2>/dev/null; then
              echo "Server exited before becoming ready"
              cat "$SERVER_LOG"
              exit 1
            fi

            if curl -fsS -o /dev/null http://localhost:8500/health; then
              echo "Server is ready"
              exit 0
            fi
            echo "Waiting for server... attempt $i/45"
            sleep 2
          done
          echo "Server failed to start"
          cat "$SERVER_LOG"
          exit 1

      - name: run promptfoo eval
        id: eval
        run: |
          PROMPTFOO_REMOTE_API_BASE_URL=http://localhost:8500 PROMPTFOO_SHARING_APP_BASE_URL=http://localhost:8500  node dist/src/main.js eval -c .github/assets/promptfooconfig.yaml --share
        env:
          PROMPTFOO_DISABLE_TELEMETRY: 1

      - name: Test that the eval results are uploaded
        run: |
          response=$(curl -s http://localhost:8500/api/results)
          echo "Response: $response"

          # Use jq to extract the array length
          count=$(echo "$response" | jq '.data | length')
          echo "Array Length: $count"

          # Check if the count is exactly 1
          if [ "$count" -ne 1 ]; then
            echo "Error: Expected 1 entry, but got $count"
            exit 1
          fi

      - name: Dump server log on failure
        if: failure()
        run: |
          if [ -f "$SERVER_LOG" ]; then
            cat "$SERVER_LOG"
          fi

      - name: Share to cloud
        if: env.PROMPTFOO_STAGING_API_KEY != ''
        env:
          PROMPTFOO_STAGING_API_KEY: ${{ secrets.PROMPTFOO_STAGING_API_KEY }}
        run: |
          node dist/src/main.js auth login -k ${{ secrets.PROMPTFOO_STAGING_API_KEY }} -h https://api.promptfoo-staging.app
          node dist/src/main.js eval -c .github/assets/promptfooconfig.yaml --share

      - name: Stop local server
        if: always()
        run: |
          if [ -n "${SERVER_PID:-}" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
            kill "$SERVER_PID"
            wait "$SERVER_PID" || true
          fi

  redteam:
    name: Redteam (Production API)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      - name: Run Redteam (Production API)
        run: |
          npm run test:redteam:integration

  redteam-staging:
    name: Redteam (Staging API)
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      # Node 24 ships with npm 11.6.2 which has a lockfile compatibility bug (npm/cli#8669).
      # Pinned exact (not @latest): npm 12.0.0 blocks postinstall scripts not covered by
      # allowScripts, which silently skips e.g. the Playwright browser download.
      - name: Upgrade npm
        run: npm install -g npm@11.18.0

      - name: Install Dependencies
        run: npm ci

      # Need to build first so we can login
      - name: Build
        run: |
          npm run build

      - name: Login
        if: env.PROMPTFOO_INTEGRATION_TEST_API_KEY != ''
        env:
          PROMPTFOO_INTEGRATION_TEST_API_KEY: ${{ secrets.PROMPTFOO_INTEGRATION_TEST_API_KEY }}
        run: |
          npm run bin auth login -- -k ${{ secrets.PROMPTFOO_INTEGRATION_TEST_API_KEY }} -h ${{ secrets.PROMPTFOO_INTEGRATION_TEST_API_HOST }}

      - name: Run Redteam with Staging API
        continue-on-error: true
        run: |
          npm run test:redteam:integration

  actionlint:
    name: GitHub Actions Lint
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: read
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Install and run actionlint
        run: |
          bash <(curl -s https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
          ./actionlint

  ruby:
    name: Check Ruby
    timeout-minutes: 5
    runs-on: ubuntu-latest
    strategy:
      matrix:
        ruby-version: ['3.0', '3.4']
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Use Ruby ${{ matrix.ruby-version }}
        uses: ruby/setup-ruby@4c56a21280b36d862b5fc31348f463d60bdc55d5 # v1
        with:
          ruby-version: ${{ matrix.ruby-version }}

      - name: Install Dependencies
        run: |
          gem install rubocop

      - name: Check Formatting
        run: |
          rubocop --autocorrect-all src/ruby/
          git diff --exit-code || (echo "Files were modified by rubocop. Please commit these changes." && exit 1)

      - name: Test Ruby Wrapper
        run: |
          # Create a simple test script
          cat > /tmp/test_script.rb << 'EOF'
          def test_function(a, b)
            { "sum" => a + b, "product" => a * b }
          end
          EOF

          # Create input JSON
          echo '[2, 3]' > /tmp/input.json

          # Run the wrapper
          ruby src/ruby/wrapper.rb /tmp/test_script.rb test_function /tmp/input.json /tmp/output.json

          # Verify output
          cat /tmp/output.json
          if ! grep -q '"sum":5' /tmp/output.json; then
            echo "Error: Ruby wrapper test failed"
            exit 1
          fi
          echo "Ruby wrapper test passed"

  golang:
    name: Go Tests
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout repo
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

      - name: Set up Go
        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
        with:
          go-version-file: src/golang/go.mod
          check-latest: true

      - name: Run wrapper tests
        working-directory: src/golang
        run: |
          go test -v wrapper.go wrapper_test.go