elizaos--eliza/.github/workflows/browser-real-bench.yml

name: Browser Benchmark Lanes

# Deferred "Needs CI infra" lanes of #9476, tracked by #10333:
# - MiniWoB++ benchmark suite against a REAL Chromium engine (puppeteer-core)
# - ScreenSpot-Web-style grounding against a REAL Chromium engine
# - committed Mind2Web/WebArena-style external dataset fixtures through the
#   plugin-browser BROWSER command router, including a REAL Chromium pass
#
# The `*.real.test.ts` lanes are excluded from the default `vitest run` and
# self-skip without a Chromium binary, so they only execute here after
# `bunx playwright install chromium` (mirroring scenario-pr.yml's
# `app-browser-core`). Gated like the other heavy real lanes — nightly +
# on-demand + on changes to the benchmark/executor — not on every PR.

on:
  schedule:
    # 10:00 UTC daily — after the external-API live lanes settle.
    - cron: "0 10 * * *"
  workflow_dispatch: {}
  pull_request:
    branches: [main]
    paths:
      - "plugins/plugin-browser/src/benchmark/**"
      - "plugins/plugin-browser/vitest.real.config.ts"
      - "plugins/plugin-browser/scripts/*benchmark*.mjs"
      - "plugins/plugin-browser/scripts/capture-web-grounding-evidence.mjs"
      - "plugins/plugin-browser/package.json"
      - ".github/workflows/browser-real-bench.yml"

concurrency:
  group: browser-real-bench-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  CI: "true"
  NODE_VERSION: "24"
  BUN_VERSION: "canary"

jobs:
  browser-real-bench:
    name: Browser benchmarks on real Chromium
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          submodules: false
          show-progress: false

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
        with:
          node-version: ${{ env.NODE_VERSION }}

      # setup-bun-workspace instead of a hand-rolled `bun install
      # --frozen-lockfile`: this repo tracks bun canary, and canary rebuilds
      # reserialize bun.lock between builds, so a bare frozen install reds with
      # "lockfile had changes, but lockfile is frozen" on schedule runs (e.g.
      # run 28513244717). The composite handles the drift fallback and restores
      # the committed lockfile; same invocation as scenario-pr.yml's
      # app-browser-core lane this workflow mirrors.
      - name: Setup workspace dependencies
        uses: ./.github/actions/setup-bun-workspace
        with:
          bun-version: ${{ env.BUN_VERSION }}
          install-command: bun install --ignore-scripts --no-frozen-lockfile
          install-native-deps: "false"
          skip-avatar-clone: "true"
          no-vision-deps: "true"

      # Real Chromium for the engine lane — the same pattern scenario-pr.yml's
      # app-browser-core uses. resolveChromiumExecutablePath() picks it up via
      # playwright-core, so no extra env wiring is needed.
      - name: Install Chromium
        run: bunx playwright install --with-deps chromium

      - name: Run real Chromium benchmark lanes
        run: bun run --cwd plugins/plugin-browser test:real-chromium

      - name: Generate MiniWoB++ run-report artifacts (oracle + noop)
        env:
          BROWSER_EVIDENCE_DIR: ${{ runner.temp }}/evidence/10333-browser-real-chromium
        run: |
          mkdir -p "$BROWSER_EVIDENCE_DIR"
          bun run --cwd plugins/plugin-browser bench:miniwob:chromium --policy oracle --seeds 3 --out "$BROWSER_EVIDENCE_DIR/miniwob-chromium-oracle-run.json"
          bun run --cwd plugins/plugin-browser bench:miniwob:chromium --policy noop --seeds 2 --out "$BROWSER_EVIDENCE_DIR/miniwob-chromium-noop-run.json"

      - name: Generate web-grounding artifacts
        env:
          OUT: ${{ runner.temp }}/evidence/10333-web-grounding
        run: bun run --cwd plugins/plugin-browser bench:grounding:chromium

      - name: Generate external-dataset artifacts
        env:
          BROWSER_EVIDENCE_DIR: ${{ runner.temp }}/evidence/10333-browser-external-dataset
        run: |
          mkdir -p "$BROWSER_EVIDENCE_DIR"
          bun run --cwd plugins/plugin-browser bench:external --policy oracle --out "$BROWSER_EVIDENCE_DIR/external-dataset-oracle-run.json"
          bun run --cwd plugins/plugin-browser bench:external --policy noop --out "$BROWSER_EVIDENCE_DIR/external-dataset-noop-run.json"
          bun run --cwd plugins/plugin-browser bench:external --policy wrong --out "$BROWSER_EVIDENCE_DIR/external-dataset-wrong-run.json"
          bun run --cwd plugins/plugin-browser bench:external:chromium --policy oracle --out "$BROWSER_EVIDENCE_DIR/external-dataset-chromium-oracle-run.json"
          bun run --cwd plugins/plugin-browser bench:external:chromium --policy noop --out "$BROWSER_EVIDENCE_DIR/external-dataset-chromium-noop-run.json"
          bun run --cwd plugins/plugin-browser bench:external:chromium --policy wrong --out "$BROWSER_EVIDENCE_DIR/external-dataset-chromium-wrong-run.json"

      - name: Upload benchmark artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: browser-real-benchmark-runs
          path: |
            ${{ runner.temp }}/evidence/10333-browser-real-chromium/*
            ${{ runner.temp }}/evidence/10333-web-grounding/*
            ${{ runner.temp }}/evidence/10333-browser-external-dataset/*.json
          if-no-files-found: warn