elizaos--eliza/.github/workflows/local-inference-bench.yml

name: Local Inference Bench

# Nightly benchmark for the on-device chat agent. The harness is host-
# agnostic — it just talks HTTP to whatever agent is reachable at
# --target — so we run it in two layers:
#
#   1. PR + nightly: harness route smoke. Boots
#      `packages/scripts/benchmark/synthetic-agent-server.mjs` on a free port and
#      runs the profile harness against its model-ensure path.
#   2. Nightly only: real agent profile. Boots `bun run dev`, points the
#      harness at the dev server, writes the report under
#      `reports/porting/<YYYY-MM-DD>/profile.{json,md}`, uploads the
#      directory as a workflow artifact, and opens/updates a tracking
#      issue with the markdown summary.
#
# Cuttlefish AVD profiling (job 3) is dispatch-only — it shares
# infrastructure with `elizaos-cuttlefish.yml`, so we don't duplicate
# that setup; instead we document the dispatch entrypoint and let the
# operator chain the two workflows.

on:
  schedule:
    # 05:00 UTC — runs after the nightly publish job at 04:00 UTC so
    # the dev server gets the latest packages.
    - cron: "0 5 * * *"
    # 04:30 UTC Sunday — weekly eliza-1 model publish window.
    - cron: "30 4 * * 0"
  pull_request:
    branches: [main]
    paths:
      - "packages/scripts/benchmark/**"
      - ".github/workflows/local-inference-bench.yml"
      - "plugins/plugin-local-embedding/**"
      - "packages/training/scripts/publish_eliza1_model.py"
      - "packages/training/scripts/sync_catalog_from_hf.py"
      - "scripts/verify-phone-download.mjs"
  workflow_dispatch:
    inputs:
      run_real_agent:
        description: "Run real-agent profile (nightly path) on-demand."
        required: false
        type: boolean
        default: false
      run_cuttlefish:
        description: "Run cuttlefish AVD profile (job 3). Coordinates with elizaos-cuttlefish.yml."
        required: false
        type: boolean
        default: false
      run_publish_models:
        description: "Run the eliza-1 model publish + catalog-sync job on-demand (gated by secrets.ELIZA_HF_TOKEN)."
        required: false
        type: boolean
        default: false

concurrency:
  group: local-inference-bench-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: write
  issues: write
  pull-requests: write

env:
  BUN_VERSION: "canary"
  NODE_VERSION: "24"
  STUB_PORT: "31338"

jobs:
  stub-validation:
    name: Harness route smoke
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          submodules: false
          show-progress: false

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Boot stub agent server
        run: |
          set -euo pipefail
          node packages/scripts/benchmark/synthetic-agent-server.mjs \
            --port "${STUB_PORT}" \
            --require-installed-models &
          echo $! > /tmp/stub.pid
          # Wait for the stub to bind (health endpoint becomes 200).
          for _ in $(seq 1 50); do
            if curl --silent --fail "http://127.0.0.1:${STUB_PORT}/api/health" >/dev/null; then
              break
            fi
            sleep 0.2
          done
          curl --silent --fail "http://127.0.0.1:${STUB_PORT}/api/health"

      - name: Verify harness health route only
        run: |
          set -euo pipefail
          curl --silent --fail "http://127.0.0.1:${STUB_PORT}/api/health"

      - name: Verify harness model ensure path
        run: |
          set -euo pipefail
          node packages/scripts/benchmark/profile-inference.mjs \
            --target "http://127.0.0.1:${STUB_PORT}" \
            --config packages/scripts/benchmark/configs/host-cpu.json \
            --label "stub-${GITHUB_SHA}-${GITHUB_RUN_ID}" \
            --out ".ci-logs/stub-profile" \
            --ensure-models \
            --download-timeout-ms 30000 \
            --load-timeout-ms 10000 \
            --request-timeout-ms 30000

      - name: Tear down stub
        if: ${{ always() }}
        run: |
          if [ -f /tmp/stub.pid ]; then
            kill "$(cat /tmp/stub.pid)" || true
          fi

  nightly-real-agent:
    name: Nightly real-agent profile
    needs: stub-validation
    if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.run_real_agent == true) }}
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          submodules: false
          show-progress: false
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Preflight — published eliza-1 manifest shape
        # The harness downloads the PUBLISHED HuggingFace bundle manifest and
        # validates it against the Eliza-1 manifest schema before fetching any
        # weight. A malformed published manifest (e.g. `files.vision` as an
        # object instead of an array — the 2026-06→07 Gemma-4 cutover defect)
        # otherwise fails the run ~5 minutes in, AFTER a full bun install +
        # agent boot, with an opaque "expected array, received object" trace.
        # Fail fast here with an operator-actionable message instead.
        run: node packages/scripts/benchmark/preflight-eliza1-manifest.mjs eliza-1-2b

      - name: Install dependencies
        run: bun install --frozen-lockfile || bun install --no-frozen-lockfile

      - name: Generate shared i18n assets
        run: bun run --cwd packages/shared build:i18n

      - name: Prepare dev workspace packages
        # `bun run dev` starts Vite from a clean checkout. Build the app's
        # workspace dependencies first so Vite can resolve package exports like
        # @elizaos/core instead of failing before the benchmark can run.
        run: bun run dev:prepare

      - name: Boot dev agent
        env:
          ELIZA_API_PORT: "31337"
          ELIZA_PORT: "2138"
          NODE_ENV: "test"
          ELIZA_DISABLE_TRAJECTORY_LOGGING: "1"
        run: |
          set -euo pipefail
          mkdir -p .ci-logs
          # Run the dev server in the background. The orchestrator
          # already auto-shifts ports if the defaults are busy and writes
          # a `.eliza/desktop-dev-console.log` we can tail if needed.
          bun run dev > .ci-logs/dev.log 2>&1 &
          echo $! > /tmp/dev.pid
          # Health probe — give the agent up to 120s to come up.
          for _ in $(seq 1 60); do
            if curl --silent --fail "http://127.0.0.1:31337/api/health" >/dev/null 2>&1; then
              echo "agent ready"
              break
            fi
            sleep 2
          done
          curl --silent --fail "http://127.0.0.1:31337/api/health"

      - name: Run profile harness (host-cpu config)
        env:
          ELIZA_API_TOKEN: ${{ secrets.ELIZA_CI_API_TOKEN }}
        run: |
          set -euo pipefail
          DATE="$(date -u +%Y-%m-%d)"
          OUT_DIR="reports/porting/${DATE}"
          mkdir -p "$OUT_DIR"
          node packages/scripts/benchmark/profile-inference.mjs \
            --target "http://127.0.0.1:31337" \
            --config packages/scripts/benchmark/configs/host-cpu.json \
            --label "ci-${GITHUB_SHA}-${GITHUB_RUN_ID}" \
            --out "$OUT_DIR" \
            --ensure-models
          echo "REPORT_DIR=$OUT_DIR" >> "$GITHUB_ENV"

      - name: Tear down dev agent
        if: ${{ always() }}
        run: |
          if [ -f /tmp/dev.pid ]; then
            kill "$(cat /tmp/dev.pid)" || true
          fi

      - name: Upload report artifact
        if: ${{ always() && env.REPORT_DIR != '' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: profile-nightly-${{ github.run_id }}
          path: ${{ env.REPORT_DIR }}
          if-no-files-found: warn
          retention-days: 90

      - name: Upload dev server log
        if: ${{ always() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: dev-log-nightly-${{ github.run_id }}
          path: .ci-logs/dev.log
          if-no-files-found: warn
          retention-days: 14

      - name: Open / update tracking issue
        if: ${{ env.REPORT_DIR != '' }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
        with:
          script: |
            const fs = require("node:fs");
            const path = require("node:path");
            const reportDir = process.env.REPORT_DIR;
            const mdPath = path.join(reportDir, "profile.md");
            if (!fs.existsSync(mdPath)) {
              core.warning(`profile.md not found at ${mdPath}; skipping issue update`);
              return;
            }
            const md = fs.readFileSync(mdPath, "utf8");
            const today = new Date().toISOString().slice(0, 10);
            const title = `Nightly local-inference profile — ${today}`;
            const artifactUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const body = [
              `Workflow run: ${artifactUrl}`,
              "",
              `Report directory: \`${reportDir}\`.`,
              "",
              md,
            ].join("\n");
            const { data: existing } = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: "nightly-local-inference",
              state: "open",
              per_page: 5,
            });
            if (existing.length === 0) {
              const created = await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title,
                body,
                labels: ["nightly-local-inference"],
              });
              core.info(`Opened tracking issue #${created.data.number}`);
            } else {
              const issue = existing[0];
              await github.rest.issues.update({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: issue.number,
                title,
                body,
              });
              core.info(`Updated tracking issue #${issue.number}`);
            }

  cuttlefish-bench:
    # Manual-only path. Cuttlefish AVD provisioning is owned by
    # elizaos-cuttlefish.yml; this job runs the bench against the AVD
    # forward port once the operator has the AVD reachable at
    # 127.0.0.1:31337. Keeping it here (rather than duplicating AVD
    # setup) makes the dispatch surface obvious without doubling the CI
    # cost.
    name: Cuttlefish AVD profile (manual)
    if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_cuttlefish == true }}
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Setup Bun
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Reminder — boot cuttlefish first
        run: |
          echo "::warning ::This job assumes elizaos-cuttlefish.yml has booted the AVD"
          echo "::warning ::and forwarded 31337 to 127.0.0.1. Run that workflow first."
          echo "Probing /api/health on 127.0.0.1:31337 ..."
          curl --silent --fail "http://127.0.0.1:31337/api/health" || {
            echo "::error ::No agent reachable at 127.0.0.1:31337. Run elizaos-cuttlefish.yml first."
            exit 1
          }

      - name: Run profile harness (aosp-arm64 config)
        run: |
          set -euo pipefail
          DATE="$(date -u +%Y-%m-%d)"
          OUT_DIR="reports/porting/${DATE}-cuttlefish"
          mkdir -p "$OUT_DIR"
          node packages/scripts/benchmark/profile-inference.mjs \
            --target "http://127.0.0.1:31337" \
            --config packages/scripts/benchmark/configs/aosp-arm64.json \
            --label "cuttlefish-${GITHUB_RUN_ID}" \
            --out "$OUT_DIR" \
            --ensure-models

      - name: Upload cuttlefish report artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: profile-cuttlefish-${{ github.run_id }}
          path: reports/porting/
          if-no-files-found: warn
          retention-days: 90

  publish-models-nightly:
    # Weekly nightly publish to the elizaos HF org. Walks W5-Pipeline's
    # output, calls publish_eliza1_model.py for each fused-kernel GGUF,
    # then runs sync_catalog_from_hf.py to produce the catalog diff and
    # opens a PR with the update.
    #
    # Gated behind secrets.ELIZA_HF_TOKEN. When the secret is unset
    # (forks, dry-test branches), the job no-ops with an explicit
    # warning so unrelated workflow runs don't fail.
    name: Publish eliza-1 models (weekly)
    if: >-
      (github.event_name == 'schedule' && github.event.schedule == '30 4 * * 0') ||
      (github.event_name == 'workflow_dispatch' && inputs.run_publish_models == true)
    runs-on: ${{ fromJSON(vars.HETZNER_FLEET_ONLINE == 'false' && '["ubuntu-24.04"]' || '["self-hosted","hetzner-robot"]') }}
    timeout-minutes: 90
    env:
      # W5-Pipeline drops finished checkpoints under this path. The
      # nightly publish step expects each subdirectory to be a complete
      # `<base>-optimized` (or `-drafter`) bundle —
      # i.e. a single .gguf + manifest.json + README.md.
      W5_PIPELINE_OUTPUT_ROOT: ${{ vars.W5_PIPELINE_OUTPUT_ROOT || '/mnt/w5-pipeline-output' }}
      HF_TOKEN: ${{ secrets.ELIZA_HF_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
        with:
          submodules: false
          show-progress: false

      - name: Guard — require ELIZA_HF_TOKEN
        run: |
          set -euo pipefail
          if [ -z "${HF_TOKEN:-}" ]; then
            echo "::warning ::secrets.ELIZA_HF_TOKEN not configured; publish job will no-op."
            echo "PUBLISH_SKIPPED=1" >> "$GITHUB_ENV"
            exit 0
          fi
          echo "PUBLISH_SKIPPED=0" >> "$GITHUB_ENV"

      - name: Setup Python
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
        with:
          python-version: "3.11"

      - name: Install huggingface_hub
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        run: |
          set -euo pipefail
          python -m pip install --upgrade pip
          python -m pip install "huggingface_hub[cli]>=0.24"

      - name: Setup Node.js
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Setup Bun
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Probe W5-Pipeline output
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        run: |
          set -euo pipefail
          if [ ! -d "$W5_PIPELINE_OUTPUT_ROOT" ]; then
            echo "::warning ::no W5-Pipeline output at $W5_PIPELINE_OUTPUT_ROOT; nothing to publish"
            echo "NO_PIPELINE_OUTPUT=1" >> "$GITHUB_ENV"
            exit 0
          fi
          echo "Pipeline output:"
          ls -la "$W5_PIPELINE_OUTPUT_ROOT"

      - name: Publish each fused-kernel bundle
        if: ${{ env.PUBLISH_SKIPPED != '1' && env.NO_PIPELINE_OUTPUT != '1' }}
        run: |
          set -euo pipefail
          shopt -s nullglob
          FAILURES=0
          for dir in "$W5_PIPELINE_OUTPUT_ROOT"/*/; do
            name=$(basename "$dir")
            # Bundle naming: <base>-optimized or <base>-drafter.
            # Anything else is rejected — these repos are for fused builds only.
            if [[ "$name" != *-optimized && "$name" != *-drafter ]]; then
              echo "::warning ::skipping $name (not a fused-kernel bundle name)"
              continue
            fi
            repo_id="elizaos/$name"
            echo "::group::publish $repo_id"
            if ! python packages/training/scripts/publish_eliza1_model.py \
                --model-dir "$dir" \
                --repo-id "$repo_id"; then
              echo "::error ::publish failed for $name"
              FAILURES=$((FAILURES + 1))
            fi
            echo "::endgroup::"
          done
          if [ "$FAILURES" -gt 0 ]; then
            echo "::error ::$FAILURES bundle(s) failed to publish"
            exit 1
          fi

      - name: Sync catalog diff from elizaos
        if: ${{ env.PUBLISH_SKIPPED != '1' }}
        run: |
          set -euo pipefail
          DATE="$(date -u +%Y-%m-%d)"
          OUT_DIR="reports/porting/${DATE}"
          mkdir -p "$OUT_DIR"
          python packages/training/scripts/sync_catalog_from_hf.py \
            --org elizaos \
            --out "$OUT_DIR/catalog-diff.json"
          echo "DIFF_PATH=$OUT_DIR/catalog-diff.json" >> "$GITHUB_ENV"

      - name: Verify a sample download (round-trip from HF)
        if: ${{ env.PUBLISH_SKIPPED != '1' && env.DIFF_PATH != '' }}
        run: |
          set -euo pipefail
          # Pick the first published id from the diff and round-trip it
          # through the phone-equivalent Downloader.
          SAMPLE_ID=$(python -c 'import json,os,sys; d=json.load(open(os.environ["DIFF_PATH"])); ids=[e["id"] for e in d.get("entries",[])]; print(ids[0] if ids else "")')
          if [ -z "${SAMPLE_ID:-}" ]; then
            echo "::warning ::diff has no published entries; skipping verify-phone-download"
            exit 0
          fi
          bun install --frozen-lockfile
          node scripts/verify-phone-download.mjs \
            --model-id "$SAMPLE_ID" \
            --catalog-diff "$DIFF_PATH"

      - name: Open catalog-update PR
        if: ${{ env.PUBLISH_SKIPPED != '1' && env.DIFF_PATH != '' }}
        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          branch: ci/eliza-1-catalog-${{ github.run_id }}
          base: develop
          commit-message: "chore(catalog): refresh eliza-1 pointers (auto)"
          title: "chore(catalog): refresh eliza-1 pointers"
          body: |
            Auto-generated catalog diff from `sync_catalog_from_hf.py`.

            - Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
            - Diff path: `${{ env.DIFF_PATH }}`

            W5-Catalog: please apply the diff to `packages/shared/src/local-inference/catalog.ts`.
          add-paths: |
            reports/porting/**/catalog-diff.json
          delete-branch: true
          labels: |
            eliza-1
            local-inference
            automated