name: Benchmark

# Nightly run of the HTTP user-journey performance benchmark
# (dev/benchmarks/omnigent). Seeds a sizeable corpus, boots a real server
# against it, drives the journeys, and uploads the JSON report as an artifact.
# Runs a backend matrix — SQLite (in-process) and Postgres (a service
# container, matching prod's Lakebase/Postgres round-trip + pooling profile).
# A workspace Databricks notebook pulls these artifacts via the GitHub API into
# a Delta table for the trend dashboard (see dev/benchmarks/omnigent/README.md)
# — so this workflow only produces artifacts; it never touches Databricks.
#
# Scheduled -> runs on the trusted default branch with the repo GITHUB_TOKEN;
# it reads no PR-authored code. Also dispatchable for an ad-hoc run.

on:
  schedule:
    - cron: "37 7 * * *" # 07:37 UTC nightly (off-peak, off the :00 mark)
  workflow_dispatch:
    inputs:
      iterations:
        description: "Requests per run"
        required: false
        default: "100"
      runs:
        description: "Timed runs per journey"
        required: false
        default: "3"
      sessions:
        description: "Seeded sessions"
        required: false
        default: "5000"
      items_per_session:
        description: "Seeded items per session"
        required: false
        default: "200"

permissions:
  contents: read

env:
  # No web SPA build during `uv sync` (setup.py _build_web_ui): this job never
  # serves the bundle, and the build otherwise times out on public npm.
  OMNIGENT_SKIP_WEB_UI: "true"
  UV_INDEX_URL: https://pypi.org/simple
  PIP_INDEX_URL: https://pypi.org/simple
  ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || '100' }}
  RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || '3' }}
  SESSIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.sessions || '5000' }}
  ITEMS: ${{ github.event_name == 'workflow_dispatch' && inputs.items_per_session || '200' }}

concurrency:
  # Never cancel a scheduled run mid-flight (each is a distinct data point);
  # coalesce manual dispatches per ref.
  group: benchmark-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}

jobs:
  benchmark:
    name: Run benchmark (${{ matrix.backend }})
    if: github.repository == 'omnigent-ai/omnigent'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        backend: [sqlite, postgres, mysql]
    services:
      # The Postgres and MySQL services are defined unconditionally (GitHub
      # Actions has no per-matrix-value service gating); each leg connects only
      # to its own backend and ignores the others. postgres:16 mirrors
      # Lakebase's major version; mysql:8.0 matches the stores-mysql CI lane.
      postgres:
        image: postgres:16
        env:
          POSTGRES_PASSWORD: bench
          POSTGRES_DB: benchdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: bench
          MYSQL_DATABASE: benchdb
        ports:
          - 3306:3306
        options: >-
          --health-cmd "mysqladmin ping -h 127.0.0.1 -u root -pbench"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
    steps:
      - name: Check out repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version-file: ".python-version"

      - name: Set up uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
        with:
          enable-cache: true

      - name: Install dependencies
        # `databricks` extra carries psycopg[binary] for the Postgres backend.
        run: uv sync --extra dev --extra databricks

      - name: Install MySQL driver
        # mysqlclient (mysql+mysqldb://) needs the system client library and is
        # not in any extra, so install it only on the mysql leg. Matches the
        # stores-mysql lane in ci.yml.
        if: matrix.backend == 'mysql'
        run: |
          sudo apt-get update -qq && sudo apt-get install -y -q libmysqlclient-dev
          uv pip install mysqlclient

      # Resolve the DB URI + a stable seed-cache key for this backend. The
      # cache key binds the DB schema head + seed.py contents + corpus config,
      # so a schema change or seed edit busts the cache and forces a reseed —
      # the "you changed the schema, refresh the seed" contract (SQLite only;
      # the Postgres/MySQL services are fresh each run so their DB is never
      # cached).
      - name: Resolve DB target
        id: db
        run: |
          HEAD="$(uv run --no-sync dev/benchmarks/omnigent/seed.py --print-head)"
          if [[ "${{ matrix.backend }}" == "postgres" ]]; then
            echo "uri=postgresql+psycopg://postgres:bench@localhost:5432/benchdb" >> "$GITHUB_OUTPUT"
            echo "cache_path=" >> "$GITHUB_OUTPUT"
          elif [[ "${{ matrix.backend }}" == "mysql" ]]; then
            echo "uri=mysql+mysqldb://root:bench@127.0.0.1:3306/benchdb" >> "$GITHUB_OUTPUT"
            echo "cache_path=" >> "$GITHUB_OUTPUT"
          else
            echo "uri=sqlite:///$PWD/bench.db" >> "$GITHUB_OUTPUT"
            echo "cache_path=bench.db" >> "$GITHUB_OUTPUT"
          fi
          echo "cache_key=benchdb-${{ matrix.backend }}-$HEAD-${SESSIONS}x${ITEMS}-${{ hashFiles('dev/benchmarks/omnigent/seed.py') }}" >> "$GITHUB_OUTPUT"

      # Reuse a previously-seeded SQLite corpus when schema + seed + config are
      # unchanged. No-op for the server-backed legs (empty path).
      - name: Restore seeded SQLite corpus
        if: matrix.backend == 'sqlite'
        id: seedcache
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
        with:
          path: ${{ steps.db.outputs.cache_path }}
          key: ${{ steps.db.outputs.cache_key }}

      - name: Seed corpus
        # The fresh-service backends (postgres, mysql) always seed; SQLite seeds
        # only on a cache miss. seed.py is itself idempotent, so a stray hit is
        # harmless.
        if: matrix.backend != 'sqlite' || steps.seedcache.outputs.cache-hit != 'true'
        run: |
          uv run --no-sync dev/benchmarks/omnigent/seed.py \
            --database-uri "${{ steps.db.outputs.uri }}" \
            --sessions "$SESSIONS" --items-per-session "$ITEMS"

      - name: Run benchmark
        run: |
          uv run --no-sync dev/benchmarks/omnigent/run.py \
            --database-uri "${{ steps.db.outputs.uri }}" \
            --iterations "$ITERATIONS" \
            --runs "$RUNS" \
            --output "benchmark-results-${{ matrix.backend }}.json"

      - name: Upload benchmark results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: benchmark-results-${{ matrix.backend }}-${{ github.run_id }}
          path: benchmark-results-${{ matrix.backend }}.json
          retention-days: 90
          if-no-files-found: warn