wehub-resource-sync
183 lines
7.3 KiB
YAML
183 lines
7.3 KiB
YAML
name: Benchmark
|
|
|
|
# Nightly run of the HTTP user-journey performance benchmark
|
|
# (dev/benchmarks/omnigent). Seeds a sizeable corpus, boots a real server
|
|
# against it, drives the journeys, and uploads the JSON report as an artifact.
|
|
# Runs a backend matrix — SQLite (in-process) and Postgres (a service
|
|
# container, matching prod's Lakebase/Postgres round-trip + pooling profile).
|
|
# A workspace Databricks notebook pulls these artifacts via the GitHub API into
|
|
# a Delta table for the trend dashboard (see dev/benchmarks/omnigent/README.md)
|
|
# — so this workflow only produces artifacts; it never touches Databricks.
|
|
#
|
|
# Scheduled -> runs on the trusted default branch with the repo GITHUB_TOKEN;
|
|
# it reads no PR-authored code. Also dispatchable for an ad-hoc run.
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "37 7 * * *" # 07:37 UTC nightly (off-peak, off the :00 mark)
|
|
workflow_dispatch:
|
|
inputs:
|
|
iterations:
|
|
description: "Requests per run"
|
|
required: false
|
|
default: "100"
|
|
runs:
|
|
description: "Timed runs per journey"
|
|
required: false
|
|
default: "3"
|
|
sessions:
|
|
description: "Seeded sessions"
|
|
required: false
|
|
default: "5000"
|
|
items_per_session:
|
|
description: "Seeded items per session"
|
|
required: false
|
|
default: "200"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
# No web SPA build during `uv sync` (setup.py _build_web_ui): this job never
|
|
# serves the bundle, and the build otherwise times out on public npm.
|
|
OMNIGENT_SKIP_WEB_UI: "true"
|
|
UV_INDEX_URL: https://pypi.org/simple
|
|
PIP_INDEX_URL: https://pypi.org/simple
|
|
ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || '100' }}
|
|
RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || '3' }}
|
|
SESSIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.sessions || '5000' }}
|
|
ITEMS: ${{ github.event_name == 'workflow_dispatch' && inputs.items_per_session || '200' }}
|
|
|
|
concurrency:
|
|
# Never cancel a scheduled run mid-flight (each is a distinct data point);
|
|
# coalesce manual dispatches per ref.
|
|
group: benchmark-${{ github.event_name }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
|
|
|
|
jobs:
|
|
benchmark:
|
|
name: Run benchmark (${{ matrix.backend }})
|
|
if: github.repository == 'omnigent-ai/omnigent'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
backend: [sqlite, postgres, mysql]
|
|
services:
|
|
# The Postgres and MySQL services are defined unconditionally (GitHub
|
|
# Actions has no per-matrix-value service gating); each leg connects only
|
|
# to its own backend and ignores the others. postgres:16 mirrors
|
|
# Lakebase's major version; mysql:8.0 matches the stores-mysql CI lane.
|
|
postgres:
|
|
image: postgres:16
|
|
env:
|
|
POSTGRES_PASSWORD: bench
|
|
POSTGRES_DB: benchdb
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 5s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
mysql:
|
|
image: mysql:8.0
|
|
env:
|
|
MYSQL_ROOT_PASSWORD: bench
|
|
MYSQL_DATABASE: benchdb
|
|
ports:
|
|
- 3306:3306
|
|
options: >-
|
|
--health-cmd "mysqladmin ping -h 127.0.0.1 -u root -pbench"
|
|
--health-interval 5s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- name: Check out repo
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
# `databricks` extra carries psycopg[binary] for the Postgres backend.
|
|
run: uv sync --extra dev --extra databricks
|
|
|
|
- name: Install MySQL driver
|
|
# mysqlclient (mysql+mysqldb://) needs the system client library and is
|
|
# not in any extra, so install it only on the mysql leg. Matches the
|
|
# stores-mysql lane in ci.yml.
|
|
if: matrix.backend == 'mysql'
|
|
run: |
|
|
sudo apt-get update -qq && sudo apt-get install -y -q libmysqlclient-dev
|
|
uv pip install mysqlclient
|
|
|
|
# Resolve the DB URI + a stable seed-cache key for this backend. The
|
|
# cache key binds the DB schema head + seed.py contents + corpus config,
|
|
# so a schema change or seed edit busts the cache and forces a reseed —
|
|
# the "you changed the schema, refresh the seed" contract (SQLite only;
|
|
# the Postgres/MySQL services are fresh each run so their DB is never
|
|
# cached).
|
|
- name: Resolve DB target
|
|
id: db
|
|
run: |
|
|
HEAD="$(uv run --no-sync dev/benchmarks/omnigent/seed.py --print-head)"
|
|
if [[ "${{ matrix.backend }}" == "postgres" ]]; then
|
|
echo "uri=postgresql+psycopg://postgres:bench@localhost:5432/benchdb" >> "$GITHUB_OUTPUT"
|
|
echo "cache_path=" >> "$GITHUB_OUTPUT"
|
|
elif [[ "${{ matrix.backend }}" == "mysql" ]]; then
|
|
echo "uri=mysql+mysqldb://root:bench@127.0.0.1:3306/benchdb" >> "$GITHUB_OUTPUT"
|
|
echo "cache_path=" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "uri=sqlite:///$PWD/bench.db" >> "$GITHUB_OUTPUT"
|
|
echo "cache_path=bench.db" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
echo "cache_key=benchdb-${{ matrix.backend }}-$HEAD-${SESSIONS}x${ITEMS}-${{ hashFiles('dev/benchmarks/omnigent/seed.py') }}" >> "$GITHUB_OUTPUT"
|
|
|
|
# Reuse a previously-seeded SQLite corpus when schema + seed + config are
|
|
# unchanged. No-op for the server-backed legs (empty path).
|
|
- name: Restore seeded SQLite corpus
|
|
if: matrix.backend == 'sqlite'
|
|
id: seedcache
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: ${{ steps.db.outputs.cache_path }}
|
|
key: ${{ steps.db.outputs.cache_key }}
|
|
|
|
- name: Seed corpus
|
|
# The fresh-service backends (postgres, mysql) always seed; SQLite seeds
|
|
# only on a cache miss. seed.py is itself idempotent, so a stray hit is
|
|
# harmless.
|
|
if: matrix.backend != 'sqlite' || steps.seedcache.outputs.cache-hit != 'true'
|
|
run: |
|
|
uv run --no-sync dev/benchmarks/omnigent/seed.py \
|
|
--database-uri "${{ steps.db.outputs.uri }}" \
|
|
--sessions "$SESSIONS" --items-per-session "$ITEMS"
|
|
|
|
- name: Run benchmark
|
|
run: |
|
|
uv run --no-sync dev/benchmarks/omnigent/run.py \
|
|
--database-uri "${{ steps.db.outputs.uri }}" \
|
|
--iterations "$ITERATIONS" \
|
|
--runs "$RUNS" \
|
|
--output "benchmark-results-${{ matrix.backend }}.json"
|
|
|
|
- name: Upload benchmark results
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always()
|
|
with:
|
|
name: benchmark-results-${{ matrix.backend }}-${{ github.run_id }}
|
|
path: benchmark-results-${{ matrix.backend }}.json
|
|
retention-days: 90
|
|
if-no-files-found: warn
|