chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,182 @@
|
||||
name: Benchmark
|
||||
|
||||
# Nightly run of the HTTP user-journey performance benchmark
|
||||
# (dev/benchmarks/omnigent). Seeds a sizeable corpus, boots a real server
|
||||
# against it, drives the journeys, and uploads the JSON report as an artifact.
|
||||
# Runs a backend matrix — SQLite (in-process) and Postgres (a service
|
||||
# container, matching prod's Lakebase/Postgres round-trip + pooling profile).
|
||||
# A workspace Databricks notebook pulls these artifacts via the GitHub API into
|
||||
# a Delta table for the trend dashboard (see dev/benchmarks/omnigent/README.md)
|
||||
# — so this workflow only produces artifacts; it never touches Databricks.
|
||||
#
|
||||
# Scheduled -> runs on the trusted default branch with the repo GITHUB_TOKEN;
|
||||
# it reads no PR-authored code. Also dispatchable for an ad-hoc run.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "37 7 * * *" # 07:37 UTC nightly (off-peak, off the :00 mark)
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
iterations:
|
||||
description: "Requests per run"
|
||||
required: false
|
||||
default: "100"
|
||||
runs:
|
||||
description: "Timed runs per journey"
|
||||
required: false
|
||||
default: "3"
|
||||
sessions:
|
||||
description: "Seeded sessions"
|
||||
required: false
|
||||
default: "5000"
|
||||
items_per_session:
|
||||
description: "Seeded items per session"
|
||||
required: false
|
||||
default: "200"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# No web SPA build during `uv sync` (setup.py _build_web_ui): this job never
|
||||
# serves the bundle, and the build otherwise times out on public npm.
|
||||
OMNIGENT_SKIP_WEB_UI: "true"
|
||||
UV_INDEX_URL: https://pypi.org/simple
|
||||
PIP_INDEX_URL: https://pypi.org/simple
|
||||
ITERATIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.iterations || '100' }}
|
||||
RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || '3' }}
|
||||
SESSIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.sessions || '5000' }}
|
||||
ITEMS: ${{ github.event_name == 'workflow_dispatch' && inputs.items_per_session || '200' }}
|
||||
|
||||
concurrency:
|
||||
# Never cancel a scheduled run mid-flight (each is a distinct data point);
|
||||
# coalesce manual dispatches per ref.
|
||||
group: benchmark-${{ github.event_name }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
benchmark:
|
||||
name: Run benchmark (${{ matrix.backend }})
|
||||
if: github.repository == 'omnigent-ai/omnigent'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
backend: [sqlite, postgres, mysql]
|
||||
services:
|
||||
# The Postgres and MySQL services are defined unconditionally (GitHub
|
||||
# Actions has no per-matrix-value service gating); each leg connects only
|
||||
# to its own backend and ignores the others. postgres:16 mirrors
|
||||
# Lakebase's major version; mysql:8.0 matches the stores-mysql CI lane.
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_PASSWORD: bench
|
||||
POSTGRES_DB: benchdb
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: bench
|
||||
MYSQL_DATABASE: benchdb
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: >-
|
||||
--health-cmd "mysqladmin ping -h 127.0.0.1 -u root -pbench"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- name: Check out repo
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
|
||||
with:
|
||||
enable-cache: true
|
||||
|
||||
- name: Install dependencies
|
||||
# `databricks` extra carries psycopg[binary] for the Postgres backend.
|
||||
run: uv sync --extra dev --extra databricks
|
||||
|
||||
- name: Install MySQL driver
|
||||
# mysqlclient (mysql+mysqldb://) needs the system client library and is
|
||||
# not in any extra, so install it only on the mysql leg. Matches the
|
||||
# stores-mysql lane in ci.yml.
|
||||
if: matrix.backend == 'mysql'
|
||||
run: |
|
||||
sudo apt-get update -qq && sudo apt-get install -y -q libmysqlclient-dev
|
||||
uv pip install mysqlclient
|
||||
|
||||
# Resolve the DB URI + a stable seed-cache key for this backend. The
|
||||
# cache key binds the DB schema head + seed.py contents + corpus config,
|
||||
# so a schema change or seed edit busts the cache and forces a reseed —
|
||||
# the "you changed the schema, refresh the seed" contract (SQLite only;
|
||||
# the Postgres/MySQL services are fresh each run so their DB is never
|
||||
# cached).
|
||||
- name: Resolve DB target
|
||||
id: db
|
||||
run: |
|
||||
HEAD="$(uv run --no-sync dev/benchmarks/omnigent/seed.py --print-head)"
|
||||
if [[ "${{ matrix.backend }}" == "postgres" ]]; then
|
||||
echo "uri=postgresql+psycopg://postgres:bench@localhost:5432/benchdb" >> "$GITHUB_OUTPUT"
|
||||
echo "cache_path=" >> "$GITHUB_OUTPUT"
|
||||
elif [[ "${{ matrix.backend }}" == "mysql" ]]; then
|
||||
echo "uri=mysql+mysqldb://root:bench@127.0.0.1:3306/benchdb" >> "$GITHUB_OUTPUT"
|
||||
echo "cache_path=" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "uri=sqlite:///$PWD/bench.db" >> "$GITHUB_OUTPUT"
|
||||
echo "cache_path=bench.db" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "cache_key=benchdb-${{ matrix.backend }}-$HEAD-${SESSIONS}x${ITEMS}-${{ hashFiles('dev/benchmarks/omnigent/seed.py') }}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Reuse a previously-seeded SQLite corpus when schema + seed + config are
|
||||
# unchanged. No-op for the server-backed legs (empty path).
|
||||
- name: Restore seeded SQLite corpus
|
||||
if: matrix.backend == 'sqlite'
|
||||
id: seedcache
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
||||
with:
|
||||
path: ${{ steps.db.outputs.cache_path }}
|
||||
key: ${{ steps.db.outputs.cache_key }}
|
||||
|
||||
- name: Seed corpus
|
||||
# The fresh-service backends (postgres, mysql) always seed; SQLite seeds
|
||||
# only on a cache miss. seed.py is itself idempotent, so a stray hit is
|
||||
# harmless.
|
||||
if: matrix.backend != 'sqlite' || steps.seedcache.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
uv run --no-sync dev/benchmarks/omnigent/seed.py \
|
||||
--database-uri "${{ steps.db.outputs.uri }}" \
|
||||
--sessions "$SESSIONS" --items-per-session "$ITEMS"
|
||||
|
||||
- name: Run benchmark
|
||||
run: |
|
||||
uv run --no-sync dev/benchmarks/omnigent/run.py \
|
||||
--database-uri "${{ steps.db.outputs.uri }}" \
|
||||
--iterations "$ITERATIONS" \
|
||||
--runs "$RUNS" \
|
||||
--output "benchmark-results-${{ matrix.backend }}.json"
|
||||
|
||||
- name: Upload benchmark results
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: always()
|
||||
with:
|
||||
name: benchmark-results-${{ matrix.backend }}-${{ github.run_id }}
|
||||
path: benchmark-results-${{ matrix.backend }}.json
|
||||
retention-days: 90
|
||||
if-no-files-found: warn
|
||||
Reference in New Issue
Block a user