wehub-resource-sync
465 lines
18 KiB
YAML
465 lines
18 KiB
YAML
name: CI
|
|
|
|
# Unit-test pytest matrix on every non-draft PR and on push to main. Tests are
|
|
# split across directory-based matrix groups (runtime-*, server-*, inner-rest,
|
|
# tools, repl-sdk, spec-llms, runner-app, stores, misc) so slow files don't
|
|
# bottleneck one runner; the slowest groups use `--dist=worksteal` to fan tests
|
|
# out within a file. The `misc` group is a catch-all so new top-level
|
|
# tests/<dir>/ are picked up automatically (it ignores the dirs that have their
|
|
# own group). Draft PRs are skipped (ready_for_review re-fires the workflow).
|
|
# A `coverage-report` job combines per-shard coverage for code-coverage.yml.
|
|
|
|
on:
|
|
pull_request:
|
|
types: [opened, synchronize, reopened, ready_for_review]
|
|
paths-ignore: ['web/**', 'tests/e2e_ui/**']
|
|
push:
|
|
branches:
|
|
- main
|
|
paths-ignore: ['web/**', 'tests/e2e_ui/**']
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
# No web SPA build during `uv sync`; this job never serves the bundle.
|
|
OMNIGENT_SKIP_WEB_UI: "true"
|
|
UV_INDEX_URL: https://pypi.org/simple
|
|
PIP_INDEX_URL: https://pypi.org/simple
|
|
|
|
concurrency:
|
|
group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# Security precondition gate (security-gate.yml): untrusted PRs wait for the
|
|
# scan; trusted authors / non-PR events pass through.
|
|
gate:
|
|
uses: ./.github/workflows/security-gate.yml
|
|
|
|
pytest:
|
|
name: Pytest (${{ matrix.group }})
|
|
needs: gate
|
|
if: ${{ !github.event.pull_request.draft }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- group: runtime-harnesses
|
|
paths: tests/runtime/harnesses
|
|
dist: worksteal
|
|
- group: runtime-policies
|
|
paths: tests/runtime/policies
|
|
- group: runtime-core
|
|
paths: >-
|
|
tests/runtime
|
|
--ignore=tests/runtime/harnesses
|
|
--ignore=tests/runtime/policies
|
|
dist: worksteal
|
|
- group: inner-rest
|
|
paths: tests/inner
|
|
- group: tools
|
|
paths: tests/tools tests/test_errors.py
|
|
- group: repl-sdk
|
|
paths: tests/frontends tests/repl tests/terminals
|
|
# Isolates the park-on-future elicitation/permission/policy files so a
|
|
# wedge there stalls one small job, not the whole server shard.
|
|
- group: server-approvals
|
|
paths: >-
|
|
tests/server/integration/test_sessions_permission_request_hook.py
|
|
tests/server/integration/test_sessions_elicitation_resolve_url.py
|
|
tests/server/integration/test_sessions_policy_evaluate.py
|
|
workers: "4"
|
|
# worksteal: the biggest file would otherwise pin one worker past the
|
|
# shard's balanced wall time. server/conftest fixtures are function-scoped.
|
|
- group: server-integration
|
|
paths: >-
|
|
tests/server/integration
|
|
--ignore=tests/server/integration/test_sessions_permission_request_hook.py
|
|
--ignore=tests/server/integration/test_sessions_elicitation_resolve_url.py
|
|
--ignore=tests/server/integration/test_sessions_policy_evaluate.py
|
|
workers: "4"
|
|
dist: worksteal
|
|
# Historical name; stays in merge-ready's REQUIRED list.
|
|
- group: server-rest
|
|
paths: tests/server --ignore=tests/server/integration tests/onboarding
|
|
workers: "4"
|
|
- group: spec-llms
|
|
paths: tests/spec tests/llms
|
|
# Integration journey tests with mock LLM (no API key).
|
|
# workers=0 (serial): session-scoped live_server + mock_llm_server
|
|
# fixtures spawn subprocesses that must share one mock server;
|
|
# xdist workers would each create their own session fixtures.
|
|
- group: integration-mock
|
|
paths: tests/integration
|
|
workers: "0"
|
|
# Carved out of misc: runner + stores were ~68% of misc's cpu and
|
|
# under loadfile a single 500s+ file (test_app_sessions_native) pinned
|
|
# one worker and set the whole misc wall time. worksteal fans each
|
|
# dir's tests across workers (biggest single test is ~40s / ~5s, so
|
|
# the floor drops from ~500s to ~100s). Both dirs' conftests are
|
|
# function-scoped, so splitting a file across workers is safe.
|
|
- group: runner-app
|
|
paths: tests/runner
|
|
dist: worksteal
|
|
- group: stores
|
|
paths: tests/stores
|
|
dist: worksteal
|
|
# Catch-all so new top-level tests/<dir>/ are covered automatically.
|
|
# worksteal keeps the biggest remaining file (the benchmark smoke
|
|
# test, ~58s) from re-pinning one worker as this catch-all grows.
|
|
- group: misc
|
|
paths: >-
|
|
tests
|
|
--ignore=tests/e2e
|
|
--ignore=tests/integration
|
|
--ignore=tests/runtime
|
|
--ignore=tests/inner
|
|
--ignore=tests/tools
|
|
--ignore=tests/test_errors.py
|
|
--ignore=tests/frontends
|
|
--ignore=tests/repl
|
|
--ignore=tests/terminals
|
|
--ignore=tests/server
|
|
--ignore=tests/onboarding
|
|
--ignore=tests/spec
|
|
--ignore=tests/llms
|
|
--ignore=tests/codex_parity
|
|
--ignore=tests/runner
|
|
--ignore=tests/stores
|
|
dist: worksteal
|
|
# Databricks-coupled tests (Lakebase token engine, psycopg). This is
|
|
# the only lane that installs the `databricks` extra; the
|
|
# @pytest.mark.databricks marker keeps these tests off the lean lanes
|
|
# (which run -m "not databricks") and selects them here.
|
|
- group: databricks
|
|
paths: tests/db tests/deploy
|
|
extra: databricks
|
|
markexpr: databricks
|
|
|
|
steps:
|
|
- name: Check out repo
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Install ripgrep + bubblewrap + tmux
|
|
# ripgrep: the Grep tool prefers it. bubblewrap: the linux_bwrap sandbox
|
|
# needs it. tmux: the integration-mock shard spawns harness terminals.
|
|
# apparmor sysctl: Ubuntu 24.04 blocks unprivileged user namespaces
|
|
# that bwrap needs; scope is the ephemeral runner.
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y ripgrep bubblewrap tmux
|
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
|
|
|
- name: Cache virtualenv
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ hashFiles('.python-version') }}-${{ hashFiles('uv.lock') }}
|
|
|
|
- name: Install dependencies
|
|
# matrix.extra (e.g. "databricks") adds an extra for lanes that need it;
|
|
# empty for the default lanes.
|
|
run: uv sync --locked --extra all --extra dev ${{ matrix.extra && format('--extra {0}', matrix.extra) || '' }}
|
|
|
|
- name: Run pytest
|
|
shell: bash
|
|
env:
|
|
PYTHONFAULTHANDLER: "1"
|
|
PYTEST_PROGRESS_LOG_DIR: artifacts/progress
|
|
OMNIGENT_TOKEN_USAGE_JSON: artifacts/tokens-${{ matrix.group }}.json
|
|
# Outside the repo: coverage.py's transient `.coverage.*` files
|
|
# under the ro-bound cwd raced the sandbox's dotfile masker. Staged
|
|
# back into artifacts/ below for the coverage-report job.
|
|
COVERAGE_FILE: ${{ runner.temp }}/omnigent-coverage/.coverage.${{ matrix.group }}
|
|
# sysmon: the default C-trace coverage backend pushed the heaviest
|
|
# shard past its timeout; only line coverage is collected.
|
|
COVERAGE_CORE: sysmon
|
|
run: |
|
|
mkdir -p artifacts artifacts/progress "$(dirname "$COVERAGE_FILE")"
|
|
# matrix.paths is unquoted on purpose: it word-splits into pytest args.
|
|
# shellcheck disable=SC2086
|
|
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY -u DATABRICKS_TOKEN \
|
|
uv run pytest ${{ matrix.paths }} \
|
|
-m "${{ matrix.markexpr || 'not databricks' }}" \
|
|
-n ${{ matrix.workers || '8' }} \
|
|
--dist=${{ matrix.dist || 'loadfile' }} \
|
|
--timeout=${{ matrix.timeout || '300' }} \
|
|
--junitxml=artifacts/pytest-${{ matrix.group }}.xml \
|
|
--cov=omnigent --cov-report= \
|
|
-v --tb=long --showlocals --log-level=INFO -r a \
|
|
|| { rc=$?; [ "$rc" -eq 5 ] && echo "::notice::No tests collected in this shard; treating as a pass." || exit "$rc"; }
|
|
|
|
- name: Stage coverage data for upload
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
cov_file="${{ runner.temp }}/omnigent-coverage/.coverage.${{ matrix.group }}"
|
|
if [[ -f "$cov_file" ]]; then
|
|
cp "$cov_file" "artifacts/.coverage.${{ matrix.group }}"
|
|
else
|
|
echo "::notice::No coverage data file at $cov_file; nothing to stage."
|
|
fi
|
|
|
|
- name: Upload pytest artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: pytest-${{ matrix.group }}-${{ github.run_id }}
|
|
path: artifacts/
|
|
retention-days: 14
|
|
include-hidden-files: true # the per-shard .coverage.<group> dotfile
|
|
|
|
stores-postgres:
|
|
name: Pytest (stores-postgres)
|
|
needs: gate
|
|
if: ${{ !github.event.pull_request.draft }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
services:
|
|
postgres:
|
|
image: postgres:16
|
|
env:
|
|
POSTGRES_PASSWORD: omnigent
|
|
POSTGRES_DB: omnigent_root
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 5s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
|
|
with:
|
|
python-version-file: ".python-version"
|
|
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a
|
|
with:
|
|
enable-cache: true
|
|
- name: Install dependencies
|
|
run: uv sync --locked --extra all --extra dev --extra databricks
|
|
- name: Run store + DB tests against PostgreSQL
|
|
env:
|
|
OMNIGENT_TEST_DB_URI: postgresql+psycopg://postgres:omnigent@localhost:5432/omnigent_root
|
|
run: |
|
|
uv run pytest tests/stores tests/db \
|
|
-m "not databricks" \
|
|
-n 4 \
|
|
--dist=loadfile \
|
|
--timeout=300 \
|
|
--junitxml=artifacts/pytest-stores-postgres.xml
|
|
- if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
|
|
with:
|
|
name: pytest-stores-postgres-${{ github.run_id }}
|
|
path: artifacts/
|
|
retention-days: 14
|
|
|
|
stores-mysql:
|
|
name: Pytest (stores-mysql)
|
|
needs: gate
|
|
if: ${{ !github.event.pull_request.draft }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
services:
|
|
mysql:
|
|
image: mysql:8.0
|
|
env:
|
|
MYSQL_ROOT_PASSWORD: omnigent
|
|
MYSQL_DATABASE: omnigent_root
|
|
ports:
|
|
- 3306:3306
|
|
options: >-
|
|
--health-cmd "mysqladmin ping -h 127.0.0.1 -u root -pomnigent"
|
|
--health-interval 5s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
|
|
with:
|
|
python-version-file: ".python-version"
|
|
- uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a
|
|
with:
|
|
enable-cache: true
|
|
- name: Install system MySQL client library
|
|
run: sudo apt-get update -qq && sudo apt-get install -y -q libmysqlclient-dev
|
|
- name: Install dependencies
|
|
run: uv sync --locked --extra all --extra dev --extra databricks && uv pip install mysqlclient
|
|
- name: Run store + DB tests against MySQL
|
|
env:
|
|
OMNIGENT_TEST_DB_URI: mysql+mysqldb://root:omnigent@127.0.0.1:3306/omnigent_root
|
|
run: |
|
|
uv run pytest tests/stores tests/db \
|
|
-m "not databricks" \
|
|
-n 4 \
|
|
--dist=loadfile \
|
|
--timeout=300 \
|
|
--junitxml=artifacts/pytest-stores-mysql.xml
|
|
- if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
|
|
with:
|
|
name: pytest-stores-mysql-${{ github.run_id }}
|
|
path: artifacts/
|
|
retention-days: 14
|
|
|
|
codex-parity:
|
|
name: Pytest (codex-parity)
|
|
needs: gate
|
|
if: ${{ !github.event.pull_request.draft }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Check out repo
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v3
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Set up Rust toolchain
|
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Capture Rust version
|
|
id: rustc
|
|
run: echo "version=$(rustc --version | tr ' ' '-')" >> "$GITHUB_OUTPUT"
|
|
|
|
# The sidecar source is frozen and its deps are rev-pinned, so the binary is
|
|
# a pure function of sidecar/** + the toolchain. Cache the built binary (not
|
|
# the 1.6 GB target dir) and skip the ~3 min compile below on a hit; the key
|
|
# self-invalidates when the source, Cargo.lock, or rustc changes.
|
|
- name: Cache parity sidecar binary
|
|
id: sidecar-cache
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: .tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
key: codex-parity-bin-${{ runner.os }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('tests/codex_parity/sidecar/**') }}
|
|
|
|
- name: Set up Node
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: 20
|
|
|
|
- name: Install codex CLI
|
|
run: |
|
|
npm install --ignore-scripts --prefix .github/ci-deps
|
|
echo "${GITHUB_WORKSPACE}/.github/ci-deps/node_modules/.bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Cache virtualenv
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ hashFiles('.python-version') }}-${{ hashFiles('uv.lock') }}
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --locked --extra all --extra dev
|
|
|
|
- name: Build parity sidecar
|
|
if: steps.sidecar-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
cargo build \
|
|
--manifest-path tests/codex_parity/sidecar/Cargo.toml \
|
|
--target-dir .tmp-codex-parity-target
|
|
|
|
- name: Run codex parity tests
|
|
shell: bash
|
|
env:
|
|
PYTHONFAULTHANDLER: "1"
|
|
# Reuse the binary from the "Build parity sidecar" step above so the
|
|
# fixture doesn't re-invoke cargo build during collection.
|
|
CODEX_PARITY_SIDECAR_BIN: ${{ github.workspace }}/.tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
run: |
|
|
mkdir -p artifacts
|
|
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY -u DATABRICKS_TOKEN \
|
|
uv run pytest tests/codex_parity/ \
|
|
--codex-parity \
|
|
--timeout=300 \
|
|
--junitxml=artifacts/pytest-codex-parity.xml \
|
|
-v --tb=long --showlocals --log-level=INFO -r a
|
|
|
|
- name: Upload pytest artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: pytest-codex-parity-${{ github.run_id }}
|
|
path: artifacts/
|
|
retention-days: 14
|
|
|
|
coverage-report:
|
|
name: Coverage report
|
|
# Combines per-shard coverage into a coverage-summary artifact. Runs in the
|
|
# unprivileged pull_request context (read-only); code-coverage.yml consumes
|
|
# the artifact and posts the status. Report-only.
|
|
needs: pytest
|
|
if: ${{ !cancelled() && !github.event.pull_request.draft }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Check out repo
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Install coverage
|
|
run: pip install "coverage>=7"
|
|
|
|
- name: Download shard coverage data
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
pattern: pytest-*
|
|
path: covdata
|
|
merge-multiple: true
|
|
|
|
- name: Combine + report
|
|
shell: bash
|
|
run: |
|
|
shopt -s globstar nullglob
|
|
files=(covdata/**/.coverage.*)
|
|
mkdir -p coverage-summary
|
|
if [[ ${#files[@]} -eq 0 ]]; then
|
|
echo "::warning::No coverage data files found; skipping coverage report."
|
|
exit 0
|
|
fi
|
|
echo "Combining ${#files[@]} shard data file(s)."
|
|
coverage combine "${files[@]}"
|
|
# --ignore-errors: a plain checkout lacks uv-sync-generated files.
|
|
{ echo "## Coverage"; echo; coverage report --format=markdown --ignore-errors; } >> "$GITHUB_STEP_SUMMARY"
|
|
coverage xml -o coverage-summary/coverage.xml --ignore-errors
|
|
coverage report --format=total --ignore-errors > coverage-summary/total.txt
|
|
echo "Total coverage: $(cat coverage-summary/total.txt)%"
|
|
|
|
- name: Upload coverage summary
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: coverage-summary-${{ github.run_id }}
|
|
path: coverage-summary/
|
|
retention-days: 14
|
|
if-no-files-found: ignore
|