wehub-resource-sync
227 lines
10 KiB
YAML
227 lines
10 KiB
YAML
name: "Run e2e suite"
|
|
description: >
|
|
Run the tests/e2e suite exactly as the e2e.yml gate does (mock LLM,
|
|
sharded). When `server_version` is set, the omnigent SERVER subprocess is
|
|
pinned to that released tag (built into an isolated venv) while the client,
|
|
runner, and tests stay on the checked-out ref — the server-version
|
|
backwards-compat configuration. Shared verbatim by e2e.yml (normal gate) and
|
|
server-compat.yml (backcompat jobs) so the two never drift. The caller is
|
|
responsible for the preceding `actions/checkout` (the checkout ref differs:
|
|
the gate tests refs/pull/N/merge; backcompat needs fetch-depth 0 for tags).
|
|
|
|
inputs:
|
|
shard_id:
|
|
description: "pytest-shard shard index"
|
|
required: true
|
|
num_shards:
|
|
description: "pytest-shard shard count"
|
|
required: true
|
|
parallelism:
|
|
description: "pytest workers (-n)"
|
|
required: false
|
|
default: "2"
|
|
nightly_full:
|
|
description: "true = full pass (schedule/dispatch); false = exclude @nightly"
|
|
required: false
|
|
default: "false"
|
|
server_version:
|
|
description: >
|
|
Empty = run the checked-out server (normal gate). Set to a release tag
|
|
(e.g. v0.1.1) = build that old server into a venv and redirect the
|
|
server subprocess to it (backwards-compat run).
|
|
required: false
|
|
default: ""
|
|
runner_version:
|
|
description: >
|
|
Empty = run the checked-out runner/host (normal gate). Set to a release
|
|
tag = build that old runner+host into a venv and redirect the runner and
|
|
host-daemon subprocesses to it (Config 2 backwards-compat run). Orthogonal
|
|
to server_version.
|
|
required: false
|
|
default: ""
|
|
artifact_suffix:
|
|
description: >
|
|
Appended to uploaded-artifact names so they stay unique across matrix
|
|
cells (e.g. "-sv0.2.0-rmain"). Default empty — the normal gate has one
|
|
cell per shard, so its names are already unique.
|
|
required: false
|
|
default: ""
|
|
|
|
runs:
|
|
using: composite
|
|
steps:
|
|
- name: Configure environment
|
|
shell: bash
|
|
run: |
|
|
# Self-contained so the action behaves identically regardless of the
|
|
# caller's env. No web SPA build during installs (this job never
|
|
# serves the bundle); blank provider keys so a spawned server can't
|
|
# pick up the runner's own credentials.
|
|
{
|
|
echo "OMNIGENT_SKIP_WEB_UI=true"
|
|
echo "ANTHROPIC_API_KEY="
|
|
echo "OPENAI_API_KEY="
|
|
echo "CODEX="
|
|
echo "CLAUDE_CODE="
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Cache virtualenv
|
|
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ hashFiles('.python-version') }}-${{ hashFiles('uv.lock') }}
|
|
|
|
- name: Install project and dev dependencies
|
|
shell: bash
|
|
run: uv sync --locked --extra all --extra dev
|
|
|
|
- name: Install binary dependencies
|
|
# npm install against .github/ci-deps/package.json with --ignore-scripts
|
|
# to block postinstall on every package. The claude-code stub binary
|
|
# needs its install.cjs (audited: platform detect + same-tree hardlink,
|
|
# no network/exec) so we run that one explicitly; codex and pi have no
|
|
# install scripts and ship prebuilt CLIs. bubblewrap: the linux_bwrap
|
|
# sandbox backend fails loud if `bwrap` is missing, and the e2e runner
|
|
# runs real agents with os_env. The apparmor sysctl mirrors ci.yml
|
|
# (Ubuntu 24.04 blocks unprivileged user namespaces, which bwrap's
|
|
# unshare(CLONE_NEWUSER) needs).
|
|
working-directory: .github/ci-deps
|
|
shell: bash
|
|
run: |
|
|
sudo apt-get install -y tmux bubblewrap
|
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
|
npm install --ignore-scripts
|
|
node node_modules/@anthropic-ai/claude-code/install.cjs
|
|
echo "${GITHUB_WORKSPACE}/.github/ci-deps/node_modules/.bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Build pinned old server (backwards-compat only)
|
|
# Only runs when server_version is set. Builds the released tag into an
|
|
# isolated venv (all three packages editable so the old ==<old> SDK
|
|
# cross-pins resolve without an index) and points the server subprocess
|
|
# at it via OMNIGENT_COMPAT_SERVER_PYTHON. The redirect also drops the
|
|
# worktree PYTHONPATH/CWD shadow (see tests/_helpers/compat.py) so the
|
|
# pinned install actually resolves. Requires fetch-depth 0 in the caller.
|
|
if: ${{ inputs.server_version != '' }}
|
|
shell: bash
|
|
env:
|
|
SERVER_VERSION_INPUT: ${{ inputs.server_version }}
|
|
run: |
|
|
tag="$SERVER_VERSION_INPUT"
|
|
if ! [[ "$tag" =~ ^v?[0-9]+\.[0-9]+(\.[0-9]+)?([a-z0-9.]*)?$ ]]; then
|
|
echo "Invalid server_version: '$tag'" >&2; exit 1
|
|
fi
|
|
src="$RUNNER_TEMP/server-src"
|
|
venv="$RUNNER_TEMP/server-env"
|
|
git worktree add --detach "$src" "$tag"
|
|
uv venv --python 3.12 "$venv"
|
|
uv pip install --python "$venv/bin/python" \
|
|
-e "$src" -e "$src/sdks/python-client" -e "$src/sdks/ui"
|
|
"$venv/bin/omnigent" --version
|
|
echo "OMNIGENT_COMPAT_SERVER_PYTHON=$venv/bin/python" >> "$GITHUB_ENV"
|
|
echo "OMNIGENT_COMPAT_SERVER_VERSION=${tag#v}" >> "$GITHUB_ENV"
|
|
|
|
- name: Build pinned old runner/host (backwards-compat only)
|
|
# Only runs when runner_version is set (Config 2). Builds the released tag
|
|
# into an isolated venv and points the runner + host-daemon subprocesses
|
|
# at it via OMNIGENT_COMPAT_RUNNER_PYTHON (apply_runner_env drops the
|
|
# worktree PYTHONPATH/CWD shadow). Distinct paths from the server build so
|
|
# both can coexist. Requires fetch-depth 0 in the caller.
|
|
if: ${{ inputs.runner_version != '' }}
|
|
shell: bash
|
|
env:
|
|
RUNNER_VERSION_INPUT: ${{ inputs.runner_version }}
|
|
run: |
|
|
tag="$RUNNER_VERSION_INPUT"
|
|
if ! [[ "$tag" =~ ^v?[0-9]+\.[0-9]+(\.[0-9]+)?([a-z0-9.]*)?$ ]]; then
|
|
echo "Invalid runner_version: '$tag'" >&2; exit 1
|
|
fi
|
|
src="$RUNNER_TEMP/runner-src"
|
|
venv="$RUNNER_TEMP/runner-env"
|
|
git worktree add --detach "$src" "$tag"
|
|
uv venv --python 3.12 "$venv"
|
|
uv pip install --python "$venv/bin/python" \
|
|
-e "$src" -e "$src/sdks/python-client" -e "$src/sdks/ui"
|
|
"$venv/bin/omnigent" --version
|
|
echo "OMNIGENT_COMPAT_RUNNER_PYTHON=$venv/bin/python" >> "$GITHUB_ENV"
|
|
echo "OMNIGENT_COMPAT_RUNNER_VERSION=${tag#v}" >> "$GITHUB_ENV"
|
|
|
|
- name: Run e2e tests
|
|
shell: bash
|
|
env:
|
|
PARALLELISM_INPUT: ${{ inputs.parallelism }}
|
|
SHARD_ID: ${{ inputs.shard_id }}
|
|
NUM_SHARDS: ${{ inputs.num_shards }}
|
|
NIGHTLY_FULL: ${{ inputs.nightly_full }}
|
|
E2E_TMP_BASE: /tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}
|
|
PYTEST_PROGRESS_LOG_DIR: /tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/progress
|
|
OMNIGENT_TOKEN_USAGE_JSON: /tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/tokens.json
|
|
run: |
|
|
# Validate parallelism (untrusted input -- bind to env, never
|
|
# interpolate a GitHub expression into the shell).
|
|
if ! [[ "$PARALLELISM_INPUT" =~ ^[1-9][0-9]?$ ]]; then
|
|
echo "Invalid parallelism input: $PARALLELISM_INPUT (expected 1-99)" >&2
|
|
exit 1
|
|
fi
|
|
WORKERS="$PARALLELISM_INPUT"
|
|
mkdir -p "$E2E_TMP_BASE"
|
|
|
|
EXTRA_ARGS=()
|
|
if [[ "$NIGHTLY_FULL" != "true" ]]; then
|
|
EXTRA_ARGS+=(-m "not nightly")
|
|
fi
|
|
|
|
# --junitxml emits per-test results eagerly so diagnostics survive a
|
|
# wall-clock overrun. --shard-id/--num-shards chunk the node IDs.
|
|
# --timeout=180 caps each test; --timeout-method=thread because our
|
|
# pty/subprocess children don't get SIGALRM. --max-worker-restart=0
|
|
# fails the shard fast instead of letting loadscope requeue deadlock
|
|
# the controller (the 2026-06-11 shard-2 wedge).
|
|
uv run pytest tests/e2e/ \
|
|
-n "$WORKERS" \
|
|
--dist=loadscope \
|
|
--max-worker-restart=0 \
|
|
--shard-id="$SHARD_ID" \
|
|
--num-shards="$NUM_SHARDS" \
|
|
--timeout=180 \
|
|
--timeout-method=thread \
|
|
--basetemp="$E2E_TMP_BASE" \
|
|
--junitxml="$E2E_TMP_BASE/junit.xml" \
|
|
-v --tb=long --showlocals --log-level=INFO -r a \
|
|
"${EXTRA_ARGS[@]}" \
|
|
|| { rc=$?; [ "$rc" -eq 5 ] && echo "::notice::No tests collected in this shard; treating as a pass." || exit "$rc"; }
|
|
|
|
- name: Upload server logs on failure
|
|
# cancelled() too: failure() misses step timeouts (#426).
|
|
if: ${{ failure() || cancelled() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: e2e-server-logs-${{ github.run_id }}-shard${{ inputs.shard_id }}${{ inputs.artifact_suffix }}
|
|
path: |
|
|
/tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/**/server.log
|
|
/tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/**/runner.log
|
|
/tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/**/.omnigent/logs/**/*.log
|
|
/tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/junit.xml
|
|
/tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/progress/progress-*.log
|
|
retention-days: 7
|
|
if-no-files-found: warn
|
|
include-hidden-files: true
|
|
|
|
- name: Upload token usage
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
|
with:
|
|
name: e2e-tokens-${{ github.run_id }}-shard${{ inputs.shard_id }}${{ inputs.artifact_suffix }}
|
|
path: /tmp/omnigent-e2e-${{ github.run_id }}-shard${{ inputs.shard_id }}/tokens*.json
|
|
retention-days: 14
|
|
if-no-files-found: warn
|