wehub-resource-sync
386 lines
18 KiB
YAML
386 lines
18 KiB
YAML
name: E2E UI Tests
|
|
|
|
# Runs the Playwright UI suite against a freshly built web SPA, split across
|
|
# a 3-shard matrix. The whole suite (including the native Claude/Codex/Cursor
|
|
# render-parity tests) runs against the in-process mock LLM and needs NO
|
|
# secrets, so it runs on ALL PRs -- same-repo AND fork -- directly, like ci.yml.
|
|
# (The native_*_mock_session fixtures use the real gateway only when LLM_API_KEY
|
|
# is set, e.g. local dev; CI never sets it.)
|
|
#
|
|
# Triggers:
|
|
# pull_request ALL PRs (same-repo + fork). Draft PRs skip.
|
|
# schedule 09:00 UTC daily, alongside nightly.yml.
|
|
# workflow_dispatch manual run. Input `branch` selects a non-main ref.
|
|
|
|
on:
|
|
pull_request:
|
|
# No labeled/unlabeled: a skip-security-scan waiver re-runs this workflow's
|
|
# Security Gate via rerun-security-gate.yml, so label churn need not re-run
|
|
# the heavy Playwright suite. (#399 added these for the gate; superseded.)
|
|
types: [opened, synchronize, reopened, ready_for_review]
|
|
schedule:
|
|
- cron: "0 9 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
branch:
|
|
description: "Branch to run UI tests against"
|
|
required: false
|
|
default: "main"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
# PRs key by number, dispatch by branch (so re-runs cancel); push /
|
|
# schedule key by SHA so each merge to `main` gets its own run.
|
|
group: e2e-ui-${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.branch || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# No SPA build during `uv sync`: this workflow builds the bundle in a
|
|
# dedicated step, so the setup.py build would be a redundant npm hit.
|
|
OMNIGENT_SKIP_WEB_UI: "true"
|
|
# Scrub harness credentials the test server must not pick up.
|
|
# OPENAI_API_KEY / OPENAI_BASE_URL are NOT scrubbed here — the
|
|
# conftest's live_server fixture overrides them to mock values
|
|
# (OPENAI_BASE_URL=<mock>/v1, OPENAI_API_KEY=mock-key) inside the
|
|
# spawned server subprocess, so ambient real credentials are a no-op.
|
|
ANTHROPIC_API_KEY: ""
|
|
DATABRICKS_TOKEN: ""
|
|
CODEX: ""
|
|
CLAUDE_CODE: ""
|
|
# Match e2e.yml's proxy choice.
|
|
UV_INDEX_URL: https://pypi.org/simple
|
|
# Runners default to TERM=dumb, which breaks the PTY shell's "clear".
|
|
# A real terminfo lets the spawned PTY resolve clear/cursor sequences;
|
|
# inherited by the agent server via the conftest's env plumbing.
|
|
TERM: xterm-256color
|
|
|
|
jobs:
|
|
# Security gate: untrusted PRs wait on the deterministic scan
|
|
# (security-gate.yml); trusted authors and non-PR events pass instantly.
|
|
gate:
|
|
uses: ./.github/workflows/security-gate.yml
|
|
|
|
# Compute the shard matrix once. A skipped run (draft / fork pull_request)
|
|
# yields an EMPTY matrix -> zero shard jobs -> no skipped placeholder check.
|
|
# Shared with e2e.yml via e2e-shard-matrix.sh (only NUM_SHARDS differs).
|
|
setup:
|
|
name: setup
|
|
needs: gate
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
outputs:
|
|
matrix: ${{ steps.matrix.outputs.matrix }}
|
|
steps:
|
|
- name: Check out CI scripts
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
with:
|
|
# Triggering ref (not main): the script must exist on it, and it
|
|
# only shards tests -- no secrets exposure, so the PR's copy is fine.
|
|
sparse-checkout: .github/scripts/ci
|
|
persist-credentials: false
|
|
- name: Compute shard matrix
|
|
id: matrix
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
IS_DRAFT: ${{ github.event.pull_request.draft }}
|
|
NUM_SHARDS: "3"
|
|
run: bash .github/scripts/ci/e2e-shard-matrix.sh
|
|
|
|
# Build the Codex-parity sidecar ONCE and publish the binary. The
|
|
# mocked_native_codex_goal_session fixture needs it, but compiling it pulls
|
|
# openai/codex's core_test_support (~1100 crates). Done lazily inside pytest
|
|
# it lands ~4min (warm) to ~7min (cold) on whichever single shard collects
|
|
# test_codex_goal_mode, lopsiding that shard against the 20min cap. Building
|
|
# here once and handing every shard the ~10MB binary (via the artifact +
|
|
# CODEX_PARITY_SIDECAR_BIN below) keeps the sidecar cost off the shard
|
|
# critical path entirely. Skips on draft PRs (empty matrix -> no shards).
|
|
build-sidecar:
|
|
name: build codex-parity sidecar
|
|
needs: setup
|
|
if: needs.setup.outputs.matrix != '{"include":[]}'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
with:
|
|
ref: ${{ github.event.inputs.branch || github.ref }}
|
|
- name: Set up Rust toolchain
|
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
|
with:
|
|
toolchain: stable
|
|
- name: Capture Rust version
|
|
id: rustc
|
|
run: echo "version=$(rustc --version | tr ' ' '-')" >> "$GITHUB_OUTPUT"
|
|
# The sidecar source is frozen and its deps are rev-pinned, so the binary
|
|
# is a pure function of sidecar/** + the toolchain. Cache the built binary
|
|
# (not the 1.6 GB target dir) and skip the ~7 min compile below on a hit;
|
|
# the key self-invalidates when the source, Cargo.lock, or rustc changes.
|
|
# Same key as ci.yml's codex-parity job -- ci.yml runs on push to main and
|
|
# populates the main-scoped cache that this PR-only workflow restores from.
|
|
- name: Cache parity sidecar binary
|
|
id: sidecar-cache
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: .tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
key: codex-parity-bin-${{ runner.os }}-${{ steps.rustc.outputs.version }}-${{ hashFiles('tests/codex_parity/sidecar/**') }}
|
|
- name: Build parity sidecar
|
|
if: steps.sidecar-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
cargo build \
|
|
--manifest-path tests/codex_parity/sidecar/Cargo.toml \
|
|
--target-dir .tmp-codex-parity-target
|
|
- name: Upload sidecar binary
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: codex-parity-sidecar
|
|
path: .tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
e2e-ui:
|
|
name: E2E UI Tests (shard ${{ matrix.shard_id }}/${{ matrix.num_shards }})
|
|
# Draft PRs resolve to an EMPTY matrix in `setup`, so no shard runs for
|
|
# them. Fork PRs DO run (mock LLM, no secrets) -- same as ci.yml.
|
|
# `ready_for_review` re-fires when a draft is converted.
|
|
needs: [setup, build-sidecar]
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 20
|
|
strategy:
|
|
# One red shard shouldn't cancel siblings -- we want every shard's signal.
|
|
fail-fast: false
|
|
max-parallel: 3
|
|
# Shards from `setup`; [] when skipped. Shard check names live in
|
|
# merge-ready/required.sh -- keep in sync with NUM_SHARDS above.
|
|
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
with:
|
|
ref: ${{ github.event.inputs.branch || github.ref }}
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
|
with:
|
|
python-version-file: ".python-version"
|
|
|
|
- name: Set up Node 20
|
|
uses: ./.github/actions/setup-node
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Cache virtualenv
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v4
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ hashFiles('.python-version') }}-${{ hashFiles('uv.lock') }}
|
|
- name: Install project + dev extras
|
|
run: uv sync --locked --extra all --extra dev
|
|
|
|
- name: Install bubblewrap + tmux
|
|
# bubblewrap: the UI tests open terminals under os_env, whose
|
|
# linux_bwrap backend fails loud if `bwrap` is missing. The apparmor
|
|
# sysctl mirrors ci.yml (Ubuntu 24.04 blocks unprivileged user
|
|
# namespaces, which bwrap's unshare(CLONE_NEWUSER) needs).
|
|
# tmux: the claude-native render-parity test drives Claude Code
|
|
# through a tmux pane, so `tmux` must be on PATH.
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y bubblewrap tmux
|
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
|
|
|
# Fetch the prebuilt Codex-parity sidecar from the build-sidecar job
|
|
# instead of compiling it here: no per-shard Rust toolchain or cargo
|
|
# build. The mocked_native_codex_goal_session fixture uses this binary
|
|
# via CODEX_PARITY_SIDECAR_BIN (set on the pytest step below).
|
|
- name: Download codex-parity sidecar binary
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
name: codex-parity-sidecar
|
|
path: .tmp-codex-parity-target/debug
|
|
- name: Make sidecar binary executable
|
|
# upload-artifact does not preserve the +x bit; restore it so the
|
|
# fixture can exec the binary.
|
|
run: chmod +x .tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: ${{ runner.os }}-playwright-${{ hashFiles('uv.lock') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-playwright-
|
|
|
|
- name: Install Playwright Chromium
|
|
run: |
|
|
uv run playwright install --with-deps chromium
|
|
|
|
- name: Build web SPA
|
|
# Build BEFORE pytest: Vite's emptyOutDir clobbers the static dir,
|
|
# so never run it under xdist or alongside the live server.
|
|
# --legacy-peer-deps avoids re-resolving the known React 19 peer
|
|
# conflict under @emoji-mart/react.
|
|
env:
|
|
NPM_CONFIG_REGISTRY: https://registry.npmjs.org/
|
|
run: |
|
|
cd web
|
|
npm ci --legacy-peer-deps --no-audit --no-fund
|
|
npm run build
|
|
|
|
# Native coding-agent harness enablement: the next steps let the
|
|
# native render-parity tests boot a real Claude Code / Codex CLI. The
|
|
# rest of the e2e-ui suite (openai-agents) ignores them.
|
|
- name: Install Claude Code CLI
|
|
# claude-code 2.1.170, NOT the 2.1.124 in .github/ci-deps: 2.1.124
|
|
# doesn't recognise the hook events the native bridge configures and
|
|
# shows a blocking startup modal that swallows the first message.
|
|
# --ignore-scripts then run install.cjs explicitly (audited: platform
|
|
# detect + same-tree hardlink, no network/exec) and put its bin on PATH.
|
|
env:
|
|
NPM_CONFIG_REGISTRY: https://registry.npmjs.org/
|
|
run: |
|
|
mkdir -p "${GITHUB_WORKSPACE}/.cc-cli" && cd "${GITHUB_WORKSPACE}/.cc-cli"
|
|
npm install --ignore-scripts --no-audit --no-fund @anthropic-ai/claude-code@2.1.170
|
|
node node_modules/@anthropic-ai/claude-code/install.cjs
|
|
echo "${GITHUB_WORKSPACE}/.cc-cli/node_modules/.bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Install Codex CLI
|
|
# @openai/codex at the .github/ci-deps pin (same build as e2e.yml's
|
|
# codex leg). `scripts: null` means no postinstall, so --ignore-scripts
|
|
# is a safety no-op; the native binary ships in the package and goes on
|
|
# PATH for the codex render-parity test's tmux pane.
|
|
env:
|
|
NPM_CONFIG_REGISTRY: https://registry.npmjs.org/
|
|
run: |
|
|
mkdir -p "${GITHUB_WORKSPACE}/.codex-cli" && cd "${GITHUB_WORKSPACE}/.codex-cli"
|
|
npm install --ignore-scripts --no-audit --no-fund @openai/codex@0.139.0
|
|
echo "${GITHUB_WORKSPACE}/.codex-cli/node_modules/.bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Run UI e2e tests
|
|
# --ui-skip-build: the SPA was built in the previous step.
|
|
# --tracing/--screenshot/--video default to off; retain-on-failure
|
|
# keeps green runs cheap while capturing artifacts on failures.
|
|
# The conftest's live_server fixture injects OPENAI_BASE_URL=mock/v1
|
|
# and OPENAI_API_KEY=mock-key into the runner subprocess env, so the
|
|
# openai-agents harness and policy classifier both hit the mock — no
|
|
# real credentials needed. Native render-parity tests write their own
|
|
# mock provider config via native_*_mock_session at terminal-creation
|
|
# time, so no ~/.omnigent/config.yaml is written in CI either.
|
|
env:
|
|
# Scheduled / manually dispatched runs are the full pass;
|
|
# PR and push runs exclude @pytest.mark.nightly tests.
|
|
NIGHTLY_FULL: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
|
|
SHARD_ID: ${{ matrix.shard_id }}
|
|
NUM_SHARDS: ${{ matrix.num_shards }}
|
|
# Prebuilt sidecar from build-sidecar; the codex goal-mode fixture
|
|
# uses this instead of running cargo build. Absolute path: the
|
|
# fixture runs with cwd at the repo root but be explicit.
|
|
CODEX_PARITY_SIDECAR_BIN: ${{ github.workspace }}/.tmp-codex-parity-target/debug/codex-parity-sidecar
|
|
run: |
|
|
# Always exclude @visual: the UI diff snapshot runs in its own
|
|
# pinned-runner gate (ui-snapshot.yml) so its baseline matches the
|
|
# comparison environment; on this unpinned ubuntu-latest it would
|
|
# flake on font drift. Add the nightly exclusion for PR/push runs.
|
|
MARKER="not visual"
|
|
if [[ "$NIGHTLY_FULL" != "true" ]]; then
|
|
MARKER="$MARKER and not nightly"
|
|
fi
|
|
# --splits/--group partition the suite via a strided slice (see
|
|
# pytest_collection_modifyitems in tests/e2e_ui/conftest.py), which
|
|
# evens out wall-clock better than pytest-shard's hash-bucketing.
|
|
# --group is 1-indexed, so map the 0-indexed shard_id with +1.
|
|
uv run pytest tests/e2e_ui \
|
|
-v --tb=long --showlocals --log-level=INFO -r a \
|
|
--ui-skip-build \
|
|
--splits="$NUM_SHARDS" \
|
|
--group="$((SHARD_ID + 1))" \
|
|
--tracing=retain-on-failure \
|
|
--screenshot=only-on-failure \
|
|
--video=retain-on-failure \
|
|
-m "$MARKER" \
|
|
|| { rc=$?; [ "$rc" -eq 5 ] && echo "::notice::No tests collected in this shard; treating as a pass." || exit "$rc"; }
|
|
|
|
- name: Upload Playwright traces / videos / screenshots on failure
|
|
id: upload_playwright
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
# Shard suffix avoids the matrix's parallel uploads colliding (v4
|
|
# 409s on dupe names).
|
|
name: e2e-ui-playwright-${{ github.run_id }}-shard${{ matrix.shard_id }}
|
|
# `playwright-report/` is the JS-runner's HTML dir, never produced
|
|
# by pytest-playwright -- kept for forward-compat (ignore-if-absent).
|
|
path: |
|
|
test-results/
|
|
playwright-report/
|
|
retention-days: 3
|
|
if-no-files-found: ignore
|
|
|
|
- name: Dump Claude transcript on failure
|
|
# Claude Code's transcript JSONL lives under ~/.claude/projects (a
|
|
# hidden dir the artifact glob misses); stage it under /tmp. NOT
|
|
# copying ~/.claude.json: its apiKeyHelper embeds the gateway token.
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/claude-home-dump
|
|
cp -r "$HOME/.claude/projects" /tmp/claude-home-dump/ 2>/dev/null || true
|
|
|
|
- name: Dump Codex transcript on failure
|
|
# Codex's per-session rollout JSONLs live under the bridged CODEX_HOME
|
|
# at ~/.omnigent/codex-native/<hash>/codex-home/sessions; stage only
|
|
# the *.jsonl. NOT copying config.toml: its auth command embeds the token.
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/codex-home-dump
|
|
find "$HOME/.omnigent/codex-native" -name '*.jsonl' -print0 2>/dev/null \
|
|
| xargs -0 -I{} cp --parents {} /tmp/codex-home-dump/ 2>/dev/null || true
|
|
|
|
- name: Upload server logs on failure
|
|
id: upload_server_logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: e2e-ui-server-logs-${{ github.run_id }}-shard${{ matrix.shard_id }}
|
|
# server.log + runner.log from the live_server fixture's tmp dir,
|
|
# plus the native bridge dirs and the Claude / Codex transcripts
|
|
# staged above -- all needed to triage a native render-parity failure.
|
|
path: |
|
|
/tmp/pytest-of-runner/**/e2e_ui_server*/server.log
|
|
/tmp/pytest-of-runner/**/e2e_ui_server*/runner.log
|
|
/tmp/omnigent-*/claude-native/**
|
|
/tmp/claude-home-dump/**
|
|
/tmp/codex-home-dump/**
|
|
retention-days: 3
|
|
if-no-files-found: ignore
|
|
|
|
- name: Surface failure artifacts on job summary
|
|
# Write a flat, always-visible block of artifact download links to
|
|
# GITHUB_STEP_SUMMARY (GH otherwise buries them inside each step).
|
|
if: failure()
|
|
env:
|
|
PLAYWRIGHT_URL: ${{ steps.upload_playwright.outputs.artifact-url }}
|
|
SERVER_LOGS_URL: ${{ steps.upload_server_logs.outputs.artifact-url }}
|
|
run: |
|
|
{
|
|
echo "## Failure artifacts"
|
|
echo
|
|
echo "Direct downloads (retention 3 days):"
|
|
echo
|
|
if [ -n "${PLAYWRIGHT_URL}" ]; then
|
|
echo "- 🎬 [Playwright trace / video / screenshots](${PLAYWRIGHT_URL})"
|
|
else
|
|
echo "- 🎬 Playwright trace: _no artifact uploaded (test-results/ was empty)_"
|
|
fi
|
|
if [ -n "${SERVER_LOGS_URL}" ]; then
|
|
echo "- 📜 [omnigent server log](${SERVER_LOGS_URL})"
|
|
else
|
|
echo "- 📜 server.log: _no artifact uploaded (glob matched nothing)_"
|
|
fi
|
|
} >> "$GITHUB_STEP_SUMMARY"
|