wehub-resource-sync
556 lines
30 KiB
YAML
556 lines
30 KiB
YAML
name: "test / e2e / showcase / on-demand"
|
|
|
|
# SECURITY — residual trust model (read before editing):
|
|
#
|
|
# This workflow EXISTS to execute PR-HEAD code (Playwright tests, Next.js dev
|
|
# server, Python agent, pip install of PR-controlled requirements.txt). Several
|
|
# hardening layers reduce blast radius:
|
|
# - `author_association` gate limits the `issue_comment` trigger to OWNER /
|
|
# MEMBER / COLLABORATOR (third-party commenters cannot spawn runs).
|
|
# - workflow-level `permissions: contents: read` means the heavy test job's
|
|
# GITHUB_TOKEN cannot mutate the repo; the `post-result` job gets write
|
|
# perms scoped to just the final PR comment.
|
|
# - `persist-credentials: false` on `actions/checkout` prevents the token
|
|
# from being left behind in `.git/config` where PR-HEAD build hooks might
|
|
# read it.
|
|
# - `pnpm install --ignore-scripts` / `npm install --ignore-scripts` block
|
|
# install-time hooks in PR-controlled JS manifests from executing on the
|
|
# runner. The Python install uses `pip install --prefer-binary` (prefers
|
|
# wheels, falls back to sdist on transitive deps that lack a wheel for
|
|
# linux-x86_64/py3.12). We used to use `--only-binary :all:` for a hard
|
|
# block against source-build hooks, but CrewAI's transitive graph
|
|
# (tiktoken / chromadb / litellm cadence releases) regularly ships a
|
|
# sdist-only revision that makes every CI run fail-loud with "Could not
|
|
# find a version that satisfies the requirement". `--prefer-binary` trades
|
|
# that hard guarantee for reliability — the `author_association` gate
|
|
# above still limits WHO can trigger this workflow, so the residual risk
|
|
# is bounded to a trusted commenter. See also the "Start Python agent"
|
|
# step for the in-context trade-off rationale.
|
|
# - A strict slug whitelist (`^[a-z0-9-]+$` + existing-dir check) and the
|
|
# `env:`-based pattern for UNTRUSTED values (comment body, dispatch slug)
|
|
# prevent shell injection / path traversal.
|
|
#
|
|
# What this is NOT: a security boundary against a malicious trusted commenter.
|
|
# The last line of defense is the SOCIAL CONTRACT that a trusted commenter
|
|
# reviews the PR diff BEFORE typing `/test-aimock` — if a compromised / rogue
|
|
# OWNER/MEMBER/COLLABORATOR comments on an attacker's PR, they get a full
|
|
# runner exec with the job's token. That is an accepted residual risk for the
|
|
# developer-velocity benefit of PR-triggered E2E runs. Do not loosen the
|
|
# `author_association` gate without revisiting the threat model above.
|
|
#
|
|
# Known TOCTOU — comment-trigger vs resolved HEAD SHA:
|
|
# "Resolve PR HEAD ref" below calls `pulls.get` at job start. There is a
|
|
# window between the trusted commenter typing `/test-aimock` (reviewed diff
|
|
# D1) and the workflow actually calling `pulls.get` (resolves whatever HEAD
|
|
# is current — possibly D2 after a force-push). A PR author who force-pushes
|
|
# malicious content AFTER the trusted comment but BEFORE the resolve call
|
|
# gets their code executed. GitHub Actions does NOT natively support pinning
|
|
# the SHA at comment time (no `comment.commit_sha` equivalent), so this gap
|
|
# is architectural. The `author_association` gate + code-review social
|
|
# contract are the mitigations; the residual TOCTOU risk is accepted. If
|
|
# GitHub ever ships a comment-time SHA field, pin to it and drop this note.
|
|
|
|
on:
|
|
issue_comment:
|
|
types: [created]
|
|
workflow_dispatch:
|
|
inputs:
|
|
slug:
|
|
description: "Package slug to test (Python integration with aimock support)"
|
|
required: true
|
|
# Only Python integrations that exercise the AIMOCK_URL path
|
|
# end-to-end belong here. Restricting the enum prevents accidental
|
|
# dispatch of a TS-only (mastra) or Java (spring-ai) slug that would
|
|
# skip the Python agent startup step and then fail with a misleading
|
|
# Playwright timeout. When a new Python slug is validated, append it.
|
|
#
|
|
# No `default:` is set — the operator must pick a slug explicitly. A
|
|
# hidden default would silently bind manual dispatches to whichever
|
|
# slug happens to be first in the enum, which contradicts the
|
|
# "no silent fallback" guarantee the comment-path extractor enforces.
|
|
type: choice
|
|
options:
|
|
- crewai-crews
|
|
- langgraph-python
|
|
|
|
# Default to read-only at the job level. The only step that needs write access
|
|
# is "Post result to PR" at the end — we grant it write perms inline there.
|
|
# Keeping the workflow-level perms read-only means every intermediate step
|
|
# (including `pip install` on attacker-controlled requirements.txt) runs with
|
|
# a token that cannot mutate the repo.
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
aimock-e2e:
|
|
# Only run on PR comments matching `/test-aimock ` (trailing space REQUIRED)
|
|
# from trusted authors, or manual dispatch. The trailing space tightens
|
|
# the match so unrelated text like `/test-aimocker` or `don't /test-aimock-like-this`
|
|
# does NOT trigger the workflow. The author_association gate additionally
|
|
# prevents arbitrary third-party commenters from triggering runs with
|
|
# attacker-controlled comment bodies (which the 'Determine slug' step then
|
|
# parses — see env-based shell interpolation below). A bare `/test-aimock`
|
|
# alone (no trailing space) is rejected by design; commenters must pick a
|
|
# slug explicitly — no silent fallback to crewai-crews (see "Determine slug"
|
|
# step below).
|
|
# `startsWith` (not `contains`) is the Actions-level gate: it requires
|
|
# `/test-aimock ` to be the FIRST token of the comment, so embedded mentions
|
|
# (in code blocks, quoted replies, or mid-sentence prose) cannot spin up a
|
|
# runner. The shell extractor in the "Determine slug" step uses the same
|
|
# leading anchor (`^/test-aimock[[:space:]]+…`) as defense-in-depth; both
|
|
# layers agree on "first token only" so a future edit that loosens either
|
|
# layer alone cannot bypass validation. Commenters who
|
|
# want to add narration around the command should put the command on its
|
|
# own line at the top of the comment.
|
|
if: >
|
|
github.event_name == 'workflow_dispatch' ||
|
|
(github.event.issue.pull_request
|
|
&& startsWith(github.event.comment.body, '/test-aimock ')
|
|
&& contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association))
|
|
# Pinned to ubuntu-latest deliberately: the 'Determine slug' step uses
|
|
# POSIX-only `grep -oE` + `sed` (no `grep -oP` / PCRE) so a future BSD
|
|
# grep would still work, but ubuntu-latest keeps the install/setup matrix
|
|
# consistent with every other showcase workflow.
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
# For issue_comment events, we need to resolve the PR HEAD SHA ourselves
|
|
# because the event payload doesn't include pull_request.head.sha
|
|
- name: Resolve PR HEAD ref
|
|
id: pr-ref
|
|
if: github.event_name == 'issue_comment'
|
|
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
|
with:
|
|
script: |
|
|
const { data: pr } = await github.rest.pulls.get({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
pull_number: context.issue.number,
|
|
});
|
|
// Refuse to run against a closed / merged PR. A trusted commenter
|
|
// typing `/test-aimock` on a stale closed PR would otherwise
|
|
// re-exec the old HEAD — either wasting CI or (if the PR was
|
|
// closed BECAUSE it was bad) re-running known-bad code. Fail loud.
|
|
if (pr.state !== 'open') {
|
|
core.setFailed(`PR #${pr.number} is ${pr.state} (not open). Refusing to run E2E on a non-open PR.`);
|
|
return;
|
|
}
|
|
core.setOutput('ref', pr.head.sha);
|
|
core.setOutput('pr_number', pr.number);
|
|
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
ref: ${{ steps.pr-ref.outputs.ref || github.sha }}
|
|
# Do NOT leave the workflow's GITHUB_TOKEN in `.git/config` after
|
|
# checkout. PR-HEAD code (pip build hooks, Next.js dev scripts,
|
|
# Playwright fixtures) runs on this runner; a credential left in the
|
|
# working tree could be read by that code and exfiltrated. The job's
|
|
# `permissions: contents: read` limits blast radius, but defense-in-
|
|
# depth cheap — disable credential persistence.
|
|
persist-credentials: false
|
|
|
|
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: 22.x
|
|
|
|
# Omit `version:` so pnpm/action-setup inherits from the repo's
|
|
# `packageManager` field in package.json (via corepack).
|
|
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
|
|
|
- name: Determine slug
|
|
id: slug
|
|
# SECURITY: comment body and dispatch slug are UNTRUSTED. Pass via env
|
|
# (NOT via `${{ ... }}` expression interpolation) so shell never parses
|
|
# attacker-controlled text. Then validate against a strict whitelist
|
|
# before anything downstream uses $SLUG as a path / package name — so
|
|
# `../../../etc/shadow` or similar cannot reach `cd`/`pip install`.
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
COMMENT_BODY: ${{ github.event.comment.body }}
|
|
DISPATCH_SLUG: ${{ github.event.inputs.slug }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
|
SLUG="$DISPATCH_SLUG"
|
|
else
|
|
# POSIX-safe extraction (no `grep -oP` / PCRE `\K`): match
|
|
# `/test-aimock` ONLY at the start of the comment body, followed
|
|
# by whitespace + a slug. The leading anchor (^) matches exactly
|
|
# what the job-level `if:` gate enforces via
|
|
# `startsWith(github.event.comment.body, '/test-aimock ')` — both
|
|
# layers agree that the command must be the FIRST token of the
|
|
# body, so an edit that loosens either layer cannot accidentally
|
|
# desynchronize from the other. This blocks
|
|
# `/test-aimocker` or mid-line mentions from matching.
|
|
# Works on both GNU grep (ubuntu-latest) and BSD grep.
|
|
SLUG=$(printf '%s' "$COMMENT_BODY" \
|
|
| grep -oE '^/test-aimock[[:space:]]+[^[:space:]]+' \
|
|
| head -n1 \
|
|
| sed 's|^/test-aimock[[:space:]]*||' \
|
|
|| true)
|
|
# No default slug fallback. A bare `/test-aimock` (no slug) or a
|
|
# match that only skimmed our boundary (e.g. `/test-aimocker x`)
|
|
# FAILS the workflow rather than silently running against
|
|
# crewai-crews. A hidden default is a footgun: a trusted commenter
|
|
# typing `don't /test-aimock-like-this` would otherwise spawn a
|
|
# full CI run against the wrong package.
|
|
if [ -z "$SLUG" ]; then
|
|
echo "::error::No slug provided. Usage: '/test-aimock <slug>' (e.g. '/test-aimock crewai-crews')"
|
|
exit 1
|
|
fi
|
|
fi
|
|
# Strict slug whitelist: lowercase alphanumerics + hyphens only. This
|
|
# blocks path traversal (`../`), absolute paths, command substitution,
|
|
# and anything else that could escape `showcase/integrations/$SLUG`.
|
|
case "$SLUG" in
|
|
''|*[!a-z0-9-]*)
|
|
echo "::error::Invalid slug '$SLUG' — must match ^[a-z0-9-]+$"
|
|
exit 1
|
|
;;
|
|
esac
|
|
# Belt-and-suspenders: the slug must correspond to an existing package
|
|
# directory. Rejects typos and anything that bypasses the regex.
|
|
if [ ! -d "showcase/integrations/$SLUG" ]; then
|
|
echo "::error::Slug '$SLUG' does not map to showcase/integrations/$SLUG"
|
|
exit 1
|
|
fi
|
|
echo "slug=$SLUG" >> "$GITHUB_OUTPUT"
|
|
|
|
# NOTE on `${{ steps.slug.outputs.slug }}` vs `env:` pattern:
|
|
# Downstream steps interpolate `steps.slug.outputs.slug` directly into
|
|
# the shell script body. This is SAFE here because the "Determine slug"
|
|
# step above whitelists the value against `^[a-z0-9-]+$` AND rejects any
|
|
# slug that doesn't map to an existing package directory — so the value
|
|
# that reaches these interpolations is always a trusted, validated
|
|
# identifier. We still use the `env:`-based defensive default for
|
|
# downstream script bodies that handle anything else UNTRUSTED (see the
|
|
# `actions/github-script` step at the bottom of the workflow).
|
|
- name: Detect package type
|
|
id: pkg-type
|
|
run: |
|
|
SLUG="${{ steps.slug.outputs.slug }}"
|
|
PKG_DIR="showcase/integrations/$SLUG"
|
|
if [ -f "$PKG_DIR/requirements.txt" ] || [ -f "$PKG_DIR/pyproject.toml" ]; then
|
|
echo "has_python=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "has_python=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
# Detect agent server type: langgraph (langgraph_cli dev on :8123)
|
|
# vs uvicorn (agent_server:app on :8000). The two use different
|
|
# start commands, ports, and health endpoints.
|
|
if [ -f "$PKG_DIR/langgraph.json" ]; then
|
|
echo "agent_type=langgraph" >> "$GITHUB_OUTPUT"
|
|
echo "agent_port=8123" >> "$GITHUB_OUTPUT"
|
|
echo "agent_health_path=/ok" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "agent_type=uvicorn" >> "$GITHUB_OUTPUT"
|
|
echo "agent_port=8000" >> "$GITHUB_OUTPUT"
|
|
echo "agent_health_path=/health" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
# aimock_toggle.py ships in crewai-crews and wires AIMOCK_URL
|
|
# end-to-end via configure_aimock(). Packages without it (e.g.
|
|
# langgraph-python) can still use aimock — the workflow injects
|
|
# OPENAI_BASE_URL directly on the agent process. Log the status
|
|
# but do not block; the toggle is a nice-to-have, not a gate.
|
|
if [ -f "$PKG_DIR/src/aimock_toggle.py" ]; then
|
|
echo "ships_toggle=true" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::Slug '$SLUG' ships aimock_toggle.py — aimock redirect handled by configure_aimock()"
|
|
else
|
|
echo "ships_toggle=false" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::Slug '$SLUG' does not ship aimock_toggle.py — aimock redirect will be injected via OPENAI_BASE_URL env var"
|
|
fi
|
|
# Still require Python — this workflow cannot exercise TS-only or
|
|
# Java slugs (no Python agent to start).
|
|
if [ ! -f "$PKG_DIR/requirements.txt" ] && [ ! -f "$PKG_DIR/pyproject.toml" ]; then
|
|
echo "::error::Slug '$SLUG' has no requirements.txt or pyproject.toml — this workflow only exercises Python-backed packages."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Install aimock
|
|
run: |
|
|
# aimock is pinned as a workspace dependency (@copilotkit/showcase-scripts)
|
|
# and installed from the frozen lockfile — no ad-hoc `npm install -g`.
|
|
# A frozen install guarantees the exact pinned version resolves (the old
|
|
# caret floor could drift to a bad publish); this keeps the CI signal
|
|
# reproducible AND satisfies zizmor's adhoc-packages audit.
|
|
#
|
|
# `--ignore-scripts`: a trusted commenter can run this workflow on a PR
|
|
# whose package.json is untrusted content, so we never execute install-time
|
|
# scripts. aimock's `llmock` bin runs fine without them.
|
|
#
|
|
# `--filter` scopes the install to just the aimock owner package so we
|
|
# don't pay for the full monorepo install here (the per-slug package deps
|
|
# are installed later in "Install package dependencies").
|
|
pnpm --filter @copilotkit/showcase-scripts install --frozen-lockfile --ignore-scripts
|
|
|
|
- name: Start aimock
|
|
run: |
|
|
# Invoke the workspace-installed `llmock` bin directly from the repo root.
|
|
# `llmock` is aimock's fixtures-based CLI (the package also ships an
|
|
# `aimock` bin, which is the newer config-only CLI that does NOT accept
|
|
# --fixtures). Running from the repo root keeps the root-relative
|
|
# --fixtures paths correct (a `pnpm --filter exec` would run inside
|
|
# showcase/scripts and break them).
|
|
AIMOCK_BIN="./showcase/scripts/node_modules/.bin/llmock"
|
|
if [ ! -x "$AIMOCK_BIN" ]; then
|
|
echo "::error::aimock binary not found at $AIMOCK_BIN after workspace install"
|
|
exit 1
|
|
fi
|
|
# Fixture layout matches docker-compose.local.yml: feature-parity.json
|
|
# was split into per-framework shared/d4/d5-recorded/d6 directories
|
|
# (directory-based loading, one --fixtures per directory).
|
|
"$AIMOCK_BIN" --port 4010 --host 127.0.0.1 \
|
|
--fixtures showcase/aimock/shared \
|
|
--fixtures showcase/aimock/d4 \
|
|
--fixtures showcase/aimock/d5-recorded \
|
|
--fixtures showcase/aimock/d6 \
|
|
--validate-on-load &
|
|
AIMOCK_PID=$!
|
|
echo "AIMOCK_PID=$AIMOCK_PID" >> "$GITHUB_ENV"
|
|
# Wait for aimock to be ready. Capture the PID + `kill -0` inside
|
|
# the loop so an aimock that crashes on startup (bad fixture path,
|
|
# port in use, binary import error) fails fast instead of burning
|
|
# the full 20s polling a dead process.
|
|
#
|
|
# Probe `/__aimock/health` — aimock's actual readiness endpoint.
|
|
# Root `/` returns HTTP 404 (aimock serves `/__aimock/*` and `/v1/*`
|
|
# only), and `curl -sf` treats 404 as failure, so probing `/` would
|
|
# loop until the budget expired and then hard-fail every run.
|
|
#
|
|
# `--max-time 2 --connect-timeout 1` caps each probe so a hung
|
|
# socket cannot blow the loop's 20-iteration budget.
|
|
for i in $(seq 1 20); do
|
|
if ! kill -0 "$AIMOCK_PID" 2>/dev/null; then
|
|
echo "::error::aimock process (PID $AIMOCK_PID) exited before becoming ready — check the preceding aimock stdout/stderr."
|
|
exit 1
|
|
fi
|
|
curl -sf --max-time 2 --connect-timeout 1 http://localhost:4010/__aimock/health > /dev/null 2>&1 && break
|
|
sleep 1
|
|
done
|
|
curl -sf --max-time 2 --connect-timeout 1 http://localhost:4010/__aimock/health || { echo "aimock failed to start"; exit 1; }
|
|
|
|
- name: Setup Python agent
|
|
if: steps.pkg-type.outputs.has_python == 'true'
|
|
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
|
with:
|
|
python-version: "3.12"
|
|
# Cache pip to avoid reinstalling CrewAI's heavy transitive dep
|
|
# tree on every PR run. Key scopes to the selected slug so each
|
|
# package gets its own cache bucket keyed on its requirements.txt.
|
|
cache: "pip"
|
|
cache-dependency-path: showcase/integrations/${{ steps.slug.outputs.slug }}/requirements.txt
|
|
|
|
- name: Start Python agent
|
|
if: steps.pkg-type.outputs.has_python == 'true'
|
|
run: |
|
|
SLUG="${{ steps.slug.outputs.slug }}"
|
|
AGENT_TYPE="${{ steps.pkg-type.outputs.agent_type }}"
|
|
AGENT_PORT="${{ steps.pkg-type.outputs.agent_port }}"
|
|
AGENT_HEALTH="${{ steps.pkg-type.outputs.agent_health_path }}"
|
|
cd "showcase/integrations/$SLUG"
|
|
# SECURITY / RELIABILITY trade-off: `pip install` runs setup.py /
|
|
# PEP 517 build hooks from PR-controlled packages. Unlike npm / pnpm
|
|
# there is no `--ignore-scripts` flag for pip; the closest equivalent
|
|
# is `--only-binary :all:` (wheel-only, blocks source-build hooks).
|
|
#
|
|
# We previously used `--only-binary :all:` but CrewAI's dependency
|
|
# graph (tiktoken / chromadb / litellm etc.) regularly ships a
|
|
# sdist-only revision of a transitive dep. That made every CI run
|
|
# fail with "Could not find a version that satisfies the requirement"
|
|
# — not a security win but a CI outage. `--prefer-binary` keeps the
|
|
# wheel-first preference (most installs remain hook-free) and only
|
|
# falls back to sdist when a wheel isn't published for
|
|
# linux-x86_64/py3.12. The `author_association` gate at the job
|
|
# level still restricts WHO can trigger this workflow, so the
|
|
# residual source-build-hook risk is bounded to a trusted commenter.
|
|
pip install --prefer-binary -r requirements.txt
|
|
|
|
if [ "$AGENT_TYPE" = "langgraph" ]; then
|
|
# langgraph-python: start via langgraph_cli dev on port 8123.
|
|
# Uses langgraph.json for graph configuration. The /ok endpoint
|
|
# is the readiness probe. Inject OPENAI_BASE_URL + dummy key
|
|
# directly since langgraph-python does not ship aimock_toggle.py.
|
|
if [ ! -f "langgraph.json" ]; then
|
|
echo "::error::Slug '$SLUG' detected as langgraph but langgraph.json is missing."
|
|
exit 1
|
|
fi
|
|
OPENAI_BASE_URL=http://localhost:4010/v1 \
|
|
OPENAI_API_KEY=sk-aimock-dev-ci-only \
|
|
python -u -m langgraph_cli dev \
|
|
--config langgraph.json \
|
|
--host 127.0.0.1 \
|
|
--port "$AGENT_PORT" \
|
|
--no-browser &
|
|
else
|
|
# uvicorn-based agent (crewai-crews): start via agent_server:app
|
|
# on port 8000. Packages that ship aimock_toggle.py wire
|
|
# OPENAI_BASE_URL internally — set AIMOCK_URL only so the toggle
|
|
# itself is exercised end-to-end.
|
|
if [ ! -f "src/agent_server.py" ]; then
|
|
echo "::error::Slug '$SLUG' is missing src/agent_server.py — uvicorn agent type requires the FastAPI entrypoint."
|
|
exit 1
|
|
fi
|
|
export PYTHONPATH="$PWD/src:${PYTHONPATH:-}"
|
|
AIMOCK_URL=http://localhost:4010/v1 \
|
|
python -m uvicorn "agent_server:app" --host 127.0.0.1 --port "$AGENT_PORT" &
|
|
fi
|
|
|
|
# Wait for agent to be ready. Cold imports (litellm + crew graph
|
|
# or langgraph compile) can exceed 60s on a cold runner, so give
|
|
# it 90s (45 iterations x 2s). Mirrors the aimock start pattern:
|
|
# loop + hard-fail so a cryptic Playwright timeout doesn't mask a
|
|
# bind/startup failure.
|
|
for i in $(seq 1 45); do
|
|
curl -sf --max-time 2 --connect-timeout 1 "http://localhost:${AGENT_PORT}${AGENT_HEALTH}" > /dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
curl -sf --max-time 2 --connect-timeout 1 "http://localhost:${AGENT_PORT}${AGENT_HEALTH}" > /dev/null 2>&1 \
|
|
|| { echo "Python agent failed to start on :${AGENT_PORT}"; exit 1; }
|
|
|
|
- name: Install package dependencies
|
|
run: |
|
|
SLUG="${{ steps.slug.outputs.slug }}"
|
|
cd "showcase/integrations/$SLUG"
|
|
# `--ignore-scripts`: a trusted commenter can run `/test-aimock` on
|
|
# a PR whose package.json is untrusted content. Without this flag
|
|
# an attacker's postinstall script would execute on the runner with
|
|
# the workflow's token. The E2E path (Playwright + Next.js dev) does
|
|
# not require install-time scripts to succeed.
|
|
pnpm install --ignore-scripts
|
|
|
|
- name: Start dev server
|
|
run: |
|
|
SLUG="${{ steps.slug.outputs.slug }}"
|
|
AGENT_TYPE="${{ steps.pkg-type.outputs.agent_type }}"
|
|
AGENT_PORT="${{ steps.pkg-type.outputs.agent_port }}"
|
|
cd "showcase/integrations/$SLUG"
|
|
# Invoke `next dev` directly instead of `pnpm dev` — the package's
|
|
# `pnpm dev` script spawns a SECOND agent process via concurrently,
|
|
# but the previous "Start Python agent" step already bound the agent
|
|
# port. A second bind would fail with EADDRINUSE. Running Next
|
|
# directly also keeps the aimock env flow clean.
|
|
#
|
|
# `OPENAI_BASE_URL` + `OPENAI_API_KEY` on Next are DEFENSIVE ONLY.
|
|
# Next proxies chat traffic to the Python agent via the CopilotKit
|
|
# runtime — it does not call OpenAI directly. Setting these prevents
|
|
# accidental real-API fallback if a future route adds a direct call.
|
|
#
|
|
# Agent URL wiring differs by agent type:
|
|
# - uvicorn (crewai-crews): AGENT_URL=http://localhost:8000
|
|
# - langgraph: LANGGRAPH_DEPLOYMENT_URL=http://localhost:8123
|
|
# (langgraph-python's Next routes read this env var, defaulting
|
|
# to localhost:8123 if unset — but we set it explicitly for clarity)
|
|
# Export the correct agent URL env var for Next.js to read.
|
|
if [ "$AGENT_TYPE" = "langgraph" ]; then
|
|
export LANGGRAPH_DEPLOYMENT_URL="http://localhost:${AGENT_PORT}"
|
|
else
|
|
export AGENT_URL="http://localhost:${AGENT_PORT}"
|
|
fi
|
|
export OPENAI_BASE_URL=http://localhost:4010/v1
|
|
export OPENAI_API_KEY=sk-aimock-dev-ci-only
|
|
npx next dev --turbopack &
|
|
# Wait for dev server. `--max-time 2 --connect-timeout 1` caps each
|
|
# probe so a hung socket can't blow the loop budget.
|
|
for i in $(seq 1 30); do
|
|
curl -sf --max-time 2 --connect-timeout 1 http://localhost:3000 > /dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
curl -sf --max-time 2 --connect-timeout 1 http://localhost:3000 || { echo "Dev server failed to start"; exit 1; }
|
|
|
|
- name: Install Playwright
|
|
run: |
|
|
cd "showcase/integrations/${{ steps.slug.outputs.slug }}"
|
|
npx playwright install chromium --with-deps
|
|
|
|
- name: Re-probe aimock liveness
|
|
# aimock was readiness-checked once right after startup, but several
|
|
# steps (Python agent start, pnpm install, Next dev startup, Playwright
|
|
# install) may have run for multiple minutes since. If aimock died
|
|
# during any of that time, Playwright would silently run against real
|
|
# OpenAI because OPENAI_BASE_URL=http://localhost:4010/v1 still points
|
|
# at the (now dead) port — curl would refuse the connection, litellm
|
|
# would fall through to the default OpenAI endpoint, and the test
|
|
# would pass/fail on REAL traffic with REAL costs. Fail loud before
|
|
# Playwright runs.
|
|
run: |
|
|
if ! curl -sf --max-time 2 --connect-timeout 1 http://localhost:4010/__aimock/health > /dev/null 2>&1; then
|
|
echo "::error::aimock is no longer responding on :4010. Refusing to run Playwright against a dead aimock (would silently hit real OpenAI)."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run Playwright tests
|
|
run: |
|
|
SLUG="${{ steps.slug.outputs.slug }}"
|
|
cd "showcase/integrations/$SLUG"
|
|
BASE_URL=http://localhost:3000 npx playwright test --reporter=list
|
|
env:
|
|
CI: "true"
|
|
# Dead env — Next.js is already running from the "Start dev server"
|
|
# step above (which set these inline on that process). Env set here
|
|
# would only affect the `npx playwright test` process, which does not
|
|
# read OPENAI_BASE_URL / OPENAI_API_KEY. Leaving unset to avoid the
|
|
# false impression that these values flow to the running Next server.
|
|
|
|
- name: Re-check aimock liveness after Playwright
|
|
if: always()
|
|
# Defense-in-depth: aimock might have OOM'd DURING the Playwright run.
|
|
# If that happened, the test either silently used stale fixtures (no-op
|
|
# after aimock died if responses were cached) or fell through to real
|
|
# OpenAI. Fail the job loudly so a dead aimock cannot masquerade as a
|
|
# green run. Keeps the 4010-is-still-alive invariant symmetric with the
|
|
# pre-Playwright re-probe above.
|
|
run: |
|
|
if [ -n "${AIMOCK_PID:-}" ] && ! kill -0 "$AIMOCK_PID" 2>/dev/null; then
|
|
echo "::error::aimock process (PID $AIMOCK_PID) died during the Playwright run. Playwright results are untrusted — it may have hit real OpenAI or returned stale fixtures."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Upload test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-${{ steps.slug.outputs.slug }}
|
|
path: showcase/integrations/${{ steps.slug.outputs.slug }}/playwright-report/
|
|
retention-days: 7
|
|
if-no-files-found: ignore
|
|
|
|
outputs:
|
|
slug: ${{ steps.slug.outputs.slug }}
|
|
|
|
# Post the final status as a PR comment. Separated into its own job so
|
|
# the write perms (pull-requests + issues) are scoped to JUST this job —
|
|
# the heavy test job above runs with `contents: read` only, so a compromised
|
|
# transitive dep in `pip install` on a PR-controlled requirements.txt
|
|
# cannot mutate PRs / issues with the workflow's token.
|
|
post-result:
|
|
needs: aimock-e2e
|
|
if: github.event_name == 'issue_comment' && always() && needs.aimock-e2e.result != 'skipped'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 2
|
|
permissions:
|
|
pull-requests: write
|
|
issues: write
|
|
steps:
|
|
- name: Post result to PR
|
|
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
|
|
# Pass dynamic values through env (NOT `${{ ... }}` interpolation into
|
|
# the script body). Even though the slug is whitelisted upstream, the
|
|
# env-var pattern is the defensive default: any future additions that
|
|
# aren't pre-validated cannot accidentally reach script text.
|
|
env:
|
|
SLUG: ${{ needs.aimock-e2e.outputs.slug }}
|
|
JOB_STATUS: ${{ needs.aimock-e2e.result }}
|
|
with:
|
|
script: |
|
|
const slug = process.env.SLUG || '(unknown)';
|
|
const jobStatus = process.env.JOB_STATUS;
|
|
const status = jobStatus === 'success' ? '✅' : '❌';
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body: `${status} **Aimock E2E Tests** (\`${slug}\`): ${jobStatus}\n\n[View run](${runUrl})`
|
|
});
|