chore: import upstream snapshot with attribution
Workflow Lint / actionlint (push) Has been cancelled
Build CI Image / build (push) Has been cancelled
Skill Docs Freshness / check-freshness (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 11:59:46 +08:00
commit dfb0b33892
1170 changed files with 352893 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
self-hosted-runner:
labels:
- ubicloud-standard-2
- ubicloud-standard-8
+125
View File
@@ -0,0 +1,125 @@
# gstack CI eval runner — pre-baked toolchain + deps
# Rebuild weekly via ci-image.yml, on Dockerfile changes, or on lockfile changes
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
# Switch apt sources to Hetzner's public mirror.
# Ubicloud runners (Hetzner FSN1-DC21) hit reliable connection timeouts to
# archive.ubuntu.com:80 — observed 90+ second outages on multiple builds.
# Hetzner's mirror is publicly accessible from any cloud and route-local for
# Ubicloud, so this fixes both reliability and latency. Ubuntu 24.04 uses
# the deb822 sources format at /etc/apt/sources.list.d/ubuntu.sources.
#
# Using HTTP (not HTTPS) intentionally: the base ubuntu:24.04 image ships
# without ca-certificates, so HTTPS apt fails with "No system certificates
# available." Apt's security model verifies via GPG-signed Release files,
# not TLS, so HTTP here is no weaker than the upstream defaults.
RUN sed -i \
-e 's|http://archive.ubuntu.com/ubuntu|http://mirror.hetzner.com/ubuntu/packages|g' \
-e 's|http://security.ubuntu.com/ubuntu|http://mirror.hetzner.com/ubuntu/packages|g' \
/etc/apt/sources.list.d/ubuntu.sources
# Also make apt itself resilient — per-package retries + generous timeouts.
# Hetzner's mirror is reliable but individual packages can still blip; the
# retry config means a single failed fetch doesn't nuke the whole build.
RUN printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' \
> /etc/apt/apt.conf.d/80-retries
# System deps (retry apt-get update + install as a unit — even Hetzner can blip).
# Includes xz-utils so the Node.js .tar.xz download below can decompress.
RUN for i in 1 2 3; do \
apt-get update && apt-get install -y --no-install-recommends \
git curl unzip xz-utils ca-certificates jq bc gpg && break || \
(echo "apt retry $i/3 after failure"; sleep 10); \
done \
&& rm -rf /var/lib/apt/lists/*
# GitHub CLI
RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
| gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
| tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& for i in 1 2 3; do \
apt-get update && apt-get install -y --no-install-recommends gh && break || \
(echo "gh install retry $i/3"; sleep 10); \
done \
&& rm -rf /var/lib/apt/lists/*
# Node.js 22 LTS (needed for claude CLI).
# Install from the official nodejs.org tarball instead of NodeSource's apt setup.
# NodeSource's setup_22.x script runs its own `apt-get update` + `apt-get install gnupg`,
# both of which depend on archive.ubuntu.com / security.ubuntu.com being reachable.
# Ubicloud CI runners frequently can't reach those mirrors (connection timeouts),
# and "gnupg" was renamed to "gpg" on Ubuntu 24.04 anyway, so NodeSource's script
# fails before it can add its own repo. Direct tarball download is network-simpler
# (one host: nodejs.org) and doesn't touch apt at all.
ENV NODE_VERSION=22.20.0
RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" -o /tmp/node.tar.xz \
&& tar -xJ -C /usr/local --strip-components=1 --no-same-owner -f /tmp/node.tar.xz \
&& rm -f /tmp/node.tar.xz \
&& node --version \
&& npm --version
# Bun (install to /usr/local so non-root users can access it)
ENV BUN_INSTALL="/usr/local"
RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/install \
| BUN_VERSION=1.3.10 bash
# Claude CLI
RUN npm i -g @anthropic-ai/claude-code
# Playwright system deps (Chromium) — needed for browse E2E tests
RUN npx playwright install-deps chromium
# Linux has neither Helvetica nor Arial. make-pdf's print CSS stacks fall back
# to Liberation Sans (metric-compatible Arial clone, SIL OFL 1.1) so PDFs don't
# render in DejaVu Sans. playwright install-deps happens to pull this in today,
# but the dep is implicit and could change — install explicitly so upgrades
# can't silently regress rendering.
#
# Xvfb is also installed here so the browse --headed integration tests
# (headed-xvfb, headed-orphan-cleanup) can exercise the Linux container
# auto-spawn path on every CI run. Without Xvfb in the image, the most
# common production --headed path goes untested.
RUN for i in 1 2 3; do \
apt-get update && apt-get install -y --no-install-recommends fonts-liberation fontconfig xvfb x11-utils && break || \
(echo "fonts-liberation install retry $i/3"; sleep 10); \
done \
&& fc-cache -f \
&& rm -rf /var/lib/apt/lists/*
# Pre-install dependencies (cached layer — only rebuilds when package.json or
# bun.lock changes). Copy BOTH so install is deterministic and matches local
# resolution. Without bun.lock here, bun install resolved transitive deps
# differently in CI vs local (observed on v1.28.0.0: socks landed but
# smart-buffer + ip-address didn't make it into the cached node_modules).
COPY package.json bun.lock /workspace/
WORKDIR /workspace
RUN bun install --frozen-lockfile && rm -rf /tmp/*
# Install Playwright Chromium to a shared location accessible by all users
ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers
RUN npx playwright install chromium \
&& chmod -R a+rX /opt/playwright-browsers
# Verify everything works
RUN bun --version && node --version && claude --version && jq --version && gh --version \
&& npx playwright --version \
&& fc-match "Liberation Sans" | grep -qi "Liberation" \
|| (echo "ERROR: fonts-liberation not installed — make-pdf PDFs will render in DejaVu Sans" && exit 1)
# At runtime: checkout overwrites /workspace, but node_modules persists
# if we move it out of the way and symlink back
# Save node_modules + package.json snapshot for cache validation at runtime
RUN mv /workspace/node_modules /opt/node_modules_cache \
&& cp /workspace/package.json /opt/node_modules_cache/.package.json
# Claude CLI refuses --dangerously-skip-permissions as root.
# Create a non-root user for eval runs (GH Actions overrides USER, so
# the workflow must set options.user or use gosu/su-exec at runtime).
RUN useradd -m -s /bin/bash runner \
&& chmod -R a+rX /opt/node_modules_cache \
&& mkdir -p /home/runner/.gstack && chown -R runner:runner /home/runner/.gstack \
&& chmod 1777 /tmp \
&& mkdir -p /home/runner/.bun && chown -R runner:runner /home/runner/.bun
+8
View File
@@ -0,0 +1,8 @@
name: Workflow Lint
on: [push, pull_request]
jobs:
actionlint:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: rhysd/actionlint@v1.7.11
+41
View File
@@ -0,0 +1,41 @@
name: Build CI Image
on:
# Rebuild weekly (Monday 6am UTC) to pick up CLI updates
schedule:
- cron: '0 6 * * 1'
# Rebuild on Dockerfile or lockfile changes
push:
branches: [main]
paths:
- '.github/docker/Dockerfile.ci'
- 'package.json'
- 'bun.lock'
# Manual trigger
workflow_dispatch:
jobs:
build:
runs-on: ubicloud-standard-8
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
# Copy lockfile + package.json into Docker build context
- run: cp package.json bun.lock .github/docker/
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/build-push-action@v6
with:
context: .github/docker
file: .github/docker/Dockerfile.ci
push: true
tags: |
ghcr.io/${{ github.repository }}/ci:latest
ghcr.io/${{ github.repository }}/ci:${{ github.sha }}
+133
View File
@@ -0,0 +1,133 @@
name: Periodic Evals
on:
schedule:
- cron: '0 6 * * 1' # Monday 6 AM UTC
workflow_dispatch:
concurrency:
group: evals-periodic
cancel-in-progress: true
env:
IMAGE: ghcr.io/${{ github.repository }}/ci
EVALS_TIER: periodic
EVALS_ALL: 1 # Ignore diff — run all periodic tests
jobs:
build-image:
runs-on: ubicloud-standard-8
permissions:
contents: read
packages: write
outputs:
image-tag: ${{ steps.meta.outputs.tag }}
steps:
- uses: actions/checkout@v4
- id: meta
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'package.json', 'bun.lock') }}" >> "$GITHUB_OUTPUT"
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check if image exists
id: check
run: |
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
- if: steps.check.outputs.exists == 'false'
run: cp package.json bun.lock .github/docker/
- if: steps.check.outputs.exists == 'false'
uses: docker/build-push-action@v6
with:
context: .github/docker
file: .github/docker/Dockerfile.ci
push: true
tags: |
${{ steps.meta.outputs.tag }}
${{ env.IMAGE }}:latest
evals:
runs-on: ubicloud-standard-8
needs: build-image
container:
image: ${{ needs.build-image.outputs.image-tag }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
options: --user runner
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
suite:
- name: e2e-plan
file: test/skill-e2e-plan.test.ts
- name: e2e-design
file: test/skill-e2e-design.test.ts
- name: e2e-qa-bugs
file: test/skill-e2e-qa-bugs.test.ts
- name: e2e-qa-workflow
file: test/skill-e2e-qa-workflow.test.ts
- name: e2e-review
file: test/skill-e2e-review.test.ts
- name: e2e-workflow
file: test/skill-e2e-workflow.test.ts
- name: e2e-routing
file: test/skill-routing-e2e.test.ts
- name: e2e-codex
file: test/codex-e2e.test.ts
- name: e2e-gemini
file: test/gemini-e2e.test.ts
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Fix bun temp
run: |
mkdir -p /home/runner/.cache/bun
{
echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun"
echo "BUN_TMPDIR=/home/runner/.cache/bun"
echo "TMPDIR=/home/runner/.cache"
} >> "$GITHUB_ENV"
# Recursive copy (cp -r) instead of symlink: bun build resolves a
# file's realpath when looking for sibling deps. See evals.yml for the
# full explanation. cp -al would be faster but /opt and /workspace
# are on different overlay-fs layers, so cross-device hardlink fails.
- name: Restore deps
run: |
if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.package.json package.json >/dev/null 2>&1; then
cp -r /opt/node_modules_cache node_modules
else
bun install
fi
- run: bun run build
- name: Run ${{ matrix.suite.name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
EVALS_CONCURRENCY: "40"
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
run: EVALS=1 bun test --retry 2 --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
- name: Upload eval results
if: always()
uses: actions/upload-artifact@v4
with:
name: eval-periodic-${{ matrix.suite.name }}
path: ~/.gstack-dev/evals/*.json
retention-days: 90
+360
View File
@@ -0,0 +1,360 @@
name: E2E Evals
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: evals-${{ github.head_ref }}
cancel-in-progress: true
env:
IMAGE: ghcr.io/${{ github.repository }}/ci
EVALS_TIER: gate
jobs:
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
build-image:
runs-on: ubicloud-standard-8
permissions:
contents: read
packages: write
outputs:
image-tag: ${{ steps.meta.outputs.tag }}
steps:
- uses: actions/checkout@v4
- id: meta
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'package.json', 'bun.lock') }}" >> "$GITHUB_OUTPUT"
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Check if image exists
id: check
run: |
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
- if: steps.check.outputs.exists == 'false'
run: cp package.json bun.lock .github/docker/
- if: steps.check.outputs.exists == 'false'
uses: docker/build-push-action@v6
with:
context: .github/docker
file: .github/docker/Dockerfile.ci
push: true
tags: |
${{ steps.meta.outputs.tag }}
${{ env.IMAGE }}:latest
evals:
runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }}
needs: build-image
container:
image: ${{ needs.build-image.outputs.image-tag }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
options: --user runner
timeout-minutes: ${{ matrix.suite.timeout || 25 }}
strategy:
fail-fast: false
matrix:
suite:
- name: llm-judge
file: test/skill-llm-eval.test.ts
- name: e2e-browse
file: test/skill-e2e-bws.test.ts
runner: ubicloud-standard-8
- name: e2e-plan
file: test/skill-e2e-plan.test.ts
- name: e2e-deploy
file: test/skill-e2e-deploy.test.ts
- name: e2e-design
file: test/skill-e2e-design.test.ts
- name: e2e-qa-bugs
file: test/skill-e2e-qa-bugs.test.ts
- name: e2e-qa-workflow
file: test/skill-e2e-qa-workflow.test.ts
- name: e2e-review
file: test/skill-e2e-review.test.ts
- name: e2e-workflow
file: test/skill-e2e-workflow.test.ts
- name: e2e-routing
file: test/skill-routing-e2e.test.ts
- name: e2e-codex
file: test/codex-e2e.test.ts
- name: e2e-gemini
file: test/gemini-e2e.test.ts
# Real-PTY plan-mode smokes. Only the deterministically-reliable ones
# are CI-gated: office-hours (asks its mode question first, caught by
# the collapsed/bullet prose-AUQ detector) and plan-mode-no-op (no
# ask-first dependency). The plan-eng/plan-design plan-mode + floor
# smokes are periodic (stochastic ask-first — see touchfiles E2E_TIERS).
# Needs the interactive-config seed step below; PTY sessions otherwise
# wedge on the fresh-container onboarding/API-key dialog.
- name: e2e-pty-plan-smoke
file: test/skill-e2e-office-hours-auto-mode.test.ts test/skill-e2e-plan-mode-no-op.test.ts
timeout: 35
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# Bun creates root-owned temp dirs during Docker build. GH Actions runs as
# runner user with HOME=/github/home. Redirect bun's cache to a writable dir.
- name: Fix bun temp
run: |
mkdir -p /home/runner/.cache/bun
{
echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun"
echo "BUN_TMPDIR=/home/runner/.cache/bun"
echo "TMPDIR=/home/runner/.cache"
} >> "$GITHUB_ENV"
# Restore pre-installed node_modules from Docker image via recursive
# copy. Symlink (`ln -s`) breaks bun's module resolution because bun
# resolves a file's realpath when walking up to find node_modules/<dep>;
# from a symlinked path, realpath escapes the workspace and sibling
# deps no longer resolve. Hardlink copy (`cp -al`) fails because /opt
# and /workspace are on different overlay-fs layers ("Invalid
# cross-device link"). Recursive copy works on every layout. Cost:
# ~5s for ~200 packages of small JS files vs ~0s for symlink — still
# vastly cheaper than rerunning `bun install` (network + resolution).
- name: Restore deps
run: |
if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.package.json package.json >/dev/null 2>&1; then
cp -r /opt/node_modules_cache node_modules
else
bun install
fi
- run: bun run build
# Verify Playwright can launch Chromium (fails fast if sandbox/deps are broken)
- name: Verify Chromium
if: matrix.suite.name == 'e2e-browse'
run: |
echo "whoami=$(whoami) HOME=$HOME TMPDIR=${TMPDIR:-unset}"
touch /tmp/.bun-test && rm /tmp/.bun-test && echo "/tmp writable"
bun -e "import {chromium} from 'playwright';const b=await chromium.launch({args:['--no-sandbox']});console.log('Chromium OK');await b.close()"
# PTY smokes spawn the interactive `claude` TUI. A fresh container has no
# ~/.claude.json, so claude wedges on the onboarding + "use detected
# ANTHROPIC_API_KEY?" dialog and the spawned session never reaches the
# skill. Seed onboarding-complete + the key approval (mirrors what the
# hermetic E2E child env seeds). Scoped to this suite; needs its OWN key
# env (the secrets block below is on the Run step only).
- name: Seed claude interactive config
if: matrix.suite.name == 'e2e-pty-plan-smoke'
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
run: |
node -e '
const fs = require("fs"), os = require("os"), path = require("path");
const p = path.join(os.homedir(), ".claude.json");
const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {};
seed.hasCompletedOnboarding = true;
const key = process.env.ANTHROPIC_API_KEY || "";
if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] };
fs.writeFileSync(p, JSON.stringify(seed, null, 2));
console.log("seeded", p);
'
# PTY smokes drive the interactive `claude` TUI and send /office-hours and
# /plan-ceo-review. Claude Code discovers user-scoped skills from
# $HOME/.claude/skills/<name>/SKILL.md, but .claude/skills is gitignored, so
# a fresh CI checkout has NO registry — claude prints "Unknown command:
# /plan-ceo-review". Mirror setup's --no-prefix registry minimally: a gstack
# root symlink (resolves the preamble's absolute ~/.claude/skills/gstack/bin/*
# and ~/.claude/skills/gstack/<skill>/sections/* paths) plus a per-skill
# top-level dir holding SKILL.md (+ sections) symlinks for the two skills
# these tests invoke. No ./setup (it builds binaries, launches Chromium,
# installs fonts, reads a /dev/tty prompt) and no binary build (SKILL.md +
# bin/ + sections/ are committed). $HOME is /github/home here; the spawned
# claude inherits it (this runner adds no HOME/CLAUDE_CONFIG_DIR override,
# no hermetic mode) and the Seed step already proved claude reads $HOME.
- name: Register gstack skills for PTY smoke
if: matrix.suite.name == 'e2e-pty-plan-smoke'
run: |
set -eu
SKILLS_DIR="$HOME/.claude/skills"
REPO="$GITHUB_WORKSPACE" # /__w/gstack/gstack
mkdir -p "$SKILLS_DIR"
# The gstack root stays a symlink — the preamble's runtime bash resolves
# ~/.claude/skills/gstack/bin/* and ~/.claude/skills/gstack/<skill>/sections/*
# through it, and bash follows cross-mount symlinks fine.
ln -snf "$REPO" "$SKILLS_DIR/gstack"
# But the per-skill SKILL.md the TUI DISCOVERS must be a REAL file on the
# same mount as $HOME. claude 2.1.187's interactive-TUI skill scanner does
# not follow the /github/home -> /__w cross-mount symlink (proven: `claude
# -p` discovered the skill — READY — while the TUI rejected /office-hours
# as "Unknown command"; a local macOS repro with the identical symlinked
# registry recognized it, isolating the failure to the container's
# cross-mount symlink). Copy SKILL.md + sections as real files so the TUI
# reads them directly.
for s in office-hours plan-ceo-review; do
rm -rf "${SKILLS_DIR:?}/$s"
mkdir -p "$SKILLS_DIR/$s"
cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md"
cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections"
done
# Also register PROJECT-scoped (cwd) skills. claude's interactive TUI
# surfaces /slash commands from <cwd>/.claude/skills, and the smokes run
# with cwd=$REPO whose .claude/skills is gitignored (absent on a fresh CI
# checkout) — the user-dir registration above feeds `claude -p` but the
# TUI looks here. No gstack symlink in the project dir: it would point at
# its own parent ($REPO). Runtime preamble paths use the user-dir
# ~/.claude/skills/gstack symlink above.
PROJ_SKILLS="$REPO/.claude/skills"
mkdir -p "$PROJ_SKILLS"
for s in office-hours plan-ceo-review; do
rm -rf "${PROJ_SKILLS:?}/$s"
mkdir -p "$PROJ_SKILLS/$s"
cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md"
cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections"
done
echo "--- registry under $SKILLS_DIR ---"
ls -la "$SKILLS_DIR/gstack" "$SKILLS_DIR/office-hours" "$SKILLS_DIR/plan-ceo-review"
# Fail fast if any committed target moved/renamed — a dangling symlink
# would otherwise resurface as a silent "Unknown command" + 35-min timeout.
for f in \
"$SKILLS_DIR/office-hours/SKILL.md" \
"$SKILLS_DIR/plan-ceo-review/SKILL.md" \
"$SKILLS_DIR/gstack/bin/gstack-update-check" \
"$SKILLS_DIR/gstack/office-hours/sections/design-and-handoff.md" \
"$SKILLS_DIR/gstack/plan-ceo-review/sections/review-sections.md"; do
if [ ! -e "$f" ]; then
echo "ERROR: skill-registry target missing (symlink dangles): $f" >&2
exit 1
fi
done
grep -m1 '^name: office-hours$' "$SKILLS_DIR/office-hours/SKILL.md" >/dev/null \
|| { echo "ERROR: office-hours SKILL.md missing 'name: office-hours' frontmatter" >&2; exit 1; }
grep -m1 '^name: plan-ceo-review$' "$SKILLS_DIR/plan-ceo-review/SKILL.md" >/dev/null \
|| { echo "ERROR: plan-ceo-review SKILL.md missing 'name: plan-ceo-review' frontmatter" >&2; exit 1; }
echo "skill registry OK"
- name: Run ${{ matrix.suite.name }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
EVALS_CONCURRENCY: "40"
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
run: EVALS=1 bun test --retry 2 --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
- name: Upload eval results
if: always()
uses: actions/upload-artifact@v4
with:
name: eval-${{ matrix.suite.name }}
path: ~/.gstack-dev/evals/*.json
retention-days: 90
report:
runs-on: ubicloud-standard-8
needs: evals
if: always() && github.event_name == 'pull_request'
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
# The comment upsert below calls the REST `/issues/{n}/comments` endpoints
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
# `issues` permission, not `pull-requests` — without it the GET returns 401
# on every PR that produces eval artifacts (PRs with no artifacts exit
# early and never hit it, which is why this stayed hidden). See #1802 CI fix.
issues: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Download all eval artifacts
uses: actions/download-artifact@v4
with:
pattern: eval-*
path: /tmp/eval-results
merge-multiple: true
- name: Post PR comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# shellcheck disable=SC2086,SC2059
RESULTS=$(find /tmp/eval-results -name '*.json' 2>/dev/null | sort)
if [ -z "$RESULTS" ]; then
echo "No eval results found"
exit 0
fi
TOTAL=0; PASSED=0; FAILED=0; COST="0"
SUITE_LINES=""
for f in $RESULTS; do
if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
echo "Skipping malformed JSON: $f"
continue
fi
T=$(jq -r '.total_tests // 0' "$f")
P=$(jq -r '.passed // 0' "$f")
F=$(jq -r '.failed // 0' "$f")
C=$(jq -r '.total_cost_usd // 0' "$f")
TIER=$(jq -r '.tier // "unknown"' "$f")
[ "$T" -eq 0 ] && continue
TOTAL=$((TOTAL + T))
PASSED=$((PASSED + P))
FAILED=$((FAILED + F))
COST=$(echo "$COST + $C" | bc)
STATUS_ICON="✅"
[ "$F" -gt 0 ] && STATUS_ICON="❌"
SUITE_LINES="${SUITE_LINES}| ${TIER} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n"
done
STATUS="✅ PASS"
[ "$FAILED" -gt 0 ] && STATUS="❌ FAIL"
BODY="## E2E Evals: ${STATUS}
**${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost | **13 parallel runners**
| Suite | Result | Status | Cost |
|-------|--------|--------|------|
$(echo -e "$SUITE_LINES")
---
*13x ubicloud-standard-8 (Docker: pre-baked toolchain + deps) | wall clock ≈ slowest suite*"
if [ "$FAILED" -gt 0 ]; then
FAILURES=""
for f in $RESULTS; do
if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
F=$(jq -r '.failed // 0' "$f")
[ "$F" -eq 0 ] && continue
FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error")
FAILURES="${FAILURES}${FAILS}\n"
done
BODY="${BODY}
### Failures
$(echo -e "$FAILURES")"
fi
# Update existing comment or create new one
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
if [ -n "$COMMENT_ID" ]; then
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
-X PATCH -f body="$BODY"
else
gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY"
fi
+91
View File
@@ -0,0 +1,91 @@
name: make-pdf copy-paste gate
on:
pull_request:
branches: [main]
paths:
- 'make-pdf/**'
- 'lib/diagram-render/**'
- 'test/diagram-render-drift.test.ts'
- 'browse/src/meta-commands.ts'
- 'browse/src/write-commands.ts'
- 'browse/src/commands.ts'
- 'browse/src/cli.ts'
- 'scripts/resolvers/make-pdf.ts'
- 'package.json'
- '.github/workflows/make-pdf-gate.yml'
workflow_dispatch:
concurrency:
group: make-pdf-gate-${{ github.head_ref }}
cancel-in-progress: true
jobs:
gate:
strategy:
fail-fast: false
matrix:
os: [ubicloud-standard-8, macos-latest]
# Windows is tolerant-mode — Xpdf / Poppler-Windows extraction
# differs enough from the Linux/macOS baseline that the strict
# exact-diff gate is unreliable. Enable once the normalized
# comparator proves tolerant enough (Codex round 2 #18).
#
# include:
# - os: windows-latest
# tolerant: true
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Install poppler (macOS)
if: matrix.os == 'macos-latest'
run: brew install poppler
- name: Install poppler-utils (Ubuntu)
if: matrix.os == 'ubicloud-standard-8'
run: sudo apt-get update && sudo apt-get install -y poppler-utils
# Install a color-emoji font BEFORE Chromium launches so the emoji render
# gate has a fallback font. macOS ships Apple Color Emoji already.
- name: Install color-emoji font (Ubuntu)
if: matrix.os == 'ubicloud-standard-8'
run: |
sudo apt-get install -y fonts-noto-color-emoji
fc-cache -f || true
fc-match -f '%{family[0]}\t%{color}\n' ':lang=und-zsye:charset=1F600' || true
- name: Install Playwright Chromium
run: bunx playwright install chromium
- name: Build binaries
run: bun run build
- name: ad-hoc codesign (Apple Silicon)
if: matrix.os == 'macos-latest'
run: |
for bin in browse/dist/browse browse/dist/find-browse design/dist/design make-pdf/dist/pdf; do
codesign --remove-signature "$bin" 2>/dev/null || true
codesign -s - -f "$bin" || true
done
- name: Log toolchain versions
run: |
echo "OS: ${{ matrix.os }}"
bun --version
which pdftotext && pdftotext -v 2>&1 | head -1 || true
- name: Run make-pdf unit tests
run: bun test make-pdf/test/*.test.ts test/diagram-render-drift.test.ts
- name: Run E2E gates (combined-features copy-paste + emoji render)
env:
BROWSE_BIN: ${{ github.workspace }}/browse/dist/browse
run: bun test make-pdf/test/e2e/
+98
View File
@@ -0,0 +1,98 @@
name: PR Title Sync
# WHY pull_request_target (not pull_request): the default GITHUB_TOKEN is
# READ-ONLY on fork PRs under `pull_request`, so the title-sync backstop could
# never `gh pr edit` a fork/agent PR. `pull_request_target` runs in the base-repo
# context with a write token, which fixes fork coverage.
#
# WHY this is SAFE (pull_request_target is the most dangerous trigger):
# - We check out the BASE repo (no `ref:`), so the only code we execute is
# trusted base-repo infra (bin/gstack-pr-title-rewrite.sh). We NEVER check
# out or run PR-head/fork code.
# - Every attacker-controlled PR field (title, head repo, head sha) arrives via
# `env:` and is referenced as a shell-quoted "$VAR". We NEVER inline a
# `${{ github.event.pull_request.* }}` expression inside the run: script
# (that would execute a crafted title as shell).
# - The PR-head VERSION is read as DATA via the API (raw media type), from the
# head repo at the head sha — never by checking out the head.
# test/pr-title-sync-workflow-safety.test.ts is the static tripwire for all of
# the above and fails CI if any of it regresses.
on:
pull_request_target:
types: [opened, synchronize, edited]
paths:
- 'VERSION'
concurrency:
group: pr-title-sync-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
sync:
name: Sync PR title to VERSION
runs-on: ubicloud-standard-8
permissions:
contents: read
pull-requests: write
if: github.actor != 'github-actions[bot]'
steps:
# Base repo only — trusted infra (the rewrite helper). No PR-head checkout.
- name: Checkout base repo (trusted)
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: Rewrite PR title to match VERSION
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUM: ${{ github.event.pull_request.number }}
# Attacker-controlled on fork PRs — env-only, never inlined into run:.
OLD_TITLE: ${{ github.event.pull_request.title }}
BASE_REPO: ${{ github.repository }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
set -euo pipefail
chmod +x ./bin/gstack-pr-title-rewrite.sh
if [ "$HEAD_REPO" = "$BASE_REPO" ]; then IS_FORK=0; else IS_FORK=1; fi
# Read the PR-head VERSION as data (raw bytes), from the head repo at
# the head sha. Guard the assignment itself: under `set -e` a bare
# `VERSION=$(...)` would abort the step before any later [ -z ] check.
if ! VERSION=$(gh api -H "Accept: application/vnd.github.raw" \
"repos/$HEAD_REPO/contents/VERSION?ref=$HEAD_SHA" 2>/dev/null | tr -d '[:space:]'); then
VERSION=""
fi
if [ -z "$VERSION" ]; then
# Same-repo read failure should never happen — fail loudly so we
# notice. A fork miss (public-contents quirk, private fork) is a
# convenience gap, not a gate — warn and skip so the check stays green.
if [ "$IS_FORK" = "0" ]; then
echo "::error::Could not read VERSION from same-repo PR head ($HEAD_SHA)."
exit 1
fi
echo "::warning::Could not read VERSION from fork $HEAD_REPO ($HEAD_SHA); skipping title sync."
exit 0
fi
# The helper rejects a malformed VERSION (exit 2). Same policy: loud for
# same-repo, soft for forks. Never echo the raw (attacker-controlled)
# title — Actions still parses ::workflow-command:: from stdout.
if ! NEW_TITLE=$(./bin/gstack-pr-title-rewrite.sh "$VERSION" "$OLD_TITLE"); then
if [ "$IS_FORK" = "0" ]; then
echo "::error::Could not compute title for VERSION '$VERSION' on PR #$PR_NUM."
exit 1
fi
echo "::warning::Could not compute title for fork PR #$PR_NUM; skipping."
exit 0
fi
if [ "$NEW_TITLE" = "$OLD_TITLE" ]; then
echo "PR #$PR_NUM title already correct; no change."
exit 0
fi
gh pr edit "$PR_NUM" --title "$NEW_TITLE"
echo "PR #$PR_NUM title synced to VERSION."
+33
View File
@@ -0,0 +1,33 @@
name: Skill Docs Freshness
on: [push, pull_request]
jobs:
check-freshness:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
- run: bun install
- name: Check Claude host freshness
run: bun run gen:skill-docs
- name: Verify Claude skill docs are fresh
run: |
git diff --exit-code || {
echo "Generated SKILL.md files are stale. Run: bun run gen:skill-docs"
exit 1
}
- name: Check Codex host freshness
run: bun run gen:skill-docs --host codex
- name: Verify Codex skill docs are fresh
run: |
git diff --exit-code -- .agents/ || {
echo "Generated Codex SKILL.md files are stale. Run: bun run gen:skill-docs --host codex"
exit 1
}
- name: Generate Factory skill docs
run: bun run gen:skill-docs --host factory
- name: Verify Factory skill docs are fresh
run: |
git diff --exit-code -- .factory/ || {
echo "Generated Factory SKILL.md files are stale. Run: bun run gen:skill-docs --host factory"
exit 1
}
+74
View File
@@ -0,0 +1,74 @@
name: Version Gate
on:
pull_request:
paths:
- 'VERSION'
- 'CHANGELOG.md'
- 'package.json'
concurrency:
group: version-gate-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
check:
name: Check VERSION is not stale vs queue
runs-on: ubicloud-standard-8
permissions:
contents: read
pull-requests: read
steps:
- name: Checkout PR head
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Read versions
id: versions
run: |
set -euo pipefail
PR_VERSION=$(cat VERSION | tr -d '[:space:]')
BASE_REF="${{ github.event.pull_request.base.ref }}"
git fetch origin "$BASE_REF" --depth=1 --quiet || true
BASE_VERSION=$(git show "origin/$BASE_REF:VERSION" 2>/dev/null | tr -d '[:space:]' || echo "0.0.0.0")
{
echo "pr_version=$PR_VERSION"
echo "base_version=$BASE_VERSION"
echo "base_ref=$BASE_REF"
} >> "$GITHUB_OUTPUT"
- name: Detect bump level
id: bump
run: |
LEVEL=$(bun run scripts/detect-bump.ts "${{ steps.versions.outputs.base_version }}" "${{ steps.versions.outputs.pr_version }}")
echo "level=$LEVEL" >> "$GITHUB_OUTPUT"
- name: Query queue (util) — fail-open on error
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
bun run bin/gstack-next-version \
--base "${{ steps.versions.outputs.base_ref }}" \
--bump "${{ steps.bump.outputs.level }}" \
--current-version "${{ steps.versions.outputs.base_version }}" \
--workspace-root null \
--exclude-pr "${{ github.event.pull_request.number }}" \
> next.json 2> next.err
RC=$?
if [ "$RC" != "0" ] || [ ! -s next.json ]; then
echo '{"offline":true}' > next.json
echo "::warning::util exit=$RC — failing open. stderr:"
cat next.err || true
fi
- name: Compare PR VERSION to next free slot
env:
PR_VERSION: ${{ steps.versions.outputs.pr_version }}
run: |
bun run scripts/compare-pr-version.ts next.json "${{ github.event.pull_request.number }}"
+123
View File
@@ -0,0 +1,123 @@
name: Windows Free Tests
# Curated subset of the free test suite that runs on a paid faster Windows runner.
#
# Codex's v1.18.0.0 review flagged that the existing evals.yml workflow uses
# a Linux container, so a windows-latest matrix entry there isn't a drop-in.
# This workflow is non-container, runs the curated Windows-safe subset, plus
# targeted resolver tests that exercise the Bun.which-based claude binary
# resolution + the GSTACK_CLAUDE_BIN override path on Windows.
#
# Runner: GitHub-hosted free `windows-latest`. The whole rest of CI runs on
# Ubicloud (Linux), but Ubicloud doesn't ship Windows runners and we don't
# want to flip on GitHub's org-level larger-runner billing for just this one
# job. 4 cores, ~60s spin-up, $0. The wave-coverage tests this runs are
# small enough that total job time stays under 2 minutes.
#
# What this DOES NOT do (still out of scope, tracked as follow-up):
# - Run the full free suite on Windows. The 24 tests that hardcode /bin/sh,
# spawn('sh',...), or raw /tmp/ paths are excluded by scripts/test-free-shards.ts
# --windows-only. They need POSIX-bound surfaces to be ported off shell
# primitives before they can run on Windows.
# - Run Playwright/browser-backed tests. Browse server bring-up on Windows is
# a separate concern (PR #1238 windows-pty-bun-pty-fix is in flight).
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: windows-free-${{ github.head_ref }}
cancel-in-progress: true
jobs:
windows-free-tests:
# Ubicloud Windows runner (same provider as the Linux evals workflow).
# To revert: swap to `windows-latest` (GitHub's free 4-core Windows runner).
runs-on: windows-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v1
with:
bun-version: latest
- name: Configure git identity (required by tests that init temp repos)
run: |
git config --global user.email "windows-ci@gstack.test"
git config --global user.name "Windows CI"
git config --global init.defaultBranch main
shell: bash
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Build server-node.mjs (required by Windows browse path)
# browse/src/cli.ts module-level throws on Windows if server-node.mjs
# is missing — Bun can't drive Playwright's Chromium on Windows
# (oven-sh/bun#4253). The bundle must exist for any test that
# transitively loads cli.ts to even import. We build only the
# Node-compatible server bundle here; full `bun run build` would
# also compile every binary which is slow and unnecessary for tests.
run: bash browse/scripts/build-node-server.sh
shell: bash
- name: Generate host SKILL.md outputs (.agents, .factory)
# The golden-file regression tests in test/gen-skill-docs.test.ts read
# .agents/skills/gstack-ship/SKILL.md and .factory/skills/gstack-ship/
# SKILL.md. Both are gitignored — generated on demand by gen:skill-docs.
# On Mac/Linux CI the existing eval workflow regenerates these as part
# of its own pipeline; the windows-free-tests lane doesn't share that
# so it must regenerate explicitly.
run: bun run gen:skill-docs --host all
shell: bash
# The Windows job verifies the new portability work this PR delivers,
# not the entire free suite. After v1.20.0.0 ships, full-suite Windows
# parity is a P4 follow-up TODO that depends on porting many tests off
# POSIX-bound surfaces (raw /tmp paths, /bin/bash hardcodes, bash
# shebang spawns, mode-bit assertions, deleted v1.14 sidebar refs, etc).
#
# The curated subset enumeration in scripts/test-free-shards.ts is
# retained for future expansion — `bun run test:windows --list` gives
# contributors a starting point to grow Windows coverage incrementally.
#
# What we verify here is exactly the new code paths v1.20.0.0 ships:
# - bin/gstack-paths state-root resolution (test/gstack-paths.test.ts)
# - browse/src/claude-bin.ts Bun.which wrapper + override + arg-prefix
# resolution including the GSTACK_CLAUDE_BIN=wsl PATHEXT path
# (browse/test/claude-bin.test.ts)
# - scripts/test-free-shards.ts curation logic itself
# (test/test-free-shards.test.ts)
- name: Show curated subset (informational — for future expansion)
run: bun run scripts/test-free-shards.ts --windows-only --list
shell: bash
continue-on-error: true
- name: Verify new portability work on Windows
# Tests targeting the v1.20.0.0 lane plus v1.30.0.0 fix-wave additions
# plus v1.36.0.0 Windows-install hardening (sanitizer + _link_or_copy
# helper + build-script subshells + doc/config-key drift guard).
# v1.30.0.0 extension covers icacls hardening (#1308), bash.exe telemetry
# wrap (#1306), and Bun.which-based binary resolvers (#1307). These must
# pass on Windows for the wave's "Windows hardening" framing to be honest.
run: |
bun test \
test/gstack-paths.test.ts \
browse/test/claude-bin.test.ts \
test/test-free-shards.test.ts \
browse/test/file-permissions.test.ts \
browse/test/security.test.ts \
browse/test/server-sanitize-surrogates.test.ts \
test/setup-windows-fallback.test.ts \
test/bin-windows-bun-import-paths.test.ts \
test/build-script-shell-compat.test.ts \
test/docs-config-keys.test.ts \
test/brain-sync-windows-paths.test.ts \
make-pdf/test/browseClient.test.ts \
make-pdf/test/pdftotext.test.ts
shell: bash
+96
View File
@@ -0,0 +1,96 @@
name: Windows Setup E2E
# End-to-end fresh-install gate for Windows. Runs `./setup` on a clean
# windows-latest checkout and asserts the build completes, binaries
# resolve via find-browse, and the gstack-paths state root resolves
# cleanly. Catches Bun shell-parser regressions in package.json's build
# chain (#1538, #1537, #1530, #1457, #1561) before they reach users.
#
# Separate from windows-free-tests.yml because that one runs a curated
# unit-test subset; this one exercises the install path itself.
#
# Runner: GitHub-hosted free windows-latest. ~3-5 min total.
on:
pull_request:
branches: [main]
paths:
- 'package.json'
- 'scripts/build.sh'
- 'scripts/write-version-files.sh'
- 'setup'
- 'browse/src/cli.ts'
- 'browse/src/find-browse.ts'
- 'bin/gstack-paths'
- '.github/workflows/windows-setup-e2e.yml'
workflow_dispatch:
concurrency:
group: windows-setup-e2e-${{ github.head_ref }}
cancel-in-progress: true
jobs:
windows-setup:
runs-on: windows-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v1
with:
bun-version: latest
- name: Configure git identity
run: |
git config --global user.email "windows-setup-e2e@gstack.test"
git config --global user.name "Windows Setup E2E"
git config --global init.defaultBranch main
shell: bash
- name: Install dependencies
run: bun install --frozen-lockfile
shell: bash
- name: Run bun run build (the previously-broken path)
# This is the regression gate. Bun's Windows shell parser rejected
# multiple constructs the old inline build chain used; the wave
# moved the build to scripts/build.sh. If this step fails on
# Windows, the build chain regressed.
run: bun run build
shell: bash
env:
GSTACK_SKIP_PLAYWRIGHT: '1'
- name: Verify binaries exist (with .exe extension on Windows)
run: |
set -e
test -f browse/dist/browse.exe || test -f browse/dist/browse || (echo "MISSING: browse" && exit 1)
test -f browse/dist/find-browse.exe || test -f browse/dist/find-browse || (echo "MISSING: find-browse" && exit 1)
test -f design/dist/design.exe || test -f design/dist/design || (echo "MISSING: design" && exit 1)
test -f bin/gstack-global-discover.exe || test -f bin/gstack-global-discover || (echo "MISSING: gstack-global-discover" && exit 1)
echo "All binaries present"
shell: bash
- name: Verify find-browse resolves to the .exe variant
run: |
set -e
OUT=$(bun browse/src/find-browse.ts 2>&1) || true
echo "find-browse output: $OUT"
# On Windows, find-browse should successfully resolve to a binary,
# whether or not it has the .exe extension on disk. Empty output
# or "not found" means the .exe extension resolver regressed.
echo "$OUT" | grep -qE '(browse\.exe|browse)$' || (echo "find-browse failed to resolve binary on Windows" && exit 1)
shell: bash
- name: Verify gstack-paths state root resolves
run: |
set -e
eval "$(bash bin/gstack-paths)"
test -n "$GSTACK_STATE_ROOT" || (echo "GSTACK_STATE_ROOT empty" && exit 1)
test -n "$PLAN_ROOT" || (echo "PLAN_ROOT empty" && exit 1)
test -n "$TMP_ROOT" || (echo "TMP_ROOT empty" && exit 1)
echo "GSTACK_STATE_ROOT=$GSTACK_STATE_ROOT"
echo "PLAN_ROOT=$PLAN_ROOT"
echo "TMP_ROOT=$TMP_ROOT"
shell: bash