wehub-resource-sync
138 lines
4.4 KiB
Bash
Executable File
138 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# ci-native-eval.sh — Start showcase integrations natively (no Docker)
|
|
# for CI evaluation. Installs deps, starts dev servers + agents,
|
|
# waits for health, then runs showcase eval --ci.
|
|
#
|
|
# Usage: ci-native-eval.sh [--level d5] [--scope affected|all] [--parallel N]
|
|
#
|
|
# Environment:
|
|
# SHOWCASE_ROOT — path to showcase/ directory (default: auto-detect)
|
|
# AIMOCK_PORT — port for aimock (default: 4010)
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SHOWCASE_ROOT="${SHOWCASE_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
|
SHARED_DIR="$SHOWCASE_ROOT/shared"
|
|
INTEGRATIONS_DIR="$SHOWCASE_ROOT/integrations"
|
|
LOCAL_PORTS="$SHARED_DIR/local-ports.json"
|
|
|
|
AIMOCK_PORT="${AIMOCK_PORT:-4010}"
|
|
LEVEL="d5"
|
|
SCOPE="affected"
|
|
PARALLEL="8"
|
|
PIDS=()
|
|
|
|
# Parse args
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--level) LEVEL="$2"; shift 2 ;;
|
|
--scope) SCOPE="$2"; shift 2 ;;
|
|
--parallel) PARALLEL="$2"; shift 2 ;;
|
|
*) echo "Unknown arg: $1" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
cleanup() {
|
|
echo "[ci-native-eval] Tearing down..."
|
|
for pid in "${PIDS[@]}"; do
|
|
kill "$pid" 2>/dev/null || true
|
|
done
|
|
wait 2>/dev/null || true
|
|
echo "[ci-native-eval] Cleanup complete."
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# ── 1. Start aimock ──────────────────────────────────────────────────
|
|
echo "[ci-native-eval] Starting aimock on :${AIMOCK_PORT}..."
|
|
npx aimock --port "$AIMOCK_PORT" &
|
|
PIDS+=($!)
|
|
|
|
for i in $(seq 1 30); do
|
|
curl -sf "http://localhost:${AIMOCK_PORT}/health" >/dev/null 2>&1 && break || sleep 1
|
|
done
|
|
curl -sf "http://localhost:${AIMOCK_PORT}/health" >/dev/null 2>&1 || {
|
|
echo "[ci-native-eval] ERROR: aimock did not become healthy" >&2
|
|
exit 1
|
|
}
|
|
echo "[ci-native-eval] aimock healthy."
|
|
|
|
# ── 2. Read slug->port mapping ──────────────────────────────────────
|
|
SLUGS=$(node -e "console.log(Object.keys(require('$LOCAL_PORTS')).join(' '))")
|
|
|
|
# ── 3. Install deps + start services ────────────────────────────────
|
|
for slug in $SLUGS; do
|
|
port=$(node -e "console.log(require('$LOCAL_PORTS')['$slug'])")
|
|
agent_port=$((port + 100))
|
|
slug_dir="$INTEGRATIONS_DIR/$slug"
|
|
|
|
[ -d "$slug_dir" ] || continue
|
|
echo "[ci-native-eval] Starting $slug (frontend :$port, agent :$agent_port)..."
|
|
|
|
cd "$slug_dir"
|
|
|
|
# Install Node deps
|
|
if [ -f package.json ]; then
|
|
pnpm install --ignore-scripts 2>/dev/null || npm install --legacy-peer-deps 2>/dev/null || true
|
|
fi
|
|
|
|
# Install Python deps + start agent
|
|
if [ -f requirements.txt ]; then
|
|
pip install -r requirements.txt --prefer-binary -q 2>/dev/null || true
|
|
PYTHONPATH=. \
|
|
OPENAI_BASE_URL="http://localhost:${AIMOCK_PORT}/v1" \
|
|
OPENAI_API_KEY="aimock" \
|
|
python -m uvicorn agent_server:app \
|
|
--host 127.0.0.1 --port "$agent_port" \
|
|
--log-level warning &
|
|
PIDS+=($!)
|
|
fi
|
|
|
|
# Start Next.js dev server
|
|
PORT="$port" \
|
|
NEXT_PUBLIC_COPILOTKIT_URL="http://localhost:${agent_port}/api/copilotkit" \
|
|
npx next dev --turbopack --port "$port" &
|
|
PIDS+=($!)
|
|
|
|
cd "$SHOWCASE_ROOT"
|
|
done
|
|
|
|
# ── 4. Health-wait ───────────────────────────────────────────────────
|
|
echo "[ci-native-eval] Waiting for services to become healthy..."
|
|
HEALTHY=0
|
|
TOTAL=0
|
|
for slug in $SLUGS; do
|
|
port=$(node -e "console.log(require('$LOCAL_PORTS')['$slug'])")
|
|
slug_dir="$INTEGRATIONS_DIR/$slug"
|
|
[ -d "$slug_dir" ] || continue
|
|
TOTAL=$((TOTAL + 1))
|
|
|
|
ok=false
|
|
for i in $(seq 1 120); do
|
|
if curl -sf "http://localhost:$port" >/dev/null 2>&1; then
|
|
ok=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
if $ok; then
|
|
echo "[ci-native-eval] + $slug :$port"
|
|
HEALTHY=$((HEALTHY + 1))
|
|
else
|
|
echo "[ci-native-eval] x $slug :$port (timeout)"
|
|
fi
|
|
done
|
|
echo "[ci-native-eval] ${HEALTHY}/${TOTAL} services healthy."
|
|
|
|
# ── 5. Run eval ──────────────────────────────────────────────────────
|
|
echo "[ci-native-eval] Running showcase eval..."
|
|
"$SHOWCASE_ROOT/bin/showcase" eval \
|
|
"--$LEVEL" \
|
|
--scope "$SCOPE" \
|
|
--parallel "$PARALLEL" \
|
|
--json \
|
|
--baseline compare \
|
|
--timeout 60000 \
|
|
--ci
|