wehub-resource-sync
29 lines
1.3 KiB
Bash
29 lines
1.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Re-run the README "Benchmark Results" A/B (with vs without codegraph) on the
|
|
# current build: the 7 README repos, same queries, RUNS per arm (default 4).
|
|
# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
|
|
# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
|
|
# $CORPUS (default /tmp/codegraph-corpus) by the build under test.
|
|
set -uo pipefail
|
|
H="$(cd "$(dirname "$0")" && pwd)"
|
|
C="${CORPUS:-/tmp/codegraph-corpus}"
|
|
RUNS="${RUNS:-4}"
|
|
ROWS=(
|
|
"vscode|How does the extension host communicate with the main process?"
|
|
"excalidraw|How does Excalidraw render and update canvas elements?"
|
|
"django|How does Django's ORM build and execute a query from a QuerySet?"
|
|
"tokio|How does tokio schedule and run async tasks on its runtime?"
|
|
"okhttp|How does OkHttp process a request through its interceptor chain?"
|
|
"gin|How does gin route requests through its middleware chain?"
|
|
"alamofire|How does Alamofire build, send, and validate a request?"
|
|
)
|
|
echo "### README A/B START $(date) RUNS=$RUNS"
|
|
for row in "${ROWS[@]}"; do
|
|
repo="${row%%|*}"; q="${row#*|}"
|
|
echo "===== $repo ====="
|
|
for run in $(seq 1 "$RUNS"); do
|
|
AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo " run$run: (no exit line)"
|
|
done
|
|
done
|
|
echo "### README A/B DONE $(date)"
|