chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Does the agent PICK codegraph_node to read a file, vs the built-in Read tool?
|
||||
# Build A/B: NEW build (HEAD, codegraph_node has Read parity) vs BASELINE build
|
||||
# (a ref where it doesn't), BOTH codegraph-attached + pre-warmed, same task. The
|
||||
# metric is tool CHOICE: Read calls vs codegraph_node[file] calls per run.
|
||||
#
|
||||
# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
|
||||
set -uo pipefail
|
||||
TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
|
||||
TASK="${2:?task required}"
|
||||
RUNS="${3:-2}"
|
||||
BASE_REF="${4:-HEAD~1}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
|
||||
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "target not indexed: run 'codegraph init $TARGET' first"; exit 1; }
|
||||
git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
|
||||
CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
|
||||
[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
|
||||
|
||||
cleanup() {
|
||||
pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
|
||||
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$OUT"
|
||||
echo "###### target=$TARGET runs/arm=$RUNS baseline=$BASE_REF"
|
||||
echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
||||
echo "###### task=$TASK"; echo
|
||||
|
||||
prewarm() {
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Per-run tool-choice counts: Read vs codegraph_node[file] vs [symbol].
|
||||
count() {
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
||||
let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
|
||||
for(const l of lines){try{const o=JSON.parse(l);
|
||||
if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;}
|
||||
const blocks=o.message?.content||[];
|
||||
for(const b of (Array.isArray(blocks)?blocks:[])){
|
||||
if(b.type!=="tool_use")continue;
|
||||
if(b.name==="Read")read++;
|
||||
else if(b.name==="mcp__codegraph__codegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
|
||||
else if(/mcp__codegraph__/.test(b.name))cgOther++;
|
||||
}
|
||||
}catch{}}
|
||||
console.log(` Read=${read} codegraph_node[file]=${cgFile} codegraph_node[symbol]=${cgSym} other_cg=${cgOther} (cg exposed=${exposed})`);
|
||||
' "$1"
|
||||
}
|
||||
|
||||
run_arm() { # label, N
|
||||
local label="$1" n="$2"
|
||||
local c="$OUT/mcp-$label.json"
|
||||
for i in $(seq 1 "$n"); do
|
||||
local tgt="$OUT/t-$label-$i"
|
||||
rm -rf "$tgt"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$TARGET/" "$tgt/"
|
||||
node "$BIN" init "$tgt" >/dev/null 2>&1
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
echo "----- [$label] run $i -----"
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
||||
count "$OUT/run-$label-$i.jsonl"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
echo "== NEW build (HEAD: codegraph_node has Read parity) =="
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
run_arm new "$RUNS"
|
||||
|
||||
echo "== BASELINE build ($BASE_REF) =="
|
||||
git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
run_arm baseline "$RUNS"
|
||||
|
||||
echo "###### DONE — compare [new] vs [baseline]: does codegraph_node[file] rise / Read fall? Logs: $OUT"
|
||||
Reference in New Issue
Block a user