wehub-resource-sync
79 lines
3.9 KiB
Bash
79 lines
3.9 KiB
Bash
#!/usr/bin/env bash
|
|
# Sufficiency A/B for an IMPLEMENTATION task (the agent edits): when it uses
|
|
# codegraph (explore/node) to understand before editing, does it still Read? Like
|
|
# ab-sufficiency.sh but copies+indexes a FRESH target per run (the agent mutates
|
|
# it), so runs don't see each other's edits.
|
|
#
|
|
# WITH codegraph (pre-warmed) vs WITHOUT (empty MCP), N runs each. Reports
|
|
# explore/node vs Read/Grep + the files Read, and whether the build still passes.
|
|
#
|
|
# Usage: ab-impl.sh <indexed-repo> "<task>" [runs] [build-cmd]
|
|
# Env: AGENT_EVAL_OUT (default: /tmp/ab-impl)
|
|
set -uo pipefail
|
|
REPO="${1:?usage: ab-impl.sh <indexed-repo> \"<task>\" [runs] [build-cmd]}"
|
|
Q="${2:?task required}"
|
|
RUNS="${3:-2}"
|
|
BUILD_CMD="${4:-}"
|
|
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
BIN="$ENGINE/dist/bin/codegraph.js"
|
|
OUT="${AGENT_EVAL_OUT:-/tmp/ab-impl}"
|
|
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
|
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
|
|
cleanup(){ pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
|
|
trap cleanup EXIT
|
|
mkdir -p "$OUT"
|
|
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built engine"
|
|
echo "###### repo=$REPO runs/arm=$RUNS"
|
|
echo "###### task=$Q"; echo
|
|
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
|
|
|
prewarm(){
|
|
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
|
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
|
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
analyze(){
|
|
node -e '
|
|
const fs=require("fs");
|
|
const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
|
let ex=0,nf=0,ns=0,oc=0,gr=0,ed=0,exposed="?";const reads=[];
|
|
for(const l of L){try{const o=JSON.parse(l);
|
|
if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;
|
|
for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
|
|
if(b.name==="mcp__codegraph__codegraph_explore")ex++;
|
|
else if(b.name==="mcp__codegraph__codegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
|
|
else if(/mcp__codegraph__/.test(b.name))oc++;
|
|
else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
|
|
else if(b.name==="Grep")gr++;
|
|
else if(b.name==="Edit"||b.name==="Write")ed++;
|
|
}}catch{}}
|
|
console.log(` explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} Edit=${ed} [cg exposed=${exposed}]`);
|
|
' "$1"
|
|
}
|
|
|
|
run(){ # label, withCodegraph(0/1)
|
|
local label="$1" wcg="$2"
|
|
for i in $(seq 1 "$RUNS"); do
|
|
local tgt="$OUT/t-$label-$i" cfg="$OUT/mcp-$label.json"
|
|
rm -rf "$tgt"
|
|
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$REPO/" "$tgt/"
|
|
node "$BIN" init "$tgt" >/dev/null 2>&1
|
|
if [ "$wcg" = "1" ]; then
|
|
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$cfg"
|
|
prewarm "$tgt"
|
|
else cp "$OUT/mcp-empty.json" "$cfg"; fi
|
|
( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
|
|
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
|
--strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
|
|
echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
|
|
if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo " build: PASS" || echo " build: FAIL" ); fi
|
|
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
|
done
|
|
echo
|
|
}
|
|
|
|
echo "== WITH codegraph =="; run with 1
|
|
echo "== WITHOUT (Read/Grep only) =="; run without 0
|
|
echo "###### DONE: $OUT"
|