chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Add-lang benchmark for ONE repo:
|
||||
# clone -> wipe+index (with the codegraph on PATH) -> verify extraction ->
|
||||
# with/without retrieval A/B (reuses scripts/agent-eval/run-all.sh).
|
||||
#
|
||||
# Assumes the codegraph dev build is already built + linked on PATH — the skill
|
||||
# runs `npm run build && ./scripts/local-install.sh` ONCE before looping repos.
|
||||
# The A/B is skipped if extraction fails its critical checks (don't burn $ on a
|
||||
# broken extractor); set FORCE_AB=1 to run it anyway.
|
||||
#
|
||||
# Usage: bench.sh <lang> <repo-name> <repo-url> "<question>" [headless|tmux|all]
|
||||
# Env: CORPUS corpus dir (default /tmp/codegraph-corpus, shared with agent-eval)
|
||||
set -uo pipefail
|
||||
|
||||
LANG_TOKEN="${1:?usage: bench.sh <lang> <repo-name> <repo-url> \"<question>\" [mode]}"
|
||||
NAME="${2:?repo-name required}"
|
||||
URL="${3:?repo-url required}"
|
||||
Q="${4:?question required}"
|
||||
MODE="${5:-headless}"
|
||||
|
||||
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
||||
AGENT_EVAL="$(cd "$HARNESS/../agent-eval" && pwd)"
|
||||
CORPUS="${CORPUS:-/tmp/codegraph-corpus}"
|
||||
REPO="$CORPUS/$NAME"
|
||||
|
||||
command -v codegraph >/dev/null || { echo "no codegraph on PATH (build + ./scripts/local-install.sh first)"; exit 1; }
|
||||
|
||||
echo "==================== add-lang bench: $NAME ($LANG_TOKEN) ===================="
|
||||
echo "codegraph: $(command -v codegraph) -> $(codegraph --version 2>/dev/null || echo '?')"
|
||||
|
||||
# 1. Ensure the repo (shallow clone, reuse if present).
|
||||
mkdir -p "$CORPUS"
|
||||
if [ -d "$REPO/.git" ]; then
|
||||
echo "→ reusing checkout: $REPO"
|
||||
else
|
||||
echo "→ cloning $URL"
|
||||
git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
|
||||
fi
|
||||
|
||||
# 2. Wipe + index with the binary under test.
|
||||
echo "→ wiping .codegraph and indexing"
|
||||
rm -rf "$REPO/.codegraph"
|
||||
( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; }
|
||||
|
||||
# 3. Verify extraction (cheap guard before the paid A/B).
|
||||
echo "→ verifying extraction"
|
||||
node "$HARNESS/verify-extraction.mjs" "$REPO" "$LANG_TOKEN"
|
||||
VERIFY=$?
|
||||
|
||||
# 4. Retrieval A/B (skipped if extraction is broken, unless FORCE_AB=1).
|
||||
if [ "$VERIFY" -ne 0 ] && [ "${FORCE_AB:-0}" != "1" ]; then
|
||||
echo "→ SKIPPING A/B — extraction failed critical checks (set FORCE_AB=1 to override)"
|
||||
else
|
||||
echo "→ retrieval A/B (mode=$MODE)"
|
||||
bash "$AGENT_EVAL/run-all.sh" "$REPO" "$Q" "$MODE"
|
||||
fi
|
||||
|
||||
echo "==================== bench complete: $NAME (verify exit=$VERIFY) ===================="
|
||||
# Exit reflects extraction: 0 = pass/warn, 1 = critical fail, 2 = couldn't read status.
|
||||
exit "$VERIFY"
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env node
|
||||
// Verify a tree-sitter grammar wasm is HEALTHY under the project's web-tree-sitter
|
||||
// runtime BEFORE writing an extractor. Prints the ABI version and parses a valid
|
||||
// sample many times in a multi-grammar context, to catch heap-corruption bugs
|
||||
// that silently drop nodes on every parse after the first.
|
||||
//
|
||||
// Why this exists: the tree-sitter-wasms Lua grammar is ABI 13 and corrupts the
|
||||
// shared WASM heap under web-tree-sitter 0.25 — Lua extraction degraded on every
|
||||
// file after the first (nested calls/imports vanished). The fix was to vendor the
|
||||
// upstream ABI-15 wasm. Run this on any new grammar first; if it FAILs, vendor a
|
||||
// newer build instead of using the tree-sitter-wasms one.
|
||||
//
|
||||
// Usage: node scripts/add-lang/check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]
|
||||
// Exit: 0 healthy, 1 corruption / parse errors, 2 could not run.
|
||||
// NOTE: the sample must be SYNTACTICALLY VALID — a broken sample fails for the
|
||||
// wrong reason.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import { Parser, Language } from 'web-tree-sitter';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const fail = (code, msg) => { console.error(`[check-grammar] ${msg}`); process.exit(code); };
|
||||
|
||||
const [token, sample, iterArg] = process.argv.slice(2);
|
||||
if (!token || !sample) fail(2, 'usage: check-grammar.mjs <lang|wasm-path> <valid-sample> [iterations]');
|
||||
if (!existsSync(sample)) fail(2, `sample not found: ${sample}`);
|
||||
const iters = iterArg ? parseInt(iterArg, 10) : 20;
|
||||
|
||||
const SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
|
||||
function resolveWasm(t) {
|
||||
if (t.endsWith('.wasm')) return existsSync(t) ? t : fail(2, `wasm not found: ${t}`);
|
||||
const base = SPECIAL[t.toLowerCase()] ?? t.toLowerCase();
|
||||
try { return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`); } catch { /* try vendored */ }
|
||||
const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
|
||||
if (existsSync(vendored)) return vendored;
|
||||
return fail(2, `no grammar for "${t}" — not in tree-sitter-wasms and not vendored`);
|
||||
}
|
||||
|
||||
const wasmPath = resolveWasm(token);
|
||||
const source = readFileSync(sample, 'utf8');
|
||||
|
||||
try { await Parser.init(); }
|
||||
catch { await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') }); }
|
||||
|
||||
// Load a second, known-good grammar — the corruption surfaces under the
|
||||
// multi-grammar runtime that real indexing uses, not a single grammar in isolation.
|
||||
try { await Language.load(require.resolve('tree-sitter-wasms/out/tree-sitter-python.wasm')); } catch { /* ok */ }
|
||||
|
||||
let language;
|
||||
try { language = await Language.load(wasmPath); }
|
||||
catch (e) { fail(2, `failed to load ${wasmPath}: ${e.message}`); }
|
||||
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(language);
|
||||
|
||||
let ok = 0, err = 0;
|
||||
for (let i = 0; i < iters; i++) {
|
||||
const tree = parser.parse(source);
|
||||
if (tree.rootNode.hasError) err++; else ok++;
|
||||
}
|
||||
|
||||
console.log(`grammar: ${wasmPath.split('/').pop()}`);
|
||||
console.log(` ABI version: ${language.abiVersion}`);
|
||||
console.log(` parses: ${ok} clean / ${err} with errors (of ${iters})`);
|
||||
if (err > 0) {
|
||||
console.log(
|
||||
`RESULT: FAIL — ${err}/${iters} parses produced ERROR trees on a valid sample. ` +
|
||||
`This grammar corrupts under web-tree-sitter; vendor a newer (ABI 14/15) wasm ` +
|
||||
`(see SKILL.md "Find a grammar"). Confirm your sample is syntactically valid first.`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('RESULT: PASS — grammar parses cleanly and reuses safely.');
|
||||
process.exit(0);
|
||||
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env node
|
||||
// Dump the tree-sitter AST for a sample file so you can write a LanguageExtractor
|
||||
// mapping. Loads a grammar .wasm directly via web-tree-sitter (the same runtime
|
||||
// codegraph uses) — you do NOT need to register the language first.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/add-lang/dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]
|
||||
// Examples:
|
||||
// node scripts/add-lang/dump-ast.mjs lua sample.lua
|
||||
// node scripts/add-lang/dump-ast.mjs src/extraction/wasm/tree-sitter-zig.wasm a.zig --depth=4
|
||||
//
|
||||
// Output: an indented AST (named nodes, with field names) followed by a
|
||||
// node-type FREQUENCY table. The frequency table is the payoff — it tells you
|
||||
// which node types to map to functionTypes / classTypes / importTypes / etc.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import { Parser, Language } from 'web-tree-sitter';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const fail = (msg) => { console.error(`[dump-ast] ${msg}`); process.exit(1); };
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const positional = argv.filter((a) => !a.startsWith('--'));
|
||||
const [langOrWasm, sampleFile] = positional;
|
||||
const depthFlag = argv.find((a) => a.startsWith('--depth='));
|
||||
const showAll = argv.includes('--full'); // also print anonymous (token) nodes
|
||||
const maxDepth = depthFlag ? parseInt(depthFlag.split('=')[1], 10) : (showAll ? Infinity : 8);
|
||||
|
||||
if (!langOrWasm || !sampleFile) {
|
||||
fail('usage: dump-ast.mjs <lang|wasm-path> <sample-file> [--depth=N] [--full]');
|
||||
}
|
||||
if (!existsSync(sampleFile)) fail(`sample file not found: ${sampleFile}`);
|
||||
|
||||
// Language tokens whose tree-sitter-wasms filename differs from the token.
|
||||
const WASM_SPECIAL = { csharp: 'c_sharp', 'c#': 'c_sharp' };
|
||||
|
||||
function resolveWasm(token) {
|
||||
if (token.endsWith('.wasm')) {
|
||||
if (!existsSync(token)) fail(`wasm not found: ${token}`);
|
||||
return token;
|
||||
}
|
||||
const base = WASM_SPECIAL[token.toLowerCase()] ?? token.toLowerCase();
|
||||
try {
|
||||
return require.resolve(`tree-sitter-wasms/out/tree-sitter-${base}.wasm`);
|
||||
} catch {
|
||||
/* not in tree-sitter-wasms — try a vendored copy */
|
||||
}
|
||||
const vendored = `src/extraction/wasm/tree-sitter-${base}.wasm`;
|
||||
if (existsSync(vendored)) return vendored;
|
||||
fail(
|
||||
`no grammar for "${token}" — not in tree-sitter-wasms and not vendored at ` +
|
||||
`${vendored}. Pass an explicit .wasm path, or vendor one (see SKILL.md "Find a grammar").`
|
||||
);
|
||||
}
|
||||
|
||||
const wasmPath = resolveWasm(langOrWasm);
|
||||
const source = readFileSync(sampleFile, 'utf8');
|
||||
|
||||
try {
|
||||
await Parser.init();
|
||||
} catch {
|
||||
await Parser.init({ locateFile: () => require.resolve('web-tree-sitter/tree-sitter.wasm') });
|
||||
}
|
||||
|
||||
let language;
|
||||
try {
|
||||
language = await Language.load(wasmPath);
|
||||
} catch (e) {
|
||||
fail(`failed to load grammar ${wasmPath}: ${e.message}`);
|
||||
}
|
||||
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(language);
|
||||
const tree = parser.parse(source);
|
||||
|
||||
const freq = new Map();
|
||||
const snippet = (node) => {
|
||||
const t = node.text.replace(/\s+/g, ' ').trim();
|
||||
return t.length > 48 ? `${t.slice(0, 48)}…` : t;
|
||||
};
|
||||
|
||||
function walk(node, depth, fieldName) {
|
||||
if (node.isNamed) freq.set(node.type, (freq.get(node.type) || 0) + 1);
|
||||
if ((node.isNamed || showAll) && depth <= maxDepth) {
|
||||
const field = fieldName ? `${fieldName}: ` : '';
|
||||
const leaf = node.childCount === 0 ? ` "${snippet(node)}"` : '';
|
||||
console.log(`${' '.repeat(depth)}${field}${node.type} @${node.startPosition.row + 1}:${node.startPosition.column}${leaf}`);
|
||||
}
|
||||
for (let i = 0; i < node.childCount; i++) {
|
||||
const child = node.child(i);
|
||||
if (child) walk(child, depth + 1, node.fieldNameForChild(i));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n# AST for ${sampleFile} (grammar: ${wasmPath.split('/').pop()})\n`);
|
||||
walk(tree.rootNode, 0, null);
|
||||
|
||||
console.log('\n# Node-type frequency (named nodes) — map the relevant ones in your extractor:\n');
|
||||
[...freq.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.forEach(([type, n]) => console.log(` ${String(n).padStart(5)} ${type}`));
|
||||
console.log();
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env node
|
||||
// Sanity-check that codegraph extracted REAL symbols (not just file/import nodes)
|
||||
// from a repo for a given language. Exits non-zero on a critical failure so it
|
||||
// can drive a write-extractor -> build -> re-check loop.
|
||||
//
|
||||
// Usage: node scripts/add-lang/verify-extraction.mjs <repo-path> <lang>
|
||||
// Reads `codegraph status <repo> --json` using whatever codegraph is on PATH,
|
||||
// so it reflects the binary that built the index.
|
||||
//
|
||||
// Exit codes: 0 = pass or soft-warn, 1 = critical fail, 2 = could not run.
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const [repo, lang] = process.argv.slice(2);
|
||||
if (!repo || !lang) {
|
||||
console.error('usage: verify-extraction.mjs <repo-path> <lang>');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
let status;
|
||||
try {
|
||||
const out = execFileSync('codegraph', ['status', repo, '--json'], { encoding: 'utf8' });
|
||||
status = JSON.parse(out);
|
||||
} catch (e) {
|
||||
console.error(`[verify] could not read codegraph status for ${repo}: ${e.message}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Kinds that prove the extractor mapped AST node types (everything except
|
||||
// 'file' and 'import', which codegraph creates structurally for any language).
|
||||
const SYMBOL_KINDS = new Set([
|
||||
'module', 'class', 'struct', 'interface', 'trait', 'protocol', 'function',
|
||||
'method', 'property', 'field', 'variable', 'constant', 'enum', 'enum_member',
|
||||
'type_alias', 'namespace', 'route', 'component',
|
||||
]);
|
||||
|
||||
const byKind = status.nodesByKind || {};
|
||||
const langs = status.languages || [];
|
||||
const files = status.fileCount || 0;
|
||||
const edges = status.edgeCount || 0;
|
||||
const symbolKinds = Object.keys(byKind).filter((k) => SYMBOL_KINDS.has(k));
|
||||
const symbolCount = symbolKinds.reduce((s, k) => s + byKind[k], 0);
|
||||
|
||||
const checks = [];
|
||||
const add = (severity, ok, label, detail) => checks.push({ severity, ok, label, detail });
|
||||
|
||||
add('critical', status.initialized === true, 'index initialized', `initialized=${status.initialized}`);
|
||||
add('critical', langs.includes(lang), `language "${lang}" detected`, `languages=[${langs.join(', ')}]`);
|
||||
add('critical', symbolCount > 0, 'structural symbols extracted', `${symbolCount} symbols (${symbolKinds.join(', ') || 'NONE — only file/import nodes!'})`);
|
||||
add('soft', symbolCount >= files, 'symbol density >= 1/file', `${symbolCount} symbols across ${files} files`);
|
||||
add('soft', edges > files, 'edges resolved', `${edges} edges across ${files} files`);
|
||||
|
||||
console.log(`\n# Extraction check — ${repo} (lang=${lang}, backend=${status.backend})`);
|
||||
console.log(` files=${files} nodes=${status.nodeCount} edges=${edges}`);
|
||||
console.log(` nodesByKind: ${JSON.stringify(byKind)}\n`);
|
||||
for (const c of checks) console.log(` ${c.ok ? '✓' : '✗'} ${c.label} — ${c.detail}`);
|
||||
|
||||
const critical = checks.filter((c) => !c.ok && c.severity === 'critical');
|
||||
const soft = checks.filter((c) => !c.ok && c.severity === 'soft');
|
||||
console.log();
|
||||
if (critical.length) {
|
||||
console.log(`RESULT: FAIL (${critical.length} critical) — extractor or grammar wiring is broken. Re-run dump-ast.mjs and fix the node-type mappings.`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (soft.length) {
|
||||
console.log(`RESULT: WARN (${soft.length} soft) — extraction works but looks thin; inspect the counts above.`);
|
||||
process.exit(0);
|
||||
}
|
||||
console.log('RESULT: PASS — extraction looks healthy.');
|
||||
process.exit(0);
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# Does the agent PICK codegraph_node to read a file, vs the built-in Read tool?
|
||||
# Build A/B: NEW build (HEAD, codegraph_node has Read parity) vs BASELINE build
|
||||
# (a ref where it doesn't), BOTH codegraph-attached + pre-warmed, same task. The
|
||||
# metric is tool CHOICE: Read calls vs codegraph_node[file] calls per run.
|
||||
#
|
||||
# Usage: ab-adoption.sh <indexed-repo> "<task>" [runs-per-arm] [baseline-ref]
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-adoption)
|
||||
set -uo pipefail
|
||||
TARGET="${1:?usage: ab-adoption.sh <indexed-repo> \"<task>\" [runs] [baseline-ref]}"
|
||||
TASK="${2:?task required}"
|
||||
RUNS="${3:-2}"
|
||||
BASE_REF="${4:-HEAD~1}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-adoption}"
|
||||
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "target not indexed: run 'codegraph init $TARGET' first"; exit 1; }
|
||||
git -C "$ENGINE" diff --quiet && git -C "$ENGINE" diff --cached --quiet || { echo "engine has uncommitted changes — commit/stash first"; exit 1; }
|
||||
CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
|
||||
[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD"; exit 1; }
|
||||
|
||||
cleanup() {
|
||||
pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
|
||||
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$OUT"
|
||||
echo "###### target=$TARGET runs/arm=$RUNS baseline=$BASE_REF"
|
||||
echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
||||
echo "###### task=$TASK"; echo
|
||||
|
||||
prewarm() {
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Per-run tool-choice counts: Read vs codegraph_node[file] vs [symbol].
|
||||
count() {
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
const lines=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
||||
let read=0,cgFile=0,cgSym=0,cgOther=0,exposed="?";
|
||||
for(const l of lines){try{const o=JSON.parse(l);
|
||||
if(o.type==="system"&&o.subtype==="init"){exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;}
|
||||
const blocks=o.message?.content||[];
|
||||
for(const b of (Array.isArray(blocks)?blocks:[])){
|
||||
if(b.type!=="tool_use")continue;
|
||||
if(b.name==="Read")read++;
|
||||
else if(b.name==="mcp__codegraph__codegraph_node"){ if(b.input&&b.input.symbol)cgSym++; else cgFile++; }
|
||||
else if(/mcp__codegraph__/.test(b.name))cgOther++;
|
||||
}
|
||||
}catch{}}
|
||||
console.log(` Read=${read} codegraph_node[file]=${cgFile} codegraph_node[symbol]=${cgSym} other_cg=${cgOther} (cg exposed=${exposed})`);
|
||||
' "$1"
|
||||
}
|
||||
|
||||
run_arm() { # label, N
|
||||
local label="$1" n="$2"
|
||||
local c="$OUT/mcp-$label.json"
|
||||
for i in $(seq 1 "$n"); do
|
||||
local tgt="$OUT/t-$label-$i"
|
||||
rm -rf "$tgt"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$TARGET/" "$tgt/"
|
||||
node "$BIN" init "$tgt" >/dev/null 2>&1
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
echo "----- [$label] run $i -----"
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label-$i.jsonl" 2>"$OUT/run-$label-$i.err" )
|
||||
count "$OUT/run-$label-$i.jsonl"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
echo "== NEW build (HEAD: codegraph_node has Read parity) =="
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
run_arm new "$RUNS"
|
||||
|
||||
echo "== BASELINE build ($BASE_REF) =="
|
||||
git -C "$ENGINE" checkout "$BASE_REF" -- $CHANGED
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
run_arm baseline "$RUNS"
|
||||
|
||||
echo "###### DONE — compare [new] vs [baseline]: does codegraph_node[file] rise / Read fall? Logs: $OUT"
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
# A/B the PreToolUse(Read) REDIRECT hook (P1): does steering Read → codegraph_node
|
||||
# file-view actually move the agent off Read during implementation? BOTH arms use
|
||||
# the CURRENT build with codegraph attached and pre-warmed; the only difference is
|
||||
# the hook. Isolates the hook's behavioral effect from the build/file-view change
|
||||
# (use ab-new-vs-baseline.sh for the build A/B).
|
||||
#
|
||||
# arm [nohook] — codegraph on, no hook (does the better file-view get picked on its own?)
|
||||
# arm [hook] — codegraph on, + redirect hook (does routing close it?)
|
||||
#
|
||||
# Reliable attach (works nested): each arm pre-warms a persistent daemon and skips
|
||||
# the startup re-exec (CODEGRAPH_WASM_RELAUNCHED=1), so claude connects before the
|
||||
# agent's first turn. Judge by ACTUAL codegraph usage in parse-run.mjs's "by type",
|
||||
# not claude's init snapshot (which can read pending even when it then connects).
|
||||
#
|
||||
# Usage: ab-hook.sh <indexed-repo> "<implementation task>" [runs-per-arm]
|
||||
# <indexed-repo> a repo with a .codegraph index (copied per arm; never mutated)
|
||||
# "<task>" a GENUINELY-NEW implementation task (verify it isn't already done)
|
||||
# [runs-per-arm] default 2 (n=1 is noisy — the doctrine says >=2)
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-hook)
|
||||
set -uo pipefail
|
||||
|
||||
TARGET="${1:?usage: ab-hook.sh <indexed-repo> \"<task>\" [runs-per-arm]}"
|
||||
TASK="${2:?task required}"
|
||||
RUNS="${3:-2}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
HOOK="$ENGINE/scripts/agent-eval/redirect-read-hook.sh"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-hook}"
|
||||
PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
|
||||
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
command -v jq >/dev/null || { echo "jq not on PATH (the hook needs it)"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "target not indexed: run 'codegraph init $TARGET' first"; exit 1; }
|
||||
chmod +x "$HOOK"
|
||||
|
||||
cleanup() { pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$OUT"
|
||||
echo "###### engine=$ENGINE"
|
||||
echo "###### target=$TARGET runs/arm=$RUNS"
|
||||
echo "###### task=$TASK"
|
||||
echo
|
||||
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
|
||||
# A settings file carrying ONLY the PreToolUse(Read) redirect hook.
|
||||
HOOK_SETTINGS="$OUT/hook-settings.json"
|
||||
jq -n --arg cmd "bash $HOOK" \
|
||||
'{hooks:{PreToolUse:[{matcher:"Read",hooks:[{type:"command",command:$cmd}]}]}}' > "$HOOK_SETTINGS"
|
||||
|
||||
prewarm() { # target — spawn a persistent daemon and wait for its socket
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
||||
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1"
|
||||
}
|
||||
|
||||
run_one() { # arm-label, run-index, use-hook(0|1)
|
||||
local label="$1" idx="$2" hook="$3"
|
||||
local tgt="$OUT/t-$label-$idx" c="$OUT/mcp-$label.json"
|
||||
rm -rf "$tgt"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$TARGET/" "$tgt/"
|
||||
node "$BIN" init "$tgt" >/dev/null 2>&1
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
local extra=()
|
||||
[ "$hook" = "1" ] && extra=(--settings "$HOOK_SETTINGS")
|
||||
echo "----- [$label] run $idx -----"
|
||||
# ${extra[@]+...} guard: bash 3.2 (macOS) under `set -u` errors on an empty
|
||||
# array expansion otherwise, which would skip the no-hook arm's claude run.
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" ${extra[@]+"${extra[@]}"} \
|
||||
</dev/null > "$OUT/run-$label-$idx.jsonl" 2>"$OUT/run-$label-$idx.err" )
|
||||
node "$PARSE" "$OUT/run-$label-$idx.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label-$idx.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
echo
|
||||
}
|
||||
|
||||
for i in $(seq 1 "$RUNS"); do run_one nohook "$i" 0; done
|
||||
for i in $(seq 1 "$RUNS"); do run_one hook "$i" 1; done
|
||||
|
||||
echo "###### DONE. Compare [nohook] vs [hook] 'by type' — Read should fall and"
|
||||
echo "###### mcp__codegraph__codegraph_node should rise in the [hook] arm. Logs: $OUT"
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sufficiency A/B for an IMPLEMENTATION task (the agent edits): when it uses
|
||||
# codegraph (explore/node) to understand before editing, does it still Read? Like
|
||||
# ab-sufficiency.sh but copies+indexes a FRESH target per run (the agent mutates
|
||||
# it), so runs don't see each other's edits.
|
||||
#
|
||||
# WITH codegraph (pre-warmed) vs WITHOUT (empty MCP), N runs each. Reports
|
||||
# explore/node vs Read/Grep + the files Read, and whether the build still passes.
|
||||
#
|
||||
# Usage: ab-impl.sh <indexed-repo> "<task>" [runs] [build-cmd]
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-impl)
|
||||
set -uo pipefail
|
||||
REPO="${1:?usage: ab-impl.sh <indexed-repo> \"<task>\" [runs] [build-cmd]}"
|
||||
Q="${2:?task required}"
|
||||
RUNS="${3:-2}"
|
||||
BUILD_CMD="${4:-}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-impl}"
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
|
||||
cleanup(){ pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null; }
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$OUT"
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built engine"
|
||||
echo "###### repo=$REPO runs/arm=$RUNS"
|
||||
echo "###### task=$Q"; echo
|
||||
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
||||
|
||||
prewarm(){
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
analyze(){
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
||||
let ex=0,nf=0,ns=0,oc=0,gr=0,ed=0,exposed="?";const reads=[];
|
||||
for(const l of L){try{const o=JSON.parse(l);
|
||||
if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;
|
||||
for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
|
||||
if(b.name==="mcp__codegraph__codegraph_explore")ex++;
|
||||
else if(b.name==="mcp__codegraph__codegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
|
||||
else if(/mcp__codegraph__/.test(b.name))oc++;
|
||||
else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
|
||||
else if(b.name==="Grep")gr++;
|
||||
else if(b.name==="Edit"||b.name==="Write")ed++;
|
||||
}}catch{}}
|
||||
console.log(` explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} Edit=${ed} [cg exposed=${exposed}]`);
|
||||
' "$1"
|
||||
}
|
||||
|
||||
run(){ # label, withCodegraph(0/1)
|
||||
local label="$1" wcg="$2"
|
||||
for i in $(seq 1 "$RUNS"); do
|
||||
local tgt="$OUT/t-$label-$i" cfg="$OUT/mcp-$label.json"
|
||||
rm -rf "$tgt"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$REPO/" "$tgt/"
|
||||
node "$BIN" init "$tgt" >/dev/null 2>&1
|
||||
if [ "$wcg" = "1" ]; then
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$cfg"
|
||||
prewarm "$tgt"
|
||||
else cp "$OUT/mcp-empty.json" "$cfg"; fi
|
||||
( cd "$tgt" && claude -p "$Q" --output-format stream-json --verbose \
|
||||
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
--strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
|
||||
echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
|
||||
if [ -n "$BUILD_CMD" ]; then ( cd "$tgt" && eval "$BUILD_CMD" >/dev/null 2>&1 && echo " build: PASS" || echo " build: FAIL" ); fi
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
echo "== WITH codegraph =="; run with 1
|
||||
echo "== WITHOUT (Read/Grep only) =="; run without 0
|
||||
echo "###### DONE: $OUT"
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# A/B a codegraph retrieval/steering change: the NEW build (current HEAD) vs a
|
||||
# BASELINE build (a git ref) — BOTH with codegraph attached — on the same
|
||||
# implementation task, measuring how many Read vs codegraph calls the agent
|
||||
# makes. ISOLATES the change (unlike run-all.sh's with-vs-without). The agent
|
||||
# works on a throwaway copy of the target, so your repos are never touched.
|
||||
#
|
||||
# Reliable attach (works even when this is itself run nested inside a Claude
|
||||
# session): each arm PRE-WARMS a persistent codegraph daemon for its target so
|
||||
# claude connects to an already-bound, index-loaded daemon instantly — before
|
||||
# the agent's first turn — and SKIPS codegraph's startup re-exec via
|
||||
# CODEGRAPH_WASM_RELAUNCHED=1. Without this, on a multi-step task the agent
|
||||
# dives into Read/grep before codegraph finishes its ~2-3s startup (worse under
|
||||
# the CPU contention of a nested run) and runs with NO codegraph.
|
||||
#
|
||||
# Gotcha: claude's `system/init` snapshot can read status:"pending" / 0 tools
|
||||
# even when the server then connects fine — judge by ACTUAL codegraph usage in
|
||||
# parse-run.mjs's "by type", not the init line.
|
||||
#
|
||||
# Usage: ab-new-vs-baseline.sh <indexed-repo> "<task>" [baseline-ref]
|
||||
# <indexed-repo> a repo with a .codegraph index (copied per arm)
|
||||
# "<task>" an implementation task, e.g. "Add X to Y and wire it through"
|
||||
# [baseline-ref] git ref for the BEFORE build (default: HEAD~1)
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-new-vs-baseline)
|
||||
set -uo pipefail
|
||||
|
||||
TARGET="${1:?usage: ab-new-vs-baseline.sh <indexed-repo> \"<task>\" [baseline-ref]}"
|
||||
TASK="${2:?task required}"
|
||||
BASE_REF="${3:-HEAD~1}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-new-vs-baseline}"
|
||||
PARSE="$ENGINE/scripts/agent-eval/parse-run.mjs"
|
||||
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "target not indexed: run 'codegraph init $TARGET' first"; exit 1; }
|
||||
if ! git -C "$ENGINE" diff --quiet || ! git -C "$ENGINE" diff --cached --quiet; then
|
||||
echo "engine repo has uncommitted changes — commit or stash first (this script checks files out)"; exit 1
|
||||
fi
|
||||
CHANGED=$(git -C "$ENGINE" diff --name-only "$BASE_REF" HEAD -- src 2>/dev/null)
|
||||
[ -n "$CHANGED" ] || { echo "no src/ changes between $BASE_REF and HEAD — nothing to A/B"; exit 1; }
|
||||
|
||||
# On exit: kill any eval daemons + restore the engine to HEAD.
|
||||
cleanup() {
|
||||
pkill -9 -f "serve --mcp --path $OUT/" 2>/dev/null
|
||||
git -C "$ENGINE" checkout HEAD -- $CHANGED 2>/dev/null
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 )
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$OUT"
|
||||
echo "###### engine=$ENGINE baseline=$BASE_REF"
|
||||
echo "###### changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
||||
echo "###### target=$TARGET"
|
||||
echo "###### task=$TASK"
|
||||
echo
|
||||
|
||||
# Two pristine copies so each arm starts clean (the agent edits its own copy).
|
||||
rm -rf "$OUT/t-new" "$OUT/t-base"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$TARGET/" "$OUT/t-new/"
|
||||
cp -R "$OUT/t-new" "$OUT/t-base"
|
||||
|
||||
prewarm() { # target — spawn a persistent daemon (current $BIN) and wait for its socket
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
||||
&& echo " daemon warm: $1" || echo " WARN: daemon never bound for $1 (arm may run without codegraph)"
|
||||
}
|
||||
|
||||
run_arm() { # label, target-copy
|
||||
local label="$1" tgt="$2" c="$OUT/mcp-$1.json"
|
||||
# Connect to the pre-warmed daemon; skip the startup re-exec for a fast attach.
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$tgt" > "$c"
|
||||
prewarm "$tgt"
|
||||
echo "############## ARM [$label] ##############"
|
||||
( cd "$tgt" && claude -p "$TASK" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 --strict-mcp-config --mcp-config "$c" \
|
||||
</dev/null > "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
||||
node "$PARSE" "$OUT/run-$label.jsonl" 2>&1 | grep -E "by type|Result" || echo " (parse failed — see $OUT/run-$label.jsonl)"
|
||||
pkill -9 -f "serve --mcp --path $tgt" 2>/dev/null
|
||||
echo
|
||||
}
|
||||
|
||||
echo "== NEW build (HEAD) =="
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo " built"
|
||||
node "$BIN" init "$OUT/t-new" >/dev/null 2>&1 && echo " indexed t-new"
|
||||
run_arm new "$OUT/t-new"
|
||||
|
||||
echo "== BASELINE build ($BASE_REF) =="
|
||||
# Per-file: a file ADDED since baseline has no pathspec on the ref — and a
|
||||
# single multi-file checkout with one bad pathspec checks out NOTHING, which
|
||||
# silently ran the NEW build in the baseline arm. Absent-on-baseline → remove.
|
||||
for f in $CHANGED; do
|
||||
git -C "$ENGINE" checkout "$BASE_REF" -- "$f" 2>/dev/null || rm -f "$ENGINE/$f"
|
||||
done
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo " built"
|
||||
node "$BIN" init "$OUT/t-base" >/dev/null 2>&1 && echo " indexed t-base"
|
||||
run_arm baseline "$OUT/t-base"
|
||||
|
||||
echo "###### DONE. Compare the [new] vs [baseline] 'by type' counts above"
|
||||
echo "###### (especially Read vs mcp__codegraph__*). Full logs in: $OUT"
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sufficiency A/B: on a real understanding/flow question, WHEN the agent uses
|
||||
# codegraph (explore/node), does it still Read? Premise under test: explore/node
|
||||
# return source WITH line numbers, so a Read should not be needed.
|
||||
#
|
||||
# WITH codegraph (pre-warmed daemon, reliable nested attach) vs WITHOUT (empty
|
||||
# MCP, Read/Grep only), N runs each, on a throwaway copy of the repo. Reports
|
||||
# explore/node vs Read/Grep, and LISTS the files Read in the WITH arm so a true
|
||||
# sufficiency gap (an indexed source file) is distinguishable from out-of-scope
|
||||
# (configs, docs, a file codegraph didn't index).
|
||||
#
|
||||
# Usage: ab-sufficiency.sh <indexed-repo> "<question>" [runs-per-arm]
|
||||
# Env: AGENT_EVAL_OUT (default: /tmp/ab-sufficiency)
|
||||
set -uo pipefail
|
||||
REPO="${1:?usage: ab-sufficiency.sh <indexed-repo> \"<question>\" [runs]}"
|
||||
Q="${2:?question required}"
|
||||
RUNS="${3:-2}"
|
||||
ENGINE="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/ab-sufficiency}"
|
||||
TGT="$OUT/target"
|
||||
command -v claude >/dev/null || { echo "claude CLI not on PATH"; exit 1; }
|
||||
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
|
||||
cleanup(){ pkill -9 -f "serve --mcp --path $TGT" 2>/dev/null; }
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$OUT"
|
||||
( cd "$ENGINE" && npm run build >/dev/null 2>&1 ) && echo "built"
|
||||
|
||||
# Throwaway copy + fresh index (the agent works here; a read-only question won't
|
||||
# edit, but isolate anyway). Excludes the source repo's index/build/vcs.
|
||||
rm -rf "$TGT"
|
||||
rsync -a --exclude node_modules --exclude .git --exclude dist --exclude .codegraph "$REPO/" "$TGT/"
|
||||
node "$BIN" init "$TGT" >/dev/null 2>&1 && echo "indexed copy ($(node "$BIN" status --json 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{console.log(JSON.parse(s).fileCount+" files")}catch{console.log("?")}})' 2>/dev/null || echo '?'))"
|
||||
|
||||
echo "###### repo=$REPO runs/arm=$RUNS"
|
||||
echo "###### Q=$Q"; echo
|
||||
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TGT" > "$OUT/mcp-cg.json"
|
||||
|
||||
prewarm(){
|
||||
pkill -9 -f "serve --mcp --path $TGT" 2>/dev/null
|
||||
CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$TGT" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$TGT" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
analyze(){
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
const L=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean);
|
||||
let ex=0,nf=0,ns=0,oc=0,gr=0,exposed="?";const reads=[];
|
||||
for(const l of L){try{const o=JSON.parse(l);
|
||||
if(o.type==="system"&&o.subtype==="init")exposed=(o.tools||[]).filter(t=>/codegraph/.test(t)).length;
|
||||
for(const b of (o.message?.content||[])){if(b.type!=="tool_use")continue;
|
||||
if(b.name==="mcp__codegraph__codegraph_explore")ex++;
|
||||
else if(b.name==="mcp__codegraph__codegraph_node"){if(b.input&&b.input.symbol)ns++;else nf++;}
|
||||
else if(/mcp__codegraph__/.test(b.name))oc++;
|
||||
else if(b.name==="Read")reads.push((b.input?.file_path||"").split("/").pop());
|
||||
else if(b.name==="Grep")gr++;
|
||||
}}catch{}}
|
||||
console.log(` explore=${ex} node[sym]=${ns} node[file]=${nf} other_cg=${oc} | Read=${reads.length}${reads.length?" ("+reads.join(", ")+")":""} Grep=${gr} [cg exposed=${exposed}]`);
|
||||
' "$1"
|
||||
}
|
||||
|
||||
run(){ # label, cfg, prewarm(0/1)
|
||||
local label="$1" cfg="$2" pw="$3"
|
||||
for i in $(seq 1 "$RUNS"); do
|
||||
[ "$pw" = "1" ] && prewarm
|
||||
( cd "$TGT" && claude -p "$Q" --output-format stream-json --verbose \
|
||||
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
--strict-mcp-config --mcp-config "$cfg" </dev/null > "$OUT/$label-$i.jsonl" 2>"$OUT/$label-$i.err" )
|
||||
echo "[$label] run $i:"; analyze "$OUT/$label-$i.jsonl"
|
||||
done
|
||||
echo
|
||||
}
|
||||
|
||||
echo "== WITH codegraph (premise: explore/node used -> Read ~0) =="; run with "$OUT/mcp-cg.json" 1
|
||||
echo "== WITHOUT (Read/Grep only — the contrast) =="; run without "$OUT/mcp-empty.json" 0
|
||||
echo "###### DONE. In the WITH arm: are explore/node>0 and Read~0? Any Read of an INDEXED source file = sufficiency gap. Logs: $OUT"
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Arm F (body-inlining trace + trace-first steering) across the same 6 repos as
|
||||
# arms-matrix.sh, so F vs B isolates the trace-enrichment effect (same surface,
|
||||
# old thin trace in B vs body-inlining trace here).
|
||||
set -uo pipefail
|
||||
H="$(cd "$(dirname "$0")" && pwd)"; RUNS="${RUNS:-2}"; C="${CORPUS:-/tmp/codegraph-corpus}"
|
||||
ROWS=(
|
||||
"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
|
||||
"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
|
||||
"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
|
||||
"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
|
||||
"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
|
||||
"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
|
||||
)
|
||||
ARM="${ARM:-F}"
|
||||
echo "### ARM $ARM START $(date) RUNS=$RUNS"
|
||||
for row in "${ROWS[@]}"; do
|
||||
repo="${row%%|*}"; q="${row#*|}"
|
||||
for r in $(seq 1 "$RUNS"); do bash "$H/run-arms.sh" "$repo" "$q" "$ARM" "$r"; done
|
||||
done
|
||||
echo "### ARM $ARM COMPLETE $(date)"
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive the tool-surface ablation across the chosen repos × arms (A–E).
|
||||
# Arms A–D ask the canonical FLOW question; arm E asks a NON-flow survey
|
||||
# question (the control probe — should degrade without explore+context).
|
||||
# Output: /tmp/arms/<repo>/<arm>-r<n>.jsonl (parse with parse-arms.mjs).
|
||||
set -uo pipefail
|
||||
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
||||
RUNS="${RUNS:-2}"
|
||||
C="${CORPUS:-/tmp/codegraph-corpus}"
|
||||
NFQ='What are the main modules/components of this codebase and what does each one do? Give an overview of how it is organized.'
|
||||
|
||||
# repo-path|flow-question (2 small, 2 medium, 2 large — spans the size range)
|
||||
ROWS=(
|
||||
"$C/flutter-samples/add_to_app/books/flutter_module_books|How does the books UI build and what child widgets does it show?"
|
||||
"$C/aspnet-realworld|How is creating an article handled? Trace the controller to the service."
|
||||
"$C/spring-mall|How is a product-list request handled? Trace the controller to the service."
|
||||
"$C/vapor-spi|How is a package-show request handled? Name the route and controller."
|
||||
"$C/excalidraw|How does updating an element re-render the canvas on screen? Trace the flow."
|
||||
"$C/spring-halo|How is publishing a post handled? Trace the controller to the service."
|
||||
)
|
||||
|
||||
echo "### ARMS MATRIX START $(date) RUNS=$RUNS"
|
||||
for row in "${ROWS[@]}"; do
|
||||
repo="${row%%|*}"; q="${row#*|}"
|
||||
for arm in A B C D; do
|
||||
for r in $(seq 1 "$RUNS"); do
|
||||
bash "$HARNESS/run-arms.sh" "$repo" "$q" "$arm" "$r"
|
||||
done
|
||||
done
|
||||
done
|
||||
# E: non-flow control probe on two repos (must degrade without explore+context)
|
||||
for repo in "$C/excalidraw" "$C/spring-mall"; do
|
||||
for r in $(seq 1 "$RUNS"); do
|
||||
bash "$HARNESS/run-arms.sh" "$repo" "$NFQ" E "$r"
|
||||
done
|
||||
done
|
||||
echo "### ARMS MATRIX COMPLETE $(date)"
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-shot CodeGraph quality audit:
|
||||
# set version -> ensure corpus repo -> wipe+reindex with that version ->
|
||||
# run with/without A/B -> restore the local dev link.
|
||||
#
|
||||
# Usage: audit.sh <version> <repo-name> <repo-url> "<question>" [headless|all]
|
||||
# <version> "local" (build + npm link this repo) | "latest" | a version (e.g. 0.7.10)
|
||||
# <repo-name> dir name under the corpus dir
|
||||
# <repo-url> git URL (cloned --depth 1 when the repo dir is missing)
|
||||
# [mode] headless (default) | all (also the interactive tmux arms)
|
||||
# Env: CORPUS corpus dir (default: /tmp/codegraph-corpus)
|
||||
set -uo pipefail
|
||||
|
||||
VERSION="${1:?usage: audit.sh <version> <repo-name> <repo-url> \"<question>\" [mode]}"
|
||||
NAME="${2:?repo-name required}"
|
||||
URL="${3:?repo-url required}"
|
||||
Q="${4:?question required}"
|
||||
MODE="${5:-headless}"
|
||||
|
||||
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$HARNESS/../.." && pwd)" # codegraph repo root
|
||||
CORPUS="${CORPUS:-/tmp/codegraph-corpus}"
|
||||
REPO="$CORPUS/$NAME"
|
||||
PKG="@colbymchenry/codegraph"
|
||||
|
||||
echo "==================== CodeGraph audit ===================="
|
||||
echo "version=$VERSION repo=$NAME mode=$MODE corpus=$CORPUS"
|
||||
echo
|
||||
|
||||
# 1. Set the codegraph version under test (mutates the global install).
|
||||
if [ "$VERSION" = local ]; then
|
||||
echo "→ [1/4] building + linking local dev build (local-install.sh)"
|
||||
( cd "$REPO_ROOT" && ./scripts/local-install.sh ) || { echo "local-install.sh failed"; exit 1; }
|
||||
else
|
||||
echo "→ [1/4] installing $PKG@$VERSION globally"
|
||||
npm install -g "$PKG@$VERSION" || { echo "npm install -g $PKG@$VERSION failed"; exit 1; }
|
||||
fi
|
||||
ACTUAL="$(codegraph --version 2>/dev/null || echo '?')"
|
||||
echo " codegraph on PATH: $(command -v codegraph) -> $ACTUAL"
|
||||
|
||||
# 2. Ensure the corpus repo exists (clone shallow if missing, reuse if present).
|
||||
mkdir -p "$CORPUS"
|
||||
if [ -d "$REPO/.git" ]; then
|
||||
echo "→ [2/4] reusing existing checkout: $REPO"
|
||||
else
|
||||
echo "→ [2/4] cloning $URL"
|
||||
git clone --depth 1 "$URL" "$REPO" || { echo "git clone failed"; exit 1; }
|
||||
fi
|
||||
|
||||
# 3. Wipe + re-index with THIS version (the index must be built by the same
|
||||
# binary that serves it — different versions extract differently).
|
||||
echo "→ [3/4] wiping .codegraph and re-indexing with $ACTUAL"
|
||||
rm -rf "$REPO/.codegraph"
|
||||
( cd "$REPO" && codegraph init -i ) || { echo "indexing failed"; exit 1; }
|
||||
|
||||
# 4. Run the with/without A/B.
|
||||
echo "→ [4/4] running A/B harness (mode=$MODE)"
|
||||
bash "$HARNESS/run-all.sh" "$REPO" "$Q" "$MODE"
|
||||
|
||||
# Restore the dev link (the normal working state in this repo).
|
||||
echo
|
||||
echo "→ restoring local dev link (local-install.sh)"
|
||||
if ( cd "$REPO_ROOT" && ./scripts/local-install.sh >/dev/null 2>&1 ); then
|
||||
echo " global codegraph restored to dev build"
|
||||
else
|
||||
echo " WARN: restore failed — run ./scripts/local-install.sh manually"
|
||||
fi
|
||||
echo "==================== audit complete ===================="
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-run the README "Benchmark Results" A/B (with vs without codegraph) on the
|
||||
# current build: the 7 README repos, same queries, RUNS per arm (default 4).
|
||||
# Output → /tmp/ab-readme/<repo>/run<n>/run-headless-{with,without}.jsonl
|
||||
# Aggregate with parse-bench-readme.mjs. Repos must be cloned + indexed under
|
||||
# $CORPUS (default /tmp/codegraph-corpus) by the build under test.
|
||||
set -uo pipefail
|
||||
H="$(cd "$(dirname "$0")" && pwd)"
|
||||
C="${CORPUS:-/tmp/codegraph-corpus}"
|
||||
RUNS="${RUNS:-4}"
|
||||
ROWS=(
|
||||
"vscode|How does the extension host communicate with the main process?"
|
||||
"excalidraw|How does Excalidraw render and update canvas elements?"
|
||||
"django|How does Django's ORM build and execute a query from a QuerySet?"
|
||||
"tokio|How does tokio schedule and run async tasks on its runtime?"
|
||||
"okhttp|How does OkHttp process a request through its interceptor chain?"
|
||||
"gin|How does gin route requests through its middleware chain?"
|
||||
"alamofire|How does Alamofire build, send, and validate a request?"
|
||||
)
|
||||
echo "### README A/B START $(date) RUNS=$RUNS"
|
||||
for row in "${ROWS[@]}"; do
|
||||
repo="${row%%|*}"; q="${row#*|}"
|
||||
echo "===== $repo ====="
|
||||
for run in $(seq 1 "$RUNS"); do
|
||||
AGENT_EVAL_OUT="/tmp/ab-readme/$repo/run$run" bash "$H/run-all.sh" "$C/$repo" "$q" headless 2>&1 | grep -E "exit [0-9]" || echo " run$run: (no exit line)"
|
||||
done
|
||||
done
|
||||
echo "### README A/B DONE $(date)"
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# One README repo, WITH-codegraph only, N runs. Each run appends a why-Read
|
||||
# diagnostic so the agent explains any Read/Grep. (The WITHOUT baseline is
|
||||
# codegraph-independent and already in the README — no point re-running it.)
|
||||
# Output -> /tmp/ab-why/<repo>/with<n>.jsonl
|
||||
# Usage: bench-why-repo.sh <repo-path> "<query>" [N]
|
||||
set -uo pipefail
|
||||
REPO="$1"; Q="$2"; N="${3:-4}"
|
||||
NAME="$(basename "$REPO")"
|
||||
CG="/Users/colby/Development/Personal/codegraph/dist/bin/codegraph.js"
|
||||
OUT="/tmp/ab-why/$NAME"; mkdir -p "$OUT"
|
||||
WHY=$'\n\nIMPORTANT — diagnostic: if you use the Read or Grep tool at ANY point, for EACH such call explain why codegraph_explore / codegraph_node did not already give you what you needed. End your entire answer with a section titled exactly "## Why I read" listing every Read and Grep you made and the precise reason codegraph fell short for it. If you used neither, write "## Why I read" then "none — codegraph was sufficient."'
|
||||
printf '{"mcpServers":{"codegraph":{"command":"%s","args":["serve","--mcp","--path","%s"]}}}' "$CG" "$REPO" > "$OUT/cg.json"
|
||||
|
||||
for i in $(seq 1 "$N"); do
|
||||
pkill -f "serve --mcp" 2>/dev/null; sleep 1; rm -f "$REPO/.codegraph/daemon.sock"
|
||||
( cd "$REPO" && claude -p "$Q$WHY" --output-format stream-json --verbose \
|
||||
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
--strict-mcp-config --mcp-config "$OUT/cg.json" > "$OUT/with$i.jsonl" 2>"$OUT/with$i.err" )
|
||||
echo "WITH run $i: exit $? ($(wc -l < "$OUT/with$i.jsonl" | tr -d ' ') lines)"
|
||||
done
|
||||
echo "DONE $NAME"
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# PreToolUse hook (experiment): deny Read of codegraph-indexed source files and
|
||||
# steer the agent to codegraph_explore/codegraph_node instead. Tests whether
|
||||
# codegraph can FULLY replace Read for code-understanding once the escape hatch
|
||||
# is removed. Non-source reads (config, .env, markdown, new files) pass through.
|
||||
#
|
||||
# Wire via: claude ... --settings scripts/agent-eval/hook-settings.json
|
||||
set -uo pipefail
|
||||
input="$(cat)"
|
||||
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
||||
|
||||
case "$fp" in
|
||||
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte)
|
||||
msg="Read is disabled for source files in this session — codegraph already has this file indexed (with line numbers, kept in sync on every change). Use codegraph_explore (several related symbols at once) or codegraph_node (one symbol's full source). If a symbol you need wasn't in a prior explore, run ANOTHER codegraph_explore with its exact name instead of reading the file."
|
||||
jq -n --arg m "$msg" '{reason:$m, hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Read",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash /Users/colby/Development/Personal/codegraph/scripts/agent-eval/block-read-hook.sh"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive an INTERACTIVE Claude Code session in tmux, send a prompt, wait for the
|
||||
# agent to finish, then print the tool-call breakdown from the session logs.
|
||||
#
|
||||
# Why interactive (not `claude -p`): headless print-mode picks the
|
||||
# general-purpose subagent, while real interactive sessions delegate to the
|
||||
# Explore subagent (or drive codegraph from the main thread). Only the
|
||||
# interactive TUI reproduces the behavior users actually see. (Idle-detection
|
||||
# technique borrowed from devpit's WaitForIdle.)
|
||||
#
|
||||
# Usage: itrun.sh <repo-path> <label> "<prompt>"
|
||||
# Output dir: $AGENT_EVAL_OUT (default /tmp/agent-eval)
|
||||
# Requires: tmux 3.0+, a logged-in `claude` CLI, codegraph MCP configured.
|
||||
set -uo pipefail
|
||||
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
||||
SESSION="cgt_${LABEL}"
|
||||
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
||||
OUT="$OUT_DIR/itrun-${LABEL}.txt"
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
cap() { tmux capture-pane -p -t "$SESSION" -S -40; }
|
||||
|
||||
tmux kill-session -t "$SESSION" 2>/dev/null
|
||||
|
||||
# Wide pane so the TUI doesn't hard-wrap tool lines.
|
||||
tmux new-session -d -s "$SESSION" -x 230 -y 60
|
||||
tmux send-keys -t "$SESSION" "cd $REPO && claude --dangerously-skip-permissions ${CLAUDE_EXTRA_ARGS:-}" Enter
|
||||
|
||||
# Wait for the ❯ prompt (claude drew its UI), up to 60s. NOTE: ❯ appears on the
|
||||
# welcome screen seconds before the input actually accepts keystrokes, so this is
|
||||
# necessary but NOT sufficient — the type-and-verify loop below is what proves
|
||||
# the input is live.
|
||||
ready=0
|
||||
for _ in $(seq 1 120); do
|
||||
cap | grep -q "❯" && { ready=1; break; }
|
||||
sleep 0.5
|
||||
done
|
||||
[ "$ready" = 1 ] || { echo "claude never drew its UI"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
||||
|
||||
# Accept the per-folder "Is this a project you trust?" dialog if it shows (first
|
||||
# time claude opens a given repo). Option 1 ("Yes, I trust this folder") is
|
||||
# pre-selected, so Enter accepts. This dialog also contains ❯, so it must be
|
||||
# cleared before the type-and-verify loop or keystrokes land on the menu.
|
||||
for _ in $(seq 1 20); do
|
||||
cap | grep -q "trust this folder" || break
|
||||
tmux send-keys -t "$SESSION" Enter
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Type-and-verify: send the prompt, confirm a distinctive chunk of it actually
|
||||
# landed in the input box, retry if it didn't (handles the early-❯ race where
|
||||
# the welcome screen shows the prompt glyph but MCP init is still eating keys).
|
||||
needle="${PROMPT:0:24}"
|
||||
typed=0
|
||||
for _ in $(seq 1 30); do
|
||||
tmux send-keys -l -t "$SESSION" "$PROMPT"
|
||||
sleep 1
|
||||
if cap | grep -Fq "$needle"; then typed=1; break; fi
|
||||
# Clear whatever partial text may have landed, then retry.
|
||||
tmux send-keys -t "$SESSION" C-u
|
||||
sleep 1
|
||||
done
|
||||
[ "$typed" = 1 ] || { echo "prompt never landed in the input box"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
||||
sleep 0.5
|
||||
tmux send-keys -t "$SESSION" Enter
|
||||
|
||||
# Busy signals. The robust one is the spinner's elapsed-time-in-parens, which
|
||||
# EVERY working state shows — both the pre-stream thinking phase
|
||||
# "(8s · thinking with max effort)" and the streaming phase
|
||||
# "(24s · ↑ 2.5k tokens · …)", and it survives the 32s→"1m 3s" rollover. We OR
|
||||
# in the token arrows, "esc to interrupt", and "Initializing" as belt-and-braces
|
||||
# (some TUI versions/states show one but not the others).
|
||||
BUSY_RE='esc to interrupt|↓ [0-9]|↑ [0-9]|Initializing|\(([0-9]+m )?[0-9]+s ·'
|
||||
|
||||
# Wait for work to START (busy indicator appears), up to 60s. If it never starts,
|
||||
# fail loudly rather than silently reporting an empty run.
|
||||
started=0
|
||||
for _ in $(seq 1 120); do
|
||||
cap | grep -qE "$BUSY_RE" && { started=1; break; }
|
||||
sleep 0.5
|
||||
done
|
||||
[ "$started" = 1 ] || { echo "agent never started working"; cap; tmux kill-session -t "$SESSION" 2>/dev/null; exit 1; }
|
||||
|
||||
# Poll for idle. CRITICAL: Opus 4.8 (extended thinking) renders NO spinner /
|
||||
# "esc to interrupt" / timer while it STREAMS its final answer — those appear
|
||||
# only during the thinking + tool-use phases ("✻ Marinating… (32s · ↓ 1.3k
|
||||
# tokens · thinking with max effort)"). So BUSY_RE reads "not busy" for the whole
|
||||
# 10-30s answer stream, and any short not-busy threshold kills the run mid-answer
|
||||
# (the truncation bug). We therefore detect "done" by CONTENT STABILITY, not by a
|
||||
# spinner string: while the agent streams, the captured pane changes every poll,
|
||||
# so stability never accrues; it accrues only once the agent has finished and the
|
||||
# static "✻ Brewed for 1m 9s" summary is all that is left. BUSY_RE still hard-
|
||||
# resets stability (covers thinking/tool-use/live-timer, where text can briefly
|
||||
# sit still). Need STABLE_NEEDED polls (~8s) of zero pane change + ❯ present.
|
||||
# Content-stability is model-agnostic — it survives future spinner re-wordings.
|
||||
STABLE_NEEDED=16
|
||||
prev=""; stable=0
|
||||
for _ in $(seq 1 2400); do # up to ~20 min
|
||||
pane="$(cap)"
|
||||
sig="$(printf '%s' "$pane" | tr -s '[:space:]' ' ')"
|
||||
if printf '%s' "$pane" | grep -qE "$BUSY_RE"; then
|
||||
stable=0 # thinking / tool use / live timer → busy
|
||||
elif [ -n "$sig" ] && [ "$sig" = "$prev" ] && printf '%s' "$pane" | grep -q "❯"; then
|
||||
stable=$((stable+1)); [ "$stable" -ge "$STABLE_NEEDED" ] && break
|
||||
else
|
||||
stable=0 # answer still streaming → pane changing
|
||||
fi
|
||||
prev="$sig"
|
||||
sleep 0.5
|
||||
done
|
||||
sleep 1
|
||||
|
||||
tmux capture-pane -p -t "$SESSION" -S - > "$OUT"
|
||||
echo "captured $(wc -l < "$OUT") lines -> $OUT"
|
||||
grep -oE "Done \([^)]*\)|[A-Z][a-z]+ for ([0-9]+m )?[0-9]+s" "$OUT" | tail -1
|
||||
grep -oE "[0-9.]+k?/[0-9.]+M" "$OUT" | tail -1 | sed 's/^/Context /'
|
||||
tmux kill-session -t "$SESSION" 2>/dev/null
|
||||
|
||||
# Clean tool breakdown from the session logs (main + subagents).
|
||||
node "$HERE/parse-session.mjs" "$REPO" 2>/dev/null || true
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# 3-arm offload eval for ONE indexed repo + ONE question, n reps each.
|
||||
# ARM offload : codegraph attached, managed offload ON (per-run AI usage log)
|
||||
# ARM raw : codegraph attached, CODEGRAPH_OFFLOAD_DISABLE=1 (raw source)
|
||||
# ARM nocg : no codegraph (empty MCP config) -> Read/Grep baseline
|
||||
# All arms: claude -p sonnet --effort high. One JSON metrics line/run -> $RESULTS.
|
||||
#
|
||||
# Usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"
|
||||
# Env: MODEL=sonnet EFFORT=high RESULTS=<file> AGENT_EVAL_OUT=<scratch dir>
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENGINE="$(cd "$HERE/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
TARGET="${1:?usage: offload-eval-3arm.sh <indexed-repo> <tier> <reps> \"<question>\"}"
|
||||
TIER="${2:?tier}"; REPS="${3:?reps}"; Q="${4:?question}"
|
||||
RUNS="$OUT/runs"
|
||||
EXTRACT="$HERE/offload-eval-metrics.mjs"
|
||||
RESULTS="${RESULTS:-$OUT/results.jsonl}"
|
||||
REPO=$(basename "$TARGET")
|
||||
mkdir -p "$RUNS"
|
||||
command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "not indexed: $TARGET (run offload-eval-setup.sh first)"; exit 1; }
|
||||
# Physical path so pkill matches the daemon's real cmdline (macOS /tmp->/private/tmp symlink
|
||||
# otherwise makes the kill miss the daemon, and the next arm connects to the SURVIVING daemon
|
||||
# — contaminating the raw arm with offload).
|
||||
TARGET=$(cd "$TARGET" && pwd -P)
|
||||
|
||||
prewarm() { # path extra-env (e.g. "FOO=bar")
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
|
||||
env ${2:-} CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
||||
&& echo " daemon warm" || echo " WARN daemon never bound"
|
||||
}
|
||||
|
||||
run() { # arm rep mcp-config usage-log-or-dash
|
||||
local arm="$1" rep="$2" cfg="$3" usage="$4" tag="$REPO-$1-$2"
|
||||
[ "$usage" != "-" ] && : > "$usage"
|
||||
# DISALLOW (optional): block sub-agent delegation across all arms so the A/B
|
||||
# measures the retrieval mode, not whether Sonnet decides to spawn a codegraph-blind
|
||||
# Explore subagent (which thrashes regardless and adds huge variance).
|
||||
( cd "$TARGET" && claude -p "$Q" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
${DISALLOW:+--disallowedTools "$DISALLOW"} \
|
||||
--strict-mcp-config --mcp-config "$cfg" \
|
||||
</dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
|
||||
node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$usage" --arm "$arm" --rep "$rep" \
|
||||
--repo "$REPO" --tier "$TIER" --q "$Q" >> "$RESULTS"
|
||||
node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(` [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
|
||||
}
|
||||
|
||||
CFG_OFF="$RUNS/mcp-offload-$REPO.json"; CFG_RAW="$RUNS/mcp-raw-$REPO.json"; CFG_NOCG="$RUNS/mcp-nocg.json"
|
||||
USAGE="$RUNS/$REPO-usage.jsonl"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' "$USAGE" "$BIN" "$TARGET" > "$CFG_OFF"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TARGET" > "$CFG_RAW"
|
||||
printf '{"mcpServers":{}}' > "$CFG_NOCG"
|
||||
|
||||
# REP_START lets a later batch ADD reps without clobbering earlier jsonls
|
||||
# (e.g. REP_START=4 REPS=3 -> reps 4,5,6; default starts at 1).
|
||||
START="${REP_START:-1}"; END=$((START + REPS - 1))
|
||||
echo "###### repo=$REPO tier=$TIER reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high}"
|
||||
echo "###### Q=$Q"
|
||||
echo "== ARM offload =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
|
||||
for r in $(seq "$START" "$END"); do run offload "$r" "$CFG_OFF" "$USAGE"; done
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1
|
||||
echo "== ARM raw =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_DISABLE=1"
|
||||
for r in $(seq "$START" "$END"); do run raw "$r" "$CFG_RAW" "-"; done
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1
|
||||
echo "== ARM nocg =="
|
||||
for r in $(seq "$START" "$END"); do run nocg "$r" "$CFG_NOCG" "-"; done
|
||||
echo "###### DONE $REPO"
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env node
|
||||
// Cost/token analysis for the 3-arm offload eval, with a MAIN-vs-SUBAGENT split.
|
||||
//
|
||||
// The explore-subagent question. With delegation ALLOWED, the nocg arm spawns a
|
||||
// Claude Code Explore subagent; the codegraph arms do all work in the main agent.
|
||||
// Two facts make naive accounting wrong:
|
||||
// 1. The Explore subagent runs on HAIKU 4.5; the main agent on SONNET 4.6.
|
||||
// So per-token cost differs ~3x between them — you cannot price both the same.
|
||||
// 2. The subagent's consumption is ~95% cache-reads. At Haiku's $0.10/MTok
|
||||
// cache-read rate, a huge TOKEN volume is a small DOLLAR cost.
|
||||
//
|
||||
// Rather than re-derive cost from raw token counts (and guess the cache TTL —
|
||||
// Claude Code uses 1-hour ephemeral cache here, 2x write, not 5-min), we read
|
||||
// Claude Code's OWN authoritative accounting from the `result` event:
|
||||
// result.modelUsage[model].costUSD — per-model cost CC itself billed
|
||||
// result.total_cost_usd — their sum (INCLUDES the Haiku subagent;
|
||||
// the handoff's "excludes subagent" was wrong)
|
||||
// The model split IS the agent split here: sonnet => main, haiku => Explore subagent
|
||||
// (only nocg spawns one, and only nocg shows haiku usage). Token volume is still
|
||||
// summed per-model from modelUsage for the separate "tokens" story.
|
||||
//
|
||||
// Usage: offload-eval-cost.mjs <runs-dir> <repo> [reps]
|
||||
// e.g. offload-eval-cost.mjs /tmp/cg-offload-eval/runs trezor 3
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
|
||||
const MAIN_TIER = /sonnet/; // main agent
|
||||
const SUB_TIER = /haiku/; // Claude Code Explore subagent
|
||||
|
||||
const [,, runsDir, repo, repsArg] = process.argv;
|
||||
if (!runsDir || !repo) { console.error('usage: offload-eval-cost.mjs <runs-dir> <repo> [reps] (env ARMS=nocg,raw,offload)'); process.exit(1); }
|
||||
const REPS = Number(repsArg || 3);
|
||||
// Arms to analyze (file stems `<repo>-<arm>-<rep>.jsonl`). Override for the style A/B:
|
||||
// ARMS=raw,refs,map,src. nocg's Haiku subagent is the only sub-tier; the rest are main-only.
|
||||
const ARMS = (process.env.ARMS || 'nocg,raw,offload').split(',').map((s) => s.trim()).filter(Boolean);
|
||||
|
||||
const toks = (u) => (u.inputTokens||0)+(u.outputTokens||0)+(u.cacheReadInputTokens||0)+(u.cacheCreationInputTokens||0);
|
||||
|
||||
function analyzeRun(file) {
|
||||
let result = null, agentCalls = 0;
|
||||
const tools = {}, subPids = new Set();
|
||||
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
||||
if (!line) continue;
|
||||
let e; try { e = JSON.parse(line); } catch { continue; }
|
||||
if (e.parent_tool_use_id && e.message?.usage) subPids.add(e.parent_tool_use_id);
|
||||
if (e.type === 'assistant' && Array.isArray(e.message?.content))
|
||||
for (const b of e.message.content)
|
||||
if (b.type === 'tool_use') { tools[b.name] = (tools[b.name]||0)+1; if (b.name === 'Agent') agentCalls++; }
|
||||
if (e.type === 'result') result = e;
|
||||
}
|
||||
// Authoritative cost + tokens from Claude Code's per-model accounting.
|
||||
const mu = result?.modelUsage || {};
|
||||
const main = { cost: 0, tok: 0 }, sub = { cost: 0, tok: 0 };
|
||||
for (const [model, u] of Object.entries(mu)) {
|
||||
const bucket = SUB_TIER.test(model) ? sub : main; // sonnet/anything-else => main
|
||||
bucket.cost += u.costUSD || 0;
|
||||
bucket.tok += toks(u);
|
||||
}
|
||||
return {
|
||||
main, sub, subagents: subPids.size, agentCalls,
|
||||
ccTotal: result?.total_cost_usd ?? null,
|
||||
ok: result?.subtype === 'success',
|
||||
durationSec: result?.duration_ms ? +(result.duration_ms/1000).toFixed(1) : null,
|
||||
models: Object.keys(mu), tools,
|
||||
};
|
||||
}
|
||||
|
||||
const k = (n) => (n/1000).toFixed(0).padStart(5) + 'K';
|
||||
const d = (n) => '$' + n.toFixed(3);
|
||||
const cost = (b) => b.cost;
|
||||
const tot = (b) => b.tok;
|
||||
|
||||
const byArm = {};
|
||||
for (const arm of ARMS) {
|
||||
const runs = [];
|
||||
for (let r = 1; r <= REPS; r++) {
|
||||
const f = `${runsDir}/${repo}-${arm}-${r}.jsonl`;
|
||||
if (existsSync(f)) runs.push({ rep: r, ...analyzeRun(f) });
|
||||
}
|
||||
byArm[arm] = runs;
|
||||
}
|
||||
|
||||
// Per-run detail. Cost is Claude Code's own modelUsage.costUSD (authoritative,
|
||||
// per-model pricing + correct cache TTL). MAIN=Sonnet, SUB=Haiku Explore subagent.
|
||||
// cc-check: main$+sub$ must equal result.total_cost_usd (delta should be ~0).
|
||||
console.log(`\n=== ${repo}: per-run main(Sonnet)/sub(Haiku) split — Claude Code's own cost accounting ===`);
|
||||
console.log('arm rep | subAg | MAIN(sonnet) tok / $ | SUB(haiku) tok / $ | TOTAL tok / $ | cc_total Δ | dur reads');
|
||||
for (const arm of ARMS) for (const r of byArm[arm]) {
|
||||
const mC = cost(r.main), sC = cost(r.sub), mT = tot(r.main), sT = tot(r.sub);
|
||||
const reads = r.tools['Read'] || 0, grep = (r.tools['Grep']||0)+(r.tools['Bash']||0)+(r.tools['Glob']||0);
|
||||
const explore = r.tools['mcp__codegraph__codegraph_explore'] || 0;
|
||||
const delta = (mC + sC) - (r.ccTotal || 0); // should be ~0
|
||||
console.log(
|
||||
`${arm.padEnd(8)} #${r.rep} | ${String(r.subagents).padStart(2)} | ${k(mT)} ${d(mC).padStart(7)} | ${k(sT)} ${d(sC).padStart(7)} | ${k(mT+sT)} ${d(mC+sC).padStart(7)} | ${d(r.ccTotal||0).padStart(7)} ${(delta>=0?'+':'')+delta.toFixed(4)} | ${String(r.durationSec).padStart(5)} r=${reads} g=${grep} x=${explore}`
|
||||
);
|
||||
}
|
||||
|
||||
// Per-arm means
|
||||
const mean = (arr, f) => arr.length ? arr.reduce((s,x)=>s+f(x),0)/arr.length : 0;
|
||||
console.log(`\n=== ${repo}: per-arm MEANS (n per arm) ===`);
|
||||
console.log('arm n | main $ sub $ TOTAL $ | main tok sub tok TOTAL tok | %$ in sub | %tok in sub');
|
||||
for (const arm of ARMS) {
|
||||
const runs = byArm[arm]; if (!runs.length) continue;
|
||||
const mC = mean(runs, r=>cost(r.main)), sC = mean(runs, r=>cost(r.sub));
|
||||
const mT = mean(runs, r=>tot(r.main)), sT = mean(runs, r=>tot(r.sub));
|
||||
const pctSubC = (mC+sC) ? (100*sC/(mC+sC)) : 0;
|
||||
const pctSubT = (mT+sT) ? (100*sT/(mT+sT)) : 0;
|
||||
console.log(
|
||||
`${arm.padEnd(8)} ${runs.length} | ${d(mC).padStart(7)} ${d(sC).padStart(7)} ${d(mC+sC).padStart(7)} | ${k(mT)} ${k(sT)} ${k(mT+sT)} | ${pctSubC.toFixed(0).padStart(3)}% | ${pctSubT.toFixed(0).padStart(3)}%`
|
||||
);
|
||||
}
|
||||
|
||||
// Headline ladders — cost, tokens, duration, all vs a baseline (nocg if present, else first arm).
|
||||
console.log(`\n=== Ladders (mean, incl. subagent) ===`);
|
||||
const totals = ARMS.map(a => ({ a, c: mean(byArm[a], r=>cost(r.main)+cost(r.sub)), t: mean(byArm[a], r=>tot(r.main)+tot(r.sub)) })).filter(x=>byArm[x.a].length);
|
||||
const base = totals.find(x=>x.a==='nocg') ?? totals[0];
|
||||
const bn = base?.a ?? '?';
|
||||
console.log(` COST (vs ${bn}):`);
|
||||
for (const x of totals) {
|
||||
const vs = base && base.c ? ` (${((x.c/base.c-1)*100>=0?'+':'')}${((x.c/base.c-1)*100).toFixed(0)}%)` : '';
|
||||
console.log(` ${x.a.padEnd(8)} ${d(x.c)}${vs}`);
|
||||
}
|
||||
console.log(` TOKENS (vs ${bn}):`);
|
||||
for (const x of totals) {
|
||||
const vs = base && base.t ? ` (${((x.t/base.t-1)*100>=0?'+':'')}${((x.t/base.t-1)*100).toFixed(0)}%)` : '';
|
||||
console.log(` ${x.a.padEnd(8)} ${k(x.t)}${vs}`);
|
||||
}
|
||||
console.log(` DURATION (wall-clock, vs ${bn}):`);
|
||||
const durs = ARMS.map(a => ({ a, s: mean(byArm[a].filter(r=>r.durationSec!=null), r=>r.durationSec) })).filter(x=>byArm[x.a].length);
|
||||
const dbase = durs.find(x=>x.a==='nocg') ?? durs[0];
|
||||
for (const x of durs) {
|
||||
const vs = dbase && dbase.s ? ` (${((x.s/dbase.s-1)*100>=0?'+':'')}${((x.s/dbase.s-1)*100).toFixed(0)}%)` : '';
|
||||
console.log(` ${x.a.padEnd(8)} ${x.s.toFixed(0)}s${vs}`);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env node
|
||||
// Effort A/B — does CODEGRAPH_OFFLOAD_EFFORT=high improve offload SYNTHESIS FIDELITY vs low?
|
||||
// Probe-based (no agent): for each repo × effort × rep, run codegraph_explore with the offload
|
||||
// ON on the canonical question, capture the synthesized answer + AI tokens/cost/latency, then
|
||||
// Sonnet-judge that answer's fidelity vs source-verified ground truth. Isolates the synthesis
|
||||
// from agent/adoption noise. Requires `codegraph login` (managed offload) + indexed repos.
|
||||
//
|
||||
// Env: REPS (default 3) · CG_ENGINE (engine repo) · AGENT_EVAL_OUT (repos under /repos) · CONC (judge concurrency)
|
||||
import { pathToFileURL, fileURLToPath } from 'node:url';
|
||||
import { resolve, dirname, join } from 'node:path';
|
||||
import { readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
|
||||
const OUT = process.env.AGENT_EVAL_OUT || '/tmp/cg-offload-eval';
|
||||
const REPOS = join(OUT, 'repos');
|
||||
const GT = JSON.parse(readFileSync(resolve(HERE, 'offload-eval-ground-truth.json'), 'utf8'));
|
||||
const REPS = Number(process.env.REPS || 3);
|
||||
const CONC = Number(process.env.CONC || 4);
|
||||
const EFFORTS = (process.env.EFFORTS_FILTER || 'low,high').split(',');
|
||||
const ONLY = process.env.REPOS_FILTER ? new Set(process.env.REPOS_FILTER.split(',')) : null;
|
||||
const TIER = { mtkruto: 'small', postybirb: 'medium', shapeshift: 'complex', trezor: 'large' };
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const toolsMod = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = toolsMod.ToolHandler ?? toolsMod.default?.ToolHandler;
|
||||
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
|
||||
console.error('could not load engine from', ENGINE); process.exit(2);
|
||||
}
|
||||
|
||||
const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools.
|
||||
|
||||
QUESTION: ${gt.question}
|
||||
|
||||
VERIFIED GROUND TRUTH (the actual call path + files):
|
||||
${gt.truth}
|
||||
|
||||
SYNTHESIZED ANSWER (to score):
|
||||
${ans || '(empty)'}
|
||||
|
||||
Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial.
|
||||
Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;
|
||||
|
||||
const askJudge = (prompt) => new Promise((res) => {
|
||||
execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
|
||||
'--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
|
||||
{ cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
|
||||
const m = (stdout || '').match(/\{[\s\S]*\}/);
|
||||
if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
|
||||
try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
|
||||
});
|
||||
});
|
||||
|
||||
// ---- 1. Probe: collect synthesized answers at each effort -------------------
|
||||
const records = [];
|
||||
for (const repo of Object.keys(GT)) {
|
||||
if (ONLY && !ONLY.has(repo)) continue;
|
||||
const dir = join(REPOS, repo);
|
||||
if (!existsSync(join(dir, '.codegraph'))) { console.error('skip (not indexed):', repo); continue; }
|
||||
const cg = CodeGraph.openSync(dir);
|
||||
const h = new ToolHandler(cg);
|
||||
for (const effort of EFFORTS) {
|
||||
for (let rep = 1; rep <= REPS; rep++) {
|
||||
process.env.CODEGRAPH_OFFLOAD_EFFORT = effort;
|
||||
const usageLog = join(tmpdir(), `effort-${repo}-${effort}-${rep}.jsonl`);
|
||||
try { rmSync(usageLog); } catch { /* none */ }
|
||||
process.env.CODEGRAPH_OFFLOAD_USAGE_LOG = usageLog;
|
||||
let answer = '';
|
||||
try { answer = (await h.execute('codegraph_explore', { query: GT[repo].question }))?.content?.[0]?.text ?? ''; }
|
||||
catch (e) { console.error(` ${repo}/${effort}#${rep} explore failed: ${e?.message}`); }
|
||||
const fired = /Synthesized by CodeGraph/.test(answer);
|
||||
const ai = { tokens: 0, cost: 0, ms: 0 };
|
||||
if (existsSync(usageLog)) for (const e of readFileSync(usageLog, 'utf8').split('\n').filter(Boolean).map(JSON.parse)) {
|
||||
ai.tokens += e.totalTokens || 0; ai.cost += e.costUsd || 0; ai.ms += e.ms || 0;
|
||||
}
|
||||
records.push({ repo, tier: TIER[repo], effort, rep, fired, ai, answer });
|
||||
console.error(` ${repo}/${effort}#${rep}: fired=${fired} ${ai.tokens}tok $${ai.cost.toFixed(4)} ${ai.ms}ms`);
|
||||
}
|
||||
}
|
||||
try { cg.close?.(); } catch { /* none */ }
|
||||
}
|
||||
|
||||
// ---- 2. Judge fidelity (concurrency) ---------------------------------------
|
||||
console.error(`\njudging ${records.length} answers (concurrency ${CONC})...`);
|
||||
let done = 0;
|
||||
const q = [...records];
|
||||
async function worker() { while (q.length) { const r = q.shift(); r.fid = await askJudge(fidPrompt(GT[r.repo], r.answer)); console.error(` [${++done}/${records.length}] ${r.repo}/${r.effort}#${r.rep}: ${r.fid.verdict} ${r.fid.score ?? ''}`); } }
|
||||
await Promise.all(Array.from({ length: CONC }, worker));
|
||||
writeFileSync(join(OUT, 'effort-results.jsonl'), records.map((r) => JSON.stringify(r)).join('\n') + '\n');
|
||||
|
||||
// ---- 3. Aggregate: low vs high per repo ------------------------------------
|
||||
const med = (a) => { a = a.filter((x) => x != null).sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
|
||||
console.log(`\n${'='.repeat(80)}\nEFFORT A/B — offload synthesis fidelity (probe, n=${REPS}/cell)\n${'='.repeat(80)}`);
|
||||
console.log(`${'repo'.padEnd(11)} ${'tier'.padEnd(8)} ${'effort'.padEnd(6)} fired ${'fid(med)'.padStart(8)} ${'fab%'.padStart(5)} ${'AItok'.padStart(7)} ${'AIcost'.padStart(8)} ${'ms(med)'.padStart(8)}`);
|
||||
for (const repo of Object.keys(GT)) {
|
||||
for (const effort of EFFORTS) {
|
||||
const rs = records.filter((r) => r.repo === repo && r.effort === effort);
|
||||
if (!rs.length) continue;
|
||||
const fids = rs.map((r) => r.fid?.score).filter((x) => x != null);
|
||||
const fab = rs.filter((r) => r.fid?.fabrication === true).length;
|
||||
console.log(`${repo.padEnd(11)} ${TIER[repo].padEnd(8)} ${effort.padEnd(6)} ${rs.filter((r) => r.fired).length}/${rs.length} ${String(med(fids) ?? '—').padStart(8)} ${String(Math.round(100 * fab / rs.length) + '%').padStart(5)} ${String(Math.round(med(rs.map((r) => r.ai.tokens)) / 1000) + 'k').padStart(7)} ${('$' + (med(rs.map((r) => r.ai.cost)) ?? 0).toFixed(4)).padStart(8)} ${String(med(rs.map((r) => r.ai.ms)) ?? '—').padStart(8)}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the FRONTLOAD arm across all 4 tiers (n reps), then judge + merge with the existing
|
||||
# matrix (offload/raw/nocg in $OUT/judged.jsonl, if present) + emit a combined summary.
|
||||
# Env: REPS (default 3) AGENT_EVAL_OUT=<scratch dir>
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
GT="$HERE/offload-eval-ground-truth.json"
|
||||
REPS="${REPS:-3}"
|
||||
export RESULTS="$OUT/results-fl.jsonl"
|
||||
: > "$RESULTS"; rm -f "$OUT/runs/hook-debug.log"
|
||||
for repo in mtkruto postybirb shapeshift trezor; do
|
||||
case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
|
||||
Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
|
||||
echo ""; echo "### $repo ($tier) $(date +%H:%M:%S)"
|
||||
bash "$HERE/offload-eval-frontload.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
|
||||
done
|
||||
echo ""
|
||||
echo "frontload: $(wc -l < "$RESULTS") runs | hook injections: $(grep -c INJECTED "$OUT/runs/hook-debug.log" 2>/dev/null) | errors: $(grep -c ERROR "$OUT/runs/hook-debug.log" 2>/dev/null)"
|
||||
echo "=== JUDGE frontload ==="
|
||||
node "$HERE/offload-eval-judge.mjs" --results "$RESULTS" --truth "$GT" --out "$OUT/judged-fl.jsonl" --concurrency 4 2>&1 | tail -4
|
||||
if [ -f "$OUT/judged.jsonl" ]; then cat "$OUT/judged.jsonl" "$OUT/judged-fl.jsonl" > "$OUT/judged-all.jsonl"; else cp "$OUT/judged-fl.jsonl" "$OUT/judged-all.jsonl"; fi
|
||||
echo "=== COMBINED SUMMARY ==="
|
||||
node "$HERE/offload-eval-summarize.mjs" "$OUT/judged-all.jsonl"
|
||||
echo "###### FRONTLOAD MATRIX DONE"
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# FRONTLOAD arm (approach 1): codegraph attached (offload-disabled) + the front-load
|
||||
# UserPromptSubmit hook (offload-eval-hook.mjs), n reps, appended to $RESULTS. Compare against
|
||||
# the matrix's raw/nocg baselines. Usage: offload-eval-frontload.sh <indexed-repo> <tier> <reps> "<Q>"
|
||||
# Env: MODEL=sonnet EFFORT=high RESULTS=<file> AGENT_EVAL_OUT=<scratch dir>
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENGINE="$(cd "$HERE/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
TARGET="${1:?repo}"; TIER="${2:?tier}"; REPS="${3:?reps}"; Q="${4:?question}"
|
||||
RUNS="$OUT/runs"
|
||||
EXTRACT="$HERE/offload-eval-metrics.mjs"
|
||||
RESULTS="${RESULTS:-$OUT/results-fl.jsonl}"
|
||||
REPO=$(basename "$TARGET")
|
||||
mkdir -p "$RUNS"
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "not indexed: $TARGET"; exit 1; }
|
||||
TARGET=$(cd "$TARGET" && pwd -P)
|
||||
|
||||
CFG="$RUNS/mcp-fl-$REPO.json"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' "$BIN" "$TARGET" > "$CFG"
|
||||
# Generate the hook settings pointing at the persisted hook; enable its debug log so we can
|
||||
# count injections (claude passes this env down to the spawned hook process).
|
||||
HOOKCFG="$RUNS/frontload-settings.json"
|
||||
printf '{"hooks":{"UserPromptSubmit":[{"hooks":[{"type":"command","command":"node %s/offload-eval-hook.mjs"}]}]}}' "$HERE" > "$HOOKCFG"
|
||||
export CG_FRONTLOAD_DEBUG="$RUNS/hook-debug.log"
|
||||
|
||||
prewarm() {
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
|
||||
env CODEGRAPH_OFFLOAD_DISABLE=1 CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
||||
&& echo " daemon warm" || echo " WARN no daemon"
|
||||
}
|
||||
|
||||
echo "###### FRONTLOAD repo=$REPO tier=$TIER reps=$REPS"
|
||||
prewarm "$TARGET"
|
||||
for r in $(seq 1 "$REPS"); do
|
||||
tag="$REPO-frontload-$r"
|
||||
( cd "$TARGET" && claude -p "$Q" --output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
--strict-mcp-config --mcp-config "$CFG" --settings "$HOOKCFG" \
|
||||
</dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
|
||||
node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "-" --arm frontload --rep "$r" --repo "$REPO" --tier "$TIER" --q "$Q" >> "$RESULTS"
|
||||
node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(` [frontload #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable}tok | read=${o.read} grep=${o.grep} agentExplore=${o.explore} | ok=${o.ok}`)' "$RESULTS"
|
||||
done
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null
|
||||
echo "###### FRONTLOAD DONE $REPO (cumulative hook injections: $(grep -c INJECTED "$CG_FRONTLOAD_DEBUG" 2>/dev/null))"
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"mtkruto": {
|
||||
"question": "How does calling the high-level client.sendMessage() method get the message serialized into a TL/MTProto request and sent over the network transport to Telegram's servers? Trace the call path.",
|
||||
"truth": "Verified call path (small TS Telegram-client lib):\n1. Client.sendMessage — client/6_client.ts:1852 → calls this.#messageManager.sendMessage(...) (1853)\n2. MessageManager.sendMessage — client/3_message_manager.ts:330 → builds the TL function {_:\"messages.sendMessage\", peer, message,...} and calls this.#c.invoke(...) (~377)\n3. c.invoke closure / Client.#invoke — client/6_client.ts:279/887 → resolves a ClientEncrypted and calls client.invoke(function_) (897)\n4. ClientEncrypted.invoke — client/2_client_encrypted.ts:324 → this.#send(function_) (325)\n5. ClientEncrypted.#send — client/2_client_encrypted.ts:261 → SERIALIZES via Api.serializeObject(function_) (290), then this.session.send(body) (296)\n - Serialization: Api.serializeObject (tl/2_telegram.ts:38) → new TLWriter().writeObject(...) (tl/1_tl_writer.ts) writes constructor id + fields\n6. SessionEncrypted.send — session/2_session_encrypted.ts:193 → ENQUEUES a PendingMessage and wakes the send loop (#awakeSendLoop) [DYNAMIC: async queue, not a direct call]\n7. SessionEncrypted.#sendLoopBody (AbortableLoop) — session/2_session_encrypted.ts:282 → #encryptMessage (serializeMessage tl/2_message.ts + IGE-256 AES) then this.transport.transport.send(payload) (339)\n8. Transport.send — TransportAbridged.send (transport/1_transport_abridged.ts:71) or TransportIntermediate → this.#connection.write(encrypt(framed))\n9. ConnectionTCP.write (connection/1_connection_tcp.ts:96) or ConnectionWebSocket.write — bytes exit the process.\nKEY SYMBOLS a correct answer must hit: sendMessage → MessageManager.sendMessage → invoke (ClientEncrypted.invoke/#send) → Api.serializeObject/TLWriter → SessionEncrypted.send (queue) → #sendLoopBody/#encryptMessage → Transport.send → Connection.write.\nDYNAMIC BOUNDARIES: the invoke indirection (closure→#invoke→ClientEncrypted), and the send-loop QUEUE between SessionEncrypted.send and #sendLoopBody (async, no direct edge)."
|
||||
},
|
||||
"postybirb": {
|
||||
"question": "How does submitting/queueing a post reach the website-specific code that actually uploads to a target website? Trace the path from the submission entry point in the NestJS server to a concrete website service's post implementation.",
|
||||
"truth": "Verified call path (NestJS+Electron, server under electron-app/src/server/):\n1. Entry: PostController.queue (submission/post/post.controller.ts:33, POST queue/:id) OR SubmissionService.queueScheduledSubmissions (submission/submission.service.ts, @Interval(60000) scheduler) — both call PostService.queue\n2. PostService.queue — submission/post/post.service.ts:68 → this.post(submission)\n3. PostService.post (private) — post.service.ts:206 → this.createPoster(...) for each non-default SubmissionPart\n4. PostService.createPoster — post.service.ts:373 → website = this.websites.getWebsiteModule(part.website) [part.website is a STRING]\n5. WebsiteProvider.getWebsiteModule — websites/website-provider.service.ts:86 → websiteModulesMap[name.toLowerCase()] [DYNAMIC: string-keyed registry of DI-injected Website singletons]; then new Poster(..., website, ...)\n6. Poster constructor — submission/post/poster.ts:117 → setTimeout(this.post.bind(this), delay) [DYNAMIC: async timer, min ~5s]\n7. Poster.post (poster.ts:131) → performPost (171) → attemptPost (217)\n8. Poster.attemptPost — poster.ts:217 → this.website.postFileSubmission(token, data, accountData) [DYNAMIC: polymorphic dispatch on abstract Website] (or postNotificationSubmission)\n9. Website.postFileSubmission (abstract) — websites/website.base.ts:102\n10. Concrete e.g. FurAffinity.postFileSubmission — websites/fur-affinity/fur-affinity.service.ts:231 → multi-step HTTP upload (GET /submit, POST /submit/upload multipart, POST /submit/finalize)\nKEY SYMBOLS: PostService.queue → PostService.post → createPoster → WebsiteProvider.getWebsiteModule (registry) → Poster (setTimeout) → Poster.attemptPost → Website.postFileSubmission (abstract base) → a concrete website service (e.g. FurAffinity).\nDYNAMIC BOUNDARIES: NestJS DI builds the website registry; string-keyed map lookup; setTimeout defers Poster.post; polymorphic dispatch on the abstract Website base. A correct answer must reach a concrete website's post via the registry + base class, not stop at PostService."
|
||||
},
|
||||
"shapeshift": {
|
||||
"question": "How does executing a swap work in ShapeShift — from the code that fetches quotes and selects a swapper down to a specific swapper's execute/trade? Name the swapper interface, the registry, and one concrete swapper, and trace the path.",
|
||||
"truth": "Verified call path (large multi-package monorepo; swap logic in packages/swapper + execution in src/lib):\nQUOTE/REGISTRY layer:\n- Swapper interface = the `Swapper` type (execute methods) + `SwapperApi` type (getTradeQuote/getUnsignedTx) — packages/swapper/src/types.ts (~846/897)\n- Registry = `swappers: Record<SwapperName,(SwapperApi & Swapper)>` — packages/swapper/src/constants.ts:52 (merges e.g. thorchainSwapper + thorchainApi)\n- Aggregator: swapperApi RTK endpoint getTradeQuote/getTradeRates (src/state/apis/swapper/swapperApi.ts:78/156) → getTradeQuotes (packages/swapper/src/swapper.ts:18/26): swapper = swappers[swapperName]; swapper.getTradeQuote(...) [DYNAMIC interface dispatch]\n- Concrete (THORChain): thorchainApi.getTradeQuote (swappers/ThorchainSwapper/endpoints.ts → getTradeQuote/getTradeQuote.ts:15) → getL1RateOrQuote → getQuote (thorService HTTP)\nSELECTION: tradeQuoteSlice selectors rank quotes (selectSortedTradeQuotes / selectActiveSwapperName) [DYNAMIC: winner chosen by ranking/user]\nEXECUTION layer:\n- useTradeExecution (src/components/MultiHopTrade/.../hooks/useTradeExecution.tsx:200/476) → new TradeExecution(); execution.execEvmTransaction(...) (CowSwap: execEvmMessage)\n- TradeExecution.execEvmTransaction — src/lib/tradeExecution.ts:326 → _execWalletAgnostic(...) (372)\n- TradeExecution._execWalletAgnostic — tradeExecution.ts:136 → swapper = swappers[swapperName] (149) [DYNAMIC registry]; buildSignBroadcast → swapper.getUnsignedEvmTransaction(...) (355) then swapper.executeEvmTransaction(unsignedTx, {signAndBroadcastTransaction}) (367) [DYNAMIC SwapperApi/Swapper interface]\n- Concrete (THORChain): thorchainSwapper.executeEvmTransaction (swappers/ThorchainSwapper/ThorchainSwapper.ts → utils.ts:181) delegates to callbacks.signAndBroadcastTransaction (wallet). CowSwap alt: cowSwapper.executeEvmMessage → signCowOrder + cowService.post.\nKEY SYMBOLS: Swapper/SwapperApi types, swappers registry (constants.ts), getTradeQuotes (swapper.ts), TradeExecution._execWalletAgnostic, swapper.getUnsignedEvmTransaction/executeEvmTransaction, one concrete swapper (thorchainSwapper/zrxSwapper/cowSwapper).\nDYNAMIC BOUNDARIES: swappers[name] registry lookup (2 sites); all hops into a concrete swapper are via the Swapper/SwapperApi interface, never a direct function. A correct answer must name the interface + registry and reach a concrete swapper through interface dispatch."
|
||||
},
|
||||
"trezor": {
|
||||
"question": "How does sending a crypto transaction flow from the send form's review/sign action through to signing it via @trezor/connect (TrezorConnect.signTransaction)? Trace the call path.",
|
||||
"truth": "Verified call path (trezor-suite monorepo; app in packages/suite, shared logic in suite-common/wallet-core, device API in packages/connect):\n1. ReviewButton.handleButtonReviewClick — packages/suite/src/views/wallet/send/TotalSent/ReviewButton.tsx:120 → signTransaction() (= useSendForm's sign)\n2. useSendForm.sign (exported as signTransaction) — packages/suite/src/hooks/wallet/useSendForm.ts:278 → dispatch(signAndPushSendFormTransactionThunk({formState, precomposedTransaction, selectedAccount})) [DYNAMIC: redux thunk]\n3. signAndPushSendFormTransactionThunk — packages/suite/src/actions/wallet/send/sendFormThunks.ts:237 → (first enhancePrecomposedTransactionThunk) then dispatch(signTransactionThunk({...})) [cross-package: thunk from suite-common/wallet-core]\n4. signTransactionThunk (coin-routing hub) — suite-common/wallet-core/src/send/sendFormThunks.ts:532/573 → networkType branch → dispatch(signBitcoinSendFormTransactionThunk) (ethereum→signEthereumSendFormTransactionThunk, etc.) [DYNAMIC: runtime coin dispatch]\n5. signBitcoinSendFormTransactionThunk — suite-common/wallet-core/src/send/sendFormBitcoinThunks.ts:394 → await TrezorConnect.signTransaction(signPayload)\n6. TrezorConnect.signTransaction (facade) — packages/connect-common/src/factory.ts → closure calls impl.call({method:'signTransaction',...}) [DYNAMIC: facade / iframe-or-module boundary]\n7. CoreInModule.call — packages/connect/src/impl/core-in-module.ts:171 → posts CORE_CALL to core (deferred promise)\n8. Core.onCall → getMethod — packages/connect/src/core/index.ts → getMethod resolves 'signTransaction' → new SignTransaction(message) [DYNAMIC: name→class]\n9. SignTransaction.run — packages/connect/src/api/signTransaction.ts:317 → signTx via device.getCommands().typedCall (protobuf to device). After signing, signAndPushSendFormTransactionThunk → pushSendFormTransactionThunk → TrezorConnect.pushTransaction.\nKEY SYMBOLS: ReviewButton → useSendForm.sign → signAndPushSendFormTransactionThunk (suite) → signTransactionThunk (wallet-core, coin hub) → signBitcoinSendFormTransactionThunk → TrezorConnect.signTransaction → connect factory/CoreInModule.call → SignTransaction.run.\nDYNAMIC BOUNDARIES: every suite→wallet-core hop is a redux thunk dispatch; signTransactionThunk branches by networkType at runtime; TrezorConnect is a dynamically-built facade crossing an iframe/module boundary; the SignTransaction class is resolved by name. A correct answer must cross suite→wallet-core→connect and reach TrezorConnect.signTransaction / SignTransaction.run, not stop at the UI."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env node
|
||||
// UserPromptSubmit hook — APPROACH 1: additive context-injection.
|
||||
// Front-loads codegraph's structural answer for flow/impact/"how/where" prompts so the
|
||||
// agent's reflex grep/read has nothing left to find. Strictly additive (never blocks),
|
||||
// gated to structural prompts (no cost otherwise), and uses RAW explore (offload disabled)
|
||||
// so the injected context is accurate — never the (currently low-fidelity) synthesis.
|
||||
//
|
||||
// Reads {prompt, cwd} as JSON on stdin; prints the explore result to stdout (which Claude
|
||||
// Code injects into the agent's context). Any failure -> silent exit 0 (degradable).
|
||||
import { pathToFileURL, fileURLToPath } from 'node:url';
|
||||
import { resolve, join, dirname } from 'node:path';
|
||||
import { existsSync, readFileSync, appendFileSync } from 'node:fs';
|
||||
|
||||
// Resolve the engine repo from this script's own location (scripts/agent-eval/ -> ../..),
|
||||
// overridable with CG_ENGINE. The hook ships inside the repo, so it finds its own dist.
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
|
||||
const BUDGET = Number(process.env.CG_FRONTLOAD_BUDGET || 16000);
|
||||
|
||||
// Debug log only when CG_FRONTLOAD_DEBUG is set to a file path (the harness points it at a
|
||||
// log to count injections); off by default so the shipped hook writes nothing extra.
|
||||
const DBG = process.env.CG_FRONTLOAD_DEBUG;
|
||||
const dbg = (m) => { if (!DBG) return; try { appendFileSync(DBG, `[${new Date().toISOString()}] ${m}\n`); } catch { /* ignore */ } };
|
||||
|
||||
let input = {};
|
||||
try { input = JSON.parse(readFileSync(0, 'utf8')); } catch (e) { dbg('stdin parse fail: ' + e.message); }
|
||||
const prompt = String(input.prompt || '');
|
||||
const cwd = String(input.cwd || process.cwd());
|
||||
dbg(`invoked: promptLen=${prompt.length} cwd=${cwd}`);
|
||||
|
||||
// Gate: only structural / flow / impact / where-how questions. Cheap regex; silent no-op
|
||||
// otherwise so non-structural prompts ("fix this typo") cost nothing.
|
||||
const STRUCTURAL = /\b(how|where|trace|flow|path|reach(es|ed)?|call(s|ed|er|ers|ee)?|depend|impact|affect|wire[ds]?|connect|implement|architect|structure|breaks?|what calls|why does)\b/i;
|
||||
if (!prompt || !STRUCTURAL.test(prompt)) { dbg('gate: non-structural, no-op'); process.exit(0); }
|
||||
dbg('gate: structural PASS');
|
||||
|
||||
// Find the index: cwd, then walk up a few levels.
|
||||
let root = cwd, found = null;
|
||||
for (let i = 0; i < 6 && root; i++) {
|
||||
if (existsSync(join(root, '.codegraph'))) { found = root; break; }
|
||||
const parent = resolve(root, '..'); if (parent === root) break; root = parent;
|
||||
}
|
||||
if (!found) { dbg(`no .codegraph found from cwd=${cwd}`); process.exit(0); }
|
||||
dbg(`found index at ${found}`);
|
||||
|
||||
try {
|
||||
process.env.CODEGRAPH_OFFLOAD_DISABLE = '1'; // raw, accurate — never the unfixed offload
|
||||
process.env.CODEGRAPH_TELEMETRY = '0'; process.env.DO_NOT_TRACK = '1';
|
||||
const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') process.exit(0);
|
||||
|
||||
// Retry once on a transient busy/locked index (the hook's openSync can race a
|
||||
// freshly-warming daemon on the first prompt of a session).
|
||||
let text = '';
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
try {
|
||||
const cg = CodeGraph.openSync(found);
|
||||
const h = new ToolHandler(cg);
|
||||
const res = await h.execute('codegraph_explore', { query: prompt });
|
||||
text = res?.content?.[0]?.text ?? '';
|
||||
try { cg.close?.(); } catch { /* ignore */ }
|
||||
dbg(`explore attempt ${attempt} returned ${text.length} chars`);
|
||||
break;
|
||||
} catch (e) {
|
||||
dbg(`explore attempt ${attempt} failed: ${e?.message || e}`);
|
||||
if (attempt === 2) throw e;
|
||||
await new Promise((r) => setTimeout(r, 800));
|
||||
}
|
||||
}
|
||||
if (!text.trim()) { dbg('empty explore result, no-op'); process.exit(0); }
|
||||
if (text.length > BUDGET) text = text.slice(0, BUDGET) + '\n…[front-load truncated to budget]';
|
||||
|
||||
process.stdout.write(
|
||||
`## CodeGraph structural context (auto-retrieved for this question)\n` +
|
||||
`The code graph was queried for your question; the relevant symbols, source, and call flow are below. ` +
|
||||
`Treat the quoted source as already read. If you need more, call codegraph_explore with specific symbol names rather than grepping or reading files.\n\n` +
|
||||
text + '\n'
|
||||
);
|
||||
dbg(`INJECTED ${text.length} chars`);
|
||||
} catch (e) { dbg('ERROR: ' + (e?.stack || e?.message || e)); process.exit(0); } // degradable
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env node
|
||||
// Accuracy judge. For each run in results.jsonl:
|
||||
// - end-to-end: agent finalAnswer vs verified ground truth (all arms)
|
||||
// - fidelity: offload synthesized answer vs ground truth (offload arm only)
|
||||
// Judge = claude -p sonnet --effort high, no tools, run from a neutral cwd,
|
||||
// JSON-only verdicts. Writes judged.jsonl (one line per run, verdicts merged).
|
||||
//
|
||||
// Usage: judge.mjs --results <f> --truth <f> --out <f> [--concurrency 4]
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { execFile } from 'child_process';
|
||||
|
||||
const A = {};
|
||||
for (let i = 2; i < process.argv.length; i += 2) A[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
|
||||
const results = readFileSync(A.results, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
|
||||
const truth = JSON.parse(readFileSync(A.truth, 'utf8'));
|
||||
const OUT = A.out || '/tmp/cg-offload-eval/judged.jsonl';
|
||||
const CONC = Number(A.concurrency || 4);
|
||||
|
||||
function askJudge(prompt) {
|
||||
return new Promise((resolve) => {
|
||||
execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high',
|
||||
'--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
|
||||
// Run from a neutral dir with no repo files so the judge can't "cheat" by reading source.
|
||||
{ cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 },
|
||||
(err, stdout) => {
|
||||
const raw = (stdout || '').trim();
|
||||
const m = raw.match(/\{[\s\S]*\}/);
|
||||
if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) });
|
||||
try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); }
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools.
|
||||
|
||||
QUESTION: ${gt.question}
|
||||
|
||||
VERIFIED GROUND TRUTH (the actual call path + files):
|
||||
${gt.truth}
|
||||
|
||||
AGENT'S ANSWER:
|
||||
${ans || '(empty)'}
|
||||
|
||||
Score how correct the agent's answer is vs the ground truth. A "pass" means it identifies the core mechanism and the major hops with the right files/symbols and makes no materially wrong claim. "partial" = right area but misses major hops or has notable errors. "fail" = wrong layer, fabricated, or misses the mechanism.
|
||||
Output ONLY minified JSON, no prose, no code fences:
|
||||
{"verdict":"pass|partial|fail","score":<0-100>,"missedHops":["..."],"wrongClaims":["..."],"note":"<=20 words"}`;
|
||||
|
||||
const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. The synthesized answer claims to trace a flow and cite file:line locations. Do NOT use any tools.
|
||||
|
||||
QUESTION: ${gt.question}
|
||||
|
||||
VERIFIED GROUND TRUTH (the actual call path + files):
|
||||
${gt.truth}
|
||||
|
||||
SYNTHESIZED ANSWER (to score):
|
||||
${ans || '(empty)'}
|
||||
|
||||
Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was that verdict honest about what it actually covered? A confident WRONG trace is the worst outcome — penalize it harder than an honest "partial/not found".
|
||||
Output ONLY minified JSON, no prose, no code fences:
|
||||
{"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"missedHops":["..."],"note":"<=20 words"}`;
|
||||
|
||||
// Build the job list
|
||||
const jobs = [];
|
||||
for (const r of results) {
|
||||
const gt = truth[r.repo];
|
||||
if (!gt) { r._nojudge = true; continue; }
|
||||
jobs.push({ r, kind: 'e2e', prompt: e2ePrompt(gt, r.finalAnswer) });
|
||||
if (r.arm === 'offload' && Array.isArray(r.offloadAnswers))
|
||||
r.offloadAnswers.forEach((ans, i) => { if (ans && ans.trim()) jobs.push({ r, kind: 'fid', idx: i, prompt: fidPrompt(gt, ans) }); });
|
||||
}
|
||||
console.error(`judging ${jobs.length} verdicts across ${results.length} runs (concurrency ${CONC})...`);
|
||||
|
||||
let done = 0;
|
||||
async function worker(queue) {
|
||||
while (queue.length) {
|
||||
const job = queue.shift();
|
||||
const v = await askJudge(job.prompt);
|
||||
if (job.kind === 'e2e') job.r.e2e = v; else (job.r._fid ??= []).push(v);
|
||||
console.error(` [${++done}/${jobs.length}] ${job.r.repo}/${job.r.arm}#${job.r.rep} ${job.kind}: ${v.verdict}${v.score != null ? ' ' + v.score : ''}`);
|
||||
}
|
||||
}
|
||||
const q = [...jobs];
|
||||
await Promise.all(Array.from({ length: CONC }, () => worker(q)));
|
||||
|
||||
// Aggregate per-answer fidelity verdicts into one fidelity object per offload run.
|
||||
const medOf = (a) => { a = [...a].sort((x, y) => x - y); return a.length ? (a.length % 2 ? a[(a.length - 1) / 2] : (a[a.length / 2 - 1] + a[a.length / 2]) / 2) : null; };
|
||||
for (const r of results) {
|
||||
if (r._fid?.length) {
|
||||
const scores = r._fid.map(v => v.score).filter(x => x != null);
|
||||
r.fidelity = {
|
||||
n: r._fid.length, scores,
|
||||
max: scores.length ? Math.max(...scores) : null,
|
||||
min: scores.length ? Math.min(...scores) : null,
|
||||
median: medOf(scores),
|
||||
anyFabrication: r._fid.some(v => v.fabrication === true),
|
||||
allCoverageHonest: r._fid.every(v => v.coverageHonest !== false),
|
||||
verdicts: r._fid.map(v => v.verdict),
|
||||
};
|
||||
}
|
||||
delete r._fid;
|
||||
}
|
||||
writeFileSync(OUT, results.map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
console.error(`wrote ${OUT}`);
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Drive the 3-arm campaign (offload/raw/nocg) across all 4 tiers, n reps each, into one
|
||||
# results.jsonl. Reads the canonical question per repo from offload-eval-ground-truth.json.
|
||||
# Env: REPS (default 3) AGENT_EVAL_OUT=<scratch dir>
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
GT="$HERE/offload-eval-ground-truth.json"
|
||||
REPS="${REPS:-3}"
|
||||
export RESULTS="$OUT/results.jsonl"
|
||||
: > "$RESULTS"
|
||||
for repo in mtkruto postybirb shapeshift trezor; do
|
||||
case "$repo" in mtkruto) tier=small;; postybirb) tier=medium;; shapeshift) tier=complex;; trezor) tier=large;; esac
|
||||
Q=$(node -e "console.log(JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'))[process.argv[2]].question)" "$GT" "$repo")
|
||||
echo ""; echo "### $repo ($tier) $(date +%H:%M:%S)"
|
||||
bash "$HERE/offload-eval-3arm.sh" "$OUT/repos/$repo" "$tier" "$REPS" "$Q"
|
||||
done
|
||||
echo ""; echo "###### MATRIX DONE -> $RESULTS ($(wc -l < "$RESULTS") runs). Judge + summarize with:"
|
||||
echo " node $HERE/offload-eval-judge.mjs --results $RESULTS --truth $GT --out $OUT/judged.jsonl"
|
||||
echo " node $HERE/offload-eval-summarize.mjs $OUT/judged.jsonl"
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env node
|
||||
// Extract one eval run's metrics from its Claude stream-json transcript + the
|
||||
// offload usage sidecar log, emit ONE merged JSON line.
|
||||
//
|
||||
// Usage: extract-metrics.mjs --run <run.jsonl> --usage <usage.jsonl|-> \
|
||||
// --arm <a> --rep <n> --repo <r> --tier <t> --q <question>
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
|
||||
const args = {};
|
||||
for (let i = 2; i < process.argv.length; i += 2) args[process.argv[i].replace(/^--/, '')] = process.argv[i + 1];
|
||||
|
||||
const runFile = args.run;
|
||||
const lines = existsSync(runFile) ? readFileSync(runFile, 'utf8').split('\n').filter(Boolean) : [];
|
||||
|
||||
const toolCounts = {};
|
||||
let result = null;
|
||||
const tok = { gen: 0, fresh: 0, cached: 0 };
|
||||
const offloadAnswers = [];
|
||||
let exploreResults = 0; // tool_results from explore (offload or raw)
|
||||
let lastAssistantText = '';
|
||||
|
||||
for (const line of lines) {
|
||||
let ev; try { ev = JSON.parse(line); } catch { continue; }
|
||||
|
||||
// per-turn token usage (authoritative token measure; result.usage is last-turn only)
|
||||
const u = ev.message?.usage;
|
||||
if (u) {
|
||||
tok.gen += u.output_tokens || 0;
|
||||
tok.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
|
||||
tok.cached += u.cache_read_input_tokens || 0;
|
||||
}
|
||||
|
||||
if (ev.type === 'assistant' && Array.isArray(ev.message?.content)) {
|
||||
for (const b of ev.message.content) {
|
||||
if (b.type === 'tool_use') toolCounts[b.name] = (toolCounts[b.name] || 0) + 1;
|
||||
if (b.type === 'text' && b.text?.trim()) lastAssistantText = b.text.trim();
|
||||
}
|
||||
}
|
||||
// tool_results arrive in user messages
|
||||
if (ev.type === 'user' && Array.isArray(ev.message?.content)) {
|
||||
for (const b of ev.message.content) {
|
||||
if (b.type !== 'tool_result') continue;
|
||||
const text = Array.isArray(b.content)
|
||||
? b.content.map(c => (typeof c === 'string' ? c : c.text || '')).join('')
|
||||
: (typeof b.content === 'string' ? b.content : '');
|
||||
// An offload answer is either the 'plain'/'report' synthesis (carries the
|
||||
// "Synthesized by CodeGraph" footer) or a 'refs' answer (carries the re-expanded
|
||||
// "### Referenced source — verbatim" appendix). A refs call that cited nothing
|
||||
// valid falls back to RAW source, which is correctly counted as a raw explore below.
|
||||
if (/Synthesized by CodeGraph|### Referenced source — verbatim/.test(text)) { offloadAnswers.push(text); exploreResults++; }
|
||||
else if (/Found \d+ symbols? across|\*\*Exploration:/.test(text)) exploreResults++;
|
||||
}
|
||||
}
|
||||
if (ev.type === 'result') result = ev;
|
||||
}
|
||||
|
||||
// offload usage sidecar (CodeGraph AI tokens + cost) — one JSON line per offload call
|
||||
const ai = { calls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, credits: 0, costUsd: 0, ms: 0 };
|
||||
if (args.usage && args.usage !== '-' && existsSync(args.usage)) {
|
||||
for (const line of readFileSync(args.usage, 'utf8').split('\n').filter(Boolean)) {
|
||||
let e; try { e = JSON.parse(line); } catch { continue; }
|
||||
ai.calls++;
|
||||
ai.promptTokens += e.promptTokens || 0;
|
||||
ai.completionTokens += e.completionTokens || 0;
|
||||
ai.totalTokens += e.totalTokens || 0;
|
||||
ai.credits += e.creditsCharged || 0;
|
||||
ai.costUsd += e.costUsd || 0;
|
||||
ai.ms += e.ms || 0;
|
||||
}
|
||||
}
|
||||
|
||||
// front-load hook fired iff its injected header appears in the transcript
|
||||
const frontload = lines.some(l => l.includes('auto-retrieved for this question'));
|
||||
const get = (n) => toolCounts[n] || 0;
|
||||
const read = get('Read');
|
||||
const grep = get('Grep') + get('Bash') + get('Glob');
|
||||
const explore = get('mcp__codegraph__codegraph_explore');
|
||||
const cgAny = Object.keys(toolCounts).filter(k => /mcp__codegraph__/.test(k)).reduce((s, k) => s + toolCounts[k], 0);
|
||||
|
||||
const out = {
|
||||
repo: args.repo, tier: args.tier, arm: args.arm, rep: Number(args.rep), question: args.q,
|
||||
ok: result?.subtype === 'success',
|
||||
durationSec: result ? +(result.duration_ms / 1000).toFixed(1) : null,
|
||||
numTurns: result?.num_turns ?? null,
|
||||
costUsdMain: result ? +(result.total_cost_usd || 0).toFixed(4) : null,
|
||||
tokGen: tok.gen, tokFresh: tok.fresh, tokCached: tok.cached, tokBillable: tok.gen + tok.fresh,
|
||||
read, grep, explore, cgAny, frontload,
|
||||
offloadFired: offloadAnswers.length,
|
||||
ai,
|
||||
// text payloads for the accuracy judge (kept separate; large)
|
||||
finalAnswer: (result?.result || lastAssistantText || '').slice(0, 8000),
|
||||
offloadAnswers: offloadAnswers.map(a => a.slice(0, 6000)),
|
||||
};
|
||||
process.stdout.write(JSON.stringify(out) + '\n');
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# ONE offload run on ONE indexed repo at a given offload STYLE (plain|refs), so we can
|
||||
# watch a single agent transcript at a time (the user's one-run-at-a-time methodology).
|
||||
# The OFFLOAD reasoning runs in the prewarmed DAEMON process, so the style env must be
|
||||
# set on BOTH the daemon and the client MCP config. Writes one metrics line to RESULTS
|
||||
# and leaves the raw stream-json at $RUNS/<repo>-<style>-<n>.jsonl for inspection.
|
||||
#
|
||||
# Usage: offload-eval-refs1.sh <indexed-repo> <style> <n> "<question>"
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"; ENGINE="$(cd "$HERE/../.." && pwd)"; BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"; RUNS="$OUT/runs"; EXTRACT="$HERE/offload-eval-metrics.mjs"
|
||||
TARGET="${1:?repo}"; STYLE="${2:?style}"; N="${3:?run-tag}"; Q="${4:?question}"
|
||||
RESULTS="${RESULTS:-$OUT/results-refs.jsonl}"; REPO=$(basename "$TARGET"); TARGET=$(cd "$TARGET" && pwd -P)
|
||||
mkdir -p "$RUNS"; command -v claude >/dev/null || { echo "no claude"; exit 1; }
|
||||
USAGE="$RUNS/$REPO-$STYLE-usage.jsonl"; : > "$USAGE"
|
||||
CFG="$RUNS/mcp-$REPO-$STYLE.json"
|
||||
# `raw` is a pseudo-style: codegraph attached but the offload DISABLED (the ceiling —
|
||||
# verbatim source, no reasoning model). Any other value is an offload style (plain|refs).
|
||||
if [ "$STYLE" = "raw" ]; then
|
||||
DAEMON_ENV="CODEGRAPH_OFFLOAD_DISABLE=1"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_DISABLE=1","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$BIN" "$TARGET" > "$CFG"
|
||||
USAGE="-"
|
||||
else
|
||||
DAEMON_ENV="CODEGRAPH_OFFLOAD_STYLE=$STYLE CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1","CODEGRAPH_OFFLOAD_STYLE=%s","CODEGRAPH_OFFLOAD_USAGE_LOG=%s","node","%s","serve","--mcp","--path","%s"]}}}' \
|
||||
"$STYLE" "$USAGE" "$BIN" "$TARGET" > "$CFG"
|
||||
fi
|
||||
|
||||
# Prewarm a persistent daemon carrying the SAME offload config (it does the reasoning).
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
|
||||
env $DAEMON_ENV CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 \
|
||||
node "$BIN" serve --mcp --path "$TARGET" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$TARGET" \
|
||||
&& echo "daemon warm ($STYLE)" || echo "WARN daemon never bound"
|
||||
|
||||
tag="$REPO-$STYLE-$N"
|
||||
echo "== run $tag =="
|
||||
# DISALLOW (optional): block tools that confound the offload-sufficiency signal —
|
||||
# chiefly "Agent" (sub-agent delegation: the spawned Explore subagent has low MCP
|
||||
# salience, ignores codegraph, and thrashes via Bash+Read, making the A/B noise).
|
||||
( cd "$TARGET" && claude -p "$Q" --output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
${DISALLOW:+--disallowedTools "$DISALLOW"} \
|
||||
--strict-mcp-config --mcp-config "$CFG" </dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
|
||||
node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$USAGE" --arm "offload-$STYLE" --rep "$N" \
|
||||
--repo "$REPO" --tier "complex" --q "$Q" >> "$RESULTS"
|
||||
node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(` [${o.arm} #${o.rep}] ${o.durationSec}s | main $${o.costUsdMain} ${o.tokBillable} tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}call/${o.ai.totalTokens}tok/$${o.ai.costUsd.toFixed(4)} | ok=${o.ok}`)' "$RESULTS"
|
||||
pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null
|
||||
echo "raw transcript: $RUNS/$tag.jsonl"
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Clone + index the 4 "not-trained-on" eval repos into $AGENT_EVAL_OUT/repos. These were
|
||||
# selected via a no-tools memory-probe gate (Sonnet cannot answer their flow questions from
|
||||
# memory — so the no-codegraph baseline is honest). Env: AGENT_EVAL_OUT=<scratch dir>
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENGINE="$(cd "$HERE/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
ROOT="$OUT/repos"; mkdir -p "$ROOT"
|
||||
export CODEGRAPH_TELEMETRY=0 DO_NOT_TRACK=1
|
||||
[ -f "$BIN" ] || { echo "engine not built: run 'npm run build' in $ENGINE first"; exit 1; }
|
||||
|
||||
clone_index() { # url name
|
||||
echo "=== $2: clone ==="; rm -rf "$ROOT/$2"
|
||||
git clone --quiet --depth 1 "$1" "$ROOT/$2" || { echo " clone FAILED"; return 1; }
|
||||
echo "=== $2: index ==="
|
||||
node "$BIN" init "$ROOT/$2" 2>&1 | grep -iE 'indexed|nodes|edges|error' | tail -2
|
||||
}
|
||||
clone_index https://github.com/MTKruto/MTKruto.git mtkruto # small (~322 TS)
|
||||
clone_index https://github.com/mvdicarlo/postybirb-plus.git postybirb # medium (~608 TS)
|
||||
clone_index https://github.com/shapeshift/web.git shapeshift # complex (~3.2k TS, 35-pkg monorepo)
|
||||
clone_index https://github.com/trezor/trezor-suite.git trezor # large (~8k TS monorepo)
|
||||
echo "###### SETUP DONE -> $ROOT"
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# Offload reasoning-OUTPUT-STYLE A/B — all codegraph-on, isolating the Worker's
|
||||
# output shape's effect on main-session tokens / latency / accuracy:
|
||||
# raw : CODEGRAPH_OFFLOAD_DISABLE=1 (verbatim explore source, the floor)
|
||||
# refs : managed offload, default (Cerebras map re-expanded to verbatim, ~24K)
|
||||
# map : managed offload, STYLE=map (compact reasoned map + file:line anchors, ~1-3K)
|
||||
# src : managed offload, STYLE=src (map + cited line ranges only, ~1-5K)
|
||||
# Delegation BLOCKED by default (DISALLOW=Agent) so we measure the offload payload's
|
||||
# effect on the main Sonnet agent, not whether it spawns a Haiku Explore subagent.
|
||||
#
|
||||
# Usage: offload-eval-styles.sh <indexed-repo> <reps> "<question>"
|
||||
# Env: RESULTS=<file> AGENT_EVAL_OUT=<dir> REP_START=1 DISALLOW=Agent MODEL/EFFORT
|
||||
set -uo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENGINE="$(cd "$HERE/../.." && pwd)"
|
||||
BIN="$ENGINE/dist/bin/codegraph.js"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/cg-offload-eval}"
|
||||
TARGET="${1:?usage: offload-eval-styles.sh <indexed-repo> <reps> \"<question>\"}"
|
||||
REPS="${2:?reps}"; Q="${3:?question}"
|
||||
RUNS="$OUT/runs"; EXTRACT="$HERE/offload-eval-metrics.mjs"
|
||||
RESULTS="${RESULTS:-$OUT/results-styles.jsonl}"
|
||||
REPO=$(basename "$TARGET")
|
||||
DISALLOW="${DISALLOW-Agent}" # default: block delegation. `DISALLOW= ` to allow.
|
||||
START="${REP_START:-1}"; END=$((START + REPS - 1))
|
||||
mkdir -p "$RUNS"
|
||||
command -v claude >/dev/null || { echo "no claude on PATH"; exit 1; }
|
||||
[ -d "$TARGET/.codegraph" ] || { echo "not indexed: $TARGET"; exit 1; }
|
||||
TARGET=$(cd "$TARGET" && pwd -P)
|
||||
|
||||
prewarm() { # path extra-env
|
||||
pkill -9 -f "serve --mcp --path $1" 2>/dev/null; rm -f "$1/.codegraph/daemon.sock" 2>/dev/null; sleep 0.6
|
||||
env ${2:-} CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS=1800000 node "$BIN" serve --mcp --path "$1" </dev/null >/dev/null 2>&1 &
|
||||
node -e 'const fs=require("fs");let n=0;const t=setInterval(()=>{if(fs.existsSync(process.argv[1]+"/.codegraph/daemon.sock")){clearInterval(t);process.exit(0)}if(n++>150){clearInterval(t);process.exit(1)}},100)' "$1" \
|
||||
&& echo " daemon warm" || echo " WARN daemon never bound"
|
||||
}
|
||||
kill_daemon() { pkill -9 -f "serve --mcp --path $TARGET" 2>/dev/null; rm -f "$TARGET/.codegraph/daemon.sock" 2>/dev/null; sleep 1; }
|
||||
|
||||
run() { # arm rep mcp-config usage-log-or-dash
|
||||
local arm="$1" rep="$2" cfg="$3" usage="$4" tag="$REPO-$1-$2"
|
||||
[ "$usage" != "-" ] && : > "$usage"
|
||||
( cd "$TARGET" && claude -p "$Q" \
|
||||
--output-format stream-json --verbose --permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4 \
|
||||
${DISALLOW:+--disallowedTools "$DISALLOW"} \
|
||||
--strict-mcp-config --mcp-config "$cfg" \
|
||||
</dev/null > "$RUNS/$tag.jsonl" 2>"$RUNS/$tag.err" )
|
||||
node "$EXTRACT" --run "$RUNS/$tag.jsonl" --usage "$usage" --arm "$arm" --rep "$rep" \
|
||||
--repo "$REPO" --tier styles --q "$Q" >> "$RESULTS"
|
||||
node -e 'const o=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8").trim().split("\n").pop());console.log(` [${o.arm} #${o.rep}] ${o.durationSec}s | ${o.tokBillable} billable tok | read=${o.read} grep=${o.grep} explore=${o.explore} offload=${o.offloadFired} | AI ${o.ai.calls}c/${o.ai.totalTokens}t | ok=${o.ok}`)' "$RESULTS"
|
||||
}
|
||||
|
||||
# MCP configs: env baked into the daemon-spawn command claude uses.
|
||||
USAGE="$RUNS/$REPO-usage.jsonl"
|
||||
mkcfg() { # file extra-env-pairs(JSON array entries, comma-led or empty)
|
||||
printf '{"mcpServers":{"codegraph":{"command":"env","args":["CODEGRAPH_WASM_RELAUNCHED=1"%s,"node","%s","serve","--mcp","--path","%s"]}}}' "$1" "$BIN" "$TARGET"
|
||||
}
|
||||
CFG_RAW="$RUNS/mcp-sty-raw-$REPO.json"; mkcfg ',"CODEGRAPH_OFFLOAD_DISABLE=1"' > "$CFG_RAW"
|
||||
CFG_REFS="$RUNS/mcp-sty-refs-$REPO.json"; mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\"" > "$CFG_REFS"
|
||||
CFG_MAP="$RUNS/mcp-sty-map-$REPO.json"; mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\",\"CODEGRAPH_OFFLOAD_STYLE=map\"" > "$CFG_MAP"
|
||||
CFG_SRC="$RUNS/mcp-sty-src-$REPO.json"; mkcfg ",\"CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE\",\"CODEGRAPH_OFFLOAD_STYLE=src\"" > "$CFG_SRC"
|
||||
|
||||
echo "###### repo=$REPO reps=$START..$END model=${MODEL:-sonnet}/${EFFORT:-high} disallow=${DISALLOW:-<none>}"
|
||||
echo "###### Q=$Q"
|
||||
echo "== ARM raw =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_DISABLE=1"
|
||||
for r in $(seq "$START" "$END"); do run raw "$r" "$CFG_RAW" "-"; done; kill_daemon
|
||||
echo "== ARM refs =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE"
|
||||
for r in $(seq "$START" "$END"); do run refs "$r" "$CFG_REFS" "$USAGE"; done; kill_daemon
|
||||
echo "== ARM map =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE CODEGRAPH_OFFLOAD_STYLE=map"
|
||||
for r in $(seq "$START" "$END"); do run map "$r" "$CFG_MAP" "$USAGE"; done; kill_daemon
|
||||
echo "== ARM src =="; prewarm "$TARGET" "CODEGRAPH_OFFLOAD_USAGE_LOG=$USAGE CODEGRAPH_OFFLOAD_STYLE=src"
|
||||
for r in $(seq "$START" "$END"); do run src "$r" "$CFG_SRC" "$USAGE"; done; kill_daemon
|
||||
echo "###### DONE $REPO — judge: node $HERE/offload-eval-judge.mjs --results $RESULTS --truth $HERE/offload-eval-ground-truth.json --out $OUT/judged-styles.jsonl"
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env node
|
||||
// Aggregate judged.jsonl (or results.jsonl) into a per-repo, per-arm report:
|
||||
// time, main tokens/cost, AI tokens/cost, total cost, tool mix, accuracy.
|
||||
// Usage: summarize.mjs <judged-or-results.jsonl>
|
||||
import { readFileSync } from 'fs';
|
||||
const rows = readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
|
||||
|
||||
const med = (xs) => { const a = xs.filter(x => x != null).sort((p, q) => p - q); if (!a.length) return null; const m = Math.floor(a.length / 2); return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2; };
|
||||
const rng = (xs) => { const a = xs.filter(x => x != null); return a.length ? `${Math.min(...a)}–${Math.max(...a)}` : '—'; };
|
||||
const d2 = (x) => x == null ? '—' : (+x).toFixed(2);
|
||||
const d3 = (x) => x == null ? '—' : (+x).toFixed(3);
|
||||
const d4 = (x) => x == null ? '—' : (+x).toFixed(4);
|
||||
|
||||
const ARM_ORDER = ['frontload', 'offload', 'raw', 'nocg'];
|
||||
const byRepo = {};
|
||||
for (const r of rows) (byRepo[r.repo] ??= {});
|
||||
for (const r of rows) ((byRepo[r.repo][r.arm] ??= []).push(r));
|
||||
|
||||
const verdictTally = (rs, field) => {
|
||||
const t = { pass: 0, partial: 0, fail: 0, error: 0 };
|
||||
for (const r of rs) { const v = r[field]?.verdict; if (v in t) t[v]++; }
|
||||
return t;
|
||||
};
|
||||
|
||||
for (const repo of Object.keys(byRepo)) {
|
||||
const tier = byRepo[repo][Object.keys(byRepo[repo])[0]][0].tier;
|
||||
console.log(`\n${'='.repeat(78)}\n${repo} [${tier}]\n${'='.repeat(78)}`);
|
||||
console.log(`${'arm'.padEnd(9)} n ${'time(s)'.padStart(9)} ${'mainCost'.padStart(9)} ${'aiCost'.padStart(8)} ${'totCost'.padStart(8)} ${'mainTok'.padStart(8)} ${'aiTok'.padStart(7)} ${'rd'.padStart(3)} ${'gr'.padStart(3)} ${'exp'.padStart(3)} ${'off'.padStart(3)} e2e(P/p/F) fidScore`);
|
||||
for (const arm of ARM_ORDER) {
|
||||
const rs = byRepo[repo][arm]; if (!rs) continue;
|
||||
const n = rs.length;
|
||||
const mainCost = med(rs.map(r => r.costUsdMain));
|
||||
const aiCost = med(rs.map(r => r.ai?.costUsd ?? 0));
|
||||
const totCost = (mainCost ?? 0) + (aiCost ?? 0);
|
||||
const e2e = verdictTally(rs, 'e2e');
|
||||
const fidScores = arm === 'offload' ? rs.flatMap(r => r.fidelity?.scores ?? []) : [];
|
||||
const fid = fidScores.length ? med(fidScores) : null;
|
||||
const fab = arm === 'offload' && rs.some(r => r.fidelity?.anyFabrication);
|
||||
const e2eScore = med(rs.map(r => r.e2e?.score).filter(x => x != null));
|
||||
console.log(
|
||||
`${arm.padEnd(9)} ${String(n).padStart(1)} ${String(med(rs.map(r => r.durationSec))).padStart(9)} ` +
|
||||
`${('$' + d3(mainCost)).padStart(9)} ${('$' + d3(aiCost)).padStart(8)} ${('$' + d3(totCost)).padStart(8)} ` +
|
||||
`${String(Math.round(med(rs.map(r => r.tokBillable)) / 1000) + 'k').padStart(8)} ${String(Math.round(med(rs.map(r => r.ai?.totalTokens ?? 0)) / 1000) + 'k').padStart(7)} ` +
|
||||
`${String(med(rs.map(r => r.read))).padStart(3)} ${String(med(rs.map(r => r.grep))).padStart(3)} ${String(med(rs.map(r => r.explore))).padStart(3)} ${String(med(rs.map(r => r.offloadFired))).padStart(3)} ` +
|
||||
`${(e2e.pass + '/' + e2e.partial + '/' + e2e.fail).padStart(9)} ${e2eScore != null ? 'e2e=' + e2eScore : ''} ${fid != null ? 'fid=' + fid + (fab ? ' FAB!' : '') : ''}`
|
||||
);
|
||||
}
|
||||
// ranges line for the two key metrics (variance matters)
|
||||
for (const arm of ARM_ORDER) {
|
||||
const rs = byRepo[repo][arm]; if (!rs) continue;
|
||||
console.log(` ${arm} ranges: time ${rng(rs.map(r => r.durationSec))}s · mainCost $${rng(rs.map(r => r.costUsdMain))} · read ${rng(rs.map(r => r.read))} · explore ${rng(rs.map(r => r.explore))} · offloadFired ${rng(rs.map(r => r.offloadFired))}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Cross-repo roll-up: offload vs raw vs nocg deltas
|
||||
console.log(`\n${'='.repeat(78)}\nCROSS-REPO SUMMARY (medians per repo, then averaged)\n${'='.repeat(78)}`);
|
||||
console.log(`${'repo'.padEnd(12)} ${'arm'.padEnd(8)} ${'time'.padStart(7)} ${'totCost'.padStart(8)} ${'read'.padStart(5)} ${'e2e pass%'.padStart(9)} ${'fid'.padStart(5)}`);
|
||||
for (const repo of Object.keys(byRepo)) {
|
||||
for (const arm of ARM_ORDER) {
|
||||
const rs = byRepo[repo][arm]; if (!rs) continue;
|
||||
const e2e = verdictTally(rs, 'e2e');
|
||||
const passPct = Math.round(100 * e2e.pass / rs.length);
|
||||
const totCost = (med(rs.map(r => r.costUsdMain)) ?? 0) + (med(rs.map(r => r.ai?.costUsd ?? 0)) ?? 0);
|
||||
const fid = arm === 'offload' ? med(rs.flatMap(r => r.fidelity?.scores ?? [])) : null;
|
||||
console.log(`${repo.padEnd(12)} ${arm.padEnd(8)} ${(med(rs.map(r => r.durationSec)) + 's').padStart(7)} ${('$' + d3(totCost)).padStart(8)} ${String(med(rs.map(r => r.read))).padStart(5)} ${(passPct + '%').padStart(9)} ${String(fid ?? '—').padStart(5)}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
@@ -0,0 +1,76 @@
|
||||
# CodeGraph AI offload — accuracy & adoption eval harness
|
||||
|
||||
Measures the managed **offload** (`codegraph_explore` → reasoning model synthesis) and the
|
||||
**front-load hook** (approach 1) against plain codegraph and no-codegraph, across repo sizes,
|
||||
on **time · main-session tokens/cost · CodeGraph-AI tokens/cost · accuracy**.
|
||||
|
||||
All agent arms run `claude -p --model sonnet --effort high` (the deliberate floor model — an
|
||||
affordance that lands on Sonnet generalizes up). Everything writes to a scratch dir
|
||||
(`AGENT_EVAL_OUT`, default `/tmp/cg-offload-eval`); nothing here is shipped to users.
|
||||
|
||||
## Repos (selected via a memory-probe gate — NOT trained on)
|
||||
|
||||
Famous repos (express, excalidraw, n8n, …) are useless for *accuracy* evals: Sonnet answers their
|
||||
flow questions from memory, so the no-codegraph baseline is dishonest. These four passed a no-tools
|
||||
probe (Sonnet could not name their real flow internals) and are cloned fresh by `offload-eval-setup.sh`:
|
||||
|
||||
| tier | repo | ~src files | canonical flow |
|
||||
|---|---|---|---|
|
||||
| small | MTKruto/MTKruto | 322 TS | `sendMessage` → invoke → TL serialize → transport |
|
||||
| medium | mvdicarlo/postybirb-plus | 608 TS | submission → queue → per-website `.post()` |
|
||||
| complex | shapeshift/web | 3.2k TS (35-pkg monorepo) | swap → swapper registry → concrete swapper |
|
||||
| large | trezor/trezor-suite | 8k TS monorepo | send-form → sign thunk → `@trezor/connect` |
|
||||
|
||||
Verified ground-truth flows (the judge's reference) live in `offload-eval-ground-truth.json`.
|
||||
|
||||
## Arms
|
||||
|
||||
- **offload** — codegraph + managed offload ON (requires `codegraph login`); records AI tokens/credits via `CODEGRAPH_OFFLOAD_USAGE_LOG`.
|
||||
- **raw** — codegraph, `CODEGRAPH_OFFLOAD_DISABLE=1` (returns raw source).
|
||||
- **nocg** — empty MCP config; Read/Grep baseline.
|
||||
- **frontload** — codegraph (offload-disabled) + a `UserPromptSubmit` hook (`offload-eval-hook.mjs`) that runs raw explore on the prompt and injects the result into context (approach 1).
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
npm run build # the harness shells out to dist/
|
||||
codegraph login # only needed for the offload arm
|
||||
export AGENT_EVAL_OUT=/tmp/cg-offload-eval
|
||||
|
||||
bash scripts/agent-eval/offload-eval-setup.sh # clone + index the 4 repos
|
||||
bash scripts/agent-eval/offload-eval-matrix.sh # 3 arms × 4 tiers × REPS (default 3)
|
||||
node scripts/agent-eval/offload-eval-judge.mjs \
|
||||
--results $AGENT_EVAL_OUT/results.jsonl \
|
||||
--truth scripts/agent-eval/offload-eval-ground-truth.json \
|
||||
--out $AGENT_EVAL_OUT/judged.jsonl
|
||||
node scripts/agent-eval/offload-eval-summarize.mjs $AGENT_EVAL_OUT/judged.jsonl
|
||||
|
||||
bash scripts/agent-eval/offload-eval-frontload-matrix.sh # frontload arm + judge + merged summary
|
||||
```
|
||||
|
||||
Single repo: `offload-eval-3arm.sh <indexed-repo> <tier> <reps> "<question>"` (or `-frontload.sh`).
|
||||
|
||||
## Files
|
||||
|
||||
- `offload-eval-setup.sh` — clone + index the 4 repos.
|
||||
- `offload-eval-3arm.sh` / `-frontload.sh` — one repo, the arms.
|
||||
- `offload-eval-matrix.sh` / `-frontload-matrix.sh` — drive all 4 tiers.
|
||||
- `offload-eval-hook.mjs` — the front-load `UserPromptSubmit` hook (resolves its own engine; `CG_FRONTLOAD_DEBUG=<path>` to log injections; `CG_FRONTLOAD_BUDGET` to cap injected chars).
|
||||
- `offload-eval-metrics.mjs` — one run's stream-json + usage log → one JSON metrics line.
|
||||
- `offload-eval-judge.mjs` — Sonnet judge: end-to-end (agent final vs ground truth) + per-answer offload fidelity.
|
||||
- `offload-eval-summarize.mjs` — per-tier, per-arm table + cross-repo roll-up.
|
||||
- `offload-eval-ground-truth.json` — source-verified canonical flows.
|
||||
|
||||
## Findings (2026-06, n=3 — direction consistent, magnitudes noisy)
|
||||
|
||||
- **Raw codegraph is the efficiency win** — ~nocg accuracy, fewer reads, faster, no AI cost.
|
||||
- **The offload is the least-accurate arm in all 4 tiers** — synthesized fidelity 12–27/100 with
|
||||
fabrication in 3/4 (e.g. invented website services; traced `ClientPlain`/`SessionPlain` instead of
|
||||
the real encrypted path). Its speed/cost win is narrow (medium-only) and inversely correlated with
|
||||
accuracy. **Use raw until offload fidelity is fixed.**
|
||||
- **The front-load hook SOLVES adoption** — reads → 0–1 in every tier (incl. large, where the agent
|
||||
otherwise read 12–24 files); fired 12/12, 0 errors. Wins on medium/complex (100% pass). But it
|
||||
**regresses small/large to partial** — it suppresses the reads that compensate for explore's gaps at
|
||||
**dynamic boundaries** (async queues, redux thunks, facade/factory indirection).
|
||||
- **Master lever for BOTH:** explore's dynamic-dispatch coverage. Fix it → front-load is complete
|
||||
everywhere and the offload has the full flow to synthesize.
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env node
|
||||
// Analyze the tool-surface ablation (/tmp/arms/<repo>/<arm>-r<n>.jsonl).
|
||||
// Compares arms A–E on trace adoption, Read/Grep fallback, codegraph payload,
|
||||
// round-trips, and duration — averaged across runs per arm.
|
||||
//
|
||||
// The decisive signal is READS: if removing a tool raises Reads on a question
|
||||
// class, that tool was load-bearing for it (not redundant). If removing it
|
||||
// changes nothing, it was redundant.
|
||||
//
|
||||
// A control all tools no steering (baseline)
|
||||
// B steer all tools trace-first (adoption)
|
||||
// C no-explore hide explore trace-first (is explore redundant?)
|
||||
// D trace-centric hide explore+context trace-first (is the survey pair redundant?)
|
||||
// E control-probe hide explore+context trace-first (NON-flow Q — should degrade)
|
||||
//
|
||||
// Usage: node scripts/agent-eval/parse-arms.mjs [/tmp/arms]
|
||||
import { readFileSync, readdirSync, existsSync, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const ROOT = process.argv[2] || '/tmp/arms';
|
||||
const cgShort = (n) => n.replace('mcp__codegraph__codegraph_', '').replace('mcp__codegraph__', '');
|
||||
|
||||
function parse(file) {
|
||||
if (!existsSync(file)) return null;
|
||||
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
||||
const calls = []; let result = null, initCg = 0;
|
||||
for (const l of lines) {
|
||||
let ev; try { ev = JSON.parse(l); } catch { continue; }
|
||||
if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /codegraph/.test(t)).length;
|
||||
if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use')
|
||||
calls.push({ id: b.id, name: b.name, out: 0 });
|
||||
if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
|
||||
const c = b.content;
|
||||
const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
|
||||
const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
|
||||
}
|
||||
if (ev.type === 'result') result = ev;
|
||||
}
|
||||
const cg = calls.filter(c => c.name.includes('codegraph'));
|
||||
return {
|
||||
initCg,
|
||||
reads: calls.filter(c => c.name === 'Read').length,
|
||||
greps: calls.filter(c => c.name === 'Grep').length + calls.filter(c => c.name === 'Glob').length,
|
||||
cgCalls: cg.length,
|
||||
cgSeq: cg.map(c => cgShort(c.name)),
|
||||
cgOut: cg.reduce((s, c) => s + c.out, 0),
|
||||
traceUsed: cg.some(c => c.name.includes('trace')),
|
||||
turns: result?.num_turns ?? null,
|
||||
dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
|
||||
cost: result?.total_cost_usd || 0,
|
||||
ok: result?.subtype === 'success',
|
||||
};
|
||||
}
|
||||
|
||||
// repo -> arm -> [runs]
|
||||
const data = {};
|
||||
if (!existsSync(ROOT)) { console.error(`no ${ROOT}`); process.exit(1); }
|
||||
for (const repo of readdirSync(ROOT)) {
|
||||
const rdir = join(ROOT, repo);
|
||||
if (!statSync(rdir).isDirectory()) continue;
|
||||
for (const f of readdirSync(rdir)) {
|
||||
const m = f.match(/^([A-I])-r(\d+)\.jsonl$/); if (!m) continue;
|
||||
const p = parse(join(rdir, f)); if (!p || !p.ok) continue;
|
||||
(((data[repo] ??= {})[m[1]]) ??= []).push(p);
|
||||
}
|
||||
}
|
||||
|
||||
const avg = (a, f) => a.length ? a.reduce((s, x) => s + (f(x) || 0), 0) / a.length : 0;
|
||||
const k = (n) => (n / 1000).toFixed(1);
|
||||
const pad = (s, n) => String(s).padEnd(n);
|
||||
const ARMS = ['A', 'H', 'I', 'B', 'F', 'G', 'C', 'D', 'E'];
|
||||
const LABEL = { A: 'A all/none(old)', H: 'H body-trace/none', I: 'I bodytrace+dest', B: 'B all/steer(thin)', F: 'F all/steer(body)', G: 'G ported(noprompt)', C: 'C no-explore', D: 'D trace-centric', E: 'E nonflow-probe' };
|
||||
|
||||
// ---- per repo × arm ----
|
||||
console.log('\n=== PER REPO × ARM (avg over runs) ===');
|
||||
console.log(pad('repo', 22), pad('arm', 16), 'tools', 'trace', pad('reads', 6), pad('cgOutK', 7), pad('turns', 6), 'dur');
|
||||
for (const repo of Object.keys(data).sort()) {
|
||||
for (const arm of ARMS) {
|
||||
const runs = data[repo][arm]; if (!runs?.length) continue;
|
||||
console.log(
|
||||
pad(repo, 22), pad(LABEL[arm], 16),
|
||||
pad(runs[0].initCg, 5),
|
||||
pad(runs.filter(r => r.traceUsed).length + '/' + runs.length, 5),
|
||||
pad(avg(runs, r => r.reads).toFixed(1), 6),
|
||||
pad(k(avg(runs, r => r.cgOut)), 7),
|
||||
pad(avg(runs, r => r.turns).toFixed(1), 6),
|
||||
avg(runs, r => r.dur).toFixed(0) + 's',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- aggregate per arm (flow arms A–D over the flow repos; E shown apart) ----
|
||||
console.log('\n=== AGGREGATE PER ARM (mean across repos) ===');
|
||||
console.log(pad('arm', 16), pad('adoption', 9), pad('reads', 7), pad('greps', 7), pad('cgOutK', 8), pad('turns', 7), pad('dur', 6), 'cost');
|
||||
for (const arm of ARMS) {
|
||||
const all = [];
|
||||
for (const repo of Object.keys(data)) for (const r of (data[repo][arm] || [])) all.push({ ...r, repo });
|
||||
if (!all.length) continue;
|
||||
const repos = new Set(all.map(r => r.repo)).size;
|
||||
const adopt = all.filter(r => r.traceUsed).length;
|
||||
console.log(
|
||||
pad(LABEL[arm], 16),
|
||||
pad(`${adopt}/${all.length}`, 9),
|
||||
pad(avg(all, r => r.reads).toFixed(2), 7),
|
||||
pad(avg(all, r => r.greps).toFixed(2), 7),
|
||||
pad(k(avg(all, r => r.cgOut)), 8),
|
||||
pad(avg(all, r => r.turns).toFixed(1), 7),
|
||||
pad(avg(all, r => r.dur).toFixed(0) + 's', 6),
|
||||
'$' + avg(all, r => r.cost).toFixed(3),
|
||||
` (${repos} repos)`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log('\nRead the signal: B vs A = does steering alone fix adoption + cut payload.');
|
||||
console.log('C vs B = is explore redundant (reads should NOT jump). D vs C = is context redundant.');
|
||||
console.log('E = non-flow under trace-centric; reads SHOULD jump (proves survey tools are load-bearing).');
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env node
|
||||
// Aggregate the README A/B (bench-readme.sh output): per repo, median of N runs
|
||||
// per arm → time, tool calls, tokens, cost, and % saved. Plus an average row.
|
||||
//
|
||||
// Tokens = SUM of per-turn assistant `usage` (input + output + cache read +
|
||||
// cache creation) — the cumulative "total tokens processed". NOTE: `result.usage`
|
||||
// is last-turn-only in current Claude Code, so it under-counts badly; don't use it.
|
||||
// `total_cost_usd` and `duration_ms` are already cumulative.
|
||||
//
|
||||
// Usage: node parse-bench-readme.mjs [/tmp/ab-readme]
|
||||
import { readFileSync, existsSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
const ROOT = process.argv[2] || '/tmp/ab-readme';
|
||||
const REPOS = ['vscode', 'excalidraw', 'django', 'tokio', 'okhttp', 'gin', 'alamofire'];
|
||||
|
||||
function parse(file) {
|
||||
if (!existsSync(file)) return null;
|
||||
const L = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
||||
let tools = 0, reads = 0, grep = 0, cg = 0, tokens = 0, r = null, raced = false;
|
||||
for (const l of L) { let e; try { e = JSON.parse(l); } catch { continue; }
|
||||
if (e.type === 'assistant') {
|
||||
const u = e.message?.usage;
|
||||
if (u) tokens += (u.input_tokens || 0) + (u.output_tokens || 0) + (u.cache_read_input_tokens || 0) + (u.cache_creation_input_tokens || 0);
|
||||
for (const b of (e.message?.content || [])) if (b.type === 'tool_use') {
|
||||
const n = b.name;
|
||||
if (n === 'ToolSearch') continue;
|
||||
tools++;
|
||||
if (n === 'Read') reads++;
|
||||
else if (n === 'Grep' || n === 'Glob') grep++;
|
||||
else if (/codegraph/.test(n)) cg++;
|
||||
}
|
||||
}
|
||||
// MCP cold-start race: the headless agent fired before `codegraph serve --mcp`
|
||||
// finished registering its tools, so early calls returned "No such tool
|
||||
// available" and the agent floundered into grep/Read. That measures CodeGraph's
|
||||
// startup latency, NOT its steady-state value — flag the run so the aggregate
|
||||
// can exclude it (an artifact of headless first-turn timing, not the tool).
|
||||
if (e.type === 'user') for (const b of (Array.isArray(e.message?.content) ? e.message.content : [])) {
|
||||
if (b.type === 'tool_result') {
|
||||
const t = Array.isArray(b.content) ? b.content.map(c => c.text || '').join('') : (b.content || '');
|
||||
if (/No such tool available/.test(t)) raced = true;
|
||||
}
|
||||
}
|
||||
if (e.type === 'result') r = e;
|
||||
}
|
||||
if (!r || r.subtype !== 'success') return null;
|
||||
return { dur: r.duration_ms / 1000, tools, reads, grep, cg, tokens, cost: r.total_cost_usd || 0, raced };
|
||||
}
|
||||
const median = (arr) => { const v = [...arr].sort((a, b) => a - b); const n = v.length; return n === 0 ? 0 : n % 2 ? v[(n - 1) / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; };
|
||||
const fmtTime = (s) => s >= 60 ? `${Math.floor(s / 60)}m ${Math.round(s % 60)}s` : `${Math.round(s)}s`;
|
||||
const fmtTok = (t) => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
|
||||
const pct = (w, wo) => wo > 0 ? Math.round((1 - w / wo) * 100) : 0;
|
||||
|
||||
console.log('repo n(w/wo) time WITH→WITHOUT tools W→WO tokens W→WO (saved) cost W→WO (saved)');
|
||||
const savings = { cost: [], tokens: [], time: [], tools: [] };
|
||||
for (const repo of REPOS) {
|
||||
const dir = join(ROOT, repo);
|
||||
const runDirs = existsSync(dir) ? readdirSync(dir).filter(d => /^run\d+$/.test(d)) : [];
|
||||
// Exclude MCP-cold-start-raced WITH runs by default — they measure a startup
|
||||
// race, not steady-state value. `CG_INCLUDE_RACED=1` keeps them (to see the raw
|
||||
// distribution). The WITHOUT arm has no MCP, so it's never raced.
|
||||
const includeRaced = process.env.CG_INCLUDE_RACED === '1';
|
||||
const W = [], WO = []; let racedExcluded = 0;
|
||||
for (const rd of runDirs) {
|
||||
const w = parse(join(dir, rd, 'run-headless-with.jsonl'));
|
||||
if (w) { if (w.raced && !includeRaced) racedExcluded++; else W.push(w); }
|
||||
const wo = parse(join(dir, rd, 'run-headless-without.jsonl')); if (wo) WO.push(wo);
|
||||
}
|
||||
if (!W.length || !WO.length) { console.log(`${repo.padEnd(11)} (incomplete: w=${W.length} wo=${WO.length})`); continue; }
|
||||
const m = (arr, k) => median(arr.map(x => x[k]));
|
||||
const wT = m(W, 'dur'), woT = m(WO, 'dur'), wTok = m(W, 'tokens'), woTok = m(WO, 'tokens');
|
||||
const wC = m(W, 'cost'), woC = m(WO, 'cost'), wTl = m(W, 'tools'), woTl = m(WO, 'tools');
|
||||
savings.time.push(pct(wT, woT)); savings.tokens.push(pct(wTok, woTok)); savings.cost.push(pct(wC, woC)); savings.tools.push(pct(wTl, woTl));
|
||||
console.log(
|
||||
`${repo.padEnd(11)} ${W.length}/${WO.length} ` +
|
||||
`${(fmtTime(wT) + '→' + fmtTime(woT)).padEnd(22)}` +
|
||||
`${(Math.round(wTl) + '→' + Math.round(woTl)).padEnd(12)}` +
|
||||
`${(fmtTok(wTok) + '→' + fmtTok(woTok) + ' (' + pct(wTok, woTok) + '%)').padEnd(24)}` +
|
||||
`$${wC.toFixed(2)}→$${woC.toFixed(2)} (${pct(wC, woC)}%)` +
|
||||
(racedExcluded ? ` [${racedExcluded} raced run${racedExcluded === 1 ? '' : 's'} excluded]` : '')
|
||||
);
|
||||
}
|
||||
const avg = (a) => a.length ? Math.round(a.reduce((s, x) => s + x, 0) / a.length) : 0;
|
||||
console.log(`\nAVERAGE saved: cost ${avg(savings.cost)}% · tokens ${avg(savings.tokens)}% · time ${avg(savings.time)}% · tool calls ${avg(savings.tools)}%`);
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env node
|
||||
// Parse a Claude Code stream-json run log: tool-call sequence + token usage.
|
||||
import { readFileSync } from 'fs';
|
||||
const file = process.argv[2];
|
||||
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
||||
|
||||
const toolCalls = [];
|
||||
let result = null;
|
||||
let initTools = null;
|
||||
|
||||
for (const line of lines) {
|
||||
let ev;
|
||||
try { ev = JSON.parse(line); } catch { continue; }
|
||||
if (ev.type === 'system' && ev.subtype === 'init') {
|
||||
initTools = (ev.tools || []).filter(t => /codegraph/.test(t));
|
||||
}
|
||||
if (ev.type === 'assistant' && ev.message?.content) {
|
||||
for (const block of ev.message.content) {
|
||||
if (block.type === 'tool_use') {
|
||||
let detail = '';
|
||||
if (block.name === 'Task') detail = ` [subagent_type=${block.input?.subagent_type ?? '?'}] ${(block.input?.description ?? '').slice(0,40)}`;
|
||||
else if (/codegraph/.test(block.name)) detail = ` ${JSON.stringify(block.input?.query ?? block.input?.task ?? block.input?.symbol ?? '').slice(0,60)}`;
|
||||
else if (block.name === 'Bash') detail = ` ${(block.input?.command ?? '').slice(0,50)}`;
|
||||
else if (block.name === 'Read') detail = ` ${(block.input?.file_path ?? '').split('/').slice(-1)[0]}`;
|
||||
toolCalls.push(`${block.name}${detail}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ev.type === 'result') result = ev;
|
||||
}
|
||||
|
||||
console.log(`\n=== ${file.split('/').pop()} ===`);
|
||||
console.log(`codegraph tools exposed: ${initTools ? initTools.length : '?'}`);
|
||||
console.log(`\nTool calls (${toolCalls.length}):`);
|
||||
const counts = {};
|
||||
for (const tc of toolCalls) { const n = tc.split(' ')[0]; counts[n] = (counts[n]||0)+1; }
|
||||
console.log(' by type:', JSON.stringify(counts));
|
||||
toolCalls.forEach((tc, i) => console.log(` ${i+1}. ${tc}`));
|
||||
|
||||
if (result) {
|
||||
const u = result.usage || {};
|
||||
const totalIn = (u.input_tokens||0) + (u.cache_read_input_tokens||0) + (u.cache_creation_input_tokens||0);
|
||||
console.log(`\nResult: ${result.subtype} | duration ${(result.duration_ms/1000).toFixed(0)}s | turns ${result.num_turns}`);
|
||||
console.log(` tokens: in=${totalIn} out=${u.output_tokens||0} | cost $${(result.total_cost_usd||0).toFixed(3)}`);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env node
|
||||
// Parse the newest Claude Code session log for a project + its subagent logs,
|
||||
// and report the tool-call breakdown (main + subagents). Works for interactive
|
||||
// runs (driven via itrun.sh) — Claude Code writes full transcripts to
|
||||
// ~/.claude/projects/<escaped-cwd>/<session>.jsonl with subagents/ alongside.
|
||||
import { readFileSync, readdirSync, statSync, existsSync, realpathSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { homedir } from 'os';
|
||||
|
||||
const projectArg = process.argv[2];
|
||||
if (!projectArg) { console.error('usage: parse-session.mjs <project-dir>'); process.exit(1); }
|
||||
|
||||
// Claude Code escapes the (real) cwd by replacing every "/" with "-".
|
||||
const real = realpathSync(projectArg);
|
||||
const escaped = real.replace(/\//g, '-');
|
||||
const projDir = join(homedir(), '.claude', 'projects', escaped);
|
||||
if (!existsSync(projDir)) { console.error('no session logs at', projDir); process.exit(1); }
|
||||
|
||||
// Newest top-level session .jsonl
|
||||
const sessions = readdirSync(projDir)
|
||||
.filter(f => f.endsWith('.jsonl'))
|
||||
.map(f => ({ f, m: statSync(join(projDir, f)).mtimeMs }))
|
||||
.sort((a, b) => b.m - a.m);
|
||||
if (sessions.length === 0) { console.error('no .jsonl sessions in', projDir); process.exit(1); }
|
||||
const sessionId = sessions[0].f.replace('.jsonl', '');
|
||||
|
||||
function tally(file) {
|
||||
const counts = {};
|
||||
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
||||
if (!line) continue;
|
||||
let ev; try { ev = JSON.parse(line); } catch { continue; }
|
||||
const content = ev.message?.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (const b of content) {
|
||||
if (b.type === 'tool_use') counts[b.name] = (counts[b.name] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
// Sum token usage from a transcript. The TUI's "Done (…Xk tokens…)" line only
|
||||
// covers a subagent's throughput; this works for main-thread runs too and is
|
||||
// consistent across both paths. `gen` = output, `fresh` = uncached input
|
||||
// (input + cache_creation), `cached` = cache reads (≈free), `total` = all.
|
||||
function sumTokens(file) {
|
||||
const t = { gen: 0, fresh: 0, cached: 0 };
|
||||
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
||||
if (!line) continue;
|
||||
let ev; try { ev = JSON.parse(line); } catch { continue; }
|
||||
const u = ev.message?.usage;
|
||||
if (!u) continue;
|
||||
t.gen += u.output_tokens || 0;
|
||||
t.fresh += (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
|
||||
t.cached += u.cache_read_input_tokens || 0;
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
const mainCounts = tally(join(projDir, sessionId + '.jsonl'));
|
||||
|
||||
// Subagent transcripts live under <session>/subagents/*.jsonl
|
||||
const subDir = join(projDir, sessionId, 'subagents');
|
||||
const subCounts = {};
|
||||
let subAgentFiles = 0;
|
||||
if (existsSync(subDir)) {
|
||||
for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) {
|
||||
subAgentFiles++;
|
||||
const c = tally(join(subDir, f));
|
||||
for (const [k, v] of Object.entries(c)) subCounts[k] = (subCounts[k] || 0) + v;
|
||||
}
|
||||
}
|
||||
|
||||
const fmt = (counts) => Object.entries(counts).sort((a, b) => b[1] - a[1])
|
||||
.map(([k, v]) => ` ${String(v).padStart(3)} ${k}`).join('\n') || ' (none)';
|
||||
|
||||
console.log(`session: ${sessionId}`);
|
||||
console.log(`\nMAIN thread tools:\n${fmt(mainCounts)}`);
|
||||
console.log(`\nSUBAGENT tools (${subAgentFiles} subagent transcript${subAgentFiles === 1 ? '' : 's'}):\n${fmt(subCounts)}`);
|
||||
|
||||
const explore = subCounts['mcp__codegraph__codegraph_explore'] || mainCounts['mcp__codegraph__codegraph_explore'] || 0;
|
||||
const reads = (subCounts['Read'] || 0) + (mainCounts['Read'] || 0);
|
||||
const greps = (subCounts['Grep'] || 0) + (mainCounts['Grep'] || 0) + (subCounts['Bash'] || 0) + (mainCounts['Bash'] || 0);
|
||||
console.log(`\nVERDICT: codegraph_explore used ${explore}x | Read ${reads} | Grep/Bash ${greps}`);
|
||||
|
||||
// Token totals (main + subagents), consistent across main-thread and subagent runs.
|
||||
const tok = { gen: 0, fresh: 0, cached: 0 };
|
||||
const addTok = (t) => { tok.gen += t.gen; tok.fresh += t.fresh; tok.cached += t.cached; };
|
||||
addTok(sumTokens(join(projDir, sessionId + '.jsonl')));
|
||||
if (existsSync(subDir)) {
|
||||
for (const f of readdirSync(subDir).filter(f => f.endsWith('.jsonl'))) addTok(sumTokens(join(subDir, f)));
|
||||
}
|
||||
const k = (n) => (n / 1000).toFixed(1) + 'k';
|
||||
console.log(`TOKENS: gen ${k(tok.gen)} | fresh-in ${k(tok.fresh)} | cached-in ${k(tok.cached)} | billable≈ ${k(tok.gen + tok.fresh)}`);
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
// Probe codegraph_context (with call-paths) against an index using the built dist.
|
||||
// Usage: node probe-context.mjs <repo-with-.codegraph> <task words...>
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const [, , repo, ...taskParts] = process.argv;
|
||||
const task = taskParts.join(' ');
|
||||
if (!repo || !task) { console.error('usage: probe-context.mjs <repo> <task...>'); process.exit(1); }
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
const cg = CodeGraph.openSync(repo);
|
||||
const h = new ToolHandler(cg);
|
||||
const res = await h.execute('codegraph_context', { task });
|
||||
console.log(res.content?.[0]?.text ?? '(no text)');
|
||||
try { cg.close?.(); } catch {}
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env node
|
||||
// One-shot probe: run handleExplore against an existing index using the built
|
||||
// dist, print the output + a few stats. Lets us verify explore's coverage fix
|
||||
// without a full agent run. Usage: node probe-explore.mjs <repo-with-.codegraph> "<query>"
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const [, , repo, query] = process.argv;
|
||||
if (!repo || !query) {
|
||||
console.error('usage: probe-explore.mjs <repo> "<query>"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
|
||||
// esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
if (typeof CodeGraph?.openSync !== 'function') {
|
||||
console.error('could not resolve CodeGraph.openSync; index keys:', Object.keys(idx), 'default keys:', idx.default && Object.keys(idx.default));
|
||||
process.exit(2);
|
||||
}
|
||||
if (typeof ToolHandler !== 'function') {
|
||||
console.error('could not resolve ToolHandler; tools keys:', Object.keys(tools));
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const cg = CodeGraph.openSync(repo);
|
||||
const h = new ToolHandler(cg);
|
||||
const res = await h.execute('codegraph_explore', { query });
|
||||
const text = res.content?.[0]?.text ?? '(no text)';
|
||||
console.log(text);
|
||||
console.error('\n--- PROBE STATS ---');
|
||||
console.error('output chars:', text.length);
|
||||
console.error('triggerRender body present (-> setState({})):', /triggerRender[\s\S]{0,400}setState\(\{\}\)/.test(text));
|
||||
console.error('App.tsx in source section:', /\*\*`.*App\.tsx`\*\* —/.test(text));
|
||||
try { cg.close?.(); } catch {}
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env node
|
||||
// Probe codegraph_node (with trail) against an index using the built dist.
|
||||
// Usage: node probe-node.mjs <repo-with-.codegraph> <symbol> [code]
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const [, , repo, symbol, code] = process.argv;
|
||||
if (!repo || !symbol) { console.error('usage: probe-node.mjs <repo> <symbol> [code]'); process.exit(1); }
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
const cg = CodeGraph.openSync(repo);
|
||||
const h = new ToolHandler(cg);
|
||||
const res = await h.execute('codegraph_node', { symbol, includeCode: code === 'code' });
|
||||
console.log(res.content?.[0]?.text ?? '(no text)');
|
||||
try { cg.close?.(); } catch {}
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
// probe-sweep — direct MCP test across N repos × N tools, no claude needed.
|
||||
//
|
||||
// Measures response characteristics (size, sections present, signals fired)
|
||||
// for each (repo, query) pair against the built dist/. Sub-second per probe;
|
||||
// the full sweep below runs in ~10-30s vs hours for a real claude audit.
|
||||
//
|
||||
// Use this to iterate on backend changes rapidly: change tools.ts /
|
||||
// context-builder, npm run build, re-run probe-sweep, compare. Once a
|
||||
// change looks good on probe metrics, run a focused claude audit for the
|
||||
// few repos that matter to confirm end-to-end cost behavior.
|
||||
//
|
||||
// Usage: node scripts/agent-eval/probe-sweep.mjs [--tool=context|explore|trace] [--repos=a,b,c]
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const args = Object.fromEntries(
|
||||
process.argv.slice(2).map(a => a.startsWith('--') ? a.slice(2).split('=') : [a, true])
|
||||
);
|
||||
const TOOL = args.tool ?? 'context';
|
||||
|
||||
const load = (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
// Each entry: repo, query, optional 2nd arg for trace (from, to).
|
||||
// The query is the same prompt used in the real claude audits, so probe
|
||||
// output is directly comparable to the agent's would-be input.
|
||||
const SWEEP = [
|
||||
// Small realworld template repos (the loss cases from the cross-language sweep)
|
||||
{ id: 'gin-rw', repo: '/tmp/codegraph-corpus/gin-realworld', q: 'How does this Gin app route a request through its middleware chain to a handler?' },
|
||||
{ id: 'go-mux', repo: '/tmp/codegraph-corpus/go-mux', q: 'How does this gorilla/mux app route a request to its handler?' },
|
||||
{ id: 'fastapi-rw', repo: '/tmp/codegraph-corpus/fastapi-realworld', q: 'How does FastAPI route a request through its dependencies to a handler?' },
|
||||
{ id: 'spring-pc', repo: '/tmp/codegraph-corpus/spring-petclinic', q: 'How does Spring route an HTTP request to a controller method?' },
|
||||
{ id: 'axum-rw', repo: '/tmp/codegraph-corpus/rust-axum-realworld', q: 'How does Axum route a request to its handler in this app?' },
|
||||
{ id: 'express-rw', repo: '/tmp/codegraph-corpus/express-realworld', q: 'How does this Express app route a request through middleware to a handler?' },
|
||||
{ id: 'kotlin-pc', repo: '/tmp/codegraph-corpus/kotlin-petclinic', q: 'How does the Kotlin Spring app route an HTTP request to its handler?' },
|
||||
{ id: 'flask-mb', repo: '/tmp/codegraph-corpus/flask-microblog', q: 'How does this Flask app route a request to a view function?' },
|
||||
{ id: 'vapor-tpl', repo: '/tmp/codegraph-corpus/vapor-template', q: 'How does Vapor route an HTTP request to its handler?' },
|
||||
{ id: 'cpp-leveldb', repo: '/tmp/codegraph-corpus/cpp-leveldb', q: 'How does LevelDB handle a Put operation through to disk?' },
|
||||
{ id: 'lualine', repo: '/tmp/codegraph-corpus/lualine.nvim', q: 'How does lualine assemble and render the statusline?' },
|
||||
{ id: 'drupal-admin', repo: '/tmp/codegraph-corpus/drupal-admintoolbar', q: 'How does the Drupal admin toolbar module render its toolbar?' },
|
||||
{ id: 'svelte-rw', repo: '/tmp/codegraph-corpus/svelte-realworld', q: 'How does this SvelteKit app route a request to a handler?' },
|
||||
{ id: 'react-rw', repo: '/tmp/codegraph-corpus/react-realworld', q: 'How does this React app fetch and display articles?' },
|
||||
{ id: 'rails-rw', repo: '/tmp/codegraph-corpus/rails-realworld', q: 'How does Rails route a request to a controller action?' },
|
||||
{ id: 'flask-rest', repo: '/tmp/codegraph-corpus/flask-restful-realworld', q: 'How does Flask-RESTful route a request to a resource method?' },
|
||||
{ id: 'laravel-rw', repo: '/tmp/codegraph-corpus/laravel-realworld', q: 'How does Laravel route a request to the controller method?' },
|
||||
{ id: 'aspnet-rw', repo: '/tmp/codegraph-corpus/aspnet-realworld', q: 'How does ASP.NET route a request to the controller action?' },
|
||||
// The iter7 wins/ties (to make sure we don't regress)
|
||||
{ id: 'cobra', repo: '/tmp/codegraph-corpus/cobra', q: 'How does cobra parse commands and flags?' },
|
||||
{ id: 'sinatra', repo: '/tmp/codegraph-corpus/sinatra', q: 'How does sinatra route a request to its handler?' },
|
||||
{ id: 'slim', repo: '/tmp/codegraph-corpus/slim', q: 'How does slim route a request and apply middleware?' },
|
||||
];
|
||||
|
||||
// Detect signals in response text — these are the levers we've added that
|
||||
// otherwise only show up via "agent ran X more tool calls" downstream.
|
||||
const detect = (text) => ({
|
||||
hasEntryPoints: /^### Entry Points/m.test(text),
|
||||
hasRelatedSymbols: /^### Related Symbols/m.test(text),
|
||||
hasFlowTrace: /^## Inline flow trace/m.test(text),
|
||||
hasRouteManifest: /^## Routing manifest/m.test(text),
|
||||
hasTopHandler: /^### Top handler file/m.test(text),
|
||||
hasSmallRepoTail: /This project is small/.test(text),
|
||||
});
|
||||
|
||||
const filterRepos = args.repos ? new Set(String(args.repos).split(',')) : null;
|
||||
const subjects = SWEEP.filter(s => !filterRepos || filterRepos.has(s.id));
|
||||
|
||||
const t0 = Date.now();
|
||||
const rows = [];
|
||||
for (const s of subjects) {
|
||||
try {
|
||||
const cg = CodeGraph.openSync(s.repo);
|
||||
const handler = new ToolHandler(cg);
|
||||
const t1 = Date.now();
|
||||
const res = await handler.execute('codegraph_' + TOOL,
|
||||
TOOL === 'context' ? { task: s.q } :
|
||||
TOOL === 'explore' ? { query: s.q } : { from: 'main', to: 'main' });
|
||||
const text = res.content?.[0]?.text ?? '';
|
||||
const signals = detect(text);
|
||||
rows.push({
|
||||
id: s.id,
|
||||
ms: Date.now() - t1,
|
||||
chars: text.length,
|
||||
lines: text.split('\n').length,
|
||||
...signals,
|
||||
});
|
||||
try { cg.close?.(); } catch {}
|
||||
} catch (e) {
|
||||
rows.push({ id: s.id, error: String(e).slice(0, 80) });
|
||||
}
|
||||
}
|
||||
|
||||
// Pretty-print as a compact table.
|
||||
const fmt = (r) =>
|
||||
r.error
|
||||
? ` ${r.id.padEnd(13)} ERROR: ${r.error}`
|
||||
: ` ${r.id.padEnd(13)} ${String(r.chars).padStart(6)}c ${String(r.lines).padStart(4)}L ${String(r.ms).padStart(4)}ms` +
|
||||
` ${r.hasEntryPoints ? 'EP ' : ' '}` +
|
||||
`${r.hasFlowTrace ? 'TRC ' : ' '}` +
|
||||
`${r.hasRouteManifest ? 'MAN ' : ' '}` +
|
||||
`${r.hasTopHandler ? 'HND ' : ' '}` +
|
||||
`${r.hasSmallRepoTail ? 'TAIL' : ' '}`;
|
||||
console.log(`=== probe-sweep tool=${TOOL} n=${subjects.length} (${Date.now() - t0}ms total) ===`);
|
||||
console.log(' id chars lines ms signals');
|
||||
console.log(' ' + '-'.repeat(56));
|
||||
for (const r of rows) console.log(fmt(r));
|
||||
|
||||
// Sum + medians for the size pillar
|
||||
const sizes = rows.filter(r => !r.error).map(r => r.chars);
|
||||
sizes.sort((a, b) => a - b);
|
||||
const median = sizes[Math.floor(sizes.length / 2)];
|
||||
const sum = sizes.reduce((a, b) => a + b, 0);
|
||||
console.log(` ${'-'.repeat(64)}`);
|
||||
console.log(` median=${median}c total=${sum}c ` +
|
||||
`manifest=${rows.filter(r => r.hasRouteManifest).length}/${rows.filter(r => !r.error).length} ` +
|
||||
`top-handler=${rows.filter(r => r.hasTopHandler).length}/${rows.filter(r => !r.error).length}`);
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env node
|
||||
// Probe codegraph_trace against an index using the built dist.
|
||||
// Usage: node probe-trace.mjs <repo-with-.codegraph> <from> <to>
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const [, , repo, from, to] = process.argv;
|
||||
if (!repo || !from || !to) { console.error('usage: probe-trace.mjs <repo> <from> <to>'); process.exit(1); }
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
const cg = CodeGraph.openSync(repo);
|
||||
const h = new ToolHandler(cg);
|
||||
const res = await h.execute('codegraph_trace', { from, to });
|
||||
console.log(res.content?.[0]?.text ?? '(no text)');
|
||||
try { cg.close?.(); } catch {}
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# PreToolUse(Read) REDIRECT hook — prototype for A/B (P1: get agents off Read and
|
||||
# onto codegraph_node during implementation, not just for Q&A).
|
||||
#
|
||||
# When the agent Reads a SOURCE file, deny it and steer to codegraph_node's
|
||||
# file-view, which (as of the Lever-1 change) returns the WHOLE file verbatim
|
||||
# WITH line numbers — imports, top-level code, comments and all — PLUS the file's
|
||||
# blast radius, in one call. That output is a strict superset of Read, so the
|
||||
# redirect is lossless: the agent loses nothing by taking it, and gains who-
|
||||
# depends-on-this for the edit it's about to make.
|
||||
#
|
||||
# Differs from block-read-hook.sh (which steers to explore/node-by-symbol): this
|
||||
# names the FILE-VIEW path explicitly (file:"<base>" + includeCode:true), the
|
||||
# 1:1 Read replacement we're trying to get picked during implementation.
|
||||
#
|
||||
# Non-source files (configs, docs, lockfiles, .env) pass through to a real Read.
|
||||
# A redirect to a file codegraph hasn't indexed SELF-CORRECTS: the file-view
|
||||
# replies "No indexed file matches … Read it directly", so a just-created file
|
||||
# never dead-ends — the agent Reads it on the next turn.
|
||||
#
|
||||
# Wire via: claude ... --settings <settings-with-this-as-PreToolUse(Read)>
|
||||
# Eval artifact only. The production version is an indexed-aware `codegraph`
|
||||
# subcommand (cross-platform — no bash/jq — and queries the index so it never
|
||||
# bounces a new/un-indexed file), wired opt-in by the installer.
|
||||
set -uo pipefail
|
||||
input="$(cat)"
|
||||
fp="$(printf '%s' "$input" | jq -r '.tool_input.file_path // empty' 2>/dev/null)"
|
||||
[ -n "$fp" ] || exit 0
|
||||
base="$(basename "$fp")"
|
||||
|
||||
case "$fp" in
|
||||
*.ts|*.tsx|*.js|*.jsx|*.mjs|*.cjs|*.py|*.go|*.rs|*.java|*.rb|*.php|*.swift|*.kt|*.kts|*.scala|*.c|*.cc|*.cpp|*.h|*.hpp|*.cs|*.lua|*.vue|*.svelte|*.m|*.mm)
|
||||
msg="codegraph has this file indexed (kept in sync on every edit). Call codegraph_node with file:\"$base\" and includeCode:true instead of Read — it returns the WHOLE file verbatim WITH line numbers (imports, top-level code and all — safe to base an Edit on) PLUS which files depend on it, in one call. Treat its output as already-Read; do not Read this file. (If it answers that the file isn't indexed — e.g. you just created it — then Read it directly.)"
|
||||
jq -n --arg m "$msg" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"deny",permissionDecisionReason:$m}}'
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
// Reproduction harness A — does the shared daemon serialize concurrent explore?
|
||||
//
|
||||
// Mirrors the daemon's reality: ONE CodeGraph + ONE ToolHandler (as MCPEngine
|
||||
// shares across all sessions), then fires N concurrent codegraph_explore calls
|
||||
// and measures:
|
||||
// - each call's wall-clock latency + completion order
|
||||
// - an event-loop HEARTBEAT (setInterval 50ms): the max gap between ticks is a
|
||||
// direct measure of how long synchronous compute blocked the loop. In the
|
||||
// real daemon a blocked loop can't flush a finished response or read the
|
||||
// next request, so this gap is what starves the MCP transport.
|
||||
//
|
||||
// Usage: node repro-concurrent-explore.mjs <repo-with-.codegraph> <N> [timeoutMs]
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { resolve } from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
const [, , repo, nRaw, timeoutRaw] = process.argv;
|
||||
if (!repo) {
|
||||
console.error('usage: repro-concurrent-explore.mjs <repo> <N=10> [timeoutMs=60000]');
|
||||
process.exit(1);
|
||||
}
|
||||
const N = Number(nRaw) || 10;
|
||||
const TIMEOUT_MS = Number(timeoutRaw) || 60000; // ~ MCP SDK default request timeout
|
||||
|
||||
const load = async (rel) => import(pathToFileURL(resolve(rel)).href);
|
||||
const idx = await load('dist/index.js');
|
||||
const tools = await load('dist/mcp/tools.js');
|
||||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||||
|
||||
// Distinct queries so no two calls are trivially identical. Mix of NL questions
|
||||
// (exercise FTS + RWR over the whole graph) — the expensive explore path.
|
||||
const QUERIES = [
|
||||
'how does the text model handle edits and undo',
|
||||
'how does the file service watch for changes on disk',
|
||||
'how does the keybinding service resolve a chord to a command',
|
||||
'how does the extension host activate an extension',
|
||||
'how does the editor render decorations in the viewport',
|
||||
'how does the search service stream results to the UI',
|
||||
'how does the terminal process manager spawn a shell',
|
||||
'how does the configuration service merge user and workspace settings',
|
||||
'how does the debug adapter forward breakpoints to the runtime',
|
||||
'how does the quick input widget filter its items',
|
||||
'how does the notification service queue and show toasts',
|
||||
'how does the git extension compute the diff for a file',
|
||||
'how does the language features registry dispatch a hover request',
|
||||
'how does the workbench layout restore editor groups on reload',
|
||||
'how does the storage service persist state between sessions',
|
||||
'how does the menu service build a context menu from contributions',
|
||||
];
|
||||
|
||||
const cg = CodeGraph.openSync(repo);
|
||||
let fileCount = 0;
|
||||
try { fileCount = cg.getStats().fileCount; } catch {}
|
||||
const handler = new ToolHandler(cg);
|
||||
|
||||
// --- event-loop heartbeat ---
|
||||
let lastTick = performance.now();
|
||||
let maxGap = 0;
|
||||
const gaps = [];
|
||||
const hb = setInterval(() => {
|
||||
const now = performance.now();
|
||||
const gap = now - lastTick;
|
||||
lastTick = now;
|
||||
if (gap > 60) gaps.push(Math.round(gap)); // expected ~50ms; record stalls
|
||||
if (gap > maxGap) maxGap = gap;
|
||||
}, 50);
|
||||
|
||||
function runOne(i) {
|
||||
const q = QUERIES[i % QUERIES.length];
|
||||
const startedAt = performance.now();
|
||||
let timer;
|
||||
const timeout = new Promise((res) => {
|
||||
timer = setTimeout(() => res({ timedOut: true }), TIMEOUT_MS);
|
||||
});
|
||||
const work = handler
|
||||
.execute('codegraph_explore', { query: q })
|
||||
.then((r) => ({ ok: !r.isError, chars: r.content?.[0]?.text?.length ?? 0 }))
|
||||
.catch((e) => ({ ok: false, err: String(e?.message ?? e) }));
|
||||
return Promise.race([work, timeout]).then((r) => {
|
||||
clearTimeout(timer);
|
||||
return { i, q, ms: Math.round(performance.now() - startedAt), ...r };
|
||||
});
|
||||
}
|
||||
|
||||
// Baseline: one warm single call (so the first-call cold paths don't skew N).
|
||||
const warmStart = performance.now();
|
||||
await runOne(0);
|
||||
const warmMs = Math.round(performance.now() - warmStart);
|
||||
|
||||
// Reset heartbeat stats for the concurrent run.
|
||||
gaps.length = 0; maxGap = 0; lastTick = performance.now();
|
||||
|
||||
const batchStart = performance.now();
|
||||
const results = await Promise.all(Array.from({ length: N }, (_, i) => runOne(i)));
|
||||
const batchMs = Math.round(performance.now() - batchStart);
|
||||
clearInterval(hb);
|
||||
|
||||
const lat = results.map((r) => r.ms).sort((a, b) => a - b);
|
||||
const timeouts = results.filter((r) => r.timedOut).length;
|
||||
const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
|
||||
|
||||
console.log('='.repeat(64));
|
||||
console.log(`repo=${repo}`);
|
||||
console.log(`fileCount=${fileCount} N=${N} perCallTimeout=${TIMEOUT_MS}ms`);
|
||||
console.log(`single warm explore: ${warmMs}ms`);
|
||||
console.log('-'.repeat(64));
|
||||
console.log(`concurrent batch wall-clock: ${batchMs}ms`);
|
||||
console.log(`per-call latency min=${lat[0]} p50=${p(0.5)} p90=${p(0.9)} max=${lat[lat.length - 1]} (ms)`);
|
||||
console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
|
||||
console.log(`event-loop max stall: ${Math.round(maxGap)}ms stalls>60ms: ${gaps.length}`);
|
||||
console.log(` sum of stalls: ${gaps.reduce((a, b) => a + b, 0)}ms biggest 5: ${gaps.sort((a,b)=>b-a).slice(0,5).join(', ')}`);
|
||||
console.log('-'.repeat(64));
|
||||
console.log('SERIALIZATION CHECK:');
|
||||
console.log(` if serialized, batch ≈ N×single = ~${N * warmMs}ms; actual=${batchMs}ms (ratio ${(batchMs / (N * warmMs)).toFixed(2)})`);
|
||||
console.log(` max latency / single = ${(lat[lat.length - 1] / warmMs).toFixed(1)}× (≈N means last call waited for all others)`);
|
||||
console.log('='.repeat(64));
|
||||
try { cg.close?.(); } catch {}
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env node
|
||||
// Reproduction harness B — the FAITHFUL opencode scenario.
|
||||
//
|
||||
// Spawns N real `codegraph serve --mcp --path <repo>` processes (each becomes a
|
||||
// proxy that attaches to ONE shared daemon — exactly what opencode does with N
|
||||
// subagents), drives clean MCP JSON-RPC over each child's stdio, then fires ONE
|
||||
// concurrent wave of codegraph_explore tools/call across all N and measures
|
||||
// end-to-end latency + timeouts. This captures transport-flush starvation: a
|
||||
// daemon event-loop blocked in synchronous explore compute can neither read the
|
||||
// next request nor flush a finished response.
|
||||
//
|
||||
// Usage: node repro-daemon-clients.mjs <repo> <N=10> [perCallTimeoutMs=60000] [warm=1]
|
||||
import { spawn } from 'node:child_process';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const [, , repoRaw, nRaw, timeoutRaw, warmRaw] = process.argv;
|
||||
const repo = resolve(repoRaw || '.');
|
||||
const N = Number(nRaw) || 10;
|
||||
const TIMEOUT_MS = Number(timeoutRaw) || 60000;
|
||||
const WARM = warmRaw === undefined ? true : warmRaw !== '0';
|
||||
const CLI = resolve('dist/bin/codegraph.js');
|
||||
|
||||
const QUERIES = [
|
||||
'how does the text model handle edits and undo',
|
||||
'how does the file service watch for changes on disk',
|
||||
'how does the keybinding service resolve a chord to a command',
|
||||
'how does the extension host activate an extension',
|
||||
'how does the editor render decorations in the viewport',
|
||||
'how does the search service stream results to the UI',
|
||||
'how does the terminal process manager spawn a shell',
|
||||
'how does the configuration service merge user and workspace settings',
|
||||
'how does the debug adapter forward breakpoints to the runtime',
|
||||
'how does the quick input widget filter its items',
|
||||
'how does the notification service queue and show toasts',
|
||||
'how does the git extension compute the diff for a file',
|
||||
];
|
||||
|
||||
function makeClient(id) {
|
||||
const child = spawn('node', [CLI, 'serve', '--mcp', '--path', repo], {
|
||||
env: { ...process.env, CODEGRAPH_TELEMETRY: '0', DO_NOT_TRACK: '1', CODEGRAPH_MCP_LOG_ATTACH: '0' },
|
||||
stdio: ['pipe', 'pipe', 'inherit'],
|
||||
});
|
||||
let buf = '';
|
||||
const waiters = new Map(); // id -> resolve
|
||||
child.stdout.setEncoding('utf8');
|
||||
child.stdout.on('data', (chunk) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf('\n')) !== -1) {
|
||||
const line = buf.slice(0, idx).trim();
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
let msg; try { msg = JSON.parse(line); } catch { continue; }
|
||||
if (msg.id !== undefined && waiters.has(msg.id)) {
|
||||
waiters.get(msg.id)(msg);
|
||||
waiters.delete(msg.id);
|
||||
}
|
||||
}
|
||||
});
|
||||
const send = (obj) => child.stdin.write(JSON.stringify(obj) + '\n');
|
||||
const request = (method, params, rpcId, timeoutMs) =>
|
||||
new Promise((res) => {
|
||||
let timer;
|
||||
if (timeoutMs) timer = setTimeout(() => { waiters.delete(rpcId); res({ __timeout: true }); }, timeoutMs);
|
||||
waiters.set(rpcId, (m) => { if (timer) clearTimeout(timer); res(m); });
|
||||
send({ jsonrpc: '2.0', id: rpcId, method, params });
|
||||
});
|
||||
return { id, child, send, request };
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
const clients = Array.from({ length: N }, (_, i) => makeClient(i));
|
||||
|
||||
// Initialize every client (handshake is answered locally by each proxy, instant).
|
||||
await Promise.all(clients.map((c) =>
|
||||
c.request('initialize', { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'repro', version: '1' } }, `init-${c.id}`, 10000)
|
||||
.then(() => c.send({ jsonrpc: '2.0', method: 'initialized' }))
|
||||
));
|
||||
|
||||
// Warm the daemon: one explore through client 0 forces daemon spawn + project
|
||||
// open + catch-up gate to complete, so the concurrent wave measures the STEADY
|
||||
// state (the user's real scenario after the first call), not cold start.
|
||||
if (WARM) {
|
||||
process.stderr.write('[repro] warming daemon (first explore triggers spawn+open+catchup)...\n');
|
||||
const t0 = performance.now();
|
||||
const r = await clients[0].request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[0] } }, 'warm-0', 120000);
|
||||
process.stderr.write(`[repro] warm explore took ${Math.round(performance.now() - t0)}ms (timeout=${!!r.__timeout})\n`);
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
// THE WAVE: fire one explore on every client as simultaneously as possible.
|
||||
process.stderr.write(`[repro] firing ${N} concurrent explores...\n`);
|
||||
const waveStart = performance.now();
|
||||
const results = await Promise.all(clients.map((c, i) => {
|
||||
const started = performance.now();
|
||||
return c.request('tools/call', { name: 'codegraph_explore', arguments: { query: QUERIES[i % QUERIES.length] } }, `call-${c.id}`, TIMEOUT_MS)
|
||||
.then((m) => ({
|
||||
id: c.id,
|
||||
ms: Math.round(performance.now() - started),
|
||||
timedOut: !!m.__timeout,
|
||||
ok: !!m.result && !m.result.isError,
|
||||
chars: m.result?.content?.[0]?.text?.length ?? 0,
|
||||
}));
|
||||
}));
|
||||
const waveMs = Math.round(performance.now() - waveStart);
|
||||
|
||||
const lat = results.map((r) => r.ms).sort((a, b) => a - b);
|
||||
const timeouts = results.filter((r) => r.timedOut).length;
|
||||
const p = (q) => lat[Math.min(lat.length - 1, Math.floor(q * lat.length))];
|
||||
|
||||
console.log('='.repeat(64));
|
||||
console.log(`HARNESS B (real daemon + ${N} proxies) repo=${repo}`);
|
||||
console.log(`warm=${WARM} perCallTimeout=${TIMEOUT_MS}ms`);
|
||||
console.log('-'.repeat(64));
|
||||
console.log(`wave wall-clock: ${waveMs}ms`);
|
||||
console.log(`per-call latency min=${lat[0]} p50=${p(0.5)} p90=${p(0.9)} max=${lat[lat.length - 1]} (ms)`);
|
||||
console.log(`TIMEOUTS (>${TIMEOUT_MS}ms): ${timeouts} / ${N}`);
|
||||
console.log(`completion order (id:ms): ${results.slice().sort((a,b)=>a.ms-b.ms).map(r=>`${r.id}:${r.ms}`).join(' ')}`);
|
||||
console.log('='.repeat(64));
|
||||
|
||||
for (const c of clients) { try { c.child.stdin.end(); c.child.kill('SIGTERM'); } catch {} }
|
||||
await sleep(300);
|
||||
process.exit(0);
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Headless Claude Code run against a repo with codegraph MCP, capturing the
|
||||
# full stream-json so we can see tool calls + token usage. Complements the
|
||||
# interactive itrun.sh: headless gives a clean per-tool breakdown + exact
|
||||
# tokens/cost, but defaults to the general-purpose subagent (not Explore).
|
||||
# To force the Explore path, ask for it in the prompt.
|
||||
#
|
||||
# Usage: run-agent.sh <repo-path> <label> "<prompt>"
|
||||
# Env: AGENT_EVAL_OUT (default /tmp/agent-eval), CG_BIN (codegraph dist binary)
|
||||
set -uo pipefail
|
||||
|
||||
REPO="$1"; LABEL="$2"; PROMPT="$3"
|
||||
CG_BIN="${CG_BIN:-$(command -v codegraph || echo /usr/local/bin/codegraph)}"
|
||||
OUT_DIR="${AGENT_EVAL_OUT:-/tmp/agent-eval}"; mkdir -p "$OUT_DIR"
|
||||
OUT="$OUT_DIR/run-${LABEL}.jsonl"
|
||||
|
||||
MCP_CONFIG=$(cat <<JSON
|
||||
{"mcpServers":{"codegraph":{"command":"${CG_BIN}","args":["serve","--mcp","--path","${REPO}"]}}}
|
||||
JSON
|
||||
)
|
||||
|
||||
echo "→ running [$LABEL] in $REPO"
|
||||
cd "$REPO" || exit 1
|
||||
|
||||
claude -p "$PROMPT" \
|
||||
--output-format stream-json --verbose \
|
||||
--permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
||||
--max-budget-usd 2 \
|
||||
--strict-mcp-config --mcp-config "$MCP_CONFIG" \
|
||||
> "$OUT" 2>"$OUT_DIR/run-${LABEL}.err"
|
||||
|
||||
echo "exit: $? | wrote $OUT ($(wc -l < "$OUT") lines)"
|
||||
node "$(cd "$(dirname "$0")" && pwd)/parse-run.mjs" "$OUT" 2>/dev/null || true
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# With/without A/B (and optional interactive) eval for a codegraph version on a
|
||||
# repo. Codegraph is the ONLY variable: both arms launch claude with
|
||||
# --strict-mcp-config — with = codegraph-only MCP (pointed at $CG_BIN),
|
||||
# without = empty MCP. Built-in Read/Grep/Bash stay available in both arms.
|
||||
#
|
||||
# Usage: run-all.sh <repo-path> "<question>" [headless|tmux|all]
|
||||
# Env: CG_BIN codegraph binary (default: command -v codegraph)
|
||||
# AGENT_EVAL_OUT output dir (default: /tmp/agent-eval)
|
||||
# MODEL / EFFORT claude model/effort (default: sonnet / high — the
|
||||
# standing A/B policy; see CLAUDE.md, don't raise)
|
||||
set -uo pipefail
|
||||
|
||||
REPO="${1:?usage: run-all.sh <repo-path> \"<question>\" [headless|tmux|all]}"
|
||||
Q="${2:?question required}"
|
||||
MODE="${3:-headless}"
|
||||
CG_BIN="${CG_BIN:-$(command -v codegraph)}"
|
||||
OUT="${AGENT_EVAL_OUT:-/tmp/agent-eval}"
|
||||
HARNESS="$(cd "$(dirname "$0")" && pwd)"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
# Neutralize any ambient CodeGraph prompt-hook (~/.claude) in BOTH arms:
|
||||
# the hook injects codegraph context into every prompt, which contaminates
|
||||
# the without-arm (free structural context) and double-counts the with-arm.
|
||||
# The A/B's only variable must be the MCP server wired below.
|
||||
export CODEGRAPH_NO_PROMPT_HOOK=1
|
||||
|
||||
[ -n "$CG_BIN" ] || { echo "no codegraph binary on PATH (set CG_BIN)"; exit 1; }
|
||||
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO — index it first"; exit 1; }
|
||||
case "$MODE" in headless|tmux|all) ;; *) echo "mode must be headless|tmux|all (got '$MODE')"; exit 1;; esac
|
||||
|
||||
# MCP config files (path form avoids inline-JSON quoting through tmux).
|
||||
cat > "$OUT/mcp-codegraph.json" <<JSON
|
||||
{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
||||
JSON
|
||||
echo '{"mcpServers":{}}' > "$OUT/mcp-empty.json"
|
||||
|
||||
echo "###### codegraph: $CG_BIN"
|
||||
echo "###### repo: $REPO"
|
||||
echo "###### question: $Q"
|
||||
echo
|
||||
|
||||
# Headless arm: claude -p with stream-json -> exact tool sequence + tokens/cost.
|
||||
headless() {
|
||||
local label="$1" cfg="$2"
|
||||
echo "############################## HEADLESS [$label] ##############################"
|
||||
( cd "$REPO" && claude -p "$Q" \
|
||||
--output-format stream-json --verbose \
|
||||
--permission-mode bypassPermissions \
|
||||
--model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" \
|
||||
--max-budget-usd 4 \
|
||||
--strict-mcp-config --mcp-config "$cfg" \
|
||||
> "$OUT/run-$label.jsonl" 2>"$OUT/run-$label.err" )
|
||||
echo "exit $? -> $OUT/run-$label.jsonl ($(wc -l < "$OUT/run-$label.jsonl" | tr -d ' ') lines)"
|
||||
tail -2 "$OUT/run-$label.err" 2>/dev/null
|
||||
node "$HARNESS/parse-run.mjs" "$OUT/run-$label.jsonl" 2>&1 || true
|
||||
echo
|
||||
}
|
||||
|
||||
if [ "$MODE" = headless ] || [ "$MODE" = all ]; then
|
||||
headless "headless-with" "$OUT/mcp-codegraph.json"
|
||||
headless "headless-without" "$OUT/mcp-empty.json"
|
||||
fi
|
||||
|
||||
if [ "$MODE" = tmux ] || [ "$MODE" = all ]; then
|
||||
echo "############################## INTERACTIVE [with] ##############################"
|
||||
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-codegraph.json" \
|
||||
bash "$HARNESS/itrun.sh" "$REPO" "int-with" "$Q" 2>&1 || echo "[itrun WITH failed]"
|
||||
echo
|
||||
echo "############################## INTERACTIVE [without] ##############################"
|
||||
CLAUDE_EXTRA_ARGS="--model ${MODEL:-sonnet} --effort ${EFFORT:-high} --strict-mcp-config --mcp-config $OUT/mcp-empty.json" \
|
||||
bash "$HARNESS/itrun.sh" "$REPO" "int-without" "$Q" 2>&1 || echo "[itrun WITHOUT failed]"
|
||||
echo
|
||||
fi
|
||||
echo "############################## RUN-ALL COMPLETE ##############################"
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tool-surface ablation — run ONE repo+question under ONE arm.
|
||||
#
|
||||
# Arms vary (exposed codegraph tools, trace-first steering). Tools are trimmed
|
||||
# SERVER-SIDE via CODEGRAPH_MCP_TOOLS in the MCP config's `env` block, so an
|
||||
# ablated tool is genuinely absent from ListTools — no deferred-ToolSearch or
|
||||
# denied-call confound (which --disallowedTools would introduce). Steering is
|
||||
# injected with --append-system-prompt, so no rebuild of the shipped
|
||||
# server-instructions is needed to A/B it.
|
||||
#
|
||||
# A control all tools no steering
|
||||
# B steer all tools trace-first
|
||||
# C no-explore hide explore trace-first
|
||||
# D trace-centric hide explore+context trace-first
|
||||
# E control-probe hide explore+context trace-first (caller passes a NON-flow Q)
|
||||
#
|
||||
# Usage: run-arms.sh <repo-path> "<question>" <A|B|C|D|E> [run-id]
|
||||
set -uo pipefail
|
||||
REPO="${1:?repo path}"; Q="${2:?question}"; ARM="${3:?arm A-E}"; RID="${4:-1}"
|
||||
CG_BIN="${CG_BIN:-$(command -v codegraph)}"
|
||||
OUT="${ARMS_OUT:-/tmp/arms}/$(basename "$REPO")"
|
||||
mkdir -p "$OUT"
|
||||
[ -n "$CG_BIN" ] || { echo "no codegraph binary (set CG_BIN)"; exit 1; }
|
||||
[ -d "$REPO/.codegraph" ] || { echo "no .codegraph index at $REPO"; exit 1; }
|
||||
|
||||
STEER='Flow questions ("how does X reach/become Y", "trace the flow", request to handler, state to render): call codegraph_trace(from,to) FIRST — one call returns the whole path. Use codegraph_context/search only to locate the two endpoint symbols if you do not know them. Do NOT reconstruct the path with repeated search/callers/explore.'
|
||||
KEEP_NO_EXPLORE="trace,search,node,context,callers,callees,impact,files,status"
|
||||
KEEP_TRACE_CENTRIC="trace,search,node,callers,callees,impact,files,status"
|
||||
|
||||
case "$ARM" in
|
||||
A|G|H|I) TOOLS=""; STEERING="" ;; # no steering; H = body-trace, I = body-trace + destination callees (sufficiency)
|
||||
B|F) TOOLS=""; STEERING="$STEER" ;; # F = B's surface, run on the body-inlining trace build
|
||||
C) TOOLS="$KEEP_NO_EXPLORE"; STEERING="$STEER" ;;
|
||||
D|E) TOOLS="$KEEP_TRACE_CENTRIC"; STEERING="$STEER" ;;
|
||||
*) echo "bad arm '$ARM' (want A|B|C|D|E)"; exit 1 ;;
|
||||
esac
|
||||
|
||||
CFG="$OUT/mcp-$ARM.json"
|
||||
if [ -n "$TOOLS" ]; then
|
||||
cat > "$CFG" <<JSON
|
||||
{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"],"env":{"CODEGRAPH_MCP_TOOLS":"$TOOLS"}}}}
|
||||
JSON
|
||||
else
|
||||
cat > "$CFG" <<JSON
|
||||
{"mcpServers":{"codegraph":{"command":"$CG_BIN","args":["serve","--mcp","--path","$REPO"]}}}
|
||||
JSON
|
||||
fi
|
||||
|
||||
LOG="$OUT/$ARM-r$RID.jsonl"; ERR="$OUT/$ARM-r$RID.err"
|
||||
ARGS=( -p "$Q" --output-format stream-json --verbose
|
||||
--permission-mode bypassPermissions --model "${MODEL:-sonnet}" --effort "${EFFORT:-high}" --max-budget-usd 4
|
||||
--strict-mcp-config --mcp-config "$CFG" )
|
||||
[ -n "$STEERING" ] && ARGS+=( --append-system-prompt "$STEERING" )
|
||||
|
||||
( cd "$REPO" && claude "${ARGS[@]}" > "$LOG" 2>"$ERR" )
|
||||
echo "[$(basename "$REPO") $ARM r$RID] exit $? -> $LOG ($(wc -l < "$LOG" | tr -d ' ') lines)"
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env node
|
||||
// Mine the surviving A/B stream-json logs (/tmp/ab-matrix/<Cell>/run-headless-*.jsonl)
|
||||
// for what the aggregate matrix can't see: the call SEQUENCE and per-call output SIZE.
|
||||
//
|
||||
// Answers three questions:
|
||||
// 1. Trace adoption — on a flow question, does the with-arm actually call codegraph_trace?
|
||||
// 2. Payload size vs repo size — is trace path-scoped (tiny, size-independent) while
|
||||
// explore is breadth-scoped (grows with the repo / over-returns on small repos)?
|
||||
// 3. Round-trips — num_turns with vs without (the real wall-clock driver).
|
||||
//
|
||||
// Usage: node scripts/agent-eval/seq-matrix.mjs [/tmp/ab-matrix]
|
||||
import { readFileSync, readdirSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const AB = process.argv[2] || '/tmp/ab-matrix';
|
||||
const MD = new URL('../../docs/benchmarks/codegraph-ab-matrix.md', import.meta.url).pathname;
|
||||
|
||||
// repo -> {lang,size,files} from the published matrix table
|
||||
const repoMeta = {};
|
||||
if (existsSync(MD)) for (const line of readFileSync(MD, 'utf8').split('\n')) {
|
||||
const m = line.match(/^\|\s*([^|]+?)\s*\|\s*(S|M|L)\s*\|\s*`([^`]+)`\s*\|\s*(\d+)\s*\|/);
|
||||
if (m) repoMeta[m[3]] = { lang: m[1].trim(), size: m[2], files: +m[4] };
|
||||
}
|
||||
|
||||
const cgShort = (n) => n.replace('mcp__codegraph__codegraph_', '').replace('mcp__codegraph__', '');
|
||||
const tag = (n) => n === 'Read' ? 'R' : n === 'Grep' ? 'G' : n === 'Glob' ? 'Gl'
|
||||
: n === 'Bash' ? 'B' : n === 'Task' ? 'Ag' : n === 'ToolSearch' ? 'TS'
|
||||
: n.includes('codegraph') ? cgShort(n) : n;
|
||||
|
||||
function parse(file) {
|
||||
if (!existsSync(file)) return null;
|
||||
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
||||
const calls = []; let result = null, initCg = 0;
|
||||
for (const l of lines) {
|
||||
let ev; try { ev = JSON.parse(l); } catch { continue; }
|
||||
if (ev.type === 'system' && ev.subtype === 'init') initCg = (ev.tools || []).filter(t => /codegraph/.test(t)).length;
|
||||
if (ev.type === 'assistant') for (const b of (ev.message?.content || [])) if (b.type === 'tool_use') {
|
||||
const i = b.input || {};
|
||||
const q = i.query ?? i.symbol ?? i.task ?? (i.from && i.to ? `${i.from}->${i.to}` : (i.file_path || i.command || ''));
|
||||
calls.push({ id: b.id, name: b.name, q: String(q ?? '').slice(0, 38), out: 0 });
|
||||
}
|
||||
if (ev.type === 'user') for (const b of (ev.message?.content || [])) if (b.type === 'tool_result') {
|
||||
const c = b.content;
|
||||
const txt = typeof c === 'string' ? c : Array.isArray(c) ? c.map(x => x?.text || '').join('') : '';
|
||||
const call = calls.find(k => k.id === b.tool_use_id); if (call) call.out = txt.length;
|
||||
}
|
||||
if (ev.type === 'result') result = ev;
|
||||
}
|
||||
const cg = calls.filter(c => c.name.includes('codegraph'));
|
||||
const perTool = {};
|
||||
for (const c of cg) { const k = cgShort(c.name); (perTool[k] ??= { n: 0, out: 0 }); perTool[k].n++; perTool[k].out += c.out; }
|
||||
const traceIdx = cg.findIndex(c => c.name.includes('trace'));
|
||||
const u = result?.usage || {};
|
||||
return {
|
||||
initCg, cg, perTool,
|
||||
cgSeq: cg.map(c => cgShort(c.name)),
|
||||
seq: calls.map(c => tag(c.name)),
|
||||
reads: calls.filter(c => c.name === 'Read').length,
|
||||
greps: calls.filter(c => c.name === 'Grep').length,
|
||||
cgOut: cg.reduce((s, c) => s + c.out, 0),
|
||||
traceUsed: traceIdx >= 0,
|
||||
afterTrace: traceIdx >= 0 ? cg.slice(traceIdx + 1).map(c => cgShort(c.name)) : null,
|
||||
turns: result?.num_turns ?? null,
|
||||
dur: result?.duration_ms ? Math.round(result.duration_ms / 1000) : null,
|
||||
cost: result?.total_cost_usd || 0,
|
||||
};
|
||||
}
|
||||
|
||||
const cells = [];
|
||||
for (const d of readdirSync(AB)) {
|
||||
const dir = join(AB, d);
|
||||
if (!existsSync(join(dir, 'run-headless-with.jsonl'))) continue;
|
||||
const log = existsSync(join(AB, d + '.log')) ? readFileSync(join(AB, d + '.log'), 'utf8') : '';
|
||||
const repo = (log.match(/repo:\s*\S*\/([^\s/]+)/) || [])[1] || d;
|
||||
const question = (log.match(/question:\s*(.+)/) || [])[1] || '';
|
||||
cells.push({ cell: d, repo, question, ...(repoMeta[repo] || {}),
|
||||
with: parse(join(dir, 'run-headless-with.jsonl')),
|
||||
without: parse(join(dir, 'run-headless-without.jsonl')) });
|
||||
}
|
||||
cells.sort((a, b) => (a.files || 0) - (b.files || 0));
|
||||
|
||||
const k = (n) => (n / 1000).toFixed(1);
|
||||
const pad = (s, n) => String(s).padEnd(n);
|
||||
|
||||
// ---- per-cell sequence table ----
|
||||
console.log('\n=== PER-CELL: with-arm codegraph sequence + payload (sorted by repo size) ===');
|
||||
console.log(pad('repo', 22), pad('files', 6), 'trace', pad('cg-call sequence', 40), pad('cgOutK', 7), 'turns(w/wo)');
|
||||
for (const c of cells) {
|
||||
const w = c.with;
|
||||
console.log(
|
||||
pad(c.repo, 22), pad(c.files ?? '?', 6),
|
||||
pad(w.traceUsed ? 'YES' : 'no', 5),
|
||||
pad(w.cgSeq.join(',') || '(none)', 40),
|
||||
pad(k(w.cgOut), 7),
|
||||
`${w.turns}/${c.without?.turns}`,
|
||||
);
|
||||
}
|
||||
|
||||
// ---- trace adoption ----
|
||||
const flow = cells; // every matrix question is a canonical flow question by design
|
||||
const used = flow.filter(c => c.with.traceUsed);
|
||||
console.log(`\n=== TRACE ADOPTION (all ${flow.length} cells are flow questions) ===`);
|
||||
console.log(`trace called in ${used.length}/${flow.length} cells`);
|
||||
console.log('used trace:', used.map(c => c.repo).join(', ') || '(none)');
|
||||
if (used.length) console.log('after-trace follow-ups:', used.map(c => `${c.repo}[${c.with.afterTrace.join(',') || 'none'}]`).join(' '));
|
||||
|
||||
// ---- payload size by repo-size tier ----
|
||||
const tier = (f) => f < 200 ? 'S(<200)' : f < 2000 ? 'M(<2000)' : 'L(>=2000)';
|
||||
const byTier = {};
|
||||
for (const c of cells) { (byTier[tier(c.files || 0)] ??= []).push(c.with.cgOut); }
|
||||
console.log('\n=== with-arm TOTAL codegraph payload by repo-size tier ===');
|
||||
for (const t of ['S(<200)', 'M(<2000)', 'L(>=2000)']) {
|
||||
const a = byTier[t] || []; if (!a.length) continue;
|
||||
const avg = a.reduce((s, x) => s + x, 0) / a.length;
|
||||
console.log(` ${pad(t, 10)} n=${a.length} avg cgOut=${k(avg)}K range ${k(Math.min(...a))}-${k(Math.max(...a))}K`);
|
||||
}
|
||||
|
||||
// ---- per-tool usage + avg payload (breadth vs path evidence) ----
|
||||
const tot = {};
|
||||
for (const c of cells) for (const [name, v] of Object.entries(c.with.perTool)) {
|
||||
(tot[name] ??= { n: 0, out: 0 }); tot[name].n += v.n; tot[name].out += v.out;
|
||||
}
|
||||
console.log('\n=== codegraph tool usage across all cells (n calls, avg payload/call) ===');
|
||||
for (const [name, v] of Object.entries(tot).sort((a, b) => b[1].n - a[1].n)) {
|
||||
console.log(` ${pad(name, 10)} calls=${pad(v.n, 4)} avg=${k(v.out / v.n)}K/call total=${k(v.out)}K`);
|
||||
}
|
||||
|
||||
// ---- round-trips ----
|
||||
const sum = (arr, f) => arr.reduce((s, x) => s + (f(x) || 0), 0);
|
||||
const wTurns = sum(cells, c => c.with.turns), woTurns = sum(cells, c => c.without?.turns);
|
||||
const wCalls = sum(cells, c => c.with.cg.length);
|
||||
const tsAll = cells.every(c => c.with.seq[0] === 'TS');
|
||||
console.log('\n=== ROUND-TRIPS ===');
|
||||
console.log(`turns: with=${wTurns} without=${woTurns} (${((1 - wTurns / woTurns) * 100).toFixed(0)}% fewer with)`);
|
||||
console.log(`avg turns/cell: with=${(wTurns / cells.length).toFixed(1)} without=${(woTurns / cells.length).toFixed(1)}`);
|
||||
console.log(`total codegraph calls=${wCalls} (avg ${(wCalls / cells.length).toFixed(1)}/cell)`);
|
||||
console.log(`every with-arm opens with a ToolSearch round-trip (deferred tools): ${tsAll ? 'YES — 1 fixed tax/run' : 'no'}`);
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build a self-contained CodeGraph bundle: an official Node runtime + the
|
||||
# compiled app + its production deps, so CodeGraph runs with NO system Node and
|
||||
# NO native build — node:sqlite is built into the bundled Node. One archive per
|
||||
# platform.
|
||||
#
|
||||
# Because dropping better-sqlite3 left zero native addons, the recipe is pure
|
||||
# file-packaging (download the target's Node, copy the app, archive) — so any
|
||||
# platform's bundle can be built on any OS. No cross-compile, no native runners.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/build-bundle.sh <target> [node-version]
|
||||
# target: darwin-arm64 | darwin-x64 | linux-x64 | linux-arm64
|
||||
# | win32-x64 | win32-arm64
|
||||
# node-version: e.g. v24.16.0 (default below; pin for reproducible builds)
|
||||
#
|
||||
# Output:
|
||||
# unix: release/codegraph-<target>.tar.gz (launcher: bin/codegraph)
|
||||
# windows: release/codegraph-<target>.zip (launcher: bin/codegraph.cmd)
|
||||
set -euo pipefail
|
||||
|
||||
TARGET="${1:?usage: build-bundle.sh <target> [node-version]}"
|
||||
NODE_VERSION="${2:-v24.16.0}"
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT="$ROOT/release"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
ARCH="${TARGET##*-}" # x64 | arm64
|
||||
OSFAM="${TARGET%-*}" # darwin | linux | win32
|
||||
|
||||
echo "[bundle] target=${TARGET} node=${NODE_VERSION}"
|
||||
|
||||
# 1. Download + extract the official Node runtime for the target platform.
|
||||
if [ "$OSFAM" = "win32" ]; then
|
||||
NODE_DIST="node-${NODE_VERSION}-win-${ARCH}"
|
||||
NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.zip"
|
||||
echo "[bundle] downloading ${NODE_URL}"
|
||||
curl -fsSL "$NODE_URL" -o "$WORK/node.zip"
|
||||
if command -v unzip >/dev/null 2>&1; then
|
||||
unzip -q "$WORK/node.zip" -d "$WORK"
|
||||
else
|
||||
tar -xf "$WORK/node.zip" -C "$WORK" # bsdtar can read zip
|
||||
fi
|
||||
NODE_BIN="$WORK/${NODE_DIST}/node.exe"
|
||||
else
|
||||
NODE_DIST="node-${NODE_VERSION}-${TARGET}"
|
||||
NODE_URL="https://nodejs.org/dist/${NODE_VERSION}/${NODE_DIST}.tar.gz"
|
||||
echo "[bundle] downloading ${NODE_URL}"
|
||||
curl -fsSL "$NODE_URL" -o "$WORK/node.tar.gz"
|
||||
tar -xzf "$WORK/node.tar.gz" -C "$WORK"
|
||||
NODE_BIN="$WORK/${NODE_DIST}/bin/node"
|
||||
fi
|
||||
[ -f "$NODE_BIN" ] || { echo "[bundle] error: node binary not found ($NODE_BIN)" >&2; exit 1; }
|
||||
|
||||
# 2. Build the app (compiled JS + copied wasm/schema assets).
|
||||
echo "[bundle] building app"
|
||||
( cd "$ROOT" && npm run build >/dev/null )
|
||||
|
||||
# 3. Stage: app + production-only deps (pure JS/wasm → portable across platforms).
|
||||
STAGE="$WORK/codegraph-${TARGET}"
|
||||
mkdir -p "$STAGE/lib" "$STAGE/bin"
|
||||
cp -R "$ROOT/dist" "$STAGE/lib/dist"
|
||||
cp "$ROOT/package.json" "$ROOT/package-lock.json" "$STAGE/lib/"
|
||||
echo "[bundle] installing production dependencies"
|
||||
( cd "$STAGE/lib" && npm ci --omit=dev --ignore-scripts >/dev/null 2>&1 )
|
||||
rm -f "$STAGE/lib/package-lock.json"
|
||||
|
||||
# 4. Vendored Node + launcher (the launcher uses the bundled Node by relative
|
||||
# path, so no system Node is ever needed).
|
||||
#
|
||||
# `--liftoff-only`: keep tree-sitter's large WASM grammars on V8's Liftoff
|
||||
# baseline compiler so they never reach the turboshaft optimizing tier, whose
|
||||
# per-compilation Zone arena OOMs the whole process (`Fatal process out of
|
||||
# memory: Zone`) on Node >= 22 — even with tens of GB free. The flag is read at
|
||||
# V8 engine init so it must be on node's command line; the parse worker inherits
|
||||
# it. See issues #293/#298 and src/extraction/wasm-runtime-flags.ts. (The CLI
|
||||
# also self-relaunches with this flag when launched without it, so non-bundled
|
||||
# runs are covered too; passing it here avoids that extra spawn.)
|
||||
if [ "$OSFAM" = "win32" ]; then
|
||||
cp "$NODE_BIN" "$STAGE/node.exe"
|
||||
printf '@"%%~dp0..\\node.exe" --liftoff-only "%%~dp0..\\lib\\dist\\bin\\codegraph.js" %%*\r\n' \
|
||||
> "$STAGE/bin/codegraph.cmd"
|
||||
else
|
||||
cp "$NODE_BIN" "$STAGE/node"
|
||||
cat > "$STAGE/bin/codegraph" <<'LAUNCH'
|
||||
#!/bin/sh
|
||||
# Resolve symlinks (e.g. the ~/.local/bin/codegraph link install.sh creates) so
|
||||
# we find the real bundle dir, not the symlink's location.
|
||||
SELF="$0"
|
||||
while [ -L "$SELF" ]; do
|
||||
target="$(readlink "$SELF")"
|
||||
case "$target" in
|
||||
/*) SELF="$target" ;;
|
||||
*) SELF="$(dirname "$SELF")/$target" ;;
|
||||
esac
|
||||
done
|
||||
DIR="$(cd "$(dirname "$SELF")/.." && pwd)"
|
||||
# Thread the MCP host's pid to the server's orphan watchdog (issue #1185).
|
||||
# $PPID is our parent — the host itself when it launched this script directly;
|
||||
# an already-threaded value (the npm shim sets the true host pid) wins.
|
||||
CODEGRAPH_HOST_PPID="${CODEGRAPH_HOST_PPID:-$PPID}"
|
||||
export CODEGRAPH_HOST_PPID
|
||||
# --liftoff-only: avoid the V8 turboshaft WASM Zone OOM (issues #293/#298).
|
||||
exec "$DIR/node" --liftoff-only "$DIR/lib/dist/bin/codegraph.js" "$@"
|
||||
LAUNCH
|
||||
chmod +x "$STAGE/bin/codegraph"
|
||||
fi
|
||||
|
||||
# 5. Archive (.zip for Windows, .tar.gz otherwise).
|
||||
mkdir -p "$OUT"
|
||||
if [ "$OSFAM" = "win32" ]; then
|
||||
ARCHIVE="$OUT/codegraph-${TARGET}.zip"
|
||||
rm -f "$ARCHIVE"
|
||||
( cd "$WORK" && zip -rqX "$ARCHIVE" "codegraph-${TARGET}" )
|
||||
else
|
||||
ARCHIVE="$OUT/codegraph-${TARGET}.tar.gz"
|
||||
# --no-xattrs: don't embed macOS xattrs that make GNU tar warn on Linux.
|
||||
tar --no-xattrs -czf "$ARCHIVE" -C "$WORK" "codegraph-${TARGET}"
|
||||
fi
|
||||
echo "[bundle] wrote ${ARCHIVE} ($(du -h "$ARCHIVE" | cut -f1))"
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Extract a release-notes block from CHANGELOG.md for a given version
|
||||
* (or unwrap text supplied on stdin), then join hard-wrapped paragraphs.
|
||||
*
|
||||
* Why: GitHub renders release-note Markdown with GFM hard breaks, so
|
||||
* every `\n` becomes `<br>`. The CHANGELOG is hard-wrapped at ~75
|
||||
* chars for readable diffs, which then renders as awkward visible
|
||||
* line breaks on the release page. This script joins indented
|
||||
* continuation lines into a single line per bullet so the GFM
|
||||
* renderer produces clean paragraphs.
|
||||
*
|
||||
* Repo-level CHANGELOG.md viewing is unaffected (CommonMark treats
|
||||
* newlines as spaces there).
|
||||
*
|
||||
* Usage:
|
||||
* extract-release-notes.mjs <version> # read CHANGELOG.md
|
||||
* extract-release-notes.mjs --stdin # read from stdin (any text)
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
const arg = process.argv[2];
|
||||
if (!arg) {
|
||||
console.error('usage: extract-release-notes.mjs <version> | --stdin');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let block;
|
||||
if (arg === '--stdin') {
|
||||
block = readFileSync(0, 'utf8').replace(/\r\n?/g, '\n').split('\n');
|
||||
} else {
|
||||
const version = arg;
|
||||
const escaped = version.replace(/\./g, '\\.');
|
||||
const headerRe = new RegExp(`^## \\[${escaped}\\]`);
|
||||
const anyHeaderRe = /^## \[/;
|
||||
const lines = readFileSync('CHANGELOG.md', 'utf8').split('\n');
|
||||
const start = lines.findIndex((l) => headerRe.test(l));
|
||||
if (start === -1) {
|
||||
console.error(`no '## [${version}]' entry found in CHANGELOG.md`);
|
||||
process.exit(1);
|
||||
}
|
||||
const after = lines.findIndex((l, i) => i > start && anyHeaderRe.test(l));
|
||||
block = lines.slice(start, after === -1 ? lines.length : after);
|
||||
}
|
||||
|
||||
// Track a stack of `{ indent: number }` frames so a continuation line
|
||||
// can attach to the right ancestor. Handles the post-nested-list
|
||||
// continuation pattern:
|
||||
//
|
||||
// - top-level
|
||||
// - nested
|
||||
// back to top-level <- 2-space indent, joins the top-level bullet
|
||||
const out = [];
|
||||
let buf = '';
|
||||
let stack = [];
|
||||
|
||||
function flushBuf() {
|
||||
if (buf !== '') {
|
||||
out.push(buf);
|
||||
buf = '';
|
||||
}
|
||||
}
|
||||
|
||||
function leadingSpaces(s) {
|
||||
const m = s.match(/^(\s*)/);
|
||||
return m ? m[1].length : 0;
|
||||
}
|
||||
|
||||
// Bullets: `-`, `*`, `digit.` only. `+` is intentionally excluded — the
|
||||
// CHANGELOG uses literal `+` inline (`config + instructions`) and we
|
||||
// don't want to misread those as nested bullets.
|
||||
const listItemRe = /^(\s*)([-*]|\d+\.)\s+/;
|
||||
const fenceRe = /^\s*```/;
|
||||
|
||||
let inFence = false;
|
||||
|
||||
for (const line of block) {
|
||||
// Fenced code blocks: pass through verbatim, no joining.
|
||||
if (fenceRe.test(line)) {
|
||||
flushBuf();
|
||||
stack = [];
|
||||
out.push(line);
|
||||
inFence = !inFence;
|
||||
continue;
|
||||
}
|
||||
if (inFence) {
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
if (/^\s*$/.test(line)) {
|
||||
flushBuf();
|
||||
out.push('');
|
||||
continue;
|
||||
}
|
||||
if (/^#/.test(line)) {
|
||||
flushBuf();
|
||||
stack = [];
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
const itemMatch = line.match(listItemRe);
|
||||
if (itemMatch) {
|
||||
flushBuf();
|
||||
const indent = itemMatch[1].length;
|
||||
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
|
||||
stack.pop();
|
||||
}
|
||||
stack.push({ indent });
|
||||
buf = line;
|
||||
continue;
|
||||
}
|
||||
if (/^\s/.test(line)) {
|
||||
const indent = leadingSpaces(line);
|
||||
while (stack.length > 1 && stack[stack.length - 1].indent >= indent) {
|
||||
flushBuf();
|
||||
stack.pop();
|
||||
}
|
||||
const trimmed = line.replace(/^\s+/, '');
|
||||
buf = buf === '' ? trimmed : `${buf} ${trimmed}`;
|
||||
continue;
|
||||
}
|
||||
flushBuf();
|
||||
stack = [];
|
||||
out.push(line);
|
||||
}
|
||||
flushBuf();
|
||||
|
||||
process.stdout.write(out.join('\n'));
|
||||
if (!out[out.length - 1]?.endsWith('\n')) process.stdout.write('\n');
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the current branch and link it as the global `codegraph` for
|
||||
# hands-on testing. Replaces any existing global install for as long
|
||||
# as the symlink is in place.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/local-install.sh # build + link
|
||||
# ./scripts/local-install.sh --undo # unlink + restore the published version
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
PKG=$(node -p "require('./package.json').name")
|
||||
VERSION=$(node -p "require('./package.json').version")
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
|
||||
if [ "${1:-}" = "--undo" ]; then
|
||||
echo "→ unlinking ${PKG}"
|
||||
npm unlink -g "${PKG}" >/dev/null 2>&1 || true
|
||||
echo "→ reinstalling published ${PKG}"
|
||||
npm install -g "${PKG}"
|
||||
echo "done: global codegraph -> $(command -v codegraph)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "→ building ${PKG} ${VERSION} (${BRANCH})"
|
||||
npm run build
|
||||
|
||||
echo "→ linking globally"
|
||||
npm link
|
||||
|
||||
LINKED=$(command -v codegraph || echo "(not on PATH)")
|
||||
echo
|
||||
echo "✓ global codegraph now points to this branch"
|
||||
echo " binary: ${LINKED}"
|
||||
echo " branch: ${BRANCH}"
|
||||
echo " version: ${VERSION}"
|
||||
echo
|
||||
echo "To restore the published version:"
|
||||
echo " ./scripts/local-install.sh --undo"
|
||||
@@ -0,0 +1,75 @@
|
||||
'use strict';
|
||||
//
|
||||
// Programmatic / embedded SDK entry for @colbymchenry/codegraph (issue #354).
|
||||
//
|
||||
// The CLI/MCP `bin` (npm-shim.js) execs the per-platform bundle's OWN Node 24 so
|
||||
// the tool never depends on the user's runtime. Embedded library consumers are
|
||||
// the opposite case: they already run their own Node and just want the compiled
|
||||
// API — `require("@colbymchenry/codegraph")` returning the CodeGraph class et al.
|
||||
//
|
||||
// The compiled library + its production dependencies (web-tree-sitter,
|
||||
// tree-sitter-wasms, …) ship INSIDE the per-platform bundle, at
|
||||
// @colbymchenry/codegraph-<platform>-<arch>/lib/dist/index.js
|
||||
// (with the deps in the sibling lib/node_modules). Re-exporting that bundle keeps
|
||||
// the main package thin — no second 50 MB copy of the grammars — while making the
|
||||
// SDK work in the consumer's process. Types are a separate concern: the main
|
||||
// package ships its own dist/**/*.d.ts tree (pointed at by `types`), built from
|
||||
// the same release so it can never skew from the runtime it re-exports.
|
||||
//
|
||||
// node:sqlite (Node >= 22.5) is required to OPEN a graph, but only lazily inside
|
||||
// the SQLite adapter — so loading this module is safe on older Node, and the
|
||||
// node:sqlite requirement surfaces with an actionable error only when a DB is
|
||||
// actually opened. Heavy extraction additionally wants the bundled launcher's
|
||||
// --liftoff-only flag (the WASM Zone-OOM guard, issues #293/#298); an embedded
|
||||
// host that drives large indexing should pass that flag to its own Node.
|
||||
|
||||
var path = require('path');
|
||||
var os = require('os');
|
||||
var fs = require('fs');
|
||||
|
||||
var target = process.platform + '-' + process.arch; // e.g. darwin-arm64, linux-x64
|
||||
var pkg = '@colbymchenry/codegraph-' + target;
|
||||
|
||||
module.exports = require(resolveLibrary());
|
||||
|
||||
// Locate the compiled library entry inside the installed per-platform bundle.
|
||||
// Throws an actionable error (rather than a bare MODULE_NOT_FOUND) when no bundle
|
||||
// is present, so an embedded consumer knows exactly what to install.
|
||||
function resolveLibrary() {
|
||||
// 1) The npm-installed optional dependency — the normal case.
|
||||
try {
|
||||
return require.resolve(pkg + '/lib/dist/index.js');
|
||||
} catch (e) {
|
||||
/* fall through to the self-healed cache */
|
||||
}
|
||||
|
||||
// 2) A bundle the CLI shim self-healed from GitHub Releases into the cache
|
||||
// (issue #303). Same node/lib/bin layout as the npm package. We only REUSE a
|
||||
// cached bundle here — unlike the CLI shim we never trigger a network
|
||||
// download from inside require(), which must stay synchronous and cheap.
|
||||
var cached = cachedLibrary();
|
||||
if (cached) return cached;
|
||||
|
||||
throw new Error(
|
||||
'codegraph: the programmatic API is unavailable because the platform bundle\n' +
|
||||
'(' + pkg + ') is not installed.\n' +
|
||||
'The compiled library ships inside that per-platform optional dependency.\n' +
|
||||
'Fixes:\n' +
|
||||
' - install from the official npm registry so the matching bundle is fetched:\n' +
|
||||
' npm i @colbymchenry/codegraph --registry=https://registry.npmjs.org\n' +
|
||||
' - or run the CLI once (e.g. `npx @colbymchenry/codegraph status`) to\n' +
|
||||
' self-heal the bundle into ~/.codegraph, then require() will find it.'
|
||||
);
|
||||
}
|
||||
|
||||
function cachedLibrary() {
|
||||
try {
|
||||
var version = require(path.join(__dirname, 'package.json')).version;
|
||||
var base = process.env.CODEGRAPH_INSTALL_DIR || path.join(os.homedir(), '.codegraph');
|
||||
var lib = path.join(base, 'bundles', target + '-' + version, 'lib', 'dist', 'index.js');
|
||||
if (fs.existsSync(lib)) return lib;
|
||||
} catch (e) {
|
||||
/* no readable cache → caller reports the install guidance */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
Executable
+275
@@ -0,0 +1,275 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
//
|
||||
// npm thin-installer launcher for CodeGraph.
|
||||
//
|
||||
// The heavy artifact (a vendored Node runtime + the app) ships as a per-platform
|
||||
// optionalDependency: @colbymchenry/codegraph-<platform>-<arch>. npm installs
|
||||
// only the one matching the host, via each package's `os`/`cpu` fields (the
|
||||
// esbuild pattern). This shim — run by the user's OWN Node — locates that bundle
|
||||
// and execs its launcher, so the real work always runs on the bundled Node 24
|
||||
// (with node:sqlite), regardless of the user's Node version. The user's Node is
|
||||
// only ever a launcher; even an ancient version can run this file.
|
||||
//
|
||||
// Self-heal (issue #303): some registries — notably the npmmirror/cnpm mirrors,
|
||||
// and some corporate proxies — don't reliably mirror the per-platform
|
||||
// optionalDependencies. npm treats an unfetchable optional dep as success and
|
||||
// silently skips it, so the bundle goes missing and every command fails. When
|
||||
// the installed bundle can't be resolved, this shim falls back to downloading
|
||||
// the matching bundle straight from GitHub Releases — the very archive
|
||||
// install.sh uses — into a cache dir, then runs that. Knobs:
|
||||
// CODEGRAPH_NO_DOWNLOAD=1 disable the network fallback (print guidance)
|
||||
// CODEGRAPH_INSTALL_DIR=DIR cache location (default: ~/.codegraph)
|
||||
// CODEGRAPH_DOWNLOAD_BASE=URL release-download base (for mirrors/air-gapped)
|
||||
//
|
||||
// Wired up at release time as the main package's `bin`:
|
||||
// "bin": { "codegraph": "npm-shim.js" }
|
||||
// with the platform packages listed in `optionalDependencies`.
|
||||
|
||||
var childProcess = require('child_process');
|
||||
var fs = require('fs');
|
||||
var os = require('os');
|
||||
var path = require('path');
|
||||
|
||||
var target = process.platform + '-' + process.arch; // e.g. darwin-arm64, linux-x64
|
||||
var pkg = '@colbymchenry/codegraph-' + target;
|
||||
var isWindows = process.platform === 'win32';
|
||||
var REPO = 'colbymchenry/codegraph';
|
||||
|
||||
main().catch(function (e) {
|
||||
process.stderr.write('codegraph: ' + (e && e.message ? e.message : String(e)) + '\n');
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
async function main() {
|
||||
// Happy path: the npm-installed optional dependency. Fall back to a download
|
||||
// when the registry didn't deliver it.
|
||||
var resolved = resolveInstalledBundle() || (await selfHealBundle());
|
||||
// Thread the MCP host's pid (our parent) down to the bundled server so its
|
||||
// orphan watchdog can poll the host directly. Without this, the server can
|
||||
// only watch THIS shim — and a shim killed during the server's first ~100ms
|
||||
// of startup used to leave the server orphaned forever (issue #1185). An
|
||||
// already-set value (an outer launcher) wins.
|
||||
var env = Object.assign({}, process.env);
|
||||
if (!env.CODEGRAPH_HOST_PPID) env.CODEGRAPH_HOST_PPID = String(process.ppid);
|
||||
var res = childProcess.spawnSync(resolved.command, resolved.args, { stdio: 'inherit', windowsHide: true, env: env });
|
||||
if (res.error) {
|
||||
process.stderr.write('codegraph: ' + res.error.message + '\n');
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(res.status === null ? 1 : res.status);
|
||||
}
|
||||
|
||||
// Resolve the launcher from the installed per-platform optionalDependency.
|
||||
// Returns {command, args} or null if the package isn't installed.
|
||||
function resolveInstalledBundle() {
|
||||
try {
|
||||
if (isWindows) {
|
||||
// Modern Node refuses to spawn the bundle's .cmd directly (EINVAL, the
|
||||
// CVE-2024-27980 hardening on Node 24), so invoke the bundled node.exe
|
||||
// against the app entry point and pass --liftoff-only here.
|
||||
var nodeExe = require.resolve(pkg + '/node.exe');
|
||||
var entry = require.resolve(pkg + '/lib/dist/bin/codegraph.js');
|
||||
return { command: nodeExe, args: liftoff(entry) };
|
||||
}
|
||||
return { command: require.resolve(pkg + '/bin/codegraph'), args: process.argv.slice(2) };
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Locate the launcher inside an extracted GitHub bundle directory (same
|
||||
// node/lib/bin layout as the npm platform package). Returns {command, args} or
|
||||
// null when the directory doesn't hold a usable bundle yet.
|
||||
function launcherIn(dir) {
|
||||
if (isWindows) {
|
||||
var nodeExe = path.join(dir, 'node.exe');
|
||||
var entry = path.join(dir, 'lib', 'dist', 'bin', 'codegraph.js');
|
||||
if (fs.existsSync(nodeExe) && fs.existsSync(entry)) {
|
||||
return { command: nodeExe, args: liftoff(entry) };
|
||||
}
|
||||
} else {
|
||||
var launcher = path.join(dir, 'bin', 'codegraph');
|
||||
if (fs.existsSync(launcher)) return { command: launcher, args: process.argv.slice(2) };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// --liftoff-only keeps tree-sitter's WASM grammars off V8's turboshaft tier to
|
||||
// avoid the Zone OOM on Node >= 22 (issues #293/#298). The unix bin/codegraph
|
||||
// launcher already passes it; on Windows we invoke node.exe directly so add it.
|
||||
function liftoff(entry) {
|
||||
return ['--liftoff-only', entry].concat(process.argv.slice(2));
|
||||
}
|
||||
|
||||
// Download + cache the platform bundle from GitHub Releases. Returns
|
||||
// {command, args}; exits the process with guidance if it can't.
|
||||
async function selfHealBundle() {
|
||||
var version = readVersion();
|
||||
var bundlesDir = path.join(process.env.CODEGRAPH_INSTALL_DIR || path.join(os.homedir(), '.codegraph'), 'bundles');
|
||||
var dest = path.join(bundlesDir, target + '-' + version);
|
||||
|
||||
// Already downloaded by a previous run? Use it even when downloads are
|
||||
// disabled — CODEGRAPH_NO_DOWNLOAD blocks fetching, not a cached bundle.
|
||||
var cached = launcherIn(dest);
|
||||
if (cached) { pruneOldBundles(bundlesDir, dest); return cached; }
|
||||
|
||||
if (process.env.CODEGRAPH_NO_DOWNLOAD) {
|
||||
fail('the network fallback is disabled (CODEGRAPH_NO_DOWNLOAD is set).');
|
||||
}
|
||||
|
||||
var asset = 'codegraph-' + target + (isWindows ? '.zip' : '.tar.gz');
|
||||
var base = process.env.CODEGRAPH_DOWNLOAD_BASE || ('https://github.com/' + REPO + '/releases/download');
|
||||
var url = base + '/v' + version + '/' + asset;
|
||||
|
||||
process.stderr.write(
|
||||
'codegraph: platform bundle missing (registry did not provide ' + pkg + ').\n' +
|
||||
'codegraph: downloading ' + asset + ' from GitHub Releases (' + version + ')...\n'
|
||||
);
|
||||
|
||||
// Stage inside bundlesDir so the final rename is on the same filesystem (atomic,
|
||||
// no EXDEV across tmpfs). Strip the archive's top-level codegraph-<target>/ dir.
|
||||
fs.mkdirSync(bundlesDir, { recursive: true });
|
||||
var stage = fs.mkdtempSync(path.join(bundlesDir, '.dl-'));
|
||||
try {
|
||||
var archivePath = path.join(stage, asset);
|
||||
await download(url, archivePath, 6);
|
||||
await verifyChecksum(archivePath, asset, base, version);
|
||||
var extracted = path.join(stage, 'bundle');
|
||||
fs.mkdirSync(extracted);
|
||||
extract(archivePath, extracted);
|
||||
|
||||
var raced = launcherIn(dest); // another process may have finished meanwhile
|
||||
if (raced) { rmrf(stage); return raced; }
|
||||
try {
|
||||
fs.renameSync(extracted, dest);
|
||||
} catch (e) {
|
||||
var other = launcherIn(dest); // lost the race but theirs is valid
|
||||
if (other) { rmrf(stage); return other; }
|
||||
throw e;
|
||||
}
|
||||
} catch (e) {
|
||||
rmrf(stage);
|
||||
fail('download failed (' + e.message + ').\n URL: ' + url);
|
||||
}
|
||||
rmrf(stage);
|
||||
|
||||
var ready = launcherIn(dest);
|
||||
if (!ready) fail('downloaded bundle is missing its launcher under ' + dest + '.');
|
||||
pruneOldBundles(bundlesDir, dest);
|
||||
process.stderr.write('codegraph: bundle ready.\n');
|
||||
return ready;
|
||||
}
|
||||
|
||||
function readVersion() {
|
||||
try {
|
||||
return require(path.join(__dirname, 'package.json')).version;
|
||||
} catch (e) {
|
||||
fail('could not read this package\'s version to locate a matching release.');
|
||||
}
|
||||
}
|
||||
|
||||
// GET with manual redirect following (GitHub release URLs redirect to a CDN).
|
||||
function download(url, dest, redirectsLeft) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
var https = require('https');
|
||||
// timeout is an idle/inactivity timeout — it won't kill a slow-but-progressing
|
||||
// download, only a stalled connection (so a blocked mirror fails fast with
|
||||
// guidance instead of hanging the user's command forever).
|
||||
var req = https.get(url, { headers: { 'User-Agent': 'codegraph-npm-shim' }, timeout: 30000 }, function (res) {
|
||||
var status = res.statusCode;
|
||||
if (status >= 300 && status < 400 && res.headers.location) {
|
||||
res.resume();
|
||||
if (redirectsLeft <= 0) { reject(new Error('too many redirects')); return; }
|
||||
download(new URL(res.headers.location, url).toString(), dest, redirectsLeft - 1).then(resolve, reject);
|
||||
return;
|
||||
}
|
||||
if (status !== 200) { res.resume(); reject(new Error('HTTP ' + status)); return; }
|
||||
var file = fs.createWriteStream(dest);
|
||||
res.on('error', reject);
|
||||
res.pipe(file);
|
||||
file.on('error', reject);
|
||||
file.on('finish', function () { file.close(function () { resolve(); }); });
|
||||
});
|
||||
req.on('timeout', function () { req.destroy(new Error('connection timed out')); });
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
// Best-effort integrity check. When the release publishes a SHA256SUMS file, the
|
||||
// downloaded archive MUST match its listed hash or we abort. When that file is
|
||||
// absent (older releases) or simply unreachable, we proceed — the archive still
|
||||
// arrived from GitHub over TLS. So tampering/corruption is caught, while a
|
||||
// missing checksum never breaks an install.
|
||||
async function verifyChecksum(archivePath, asset, base, version) {
|
||||
var sumsPath = archivePath + '.SHA256SUMS';
|
||||
try {
|
||||
await download(base + '/v' + version + '/SHA256SUMS', sumsPath, 6);
|
||||
} catch (e) {
|
||||
return; // not published / unreachable → skip
|
||||
}
|
||||
var expected = null;
|
||||
var lines = fs.readFileSync(sumsPath, 'utf8').split('\n');
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var m = lines[i].trim().match(/^([0-9a-fA-F]{64})\s+\*?(.+)$/);
|
||||
if (m && path.basename(m[2].trim()) === asset) { expected = m[1].toLowerCase(); break; }
|
||||
}
|
||||
if (!expected) return; // asset not listed → nothing to check
|
||||
var actual = require('crypto').createHash('sha256').update(fs.readFileSync(archivePath)).digest('hex');
|
||||
if (actual !== expected) {
|
||||
throw new Error('checksum mismatch for ' + asset +
|
||||
' (expected ' + expected.slice(0, 12) + '…, got ' + actual.slice(0, 12) + '…)');
|
||||
}
|
||||
process.stderr.write('codegraph: checksum verified.\n');
|
||||
}
|
||||
|
||||
// Extract via the system tar — present on macOS, Linux, and Windows 10+
|
||||
// (bsdtar reads .zip too). No third-party dependency in the shim.
|
||||
function extract(archive, destDir) {
|
||||
var args = isWindows
|
||||
? ['-xf', archive, '-C', destDir, '--strip-components=1']
|
||||
: ['-xzf', archive, '-C', destDir, '--strip-components=1'];
|
||||
var res = childProcess.spawnSync('tar', args, { stdio: 'ignore', windowsHide: true });
|
||||
if (res.error) throw new Error('tar unavailable: ' + res.error.message);
|
||||
if (res.status !== 0) throw new Error('tar exited ' + res.status);
|
||||
}
|
||||
|
||||
function rmrf(p) {
|
||||
try { fs.rmSync(p, { recursive: true, force: true }); } catch (e) { /* best effort */ }
|
||||
}
|
||||
|
||||
// Drop sibling bundles for OTHER versions of this same platform target, keeping
|
||||
// only keepDir. The self-heal cache otherwise accumulates a full ~50 MB bundle
|
||||
// per version forever (issue #1074). Best-effort: a locked/busy dir (a
|
||||
// concurrent run still mapping an older node.exe on Windows) just stays — rmrf
|
||||
// already swallows its own errors, and the readdir is guarded — so cleanup can
|
||||
// never break a working command. Only this target's "<target>-<version>" dirs
|
||||
// are touched; other platforms' bundles and the ".dl-*" staging dirs are left
|
||||
// alone.
|
||||
function pruneOldBundles(bundlesDir, keepDir) {
|
||||
var keep = path.basename(keepDir);
|
||||
try {
|
||||
var names = fs.readdirSync(bundlesDir);
|
||||
for (var i = 0; i < names.length; i++) {
|
||||
var name = names[i];
|
||||
if (name === keep) continue;
|
||||
if (name.indexOf(target + '-') !== 0) continue;
|
||||
rmrf(path.join(bundlesDir, name));
|
||||
}
|
||||
} catch (e) { /* best effort — never break a working run over cleanup */ }
|
||||
}
|
||||
|
||||
function fail(reason) {
|
||||
process.stderr.write(
|
||||
'codegraph: no prebuilt bundle for ' + target + '.\n' +
|
||||
(reason ? 'codegraph: ' + reason + '\n' : '') +
|
||||
'Expected the optional package ' + pkg + ' to be installed.\n' +
|
||||
'A registry mirror (e.g. npmmirror/cnpm) that did not mirror the per-platform\n' +
|
||||
'package is the usual cause. Fixes:\n' +
|
||||
' - install from the official registry:\n' +
|
||||
' npm i -g @colbymchenry/codegraph --registry=https://registry.npmjs.org\n' +
|
||||
' - or use the standalone installer (no Node required):\n' +
|
||||
' curl -fsSL https://raw.githubusercontent.com/' + REPO + '/main/install.sh | sh\n'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Assemble the npm thin-installer packages from built bundles (esbuild pattern).
|
||||
#
|
||||
# Produces, under release/npm/:
|
||||
# codegraph-<target>/ one per built bundle — the vendored Node + app, tagged
|
||||
# with os/cpu so npm installs only the matching one.
|
||||
# main/ the @colbymchenry/codegraph shim package: a tiny bin
|
||||
# that execs the matching platform bundle, with every
|
||||
# platform package in optionalDependencies.
|
||||
#
|
||||
# The release pipeline then `npm publish`es each dir. This does NOT touch the
|
||||
# repo's package.json — the dev/from-source path keeps working; the *published*
|
||||
# main package's shape is generated here.
|
||||
#
|
||||
# Prereq: run build-bundle.sh for each target first (release/codegraph-*.tar.gz).
|
||||
# Usage: scripts/pack-npm.sh [version] (default: version from package.json)
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
VERSION="${1:-$(node -p "require('$ROOT/package.json').version")}"
|
||||
SCOPE="@colbymchenry"
|
||||
REL="$ROOT/release"
|
||||
NPM="$REL/npm"
|
||||
|
||||
rm -rf "$NPM"
|
||||
mkdir -p "$NPM/main"
|
||||
|
||||
shopt -s nullglob
|
||||
archives=("$REL"/codegraph-*.tar.gz "$REL"/codegraph-*.zip)
|
||||
[ ${#archives[@]} -gt 0 ] || { echo "[pack-npm] no bundles in $REL — run build-bundle.sh first" >&2; exit 1; }
|
||||
|
||||
targets=()
|
||||
for archive in "${archives[@]}"; do
|
||||
fname="$(basename "$archive")"
|
||||
case "$fname" in
|
||||
*.tar.gz) base="${fname%.tar.gz}" ;; # codegraph-<target>
|
||||
*.zip) base="${fname%.zip}" ;;
|
||||
esac
|
||||
target="${base#codegraph-}" # <target>, e.g. darwin-arm64 / win32-x64
|
||||
os="${target%-*}" # darwin | linux | win32
|
||||
arch="${target##*-}" # arm64 | x64
|
||||
pkgdir="$NPM/$base"
|
||||
mkdir -p "$pkgdir"
|
||||
case "$fname" in
|
||||
*.zip)
|
||||
tmpx="$(mktemp -d)"
|
||||
unzip -q "$archive" -d "$tmpx"
|
||||
mv "$tmpx/codegraph-${target}"/* "$pkgdir"/
|
||||
rm -rf "$tmpx"
|
||||
nodefile="node.exe"
|
||||
;;
|
||||
*)
|
||||
tar -xzf "$archive" -C "$pkgdir" --strip-components=1
|
||||
nodefile="node"
|
||||
;;
|
||||
esac
|
||||
VERSION="$VERSION" SCOPE="$SCOPE" TARGET="$target" OSV="$os" ARCHV="$arch" NODEFILE="$nodefile" \
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
fs.writeFileSync(process.argv[1], JSON.stringify({
|
||||
name: `${process.env.SCOPE}/codegraph-${process.env.TARGET}`,
|
||||
version: process.env.VERSION,
|
||||
description: `CodeGraph self-contained bundle for ${process.env.TARGET}`,
|
||||
os: [process.env.OSV], cpu: [process.env.ARCHV],
|
||||
files: [process.env.NODEFILE, "lib", "bin"],
|
||||
license: "MIT"
|
||||
}, null, 2) + "\n");
|
||||
' "$pkgdir/package.json"
|
||||
targets+=("$target")
|
||||
echo "[pack-npm] ${SCOPE}/codegraph-${target}@${VERSION}"
|
||||
done
|
||||
|
||||
# Main shim package.
|
||||
# npm-shim.js CLI/MCP launcher (execs the bundled Node) — the `bin`.
|
||||
# npm-sdk.js programmatic/embedded entry (#354): re-exports the installed
|
||||
# platform bundle's compiled library — the `main`.
|
||||
# dist/ the .d.ts tree only (types). The runtime .js stays in the
|
||||
# per-platform bundle so its deps aren't duplicated here.
|
||||
cp "$ROOT/scripts/npm-shim.js" "$NPM/main/npm-shim.js"
|
||||
cp "$ROOT/scripts/npm-sdk.js" "$NPM/main/npm-sdk.js"
|
||||
[ -f "$ROOT/README.md" ] && cp "$ROOT/README.md" "$NPM/main/README.md"
|
||||
|
||||
# Ship the type declarations so `types`/`exports.types` resolve. Built from this
|
||||
# same release, so they can't skew from the runtime npm-sdk.js re-exports.
|
||||
[ -f "$ROOT/dist/index.d.ts" ] || ( echo "[pack-npm] building dist for .d.ts" >&2 && cd "$ROOT" && npm run build >/dev/null )
|
||||
ROOT="$ROOT" DEST="$NPM/main" node -e '
|
||||
const fs=require("fs"), path=require("path");
|
||||
const src=path.join(process.env.ROOT,"dist"), dest=path.join(process.env.DEST,"dist");
|
||||
fs.cpSync(src, dest, { recursive:true, filter(s){
|
||||
try { return fs.statSync(s).isDirectory() || s.endsWith(".d.ts"); } catch (e) { return false; }
|
||||
}});
|
||||
'
|
||||
|
||||
VERSION="$VERSION" SCOPE="$SCOPE" TARGETS="${targets[*]}" \
|
||||
node -e '
|
||||
const fs=require("fs");
|
||||
const opt={};
|
||||
for (const t of process.env.TARGETS.split(/\s+/).filter(Boolean))
|
||||
opt[`${process.env.SCOPE}/codegraph-${t}`]=process.env.VERSION;
|
||||
fs.writeFileSync(process.argv[1], JSON.stringify({
|
||||
name: `${process.env.SCOPE}/codegraph`,
|
||||
version: process.env.VERSION,
|
||||
description: "Local-first code intelligence for AI agents (MCP). Self-contained — bundles its own runtime.",
|
||||
bin: { codegraph: "npm-shim.js" },
|
||||
main: "npm-sdk.js",
|
||||
types: "dist/index.d.ts",
|
||||
exports: {
|
||||
".": { types: "./dist/index.d.ts", default: "./npm-sdk.js" },
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
optionalDependencies: opt,
|
||||
files: ["npm-shim.js","npm-sdk.js","dist","README.md"],
|
||||
license: "MIT"
|
||||
}, null, 2) + "\n");
|
||||
' "$NPM/main/package.json"
|
||||
|
||||
echo "[pack-npm] ${SCOPE}/codegraph@${VERSION} (${#targets[@]} platform packages in optionalDependencies)"
|
||||
echo "[pack-npm] output: $NPM"
|
||||
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Promote `## [Unreleased]` content into `## [<version>]` in CHANGELOG.md
|
||||
* so the release.yml workflow's `extract-release-notes.mjs <version>` call
|
||||
* picks up everything that landed since the last release.
|
||||
*
|
||||
* **Why this exists:** the release workflow used to do a literal
|
||||
* `extract-release-notes.mjs <version>` lookup with an `[Unreleased]`
|
||||
* fallback. The fallback only triggers if the `[<version>]` block
|
||||
* doesn't exist at all — and in practice maintainers sometimes had a
|
||||
* sparse `[<version>]` block pre-populated (e.g. one early fix
|
||||
* documented before the rest of the work landed). The workflow then
|
||||
* extracted that sparse block, ignoring the much larger `[Unreleased]`
|
||||
* section above it — so the published release notes were missing most
|
||||
* of what shipped. See v0.9.5 for the canonical post-mortem.
|
||||
*
|
||||
* **What it does**, idempotently:
|
||||
*
|
||||
* Case A — `[<version>]` does not exist yet:
|
||||
* Rename the `[Unreleased]` header to `[<version>] - <YYYY-MM-DD>`
|
||||
* and add a fresh empty `## [Unreleased]` block above it. This is
|
||||
* the common case.
|
||||
*
|
||||
* Case B — `[<version>]` exists AND `[Unreleased]` has content:
|
||||
* Merge `[Unreleased]`'s sub-sections (### Added / ### Fixed /
|
||||
* ### Changed / ### Removed / ### Deprecated / ### Security) into
|
||||
* the corresponding sub-sections of `[<version>]`. Unmatched
|
||||
* sub-sections are appended to `[<version>]`. The `[Unreleased]`
|
||||
* block is then emptied.
|
||||
*
|
||||
* Case C — `[Unreleased]` has no content:
|
||||
* No-op. Exit 0. Re-runs of the workflow are safe.
|
||||
*
|
||||
* **Where the date comes from:** for Case A, `<YYYY-MM-DD>` is the
|
||||
* UTC date at run time. Matches the existing CHANGELOG convention.
|
||||
*
|
||||
* **Usage:**
|
||||
*
|
||||
* node scripts/prepare-release.mjs # reads version from package.json
|
||||
* node scripts/prepare-release.mjs 1.2.3 # explicit version
|
||||
*
|
||||
* **Output:**
|
||||
*
|
||||
* Writes CHANGELOG.md in place. Prints a summary line to stdout
|
||||
* like `prepare-release: 0.9.5 — promoted 6 Unreleased entries`.
|
||||
* Exits non-zero on parse failures.
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const CHANGELOG_PATH = resolve(process.cwd(), 'CHANGELOG.md');
|
||||
|
||||
function readPackageVersion() {
|
||||
const pkg = JSON.parse(readFileSync(resolve(process.cwd(), 'package.json'), 'utf8'));
|
||||
if (!pkg.version) throw new Error('package.json has no "version" field');
|
||||
return pkg.version;
|
||||
}
|
||||
|
||||
function todayUtcIsoDate() {
|
||||
// YYYY-MM-DD in UTC. Matches the CHANGELOG's existing convention
|
||||
// (the existing dated entries don't disclose a timezone, but UTC is
|
||||
// stable across runners and is what the workflow's runner produces
|
||||
// by default anyway).
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Split the CHANGELOG into a header preface + an ordered list of
|
||||
* version blocks `{ header, body[] }`, preserving line content
|
||||
* verbatim so we can re-join without surprises.
|
||||
*/
|
||||
function parseChangelog(text) {
|
||||
const lines = text.split('\n');
|
||||
const versionHeaderRe = /^## \[([^\]]+)\](?:\s+-\s+(.+))?\s*$/;
|
||||
const preface = [];
|
||||
const blocks = []; // { header: string, name: string, body: string[] }
|
||||
let cur = null;
|
||||
for (const line of lines) {
|
||||
const m = line.match(versionHeaderRe);
|
||||
if (m) {
|
||||
if (cur) blocks.push(cur);
|
||||
cur = { header: line, name: m[1], date: m[2] ?? null, body: [] };
|
||||
} else if (cur) {
|
||||
cur.body.push(line);
|
||||
} else {
|
||||
preface.push(line);
|
||||
}
|
||||
}
|
||||
if (cur) blocks.push(cur);
|
||||
return { preface, blocks };
|
||||
}
|
||||
|
||||
function joinChangelog({ preface, blocks }) {
|
||||
const parts = [preface.join('\n')];
|
||||
for (const b of blocks) {
|
||||
// Reconstruct: header + body. The block body INCLUDES the blank
|
||||
// line after the header (it was captured verbatim).
|
||||
parts.push([b.header, ...b.body].join('\n'));
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a block body into ordered sub-sections keyed by their
|
||||
* `### Heading`. Lines before the first `### Heading` go in
|
||||
* `leading`. Preserves the original (line-array) body inside each
|
||||
* sub-section so we can splice cleanly when merging.
|
||||
*/
|
||||
function splitSubsections(body) {
|
||||
const subsectionRe = /^### (\w+)\s*$/;
|
||||
const leading = [];
|
||||
const subs = []; // { heading: 'Added' | 'Fixed' | …, headerLine: string, body: string[] }
|
||||
let cur = null;
|
||||
for (const line of body) {
|
||||
const m = line.match(subsectionRe);
|
||||
if (m) {
|
||||
if (cur) subs.push(cur);
|
||||
cur = { heading: m[1], headerLine: line, body: [] };
|
||||
} else if (cur) {
|
||||
cur.body.push(line);
|
||||
} else {
|
||||
leading.push(line);
|
||||
}
|
||||
}
|
||||
if (cur) subs.push(cur);
|
||||
return { leading, subs };
|
||||
}
|
||||
|
||||
function rebuildBody({ leading, subs }) {
|
||||
const parts = [];
|
||||
if (leading.length) parts.push(leading.join('\n'));
|
||||
for (const s of subs) {
|
||||
parts.push([s.headerLine, ...s.body].join('\n'));
|
||||
}
|
||||
return parts.join('\n').split('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the block has any meaningful entries (a bullet line
|
||||
* starting with `-`, `*`, or a digit) — vs. being empty / just
|
||||
* whitespace / just sub-section headers with nothing under them.
|
||||
*/
|
||||
function blockHasContent(body) {
|
||||
for (const line of body) {
|
||||
if (/^\s*([-*]|\d+\.)\s+/.test(line)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim trailing blank lines from an array of lines, then return.
|
||||
* Keeps the output tidy when merging.
|
||||
*/
|
||||
function trimTrailingBlank(arr) {
|
||||
let i = arr.length;
|
||||
while (i > 0 && /^\s*$/.test(arr[i - 1])) i--;
|
||||
return arr.slice(0, i);
|
||||
}
|
||||
|
||||
function main() {
|
||||
const versionArg = process.argv[2];
|
||||
const version = versionArg || readPackageVersion();
|
||||
|
||||
const text = readFileSync(CHANGELOG_PATH, 'utf8');
|
||||
const parsed = parseChangelog(text);
|
||||
|
||||
const unrelIdx = parsed.blocks.findIndex((b) => b.name === 'Unreleased');
|
||||
const verIdx = parsed.blocks.findIndex((b) => b.name === version);
|
||||
|
||||
if (unrelIdx === -1) {
|
||||
console.log(`prepare-release: no [Unreleased] block — nothing to do`);
|
||||
return;
|
||||
}
|
||||
|
||||
const unrel = parsed.blocks[unrelIdx];
|
||||
if (!blockHasContent(unrel.body)) {
|
||||
console.log(`prepare-release: [Unreleased] is empty — nothing to do`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (verIdx === -1) {
|
||||
// Case A — promote Unreleased → [version].
|
||||
const today = todayUtcIsoDate();
|
||||
const promoted = {
|
||||
header: `## [${version}] - ${today}`,
|
||||
name: version,
|
||||
date: today,
|
||||
body: trimTrailingBlank(unrel.body).concat(['']), // single trailing blank
|
||||
};
|
||||
const emptied = {
|
||||
header: `## [Unreleased]`,
|
||||
name: 'Unreleased',
|
||||
date: null,
|
||||
body: ['', ''], // two blank lines for the next round of entries
|
||||
};
|
||||
parsed.blocks.splice(unrelIdx, 1, emptied, promoted);
|
||||
const next = joinChangelog(parsed);
|
||||
writeFileSync(CHANGELOG_PATH, appendLinkRef(next, version));
|
||||
console.log(`prepare-release: ${version} — renamed [Unreleased] to [${version}] - ${today}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Case B — merge Unreleased sub-sections into the existing
|
||||
// [version] sub-sections. New sub-section headings encountered in
|
||||
// Unreleased that don't exist in [version] get appended.
|
||||
const ver = parsed.blocks[verIdx];
|
||||
const unrelSubs = splitSubsections(unrel.body);
|
||||
const verSubs = splitSubsections(ver.body);
|
||||
|
||||
let merged = 0;
|
||||
for (const us of unrelSubs.subs) {
|
||||
const target = verSubs.subs.find((s) => s.heading === us.heading);
|
||||
const usBody = trimTrailingBlank(us.body);
|
||||
if (usBody.length === 0) continue;
|
||||
if (target) {
|
||||
// Append Unreleased's entries to the end of the version's matching
|
||||
// sub-section, keeping their original ordering. Insert a separating
|
||||
// blank line if the existing sub-section doesn't already end in one.
|
||||
const existing = trimTrailingBlank(target.body);
|
||||
const sep = existing.length && !/^\s*$/.test(existing[existing.length - 1]) ? [''] : [];
|
||||
target.body = existing.concat(sep, usBody, ['']);
|
||||
} else {
|
||||
// Append the whole sub-section to the end.
|
||||
verSubs.subs.push({
|
||||
heading: us.heading,
|
||||
headerLine: us.headerLine,
|
||||
body: usBody.concat(['']),
|
||||
});
|
||||
}
|
||||
merged += usBody.filter((l) => /^\s*([-*]|\d+\.)\s+/.test(l)).length;
|
||||
}
|
||||
|
||||
ver.body = rebuildBody(verSubs);
|
||||
// Empty out Unreleased.
|
||||
unrel.body = ['', ''];
|
||||
|
||||
const merged_text = joinChangelog(parsed);
|
||||
writeFileSync(CHANGELOG_PATH, appendLinkRef(merged_text, version));
|
||||
console.log(`prepare-release: ${version} — merged ${merged} Unreleased entries into existing [${version}] block`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a `[X.Y.Z]: https://github.com/colbymchenry/codegraph/releases/tag/vX.Y.Z`
|
||||
* link reference at the end of the file IF one doesn't already exist. The
|
||||
* link ref is what makes `## [X.Y.Z]` heading text auto-link to its tag in
|
||||
* GitHub's renderer; without it the heading still renders, just unlinked.
|
||||
*
|
||||
* Idempotent. The existing CHANGELOG mixes link refs scattered through the
|
||||
* file and a sorted block at the bottom — we just append at the very end,
|
||||
* which CommonMark accepts regardless.
|
||||
*/
|
||||
function appendLinkRef(text, version) {
|
||||
const refLine = `[${version}]: https://github.com/colbymchenry/codegraph/releases/tag/v${version}`;
|
||||
// Already there? Look for a line that EQUALS this (anywhere in the file)
|
||||
// to keep idempotency robust against the scattered-vs-block layout.
|
||||
const lines = text.split('\n');
|
||||
if (lines.some((l) => l.trim() === refLine)) return text;
|
||||
// Append, separated by a blank line from the prior content. Preserve a
|
||||
// single trailing newline at EOF.
|
||||
const trailingNewline = text.endsWith('\n') ? '' : '\n';
|
||||
return text + trailingNewline + refLine + '\n';
|
||||
}
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (err) {
|
||||
console.error(`prepare-release: ${err?.message ?? err}`);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user