wehub-resource-sync
120 lines
7.1 KiB
JavaScript
Executable File
120 lines
7.1 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
||
// probe-sweep — direct MCP test across N repos × N tools, no claude needed.
|
||
//
|
||
// Measures response characteristics (size, sections present, signals fired)
|
||
// for each (repo, query) pair against the built dist/. Sub-second per probe;
|
||
// the full sweep below runs in ~10-30s vs hours for a real claude audit.
|
||
//
|
||
// Use this to iterate on backend changes rapidly: change tools.ts /
|
||
// context-builder, npm run build, re-run probe-sweep, compare. Once a
|
||
// change looks good on probe metrics, run a focused claude audit for the
|
||
// few repos that matter to confirm end-to-end cost behavior.
|
||
//
|
||
// Usage: node scripts/agent-eval/probe-sweep.mjs [--tool=context|explore|trace] [--repos=a,b,c]
|
||
import { pathToFileURL } from 'node:url';
|
||
import { resolve } from 'node:path';
|
||
|
||
const args = Object.fromEntries(
|
||
process.argv.slice(2).map(a => a.startsWith('--') ? a.slice(2).split('=') : [a, true])
|
||
);
|
||
const TOOL = args.tool ?? 'context';
|
||
|
||
const load = (rel) => import(pathToFileURL(resolve(rel)).href);
|
||
const idx = await load('dist/index.js');
|
||
const tools = await load('dist/mcp/tools.js');
|
||
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
||
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
||
|
||
// Each entry: repo, query, optional 2nd arg for trace (from, to).
|
||
// The query is the same prompt used in the real claude audits, so probe
|
||
// output is directly comparable to the agent's would-be input.
|
||
const SWEEP = [
|
||
// Small realworld template repos (the loss cases from the cross-language sweep)
|
||
{ id: 'gin-rw', repo: '/tmp/codegraph-corpus/gin-realworld', q: 'How does this Gin app route a request through its middleware chain to a handler?' },
|
||
{ id: 'go-mux', repo: '/tmp/codegraph-corpus/go-mux', q: 'How does this gorilla/mux app route a request to its handler?' },
|
||
{ id: 'fastapi-rw', repo: '/tmp/codegraph-corpus/fastapi-realworld', q: 'How does FastAPI route a request through its dependencies to a handler?' },
|
||
{ id: 'spring-pc', repo: '/tmp/codegraph-corpus/spring-petclinic', q: 'How does Spring route an HTTP request to a controller method?' },
|
||
{ id: 'axum-rw', repo: '/tmp/codegraph-corpus/rust-axum-realworld', q: 'How does Axum route a request to its handler in this app?' },
|
||
{ id: 'express-rw', repo: '/tmp/codegraph-corpus/express-realworld', q: 'How does this Express app route a request through middleware to a handler?' },
|
||
{ id: 'kotlin-pc', repo: '/tmp/codegraph-corpus/kotlin-petclinic', q: 'How does the Kotlin Spring app route an HTTP request to its handler?' },
|
||
{ id: 'flask-mb', repo: '/tmp/codegraph-corpus/flask-microblog', q: 'How does this Flask app route a request to a view function?' },
|
||
{ id: 'vapor-tpl', repo: '/tmp/codegraph-corpus/vapor-template', q: 'How does Vapor route an HTTP request to its handler?' },
|
||
{ id: 'cpp-leveldb', repo: '/tmp/codegraph-corpus/cpp-leveldb', q: 'How does LevelDB handle a Put operation through to disk?' },
|
||
{ id: 'lualine', repo: '/tmp/codegraph-corpus/lualine.nvim', q: 'How does lualine assemble and render the statusline?' },
|
||
{ id: 'drupal-admin', repo: '/tmp/codegraph-corpus/drupal-admintoolbar', q: 'How does the Drupal admin toolbar module render its toolbar?' },
|
||
{ id: 'svelte-rw', repo: '/tmp/codegraph-corpus/svelte-realworld', q: 'How does this SvelteKit app route a request to a handler?' },
|
||
{ id: 'react-rw', repo: '/tmp/codegraph-corpus/react-realworld', q: 'How does this React app fetch and display articles?' },
|
||
{ id: 'rails-rw', repo: '/tmp/codegraph-corpus/rails-realworld', q: 'How does Rails route a request to a controller action?' },
|
||
{ id: 'flask-rest', repo: '/tmp/codegraph-corpus/flask-restful-realworld', q: 'How does Flask-RESTful route a request to a resource method?' },
|
||
{ id: 'laravel-rw', repo: '/tmp/codegraph-corpus/laravel-realworld', q: 'How does Laravel route a request to the controller method?' },
|
||
{ id: 'aspnet-rw', repo: '/tmp/codegraph-corpus/aspnet-realworld', q: 'How does ASP.NET route a request to the controller action?' },
|
||
// The iter7 wins/ties (to make sure we don't regress)
|
||
{ id: 'cobra', repo: '/tmp/codegraph-corpus/cobra', q: 'How does cobra parse commands and flags?' },
|
||
{ id: 'sinatra', repo: '/tmp/codegraph-corpus/sinatra', q: 'How does sinatra route a request to its handler?' },
|
||
{ id: 'slim', repo: '/tmp/codegraph-corpus/slim', q: 'How does slim route a request and apply middleware?' },
|
||
];
|
||
|
||
// Detect signals in response text — these are the levers we've added that
|
||
// otherwise only show up via "agent ran X more tool calls" downstream.
|
||
const detect = (text) => ({
|
||
hasEntryPoints: /^### Entry Points/m.test(text),
|
||
hasRelatedSymbols: /^### Related Symbols/m.test(text),
|
||
hasFlowTrace: /^## Inline flow trace/m.test(text),
|
||
hasRouteManifest: /^## Routing manifest/m.test(text),
|
||
hasTopHandler: /^### Top handler file/m.test(text),
|
||
hasSmallRepoTail: /This project is small/.test(text),
|
||
});
|
||
|
||
const filterRepos = args.repos ? new Set(String(args.repos).split(',')) : null;
|
||
const subjects = SWEEP.filter(s => !filterRepos || filterRepos.has(s.id));
|
||
|
||
const t0 = Date.now();
|
||
const rows = [];
|
||
for (const s of subjects) {
|
||
try {
|
||
const cg = CodeGraph.openSync(s.repo);
|
||
const handler = new ToolHandler(cg);
|
||
const t1 = Date.now();
|
||
const res = await handler.execute('codegraph_' + TOOL,
|
||
TOOL === 'context' ? { task: s.q } :
|
||
TOOL === 'explore' ? { query: s.q } : { from: 'main', to: 'main' });
|
||
const text = res.content?.[0]?.text ?? '';
|
||
const signals = detect(text);
|
||
rows.push({
|
||
id: s.id,
|
||
ms: Date.now() - t1,
|
||
chars: text.length,
|
||
lines: text.split('\n').length,
|
||
...signals,
|
||
});
|
||
try { cg.close?.(); } catch {}
|
||
} catch (e) {
|
||
rows.push({ id: s.id, error: String(e).slice(0, 80) });
|
||
}
|
||
}
|
||
|
||
// Pretty-print as a compact table.
|
||
const fmt = (r) =>
|
||
r.error
|
||
? ` ${r.id.padEnd(13)} ERROR: ${r.error}`
|
||
: ` ${r.id.padEnd(13)} ${String(r.chars).padStart(6)}c ${String(r.lines).padStart(4)}L ${String(r.ms).padStart(4)}ms` +
|
||
` ${r.hasEntryPoints ? 'EP ' : ' '}` +
|
||
`${r.hasFlowTrace ? 'TRC ' : ' '}` +
|
||
`${r.hasRouteManifest ? 'MAN ' : ' '}` +
|
||
`${r.hasTopHandler ? 'HND ' : ' '}` +
|
||
`${r.hasSmallRepoTail ? 'TAIL' : ' '}`;
|
||
console.log(`=== probe-sweep tool=${TOOL} n=${subjects.length} (${Date.now() - t0}ms total) ===`);
|
||
console.log(' id chars lines ms signals');
|
||
console.log(' ' + '-'.repeat(56));
|
||
for (const r of rows) console.log(fmt(r));
|
||
|
||
// Sum + medians for the size pillar
|
||
const sizes = rows.filter(r => !r.error).map(r => r.chars);
|
||
sizes.sort((a, b) => a - b);
|
||
const median = sizes[Math.floor(sizes.length / 2)];
|
||
const sum = sizes.reduce((a, b) => a + b, 0);
|
||
console.log(` ${'-'.repeat(64)}`);
|
||
console.log(` median=${median}c total=${sum}c ` +
|
||
`manifest=${rows.filter(r => r.hasRouteManifest).length}/${rows.filter(r => !r.error).length} ` +
|
||
`top-handler=${rows.filter(r => r.hasTopHandler).length}/${rows.filter(r => !r.error).length}`);
|