wehub-resource-sync
124 lines
4.1 KiB
TypeScript
124 lines
4.1 KiB
TypeScript
import { execSync } from 'child_process';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { CodeGraph } from '../../src/index.js';
|
|
import { scoreSearchNodes, scoreFindRelevantContext } from './scoring.js';
|
|
import { testCases } from './test-cases.js';
|
|
import type { EvalReport, EvalResult } from './types.js';
|
|
|
|
const codebasePath = process.env.EVAL_CODEBASE || process.argv[2];
|
|
if (!codebasePath) {
|
|
console.error('Usage: EVAL_CODEBASE=/path/to/codebase npx tsx __tests__/evaluation/runner.ts');
|
|
console.error(' or: npx tsx __tests__/evaluation/runner.ts /path/to/codebase');
|
|
process.exit(1);
|
|
}
|
|
|
|
const resolvedPath = path.resolve(codebasePath);
|
|
if (!fs.existsSync(path.join(resolvedPath, '.codegraph', 'codegraph.db'))) {
|
|
console.error(`No .codegraph/codegraph.db found at ${resolvedPath}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
let codegraphSha = 'unknown';
|
|
try {
|
|
codegraphSha = execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
|
|
} catch {}
|
|
|
|
console.log(`\nCodeGraph Eval — ${path.basename(resolvedPath)}`);
|
|
console.log(`Codebase: ${resolvedPath}`);
|
|
console.log(`Commit: ${codegraphSha}`);
|
|
console.log(`Cases: ${testCases.length}`);
|
|
console.log('');
|
|
|
|
async function run() {
|
|
const cg = CodeGraph.openSync(resolvedPath);
|
|
const results: EvalResult[] = [];
|
|
|
|
for (const tc of testCases) {
|
|
const start = performance.now();
|
|
|
|
if (tc.api === 'searchNodes') {
|
|
const searchResults = cg.searchNodes(tc.query, {
|
|
limit: 10,
|
|
kinds: tc.kinds,
|
|
...(tc.options as Record<string, unknown>),
|
|
});
|
|
const latency = performance.now() - start;
|
|
const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency);
|
|
results.push(result);
|
|
} else {
|
|
const subgraph = await cg.findRelevantContext(tc.query, {
|
|
searchLimit: 8,
|
|
traversalDepth: 3,
|
|
maxNodes: 80,
|
|
minScore: 0.2,
|
|
...(tc.options as Record<string, unknown>),
|
|
});
|
|
const latency = performance.now() - start;
|
|
const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency);
|
|
results.push(result);
|
|
}
|
|
}
|
|
|
|
cg.close();
|
|
|
|
// Print results table
|
|
const maxIdLen = Math.max(...results.map((r) => r.caseId.length));
|
|
|
|
for (const r of results) {
|
|
const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
|
|
const id = r.caseId.padEnd(maxIdLen);
|
|
const recall = `recall=${r.recall.toFixed(2)}`;
|
|
const extra =
|
|
r.edgeDensity !== undefined
|
|
? `density=${r.edgeDensity.toFixed(2)}`
|
|
: `mrr=${r.mrr.toFixed(2)}`;
|
|
const latency = `${Math.round(r.latencyMs)}ms`;
|
|
|
|
console.log(` ${id} ${status} ${recall} ${extra} ${latency}`);
|
|
|
|
if (r.missedSymbols.length > 0) {
|
|
console.log(` ${' '.repeat(maxIdLen)} missed: ${r.missedSymbols.join(', ')}`);
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
const passed = results.filter((r) => r.pass).length;
|
|
const failed = results.length - passed;
|
|
const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length;
|
|
const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-'));
|
|
const meanMRR =
|
|
mrrResults.length > 0 ? mrrResults.reduce((s, r) => s + r.mrr, 0) / mrrResults.length : 0;
|
|
|
|
console.log('');
|
|
const summaryColor = failed === 0 ? '\x1b[32m' : '\x1b[33m';
|
|
console.log(
|
|
`${summaryColor}SUMMARY: ${passed}/${results.length} passed | recall=${meanRecall.toFixed(2)} | mrr=${meanMRR.toFixed(2)}\x1b[0m`
|
|
);
|
|
|
|
// Save JSON report
|
|
const report: EvalReport = {
|
|
timestamp: new Date().toISOString(),
|
|
codebasePath: resolvedPath,
|
|
codegraphSha,
|
|
summary: { total: results.length, passed, failed, meanRecall, meanMRR },
|
|
results,
|
|
};
|
|
|
|
const resultsDir = path.join(__dirname, 'results');
|
|
fs.mkdirSync(resultsDir, { recursive: true });
|
|
const reportFile = path.join(
|
|
resultsDir,
|
|
`${new Date().toISOString().replace(/[:.]/g, '-')}.json`
|
|
);
|
|
fs.writeFileSync(reportFile, JSON.stringify(report, null, 2));
|
|
console.log(`\nReport saved: ${reportFile}`);
|
|
|
|
process.exit(failed > 0 ? 1 : 0);
|
|
}
|
|
|
|
run().catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|