chore: import upstream snapshot with attribution

__tests__/evaluation/runner.ts

@@ -0,0 +1,123 @@
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { CodeGraph } from '../../src/index.js';
+import { scoreSearchNodes, scoreFindRelevantContext } from './scoring.js';
+import { testCases } from './test-cases.js';
+import type { EvalReport, EvalResult } from './types.js';
+
+const codebasePath = process.env.EVAL_CODEBASE || process.argv[2];
+if (!codebasePath) {
+  console.error('Usage: EVAL_CODEBASE=/path/to/codebase npx tsx __tests__/evaluation/runner.ts');
+  console.error('   or: npx tsx __tests__/evaluation/runner.ts /path/to/codebase');
+  process.exit(1);
+}
+
+const resolvedPath = path.resolve(codebasePath);
+if (!fs.existsSync(path.join(resolvedPath, '.codegraph', 'codegraph.db'))) {
+  console.error(`No .codegraph/codegraph.db found at ${resolvedPath}`);
+  process.exit(1);
+}
+
+let codegraphSha = 'unknown';
+try {
+  codegraphSha = execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
+} catch {}
+
+console.log(`\nCodeGraph Eval — ${path.basename(resolvedPath)}`);
+console.log(`Codebase: ${resolvedPath}`);
+console.log(`Commit:   ${codegraphSha}`);
+console.log(`Cases:    ${testCases.length}`);
+console.log('');
+
+async function run() {
+  const cg = CodeGraph.openSync(resolvedPath);
+  const results: EvalResult[] = [];
+
+  for (const tc of testCases) {
+    const start = performance.now();
+
+    if (tc.api === 'searchNodes') {
+      const searchResults = cg.searchNodes(tc.query, {
+        limit: 10,
+        kinds: tc.kinds,
+        ...(tc.options as Record<string, unknown>),
+      });
+      const latency = performance.now() - start;
+      const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency);
+      results.push(result);
+    } else {
+      const subgraph = await cg.findRelevantContext(tc.query, {
+        searchLimit: 8,
+        traversalDepth: 3,
+        maxNodes: 80,
+        minScore: 0.2,
+        ...(tc.options as Record<string, unknown>),
+      });
+      const latency = performance.now() - start;
+      const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency);
+      results.push(result);
+    }
+  }
+
+  cg.close();
+
+  // Print results table
+  const maxIdLen = Math.max(...results.map((r) => r.caseId.length));
+
+  for (const r of results) {
+    const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
+    const id = r.caseId.padEnd(maxIdLen);
+    const recall = `recall=${r.recall.toFixed(2)}`;
+    const extra =
+      r.edgeDensity !== undefined
+        ? `density=${r.edgeDensity.toFixed(2)}`
+        : `mrr=${r.mrr.toFixed(2)}`;
+    const latency = `${Math.round(r.latencyMs)}ms`;
+
+    console.log(`  ${id}  ${status}  ${recall}  ${extra}  ${latency}`);
+
+    if (r.missedSymbols.length > 0) {
+      console.log(`  ${' '.repeat(maxIdLen)}        missed: ${r.missedSymbols.join(', ')}`);
+    }
+  }
+
+  // Summary
+  const passed = results.filter((r) => r.pass).length;
+  const failed = results.length - passed;
+  const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length;
+  const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-'));
+  const meanMRR =
+    mrrResults.length > 0 ? mrrResults.reduce((s, r) => s + r.mrr, 0) / mrrResults.length : 0;
+
+  console.log('');
+  const summaryColor = failed === 0 ? '\x1b[32m' : '\x1b[33m';
+  console.log(
+    `${summaryColor}SUMMARY: ${passed}/${results.length} passed | recall=${meanRecall.toFixed(2)} | mrr=${meanMRR.toFixed(2)}\x1b[0m`
+  );
+
+  // Save JSON report
+  const report: EvalReport = {
+    timestamp: new Date().toISOString(),
+    codebasePath: resolvedPath,
+    codegraphSha,
+    summary: { total: results.length, passed, failed, meanRecall, meanMRR },
+    results,
+  };
+
+  const resultsDir = path.join(__dirname, 'results');
+  fs.mkdirSync(resultsDir, { recursive: true });
+  const reportFile = path.join(
+    resultsDir,
+    `${new Date().toISOString().replace(/[:.]/g, '-')}.json`
+  );
+  fs.writeFileSync(reportFile, JSON.stringify(report, null, 2));
+  console.log(`\nReport saved: ${reportFile}`);
+
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+run().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

__tests__/evaluation/scoring.ts

@@ -0,0 +1,82 @@
+import type { EvalResult } from './types.js';
+
+export const PASS_THRESHOLD = 0.5;
+
+export function scoreSearchNodes(
+  caseId: string,
+  expectedSymbols: string[],
+  results: Array<{ node: { name: string }; score: number }>,
+  latencyMs: number
+): EvalResult {
+  const expectedLower = expectedSymbols.map((s) => s.toLowerCase());
+  const resultNames = results.map((r) => r.node.name.toLowerCase());
+
+  const found: string[] = [];
+  const missed: string[] = [];
+  let firstRank = 0;
+
+  for (let i = 0; i < expectedLower.length; i++) {
+    const idx = resultNames.indexOf(expectedLower[i]);
+    if (idx !== -1) {
+      found.push(expectedSymbols[i]);
+      if (firstRank === 0) firstRank = idx + 1;
+    } else {
+      missed.push(expectedSymbols[i]);
+    }
+  }
+
+  const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0;
+  const mrr = firstRank > 0 ? 1 / firstRank : 0;
+
+  return {
+    caseId,
+    pass: recall >= PASS_THRESHOLD,
+    recall,
+    mrr,
+    foundSymbols: found,
+    missedSymbols: missed,
+    latencyMs,
+  };
+}
+
+export function scoreFindRelevantContext(
+  caseId: string,
+  expectedSymbols: string[],
+  subgraph: { nodes: Map<string, { name: string }>; edges: unknown[]; roots: string[] },
+  latencyMs: number
+): EvalResult {
+  const expectedLower = new Set(expectedSymbols.map((s) => s.toLowerCase()));
+  const nodeNames = new Set<string>();
+  for (const node of subgraph.nodes.values()) {
+    nodeNames.add(node.name.toLowerCase());
+  }
+
+  const found: string[] = [];
+  const missed: string[] = [];
+
+  for (const sym of expectedSymbols) {
+    if (nodeNames.has(sym.toLowerCase())) {
+      found.push(sym);
+    } else {
+      missed.push(sym);
+    }
+  }
+
+  const recall = expectedSymbols.length > 0 ? found.length / expectedSymbols.length : 0;
+  const nodeCount = subgraph.nodes.size;
+  const edgeCount = subgraph.edges.length;
+  const edgeDensity = nodeCount > 0 ? edgeCount / nodeCount : 0;
+
+  return {
+    caseId,
+    pass: recall >= PASS_THRESHOLD,
+    recall,
+    mrr: 0,
+    foundSymbols: found,
+    missedSymbols: missed,
+    nodeCount,
+    edgeCount,
+    edgeDensity,
+    latencyMs,
+  };
+}

__tests__/evaluation/test-cases.ts

@@ -0,0 +1,93 @@
+import type { EvalTestCase } from './types.js';
+
+export const testCases: EvalTestCase[] = [
+  // === searchNodes: Symbol Lookup Precision ===
+
+  {
+    id: 'search-class-exact',
+    query: 'TransportService',
+    api: 'searchNodes',
+    expectedSymbols: ['TransportService'],
+    kinds: ['class'],
+  },
+  {
+    id: 'search-method-qualified',
+    query: 'TransportService sendRequest',
+    api: 'searchNodes',
+    expectedSymbols: ['sendRequest'],
+    kinds: ['method'],
+  },
+  {
+    id: 'search-interface',
+    query: 'ActionListener',
+    api: 'searchNodes',
+    expectedSymbols: ['ActionListener'],
+    kinds: ['interface'],
+  },
+  {
+    id: 'search-enum',
+    query: 'RestStatus',
+    api: 'searchNodes',
+    expectedSymbols: ['RestStatus'],
+    kinds: ['enum'],
+  },
+  {
+    id: 'search-exception',
+    query: 'SearchPhaseExecutionException',
+    api: 'searchNodes',
+    expectedSymbols: ['SearchPhaseExecutionException'],
+    kinds: ['class'],
+  },
+  {
+    id: 'search-nested-class',
+    query: 'Engine Index',
+    api: 'searchNodes',
+    expectedSymbols: ['Index'],
+    kinds: ['class'],
+  },
+
+  // === findRelevantContext: Exploration Quality ===
+
+  {
+    id: 'explore-rest-layer',
+    query: 'How does the REST layer handle HTTP requests?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['RestController', 'RestHandler', 'BaseRestHandler', 'RestRequest'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+  {
+    id: 'explore-search-execution',
+    query: 'How does search execution work from request to shard?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['ShardSearchRequest', 'SearchShardsRequest', 'SearchShardsGroup'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+  {
+    id: 'explore-bulk-indexing',
+    query: 'How does bulk indexing work?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['TransportBulkAction', 'BulkRequest', 'BulkResponse'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+  {
+    id: 'explore-shard-allocation',
+    query: 'How does shard rebalancing and allocation work?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['AllocationService', 'BalancedShardsAllocator'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+  {
+    id: 'explore-transport-search',
+    query: 'How does TransportService connect to SearchTransportService?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['TransportService', 'SearchTransportService'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+  {
+    id: 'explore-engine-implementations',
+    query: 'What are the Engine implementations for indexing?',
+    api: 'findRelevantContext',
+    expectedSymbols: ['InternalEngine', 'ReadOnlyEngine', 'Engine'],
+    options: { searchLimit: 8, traversalDepth: 3, maxNodes: 80, minScore: 0.2 },
+  },
+];

__tests__/evaluation/types.ts

@@ -0,0 +1,37 @@
+import type { NodeKind } from '../../src/types.js';
+
+export interface EvalTestCase {
+  id: string;
+  query: string;
+  api: 'searchNodes' | 'findRelevantContext';
+  expectedSymbols: string[];
+  kinds?: NodeKind[];
+  options?: Record<string, unknown>;
+}
+
+export interface EvalResult {
+  caseId: string;
+  pass: boolean;
+  recall: number;
+  mrr: number;
+  foundSymbols: string[];
+  missedSymbols: string[];
+  nodeCount?: number;
+  edgeCount?: number;
+  edgeDensity?: number;
+  latencyMs: number;
+}
+
+export interface EvalReport {
+  timestamp: string;
+  codebasePath: string;
+  codegraphSha: string;
+  summary: {
+    total: number;
+    passed: number;
+    failed: number;
+    meanRecall: number;
+    meanMRR: number;
+  };
+  results: EvalResult[];
+}