inside +

, so a bare `.header { border-bottom }` draws two + lines — put the rule on the outer

only and clear the inner div. */ +.header { + background: var(--cg-paper); + -webkit-backdrop-filter: none; + backdrop-filter: none; +} +header.header { + border-bottom: 1px solid var(--cg-rule); +} +.header .header { + border-bottom: 0; +} + +/* Sidebar: crisp right rule */ +#starlight__sidebar, +.sidebar-pane { + border-inline-end: 1px solid var(--cg-rule); + background: var(--cg-paper); +} + +/* Sidebar group labels — small caps, committed editorial direction */ +.sidebar-content details > summary, +.sidebar-content > ul > li > span, +.sidebar-content .large { + font-weight: 700; + letter-spacing: 0.07em; + text-transform: uppercase; + font-size: 0.72rem; + color: var(--cg-ink-2); +} + +/* Sidebar links */ +.sidebar-content a { + color: var(--cg-ink-2); +} +.sidebar-content a:hover { + background: var(--cg-paper-press); + color: var(--cg-ink); +} +.sidebar-content a[aria-current='page'], +.sidebar-content a[aria-current='page']:hover { + background: transparent; + color: var(--cg-ink); + font-weight: 700; + border-inline-start: 2px solid var(--cg-ink); +} + +/* Right "On this page" rail */ +starlight-toc a { + color: var(--cg-ink-3); +} +starlight-toc a[aria-current='true'] { + color: var(--cg-ink); + font-weight: 600; +} + +/* Prev / next pagination: flat bordered boxes */ +.pagination-links a { + border: 1px solid var(--cg-rule); + box-shadow: none; + background: var(--cg-paper); +} +.pagination-links a:hover { + background: var(--cg-paper-press); +} + +/* Inline code */ +.sl-markdown-content :not(pre) > code { + border: 1px solid var(--cg-rule-soft); + background: var(--cg-paper-2); + font-size: 0.875em; +} + +/* Cards / asides: square, hairline */ +.card, +.starlight-aside { + border: 1px solid var(--cg-rule); + box-shadow: none; +} + +/* Search trigger */ +button[data-open-modal] { + border: 1px solid var(--cg-rule); + background: var(--cg-paper); +} + +/* Content horizontal rules */ +.sl-markdown-content hr { + border: 0; + border-top: 1px solid var(--cg-rule); +} + +/* Links in prose */ +.sl-markdown-content a { + color: var(--cg-ink); + text-underline-offset: 3px; +} + +/* On wide screens Starlight right-aligns the content against the TOC + (margin-inline: auto 0), piling all the empty space on the left. Center the + content within its pane so it sits balanced between the sidebar and the TOC. */ +@media (min-width: 72rem) { + .main-pane { + --sl-content-margin-inline: auto; + } +} diff --git a/site/tsconfig.json b/site/tsconfig.json new file mode 100644 index 0000000..8bf91d3 --- /dev/null +++ b/site/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "astro/tsconfigs/strict", + "include": [".astro/types.d.ts", "**/*"], + "exclude": ["dist"] +} diff --git a/src/bin/codegraph.ts b/src/bin/codegraph.ts new file mode 100644 index 0000000..4d85e20 --- /dev/null +++ b/src/bin/codegraph.ts @@ -0,0 +1,2420 @@ +#!/usr/bin/env node +/** + * CodeGraph CLI + * + * Command-line interface for CodeGraph code intelligence. + * + * Usage: + * codegraph Run interactive installer (when no args) + * codegraph install Run interactive installer + * codegraph uninstall Remove CodeGraph from your agents + * codegraph init [path] Initialize CodeGraph in a project + * codegraph uninit [path] Remove CodeGraph from a project + * codegraph index [path] Index all files in the project + * codegraph sync [path] Sync changes since last index + * codegraph status [path] Show index status + * codegraph query Search for symbols + * codegraph files [options] Show project file structure + * codegraph context Build context for a task + * codegraph callers Find what calls a function/method + * codegraph callees Find what a function/method calls + * codegraph impact Analyze what code is affected by changing a symbol + * codegraph affected [files] Find test files affected by changes + * codegraph upgrade [version] Update CodeGraph to the latest release + */ + +// FIRST import, before anything else loads: capture process.ppid while our +// launcher is (almost certainly) still alive. A launcher killed mid-startup +// otherwise blinds the PPID watchdog forever (#1185) — see early-ppid.ts. +import '../mcp/early-ppid'; + +import { Command } from 'commander'; +import * as path from 'path'; +import * as fs from 'fs'; +import { getCodeGraphDir, isInitialized, unsafeIndexRootReason, findNearestCodeGraphRoot, planFrontload, hasStructuralKeyword, extractCodeTokens } from '../directory'; +import { extractProseCandidates } from '../search/identifier-segments'; +import { detectWorktreeIndexMismatch, worktreeMismatchWarning } from '../sync/worktree'; +import { createShimmerProgress } from '../ui/shimmer-progress'; +import { getGlyphs } from '../ui/glyphs'; + +import { buildNode25BlockBanner, buildNodeTooOldBanner, MIN_NODE_MAJOR } from './node-version-check'; +import { installFatalHandlers } from './fatal-handler'; +import { relaunchWithWasmRuntimeFlagsIfNeeded } from '../extraction/wasm-runtime-flags'; +import { installCommandSupervision } from './command-supervision'; +import { EXTRACTION_VERSION } from '../extraction/extraction-version'; +import { getTelemetry, TELEMETRY_DOCS, recordIndexEvent } from '../telemetry'; + +// Lazy-load heavy modules (CodeGraph, runInstaller) to keep CLI startup fast. +async function loadCodeGraph(): Promise { + try { + return await import('../index'); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`\x1b[31m${getGlyphs().err}\x1b[0m Failed to load CodeGraph modules.`); + console.error(`\n Node: ${process.version} Platform: ${process.platform} ${process.arch}`); + console.error(`\n Error: ${msg}`); + console.error('\n Try reinstalling with: npm install -g @colbymchenry/codegraph\n'); + process.exit(1); + } +} + +// Dynamic import helper — tsc compiles import() to require() in CJS mode, +// which fails for ESM-only packages. This bypasses the transformation. +// eslint-disable-next-line @typescript-eslint/no-implied-eval +const importESM = new Function('specifier', 'return import(specifier)') as + (specifier: string) => Promise; + +// Block CodeGraph on Node.js 25.x — V8's turboshaft WASM JIT has a Zone +// allocator bug that reliably crashes when compiling tree-sitter +// grammars (see #54, #81, #140). The previous behaviour was a soft +// console.warn that scrolls off-screen before the OOM crash 30 seconds +// later, leading to a steady stream of "what is this OOM" reports. +// Hard-exit before any WASM work; allow override via env var for users +// who patched V8 themselves or want to test a future fix. +const nodeVersion = process.versions.node; +const nodeMajor = parseInt(nodeVersion.split('.')[0] ?? '0', 10); +if (nodeMajor >= 25) { + process.stderr.write(buildNode25BlockBanner(nodeVersion) + '\n'); + if (!process.env.CODEGRAPH_ALLOW_UNSAFE_NODE) { + process.exit(1); + } + // Override active — banner shown for visibility, continuing. +} +// Enforce the supported Node floor. `engines` in package.json only *warns* on +// install (unless engine-strict), so hard-block here to actually keep users off +// unsupported versions. Mirrors the 25+ block above. See package.json `engines`. +if (nodeMajor < MIN_NODE_MAJOR) { + process.stderr.write(buildNodeTooOldBanner(nodeVersion) + '\n'); + if (!process.env.CODEGRAPH_ALLOW_UNSAFE_NODE) { + process.exit(1); + } + // Override active — banner shown for visibility, continuing. +} + +// Re-exec with V8's `--liftoff-only` if it isn't already set, so tree-sitter's +// large WASM grammars never hit the turboshaft Zone OOM (`Fatal process out of +// memory: Zone`) on Node >= 22. No-op under the bundled launcher, which already +// passes the flag. Must run before any grammar (in the parse worker, which +// inherits this process's flags) is compiled. See ../extraction/wasm-runtime-flags. +relaunchWithWasmRuntimeFlagsIfNeeded(__filename); + +// Last-resort fatal handlers: log a bounded line and exit non-zero. A fault +// that reaches here escaped every boundary, so the process is in an undefined +// state — keeping it alive is what let the detached MCP daemon orphan and pin a +// CPU core with no recovery (#799, #850). Installed before the command branch +// so it also covers a synchronous throw during startup. See ./fatal-handler. +installFatalHandlers(); + +// Check if running with no arguments - run installer +if (process.argv.length === 2) { + import('../installer').then(({ runInstaller }) => + runInstaller() + ).catch((err) => { + console.error('Installation failed:', err instanceof Error ? err.message : String(err)); + process.exit(1); + }); +} else { + // Normal CLI flow + main(); +} + +function main() { + +const program = new Command(); + +// Version from package.json +const packageJson = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', '..', 'package.json'), 'utf-8') +); + +// Make the version trivial to reach. commander's `.version()` (below) wires up +// `--version` and `-V`; intercept the spellings it can't — lowercase `-v` and +// single-dash `-version` — before any parsing. (commander's version short flag +// is the capital `-V`, and its parser rejects a multi-character single-dash +// flag.) The bare `codegraph version` subcommand is registered further down so +// the affordance also shows up in `codegraph --help`. +const firstArg = process.argv[2]; +if (firstArg === '-v' || firstArg === '-version') { + console.log(packageJson.version); + return; +} + +// ============================================================================= +// ANSI Color Helpers (avoid chalk ESM issues) +// ============================================================================= + +const colors = { + reset: '\x1b[0m', + bold: '\x1b[1m', + dim: '\x1b[2m', + red: '\x1b[31m', + green: '\x1b[32m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + cyan: '\x1b[36m', + white: '\x1b[37m', + gray: '\x1b[90m', +}; + +const chalk = { + bold: (s: string) => `${colors.bold}${s}${colors.reset}`, + dim: (s: string) => `${colors.dim}${s}${colors.reset}`, + red: (s: string) => `${colors.red}${s}${colors.reset}`, + green: (s: string) => `${colors.green}${s}${colors.reset}`, + yellow: (s: string) => `${colors.yellow}${s}${colors.reset}`, + blue: (s: string) => `${colors.blue}${s}${colors.reset}`, + cyan: (s: string) => `${colors.cyan}${s}${colors.reset}`, + white: (s: string) => `${colors.white}${s}${colors.reset}`, + gray: (s: string) => `${colors.gray}${s}${colors.reset}`, +}; + +program + .name('codegraph') + .description('Code intelligence and knowledge graph for any codebase') + .version(packageJson.version); + +// Anonymous usage telemetry (see TELEMETRY.md): record the invoked subcommand +// NAME only — never arguments or paths. Counts buffer locally; network sends +// piggyback on commands that run long anyway (quick commands only append to +// the local buffer at exit, costing nothing). +// install/uninstall are absent on purpose: the installer flushes at its own +// end, AFTER its consent prompt — a flush here would fire the first-run +// notice before the user ever sees the toggle. +const TELEMETRY_FLUSH_COMMANDS = new Set(['init', 'uninit', 'index', 'sync', 'upgrade']); +program.hook('preAction', (_thisCommand, actionCommand) => { + try { + // The detached daemon re-invokes `serve --mcp` internally — not a user action. + if (process.env.CODEGRAPH_DAEMON_INTERNAL) return; + const name = actionCommand.name(); + if (name === 'telemetry') return; // managing telemetry is not usage + getTelemetry().recordUsage('cli_command', name, true); + if (TELEMETRY_FLUSH_COMMANDS.has(name)) getTelemetry().maybeFlush(); + } catch { + /* telemetry must never break the CLI */ + } +}); + +// ============================================================================= +// Helper Functions +// ============================================================================= + +/** + * Resolve project path from argument or current directory + * Walks up parent directories to find nearest initialized CodeGraph project + * (must have .codegraph/codegraph.db, not just .codegraph/lessons.db) + */ +function resolveProjectPath(pathArg?: string): string { + const absolutePath = path.resolve(pathArg || process.cwd()); + + // If exact path is initialized (has codegraph.db), use it + if (isInitialized(absolutePath)) { + return absolutePath; + } + + // Walk up to find nearest parent with CodeGraph initialized + // Note: findNearestCodeGraphRoot finds any .codegraph folder, but we need one with codegraph.db + let current = absolutePath; + const root = path.parse(current).root; + + while (current !== root) { + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + + if (isInitialized(current)) { + return current; + } + } + + // Not found - return original path (will fail later with helpful error) + return absolutePath; +} + +/** + * Format a number with commas + */ +function formatNumber(n: number): string { + return n.toLocaleString(); +} + +/** + * Format duration in milliseconds to human readable + */ +function formatDuration(ms: number): string { + if (ms < 1000) { + return `${ms}ms`; + } + const seconds = ms / 1000; + if (seconds < 60) { + return `${seconds.toFixed(1)}s`; + } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return `${minutes}m ${remainingSeconds.toFixed(0)}s`; +} + +// Shimmer progress renderer (runs in a worker thread for smooth animation) +// Imported at top of file from '../ui/shimmer-progress' + +/** + * Create a plain-text progress callback for --verbose mode. + * No animations, no ANSI tricks — just timestamped lines to stdout. + */ +function createVerboseProgress(): (progress: { phase: string; current: number; total: number; currentFile?: string }) => void { + let lastPhase = ''; + let lastPct = -1; + const startTime = Date.now(); + + return (progress) => { + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + if (progress.phase !== lastPhase) { + lastPhase = progress.phase; + lastPct = -1; + console.log(`[${elapsed}s] Phase: ${progress.phase}`); + } + + if (progress.total > 0) { + const pct = Math.floor((progress.current / progress.total) * 100); + // Log every 5% to keep output manageable + if (pct >= lastPct + 5 || progress.current === progress.total) { + lastPct = pct; + console.log(`[${elapsed}s] ${progress.current}/${progress.total} (${pct}%)${progress.currentFile ? ` ${getGlyphs().dash} ${progress.currentFile}` : ''}`); + } + } else if (progress.current > 0) { + // Scanning phase (no total yet) — log periodically + if (progress.current % 1000 === 0 || progress.current === 1) { + console.log(`[${elapsed}s] ${formatNumber(progress.current)} files found`); + } + } + }; +} + +/** + * Print success message + */ +function success(message: string): void { + console.log(chalk.green(getGlyphs().ok) + ' ' + message); +} + +/** + * Print error message + */ +function error(message: string): void { + console.error(chalk.red(getGlyphs().err) + ' ' + message); +} + +/** + * Print info message + */ +function info(message: string): void { + console.log(chalk.blue(getGlyphs().info) + ' ' + message); +} + +/** + * Print warning message + */ +function warn(message: string): void { + console.log(chalk.yellow(getGlyphs().warn) + ' ' + message); +} + +type IndexResult = { + success: boolean; + filesIndexed: number; + filesSkipped: number; + filesErrored: number; + nodesCreated: number; + edgesCreated: number; + errors: Array<{ message: string; filePath?: string; severity: string; code?: string }>; + durationMs: number; +}; + +/** + * Print indexing results using clack log methods + */ +function printIndexResult(clack: typeof import('@clack/prompts'), result: IndexResult, projectPath?: string): void { + const hasErrors = result.filesErrored > 0; + + // Surface non-file-level failures (e.g. lock-acquisition failure + // when another indexer is running) before the file-count branches. + // Without this the CLI falls through to "No files found to index", + // which is actively misleading — the index DID run, it just couldn't + // get the lock. + // + // If success is false but no severity:'error' entry exists in + // `result.errors` (degenerate case — shouldn't happen in practice + // but worth guarding because the result shape is plumbed through + // multiple call sites), fall back to a generic message rather than + // continuing to the misleading "No files found" branch or throwing. + if (!result.success && !hasErrors && result.filesIndexed === 0) { + const generic = result.errors.find((e) => e.severity === 'error'); + clack.log.error(generic?.message ?? `Indexing failed ${getGlyphs().dash} no further details available`); + return; + } + + if (result.filesIndexed > 0) { + if (hasErrors) { + clack.log.success(`Indexed ${formatNumber(result.filesIndexed)} files (${formatNumber(result.filesErrored)} could not be parsed)`); + } else { + clack.log.success(`Indexed ${formatNumber(result.filesIndexed)} files`); + } + clack.log.info(`${formatNumber(result.nodesCreated)} nodes, ${formatNumber(result.edgesCreated)} edges in ${formatDuration(result.durationMs)}`); + // A PARTIAL index (files silently dropped mid-pipeline) must not pass + // as a clean run — it's the difference between "indexed the repo" and + // "indexed most of the repo, quietly". Only the completeness + // reconciliation warning; per-file extractor warnings stay in the + // error-code summary below. + for (const w of result.errors.filter((e) => e.code === 'index_partial')) { + clack.log.warn(w.message); + } + } else if (hasErrors) { + clack.log.error(`Indexing failed ${getGlyphs().dash} all ${formatNumber(result.filesErrored)} files had errors`); + } else { + clack.log.warn('No files found to index'); + } + + if (hasErrors) { + const errorsByCode = new Map(); + for (const err of result.errors) { + if (err.severity === 'error') { + const code = err.code || 'unknown'; + errorsByCode.set(code, (errorsByCode.get(code) || 0) + 1); + } + } + + const codeLabels: Record = { + parse_error: 'files failed to parse', + read_error: 'files could not be read', + size_exceeded: 'files exceeded size limit', + path_traversal: 'blocked paths', + unsupported_language: 'unsupported language', + parser_error: 'parser initialization failures', + }; + + const breakdown = Array.from(errorsByCode) + .map(([code, count]) => `${formatNumber(count)} ${codeLabels[code] || code}`) + .join('\n'); + clack.note(breakdown, 'Error breakdown'); + + if (projectPath) { + writeErrorLog(projectPath, result.errors); + clack.log.info('See .codegraph/errors.log for details'); + } + + if (result.filesIndexed > 0) { + clack.log.info(`The index is fully usable ${getGlyphs().dash} only the failed files are missing.`); + } + } else if (projectPath) { + const logPath = path.join(getCodeGraphDir(projectPath), 'errors.log'); + if (fs.existsSync(logPath)) { + fs.unlinkSync(logPath); + } + } +} + +/** + * When an `init`/`index` produced an EMPTY graph and the reason is that the + * project's own `.gitignore` excludes nested git repositories — the "super-repo + * gitignores its child repos" layout (#1156), where `init` at the parent + * correctly indexes ~nothing while `init` inside each child works — name those + * repos and offer to index them. An interactive terminal gets a yes/no prompt + * that writes `includeIgnored` to codegraph.json and re-indexes; a + * non-interactive run just prints the one-line opt-in snippet. The caller gates + * this on `nodesCreated === 0`, so a project that DID index real content is + * never nagged about the gitignored reference clones it deliberately keeps out + * (#970, #1065). Best-effort throughout: detection never breaks the command. + */ +async function offerIndexIgnoredRepos( + clack: typeof import('@clack/prompts'), + projectPath: string, + reindex: () => Promise, + opts: { interactive: boolean }, +): Promise { + let repos: string[]; + try { + const { findUnindexedIgnoredRepos } = await import('../extraction'); + repos = findUnindexedIgnoredRepos(projectPath); + } catch { + return; // detection is advisory — never let it break the command + } + if (repos.length === 0) return; + + const { PROJECT_CONFIG_FILENAME } = await import('../project-config'); + const isOne = repos.length === 1; + const SHOWN = 6; + const names = repos.slice(0, SHOWN).map((r) => r.replace(/\/$/, '')); + const extra = repos.length > SHOWN ? ` (+${formatNumber(repos.length - SHOWN)} more)` : ''; + const snippet = `{ "includeIgnored": [${repos.map((p) => JSON.stringify(p)).join(', ')}] }`; + + clack.log.warn( + `Your .gitignore excludes ${isOne ? 'a nested git repository' : `${formatNumber(repos.length)} nested git repositories`} here, ` + + `so ${isOne ? 'it was' : 'they were'} not indexed: ${names.join(', ')}${extra}.`, + ); + + const manualHint = () => { + clack.log.info( + `If ${isOne ? "it's" : "they're"} your code, add ${isOne ? 'it' : 'them'} to ${PROJECT_CONFIG_FILENAME} and re-index:`, + ); + clack.log.info(` ${snippet}`); + }; + + if (!opts.interactive || !process.stdin.isTTY) { + manualHint(); + return; + } + + const yes = await clack.confirm({ + message: `Index ${isOne ? 'it' : `these ${formatNumber(repos.length)}`} now? Adds ${isOne ? 'it' : 'them'} to ${PROJECT_CONFIG_FILENAME}.`, + initialValue: true, + }); + if (clack.isCancel(yes) || !yes) { + manualHint(); + return; + } + + let added: number; + try { + const { addIncludeIgnoredPatterns } = await import('../project-config'); + added = addIncludeIgnoredPatterns(projectPath, repos); + } catch (err) { + clack.log.error(`Could not update ${PROJECT_CONFIG_FILENAME}: ${err instanceof Error ? err.message : String(err)}`); + manualHint(); + return; + } + clack.log.success(`Added ${formatNumber(added)} ${added === 1 ? 'entry' : 'entries'} to ${PROJECT_CONFIG_FILENAME} ${getGlyphs().dash} re-indexing…`); + + const result = await reindex(); + printIndexResult(clack, result, projectPath); + return result; +} + +/** + * Write detailed error log to .codegraph/errors.log + */ +function writeErrorLog(projectPath: string, errors: Array<{ message: string; filePath?: string; severity: string; code?: string }>): void { + const cgDir = getCodeGraphDir(projectPath); + if (!fs.existsSync(cgDir)) return; + + const logPath = path.join(cgDir, 'errors.log'); + + // Group errors by file path + const errorsByFile = new Map>(); + const noFileErrors: Array<{ message: string; code?: string }> = []; + + for (const err of errors) { + if (err.severity !== 'error') continue; + if (err.filePath) { + let list = errorsByFile.get(err.filePath); + if (!list) { + list = []; + errorsByFile.set(err.filePath, list); + } + list.push({ message: err.message, code: err.code }); + } else { + noFileErrors.push({ message: err.message, code: err.code }); + } + } + + const lines: string[] = [ + `CodeGraph Error Log - ${new Date().toISOString()}`, + `${errorsByFile.size} files with errors`, + '', + ]; + + for (const [filePath, fileErrors] of errorsByFile) { + for (const err of fileErrors) { + lines.push(`${filePath}: ${err.message}`); + } + } + + for (const err of noFileErrors) { + lines.push(err.message); + } + + fs.writeFileSync(logPath, lines.join('\n') + '\n'); +} + +/** + * Telemetry for a completed full index (see TELEMETRY.md). The bounded flush + * keeps init/index responsive (these commands just ran for seconds anyway) + * while delivering the event promptly. + */ +async function recordIndexTelemetry( + cg: { getStats(): { filesByLanguage: Record }; getBackend(): string }, + result: IndexResult, +): Promise { + recordIndexEvent(cg, result); + await getTelemetry().flushNow(); +} + +// ============================================================================= +// Commands +// ============================================================================= + +/** + * codegraph init [path] + */ +program + .command('init [path]') + .description('Initialize CodeGraph in a project directory and build the initial index') + .option('-i, --index', 'Deprecated: indexing now runs by default; flag accepted for backward compatibility') + .option('-f, --force', 'Initialize even if the path looks like your home directory or a filesystem root') + .option('-v, --verbose', 'Show detailed worker lifecycle and memory info') + .action(async (pathArg: string | undefined, options: { index?: boolean; force?: boolean; verbose?: boolean }) => { + const projectPath = path.resolve(pathArg || process.cwd()); + const clack = await importESM('@clack/prompts'); + + clack.intro('Initializing CodeGraph'); + + try { + // Refuse to index your home directory / a filesystem root — it pulls in + // caches, other projects, and your whole tree (a multi-GB index + watcher + // churn, and on pre-1.0 macOS a machine-crashing fd blowup, #845). + const unsafe = unsafeIndexRootReason(projectPath); + if (unsafe && !options.force) { + clack.log.error(`Refusing to initialize in ${projectPath} — it looks like ${unsafe}.`); + clack.log.info('Run this inside a specific project directory, or pass --force if you really mean to index everything under it.'); + clack.outro(''); + process.exitCode = 1; + return; + } + + if (isInitialized(projectPath)) { + clack.log.warn(`Already initialized in ${projectPath}`); + clack.log.info('Use "codegraph index" to re-index or "codegraph sync" to update'); + try { + const { offerWatchFallback } = await import('../installer'); + await offerWatchFallback(clack, projectPath); + } catch { /* non-fatal */ } + clack.outro(''); + return; + } + + const { default: CodeGraph, getDatabasePath } = await loadCodeGraph(); + const cg = await CodeGraph.init(projectPath, { index: false }); + clack.log.success(`Initialized in ${projectPath}`); + + // Indexing runs by default now. The legacy -i/--index flag is still + // accepted (so existing muscle memory and scripts don't break) but is a + // no-op — initializing always builds the initial index. + // Supervise the index: self-terminate if orphaned or wedged (#999). + // The DB + WAL paths let the liveness watchdog tell a slow store on + // degraded storage from a true wedge (#1231). + // A closure so we can re-run the exact same supervised, progress-rendered + // index if the user opts gitignored child repos in below (#1156). + const dbPath = getDatabasePath(projectPath); + const runIndex = async (): Promise => { + const supervision = installCommandSupervision('init', { progressPaths: [dbPath, `${dbPath}-wal`] }); + try { + if (options.verbose) { + return await cg.indexAll({ onProgress: createVerboseProgress(), verbose: true }); + } + process.stdout.write(`${colors.dim}${getGlyphs().rail}${colors.reset}\n`); + const progress = createShimmerProgress(); + const r = await cg.indexAll({ onProgress: progress.onProgress }); + await progress.stop(); + return r; + } finally { + supervision.stop(); + } + }; + const result = await runIndex(); + printIndexResult(clack, result, projectPath); + await recordIndexTelemetry(cg, result); + + // An empty graph at a git super-repo usually means `.gitignore` excludes + // the child repos that hold the code — surface them and offer to opt in + // rather than leaving the user with a silent 0-node "Done". (#1156) + if (result.nodesCreated === 0) { + await offerIndexIgnoredRepos(clack, projectPath, runIndex, { interactive: true }); + } + + try { + const { offerWatchFallback } = await import('../installer'); + await offerWatchFallback(clack, projectPath); + } catch { /* non-fatal */ } + + clack.outro('Done'); + cg.destroy(); + } catch (err) { + clack.log.error(`Failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph uninit [path] + */ +program + .command('uninit [path]') + .description('Remove CodeGraph from a project (deletes .codegraph/ directory)') + .option('-f, --force', 'Skip confirmation prompt') + .action(async (pathArg: string | undefined, options: { force?: boolean }) => { + const projectPath = resolveProjectPath(pathArg); + + try { + if (!isInitialized(projectPath)) { + warn(`CodeGraph is not initialized in ${projectPath}`); + return; + } + + if (!options.force) { + // Confirm with user + const readline = await import('readline'); + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const answer = await new Promise((resolve) => { + rl.question( + chalk.yellow(`${getGlyphs().warn} This will permanently delete all CodeGraph data. Continue? (y/N) `), + resolve + ); + }); + rl.close(); + + if (answer.toLowerCase() !== 'y') { + info('Cancelled'); + return; + } + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = CodeGraph.openSync(projectPath); + cg.uninitialize(); + + // Clean up any git sync hooks we installed (no-op if none / not a repo). + try { + const { removeGitSyncHook } = await import('../sync/git-hooks'); + const removed = removeGitSyncHook(projectPath); + if (removed.installed.length > 0) { + info(`Removed git ${removed.installed.join(', ')} sync hook${removed.installed.length > 1 ? 's' : ''}`); + } + } catch { /* non-fatal */ } + + success(`Removed CodeGraph from ${projectPath}`); + + // Churn signal — and flush now, since after an uninit there may be no + // "next run" to deliver it. + try { + getTelemetry().recordLifecycle('uninstall', {}); + await getTelemetry().flushNow(); + } catch { /* non-fatal */ } + } catch (err) { + error(`Failed to uninitialize: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph index [path] + */ +program + .command('index [path]') + .description('Rebuild the full index from scratch (same result as a fresh init)') + .option('-f, --force', 'Index even if the path looks like your home directory or a filesystem root') + .option('-q, --quiet', 'Suppress progress output') + .option('-v, --verbose', 'Show detailed worker lifecycle and memory info') + .action(async (pathArg: string | undefined, options: { force?: boolean; quiet?: boolean; verbose?: boolean }) => { + const projectPath = resolveProjectPath(pathArg); + + try { + // Don't (re)index your home directory / a filesystem root (#845). --force + // doubles as the override. + const unsafe = unsafeIndexRootReason(projectPath); + if (unsafe && !options.force) { + error(`Refusing to index ${projectPath} — it looks like ${unsafe}. Pass --force to override.`); + process.exit(1); + } + + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + info('Run "codegraph init" first'); + process.exit(1); + } + + const { default: CodeGraph, getDatabasePath } = await loadCodeGraph(); + // `index` is a FULL re-index — identical to a fresh `init`. RECREATE the + // database from scratch (discard .codegraph/codegraph.db + its WAL) rather + // than opening the old graph and DELETE-ing every row. The clear-then-index + // approach reported "0 nodes" without the clear (#874); the recreate keeps + // that fixed AND avoids the failure mode where, on a large or pre-fix + // poisoned index, the per-row FTS delete churn wedged the main thread long + // enough to trip the liveness watchdog before scanning even began (#1067). + // recreate() hands back a fresh, empty instance — no clear() needed. For + // fast incremental updates use `sync`. + const cg = await CodeGraph.recreate(projectPath); + + // Supervise the indexer: self-terminate if orphaned (parent shim killed) + // or if the main thread wedges — neither was guarded on this path (#999). + // The DB + WAL paths let the liveness watchdog tell a slow store on + // degraded storage from a true wedge (#1231). + const dbPath = getDatabasePath(projectPath); + const supervision = installCommandSupervision('index', { progressPaths: [dbPath, `${dbPath}-wal`] }); + try { + if (options.quiet) { + // Quiet mode: no UI, just run against the freshly-recreated graph. + const result = await cg.indexAll(); + if (!result.success) process.exit(1); + cg.destroy(); + return; + } + + const clack = await importESM('@clack/prompts'); + clack.intro('Indexing project'); + + // A closure so a re-index (after opting gitignored child repos in, #1156) + // renders identically. Supervision already wraps the whole command. + const renderIndex = async (): Promise => { + if (options.verbose) { + return await cg.indexAll({ onProgress: createVerboseProgress(), verbose: true }); + } + process.stdout.write(`${colors.dim}${getGlyphs().rail}${colors.reset}\n`); + const progress = createShimmerProgress(); + const r = await cg.indexAll({ onProgress: progress.onProgress }); + await progress.stop(); + return r; + }; + + const result = await renderIndex(); + + printIndexResult(clack, result, projectPath); + await recordIndexTelemetry(cg, result); + + // Empty graph at a git super-repo → likely `.gitignore`d child repos; + // name them and offer to opt in instead of a silent 0-node result (#1156). + let finalResult = result; + if (result.nodesCreated === 0) { + finalResult = (await offerIndexIgnoredRepos(clack, projectPath, renderIndex, { interactive: true })) ?? result; + } + + if (!finalResult.success) { + process.exit(1); + } + + clack.outro('Done'); + cg.destroy(); + } finally { + supervision.stop(); + } + } catch (err) { + error(`Failed to index: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph sync [path] + */ +program + .command('sync [path]') + .description('Sync changes since last index') + .option('-q, --quiet', 'Suppress output (for git hooks)') + .action(async (pathArg: string | undefined, options: { quiet?: boolean }) => { + const projectPath = resolveProjectPath(pathArg); + + try { + if (!isInitialized(projectPath)) { + if (!options.quiet) { + error(`CodeGraph not initialized in ${projectPath}`); + } + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + + if (options.quiet) { + await cg.sync(); + cg.destroy(); + return; + } + + const clack = await importESM('@clack/prompts'); + clack.intro('Syncing CodeGraph'); + + process.stdout.write(`${colors.dim}${getGlyphs().rail}${colors.reset}\n`); + const progress = createShimmerProgress(); + + const result = await cg.sync({ + onProgress: progress.onProgress, + }); + + await progress.stop(); + + const totalChanges = result.filesAdded + result.filesModified + result.filesRemoved; + + if (totalChanges === 0) { + clack.log.info('Already up to date'); + } else { + clack.log.success(`Synced ${formatNumber(totalChanges)} changed files`); + const details: string[] = []; + if (result.filesAdded > 0) details.push(`Added: ${result.filesAdded}`); + if (result.filesModified > 0) details.push(`Modified: ${result.filesModified}`); + if (result.filesRemoved > 0) details.push(`Removed: ${result.filesRemoved}`); + clack.log.info(`${details.join(', ')} ${getGlyphs().dash} ${formatNumber(result.nodesUpdated)} nodes in ${formatDuration(result.durationMs)}`); + } + + clack.outro('Done'); + cg.destroy(); + } catch (err) { + if (!options.quiet) { + error(`Failed to sync: ${err instanceof Error ? err.message : String(err)}`); + } + process.exit(1); + } + }); + +/** + * codegraph status [path] + */ +program + .command('status [path]') + .description('Show index status and statistics') + .option('-j, --json', 'Output as JSON') + .action(async (pathArg: string | undefined, options: { json?: boolean }) => { + const projectPath = resolveProjectPath(pathArg); + // The directory the user actually ran from, before walking up to the index + // root. Used to detect when the resolved index lives in a different git + // working tree (e.g. a nested worktree borrowing the main checkout's index). + const startPath = path.resolve(pathArg || process.cwd()); + const worktreeMismatch = detectWorktreeIndexMismatch(startPath, projectPath); + + try { + if (!isInitialized(projectPath)) { + if (options.json) { + console.log(JSON.stringify({ + initialized: false, + version: packageJson.version, + projectPath, + indexPath: getCodeGraphDir(projectPath), + lastIndexed: null, + })); + return; + } + console.log(chalk.bold('\nCodeGraph Status\n')); + info(`Project: ${projectPath}`); + warn('Not initialized'); + info('Run "codegraph init" to initialize'); + return; + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const stats = cg.getStats(); + const changes = cg.getChangedFiles(); + const backend = cg.getBackend(); + const journalMode = cg.getJournalMode(); + + const buildInfo = cg.getIndexBuildInfo(); + const reindexRecommended = cg.isIndexStale(); + const indexState = cg.getIndexState(); + // Zero on a healthy index; non-zero at rest means a resolution pass was + // interrupted, so some files' call edges are missing (#1187). + const pendingRefs = cg.getPendingReferenceCount(); + + // JSON output mode + if (options.json) { + const lastIndexedMs = cg.getLastIndexedAt(); + console.log(JSON.stringify({ + initialized: true, + version: packageJson.version, + projectPath, + indexPath: getCodeGraphDir(projectPath), + lastIndexed: lastIndexedMs != null ? new Date(lastIndexedMs).toISOString() : null, + fileCount: stats.fileCount, + nodeCount: stats.nodeCount, + edgeCount: stats.edgeCount, + dbSizeBytes: stats.dbSizeBytes, + backend, + journalMode, + nodesByKind: stats.nodesByKind, + languages: Object.entries(stats.filesByLanguage).filter(([, count]) => count > 0).map(([lang]) => lang), + pendingChanges: { + added: changes.added.length, + modified: changes.modified.length, + removed: changes.removed.length, + }, + worktreeMismatch: worktreeMismatch + ? { worktreeRoot: worktreeMismatch.worktreeRoot, indexRoot: worktreeMismatch.indexRoot } + : null, + index: { + builtWithVersion: buildInfo.version, + builtWithExtractionVersion: buildInfo.extractionVersion, + currentExtractionVersion: EXTRACTION_VERSION, + reindexRecommended, + // 'complete' | 'partial' (files silently dropped) | 'indexing' + // (a run was killed mid-index — the index is truncated) | + // 'failed' | null (predates the marker). + state: indexState, + // References awaiting resolution. Non-zero at rest means an + // interrupted resolution pass left edges missing; the next + // sync sweeps them (#1187). + pendingRefs, + }, + })); + cg.destroy(); + return; + } + + console.log(chalk.bold('\nCodeGraph Status\n')); + + // Project info + console.log(chalk.cyan('Project:'), projectPath); + if (worktreeMismatch) { + warn(worktreeMismatchWarning(worktreeMismatch)); + } + if (indexState === 'indexing') { + warn('The last index run never finished (killed mid-index?) — the index is truncated. Re-run "codegraph index".'); + } else if (indexState === 'partial') { + warn('The last index run silently dropped files — the index is partial. Re-run "codegraph index".'); + } else if (indexState === 'failed') { + warn('The last index run failed — results may be incomplete. Re-run "codegraph index".'); + } + if (pendingRefs > 0) { + warn(`${formatNumber(pendingRefs)} references from an interrupted run are awaiting resolution — some callers/impact edges are missing. Run "codegraph sync" to resolve them.`); + } + console.log(); + + // Index stats + console.log(chalk.bold('Index Statistics:')); + console.log(` Files: ${formatNumber(stats.fileCount)}`); + console.log(` Nodes: ${formatNumber(stats.nodeCount)}`); + console.log(` Edges: ${formatNumber(stats.edgeCount)}`); + console.log(` DB Size: ${(stats.dbSizeBytes / 1024 / 1024).toFixed(2)} MB`); + // Surface the active SQLite backend (node:sqlite — Node's built-in real + // SQLite, full WAL + FTS5, no native build). + const backendLabel = chalk.green(`node:sqlite ${getGlyphs().dash} built-in (full WAL)`); + console.log(` Backend: ${backendLabel}`); + // Effective journal mode: 'wal' means concurrent reads never block on a + // writer; anything else means they can ("database is locked"). node:sqlite + // supports WAL everywhere, so a non-wal mode means the filesystem can't + // (network mounts, WSL2 /mnt). See issue #238. + const journalLabel = journalMode === 'wal' + ? chalk.green('wal') + : chalk.yellow(`${journalMode || 'unknown'} ${getGlyphs().dash} WAL inactive; reads can block on writes`); + console.log(` Journal: ${journalLabel}`); + console.log(); + + // Node breakdown + console.log(chalk.bold('Nodes by Kind:')); + const nodesByKind = Object.entries(stats.nodesByKind) + .filter(([, count]) => count > 0) + .sort((a, b) => b[1] - a[1]); + for (const [kind, count] of nodesByKind) { + console.log(` ${kind.padEnd(15)} ${formatNumber(count)}`); + } + console.log(); + + // Language breakdown + console.log(chalk.bold('Files by Language:')); + const filesByLang = Object.entries(stats.filesByLanguage) + .filter(([, count]) => count > 0) + .sort((a, b) => b[1] - a[1]); + for (const [lang, count] of filesByLang) { + console.log(` ${lang.padEnd(15)} ${formatNumber(count)}`); + } + console.log(); + + // Pending changes + const totalChanges = changes.added.length + changes.modified.length + changes.removed.length; + if (totalChanges > 0) { + console.log(chalk.bold('Pending Changes:')); + if (changes.added.length > 0) { + console.log(` Added: ${changes.added.length} files`); + } + if (changes.modified.length > 0) { + console.log(` Modified: ${changes.modified.length} files`); + } + if (changes.removed.length > 0) { + console.log(` Removed: ${changes.removed.length} files`); + } + info('Run "codegraph sync" to update the index'); + } else { + success('Index is up to date'); + } + console.log(); + + // Re-index hint: the index was built by an older engine than the one now + // running, so a rebuild would add data a migration can't backfill. + if (reindexRecommended) { + const builtWith = buildInfo.version ? `v${buildInfo.version.replace(/^v/, '')}` : 'an earlier version'; + warn(`Index was built by ${builtWith}; re-index to pick up this engine's improvements.`); + info('Run "codegraph index" (full rebuild) or "codegraph sync"'); + console.log(); + } + + cg.destroy(); + } catch (err) { + error(`Failed to get status: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph query + */ +program + .command('query ') + .description('Search for symbols in the codebase') + .option('-p, --path ', 'Project path') + .option('-l, --limit ', 'Maximum results', '10') + .option('-k, --kind ', 'Filter by node kind (function, class, etc.)') + .option('-j, --json', 'Output as JSON') + .action(async (search: string, options: { path?: string; limit?: string; kind?: string; json?: boolean }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + + const limit = parseInt(options.limit || '10', 10); + const rawResults = cg.searchNodes(search, { + limit, + kinds: options.kind ? [options.kind as any] : undefined, + }); + + // Mirror the MCP search down-rank so the CLI also surfaces the + // hand-written implementation before protobuf/gRPC scaffolding + // when both share a name. See extraction/generated-detection.ts. + const { isGeneratedFile } = await import('../extraction/generated-detection'); + const results = [...rawResults].sort((a, b) => { + const aGen = isGeneratedFile(a.node.filePath) ? 1 : 0; + const bGen = isGeneratedFile(b.node.filePath) ? 1 : 0; + return aGen - bGen; + }); + + if (options.json) { + console.log(JSON.stringify(results, null, 2)); + } else { + if (results.length === 0) { + info(`No results found for "${search}"`); + } else { + console.log(chalk.bold(`\nSearch Results for "${search}":\n`)); + + // Results arrive already ranked by relevance, so the order conveys + // it. We don't print the raw score: it's an unbounded BM25/FTS value + // (relative-ranking only), and the old `(score * 100)%` rendered it + // as nonsensical percentages like "12042%" (#1045). The MCP search + // tool likewise shows no score. Raw `score` stays in --json output. + for (const result of results) { + const node = result.node; + const location = `${node.filePath}:${node.startLine}`; + + console.log( + chalk.cyan(node.kind.padEnd(12)) + + chalk.white(node.name) + ); + console.log(chalk.dim(` ${location}`)); + if (node.signature) { + console.log(chalk.dim(` ${node.signature}`)); + } + console.log(); + } + } + } + + cg.destroy(); + } catch (err) { + error(`Search failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph explore + * + * The CLI face of the MCP codegraph_explore tool — same handler, same + * output (source of the relevant symbols grouped by file + the call path + * among them). Exists so agents WITHOUT the MCP tools — Task-tool + * subagents (which don't inherit MCP tools, #704) and non-MCP harnesses — + * can reach the graph through a plain shell command. + */ +program + .command('explore ') + .description('Explore an area: relevant symbols\' source + call paths in one shot (same output as the codegraph_explore MCP tool)') + .option('-p, --path ', 'Project path') + .option('--max-files ', 'Maximum number of files to include source from') + .action(async (queryParts: string[], options: { path?: string; maxFiles?: string }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph isn't available here — no .codegraph/ index exists in ${projectPath}. If you are an AI agent: continue with your usual tools; indexing is the user's decision, do not run it yourself. (The project owner can enable CodeGraph with 'codegraph init'.)`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const { ToolHandler } = await import('../mcp/tools'); + const handler = new ToolHandler(cg); + + const args: Record = { query: queryParts.join(' ') }; + if (options.maxFiles) args.maxFiles = parseInt(options.maxFiles, 10); + const result = await handler.execute('codegraph_explore', args); + + console.log(result.content[0]?.text ?? ''); + cg.destroy(); + if (result.isError) process.exit(1); + } catch (err) { + error(`Explore failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph prompt-hook (hidden) + * + * A Claude Code `UserPromptSubmit` hook entry point. Reads `{prompt, cwd}` JSON + * on stdin; for a structural/flow/impact prompt it runs `codegraph_explore` on + * the indexed project and prints the result to stdout, which Claude injects into + * the agent's context — so the agent's reflex grep/read has nothing left to find + * and reliably uses CodeGraph (the adoption problem). Installed by the installer + * into Claude's settings.json (opt-in, default-yes). + * + * LOAD-BEARING: this must NEVER break the user's prompt. Every failure path — + * kill-switch, non-structural prompt, no index, engine error — exits 0 with no + * output. The only effect is additive context when it can confidently provide it. + */ +program + .command('prompt-hook', { hidden: true }) + .description('Claude UserPromptSubmit hook: inject CodeGraph context for structural prompts (reads {prompt,cwd} JSON on stdin)') + .action(async () => { + try { + // Kill-switch: lets a user disable the nudge without uninstalling / + // editing settings.json (CI, low-power machines, personal preference). + if (process.env.CODEGRAPH_NO_PROMPT_HOOK === '1' || process.env.CODEGRAPH_PROMPT_HOOK === '0') return; + if (process.stdin.isTTY) return; // invoked by hand, no piped payload + + const raw = await new Promise((resolve) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (c) => { data += c; }); + process.stdin.on('end', () => resolve(data)); + process.stdin.on('error', () => resolve(data)); + }); + + let input: { prompt?: string; cwd?: string } = {}; + try { input = JSON.parse(raw); } catch { return; } + const prompt = String(input.prompt || ''); + + // Gate telemetry: how often each tier fires vs. no-ops — counter names + // only, NEVER prompt content (see TELEMETRY.md). This is the data that + // turns "is the gate any good" from vibes into a measured recall rate. + const gate = (outcome: string): void => { + try { getTelemetry().recordUsage('cli_command', `prompt-hook-gate-${outcome}`, true); } catch { /* never break the hook */ } + }; + + // Gate, tiered by confidence (#994, #1126): + // HIGH — a structural keyword (any covered language), or a code-shaped + // token verified in the index → full explore injection. + // MEDIUM — no keyword/token, but prose words match indexed symbol-name + // SEGMENTS ("state machine" → OrderStateMachine, in any + // language): inject a short list of the matching symbols and + // let the AGENT write the explore query — the graph-derived + // tier, no vocabulary involved. + // silent — nothing verified. Every other prompt ("fix this typo") + // stays a zero-cost no-op. + // Keywords fire on their own; a token or prose word is only a CANDIDATE + // verified against the graph below, so a tech brand ("JavaScript") that + // merely looks like code doesn't inject spurious context. + const keyworded = hasStructuralKeyword(prompt); + const codeTokens = keyworded ? [] : extractCodeTokens(prompt); + const proseWords = keyworded ? [] : extractProseCandidates(prompt); + if (!keyworded && codeTokens.length === 0 && proseWords.length === 0) { gate('noop-shape'); return; } + + // Decide what to inject, shaped by WHERE the index(es) are: the nearest + // indexed ancestor of cwd, or — when cwd is an un-indexed workspace root + // whose indexed project(s) live in sub-dirs (the monorepo case, #964) — + // the sub-project the prompt points at, plus a `projectPath` nudge for any + // others. Without the down-scan the hook injected nothing at a monorepo + // root (it only walked up), so the validated adoption lever never fired + // exactly where the agent most needs it. + const plan = planFrontload(String(input.cwd || process.cwd()), prompt); + if (!plan.exploreRoot && plan.nudgeProjects.length === 0) { gate('noop-no-index'); return; } // nothing reachable — the agent's normal tools apply + + // A "pass projectPath" line for indexed sub-projects we did NOT front-load. + // Follow-up codegraph_explore calls against a sub-project (cwd isn't its + // index root) need an explicit projectPath, so spell it out. + const nudge = (projects: string[], lead: string): string => + `${lead}\n${projects.map((p) => ` - projectPath: "${p}"`).join('\n')}\n`; + + if (plan.exploreRoot) { + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(plan.exploreRoot); + try { + const others = plan.nudgeProjects.length + ? `\n${nudge(plan.nudgeProjects, 'Other indexed projects in this workspace — pass projectPath to query them:')}` + : ''; + + // Tier decision against THIS index (issue #994 follow-up: candidates + // must be real here — a brand name or prose about another domain + // must not inject). Keyword-bearing prompts skip verification — the + // keyword is signal enough. + const tokenVerified = !keyworded && codeTokens.some((t) => cg.getNodesByName(t).length > 0); + if (keyworded || tokenVerified) { + const { ToolHandler } = await import('../mcp/tools'); + const handler = new ToolHandler(cg); + const result = await handler.execute('codegraph_explore', { query: prompt }); + const text = result.content[0]?.text ?? ''; + if (!result.isError && text.trim()) { + // Cap the injection so a large-repo explore can't flood the prompt. + const MAX = 16000; + const body = text.length > MAX ? `${text.slice(0, MAX)}\n…(truncated; call codegraph_explore for the rest)` : text; + // For a front-loaded SUB-project, a follow-up explore needs its path. + const more = plan.viaSubScan + ? `call codegraph_explore with projectPath: "${plan.exploreRoot}" for more` + : 'call codegraph_explore for more'; + process.stdout.write( + `\n${body}${others}\n\n`, + ); + gate(keyworded ? 'high-keyword' : 'high-token'); + } else { + // A high-* outcome must mean context was actually delivered — + // the funnel's noop-vs-high split is how gate recall is + // measured (#1143). An explore error or empty result is a + // delivery failure, not a gate success. + gate(keyworded ? 'noop-explore-keyword' : 'noop-explore-token'); + } + return; + } + + // MEDIUM: prose words → symbol-name segments, co-occurrence/rarity + // scored, each hit re-verified to exist (see getSegmentMatches). The + // payload names the symbols but does NOT run explore — the agent owns + // the query where the hook's confidence is only "these are related". + // + // A database indexed before the vocab table existed starts with it + // EMPTY, and only sync() backfills it — which this hook never runs + // (#1142). Heal it here: on a populated vocab this is one SELECT; + // the actual backfill is a one-time batched pass whose cost the MCP + // server's own catch-up sync usually pays first (it runs at every + // session start). A distinct noop outcome keeps a dormant vocab + // from polluting the noop-unverified recall signal. + const vocabReady = await cg.healSegmentVocabIfEmpty().catch(() => false); + if (!vocabReady) { gate('noop-vocab-empty'); return; } + const related = cg.getSegmentMatches(proseWords); + if (related.length === 0) { gate('noop-unverified'); return; } + const lines = related + .map((m) => ` - ${m.name} (${m.kind} — ${m.filePath}:${m.startLine})`) + .join('\n'); + const exampleQuery = related.slice(0, 3).map((m) => m.name).join(' '); + const projectHint = plan.viaSubScan ? ` with projectPath: "${plan.exploreRoot}"` : ''; + process.stdout.write( + `\n` + + `This project's CodeGraph index contains symbols matching this request:\n${lines}\n` + + `Call codegraph_explore ONCE${projectHint} with the relevant names in one query (e.g. "${exampleQuery}") ` + + `to get their source, call paths, and blast radius — cheaper and more complete than Read/Grep.\n${others}` + + `\n`, + ); + gate('medium-segment'); + } finally { + cg.destroy(); + } + } else { + // Several indexed sub-projects, none a clear match — don't guess; tell + // the agent they exist and how to query one. + process.stdout.write( + `\n` + + nudge(plan.nudgeProjects, "This workspace's CodeGraph indexes live in sub-projects. To use CodeGraph, call codegraph_explore with the projectPath of the relevant one:") + + `\n`, + ); + gate('nudge-projects'); + } + } catch { + // Degradable by contract: never surface an error to the prompt pipeline. + } + }); + +/** + * codegraph node [name] + * + * The CLI face of the MCP codegraph_node tool: one symbol's source + + * caller/callee trail, or a whole file with line numbers + dependents + * (Read-parity). Same subagent/non-MCP rationale as `explore`. + * + * `name` is OPTIONAL because `--file` (file-read mode) carries no symbol — + * a required `` made `codegraph node -f ` unreachable (#1044). + */ +program + .command('node [name]') + .description('One symbol\'s source + caller/callee trail, or read a file with line numbers + dependents (same output as the codegraph_node MCP tool)') + .option('-p, --path ', 'Project path') + .option('-f, --file ', 'Treat as file mode (or disambiguate a symbol to this file)') + .option('--offset ', 'File mode: 1-based start line') + .option('--limit ', 'File mode: maximum lines') + .option('--symbols-only', 'File mode: just the symbol map + dependents') + .action(async (name: string | undefined, options: { path?: string; file?: string; offset?: string; limit?: string; symbolsOnly?: boolean }) => { + // Need a symbol (positional) OR a file (--file / a path-like positional). + // With [name] optional, a bare `codegraph node` reaches here with neither + // and must be told what to pass, rather than crashing downstream. + if (!name && !options.file) { + error("Pass a symbol name (e.g. 'codegraph node parseToken') or a file (e.g. 'codegraph node -f src/auth.ts', or 'codegraph node src/auth.ts')."); + process.exit(1); + } + + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph isn't available here — no .codegraph/ index exists in ${projectPath}. If you are an AI agent: continue with your usual tools; indexing is the user's decision, do not run it yourself. (The project owner can enable CodeGraph with 'codegraph init'.)`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const { ToolHandler } = await import('../mcp/tools'); + const handler = new ToolHandler(cg); + + // A name with a path separator is a file read; otherwise a symbol + // (use --file for basename-only file reads or to pin an overload). + // Both separators: Windows users type src\auth\session.ts. Symbols + // never contain either ('/' isn't an identifier char anywhere we + // index; C++ scope is '::', JS members '.'). + const args: Record = {}; + if (options.file) { + args.file = options.file; + if (name && name !== options.file) args.symbol = name; + } else if (name && (name.includes('/') || name.includes('\\'))) { + args.file = name.replace(/\\/g, '/'); + } else if (name) { + args.symbol = name; + args.includeCode = true; + } + if (options.offset) args.offset = parseInt(options.offset, 10); + if (options.limit) args.limit = parseInt(options.limit, 10); + if (options.symbolsOnly) args.symbolsOnly = true; + + const result = await handler.execute('codegraph_node', args); + + console.log(result.content[0]?.text ?? ''); + cg.destroy(); + if (result.isError) process.exit(1); + } catch (err) { + error(`Node lookup failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph files [path] + */ +program + .command('files') + .description('Show project file structure from the index') + .option('-p, --path ', 'Project path') + .option('--filter ', 'Filter to files under this directory') + .option('--pattern ', 'Filter files matching this glob pattern') + .option('--format ', 'Output format (tree, flat, grouped)', 'tree') + .option('--max-depth ', 'Maximum directory depth for tree format') + .option('--no-metadata', 'Hide file metadata (language, symbol count)') + .option('-j, --json', 'Output as JSON') + .action(async (options: { + path?: string; + filter?: string; + pattern?: string; + format?: string; + maxDepth?: string; + metadata?: boolean; + json?: boolean; + }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + let files = cg.getFiles(); + + if (files.length === 0) { + info('No files indexed. Run "codegraph index" first.'); + cg.destroy(); + return; + } + + // Filter by path prefix + if (options.filter) { + const filter = options.filter; + files = files.filter(f => f.path.startsWith(filter) || f.path.startsWith('./' + filter)); + } + + // Filter by glob pattern + if (options.pattern) { + const regex = globToRegex(options.pattern); + files = files.filter(f => regex.test(f.path)); + } + + if (files.length === 0) { + info('No files found matching the criteria.'); + cg.destroy(); + return; + } + + // JSON output + if (options.json) { + const output = files.map(f => ({ + path: f.path, + language: f.language, + nodeCount: f.nodeCount, + size: f.size, + })); + console.log(JSON.stringify(output, null, 2)); + cg.destroy(); + return; + } + + const includeMetadata = options.metadata !== false; + const format = options.format || 'tree'; + const maxDepth = options.maxDepth ? parseInt(options.maxDepth, 10) : undefined; + + // Format output + switch (format) { + case 'flat': + console.log(chalk.bold(`\nFiles (${files.length}):\n`)); + for (const file of files.sort((a, b) => a.path.localeCompare(b.path))) { + if (includeMetadata) { + console.log(` ${file.path} ${chalk.dim(`(${file.language}, ${file.nodeCount} symbols)`)}`); + } else { + console.log(` ${file.path}`); + } + } + break; + + case 'grouped': + console.log(chalk.bold(`\nFiles by Language (${files.length} total):\n`)); + const byLang = new Map(); + for (const file of files) { + const existing = byLang.get(file.language) || []; + existing.push(file); + byLang.set(file.language, existing); + } + const sortedLangs = [...byLang.entries()].sort((a, b) => b[1].length - a[1].length); + for (const [lang, langFiles] of sortedLangs) { + console.log(chalk.cyan(`${lang} (${langFiles.length}):`)); + for (const file of langFiles.sort((a, b) => a.path.localeCompare(b.path))) { + if (includeMetadata) { + console.log(` ${file.path} ${chalk.dim(`(${file.nodeCount} symbols)`)}`); + } else { + console.log(` ${file.path}`); + } + } + console.log(); + } + break; + + case 'tree': + default: + console.log(chalk.bold(`\nProject Structure (${files.length} files):\n`)); + printFileTree(files, includeMetadata, maxDepth, chalk); + break; + } + + console.log(); + cg.destroy(); + } catch (err) { + error(`Failed to list files: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * Normalize a user-supplied file path to the project-relative, forward-slash + * form CodeGraph stores in the index. Accepts an absolute path, a `./`-prefixed + * path, or Windows back-slashes; an empty string when the input is blank. Used + * by `codegraph affected` so `./src/x.ts`, `/abs/repo/src/x.ts`, and + * `src/x.ts` all match the same indexed file. (#825) + */ +function normalizeIndexPath(filePath: string, projectPath: string): string { + let f = filePath.trim(); + if (!f) return ''; + if (path.isAbsolute(f)) f = path.relative(projectPath, f); + // Collapse `.`/`..` segments, then force forward slashes and drop a leading + // `./` (path.normalize already strips it on POSIX; explicit for Windows). + f = path.normalize(f).replace(/\\/g, '/').replace(/^\.\//, ''); + return f; +} + +/** + * Convert glob pattern to regex + */ +function globToRegex(pattern: string): RegExp { + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*\*/g, '{{GLOBSTAR}}') + .replace(/\*/g, '[^/]*') + .replace(/\?/g, '[^/]') + .replace(/\{\{GLOBSTAR\}\}/g, '.*'); + return new RegExp(escaped); +} + +/** + * Print files as a tree + */ +function printFileTree( + files: { path: string; language: string; nodeCount: number }[], + includeMetadata: boolean, + maxDepth: number | undefined, + chalk: { dim: (s: string) => string; cyan: (s: string) => string } +): void { + interface TreeNode { + name: string; + children: Map; + file?: { language: string; nodeCount: number }; + } + + const root: TreeNode = { name: '', children: new Map() }; + + for (const file of files) { + const parts = file.path.split('/'); + let current = root; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (!part) continue; + + if (!current.children.has(part)) { + current.children.set(part, { name: part, children: new Map() }); + } + current = current.children.get(part)!; + + if (i === parts.length - 1) { + current.file = { language: file.language, nodeCount: file.nodeCount }; + } + } + } + + const renderNode = (node: TreeNode, prefix: string, isLast: boolean, depth: number): void => { + if (maxDepth !== undefined && depth > maxDepth) return; + + const glyphs = getGlyphs(); + const connector = isLast ? glyphs.treeLast : glyphs.treeBranch; + const childPrefix = isLast ? ' ' : glyphs.treePipe; + + if (node.name) { + let line = prefix + connector + node.name; + if (node.file && includeMetadata) { + line += chalk.dim(` (${node.file.language}, ${node.file.nodeCount} symbols)`); + } + console.log(line); + } + + const children = [...node.children.values()]; + children.sort((a, b) => { + const aIsDir = a.children.size > 0 && !a.file; + const bIsDir = b.children.size > 0 && !b.file; + if (aIsDir !== bIsDir) return aIsDir ? -1 : 1; + return a.name.localeCompare(b.name); + }); + + for (let i = 0; i < children.length; i++) { + const child = children[i]!; + const nextPrefix = node.name ? prefix + childPrefix : prefix; + renderNode(child, nextPrefix, i === children.length - 1, depth + 1); + } + }; + + renderNode(root, '', true, 0); +} + +/** + * codegraph daemon — interactive manager for the background daemons. Arrow keys + * to pick one (the current project's daemon floats to the top, auto-selected), + * enter to stop it. Falls back to a plain list when output isn't a TTY. + */ +program + .command('daemon') + .aliases(['daemons']) + .description('Manage running CodeGraph background daemons — pick one and press enter to stop it') + .action(async () => { + const { listDaemons, stopDaemonAt, stopAllDaemons } = await import('../mcp/daemon-registry'); + const { runDaemonPicker } = await import('../mcp/daemon-manager'); + + const daemons = listDaemons(); + if (daemons.length === 0) { + info('No CodeGraph daemons running.'); + return; + } + + // No TTY (piped / CI / non-interactive) — can't do arrow-key selection, so + // just print what's running instead of crashing on a prompt with no input. + if (!process.stdout.isTTY || !process.stdin.isTTY) { + for (const d of daemons) { + console.log(`pid ${d.pid} v${d.version} up ${formatDuration(Date.now() - d.startedAt)} ${d.root}`); + } + return; + } + + // The current project's daemon floats to the top and is pre-selected. + let cwdRoot: string | null = null; + const found = findNearestCodeGraphRoot(process.cwd()); + if (found) { try { cwdRoot = fs.realpathSync(found); } catch { cwdRoot = found; } } + + const clack = await importESM('@clack/prompts'); + clack.intro('CodeGraph daemons'); + await runDaemonPicker({ + list: listDaemons, + stop: stopDaemonAt, + stopAll: stopAllDaemons, + cwdRoot, + now: () => Date.now(), + select: (opts) => clack.select(opts), + isCancel: (v) => clack.isCancel(v), + note: (m) => clack.log.success(m), + done: (m) => clack.outro(m), + }); + }); + +/** + * codegraph serve + */ +program + // Hidden from `--help`: this is the stdio entry point an AI agent launches + // for itself (the installer wires `args: ['serve','--mcp']` into every + // agent's MCP config), not a command a human runs. It still works when + // invoked — hiding only removes it from the listing. See the interactive-TTY + // guard below, which explains this to anyone who runs it by hand. + .command('serve', { hidden: true }) + .description('Start CodeGraph as an MCP server for AI assistants') + .option('-p, --path ', 'Project path (optional for MCP mode, uses rootUri from client)') + .option('--mcp', 'Run as MCP server (stdio transport)') + .option('--no-watch', 'Disable the file watcher (no auto-sync; useful on slow filesystems like WSL2 /mnt drives)') + .action(async (options: { path?: string; mcp?: boolean; watch?: boolean }) => { + const projectPath = options.path ? resolveProjectPath(options.path) : undefined; + + // Commander sets watch=false when --no-watch is passed. Route it through + // the same env-var chokepoint the watcher and MCP server already honor. + if (options.watch === false) { + process.env.CODEGRAPH_NO_WATCH = '1'; + } + + try { + if (options.mcp) { + // `serve --mcp` is the stdio MCP server an AI agent launches for itself, + // not a command to run by hand. A human in a terminal would otherwise + // see it hang waiting for JSON-RPC on stdin, which reads as broken. If + // stdin is an interactive TTY, explain instead of hanging. The agent's + // pipe and the detached daemon both have a non-TTY stdin, so this only + // ever fires for a person who typed it. + if (process.stdin.isTTY && !process.env.CODEGRAPH_DAEMON_INTERNAL) { + console.error(chalk.bold('\nCodeGraph MCP server\n')); + console.error("This is the MCP server your AI agent (Claude Code, Cursor, Codex, opencode, …)"); + console.error("starts automatically — you don't run it yourself."); + console.error(`\nIt's already wired up by ${chalk.cyan('codegraph install')}. To check on things:`); + console.error(` ${chalk.cyan('codegraph status')} ${chalk.dim('— is this project indexed and healthy?')}`); + console.error(` ${chalk.cyan('codegraph daemon')} ${chalk.dim('— list or stop background MCP servers')}`); + console.error(chalk.dim('\n(Running it directly only does something when an MCP client drives it over stdin.)')); + return; + } + // Start MCP server - it handles initialization lazily based on rootUri from client + const { MCPServer } = await import('../mcp/index'); + const server = new MCPServer(projectPath); + await server.start(); + // Server will run until terminated + } else { + // Default: show info about MCP mode. + // Use stderr so stdout stays clean for any piped/stdio usage. + console.error(chalk.bold('\nCodeGraph MCP Server\n')); + console.error(chalk.blue(getGlyphs().info) + ' Use --mcp flag to start the MCP server'); + console.error('\nTo use with Claude Code, add to your MCP configuration:'); + console.error(chalk.dim(` +{ + "mcpServers": { + "codegraph": { + "command": "codegraph", + "args": ["serve", "--mcp"] + } + } +} +`)); + console.error('Available tools:'); + console.error(chalk.cyan(' codegraph_explore') + ' - Primary: source of the relevant symbols for any question'); + console.error(chalk.cyan(' codegraph_search') + ' - Search for code symbols'); + console.error(chalk.cyan(' codegraph_callers') + ' - Find callers of a symbol'); + console.error(chalk.cyan(' codegraph_callees') + ' - Find what a symbol calls'); + console.error(chalk.cyan(' codegraph_impact') + ' - Analyze impact of changes'); + console.error(chalk.cyan(' codegraph_node') + ' - Get symbol details'); + console.error(chalk.cyan(' codegraph_files') + ' - Get project file structure'); + console.error(chalk.cyan(' codegraph_status') + ' - Get index status'); + } + } catch (err) { + error(`Failed to start server: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph unlock [path] + */ +program + .command('unlock [path]') + .description('Remove a stale lock file that is blocking indexing') + .action(async (pathArg: string | undefined) => { + const projectPath = resolveProjectPath(pathArg); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + return; + } + + const lockPath = path.join(getCodeGraphDir(projectPath), 'codegraph.lock'); + + if (!fs.existsSync(lockPath)) { + info(`No lock file found ${getGlyphs().dash} nothing to do`); + return; + } + + fs.unlinkSync(lockPath); + success('Removed lock file. You can now run indexing again.'); + } catch (err) { + error(`Failed to remove lock: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph callers + * + * CLI parity with the MCP graph tools (codegraph_callers/callees/impact) so the + * traversal queries work in scripts, CI, and git hooks without a running MCP + * server. + */ +program + .command('callers ') + .description('Find all functions/methods that call a specific symbol') + .option('-p, --path ', 'Project path') + .option('-l, --limit ', 'Maximum results', '20') + .option('-j, --json', 'Output as JSON') + .action(async (symbol: string, options: { path?: string; limit?: string; json?: boolean }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const limit = parseInt(options.limit || '20', 10); + + const matches = cg.searchNodes(symbol, { limit: 50 }); + if (matches.length === 0) { + info(`Symbol "${symbol}" not found`); + cg.destroy(); + return; + } + + const seen = new Set(); + const allCallers: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; + + for (const match of matches) { + const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); + if (!exactMatch && matches.length > 1) continue; + for (const c of cg.getCallers(match.node.id)) { + if (!seen.has(c.node.id)) { + seen.add(c.node.id); + allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); + } + } + } + + // Fallback: if exact filter removed everything, use the top match + if (allCallers.length === 0 && matches[0]) { + for (const c of cg.getCallers(matches[0].node.id)) { + if (!seen.has(c.node.id)) { + seen.add(c.node.id); + allCallers.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); + } + } + } + + const limited = allCallers.slice(0, limit); + + if (options.json) { + console.log(JSON.stringify({ symbol, callers: limited }, null, 2)); + } else if (limited.length === 0) { + info(`No callers found for "${symbol}"`); + } else { + console.log(chalk.bold(`\nCallers of "${symbol}" (${limited.length}):\n`)); + for (const node of limited) { + const loc = node.startLine ? `:${node.startLine}` : ''; + console.log( + chalk.cyan(node.kind.padEnd(12)) + + chalk.white(node.name) + ); + console.log(chalk.dim(` ${node.filePath}${loc}`)); + console.log(); + } + } + + cg.destroy(); + } catch (err) { + error(`callers failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph callees + */ +program + .command('callees ') + .description('Find all functions/methods that a specific symbol calls') + .option('-p, --path ', 'Project path') + .option('-l, --limit ', 'Maximum results', '20') + .option('-j, --json', 'Output as JSON') + .action(async (symbol: string, options: { path?: string; limit?: string; json?: boolean }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const limit = parseInt(options.limit || '20', 10); + + const matches = cg.searchNodes(symbol, { limit: 50 }); + if (matches.length === 0) { + info(`Symbol "${symbol}" not found`); + cg.destroy(); + return; + } + + const seen = new Set(); + const allCallees: Array<{ name: string; kind: string; filePath: string; startLine?: number }> = []; + + for (const match of matches) { + const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); + if (!exactMatch && matches.length > 1) continue; + for (const c of cg.getCallees(match.node.id)) { + if (!seen.has(c.node.id)) { + seen.add(c.node.id); + allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); + } + } + } + + if (allCallees.length === 0 && matches[0]) { + for (const c of cg.getCallees(matches[0].node.id)) { + if (!seen.has(c.node.id)) { + seen.add(c.node.id); + allCallees.push({ name: c.node.name, kind: c.node.kind, filePath: c.node.filePath, startLine: c.node.startLine }); + } + } + } + + const limited = allCallees.slice(0, limit); + + if (options.json) { + console.log(JSON.stringify({ symbol, callees: limited }, null, 2)); + } else if (limited.length === 0) { + info(`No callees found for "${symbol}"`); + } else { + console.log(chalk.bold(`\nCallees of "${symbol}" (${limited.length}):\n`)); + for (const node of limited) { + const loc = node.startLine ? `:${node.startLine}` : ''; + console.log( + chalk.cyan(node.kind.padEnd(12)) + + chalk.white(node.name) + ); + console.log(chalk.dim(` ${node.filePath}${loc}`)); + console.log(); + } + } + + cg.destroy(); + } catch (err) { + error(`callees failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph impact + */ +program + .command('impact ') + .description('Analyze what code is affected by changing a symbol') + .option('-p, --path ', 'Project path') + .option('-d, --depth ', 'Traversal depth', '2') + .option('-j, --json', 'Output as JSON') + .action(async (symbol: string, options: { path?: string; depth?: string; json?: boolean }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const depth = Math.min(Math.max(parseInt(options.depth || '2', 10), 1), 10); + + const matches = cg.searchNodes(symbol, { limit: 50 }); + if (matches.length === 0) { + info(`Symbol "${symbol}" not found`); + cg.destroy(); + return; + } + + // Merge impact subgraphs across all exact-matching symbols + const mergedNodes = new Map(); + const seenEdges = new Set(); + let edgeCount = 0; + + for (const match of matches) { + const exactMatch = match.node.name === symbol || match.node.name.endsWith(`.${symbol}`) || match.node.name.endsWith(`::${symbol}`); + if (!exactMatch && matches.length > 1) continue; + const impact = cg.getImpactRadius(match.node.id, depth); + for (const [id, n] of impact.nodes) { + mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); + } + for (const e of impact.edges) { + const key = `${e.source}->${e.target}:${e.kind}`; + if (!seenEdges.has(key)) { + seenEdges.add(key); + edgeCount++; + } + } + } + + // Fallback to top match if exact filter removed everything + if (mergedNodes.size === 0 && matches[0]) { + const impact = cg.getImpactRadius(matches[0].node.id, depth); + for (const [id, n] of impact.nodes) { + mergedNodes.set(id, { name: n.name, kind: n.kind, filePath: n.filePath, startLine: n.startLine }); + } + edgeCount = impact.edges.length; + } + + if (options.json) { + console.log(JSON.stringify({ + symbol, + depth, + nodeCount: mergedNodes.size, + edgeCount, + affected: Array.from(mergedNodes.values()), + }, null, 2)); + } else if (mergedNodes.size === 0) { + info(`No affected symbols found for "${symbol}"`); + } else { + console.log(chalk.bold(`\nImpact of changing "${symbol}" — ${mergedNodes.size} affected symbols:\n`)); + + // Group by file + const byFile = new Map>(); + for (const node of mergedNodes.values()) { + const list = byFile.get(node.filePath) || []; + list.push({ name: node.name, kind: node.kind, startLine: node.startLine }); + byFile.set(node.filePath, list); + } + + for (const [file, nodes] of byFile) { + console.log(chalk.cyan(file)); + for (const node of nodes) { + const loc = node.startLine ? `:${node.startLine}` : ''; + console.log(` ${chalk.dim(node.kind.padEnd(12))}${node.name}${chalk.dim(loc)}`); + } + console.log(); + } + } + + cg.destroy(); + } catch (err) { + error(`impact failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph affected [files...] + * + * Find test files affected by the given source files. + * Traces dependency edges transitively to find test files that depend on changed code. + * + * Usage: + * git diff --name-only | codegraph affected --stdin + * codegraph affected src/lib/components/Editor.svelte src/routes/+page.svelte + */ +program + .command('affected [files...]') + .description('Find test files affected by changed source files') + .option('-p, --path ', 'Project path') + .option('--stdin', 'Read file list from stdin (one per line)') + .option('-d, --depth ', 'Max dependency traversal depth', '5') + .option('-f, --filter ', 'Custom glob filter for test files (e.g. "e2e/*.spec.ts")') + .option('-j, --json', 'Output as JSON') + .option('-q, --quiet', 'Only output file paths, no decoration') + .action(async (fileArgs: string[], options: { path?: string; stdin?: boolean; depth?: string; filter?: string; json?: boolean; quiet?: boolean }) => { + const projectPath = resolveProjectPath(options.path); + + try { + if (!isInitialized(projectPath)) { + error(`CodeGraph not initialized in ${projectPath}`); + process.exit(1); + } + + // Collect changed files from args or stdin + let changedFiles: string[] = [...(fileArgs || [])]; + + if (options.stdin) { + const stdinData = fs.readFileSync(0, 'utf-8'); + const stdinFiles = stdinData.split('\n').map(f => f.trim()).filter(Boolean); + changedFiles.push(...stdinFiles); + } + + // Normalize inputs to the project-relative, forward-slash form the index + // stores. Without this, `affected ./src/x.ts`, an absolute path (what a + // wrapping script often passes), or a Windows back-slash path silently + // matches nothing and reports 0 affected tests. (#825) + changedFiles = changedFiles + .map((f) => normalizeIndexPath(f, projectPath)) + .filter(Boolean); + + if (changedFiles.length === 0) { + if (!options.quiet) info('No files provided. Use file arguments or --stdin.'); + process.exit(0); + } + + const { default: CodeGraph } = await loadCodeGraph(); + const cg = await CodeGraph.open(projectPath); + const maxDepth = parseInt(options.depth || '5', 10); + + // Common test file patterns + const defaultTestPatterns = [ + /\.spec\./, + /\.test\./, + /\/__tests__\//, + /\/tests?\//, + /\/e2e\//, + /\/spec\//, + ]; + + // Custom filter pattern + let customFilter: RegExp | null = null; + if (options.filter) { + // Convert glob to regex: ** → .+, * → [^/]*, . → \. + const regex = options.filter + .replace(/[+[\]{}()^$|\\]/g, '\\$&') + .replace(/\./g, '\\.') + .replace(/\*\*/g, '.+') + .replace(/\*/g, '[^/]*'); + customFilter = new RegExp(regex); + } + + function isTestFile(filePath: string): boolean { + if (customFilter) return customFilter.test(filePath); + return defaultTestPatterns.some(p => p.test(filePath)); + } + + // BFS to find all transitive dependents of changed files, filtered to test files + const affectedTests = new Set(); + const allDependents = new Set(); + + for (const file of changedFiles) { + // If the changed file is itself a test file, include it + if (isTestFile(file)) { + affectedTests.add(file); + continue; + } + + // BFS through dependents + const queue: Array<{ file: string; depth: number }> = [{ file, depth: 0 }]; + const visited = new Set(); + visited.add(file); + + while (queue.length > 0) { + const current = queue.shift()!; + if (current.depth >= maxDepth) continue; + + const dependents = cg.getFileDependents(current.file); + for (const dep of dependents) { + if (visited.has(dep)) continue; + visited.add(dep); + allDependents.add(dep); + + if (isTestFile(dep)) { + affectedTests.add(dep); + } else { + queue.push({ file: dep, depth: current.depth + 1 }); + } + } + } + } + + const sortedTests = Array.from(affectedTests).sort(); + + // Output + if (options.json) { + console.log(JSON.stringify({ + changedFiles, + affectedTests: sortedTests, + totalDependentsTraversed: allDependents.size, + }, null, 2)); + } else if (options.quiet) { + for (const t of sortedTests) console.log(t); + } else { + if (sortedTests.length === 0) { + info('No test files affected by the changed files.'); + } else { + console.log(chalk.bold(`\nAffected test files (${sortedTests.length}):\n`)); + for (const t of sortedTests) { + console.log(' ' + chalk.cyan(t)); + } + console.log(); + } + } + + cg.destroy(); + } catch (err) { + error(`Affected analysis failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + }); + +/** + * codegraph install + */ +program + .command('install') + .description('Install codegraph MCP server into one or more agents (Claude Code, Cursor, Codex CLI, opencode, Hermes Agent)') + .option('-t, --target ', 'Target agent(s): comma-separated ids, or "auto"|"all"|"none". Default: prompt') + .option('-l, --location ', 'Install location: "global" or "local". Default: prompt') + .option('-y, --yes', 'Non-interactive: defaults to --location=global --target=auto, auto-allow on') + .option('--no-permissions', 'Skip writing the auto-allow permissions list (Claude Code only)') + .option('--print-config ', 'Print MCP config snippet for the named agent and exit (no file writes)') + .option('--refresh', 'Rewrite what previous installs configured, for already-configured agents only (never adds new ones). Run automatically by `codegraph upgrade`') + .action(async (opts: { + target?: string; + location?: string; + yes?: boolean; + permissions?: boolean; + printConfig?: string; + refresh?: boolean; + }) => { + if (opts.printConfig) { + const { getTarget, listTargetIds } = await import('../installer/targets/registry'); + const target = getTarget(opts.printConfig); + if (!target) { + const known = listTargetIds().join(', '); + error(`Unknown target "${opts.printConfig}". Known: ${known}.`); + process.exit(1); + } + const loc = (opts.location === 'local' ? 'local' : 'global') as 'global' | 'local'; + process.stdout.write(target.printConfig(loc)); + return; + } + + // --refresh: non-interactive sweep that re-writes what previous + // installs configured (instructions section, MCP entry, legacy-hook + // cleanups) for already-configured agents, so those surfaces match + // THIS binary's templates. Skips everything else — never a first + // install, never touches permissions or the prompt hook. Sweeps both + // locations unless --location narrows it. + if (opts.refresh) { + const { refreshTargets } = await import('../installer'); + const { ALL_TARGETS } = await import('../installer/targets/registry'); + if (opts.location && opts.location !== 'global' && opts.location !== 'local') { + error(`--location must be "global" or "local" (got "${opts.location}").`); + process.exit(1); + } + const locs: Array<'global' | 'local'> = opts.location + ? [opts.location as 'global' | 'local'] + : ['global', 'local']; + let changed = 0; + for (const loc of locs) { + for (const report of refreshTargets(ALL_TARGETS, loc)) { + for (const p of report.changedPaths) { + changed += 1; + console.log(` ${report.displayName}: refreshed ${p}`); + } + } + } + if (changed === 0) { + console.log('All configured agent surfaces are already current.'); + } + return; + } + + const { runInstallerWithOptions } = await import('../installer'); + if (opts.location && opts.location !== 'global' && opts.location !== 'local') { + error(`--location must be "global" or "local" (got "${opts.location}").`); + process.exit(1); + } + try { + // Commander's `--no-permissions` makes `opts.permissions === false`; + // omitting the flag leaves it `true` (the positive-form default). + // We MUST treat the default-true as "user did not override — let + // the orchestrator prompt" and only forward an explicit `false` + // (or `true` when --yes implies it). Otherwise the auto-allow + // prompt is silently skipped on every interactive run. + const explicitNoPermissions = opts.permissions === false; + const autoAllow: boolean | undefined = explicitNoPermissions + ? false + : opts.yes + ? true + : undefined; + + await runInstallerWithOptions({ + target: opts.target, + location: opts.location as 'global' | 'local' | undefined, + autoAllow, + yes: opts.yes, + }); + } catch (err) { + error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } + }); + +/** + * codegraph uninstall + * + * Inverse of `install`. Removes the codegraph MCP server entry, + * instructions block, and permissions from every agent (or a + * `--target` subset). Prompts global-vs-local when not given. Does NOT + * delete the `.codegraph/` index — that's `codegraph uninit`. + */ +program + .command('uninstall') + .description('Remove codegraph from your agents (Claude Code, Cursor, Codex CLI, opencode, Hermes Agent)') + .option('-t, --target ', 'Target agent(s): comma-separated ids, or "all". Default: all') + .option('-l, --location ', 'Uninstall location: "global" or "local". Default: prompt') + .option('-y, --yes', 'Non-interactive: defaults to --location=global --target=all') + .option('--keep-cli', 'Remove agent configs only — leave the codegraph CLI installed') + .action(async (opts: { + target?: string; + location?: string; + yes?: boolean; + keepCli?: boolean; + }) => { + const { runUninstaller } = await import('../installer'); + if (opts.location && opts.location !== 'global' && opts.location !== 'local') { + error(`--location must be "global" or "local" (got "${opts.location}").`); + process.exit(1); + } + try { + await runUninstaller({ + target: opts.target, + location: opts.location as 'global' | 'local' | undefined, + yes: opts.yes, + keepCli: opts.keepCli, + cliFilename: __filename, + }); + } catch (err) { + error(err instanceof Error ? err.message : String(err)); + process.exit(1); + } + }); + +/** + * codegraph telemetry [on|off|status] + */ +program + .command('telemetry [action]') + .description('Show or change anonymous usage telemetry (status, on, off)') + .action((action?: string) => { + const t = getTelemetry(); + + if (action === 'on' || action === 'off') { + t.setEnabled(action === 'on', 'cli'); + if (action === 'on') { + success('Telemetry enabled — anonymous usage stats only (no code, paths, or names).'); + } else { + success('Telemetry disabled. Buffered, unsent data was deleted.'); + } + const effective = t.getStatus(); + if (effective.decidedBy === 'DO_NOT_TRACK' || effective.decidedBy === 'CODEGRAPH_TELEMETRY') { + warn( + `The ${effective.decidedBy} environment variable overrides this choice — ` + + `effective state right now: ${effective.enabled ? 'enabled' : 'disabled'}.` + ); + } + return; + } + + if (action !== undefined && action !== 'status') { + error(`Unknown action: ${action} (expected status, on, or off)`); + process.exit(1); + } + + const s = t.getStatus(); + const decidedBy: Record = { + DO_NOT_TRACK: 'DO_NOT_TRACK environment variable', + CODEGRAPH_TELEMETRY: 'CODEGRAPH_TELEMETRY environment variable', + config: 'your saved choice', + default: 'default', + }; + console.log(`\nTelemetry: ${s.enabled ? chalk.green('enabled') : chalk.yellow('disabled')} ${chalk.dim(`(${decidedBy[s.decidedBy]})`)}`); + console.log(`Machine ID: ${s.machineId ?? chalk.dim('(random UUID, created on first use)')}`); + console.log(`Config: ${s.configPath}`); + console.log(chalk.dim(`\nExactly what is collected (and never collected): ${TELEMETRY_DOCS}\n`)); + }); + +/** + * codegraph upgrade [version] + * + * Self-update, however CodeGraph was installed (bundle via install.sh/.ps1, + * npm-global, npx, or a source checkout). See ../upgrade for the detection and + * per-method upgrade logic. + */ +program + .command('upgrade [version]') + .description('Update CodeGraph to the latest release (or a specific version)') + .option('--check', 'Check whether an update is available without installing') + .option('-f, --force', 'Reinstall even if already on the target version') + .action(async (versionArg: string | undefined, options: { check?: boolean; force?: boolean }) => { + const up = await import('../upgrade'); + const method = up.detectInstallMethod({ + filename: __filename, + platform: process.platform, + cwd: process.cwd(), + }); + const pin = versionArg || process.env.CODEGRAPH_VERSION || undefined; + const code = await up.runUpgrade( + { version: pin, check: options.check, force: options.force }, + { + currentVersion: packageJson.version, + method, + resolveLatest: () => up.resolveLatestVersion(), + run: up.defaultRun, + capture: up.defaultCapture, + hasCommand: up.hasCommand, + log: (m: string) => console.log(m), + warn: (m: string) => warn(m), + error: (m: string) => error(m), + platform: process.platform, + } + ); + process.exit(code); + }); + +/** + * codegraph version + * + * The bare-noun form of `--version`. commander already provides `--version` + * and `-V`, and the `-v` / `-version` spellings are intercepted before parse + * (see top of main). This subcommand makes `codegraph version` work and lists + * the version affordance in `codegraph --help`. + */ +program + .command('version') + .description('Print the installed CodeGraph version (also: -v, --version)') + .action(() => { + console.log(packageJson.version); + }); + +// Parse and run +program.parse(); + +} // end main() diff --git a/src/bin/command-supervision.ts b/src/bin/command-supervision.ts new file mode 100644 index 0000000..6031a49 --- /dev/null +++ b/src/bin/command-supervision.ts @@ -0,0 +1,96 @@ +/** + * Process supervision for long-running CLI commands (`index` / `init --index`). + * + * Indexing a large repo can run for a while on the main thread, and #999 + * surfaced two ways that goes wrong when nothing is watching it: + * + * 1. **Orphaned worker.** `index` runs in a child re-exec'd with + * `--liftoff-only` (the WASM-flag relaunch). Its parent blocks in + * `spawnSync`, so when the parent shim is killed it cannot forward the + * signal — the child keeps running, now orphaned, pinning a core. The PPID + * watchdog (#277) notices the parent/host went away and exits the child. + * 2. **Wedged indexer.** The `#850` main-thread liveness watchdog — which + * SIGKILLs a process whose event loop stops turning — was wired only into + * the MCP `serve` path, so a wedged `index`/`init` was never auto-killed. + * + * Both reuse the exact mechanisms `serve` already uses; this just makes them + * available to a one-shot command. Best-effort and self-disabling: a missing + * watchdog never blocks the command from running. Both honour the same env + * switches as `serve` (`CODEGRAPH_NO_WATCHDOG`, `CODEGRAPH_PPID_POLL_MS=0`). + * + * Unlike the daemon — whose main thread only does fast, bounded work — the + * `index`/`init` path runs reference resolution and dynamic-edge synthesis + * SYNCHRONOUSLY on this thread, and on a large repo that is legitimately many + * seconds of work. So those spans yield cooperatively to the event loop + * (`src/resolution/cooperative-yield.ts`) to keep the heartbeat alive; without + * that the watchdog would SIGKILL a valid, in-progress index (#1091). The + * distinction it must preserve — kill a TRUE wedge, spare slow-but-progressing + * work — is exactly what cooperative yielding buys: a genuinely stuck span never + * reaches its next yield, so it still trips the timeout. + */ +import { installMainThreadWatchdog, WatchdogOptions } from '../mcp/liveness-watchdog'; +import { supervisionLostReason, parsePpidPollMs, parseHostPpid } from '../mcp/ppid-watchdog'; +import { isProcessAlive } from '../mcp/daemon-registry'; +import { EARLY_PPID } from '../mcp/early-ppid'; +import { HOST_PPID_ENV } from '../extraction/wasm-runtime-flags'; + +export interface CommandSupervision { + /** Tear down both watchdogs. Idempotent; call when the command finishes. */ + stop(): void; +} + +/** + * Install the liveness + PPID watchdogs for the duration of a CLI command. + * `label` is used in the shutdown notice (e.g. `"index"`). Returns a handle + * whose `stop()` must be called when the command completes so neither watchdog + * outlives it. + * + * Pass `watchdog.progressPaths` (the project's SQLite DB + `-wal`) so the + * liveness watchdog can tell a slow-but-progressing store on degraded storage + * (files advancing) from a true wedge (they aren't) — one long synchronous + * SQLite statement on a 150-IOPS disk otherwise gets a healthy index + * SIGKILLed (#1231). + */ +export function installCommandSupervision(label: string, watchdog: WatchdogOptions = {}): CommandSupervision { + // Liveness watchdog: a separate process that SIGKILLs us if our event loop + // stops turning for too long (a wedged synchronous loop). Self-disables on + // CODEGRAPH_NO_WATCHDOG. + const liveness = installMainThreadWatchdog(watchdog); + + // PPID watchdog: detect that the parent (or the host threaded past the + // relaunch shim) died and we've been orphaned, then exit instead of leaking. + // Baseline from the CLI entry's earliest-possible capture — reading + // process.ppid here would miss a launcher killed during startup (#1185). + const originalPpid = EARLY_PPID; + const hostPpid = parseHostPpid(process.env[HOST_PPID_ENV]); + const pollMs = parsePpidPollMs(process.env.CODEGRAPH_PPID_POLL_MS); + let ppidTimer: ReturnType | null = null; + if (pollMs > 0) { + ppidTimer = setInterval(() => { + const reason = supervisionLostReason({ + originalPpid, + currentPpid: process.ppid, + hostPpid, + isAlive: isProcessAlive, + }); + if (reason) { + try { + process.stderr.write(`[CodeGraph ${label}] Parent process exited (${reason}); aborting.\n`); + } catch { /* stderr gone with the parent — exit anyway */ } + process.exit(1); + } + }, pollMs); + // Never let the watchdog itself keep the process alive past its real work. + ppidTimer.unref(); + } + + let stopped = false; + return { + stop(): void { + if (stopped) return; + stopped = true; + if (ppidTimer) clearInterval(ppidTimer); + liveness?.stop(); + }, + }; +} diff --git a/src/bin/fatal-handler.ts b/src/bin/fatal-handler.ts new file mode 100644 index 0000000..689e7e5 --- /dev/null +++ b/src/bin/fatal-handler.ts @@ -0,0 +1,93 @@ +/** + * Last-resort handlers for uncaught exceptions and unhandled rejections. + * + * Reaching one of these means a fault escaped every boundary (per-request + * try/catch in the MCP transport, the file watcher's own `'error'` handlers, + * telemetry's fail-silent contract) — i.e. the process is in an undefined + * state. Node's default in that case is to print and exit non-zero. The CLI + * previously OVERRODE that to "log the error and keep running", which is the + * bug behind two production incidents: + * + * - #799 — a stdin socket `'error'` escalated here; the server logged it and + * kept running, orphaning the detached MCP daemon and (on Linux) spinning a + * POLLHUP fd at 100% CPU. Fixed for that one trigger by treating stdin + * failure as shutdown (`src/mcp/stdin-teardown.ts`). + * - #850 — a *different* uncaught exception hit the same handler. Logging it + * forced V8 to lazily format the Error's `.stack`, which entered a + * non-terminating source-position walk and pinned a core. Because the + * handler kept the process alive, the detached daemon was left wedged: its + * PPID watchdog and idle-timer (both `setInterval`s) could no longer fire, + * and nothing respawned it — unrecoverable without a manual `kill`. + * + * The fix restores the safe default: log a BOUNDED, hang-proof line, then exit + * non-zero so a fresh daemon starts on the next connection. + * + * Two properties are load-bearing and covered by tests: + * 1. {@link describeFatal} never reads `error.stack` and never hands the raw + * Error to `console.*`. The lazy stack getter is exactly the step that can + * wedge (#850); since it would run *inside* this handler, touching it could + * block the very `exit()` below. Name + message are plain string + * properties and are always safe. + * 2. We write synchronously to fd 2 and then exit, so the message is flushed + * even though `process.exit()` doesn't drain async streams. + */ +import * as fs from 'fs'; + +/** + * Render an uncaught value for the last-resort log WITHOUT triggering stack + * formatting. Pure and total — never throws, never touches `.stack`. + */ +export function describeFatal(value: unknown): string { + if (value instanceof Error) { + const name = typeof value.name === 'string' && value.name ? value.name : 'Error'; + // `message` is a plain own/proto string property — reading it does NOT + // format the stack (which is what can loop forever, #850). + const message = typeof value.message === 'string' ? value.message : ''; + return message ? `${name}: ${message}` : name; + } + try { + return String(value); + } catch { + // e.g. an object with a throwing `toString` / `Symbol.toPrimitive`. + return ''; + } +} + +/** Best-effort synchronous stderr write that can never keep a doomed process alive. */ +function writeStderr(line: string): void { + try { + fs.writeSync(2, line); + } catch { + /* stderr closed/gone — nothing more we can safely do */ + } +} + +/** Injectable seams so the wiring is testable without registering real handlers. */ +export interface FatalHandlerDeps { + /** Event target to attach to. Defaults to `process`. */ + target?: NodeJS.EventEmitter; + /** How to terminate. Defaults to `process.exit`. */ + exit?: (code: number) => void; + /** How to emit the bounded line. Defaults to a synchronous fd-2 write. */ + write?: (line: string) => void; +} + +/** + * Install the uncaught-exception / unhandled-rejection handlers. Both log a + * bounded line and then exit non-zero (Node's default fatal semantics). + */ +export function installFatalHandlers(deps: FatalHandlerDeps = {}): void { + const target = deps.target ?? process; + const exit = deps.exit ?? ((code: number) => process.exit(code)); + const write = deps.write ?? writeStderr; + + target.on('uncaughtException', (error: unknown) => { + write(`[CodeGraph] Uncaught exception: ${describeFatal(error)}\n`); + exit(1); + }); + + target.on('unhandledRejection', (reason: unknown) => { + write(`[CodeGraph] Unhandled rejection: ${describeFatal(reason)}\n`); + exit(1); + }); +} diff --git a/src/bin/node-version-check.ts b/src/bin/node-version-check.ts new file mode 100644 index 0000000..cea0a43 --- /dev/null +++ b/src/bin/node-version-check.ts @@ -0,0 +1,76 @@ +/** + * Node.js version compatibility check. + * + * Node 25.x has a V8 turboshaft WASM JIT Zone allocator bug that + * reliably crashes CodeGraph with `Fatal process out of memory: Zone` + * during tree-sitter grammar compilation. This module owns the + * user-facing banner shown before exit. Kept side-effect-free so it's + * safe to import from tests without triggering CLI bootstrap. + */ + +/** + * Build the bordered banner shown when CodeGraph detects an + * unsupported Node.js major version (currently 25+). Pinned via unit + * test so the recovery commands and override instructions can't be + * silently stripped by future edits. + * + * Uses ASCII glyphs to stay readable on Windows OEM-codepage consoles + * (see ../ui/glyphs.ts for the rationale). + */ +export function buildNode25BlockBanner(nodeVersion: string): string { + const sep = '-'.repeat(72); + return [ + sep, + `[CodeGraph] Unsupported Node.js version: ${nodeVersion}`, + sep, + 'Node.js 25.x has a V8 WASM JIT (turboshaft) Zone allocator bug that', + 'crashes with `Fatal process out of memory: Zone` when CodeGraph', + 'compiles tree-sitter grammars. CodeGraph WILL crash on this Node', + 'version mid-indexing. See https://github.com/colbymchenry/codegraph/issues/81', + '', + 'Fix: install Node.js 22 LTS:', + ' nvm install 22 && nvm use 22 # nvm', + ' brew install node@22 && brew link --overwrite --force node@22 # Homebrew', + '', + 'To override (NOT recommended - you will likely OOM):', + ' CODEGRAPH_ALLOW_UNSAFE_NODE=1 codegraph ...', + sep, + ].join('\n'); +} + +/** + * Lowest supported Node.js major version. Matches the `engines` floor in + * package.json. Below this, CodeGraph relies on language features / native APIs + * that aren't present, and the combination is untested. `engines` alone only + * *warns* on install (unless the user set `engine-strict`), so the CLI bootstrap + * also hard-blocks here to actually enforce the floor. + */ +export const MIN_NODE_MAJOR = 20; + +/** + * Build the bordered banner shown when CodeGraph detects a Node.js major below + * {@link MIN_NODE_MAJOR}. Pinned via unit test so the recovery commands and the + * override env var can't be silently stripped by future edits. + * + * Uses ASCII glyphs to stay readable on Windows OEM-codepage consoles + * (see ../ui/glyphs.ts for the rationale). + */ +export function buildNodeTooOldBanner(nodeVersion: string): string { + const sep = '-'.repeat(72); + return [ + sep, + `[CodeGraph] Unsupported Node.js version: ${nodeVersion}`, + sep, + `CodeGraph requires Node.js ${MIN_NODE_MAJOR} or newer. Older versions lack`, + 'language features and native APIs CodeGraph depends on, and are not', + 'tested or supported.', + '', + 'Fix: install Node.js 22 LTS:', + ' nvm install 22 && nvm use 22 # nvm', + ' brew install node@22 && brew link --overwrite --force node@22 # Homebrew', + '', + 'To override (NOT recommended - unsupported):', + ' CODEGRAPH_ALLOW_UNSAFE_NODE=1 codegraph ...', + sep, + ].join('\n'); +} diff --git a/src/bin/uninstall.ts b/src/bin/uninstall.ts new file mode 100644 index 0000000..a168d80 --- /dev/null +++ b/src/bin/uninstall.ts @@ -0,0 +1,34 @@ +#!/usr/bin/env node +/** + * CodeGraph preuninstall cleanup script + * + * Runs automatically when `npm uninstall -g @colbymchenry/codegraph` + * is called. Loops over every known agent target's `uninstall(loc)` + * for the global location only — local-location entries live inside + * project working trees and aren't ours to nuke at npm-uninstall + * time. + * + * This script must never throw — a failed cleanup must not block + * uninstall. + */ + +try { + // Lazy require so any module-level error in the registry can't + // bubble out and abort the npm uninstall. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { ALL_TARGETS } = require('../installer/targets/registry') as + typeof import('../installer/targets/registry'); + + for (const target of ALL_TARGETS) { + if (!target.supportsLocation('global')) continue; + try { + target.uninstall('global'); + } catch { + // Each target is independently safe-to-skip; per-target failure + // must not stop the loop. + } + } +} catch { + // If the registry itself can't be loaded (e.g. partial install), + // we silently skip cleanup. Uninstall still completes. +} diff --git a/src/context/formatter.ts b/src/context/formatter.ts new file mode 100644 index 0000000..748d172 --- /dev/null +++ b/src/context/formatter.ts @@ -0,0 +1,290 @@ +/** + * Context Formatter + * + * Formats TaskContext as markdown or JSON for consumption by Claude. + */ + +import { Node, Edge, TaskContext, Subgraph } from '../types'; +import { isGeneratedFile } from '../extraction/generated-detection'; + +/** + * Format context as markdown + * + * Creates a compact markdown document optimized for Claude with minimal context usage: + * - Brief summary + * - Entry points with locations + * - Code blocks only for key symbols + */ +export function formatContextAsMarkdown(context: TaskContext): string { + const lines: string[] = []; + + // Header with query + lines.push('## Code Context\n'); + lines.push(`**Query:** ${context.query}\n`); + + // Entry points - compact format. Re-sort so generated files (.pb.go, + // .pulsar.go, mocks, …) rank LAST — a flow query should lead with the + // hand-written implementation, not protobuf scaffolding. + const orderedEntries = [...context.entryPoints].sort((a, b) => { + const aGen = isGeneratedFile(a.filePath) ? 1 : 0; + const bGen = isGeneratedFile(b.filePath) ? 1 : 0; + return aGen - bGen; + }); + if (orderedEntries.length > 0) { + lines.push('### Entry Points\n'); + for (const node of orderedEntries) { + const location = node.startLine ? `:${node.startLine}` : ''; + lines.push(`- **${node.name}** (${node.kind}) - ${node.filePath}${location}`); + if (node.signature) { + lines.push(` \`${node.signature}\``); + } + } + lines.push(''); + } + + // Related symbols - compact list (skip verbose structure tree). Drop nodes + // in generated source files (`.pb.go` / `.pulsar.go` / mocks / …) — agents + // chasing a flow never want to land on protobuf scaffolding (cosmos-Q3 used + // to list `gov.pulsar.go::GetExpeditedThreshold` and `1.pulsar.go::Get` in + // Related Symbols, pure noise that displaced real-flow entries). + const otherSymbols = Array.from(context.subgraph.nodes.values()) + .filter(n => !context.entryPoints.some(e => e.id === n.id)) + .filter(n => !isGeneratedFile(n.filePath)) + .slice(0, 10); // Limit to 10 related symbols + + if (otherSymbols.length > 0) { + lines.push('### Related Symbols\n'); + const byFile = new Map(); + for (const node of otherSymbols) { + const existing = byFile.get(node.filePath) || []; + existing.push(node); + byFile.set(node.filePath, existing); + } + + for (const [file, nodes] of byFile) { + const nodeList = nodes.map(n => `${n.name}:${n.startLine}`).join(', '); + lines.push(`- ${file}: ${nodeList}`); + } + lines.push(''); + } + + // Code blocks - only for key entry points. Re-sort so non-generated blocks + // show first (consistent with Entry Points reordering above). + if (context.codeBlocks.length > 0) { + const orderedBlocks = [...context.codeBlocks].sort((a, b) => { + const aGen = isGeneratedFile(a.filePath) ? 1 : 0; + const bGen = isGeneratedFile(b.filePath) ? 1 : 0; + return aGen - bGen; + }); + lines.push('### Code\n'); + for (const block of orderedBlocks) { + const nodeName = block.node?.name ?? 'Unknown'; + lines.push(`#### ${nodeName} (${block.filePath}:${block.startLine})\n`); + lines.push('```' + block.language); + lines.push(block.content); + lines.push('```\n'); + } + } + + return lines.join('\n'); +} + +/** + * Format context as JSON + * + * Returns a structured JSON representation suitable for programmatic use. + */ +export function formatContextAsJson(context: TaskContext): string { + // Convert Map to array for JSON serialization + const serializable = { + query: context.query, + summary: context.summary, + entryPoints: context.entryPoints.map(serializeNode), + nodes: Array.from(context.subgraph.nodes.values()).map(serializeNode), + edges: context.subgraph.edges.map(serializeEdge), + codeBlocks: context.codeBlocks.map((block) => ({ + filePath: block.filePath, + startLine: block.startLine, + endLine: block.endLine, + language: block.language, + content: block.content, + nodeName: block.node?.name, + nodeKind: block.node?.kind, + })), + relatedFiles: context.relatedFiles, + stats: context.stats, + }; + + return JSON.stringify(serializable, null, 2); +} + +/** + * Format a subgraph as an ASCII tree structure + */ +export function formatSubgraphTree(subgraph: Subgraph, entryPoints: Node[]): string { + const lines: string[] = []; + const printed = new Set(); + + // Build adjacency list for outgoing edges + const outgoing = new Map(); + for (const edge of subgraph.edges) { + const existing = outgoing.get(edge.source) ?? []; + existing.push(edge); + outgoing.set(edge.source, existing); + } + + // Print each entry point as a tree root + for (const entry of entryPoints) { + formatNodeTree(entry, subgraph, outgoing, printed, lines, 0, ''); + lines.push(''); // Blank line between trees + } + + // Print any remaining nodes not reached from entry points + const remaining: Node[] = []; + for (const node of subgraph.nodes.values()) { + if (!printed.has(node.id)) { + remaining.push(node); + } + } + + if (remaining.length > 0 && remaining.length <= 10) { + lines.push('Other relevant symbols:'); + for (const node of remaining) { + const location = node.startLine ? `:${node.startLine}` : ''; + lines.push(` ${node.kind}: ${node.name} (${node.filePath}${location})`); + } + } else if (remaining.length > 10) { + lines.push(`... and ${remaining.length} more related symbols`); + } + + return lines.join('\n').trim(); +} + +/** + * Format a single node and its relationships + */ +function formatNodeTree( + node: Node, + subgraph: Subgraph, + outgoing: Map, + printed: Set, + lines: string[], + depth: number, + prefix: string +): void { + if (printed.has(node.id)) { + return; + } + printed.add(node.id); + + // Node header + const location = node.startLine ? `:${node.startLine}` : ''; + const signature = node.signature ? ` - ${truncate(node.signature, 50)}` : ''; + lines.push(`${prefix}${node.kind}: ${node.name} (${node.filePath}${location})${signature}`); + + // Outgoing edges + const edges = outgoing.get(node.id) ?? []; + const significantEdges = edges.filter((e) => + ['calls', 'extends', 'implements', 'imports', 'references'].includes(e.kind) + ); + + // Group by kind + const edgesByKind = new Map(); + for (const edge of significantEdges) { + const existing = edgesByKind.get(edge.kind) ?? []; + existing.push(edge); + edgesByKind.set(edge.kind, existing); + } + + // Print edges grouped by kind + const newPrefix = prefix + ' '; + for (const [kind, kindEdges] of edgesByKind) { + if (kindEdges.length > 3) { + // Summarize if too many + const names = kindEdges + .slice(0, 3) + .map((e) => { + const target = subgraph.nodes.get(e.target); + return target?.name ?? 'unknown'; + }) + .join(', '); + lines.push(`${newPrefix}├── ${kind}: ${names} and ${kindEdges.length - 3} more`); + } else { + for (let i = 0; i < kindEdges.length; i++) { + const edge = kindEdges[i]!; + const target = subgraph.nodes.get(edge.target); + const targetName = target?.name ?? 'unknown'; + const connector = i === kindEdges.length - 1 ? '└──' : '├──'; + lines.push(`${newPrefix}${connector} ${kind} → ${targetName}`); + } + } + } + + // Recurse for directly connected nodes (limited depth) + if (depth < 1) { + for (const edge of significantEdges.slice(0, 3)) { + const target = subgraph.nodes.get(edge.target); + if (target && !printed.has(target.id)) { + formatNodeTree(target, subgraph, outgoing, printed, lines, depth + 1, newPrefix); + } + } + } +} + +/** + * Serialize a node for JSON output + */ +function serializeNode(node: Node): Record { + return { + id: node.id, + kind: node.kind, + name: node.name, + qualifiedName: node.qualifiedName, + filePath: node.filePath, + language: node.language, + startLine: node.startLine, + endLine: node.endLine, + signature: node.signature, + docstring: node.docstring, + visibility: node.visibility, + isExported: node.isExported, + isAsync: node.isAsync, + isStatic: node.isStatic, + }; +} + +/** + * Serialize an edge for JSON output + */ +function serializeEdge(edge: Edge): Record { + return { + source: edge.source, + target: edge.target, + kind: edge.kind, + line: edge.line, + column: edge.column, + }; +} + +/** + * Truncate a string with ellipsis + */ +function truncate(str: string, maxLength: number): string { + if (str.length <= maxLength) { + return str; + } + return str.slice(0, maxLength - 3) + '...'; +} + +/** + * Format bytes as human-readable string + */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) { + return `${bytes} bytes`; + } else if (bytes < 1024 * 1024) { + return `${(bytes / 1024).toFixed(1)} KB`; + } else { + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + } +} diff --git a/src/context/index.ts b/src/context/index.ts new file mode 100644 index 0000000..68123c2 --- /dev/null +++ b/src/context/index.ts @@ -0,0 +1,1372 @@ +/** + * Context Builder + * + * Builds rich context for tasks by combining FTS search with graph traversal. + * Outputs structured context ready to inject into Claude. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { + Node, + Edge, + NodeKind, + EdgeKind, + Subgraph, + CodeBlock, + TaskContext, + TaskInput, + BuildContextOptions, + FindRelevantContextOptions, + SearchResult, +} from '../types'; +import { QueryBuilder } from '../db/queries'; +import { GraphTraverser } from '../graph'; +import { formatContextAsMarkdown, formatContextAsJson } from './formatter'; +import { logDebug } from '../errors'; +import { validatePathWithinRoot, isConfigLeafNode } from '../utils'; +import { isTestFile, extractSearchTerms, scorePathRelevance, getStemVariants, isDistinctiveIdentifier } from '../search/query-utils'; +import { LOW_CONFIDENCE_MARKER } from './markers'; + +/** + * Extract likely symbol names from a natural language query + * + * Identifies potential code symbols using patterns: + * - CamelCase: UserService, signInWithGoogle + * - snake_case: user_service, sign_in + * - SCREAMING_SNAKE: MAX_RETRIES + * - dot.notation: app.isPackaged (extracts both sides) + * - Single words that look like identifiers (no spaces, not common English words) + * + * @param query - Natural language query + * @returns Array of potential symbol names + */ +function extractSymbolsFromQuery(query: string): string[] { + const symbols = new Set(); + + // Extract CamelCase identifiers (2+ chars, starts with letter) + const camelCasePattern = /\b([A-Z][a-z]+(?:[A-Z][a-z]*)*|[a-z]+(?:[A-Z][a-z]*)+)\b/g; + let match; + while ((match = camelCasePattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 2) { + symbols.add(match[1]); + } + } + + // Extract snake_case identifiers + const snakeCasePattern = /\b([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b/gi; + while ((match = snakeCasePattern.exec(query)) !== null) { + if (match[1] && match[1].length >= 3) { + symbols.add(match[1]); + } + } + + // Extract SCREAMING_SNAKE_CASE + const screamingPattern = /\b([A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)\b/g; + while ((match = screamingPattern.exec(query)) !== null) { + if (match[1]) { + symbols.add(match[1]); + } + } + + // Extract ALL_CAPS acronyms (2+ chars, e.g., REST, HTTP, LRU, API) + const acronymPattern = /\b([A-Z]{2,})\b/g; + while ((match = acronymPattern.exec(query)) !== null) { + if (match[1]) { + symbols.add(match[1]); + } + } + + // Extract dot.notation and split into parts (e.g., "app.isPackaged" -> ["app", "isPackaged"]) + const dotPattern = /\b([a-zA-Z][a-zA-Z0-9]*(?:\.[a-zA-Z][a-zA-Z0-9]*)+)\b/g; + while ((match = dotPattern.exec(query)) !== null) { + if (match[1]) { + // Add both the full path and individual parts + symbols.add(match[1]); + const parts = match[1].split('.'); + for (const part of parts) { + if (part.length >= 2) { + symbols.add(part); + } + } + } + } + + // Extract plain lowercase identifiers (3+ chars, not already matched) + // Catches symbol names like "undo", "redo", "history", "render", "parse" + const lowercasePattern = /\b([a-z][a-z0-9]{2,})\b/g; + while ((match = lowercasePattern.exec(query)) !== null) { + if (match[1]) { + symbols.add(match[1]); + } + } + + // Filter out common English words that aren't likely symbol names + const commonWords = new Set([ + 'the', 'and', 'for', 'with', 'from', 'this', 'that', 'have', 'been', + 'will', 'would', 'could', 'should', 'does', 'done', 'make', 'made', + 'use', 'used', 'using', 'work', 'works', 'find', 'found', 'show', + 'call', 'called', 'calling', 'get', 'set', 'add', 'all', 'any', + 'how', 'what', 'when', 'where', 'which', 'who', 'why', + 'not', 'but', 'are', 'was', 'were', 'has', 'had', 'its', + 'can', 'did', 'may', 'also', 'into', 'than', 'then', 'them', + 'each', 'other', 'some', 'such', 'only', 'same', 'about', + 'after', 'before', 'between', 'through', 'during', 'without', + 'again', 'further', 'once', 'here', 'there', 'both', 'just', + 'more', 'most', 'very', 'being', 'having', 'doing', + 'system', 'need', 'needs', 'want', 'wants', 'like', 'look', + 'change', 'changes', 'changed', 'changing', + // Common English nouns/verbs that match thousands of unrelated code symbols + 'layer', 'handle', 'handles', 'handling', 'incoming', 'outgoing', + 'data', 'flow', 'flows', 'level', 'levels', 'request', 'requests', + 'response', 'responses', 'implement', 'implements', 'implementation', + 'interface', 'interfaces', 'class', 'classes', 'method', 'methods', + 'trigger', 'triggers', 'affected', 'affect', 'affects', + 'else', 'code', 'failing', 'failed', 'silently', 'decide', 'decides', + 'return', 'returns', 'returned', 'take', 'takes', 'taken', + 'check', 'checks', 'checked', 'create', 'creates', 'created', + 'read', 'reads', 'write', 'writes', 'written', + 'start', 'starts', 'stop', 'stops', 'run', 'runs', 'running', + ]); + + return Array.from(symbols).filter(s => !commonWords.has(s.toLowerCase())); +} + +/** + * Default options for context building + * + * Tuned for minimal context usage while still providing useful results: + * - Fewer nodes and code blocks by default + * - Smaller code block size limit + * - Shallower traversal + */ +const DEFAULT_BUILD_OPTIONS: Required = { + maxNodes: 20, // Reduced from 50 - most tasks don't need 50 symbols + maxCodeBlocks: 5, // Reduced from 10 - only show most relevant code + maxCodeBlockSize: 1500, // Reduced from 2000 + includeCode: true, + format: 'markdown', + searchLimit: 3, // Reduced from 5 - fewer entry points + traversalDepth: 1, // Reduced from 2 - shallower graph expansion + minScore: 0.3, +}; + +/** + * Node kinds that provide high information value in context results. + * Imports/exports are excluded because they have near-zero information density - + * they tell you something exists, not how it works. + */ +const HIGH_VALUE_NODE_KINDS: NodeKind[] = [ + 'function', 'method', 'class', 'interface', 'type_alias', 'struct', 'trait', + 'component', 'route', 'variable', 'constant', 'enum', 'module', 'namespace', +]; + +/** + * Default options for finding relevant context + */ +const DEFAULT_FIND_OPTIONS: Required = { + searchLimit: 3, // Reduced from 5 + traversalDepth: 1, // Reduced from 2 + maxNodes: 20, // Reduced from 50 + minScore: 0.3, + edgeKinds: [], + nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default +}; + +// Re-export the low-confidence sentinel (defined in a dependency-free leaf so +// the MCP layer can import it without pulling this module's deps onto the +// cold-start path). Builder code below uses the imported binding directly. +export { LOW_CONFIDENCE_MARKER } from './markers'; + +/** + * Context Builder + * + * Coordinates semantic search and graph traversal to build + * comprehensive context for tasks. + */ +export class ContextBuilder { + private projectRoot: string; + private queries: QueryBuilder; + private traverser: GraphTraverser; + + constructor( + projectRoot: string, + queries: QueryBuilder, + traverser: GraphTraverser + ) { + this.projectRoot = projectRoot; + this.queries = queries; + this.traverser = traverser; + } + + /** + * Build context for a task + * + * Pipeline: + * 1. Parse task input (string or {title, description}) + * 2. Run semantic search to find entry points + * 3. Expand graph around entry points + * 4. Extract code blocks for key nodes + * 5. Format output for Claude + * + * @param input - Task description or object with title/description + * @param options - Build options + * @returns TaskContext (structured) or formatted string + */ + async buildContext( + input: TaskInput, + options: BuildContextOptions = {} + ): Promise { + const opts = { ...DEFAULT_BUILD_OPTIONS, ...options }; + + // Parse input + const query = typeof input === 'string' ? input : `${input.title}${input.description ? `: ${input.description}` : ''}`; + + // Find relevant context (semantic search + graph expansion) + const subgraph = await this.findRelevantContext(query, { + searchLimit: opts.searchLimit, + traversalDepth: opts.traversalDepth, + maxNodes: opts.maxNodes, + minScore: opts.minScore, + }); + + // Get entry points (nodes from semantic search) + const entryPoints = this.getEntryPoints(subgraph); + + // Extract code blocks for key nodes + const codeBlocks = opts.includeCode + ? await this.extractCodeBlocks(subgraph, opts.maxCodeBlocks, opts.maxCodeBlockSize) + : []; + + // Get related files + const relatedFiles = this.getRelatedFiles(subgraph); + + // Generate summary + const summary = this.generateSummary(query, subgraph, entryPoints); + + // Calculate stats + const stats = { + nodeCount: subgraph.nodes.size, + edgeCount: subgraph.edges.length, + fileCount: relatedFiles.length, + codeBlockCount: codeBlocks.length, + totalCodeSize: codeBlocks.reduce((sum, block) => sum + block.content.length, 0), + }; + + const context: TaskContext = { + query, + subgraph, + entryPoints, + codeBlocks, + relatedFiles, + summary, + stats, + }; + + // Return formatted output or raw context + if (opts.format === 'markdown') { + return formatContextAsMarkdown(context) + + this.buildCallPathsSection(subgraph) + + (subgraph.confidence === 'low' ? this.buildLowConfidenceNote(entryPoints) : ''); + } else if (opts.format === 'json') { + return formatContextAsJson(context); + } + + return context; + } + + /** + * Honest handoff appended when retrieval confidence is low (the query matched + * mostly common words). Instead of the usual "this covers the surface" framing + * — which, when wrong, sends the agent off to Read/Grep — it admits the + * uncertainty and routes the agent to the precise tools (explore with real + * symbol names, search, or files to browse the closest areas we *did* surface). + */ + private buildLowConfidenceNote(entryPoints: Node[]): string { + const dirs: string[] = []; + const seen = new Set(); + for (const n of entryPoints) { + const slash = n.filePath.lastIndexOf('/'); + const dir = slash > 0 ? n.filePath.slice(0, slash) : n.filePath; + if (!seen.has(dir)) { seen.add(dir); dirs.push(dir); } + if (dirs.length >= 4) break; + } + const dirLine = dirs.length + ? `\n- \`codegraph_files\` a likely area: ${dirs.map(d => `\`${d}\``).join(', ')}` + : ''; + return `\n\n${LOW_CONFIDENCE_MARKER}\n\n` + + 'This query matched mostly on common words, so the entry points above may ' + + 'be off-target — treat them as a starting point, not a complete answer. ' + + 'For a reliable result:\n' + + '- `codegraph_explore` with the **exact symbol names** you are after ' + + '(class / function / method names), or\n' + + '- `codegraph_search ` for one specific symbol' + + dirLine + + '\n\nDo not assume the list above is comprehensive.'; + } + + /** + * Surface short call-paths among the symbols this context already found, + * derived in-memory from the subgraph's `calls` edges (no extra queries). + * + * This bakes the value of path-finding INTO the always-loaded `context` tool. + * Agents reliably read context's output but do NOT discover/adopt a standalone + * trace tool (in deferred-MCP harnesses they only ToolSearch-select tools they + * already know). Delivering the flow here means "how does X reach Y" is + * answered without the agent needing to find, load, or choose a new tool. + * Chains stop where the static call graph ends (e.g. dynamic dispatch) — that + * truncation is honest, and the agent can codegraph_node the last hop to bridge. + */ + private buildCallPathsSection(subgraph: Subgraph): string { + const adj = new Map(); + for (const e of subgraph.edges) { + if (e.kind !== 'calls') continue; + if (!subgraph.nodes.has(e.source) || !subgraph.nodes.has(e.target)) continue; + const list = adj.get(e.source); + if (list) list.push(e.target); + else adj.set(e.source, [e.target]); + } + if (adj.size === 0) return ''; + + const MAX_HOPS = 6; + const chains: string[][] = []; + let budget = 2000; // bound DFS work on dense subgraphs + const dfs = (id: string, path: string[], seen: Set): void => { + if (budget-- <= 0) return; + const next = (adj.get(id) ?? []).filter((t) => !seen.has(t)); + if (next.length === 0 || path.length >= MAX_HOPS) { + if (path.length >= 3) chains.push([...path]); // >=3 nodes = a real flow, not a single call + return; + } + for (const t of next) { + seen.add(t); + dfs(t, [...path, t], seen); + seen.delete(t); + } + }; + const starts = (subgraph.roots.length > 0 + ? subgraph.roots.filter((id) => adj.has(id)) + : [...adj.keys()] + ).slice(0, 5); + for (const s of starts) dfs(s, [s], new Set([s])); + if (chains.length === 0) return ''; + + // Keep only chains that connect TWO OR MORE query-relevant symbols (roots). + // A chain from a root into an arbitrary callee (render → onMagicFrameGenerate) + // is structurally valid but tangential to the question; requiring ≥2 roots + // keeps the chain anchored to what the user actually asked about. Rank by + // #roots then length, and drop any that are a sub-path of a longer kept chain. + const rootSet = new Set(subgraph.roots); + const rootCount = (c: string[]): number => c.reduce((n, id) => n + (rootSet.has(id) ? 1 : 0), 0); + const relevant = chains.filter((c) => rootCount(c) >= 2); + relevant.sort((a, b) => rootCount(b) - rootCount(a) || b.length - a.length); + const kept: string[][] = []; + for (const c of relevant) { + const key = c.join('>'); + if (kept.some((k) => k.join('>').includes(key))) continue; + kept.push(c); + if (kept.length >= 3) break; + } + if (kept.length === 0) return ''; + const name = (id: string): string => subgraph.nodes.get(id)?.name ?? id; + + // Synthesized (dynamic-dispatch) hops are real `calls` edges but invisible to + // static parsing — mark them inline so the agent sees WHERE the callback was + // wired up (`registered @file:line`) instead of grepping for it. Keyed by + // "source>target". + const synthByPair = new Map(); + for (const e of subgraph.edges) { + if (e.kind !== 'calls' || e.provenance !== 'heuristic') continue; + const m = e.metadata as Record | undefined; + if (!m?.synthesizedBy) continue; + const at = typeof m.registeredAt === 'string' ? ` @${m.registeredAt}` : ''; + const label = m.synthesizedBy === 'callback' + ? `callback via ${m.via ? `\`${String(m.via)}\`` : 'registrar'}${at}` + : m.synthesizedBy === 'react-render' + ? `React re-render via setState${at}` + : m.synthesizedBy === 'jsx-render' + ? `renders <${String(m.via || 'child')}>` + : m.synthesizedBy === 'vue-handler' + ? `Vue @${String(m.event || 'event')} handler` + : `event ${m.event ? `\`${String(m.event)}\`` : ''}${at}`; + synthByPair.set(`${e.source}>${e.target}`, label); + } + const renderChain = (c: string[]): string => { + let s = name(c[0]!); + for (let i = 1; i < c.length; i++) { + const synth = synthByPair.get(`${c[i - 1]}>${c[i]}`); + s += synth ? ` →[${synth}] ${name(c[i]!)}` : ` → ${name(c[i]!)}`; + } + return s; + }; + const hasSynth = kept.some((c) => c.some((_, i) => i > 0 && synthByPair.has(`${c[i - 1]}>${c[i]}`))); + const lines = [ + '', + '## Call paths', + '', + 'Execution flow among the key symbols (traced through the call graph):', + '', + ...kept.map((c) => `- ${renderChain(c)}`), + '', + hasSynth + ? '_Hops marked `[callback/event …]` are dynamic dispatch bridged by codegraph (with the registration site); the rest are direct calls. codegraph_node any symbol for its body._' + : '_codegraph_node any symbol above for its source + its own callers/callees._', + ]; + return '\n' + lines.join('\n') + '\n'; + } + + /** + * Find relevant subgraph for a query + * + * Uses hybrid search combining exact symbol lookup with semantic search: + * 1. Extract potential symbol names from query + * 2. Look up exact matches for those symbols (high confidence) + * 3. Use semantic search for concept matching + * 4. Merge results, prioritizing exact matches + * 5. Traverse graph from entry points + * + * @param query - Natural language query + * @param options - Search and traversal options + * @returns Subgraph of relevant nodes and edges + */ + async findRelevantContext( + query: string, + options: FindRelevantContextOptions = {} + ): Promise { + const opts = { ...DEFAULT_FIND_OPTIONS, ...options }; + + // Start with empty subgraph + const nodes = new Map(); + const edges: Edge[] = []; + const roots: string[] = []; + + // Handle empty query - return empty subgraph + if (!query || query.trim().length === 0) { + return { nodes, edges, roots }; + } + + // === HYBRID SEARCH === + + // Step 1: Extract potential symbol names from query + const symbolsFromQuery = extractSymbolsFromQuery(query); + logDebug('Extracted symbols from query', { query, symbols: symbolsFromQuery }); + + // Step 2: Look up exact matches for extracted symbols + let exactMatches: SearchResult[] = []; + if (symbolsFromQuery.length > 0) { + try { + // Get more results so we can apply co-location boosting before trimming + exactMatches = this.queries.findNodesByExactName(symbolsFromQuery, { + limit: Math.ceil(opts.searchLimit * 5), + kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + }); + + // Co-location boost: when multiple extracted symbols appear in the same file, + // those results are much more likely to be what the user is looking for. + // E.g., "scrapeLoop" + "run" both in scrape/scrape.go → boost both. + if (exactMatches.length > 1) { + // Build a map of files → how many distinct symbol names matched in that file + const fileSymbolCounts = new Map>(); + for (const r of exactMatches) { + const names = fileSymbolCounts.get(r.node.filePath) || new Set(); + names.add(r.node.name.toLowerCase()); + fileSymbolCounts.set(r.node.filePath, names); + } + // Boost results in files where multiple query symbols co-occur + exactMatches = exactMatches.map(r => { + const symbolCount = fileSymbolCounts.get(r.node.filePath)?.size || 1; + return { + ...r, + score: symbolCount > 1 ? r.score + (symbolCount - 1) * 20 : r.score, + }; + }); + exactMatches.sort((a, b) => b.score - a.score); + } + + // Trim back to reasonable size + exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 2)); + logDebug('Exact symbol matches', { count: exactMatches.length }); + } catch (error) { + logDebug('Exact symbol lookup failed', { error: String(error) }); + } + } + + // Step 2b: Search for extracted symbols as definition (class/interface) prefixes. + // When the user writes "REST", "bulk", or "allocation", they usually mean classes + // like RestController, BulkRequest, AllocationService — not nodes named exactly that. + // Also tries stem variants: "caching" → "cache" finds Cache, CacheBuilder. + if (symbolsFromQuery.length > 0) { + const definitionKinds: NodeKind[] = ['class', 'interface', 'struct', 'trait', + 'protocol', 'enum', 'type_alias']; + // Expand symbols with stem variants for broader definition matching + const expandedSymbols = new Set(symbolsFromQuery); + for (const sym of symbolsFromQuery) { + for (const variant of getStemVariants(sym)) { + expandedSymbols.add(variant); + } + } + for (const sym of expandedSymbols) { + // Title-case the symbol: "REST" → "Rest", "bulk" → "Bulk", "allocation" → "Allocation" + const titleCased = sym.charAt(0).toUpperCase() + sym.slice(1).toLowerCase(); + if (titleCased === sym) continue; // already title-case (e.g., "Engine") — handled by exact match + // Fetch more results since popular prefixes have many matches + const prefixResults = this.queries.searchNodes(titleCased, { + limit: 30, + kinds: definitionKinds, + }); + const matched: SearchResult[] = []; + for (const r of prefixResults) { + if (r.node.name.toLowerCase().startsWith(titleCased.toLowerCase())) { + // Favor shorter names: "AllocationService" (18 chars) over + // "AllocationBalancingRoundMetrics" (31 chars). Core classes tend + // to have concise names; test/helper classes are verbose. + const brevityBonus = Math.max(0, 10 - (r.node.name.length - titleCased.length) / 3); + matched.push({ ...r, score: r.score + 15 + brevityBonus }); + } + } + matched.sort((a, b) => b.score - a.score); + for (const r of matched.slice(0, Math.ceil(opts.searchLimit))) { + const existing = exactMatches.find(e => e.node.id === r.node.id); + if (!existing) { + exactMatches.push(r); + } + } + } + exactMatches.sort((a, b) => b.score - a.score); + exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); + } + + // Step 3: Run text search for natural language term matching + // This catches file-name and node-name matches that semantic search may miss, + // which is critical for template-heavy codebases (e.g., Liquid/Shopify themes) + // where file names are the primary identifiers. + let textResults: SearchResult[] = []; + try { + const searchTerms = extractSearchTerms(query); + if (searchTerms.length > 0) { + // Search each term individually to get broader coverage, + // then boost results that match multiple terms + const termResultsMap = new Map(); + // When no explicit kind filter is set, exclude imports — they flood FTS + // results with qualified name matches (e.g., "REST" matches 445K import paths) + // but are almost never what exploration queries want. + const searchKinds = opts.nodeKinds && opts.nodeKinds.length > 0 + ? opts.nodeKinds + : ['file', 'module', 'class', 'struct', 'interface', 'trait', 'protocol', + 'function', 'method', 'property', 'field', 'variable', 'constant', + 'enum', 'enum_member', 'type_alias', 'namespace', 'export', + 'route', 'component'] as NodeKind[]; + for (const term of searchTerms) { + const termResults = this.queries.searchNodes(term, { + limit: opts.searchLimit * 2, + kinds: searchKinds, + }); + for (const r of termResults) { + const existing = termResultsMap.get(r.node.id); + if (existing) { + existing.termHits++; + existing.result.score = Math.max(existing.result.score, r.score); + } else { + termResultsMap.set(r.node.id, { result: r, termHits: 1 }); + } + } + } + // Boost results matching multiple terms and sort + textResults = Array.from(termResultsMap.values()) + .map(({ result, termHits }) => ({ + ...result, + score: result.score + (termHits - 1) * 5, + })) + .sort((a, b) => b.score - a.score) + .slice(0, opts.searchLimit * 2); + } + logDebug('Text search results', { count: textResults.length }); + } catch (error) { + logDebug('Text search failed', { query, error: String(error) }); + } + + // Step 4: Merge results, taking the max score when duplicates appear + // across search channels. Exact matches may have lower scores than FTS + // results for the same node — use the best score from any channel. + const resultById = new Map(); + let searchResults: SearchResult[] = []; + + // Add exact matches first + for (const result of exactMatches) { + const existing = resultById.get(result.node.id); + if (existing) { + existing.score = Math.max(existing.score, result.score); + } else { + resultById.set(result.node.id, result); + searchResults.push(result); + } + } + + // Add text search results, upgrading scores for duplicates + for (const result of textResults) { + const existing = resultById.get(result.node.id); + if (existing) { + existing.score = Math.max(existing.score, result.score); + } else { + resultById.set(result.node.id, result); + searchResults.push(result); + } + } + + const queryLower = query.toLowerCase(); + const isTestQuery = queryLower.includes('test') || queryLower.includes('spec'); + + // Deprioritize test files early so they don't take multi-term boost slots + if (!isTestQuery) { + for (const result of searchResults) { + if (isTestFile(result.node.filePath)) { + result.score *= 0.3; + } + } + } + + // Iter7 — Core-directory boost. On projects with one file that holds + // the dense majority of internal call edges (e.g. sinatra's + // `lib/sinatra/base.rb` at 85% of all in-file edges), the agent's + // task usually asks about the framework's core. Without this boost, + // ranking favors small focused extension files (e.g. text search + // picks `sinatra-contrib/lib/sinatra/multi_route.rb`'s 10-line + // `route` method over `base.rb`'s `route!` because the extension + // file's `route` matches the query verbatim AND the file is small, + // dwarfing the longer name `route!` in a 1500-line file). Boost + // results that share a directory prefix with the dominant file's + // directory so the core file's siblings outrank sibling-package + // extensions. + try { + const dominant = this.queries.getDominantFile?.(); + if (dominant && dominant.edgeCount >= 3 * dominant.nextEdgeCount) { + // Take the directory of the dominant file (everything up to the + // last slash). For `lib/sinatra/base.rb` → `lib/sinatra/`. + const slash = dominant.filePath.lastIndexOf('/'); + if (slash > 0) { + const coreDir = dominant.filePath.slice(0, slash + 1); + for (const result of searchResults) { + if (result.node.filePath.startsWith(coreDir)) { + result.score += 25; + } + } + } + } + } catch { + // SQL failure — fall through, scoring works without the boost + } + + // Step 5a: Multi-term co-occurrence re-ranking (applied BEFORE truncation). + // For multi-word queries like "search execution from request to shard", + // nodes matching 2+ query terms in their name or path are far more relevant + // than nodes matching just one generic term. Without this, "ExecutionUtils" + // (matches only "execution") fills budget slots meant for "ShardSearchRequest" + // (matches "shard" + "search" + "request"). + const queryTermsForBoost = extractSearchTerms(query); + if (queryTermsForBoost.length >= 2) { + // Group terms that are substrings of each other (stem variants of the same + // root word). "indexed", "indexe", "index" should count as ONE concept match, + // not three. Without this, stem variants inflate matchCount and give false + // multi-term boosts to symbols matching one root word multiple times. + const termGroups: string[][] = []; + const sorted = [...queryTermsForBoost].sort((a, b) => b.length - a.length); + const assigned = new Set(); + for (const term of sorted) { + if (assigned.has(term)) continue; + const group = [term]; + assigned.add(term); + for (const other of sorted) { + if (assigned.has(other)) continue; + if (term.includes(other) || other.includes(term)) { + group.push(other); + assigned.add(other); + } + } + termGroups.push(group); + } + + // Build a set of exact-match node IDs so we can exempt them from dampening. + // When the query is "LiveEditMode DevServerPreview", these are specific + // symbols the user asked for — dampening them because they only match 1 + // term group is counter-productive. + const exactMatchIds = new Set(exactMatches.map(r => r.node.id)); + + // ...but only exempt exact matches the user *named as an identifier* + // (camelCase/snake_case/acronym). A plain dictionary word that happens to + // exact-match an unrelated symbol — query "flat object" → a constant named + // FLAT — must NOT be exempt, or the +exact-name bonus floats it to the top + // of a prose query with zero corroboration from any other term. Classify by + // the QUERY token (what the user typed), not the matched symbol's name. + const distinctiveTokens = new Set( + symbolsFromQuery.filter(isDistinctiveIdentifier).map(s => s.toLowerCase()) + ); + const distinctiveExactMatchIds = new Set( + exactMatches + .filter(r => distinctiveTokens.has(r.node.name.toLowerCase())) + .map(r => r.node.id) + ); + + for (const result of searchResults) { + // Check term matches in name (substring) and path DIRECTORIES (exact). + // Directory segments must match exactly — "search" matches directory + // "search/" but NOT "elasticsearch/". The class name is checked + // separately via substring match on the node name. + const nameLower = result.node.name.toLowerCase(); + const dirSegments = path.dirname(result.node.filePath).toLowerCase().split('/'); + let matchCount = 0; + for (const group of termGroups) { + const groupMatches = group.some(term => { + const inName = nameLower.includes(term); + const inDir = dirSegments.some(seg => seg === term); + return inName || inDir; + }); + if (groupMatches) matchCount++; + } + if (matchCount >= 2) { + // Multiplicative boost — 2 terms → 2x, 3 terms → 2.5x + result.score *= 1 + matchCount * 0.5; + } else if (distinctiveExactMatchIds.has(result.node.id)) { + // Exact match on a distinctive identifier the user explicitly named — + // keep full score (e.g. "LiveEditMode DevServerPreview"). + } else if (exactMatchIds.has(result.node.id)) { + // Exact match on a COMMON word (e.g. "flat" → FLAT): high-scoring noise + // inflated by the +exact-name bonus, corroborated by no other query + // term. Demote hard so corroborated matches win. + result.score *= 0.3; + } else { + // Mild dampen for generic single-term matches — they might be generic + // but could also be the right result (e.g., "Protocol" class for an IPC query). + result.score *= 0.6; + } + } + searchResults.sort((a, b) => b.score - a.score); + } + + // Step 5b: CamelCase-boundary matching via LIKE query. + // FTS can't find "Search" inside "TransportSearchAction" (one FTS token). + // LIKE reliably finds these substring matches. Results are appended with + // guaranteed slots so they don't compete with higher-scoring prefix matches. + if (symbolsFromQuery.length > 0) { + const camelDefinitionKinds: NodeKind[] = ['class', 'interface', 'struct', 'trait', + 'protocol', 'enum', 'type_alias']; + const camelSearchedTerms = new Set(); + const searchIdSet = new Set(searchResults.map(r => r.node.id)); + // Track per-node term hits for multi-term boosting + const camelNodeTerms = new Map(); + const maxCamelPerTerm = Math.ceil(opts.searchLimit / 2); + + for (const sym of symbolsFromQuery) { + const titleCased = sym.charAt(0).toUpperCase() + sym.slice(1).toLowerCase(); + if (titleCased.length < 3) continue; + const termKey = titleCased.toLowerCase(); + if (camelSearchedTerms.has(termKey)) continue; + camelSearchedTerms.add(termKey); + + // Fetch a large batch — popular terms like "Search" in Elasticsearch + // have hundreds of substring matches. The LIKE scan cost is the same + // regardless of LIMIT (SQLite scans all matches to sort), so we fetch + // generously and let path-relevance scoring pick the best ones. + const likeResults = this.queries.findNodesByNameSubstring(titleCased, { + limit: 200, + kinds: camelDefinitionKinds, + excludePrefix: true, + }); + + // Filter to CamelCase boundaries, score by path relevance, and take top N + const termCandidates: SearchResult[] = []; + for (const r of likeResults) { + const name = r.node.name; + const idx = name.indexOf(titleCased); + if (idx <= 0) continue; + // Accept CamelCase boundary (lowercase before match) OR + // acronym boundary (uppercase before match, e.g., RPCProtocol) + if (!/[a-zA-Z]/.test(name.charAt(idx - 1))) continue; + if (searchIdSet.has(r.node.id)) continue; + if (isTestFile(r.node.filePath) && !isTestQuery) continue; + + const pathScore = scorePathRelevance(r.node.filePath, query); + const brevityBonus = Math.max(0, 6 - (name.length - titleCased.length) / 4); + termCandidates.push({ node: r.node, score: 8 + brevityBonus + pathScore }); + } + termCandidates.sort((a, b) => b.score - a.score); + + // Widen the per-term pool for accumulation so multi-term co-occurrences + // can be discovered. A class matching 3 query terms at CamelCase boundaries + // is far more relevant than one matching just 1, but it needs to survive + // the per-term cut for EACH term to accumulate its count. + const accumPerTerm = maxCamelPerTerm * 4; + for (const r of termCandidates.slice(0, accumPerTerm)) { + const existing = camelNodeTerms.get(r.node.id); + if (existing) { + existing.termCount++; + } else { + camelNodeTerms.set(r.node.id, { + result: r, + termCount: 1, + }); + } + } + } + + // Append CamelCase matches with multi-term boost. + // These are structurally important (class names containing query terms at + // CamelCase boundaries) but score much lower than FTS results. Scale their + // scores up so multi-term CamelCase matches can compete with FTS results. + const camelResults: SearchResult[] = []; + for (const [, info] of camelNodeTerms) { + // Multi-term CamelCase matches are extremely relevant — a class matching + // 3+ query terms in its name (e.g., ExtensionHostProcess) is almost + // certainly what the user wants. Scale aggressively. + info.result.score = info.result.score * (1 + info.termCount) + (info.termCount - 1) * 30; + camelResults.push(info.result); + } + camelResults.sort((a, b) => b.score - a.score); + const maxCamelTotal = opts.searchLimit; + for (const r of camelResults.slice(0, maxCamelTotal)) { + searchResults.push(r); + searchIdSet.add(r.node.id); + } + + // Step 5c: Compound term matching — find classes whose name contains 2+ + // query terms at ANY position (not just CamelCase boundaries). + // The CamelCase step above requires idx > 0, which misses classes that + // START with a query term (e.g., "SearchShardsRequest" starts with "Search"). + // For multi-word queries, a class matching multiple query terms in its name + // is almost certainly relevant regardless of position. + if (symbolsFromQuery.length >= 2) { + // Collect ALL LIKE results per term (reusing findNodesByNameSubstring) + // but without the CamelCase boundary or prefix exclusion filters. + const compoundTermMap = new Map }>(); + for (const sym of symbolsFromQuery) { + const titleCased = sym.charAt(0).toUpperCase() + sym.slice(1).toLowerCase(); + if (titleCased.length < 3) continue; + + const likeResults = this.queries.findNodesByNameSubstring(titleCased, { + limit: 200, + kinds: camelDefinitionKinds, + excludePrefix: false, + }); + + for (const r of likeResults) { + if (searchIdSet.has(r.node.id)) continue; + if (isTestFile(r.node.filePath) && !isTestQuery) continue; + const entry = compoundTermMap.get(r.node.id); + if (entry) { + entry.terms.add(titleCased); + } else { + compoundTermMap.set(r.node.id, { node: r.node, terms: new Set([titleCased]) }); + } + } + } + + // Keep only nodes matching 2+ distinct terms + const compoundResults: SearchResult[] = []; + for (const [, entry] of compoundTermMap) { + if (entry.terms.size >= 2) { + const pathScore = scorePathRelevance(entry.node.filePath, query); + const brevityBonus = Math.max(0, 6 - entry.node.name.length / 8); + compoundResults.push({ + node: entry.node, + score: 10 + (entry.terms.size - 1) * 20 + pathScore + brevityBonus, + }); + } + } + compoundResults.sort((a, b) => b.score - a.score); + const maxCompound = Math.ceil(opts.searchLimit / 2); + for (const r of compoundResults.slice(0, maxCompound)) { + searchResults.push(r); + searchIdSet.add(r.node.id); + } + } + } + + // Final sort and truncation — all search channels (exact, text, CamelCase, + // compound) have now contributed. Sort by score so multi-term matches from + // later steps can outrank dampened single-term matches from earlier steps. + searchResults.sort((a, b) => b.score - a.score); + searchResults = searchResults.slice(0, opts.searchLimit * 3); + + // Filter by minimum score + let filteredResults = searchResults.filter((r) => r.score >= opts.minScore); + + // Resolve imports/exports to their actual definitions + // If someone searches "terminal" and finds `import { TerminalPanel }`, + // they want the TerminalPanel class, not the import statement + filteredResults = this.resolveImportsToDefinitions(filteredResults); + + // Cap entry points so traversal budget isn't spread too thin. + // With 36 entry points and maxNodes=120, each gets only 3 nodes — useless. + // Cap to searchLimit so each entry point gets a meaningful traversal budget. + if (filteredResults.length > opts.searchLimit) { + filteredResults = filteredResults.slice(0, opts.searchLimit); + } + + // Confidence signal for the honest-handoff footer (consumed in buildContext). + // A multi-term prose query that resolves only to isolated common-word matches + // — no entry point corroborated by 2+ distinct query terms, and none a + // distinctive identifier the user explicitly named — is LOW confidence: the + // results are best-effort, not a located answer, so the agent should be told + // to drill in with explore/trace rather than trust the list as comprehensive. + // Single-keyword and symbol-name queries are exempt (their single match IS the + // answer), so the handoff never fires on them. + let confidence: 'high' | 'low' = 'high'; + const confTerms = extractSearchTerms(query, { stems: false }).filter(t => t.length >= 3); + if (confTerms.length >= 2 && filteredResults.length > 0) { + const distinctive = new Set( + symbolsFromQuery.filter(isDistinctiveIdentifier).map(s => s.toLowerCase()) + ); + const anyStrong = filteredResults.some(r => { + if (distinctive.has(r.node.name.toLowerCase())) return true; + const nameLower = r.node.name.toLowerCase(); + const dirSegs = path.dirname(r.node.filePath).toLowerCase().split('/'); + let hits = 0; + for (const t of confTerms) { + if (nameLower.includes(t) || dirSegs.includes(t)) { + if (++hits >= 2) return true; + } + } + return false; + }); + if (!anyStrong) confidence = 'low'; + } + + // Add entry points to subgraph + for (const result of filteredResults) { + nodes.set(result.node.id, result.node); + roots.push(result.node.id); + } + + // Expand type hierarchy for class/interface entry points. + // BFS often exhausts its per-entry-point budget on contained methods + // before reaching extends/implements neighbors. This dedicated step + // ensures subclasses and superclasses always appear in results. + // Budget: up to maxNodes/4 hierarchy nodes to avoid flooding. + const typeHierarchyKinds = new Set(['class', 'interface', 'struct', 'trait', 'protocol']); + const maxHierarchyNodes = Math.ceil(opts.maxNodes / 4); + let hierarchyNodesAdded = 0; + for (const result of filteredResults) { + if (hierarchyNodesAdded >= maxHierarchyNodes) break; + if (typeHierarchyKinds.has(result.node.kind)) { + const hierarchy = this.traverser.getTypeHierarchy(result.node.id); + for (const [id, node] of hierarchy.nodes) { + if (!nodes.has(id)) { + nodes.set(id, node); + hierarchyNodesAdded++; + } + } + for (const edge of hierarchy.edges) { + const exists = edges.some( + (e) => e.source === edge.source && e.target === edge.target && e.kind === edge.kind + ); + if (!exists) { + edges.push(edge); + } + } + } + } + + // Pass 2: expand hierarchy of newly-discovered parent types to find siblings. + // E.g., InternalEngine → Engine (parent, from pass 1) → ReadOnlyEngine (sibling). + if (hierarchyNodesAdded > 0) { + const pass2Candidates = [...nodes.values()].filter( + n => typeHierarchyKinds.has(n.kind) && !roots.includes(n.id) + ); + for (const candidate of pass2Candidates) { + if (hierarchyNodesAdded >= maxHierarchyNodes) break; + const siblingHierarchy = this.traverser.getTypeHierarchy(candidate.id); + for (const [id, node] of siblingHierarchy.nodes) { + if (!nodes.has(id) && hierarchyNodesAdded < maxHierarchyNodes) { + nodes.set(id, node); + hierarchyNodesAdded++; + } + } + for (const edge of siblingHierarchy.edges) { + if (nodes.has(edge.source) && nodes.has(edge.target)) { + const exists = edges.some( + (e) => e.source === edge.source && e.target === edge.target && e.kind === edge.kind + ); + if (!exists) { + edges.push(edge); + } + } + } + } + } + + // Traverse from each entry point + for (const result of filteredResults) { + const traversalResult = this.traverser.traverseBFS(result.node.id, { + maxDepth: opts.traversalDepth, + edgeKinds: opts.edgeKinds && opts.edgeKinds.length > 0 ? opts.edgeKinds : undefined, + nodeKinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, + direction: 'both', + limit: Math.ceil(opts.maxNodes / Math.max(1, filteredResults.length)), + }); + + // Merge nodes + for (const [id, node] of traversalResult.nodes) { + if (!nodes.has(id)) { + nodes.set(id, node); + } + } + + // Merge edges (avoid duplicates) + for (const edge of traversalResult.edges) { + const exists = edges.some( + (e) => e.source === edge.source && e.target === edge.target && e.kind === edge.kind + ); + if (!exists) { + edges.push(edge); + } + } + } + + // Trim to max nodes if needed + let finalNodes = nodes; + let finalEdges = edges; + if (nodes.size > opts.maxNodes) { + // Prioritize entry points and their direct neighbors + const priorityIds = new Set(roots); + for (const edge of edges) { + if (priorityIds.has(edge.source)) { + priorityIds.add(edge.target); + } + if (priorityIds.has(edge.target)) { + priorityIds.add(edge.source); + } + } + + // Keep priority nodes, then fill remaining slots + finalNodes = new Map(); + for (const id of priorityIds) { + const node = nodes.get(id); + if (node && finalNodes.size < opts.maxNodes) { + finalNodes.set(id, node); + } + } + + // Fill remaining from other nodes + for (const [id, node] of nodes) { + if (finalNodes.size >= opts.maxNodes) break; + if (!finalNodes.has(id)) { + finalNodes.set(id, node); + } + } + + // Filter edges to only include kept nodes + finalEdges = edges.filter( + (e) => finalNodes.has(e.source) && finalNodes.has(e.target) + ); + } + + // Per-file diversity cap: prevent any single file from monopolizing the + // node budget. When BFS traverses from a method, it follows `contains` + // to the parent class, then back down to all sibling methods. With + // multiple entry points in the same class, one file can consume 30-40% + // of maxNodes. Cap each file to ~20% to ensure cross-file diversity. + const maxPerFile = Math.max(5, Math.ceil(opts.maxNodes * 0.2)); + const fileCounts = new Map(); + for (const [id, node] of finalNodes) { + const ids = fileCounts.get(node.filePath) || []; + ids.push(id); + fileCounts.set(node.filePath, ids); + } + const rootSet = new Set(roots); + for (const [, nodeIds] of fileCounts) { + if (nodeIds.length <= maxPerFile) continue; + // Sort: entry points first, then classes/interfaces, then others + const kindPriority: Record = { + class: 3, interface: 3, struct: 3, trait: 3, protocol: 3, enum: 3, + method: 1, function: 1, property: 0, field: 0, variable: 0, + }; + nodeIds.sort((a, b) => { + const aRoot = rootSet.has(a) ? 10 : 0; + const bRoot = rootSet.has(b) ? 10 : 0; + const aKind = kindPriority[finalNodes.get(a)!.kind] ?? 0; + const bKind = kindPriority[finalNodes.get(b)!.kind] ?? 0; + return (bRoot + bKind) - (aRoot + aKind); + }); + // Remove excess nodes (keep the highest-priority ones) + for (const id of nodeIds.slice(maxPerFile)) { + finalNodes.delete(id); + } + } + // Non-production node cap: limit test/sample/integration/example files to + // at most 15% of the budget. Many codebases have dozens of near-identical + // test implementations (e.g., 6 Guard classes in integration tests) that + // individually survive score dampening but collectively flood the result. + // Test entry points are NOT exempt — they should be evicted too. + if (!isTestQuery) { + const maxNonProd = Math.max(3, Math.ceil(opts.maxNodes * 0.15)); + const nonProdIds: string[] = []; + for (const [id, node] of finalNodes) { + if (isTestFile(node.filePath)) { + nonProdIds.push(id); + } + } + if (nonProdIds.length > maxNonProd) { + for (const id of nonProdIds.slice(maxNonProd)) { + finalNodes.delete(id); + // Also remove from roots — test file entry points shouldn't anchor results + const rootIdx = roots.indexOf(id); + if (rootIdx !== -1) roots.splice(rootIdx, 1); + } + } + } + + // Re-filter edges after per-file and non-production caps + finalEdges = finalEdges.filter( + (e) => finalNodes.has(e.source) && finalNodes.has(e.target) + ); + + // Edge recovery: BFS with many entry points leaves most nodes disconnected. + // Discover edges between already-selected nodes to recover connectivity. + const recoveryKinds: EdgeKind[] = ['calls', 'extends', 'implements', 'references', 'overrides']; + const recoveredEdges = this.queries.findEdgesBetweenNodes( + [...finalNodes.keys()], + recoveryKinds, + ); + const existingEdgeKeys = new Set( + finalEdges.map((e) => `${e.source}:${e.target}:${e.kind}`) + ); + for (const edge of recoveredEdges) { + const key = `${edge.source}:${edge.target}:${edge.kind}`; + if (!existingEdgeKeys.has(key)) { + finalEdges.push(edge); + existingEdgeKeys.add(key); + } + } + + return { nodes: finalNodes, edges: finalEdges, roots, confidence }; + } + + /** + * Get the source code for a node + * + * Reads the file and extracts the code between startLine and endLine. + * + * @param nodeId - ID of the node + * @returns Code string or null if not found + */ + async getCode(nodeId: string): Promise { + const node = this.queries.getNodeById(nodeId); + if (!node) { + return null; + } + + return this.extractNodeCode(node); + } + + /** + * Extract code from a node's source file + */ + private async extractNodeCode(node: Node): Promise { + // SECURITY (#383): a config-leaf node's on-disk line is `key = `. + // Return the KEY only — never read the value off disk. This closes the + // includeCode / buildContext code-block path, mirroring the explore source + // renderer; an agent that genuinely needs a value can read the file itself. + if (isConfigLeafNode(node)) { + return node.signature || node.qualifiedName || node.name; + } + + const filePath = validatePathWithinRoot(this.projectRoot, node.filePath); + + if (!filePath || !fs.existsSync(filePath)) { + return null; + } + + try { + const content = fs.readFileSync(filePath, 'utf-8'); + const lines = content.split('\n'); + + // Extract lines (1-indexed to 0-indexed) + const startIdx = Math.max(0, node.startLine - 1); + const endIdx = Math.min(lines.length, node.endLine); + + return lines.slice(startIdx, endIdx).join('\n'); + } catch (error) { + logDebug('Failed to extract code from node', { nodeId: node.id, filePath: node.filePath, error: String(error) }); + return null; + } + } + + /** + * Get entry points from a subgraph (the root nodes) + */ + private getEntryPoints(subgraph: Subgraph): Node[] { + return subgraph.roots + .map((id) => subgraph.nodes.get(id)) + .filter((n): n is Node => n !== undefined); + } + + /** + * Extract code blocks for key nodes in the subgraph + */ + private async extractCodeBlocks( + subgraph: Subgraph, + maxBlocks: number, + maxBlockSize: number + ): Promise { + const blocks: CodeBlock[] = []; + + // Prioritize entry points, then functions/methods + const priorityNodes: Node[] = []; + + // First: entry points + for (const id of subgraph.roots) { + const node = subgraph.nodes.get(id); + if (node) { + priorityNodes.push(node); + } + } + + // Then: functions and methods + for (const node of subgraph.nodes.values()) { + if (!subgraph.roots.includes(node.id)) { + if (node.kind === 'function' || node.kind === 'method') { + priorityNodes.push(node); + } + } + } + + // Then: classes + for (const node of subgraph.nodes.values()) { + if (!subgraph.roots.includes(node.id)) { + if (node.kind === 'class') { + priorityNodes.push(node); + } + } + } + + // Extract code for priority nodes + for (const node of priorityNodes) { + if (blocks.length >= maxBlocks) break; + + const code = await this.extractNodeCode(node); + if (code) { + // Truncate if too long. Language-neutral marker (no `//` — not a + // comment in Python, Ruby, etc.); this renders inside a fenced + // source block whose language varies. + const truncated = code.length > maxBlockSize + ? code.slice(0, maxBlockSize) + '\n... (truncated) ...' + : code; + + blocks.push({ + content: truncated, + filePath: node.filePath, + startLine: node.startLine, + endLine: node.endLine, + language: node.language, + node, + }); + } + } + + return blocks; + } + + /** + * Get unique files from a subgraph + */ + private getRelatedFiles(subgraph: Subgraph): string[] { + const files = new Set(); + for (const node of subgraph.nodes.values()) { + files.add(node.filePath); + } + return Array.from(files).sort(); + } + + /** + * Generate a summary of the context + */ + private generateSummary(_query: string, subgraph: Subgraph, entryPoints: Node[]): string { + const nodeCount = subgraph.nodes.size; + const edgeCount = subgraph.edges.length; + const files = this.getRelatedFiles(subgraph); + + const entryPointNames = entryPoints + .slice(0, 3) + .map((n) => n.name) + .join(', '); + + const remaining = entryPoints.length > 3 ? ` and ${entryPoints.length - 3} more` : ''; + + return `Found ${nodeCount} relevant code symbols across ${files.length} files. ` + + `Key entry points: ${entryPointNames}${remaining}. ` + + `${edgeCount} relationships identified.`; + } + + /** + * Resolve import/export nodes to their actual definitions + * + * When search returns `import { TerminalPanel }`, users want the TerminalPanel + * class definition, not the import statement. This follows the `imports` edge + * to find and return the actual definition instead. + * + * @param results - Search results that may include import/export nodes + * @returns Results with imports resolved to definitions where possible + */ + private resolveImportsToDefinitions(results: SearchResult[]): SearchResult[] { + const resolved: SearchResult[] = []; + const seenIds = new Set(); + + for (const result of results) { + const { node, score } = result; + + // If it's not an import/export, keep it as-is + if (node.kind !== 'import' && node.kind !== 'export') { + if (!seenIds.has(node.id)) { + seenIds.add(node.id); + resolved.push(result); + } + continue; + } + + // For imports/exports, try to find what they reference + // Imports have outgoing 'imports' edges to the definition + // Exports have outgoing 'exports' edges to the definition + const edgeKind = node.kind === 'import' ? 'imports' : 'exports'; + const outgoingEdges = this.queries.getOutgoingEdges(node.id, [edgeKind as EdgeKind]); + + let foundDefinition = false; + for (const edge of outgoingEdges) { + const targetNode = this.queries.getNodeById(edge.target); + if (targetNode && !seenIds.has(targetNode.id)) { + // Found the definition - use it instead of the import + seenIds.add(targetNode.id); + resolved.push({ + node: targetNode, + score: score, // Preserve the original score + }); + foundDefinition = true; + logDebug('Resolved import to definition', { + import: node.name, + definition: targetNode.name, + kind: targetNode.kind, + }); + } + } + + // If we couldn't resolve the import, skip it (it's low-value on its own) + if (!foundDefinition) { + logDebug('Skipping unresolved import', { name: node.name, file: node.filePath }); + } + } + + return resolved; + } +} + +/** + * Create a context builder + */ +export function createContextBuilder( + projectRoot: string, + queries: QueryBuilder, + traverser: GraphTraverser +): ContextBuilder { + return new ContextBuilder(projectRoot, queries, traverser); +} + +// Re-export formatter +export { formatContextAsMarkdown, formatContextAsJson } from './formatter'; diff --git a/src/context/markers.ts b/src/context/markers.ts new file mode 100644 index 0000000..13f1968 --- /dev/null +++ b/src/context/markers.ts @@ -0,0 +1,19 @@ +/** + * Stable sentinel strings shared between the context builder (which emits them + * into its markdown) and the MCP layer (which detects them to adjust framing). + * + * Intentionally a dependency-free leaf module: the MCP tool layer imports this + * to recognise a low-confidence response, and routing that recognition through + * the full context module would drag its dependencies onto the cold-start path. + * Keep this file import-free. + */ + +/** + * Heading that leads the honest low-confidence handoff appended to a context + * response when the query resolved only to weak/isolated matches. The MCP layer + * checks for it to suppress the contradictory "this is comprehensive, don't call + * explore" small-repo footer. Changing the text is a breaking sentinel change — + * both the emitter (`ContextBuilder`) and the detector (`src/mcp/tools.ts`) + * import this constant, so they stay in sync automatically. + */ +export const LOW_CONFIDENCE_MARKER = '### ⚠️ Low-confidence match'; diff --git a/src/db/index.ts b/src/db/index.ts new file mode 100644 index 0000000..0cd2823 --- /dev/null +++ b/src/db/index.ts @@ -0,0 +1,475 @@ +/** + * Database Layer + * + * Handles SQLite database initialization and connection management. + */ + +import { SqliteDatabase, SqliteBackend, createDatabase } from './sqlite-adapter'; +import * as fs from 'fs'; +import * as path from 'path'; +import { SchemaVersion } from '../types'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from './migrations'; +import { getCodeGraphDir } from '../directory'; + +export { SqliteDatabase, SqliteBackend } from './sqlite-adapter'; + +/** + * Apply connection-level PRAGMAs. Shared by `initialize` and `open` so the two + * paths can't drift. + * + * `busy_timeout` is set FIRST, before any pragma that might touch the database + * file (notably `journal_mode`). If another process holds a write lock at open + * time, the later pragmas — and the connection's first query — then wait out + * the lock instead of throwing "database is locked" immediately. See issue #238. + * + * The 5s window (was 120s) rides out a normal incremental sync; the old + * 2-minute wait presented as a frozen, hung agent. With WAL, reads never block + * on a writer, so this timeout only governs cross-process write contention + * (e.g. the git-hook `codegraph sync` running while the MCP server writes). + */ +function configureConnection(db: SqliteDatabase): void { + db.pragma('busy_timeout = 5000'); // MUST be first — see above + db.pragma('foreign_keys = ON'); + db.pragma('journal_mode = WAL'); // node:sqlite supports WAL on every platform + db.pragma('synchronous = NORMAL'); // safe with WAL mode + db.pragma('cache_size = -64000'); // 64 MB page cache + db.pragma('temp_store = MEMORY'); // temp tables in memory + db.pragma('mmap_size = 268435456'); // 256 MB memory-mapped I/O +} + +/** + * Database connection wrapper with lifecycle management + */ +export class DatabaseConnection { + private db: SqliteDatabase; + private dbPath: string; + private backend: SqliteBackend; + /** + * `dev:ino` of the DB file at the moment we opened it (or null when the + * platform/filesystem reports no usable inode). Lets us notice when the file + * we hold open has been unlinked and REPLACED by a new file at the same path + * — a git worktree removed and re-added, or `.codegraph/` deleted and + * re-`init`ed under a long-lived server — at which point our fd reads a now + * dead inode forever (#925). See `isReplacedOnDisk`. + */ + private openedInode: string | null; + + private constructor(db: SqliteDatabase, dbPath: string, backend: SqliteBackend) { + this.db = db; + this.dbPath = dbPath; + this.backend = backend; + this.openedInode = statInode(dbPath); + } + + /** + * Initialize a new database at the given path + */ + static initialize(dbPath: string): DatabaseConnection { + // Ensure parent directory exists + const dir = path.dirname(dbPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Create and configure database + const { db, backend } = createDatabase(dbPath); + + configureConnection(db); + + // Run schema initialization + const schemaPath = path.join(__dirname, 'schema.sql'); + const schema = fs.readFileSync(schemaPath, 'utf-8'); + db.exec(schema); + + // Record current schema version so migrations aren't re-applied on open + const currentVersion = getCurrentVersion(db); + if (currentVersion < CURRENT_SCHEMA_VERSION) { + db.prepare( + 'INSERT OR IGNORE INTO schema_versions (version, applied_at, description) VALUES (?, ?, ?)' + ).run(CURRENT_SCHEMA_VERSION, Date.now(), 'Initial schema includes all migrations'); + } + + return new DatabaseConnection(db, dbPath, backend); + } + + /** + * Open an existing database + */ + static open(dbPath: string): DatabaseConnection { + if (!fs.existsSync(dbPath)) { + throw new Error(`Database not found: ${dbPath}`); + } + + const { db, backend } = createDatabase(dbPath); + + configureConnection(db); + + // Check and run migrations if needed + const conn = new DatabaseConnection(db, dbPath, backend); + const currentVersion = getCurrentVersion(db); + + if (currentVersion < CURRENT_SCHEMA_VERSION) { + runMigrations(db, currentVersion); + } + + return conn; + } + + /** + * Get the underlying database instance + */ + getDb(): SqliteDatabase { + return this.db; + } + + /** + * Get the SQLite backend serving this connection. Per-instance so + * MCP cross-project queries report the right backend even when + * multiple project DBs are open in the same process. + */ + getBackend(): SqliteBackend { + return this.backend; + } + + /** + * Get database file path + */ + getPath(): string { + return this.dbPath; + } + + /** + * The journal mode actually in effect (e.g. 'wal', 'delete'). + * + * SQLite silently keeps the prior mode if WAL can't be enabled — e.g. on + * filesystems without shared-memory support (some network/virtualized mounts, + * WSL2 /mnt). So the effective mode can differ + * from what `configureConnection` requested. Surfaced in `codegraph status` so + * a "database is locked" report is triageable: 'wal' ⇒ readers never block on a + * writer; anything else ⇒ they can. See issue #238. + */ + getJournalMode(): string { + const raw = this.db.pragma('journal_mode'); + const row = Array.isArray(raw) ? raw[0] : raw; + const mode = row && typeof row === 'object' + ? (row as Record).journal_mode + : row; + return String(mode ?? '').toLowerCase(); + } + + /** + * Get current schema version + */ + getSchemaVersion(): SchemaVersion | null { + const row = this.db + .prepare('SELECT version, applied_at, description FROM schema_versions ORDER BY version DESC LIMIT 1') + .get() as { version: number; applied_at: number; description: string | null } | undefined; + + if (!row) return null; + + return { + version: row.version, + appliedAt: row.applied_at, + description: row.description ?? undefined, + }; + } + + /** + * Execute a function within a transaction + */ + transaction(fn: () => T): T { + return this.db.transaction(fn)(); + } + + /** + * Get database file size in bytes + */ + getSize(): number { + const stats = fs.statSync(this.dbPath); + return stats.size; + } + + /** + * Size of the `-wal` sidecar file in bytes. 0 when it doesn't exist (non-WAL + * journal mode, in-memory DB, or no write since the last checkpoint+reset). + */ + getWalSizeBytes(): number { + if (!this.dbPath || this.dbPath === ':memory:') return 0; + try { + return fs.statSync(`${this.dbPath}-wal`).size; + } catch { + return 0; + } + } + + /** Current `wal_autocheckpoint` interval in pages (0 = disabled). */ + getWalAutocheckpoint(): number { + const v = this.db.pragma('wal_autocheckpoint', { simple: true }); + const n = Number(v); + return Number.isFinite(n) ? n : 0; + } + + /** + * Set the connection's `wal_autocheckpoint` interval (pages; 0 disables). + * Bulk indexing defers checkpoints entirely (#1231): the default 1000-page + * auto-checkpoint re-writes hot B-tree/FTS pages into the main DB file over + * and over — measured at ~95% of ALL disk I/O during a bulk index, and the + * difference between 45s and 19+ minutes on HDD-class storage. During + * deferral a {@link WalCheckpointValve} bounds WAL growth off-thread. + */ + setWalAutocheckpoint(pages: number): void { + this.db.pragma(`wal_autocheckpoint = ${Math.max(0, Math.floor(pages))}`); + } + + /** + * `PRAGMA wal_checkpoint(PASSIVE)` on a worker thread with its own + * connection. PASSIVE never blocks the writer, and running it off-thread + * means the main thread — and the #850 watchdog heartbeat — keep turning + * even when the backfill is minutes of I/O on slow storage (a synchronous + * checkpoint that exceeds the watchdog's 60s window gets a healthy index + * SIGKILLed — observed in the #1231 repro). + * + * Returns SQLite's checkpoint result row — `log === checkpointed` with + * `busy === 0` means the ENTIRE WAL was backfilled, so the writer's next + * commit restarts the WAL from the top and the file stops growing. The + * WAL valve needs that signal because a WAL file's SIZE never shrinks: + * after the first wrap, raw file size says nothing about the un-backfilled + * backlog. Best-effort: returns null on any failure (including worker + * threads being unavailable — a potentially minutes-long checkpoint must + * never run inline on the main thread). + */ + async checkpointWalPassive(): Promise<{ busy: number; log: number; checkpointed: number } | null> { + if (!this.dbPath || this.dbPath === ':memory:') { + try { + const row = this.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get() as Record | undefined; + return row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null; + } catch { + return null; + } + } + try { + const { Worker } = await import('node:worker_threads'); + const workerSource = ` + const { workerData, parentPort } = require('node:worker_threads'); + let row = null; + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(workerData.dbPath); + try { row = db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get(); } catch {} + try { db.close(); } catch {} + } catch {} + parentPort.postMessage({ row }); + `; + return await new Promise((resolve) => { + let settled = false; + const finish = (row?: Record | null): void => { + if (settled) return; + settled = true; + resolve(row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null); + }; + try { + const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath } }); + worker.once('message', (m: { row?: Record | null }) => { void worker.terminate(); finish(m?.row ?? null); }); + worker.once('error', () => { void worker.terminate(); finish(null); }); + worker.once('exit', () => finish(null)); + } catch { + finish(null); + } + }); + } catch { + return null; + } + } + + /** + * Optimize database (vacuum and analyze) + */ + optimize(): void { + this.db.exec('VACUUM'); + this.db.exec('ANALYZE'); + } + + /** + * Lightweight maintenance to run after bulk writes (indexAll, sync). + * Two operations: + * + * - `PRAGMA optimize` — incremental ANALYZE; SQLite only re-analyzes + * tables whose row counts changed materially since the last + * ANALYZE. Without it, the query planner has no statistics on the + * freshly-bulk-loaded tables and can pick suboptimal indexes. + * + * - `PRAGMA wal_checkpoint(PASSIVE)` — fold pending WAL pages back + * into the main database file so the WAL file doesn't grow + * unboundedly between automatic checkpoints (auto-fires at 1000 + * pages by default; large indexAll runs blow past that). + * + * Runs on a WORKER THREAD with its own connection: on a multi-GB index + * these pragmas are minutes of synchronous IO (a 95k-file kernel index + * left a 593MB WAL whose checkpoint alone blew the #850 watchdog's 60s + * window and got a COMPLETED index SIGKILLed at the finish line). WAL + * checkpointing from a second connection is standard SQLite; `PRAGMA + * optimize` persists its statistics in sqlite_stat tables, so the main + * connection benefits the same. The main thread just awaits a message, + * so the event loop — and the watchdog heartbeat — keep turning. + * + * Everything is silently swallowed on failure — best-effort + * optimization, never load-bearing for correctness. If worker threads + * are unavailable, falls back to a bounded in-line `PRAGMA optimize` + * and SKIPS the checkpoint (the final close() checkpoints after the + * CLI has already disarmed its watchdog). + */ + async runMaintenance(): Promise { + // In-memory / test databases: nothing worth a worker round-trip. + if (!this.dbPath || this.dbPath === ':memory:') { + try { this.db.exec('PRAGMA optimize'); } catch { /* ignore */ } + try { this.db.exec('PRAGMA wal_checkpoint(PASSIVE)'); } catch { /* ignore */ } + return; + } + await this.runPragmasOffThread( + ['PRAGMA analysis_limit=1000', 'PRAGMA optimize', 'PRAGMA wal_checkpoint(PASSIVE)'], + // Worker threads unavailable — bounded in-line fallback, no checkpoint. + ['PRAGMA analysis_limit=1000', 'PRAGMA optimize'] + ); + } + + /** + * Run pragmas on a worker thread against its own connection to this DB + * (shared machinery for {@link runMaintenance} and + * {@link checkpointWalPassive}). Each pragma is individually best-effort; + * the whole call is best-effort. `inlineFallback` (if any) runs on THIS + * connection only when worker threads are unavailable — keep it to pragmas + * that are safe to run synchronously on the main thread. + */ + private async runPragmasOffThread(pragmas: string[], inlineFallback: string[] = []): Promise { + try { + const { Worker } = await import('node:worker_threads'); + const workerSource = ` + const { workerData, parentPort } = require('node:worker_threads'); + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(workerData.dbPath); + for (const p of workerData.pragmas) { try { db.exec(p); } catch {} } + try { db.close(); } catch {} + } catch {} + parentPort.postMessage('done'); + `; + await new Promise((resolve) => { + let settled = false; + const finish = (): void => { + if (!settled) { settled = true; resolve(); } + }; + try { + const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath, pragmas } }); + worker.once('message', () => { void worker.terminate(); finish(); }); + worker.once('error', () => { void worker.terminate(); finish(); }); + worker.once('exit', finish); + } catch { + finish(); + } + }); + } catch { + for (const p of inlineFallback) { + try { this.db.exec(p); } catch { /* ignore */ } + } + } + } + + /** + * Close the database connection + */ + close(): void { + this.db.close(); + } + + /** + * Check if the database connection is open + */ + isOpen(): boolean { + return this.db.open; + } + + /** + * True when the DB file at our path has been REPLACED on disk since we opened + * it — a different inode now lives at the same path, so the fd we still hold + * points at a now-unlinked inode that can never receive new writes (#925). + * The trigger is removing and recreating `.codegraph/` at the same path under + * a long-lived process (`git worktree remove` + re-add, or `rm -rf + * .codegraph` + `codegraph init`). Returns false when the inode is unchanged, + * when the file is momentarily absent (mid-recreate — nothing to reopen onto + * yet), or when the platform doesn't report a usable inode (Windows can't + * unlink an open file and its st_ino is unreliable, so this never fires there). + */ + isReplacedOnDisk(): boolean { + if (this.openedInode === null) return false; + const current = statInode(this.dbPath); + return current !== null && current !== this.openedInode; + } +} + +/** + * `dev:ino` for a path, or null if it can't be stat'd or the platform doesn't + * report a usable inode. Windows st_ino is unreliable across handle reopens, so + * we deliberately return null there — the deleted-but-open-inode hazard this + * guards (#925) is a POSIX file-semantics issue that doesn't arise on Windows + * (an open file can't be unlinked). + */ +function statInode(p: string): string | null { + if (process.platform === 'win32') return null; + try { + const s = fs.statSync(p); + return `${s.dev}:${s.ino}`; + } catch { + return null; + } +} + +/** + * Default database filename + */ +export const DATABASE_FILENAME = 'codegraph.db'; + +/** + * SQLite's sidecar files in WAL mode — the write-ahead log and its shared-memory + * index. They sit beside the main DB file and are removed alongside it when the + * database is discarded (see `removeDatabaseFiles`). + */ +const WAL_SIDECAR_SUFFIXES = ['-wal', '-shm'] as const; + +/** + * Get the default database path for a project + */ +export function getDatabasePath(projectRoot: string): string { + return path.join(getCodeGraphDir(projectRoot), DATABASE_FILENAME); +} + +/** + * Delete a database file and its WAL sidecars (`-wal`/`-shm`). + * + * This is how a FULL re-index discards an existing database — rather than + * opening the old graph and DELETE-ing every row. On a large or pre-fix + * poisoned index (e.g. an old graph that scanned an ignored gitlink corpus into + * ~1.6M nodes with a multi-GB WAL, #1065) the per-row `nodes_fts` delete-trigger + * churn blocks the main thread long enough to trip the #850 liveness watchdog + * before indexing even starts, so the rebuild could never recover the bad state + * (#1067). Unlinking is O(1) regardless of DB size and also reclaims the disk + * the bloated WAL would otherwise keep. + * + * POSIX removes the directory entry even while another process (a daemon/MCP + * server) still holds the file open; that holder heals via `reopenIfReplaced` + * (#925). On Windows a live holder can make the unlink fail with EBUSY/EPERM — + * that is thrown for the caller to surface ("stop the other process and retry"). + * The `-wal`/`-shm` sidecars are best-effort: SQLite recreates them on the next + * open, so a leftover sidecar is harmless. + */ +export function removeDatabaseFiles(dbPath: string): void { + // The main DB file first — its removal is the operation that must succeed (or + // report why it couldn't). force:true treats an already-missing file as done. + fs.rmSync(dbPath, { force: true }); + for (const suffix of WAL_SIDECAR_SUFFIXES) { + try { + fs.rmSync(dbPath + suffix, { force: true }); + } catch { + // A sidecar still held/locked is harmless — SQLite rebuilds it on open. + } + } +} diff --git a/src/db/migrations.ts b/src/db/migrations.ts new file mode 100644 index 0000000..d083095 --- /dev/null +++ b/src/db/migrations.ts @@ -0,0 +1,234 @@ +/** + * Database Migrations + * + * Schema versioning and migration support. + */ + +import { SqliteDatabase } from './sqlite-adapter'; + +/** + * Current schema version + */ +export const CURRENT_SCHEMA_VERSION = 8; + +/** + * Migration definition + */ +interface Migration { + version: number; + description: string; + up: (db: SqliteDatabase) => void; +} + +/** + * All migrations in order + * + * Note: Version 1 is the initial schema, handled by schema.sql + * Future migrations go here. + */ +const migrations: Migration[] = [ + { + version: 2, + description: 'Add project metadata, provenance tracking, and unresolved ref context', + up: (db) => { + db.exec(` + CREATE TABLE IF NOT EXISTS project_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + ALTER TABLE unresolved_refs ADD COLUMN file_path TEXT NOT NULL DEFAULT ''; + ALTER TABLE unresolved_refs ADD COLUMN language TEXT NOT NULL DEFAULT 'unknown'; + ALTER TABLE edges ADD COLUMN provenance TEXT DEFAULT NULL; + CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path); + CREATE INDEX IF NOT EXISTS idx_edges_provenance ON edges(provenance); + `); + }, + }, + { + version: 3, + description: 'Add lower(name) expression index for memory-efficient case-insensitive lookups', + up: (db) => { + db.exec(` + CREATE INDEX IF NOT EXISTS idx_nodes_lower_name ON nodes(lower(name)); + `); + }, + }, + { + version: 4, + description: + 'Drop redundant idx_edges_source / idx_edges_target (covered by source_kind / target_kind composites)', + up: (db) => { + db.exec(` + DROP INDEX IF EXISTS idx_edges_source; + DROP INDEX IF EXISTS idx_edges_target; + `); + }, + }, + { + version: 5, + description: + 'Add nodes.return_type — normalized return/result type for receiver-type inference (C++ singletons/factories, #645)', + up: (db) => { + db.exec(` + ALTER TABLE nodes ADD COLUMN return_type TEXT; + `); + }, + }, + { + version: 6, + description: + 'Dedup duplicate edge rows and add a UNIQUE identity index so INSERT OR IGNORE actually dedups (#1034)', + up: (db) => { + // `insertEdge` has always used `INSERT OR IGNORE`, but the edges table had + // no UNIQUE constraint, so nothing conflicted and byte-identical rows + // accumulated whenever two passes emitted the same edge. Collapse each + // identity group to its lowest id, then add the constraint that makes + // `OR IGNORE` keep its promise. IFNULL folds nullable line/col so + // coordinate-less edges dedup too (SQLite treats each NULL as distinct) — + // and it MUST match the GROUP BY exactly, or the index creation would + // fail on a pair the DELETE left behind. Idempotent: the index is + // `IF NOT EXISTS` and the DELETE is a no-op once the table is unique. + db.exec(` + DELETE FROM edges + WHERE id NOT IN ( + SELECT MIN(id) FROM edges + GROUP BY source, target, kind, IFNULL(line, -1), IFNULL(col, -1) + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_identity + ON edges(source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); + `); + }, + }, + { + version: 7, + description: + 'Add name_segment_vocab — prose-word → symbol-name lookup for the prompt hook’s graph-derived gate', + up: (db) => { + // DDL only — instant on any size database (the row-churn hazards of #1067 + // don't apply). The table starts EMPTY on migrated databases; `sync` + // detects that over a populated graph and backfills batched+yielding + // (CodeGraph.rebuildNameSegmentVocab), and any full index rebuilds it + // from scratch. Keep the definition in lockstep with schema.sql. + db.exec(` + CREATE TABLE IF NOT EXISTS name_segment_vocab ( + segment TEXT NOT NULL, + name TEXT NOT NULL, + PRIMARY KEY (segment, name) + ) WITHOUT ROWID; + `); + }, + }, + { + version: 8, + description: + 'Track attempted-but-unresolvable refs as status=failed so sync can retry them when a changed file adds a matching symbol (#1240)', + up: (db) => { + // DDL only — instant on any size database. No backfill needed: rows are + // only ever queried by name_tail once they carry status='failed', and + // both fields are written together by markReferencesFailed. Legacy rows + // (all 'pending' after this migration) are orphans from interrupted runs + // that the #1187 sweep grinds down on the next sync, marking survivors + // failed with their tails as it goes. The tail index is partial: on a + // healthy index the pending set is empty and the failed set is the only + // population worth indexing. Keep the definitions in lockstep with + // schema.sql. ALTER TABLE has no IF NOT EXISTS, so guard each column for + // idempotency — a database created from current schema.sql already has + // both (matters when migrations are re-run from an older recorded + // version, as the v6 regression test does). + const cols = db.prepare('PRAGMA table_info(unresolved_refs)').all() as Array<{ name: string }>; + const hasColumn = (name: string) => cols.some((c) => c.name === name); + if (!hasColumn('status')) { + db.exec("ALTER TABLE unresolved_refs ADD COLUMN status TEXT NOT NULL DEFAULT 'pending'"); + } + if (!hasColumn('name_tail')) { + db.exec("ALTER TABLE unresolved_refs ADD COLUMN name_tail TEXT NOT NULL DEFAULT ''"); + } + db.exec(` + CREATE INDEX IF NOT EXISTS idx_unresolved_status ON unresolved_refs(status); + CREATE INDEX IF NOT EXISTS idx_unresolved_failed_tail ON unresolved_refs(name_tail) WHERE status = 'failed'; + `); + }, + }, +]; + +/** + * Get the current schema version from the database + */ +export function getCurrentVersion(db: SqliteDatabase): number { + try { + const row = db + .prepare('SELECT MAX(version) as version FROM schema_versions') + .get() as { version: number | null } | undefined; + return row?.version ?? 0; + } catch { + // Table doesn't exist yet + return 0; + } +} + +/** + * Record a migration as applied + */ +function recordMigration(db: SqliteDatabase, version: number, description: string): void { + db.prepare( + 'INSERT INTO schema_versions (version, applied_at, description) VALUES (?, ?, ?)' + ).run(version, Date.now(), description); +} + +/** + * Run all pending migrations + */ +export function runMigrations(db: SqliteDatabase, fromVersion: number): void { + const pending = migrations.filter((m) => m.version > fromVersion); + + if (pending.length === 0) { + return; + } + + // Sort by version + pending.sort((a, b) => a.version - b.version); + + // Run each migration in a transaction + for (const migration of pending) { + db.transaction(() => { + migration.up(db); + recordMigration(db, migration.version, migration.description); + })(); + } +} + +/** + * Check if the database needs migration + */ +export function needsMigration(db: SqliteDatabase): boolean { + const current = getCurrentVersion(db); + return current < CURRENT_SCHEMA_VERSION; +} + +/** + * Get list of pending migrations + */ +export function getPendingMigrations(db: SqliteDatabase): Migration[] { + const current = getCurrentVersion(db); + return migrations + .filter((m) => m.version > current) + .sort((a, b) => a.version - b.version); +} + +/** + * Get migration history from database + */ +export function getMigrationHistory( + db: SqliteDatabase +): Array<{ version: number; appliedAt: number; description: string | null }> { + const rows = db + .prepare('SELECT version, applied_at, description FROM schema_versions ORDER BY version') + .all() as Array<{ version: number; applied_at: number; description: string | null }>; + + return rows.map((row) => ({ + version: row.version, + appliedAt: row.applied_at, + description: row.description, + })); +} diff --git a/src/db/queries.ts b/src/db/queries.ts new file mode 100644 index 0000000..15f2611 --- /dev/null +++ b/src/db/queries.ts @@ -0,0 +1,2245 @@ +/** + * Database Queries + * + * Prepared statements for CRUD operations on the knowledge graph. + */ + +import { SqliteDatabase, SqliteStatement } from './sqlite-adapter'; +import { + Node, + Edge, + FileRecord, + UnresolvedReference, + NodeKind, + EdgeKind, + Language, + GraphStats, + SearchOptions, + SearchResult, +} from '../types'; +import { safeJsonParse } from '../utils'; +import { kindBonus, nameMatchBonus, scorePathRelevance } from '../search/query-utils'; +import { parseQuery, boundedEditDistance } from '../search/query-parser'; +import { isGeneratedFile } from '../extraction/generated-detection'; +import { splitIdentifierSegments } from '../search/identifier-segments'; + +/** + * Path-only heuristic for files that should not be candidates for + * "dominant file" detection: test/spec files and tool-generated files. + * Generated files (`*.pb.go`, `*.pulsar.go`, mock outputs, …) often + * have huge in-file edge counts that dwarf the real source — etcd's + * `rpc.pb.go` has 4× the in-file edges of `server.go`. + */ +function isLowValueFile(filePath: string): boolean { + const lp = filePath.toLowerCase(); + return ( + /(?:^|\/)(tests?|__tests?__|spec)\//.test(lp) || + /_test\.go$/.test(lp) || + /(?:^|\/)test_[^/]+\.py$/.test(lp) || + /_test\.py$/.test(lp) || + /_spec\.rb$/.test(lp) || + /_test\.rb$/.test(lp) || + /\.(test|spec)\.[jt]sx?$/.test(lp) || + /(test|spec|tests)\.(java|kt|scala)$/.test(lp) || + /(tests?|spec)\.cs$/.test(lp) || + /tests?\.swift$/.test(lp) || + /_test\.dart$/.test(lp) || + isGeneratedFile(filePath) + ); +} + +const SQLITE_PARAM_CHUNK_SIZE = 500; + +/** + * Database row types (snake_case from SQLite) + */ +interface NodeRow { + id: string; + kind: string; + name: string; + qualified_name: string; + file_path: string; + language: string; + start_line: number; + end_line: number; + start_column: number; + end_column: number; + docstring: string | null; + signature: string | null; + visibility: string | null; + is_exported: number; + is_async: number; + is_static: number; + is_abstract: number; + decorators: string | null; + type_parameters: string | null; + return_type: string | null; + updated_at: number; +} + +interface EdgeRow { + id: number; + source: string; + target: string; + kind: string; + metadata: string | null; + line: number | null; + col: number | null; + provenance: string | null; +} + +interface FileRow { + path: string; + content_hash: string; + language: string; + size: number; + modified_at: number; + indexed_at: number; + node_count: number; + errors: string | null; +} + +interface UnresolvedRefRow { + id: number; + from_node_id: string; + reference_name: string; + reference_kind: string; + line: number; + col: number; + candidates: string | null; + file_path: string; + language: string; + status: string; + name_tail: string; +} + +/** + * Last segment of a (possibly dotted/qualified) reference name — the part a + * new symbol's plain node name could match: 'util.greet' → 'greet', + * 'mod::fn' → 'fn', 'greet' → 'greet'. Written to unresolved_refs.name_tail + * when a ref is marked failed, so the #1240 retry lookup can match dotted + * refs against newly-added node names. + */ +function referenceNameTail(referenceName: string): string { + const idx = Math.max(referenceName.lastIndexOf('.'), referenceName.lastIndexOf(':')); + return idx >= 0 ? referenceName.slice(idx + 1) : referenceName; +} + +/** + * Convert database row to Node object + */ +function rowToNode(row: NodeRow): Node { + return { + id: row.id, + kind: row.kind as NodeKind, + name: row.name, + qualifiedName: row.qualified_name, + filePath: row.file_path, + language: row.language as Language, + startLine: row.start_line, + endLine: row.end_line, + startColumn: row.start_column, + endColumn: row.end_column, + docstring: row.docstring ?? undefined, + signature: row.signature ?? undefined, + visibility: row.visibility as Node['visibility'], + isExported: row.is_exported === 1, + isAsync: row.is_async === 1, + isStatic: row.is_static === 1, + isAbstract: row.is_abstract === 1, + decorators: row.decorators ? safeJsonParse(row.decorators, undefined) : undefined, + typeParameters: row.type_parameters ? safeJsonParse(row.type_parameters, undefined) : undefined, + returnType: row.return_type ?? undefined, + updatedAt: row.updated_at, + }; +} + +/** + * Convert database row to Edge object + */ +function rowToEdge(row: EdgeRow): Edge { + return { + source: row.source, + target: row.target, + kind: row.kind as EdgeKind, + metadata: row.metadata ? safeJsonParse(row.metadata, undefined) : undefined, + line: row.line ?? undefined, + column: row.col ?? undefined, + provenance: row.provenance as Edge['provenance'], + }; +} + +/** + * Convert database row to FileRecord object + */ +function rowToFileRecord(row: FileRow): FileRecord { + return { + path: row.path, + contentHash: row.content_hash, + language: row.language as Language, + size: row.size, + modifiedAt: row.modified_at, + indexedAt: row.indexed_at, + nodeCount: row.node_count, + errors: row.errors ? safeJsonParse(row.errors, undefined) : undefined, + }; +} + +/** + * Query builder for the knowledge graph database + */ +export class QueryBuilder { + private db: SqliteDatabase; + + // Project-name tokens (go.mod / package.json / repo dir), normalized. A query + // word matching one is dropped from path-relevance scoring — it names the + // whole project, not a symbol, so it carries no discriminative signal (#720). + // Set once by the CodeGraph instance; empty by default (no down-weighting). + private projectNameTokens: Set = new Set(); + + // Node cache for frequently accessed nodes (LRU-style, max 1000 entries) + private nodeCache: Map = new Map(); + private readonly maxCacheSize = 1000; + + // Prepared statements (lazily initialized) + private stmts: { + insertNode?: SqliteStatement; + updateNode?: SqliteStatement; + deleteNode?: SqliteStatement; + deleteNodesByFile?: SqliteStatement; + getNodeById?: SqliteStatement; + getNodesByFile?: SqliteStatement; + getNodesByKind?: SqliteStatement; + insertEdge?: SqliteStatement; + upsertFile?: SqliteStatement; + deleteEdgesBySource?: SqliteStatement; + deleteEdgesByTarget?: SqliteStatement; + getEdgesBySource?: SqliteStatement; + getEdgesByTarget?: SqliteStatement; + insertFile?: SqliteStatement; + updateFile?: SqliteStatement; + deleteFile?: SqliteStatement; + getFileByPath?: SqliteStatement; + getAllFiles?: SqliteStatement; + insertUnresolved?: SqliteStatement; + deleteUnresolvedByNode?: SqliteStatement; + getUnresolvedByName?: SqliteStatement; + getNodesByName?: SqliteStatement; + getNodesByNamePrefix?: SqliteStatement; + getNodesByQualifiedNameExact?: SqliteStatement; + getNodesByLowerName?: SqliteStatement; + getUnresolvedCount?: SqliteStatement; + getUnresolvedBatch?: SqliteStatement; + getAllFilePaths?: SqliteStatement; + getAllNodeNames?: SqliteStatement; + getDominantFile?: SqliteStatement; + getTopRouteFile?: SqliteStatement; + getRoutingManifest?: SqliteStatement; + insertNameSegment?: SqliteStatement; + } = {}; + + // Names whose segments were already written this session — skips re-splitting + // and re-inserting for the same-named nodes that repeat across files ("get", + // "render", …). Purely a write-path fast path; INSERT OR IGNORE is the + // correctness backstop. Bounded so a pathological repo can't grow it forever. + private segmentedNames: Set = new Set(); + private static readonly MAX_SEGMENTED_NAMES = 65536; + + constructor(db: SqliteDatabase) { + this.db = db; + } + + /** Set the normalized project-name tokens used to down-weight non-discriminative + * query words in path scoring (#720). Called once when the project opens. */ + setProjectNameTokens(tokens: Set): void { + this.projectNameTokens = tokens; + } + + /** The normalized project-name tokens (#720); empty if none were derived. */ + getProjectNameTokens(): Set { + return this.projectNameTokens; + } + + // =========================================================================== + // Node Operations + // =========================================================================== + + /** + * Insert a new node + */ + insertNode(node: Node): void { + if (!this.stmts.insertNode) { + this.stmts.insertNode = this.db.prepare(` + INSERT OR REPLACE INTO nodes ( + id, kind, name, qualified_name, file_path, language, + start_line, end_line, start_column, end_column, + docstring, signature, visibility, + is_exported, is_async, is_static, is_abstract, + decorators, type_parameters, return_type, updated_at + ) VALUES ( + @id, @kind, @name, @qualifiedName, @filePath, @language, + @startLine, @endLine, @startColumn, @endColumn, + @docstring, @signature, @visibility, + @isExported, @isAsync, @isStatic, @isAbstract, + @decorators, @typeParameters, @returnType, @updatedAt + ) + `); + } + + // Validate required fields to prevent SQLite bind errors + if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { + console.error('[CodeGraph] Skipping node with missing required fields:', { + id: node.id, + kind: node.kind, + name: node.name, + filePath: node.filePath, + language: node.language, + }); + return; + } + + // INSERT OR REPLACE may overwrite a node we have cached. Drop the + // stale entry so the next getNodeById sees the new row, not the old + // one (matches the cache-invalidation pattern used by updateNode and + // deleteNode below). + this.nodeCache.delete(node.id); + + this.stmts.insertNode.run({ + id: node.id, + kind: node.kind, + name: node.name, + qualifiedName: node.qualifiedName ?? node.name, + filePath: node.filePath, + language: node.language, + startLine: node.startLine ?? 0, + endLine: node.endLine ?? 0, + startColumn: node.startColumn ?? 0, + endColumn: node.endColumn ?? 0, + docstring: node.docstring ?? null, + signature: node.signature ?? null, + visibility: node.visibility ?? null, + isExported: node.isExported ? 1 : 0, + isAsync: node.isAsync ? 1 : 0, + isStatic: node.isStatic ? 1 : 0, + isAbstract: node.isAbstract ? 1 : 0, + decorators: node.decorators ? JSON.stringify(node.decorators) : null, + typeParameters: node.typeParameters ? JSON.stringify(node.typeParameters) : null, + returnType: node.returnType ?? null, + updatedAt: node.updatedAt ?? Date.now(), + }); + + // Segment vocabulary rides the same write path (and transaction) so it can + // never drift ahead of the nodes it describes. Deletes intentionally leave + // orphans behind — vocab rows are proposals re-verified against nodes + // before use, and a full index clears the table at its start. File nodes + // are excluded: a file's basename duplicates the symbols inside it + // (state-machine.ts / OrderStateMachine), which double-counts every + // concept and defeats the singleton-vs-cluster rarity statistics. Import + // nodes are excluded too (#1144): they're named after module specifiers + // ("external-unindexed-pkg", "./utils/helpers"), not symbols — an + // import-only name can never be surfaced (getSegmentMatches requires a + // real definition), so its rows would only inflate the rarity statistics. + if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); + } + + /** Which node kinds contribute their name to the segment vocabulary — the + * single gate shared by insertNode, updateNode, and the rebuild page query + * (getDistinctNodeNames), so the write paths can't drift apart. */ + private isSegmentableKind(kind: string): boolean { + return kind !== 'file' && kind !== 'import'; + } + + /** Write `name`'s segments into name_segment_vocab (idempotent). */ + private insertNameSegments(name: string): void { + if (this.segmentedNames.has(name)) return; + if (this.segmentedNames.size >= QueryBuilder.MAX_SEGMENTED_NAMES) this.segmentedNames.clear(); + this.segmentedNames.add(name); + if (!this.stmts.insertNameSegment) { + this.stmts.insertNameSegment = this.db.prepare( + 'INSERT OR IGNORE INTO name_segment_vocab (segment, name) VALUES (?, ?)', + ); + } + for (const segment of splitIdentifierSegments(name)) { + this.stmts.insertNameSegment.run(segment, name); + } + } + + /** + * Insert multiple nodes in a transaction + */ + insertNodes(nodes: Node[]): void { + this.db.transaction(() => { + for (const node of nodes) { + this.insertNode(node); + } + })(); + } + + /** + * Update an existing node + */ + updateNode(node: Node): void { + if (!this.stmts.updateNode) { + this.stmts.updateNode = this.db.prepare(` + UPDATE nodes SET + kind = @kind, + name = @name, + qualified_name = @qualifiedName, + file_path = @filePath, + language = @language, + start_line = @startLine, + end_line = @endLine, + start_column = @startColumn, + end_column = @endColumn, + docstring = @docstring, + signature = @signature, + visibility = @visibility, + is_exported = @isExported, + is_async = @isAsync, + is_static = @isStatic, + is_abstract = @isAbstract, + decorators = @decorators, + type_parameters = @typeParameters, + return_type = @returnType, + updated_at = @updatedAt + WHERE id = @id + `); + } + + // Invalidate cache before update + this.nodeCache.delete(node.id); + + // Validate required fields + if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { + console.error('[CodeGraph] Skipping node update with missing required fields:', node.id); + return; + } + + this.stmts.updateNode.run({ + id: node.id, + kind: node.kind, + name: node.name, + qualifiedName: node.qualifiedName ?? node.name, + filePath: node.filePath, + language: node.language, + startLine: node.startLine ?? 0, + endLine: node.endLine ?? 0, + startColumn: node.startColumn ?? 0, + endColumn: node.endColumn ?? 0, + docstring: node.docstring ?? null, + signature: node.signature ?? null, + visibility: node.visibility ?? null, + isExported: node.isExported ? 1 : 0, + isAsync: node.isAsync ? 1 : 0, + isStatic: node.isStatic ? 1 : 0, + isAbstract: node.isAbstract ? 1 : 0, + decorators: node.decorators ? JSON.stringify(node.decorators) : null, + typeParameters: node.typeParameters ? JSON.stringify(node.typeParameters) : null, + returnType: node.returnType ?? null, + updatedAt: node.updatedAt ?? Date.now(), + }); + + // updateNode is a second real write path to `nodes` — framework + // post-extract passes rewrite names through it (NestJS route prefixing), + // and a renamed node's new name must reach the segment vocabulary just + // like an inserted one's (#1141). Without this the rename left the new + // name permanently unsearchable: the old name's rows became honest-gate + // orphans and the only backfill is gated on the vocab being EMPTY. + // insertNameSegments is idempotent (in-memory set + INSERT OR IGNORE), + // so no name-changed check is needed. + if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); + } + + /** + * Delete a node by ID + */ + deleteNode(id: string): void { + if (!this.stmts.deleteNode) { + this.stmts.deleteNode = this.db.prepare('DELETE FROM nodes WHERE id = ?'); + } + // Invalidate cache + this.nodeCache.delete(id); + this.stmts.deleteNode.run(id); + } + + /** + * Delete all nodes for a file + */ + deleteNodesByFile(filePath: string): void { + if (!this.stmts.deleteNodesByFile) { + this.stmts.deleteNodesByFile = this.db.prepare('DELETE FROM nodes WHERE file_path = ?'); + } + // Invalidate cache for nodes in this file + for (const [id, node] of this.nodeCache) { + if (node.filePath === filePath) { + this.nodeCache.delete(id); + } + } + this.stmts.deleteNodesByFile.run(filePath); + } + + // =========================================================================== + // Name-segment vocabulary (prompt-hook graph-derived gate) + // =========================================================================== + + /** Wipe the segment vocabulary. A full index calls this at its start; the + * node write path repopulates it as files (re-)index, so the end state is + * exactly the current names with no orphan rows. */ + clearNameSegmentVocab(): void { + this.db.exec('DELETE FROM name_segment_vocab'); + this.segmentedNames.clear(); + } + + /** True when the vocab has no rows — an index built before the table existed. + * `sync` uses this to heal such databases (see rebuildNameSegmentVocabFrom). */ + isNameSegmentVocabEmpty(): boolean { + const row = this.db.prepare('SELECT 1 FROM name_segment_vocab LIMIT 1').get(); + return row === undefined; + } + + /** One page of distinct segmentable node names, for batched vocab rebuilds + * (file basenames and import specifiers are excluded from the vocab — see + * insertNode). */ + getDistinctNodeNames(limit: number, offset: number): string[] { + const rows = this.db + .prepare("SELECT DISTINCT name FROM nodes WHERE kind NOT IN ('file', 'import') ORDER BY name LIMIT ? OFFSET ?") + .all(limit, offset) as Array<{ name: string }>; + return rows.map((r) => r.name); + } + + /** Insert segments for a batch of names in one transaction (vocab heal path). */ + insertNameSegmentsBatch(names: string[]): void { + this.db.transaction(() => { + for (const name of names) this.insertNameSegments(name); + })(); + } + + /** + * Names whose segments cover at least `minWords` distinct PROMPT WORDS — + * the co-occurrence probe behind the prompt hook's medium tier: the words + * "state" and "machine" both being segments of `OrderStateMachine` is strong + * evidence the prompt names that symbol in prose. Ordered by coverage. + * + * Takes (segment variant → original word) pairs and folds variants back to + * their word INSIDE the SQL: a name matching both `service` and `services` + * counts ONE word, not two. Counting raw variants let plural-variant pairs + * of a single word tie with genuine two-word matches and — because ORDER + * BY/LIMIT run here, before any JS-side re-check — crowd a real match past + * the LIMIT on vocab-heavy repos (#1146). + */ + getSegmentCoOccurrence( + variants: Array<{ segment: string; word: string }>, + minWords: number, + limit: number, + ): Array<{ name: string; matches: number }> { + if (variants.length === 0) return []; + const placeholders = variants.map(() => '?').join(', '); + const whens = variants.map(() => 'WHEN ? THEN ?').join(' '); + const rows = this.db + .prepare( + `SELECT name, COUNT(DISTINCT CASE segment ${whens} END) AS matches + FROM name_segment_vocab + WHERE segment IN (${placeholders}) + GROUP BY name + HAVING matches >= ? + ORDER BY matches DESC, length(name) ASC + LIMIT ?`, + ) + .all( + ...variants.flatMap((v) => [v.segment, v.word]), + ...variants.map((v) => v.segment), + minWords, + limit, + ) as Array<{ name: string; matches: number }>; + return rows; + } + + /** How many distinct names each segment appears in — the rarity signal that + * separates a discriminative word ("checkout") from a ubiquitous one ("state"). */ + getSegmentNameCounts(segments: string[]): Map { + if (segments.length === 0) return new Map(); + const placeholders = segments.map(() => '?').join(', '); + const rows = this.db + .prepare( + `SELECT segment, COUNT(*) AS n FROM name_segment_vocab + WHERE segment IN (${placeholders}) GROUP BY segment`, + ) + .all(...segments) as Array<{ segment: string; n: number }>; + return new Map(rows.map((r) => [r.segment, r.n])); + } + + /** Names containing the given segment (rare-single-word tier). */ + getNamesForSegment(segment: string, limit: number): string[] { + const rows = this.db + .prepare('SELECT name FROM name_segment_vocab WHERE segment = ? ORDER BY length(name) ASC LIMIT ?') + .all(segment, limit) as Array<{ name: string }>; + return rows.map((r) => r.name); + } + + /** + * Get a node by ID + */ + getNodeById(id: string): Node | null { + // Check cache first + if (this.nodeCache.has(id)) { + const cached = this.nodeCache.get(id)!; + // Move to end to implement LRU (delete and re-add) + this.nodeCache.delete(id); + this.nodeCache.set(id, cached); + return cached; + } + + if (!this.stmts.getNodeById) { + this.stmts.getNodeById = this.db.prepare('SELECT * FROM nodes WHERE id = ?'); + } + const row = this.stmts.getNodeById.get(id) as NodeRow | undefined; + if (!row) { + return null; + } + + const node = rowToNode(row); + this.cacheNode(node); + return node; + } + + /** + * Batch lookup: fetch many nodes by ID in a single SQL round-trip. + * + * Replaces the N+1 pattern in graph traversal where every edge would + * trigger its own `getNodeById` call. For a function with 50 callers + * this collapses 50 point reads into one IN-list query (~10-50x + * faster end-to-end). + * + * Returns a Map keyed by id so callers can preserve their own ordering + * (typically the order edges were returned from the graph). Missing IDs + * are simply absent from the map. + * + * Cache-aware: ids already in the LRU cache are served from memory and + * the SQL query only touches the misses. + */ + getNodesByIds(ids: readonly string[]): Map { + const out = new Map(); + if (ids.length === 0) return out; + + // Serve cache hits first; build the miss list for SQL. + const misses: string[] = []; + for (const id of ids) { + const cached = this.nodeCache.get(id); + if (cached !== undefined) { + // LRU touch + this.nodeCache.delete(id); + this.nodeCache.set(id, cached); + out.set(id, cached); + } else { + misses.push(id); + } + } + if (misses.length === 0) return out; + + // Chunk under SQLite's parameter limit (default 999, raised to 32766 + // in better-sqlite3 builds — chunk at 500 for safety across both + // backends and to keep the query plan simple). + for (let i = 0; i < misses.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = misses.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT * FROM nodes WHERE id IN (${placeholders})`) + .all(...chunk) as NodeRow[]; + for (const row of rows) { + const node = rowToNode(row); + out.set(node.id, node); + this.cacheNode(node); + } + } + return out; + } + + private getExistingNodeIds(ids: readonly string[]): Set { + const out = new Set(); + if (ids.length === 0) return out; + + const uniqueIds = [...new Set(ids)]; + for (let i = 0; i < uniqueIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = uniqueIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT id FROM nodes WHERE id IN (${placeholders})`) + .all(...chunk) as { id: string }[]; + for (const row of rows) { + out.add(row.id); + } + } + + return out; + } + + /** + * Add a node to the cache, evicting oldest if needed + */ + private cacheNode(node: Node): void { + if (this.nodeCache.size >= this.maxCacheSize) { + // Evict oldest (first) entry + const firstKey = this.nodeCache.keys().next().value; + if (firstKey) { + this.nodeCache.delete(firstKey); + } + } + this.nodeCache.set(node.id, node); + } + + /** + * Clear the node cache + */ + clearCache(): void { + this.nodeCache.clear(); + } + + /** + * Get all nodes in a file + */ + getNodesByFile(filePath: string): Node[] { + if (!this.stmts.getNodesByFile) { + this.stmts.getNodesByFile = this.db.prepare( + 'SELECT * FROM nodes WHERE file_path = ? ORDER BY start_line' + ); + } + const rows = this.stmts.getNodesByFile.all(filePath) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Find the file that holds the densest concentration of the project's + * internal call graph — the "core" file. Used by context-builder to + * boost ranking of symbols in that file's directory (so e.g. sinatra + * queries surface `lib/sinatra/base.rb`'s `route!` instead of + * `sinatra-contrib/lib/sinatra/multi_route.rb`'s `route` extension). + * + * Returns null if no file has a meaningful concentration (e.g. spread + * evenly across many files, or empty index). + * + * "Internal" = source and target are in the same file. Cross-file + * edges aren't useful here — they don't tell us which file is the + * functional center. + * + * Excludes test/spec files from candidacy via path-pattern. The agent's + * typical question is "how does X work", not "how is X tested", so + * boosting a test file's directory would be a misfire. + */ + getDominantFile(): { filePath: string; edgeCount: number; nextEdgeCount: number } | null { + if (!this.stmts.getDominantFile) { + // Pull top 20 candidates; we then filter out test/generated files + // in code (regex-grade matching that SQL LIKE can't express). The + // generated-file filter is critical — without it, etcd's + // `api/etcdserverpb/rpc.pb.go` (1916 in-file edges, generated + // protobuf stub) outranks the real `server/etcdserver/server.go` + // (470 edges) by 4×, and the boost would push the agent toward + // generated code. + this.stmts.getDominantFile = this.db.prepare(` + SELECT n.file_path AS file_path, COUNT(*) AS edge_count + FROM edges e + JOIN nodes n ON e.source = n.id + JOIN nodes m ON e.target = m.id + WHERE n.file_path = m.file_path + GROUP BY n.file_path + ORDER BY edge_count DESC + LIMIT 20 + `); + } + const rows = this.stmts.getDominantFile.all() as Array<{ file_path: string; edge_count: number }>; + const filtered = rows.filter(r => !isLowValueFile(r.file_path)); + if (filtered.length === 0 || filtered[0]!.edge_count < 20) return null; + return { + filePath: filtered[0]!.file_path, + edgeCount: filtered[0]!.edge_count, + nextEdgeCount: filtered[1]?.edge_count ?? 0, + }; + } + + /** + * Find the file that holds the densest concentration of the project's + * `route` nodes (framework-emitted: Express/Gin/Flask/Rails/Drupal/etc.). + * Used by handleContext on small repos to inline the project's routing + * config when the agent's query is about request flow — eliminating the + * "Glob + Read routes.rb" pattern that beats codegraph on tiny realworld + * template repos. + * + * Excludes test/generated files from candidacy. Returns null if there + * are fewer than 3 non-test routes total, or if no file holds at least + * 30% of them (diffuse routing → no single answer file). + */ + getTopRouteFile(): { filePath: string; routeCount: number; totalRoutes: number } | null { + if (!this.stmts.getTopRouteFile) { + this.stmts.getTopRouteFile = this.db.prepare(` + SELECT file_path, COUNT(*) AS cnt + FROM nodes + WHERE kind = 'route' + GROUP BY file_path + ORDER BY cnt DESC + LIMIT 20 + `); + } + const rows = this.stmts.getTopRouteFile.all() as Array<{ file_path: string; cnt: number }>; + const filtered = rows.filter(r => !isLowValueFile(r.file_path)); + if (filtered.length === 0) return null; + const totalRoutes = filtered.reduce((sum, r) => sum + r.cnt, 0); + const top = filtered[0]!; + if (totalRoutes < 3 || top.cnt < 3) return null; + if (top.cnt / totalRoutes < 0.30) return null; + return { filePath: top.file_path, routeCount: top.cnt, totalRoutes }; + } + + /** + * Build a URL → handler manifest from the index. Each route node's + * `references` edge points at the function/method that handles the + * request. We join them in one pass; the agent gets the canonical + * routing answer ("POST /users/login → AuthController#login") without + * having to parse the framework's route DSL itself. + * + * Also returns the file with the most handler endpoints — used as the + * "top handler file" to inline source for, so the agent has both the + * mapping AND the handler implementations. + */ + getRoutingManifest(limit: number = 40): { + entries: Array<{ url: string; handler: string; handlerFile: string; handlerLine: number; handlerKind: string }>; + topHandlerFile: string | null; + topHandlerFileCount: number; + totalRoutes: number; + } | null { + if (!this.stmts.getRoutingManifest) { + // Edge kind varies across framework resolvers: Spring/Rails/ + // Laravel/Drupal emit `references`, Express emits `calls`. Accept + // both — the semantic is the same (route → its handler). + this.stmts.getRoutingManifest = this.db.prepare(` + SELECT + r.name AS url, + h.name AS handler, + h.file_path AS handler_file, + h.start_line AS handler_line, + h.kind AS handler_kind + FROM nodes r + JOIN edges e ON e.source = r.id + JOIN nodes h ON e.target = h.id + WHERE r.kind = 'route' + AND e.kind IN ('references', 'calls') + AND h.kind IN ('function', 'method', 'class') + ORDER BY r.file_path, r.start_line + LIMIT ? + `); + } + const rows = this.stmts.getRoutingManifest.all(limit) as Array<{ + url: string; handler: string; handler_file: string; handler_line: number; handler_kind: string; + }>; + // Drop test/generated handlers — same hygiene as elsewhere. + const filtered = rows.filter(r => !isLowValueFile(r.handler_file)); + if (filtered.length < 3) return null; + // Identify the file holding the most handlers (the "primary handler file"). + const fileCounts = new Map(); + for (const r of filtered) { + fileCounts.set(r.handler_file, (fileCounts.get(r.handler_file) ?? 0) + 1); + } + let topHandlerFile: string | null = null; + let topHandlerFileCount = 0; + for (const [file, count] of fileCounts) { + if (count > topHandlerFileCount) { + topHandlerFile = file; + topHandlerFileCount = count; + } + } + return { + entries: filtered.map(r => ({ + url: r.url, + handler: r.handler, + handlerFile: r.handler_file, + handlerLine: r.handler_line, + handlerKind: r.handler_kind, + })), + topHandlerFile, + topHandlerFileCount, + totalRoutes: filtered.length, + }; + } + + /** + * Get all nodes of a specific kind + */ + getNodesByKind(kind: NodeKind): Node[] { + if (!this.stmts.getNodesByKind) { + this.stmts.getNodesByKind = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); + } + const rows = this.stmts.getNodesByKind.all(kind) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Stream every node of a kind one at a time (lazy) instead of materializing + * them all like {@link getNodesByKind}. For unbounded kinds (`function`, + * `method`) on a symbol-dense project the full array is gigabytes; the + * dynamic-edge synthesizers only scan-and-filter, so they iterate to keep + * memory O(1) in the node count rather than O(nodes) (#610). + */ + *iterateNodesByKind(kind: NodeKind): IterableIterator { + // Fresh statement per call (not a cached one): an iterator holds an open + // cursor, so a shared statement would conflict across overlapping scans. + const stmt = this.db.prepare('SELECT * FROM nodes WHERE kind = ?'); + for (const row of stmt.iterate(kind)) { + yield rowToNode(row as NodeRow); + } + } + + /** + * Get all nodes in the database + */ + getAllNodes(): Node[] { + const rows = this.db.prepare('SELECT * FROM nodes').all() as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Stream nodes of one language whose `decorators` JSON array contains + * `decorator`. The LIKE on the JSON text is a cheap index-free pre-filter + * (a decorator name can appear as a substring of another), so callers must + * still exact-check `node.decorators.includes(decorator)`. Exists so the + * kotlin expect/actual synthesizer never materializes the whole node table + * the way `getAllNodes().filter(...)` did — that array alone OOM'd Node's + * default heap on a 2M-node graph (#1212). + */ + *iterateNodesByLanguageWithDecorator(language: Language, decorator: string): IterableIterator { + // Fresh statement per call — an iterator holds an open cursor (see + // iterateNodesByKind). + const stmt = this.db.prepare( + "SELECT * FROM nodes WHERE language = ? AND decorators LIKE '%' || ? || '%'" + ); + for (const row of stmt.iterate(language, `"${decorator}"`)) { + yield rowToNode(row as NodeRow); + } + } + + /** + * Distinct languages present in the files table. One indexed aggregate — + * lets the dynamic-edge synthesizers skip passes for languages the project + * doesn't contain at all (a Kotlin pass has no work on a pure-C repo), so + * their cost is zero rather than a full-graph scan that finds nothing (#1212). + */ + getDistinctFileLanguages(): Set { + const rows = this.db.prepare('SELECT DISTINCT language FROM files').all() as Array<{ language: string }>; + return new Set(rows.map((r) => r.language)); + } + + /** + * Get nodes by exact name match (uses idx_nodes_name index) + */ + getNodesByName(name: string): Node[] { + if (!this.stmts.getNodesByName) { + this.stmts.getNodesByName = this.db.prepare('SELECT * FROM nodes WHERE name = ?'); + } + const rows = this.stmts.getNodesByName.all(name) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Nodes whose name starts with `prefix`, by index range scan (a LIKE would + * skip idx_nodes_name under SQLite's default case-insensitive LIKE). + */ + getNodesByNamePrefix(prefix: string, limit = 20): Node[] { + if (!this.stmts.getNodesByNamePrefix) { + this.stmts.getNodesByNamePrefix = this.db.prepare( + 'SELECT * FROM nodes WHERE name >= ? AND name < ? ORDER BY name LIMIT ?' + ); + } + const rows = this.stmts.getNodesByNamePrefix.all(prefix, prefix + '', limit) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Get nodes by exact qualified name match (uses idx_nodes_qualified_name index) + */ + getNodesByQualifiedNameExact(qualifiedName: string): Node[] { + if (!this.stmts.getNodesByQualifiedNameExact) { + this.stmts.getNodesByQualifiedNameExact = this.db.prepare( + 'SELECT * FROM nodes WHERE qualified_name = ?' + ); + } + const rows = this.stmts.getNodesByQualifiedNameExact.all(qualifiedName) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Get nodes by lowercase name match (uses idx_nodes_lower_name expression index) + */ + getNodesByLowerName(lowerName: string): Node[] { + if (!this.stmts.getNodesByLowerName) { + this.stmts.getNodesByLowerName = this.db.prepare( + 'SELECT * FROM nodes WHERE lower(name) = ?' + ); + } + const rows = this.stmts.getNodesByLowerName.all(lowerName) as NodeRow[]; + return rows.map(rowToNode); + } + + /** + * Search nodes by name using FTS with fallback to LIKE for better matching + * + * Search strategy: + * 1. Try FTS5 prefix match (query*) for word-start matching + * 2. If no results, try LIKE for substring matching (e.g., "signIn" finds "signInWithGoogle") + * 3. Score results based on match quality + */ + searchNodes(query: string, options: SearchOptions = {}): SearchResult[] { + const { limit = 100, offset = 0 } = options; + + // Parse field-qualified bits out of the raw query (kind:, lang:, + // path:, name:). Anything not recognised stays in `text` and goes + // to FTS unchanged. Filters compose with the SearchOptions arg — + // both are applied (intersection-style). + const parsed = parseQuery(query); + const mergedKinds = + parsed.kinds.length > 0 + ? Array.from(new Set([...(options.kinds ?? []), ...parsed.kinds])) + : options.kinds; + const mergedLanguages = + parsed.languages.length > 0 + ? Array.from(new Set([...(options.languages ?? []), ...parsed.languages])) + : options.languages; + const pathFilters = parsed.pathFilters; + const nameFilters = parsed.nameFilters; + // The text portion drives FTS/LIKE; if all the user typed was + // filters (`kind:function`), we still need *some* candidate set, + // so synthesise an empty-text path that returns everything matching + // the filters. + const text = parsed.text; + const kinds = mergedKinds; + const languages = mergedLanguages; + + // First try FTS5 with prefix matching + let results = text + ? this.searchNodesFTS(text, { kinds, languages, limit, offset }) + // Over-fetch by 5× when running filter-only (no text). The + // post-scoring path: + name: filters can be very selective, so + // a smaller multiplier risks returning fewer than `limit` + // results despite the DB having plenty of matches. + : this.searchAllByFilters({ kinds, languages, limit: limit * 5 }); + + // If no FTS results, try LIKE-based substring search + if (results.length === 0 && text.length >= 2) { + results = this.searchNodesLike(text, { kinds, languages, limit, offset }); + } + + // Final fuzzy fallback: scan all known names and keep those within + // a tight Levenshtein distance. Only fires when both FTS and LIKE + // returned nothing AND there's a text portion long enough to be + // worth fuzzing (1-char queries would match too much). + if (results.length === 0 && text.length >= 3) { + results = this.searchNodesFuzzy(text, { kinds, languages, limit }); + } + + // Supplement: ensure exact name matches are always candidates. + // BM25 can bury short exact-match names (e.g. "getBean") under hundreds of + // compound names (e.g. "getBeanDescriptor") in large codebases, + // pushing them past the FTS fetch limit before post-hoc scoring can help. + // Use the max BM25 score as the base so the nameMatchBonus (exact=30 vs + // prefix=20) actually differentiates them after rescoring. + if (results.length > 0 && query) { + const existingIds = new Set(results.map(r => r.node.id)); + const maxFtsScore = Math.max(...results.map(r => r.score)); + const terms = query.split(/\s+/).filter(t => t.length >= 2); + for (const term of terms) { + let sql = 'SELECT * FROM nodes WHERE name = ? COLLATE NOCASE'; + const params: (string | number)[] = [term]; + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + sql += ' LIMIT 20'; + const rows = this.db.prepare(sql).all(...params) as NodeRow[]; + for (const row of rows) { + if (!existingIds.has(row.id)) { + results.push({ node: rowToNode(row), score: maxFtsScore }); + existingIds.add(row.id); + } + } + } + } + + // Apply multi-signal scoring + if (results.length > 0 && (text || query)) { + const scoringQuery = text || query; + results = results.map(r => ({ + ...r, + score: r.score + + kindBonus(r.node.kind) + + scorePathRelevance(r.node.filePath, scoringQuery, this.projectNameTokens) + + nameMatchBonus(r.node.name, scoringQuery), + })); + results.sort((a, b) => b.score - a.score); + // Trim to requested limit after rescoring + if (results.length > limit) { + results = results.slice(0, limit); + } + } + + // Apply path: + name: filters AFTER scoring. Scoring already uses + // path/name as a soft signal; the explicit filters here are a hard + // gate. Done last so the FTS limit fetched plenty of candidates to + // narrow from. + if (pathFilters.length > 0) { + const lowered = pathFilters.map((p) => p.toLowerCase()); + results = results.filter((r) => { + const fp = r.node.filePath.toLowerCase(); + return lowered.some((p) => fp.includes(p)); + }); + } + if (nameFilters.length > 0) { + const lowered = nameFilters.map((n) => n.toLowerCase()); + results = results.filter((r) => { + const nm = r.node.name.toLowerCase(); + return lowered.some((n) => nm.includes(n)); + }); + } + + return results; + } + + /** + * Match-everything path used when the user supplied only field + * filters (`kind:function lang:typescript`) with no text. Returns + * candidates ordered by name; the caller's filter pass narrows to + * what was asked for. + */ + private searchAllByFilters(options: { + kinds?: NodeKind[]; + languages?: Language[]; + limit: number; + }): SearchResult[] { + const { kinds, languages, limit } = options; + let sql = 'SELECT * FROM nodes WHERE 1=1'; + const params: (string | number)[] = []; + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + sql += ' ORDER BY name LIMIT ?'; + params.push(limit); + const rows = this.db.prepare(sql).all(...params) as NodeRow[]; + return rows.map((row) => ({ node: rowToNode(row), score: 1 })); + } + + /** + * Fuzzy fallback: when zero FTS/LIKE hits, try an edit-distance + * sweep over the distinct symbol-name set. Caps `maxDist` at 2 so + * `getUssr` finds `getUser` but `process` doesn't match `prosody`. + * Bounded edit distance keeps each comparison cheap; the per-query + * scan is O(distinct-name-count) which is far smaller than total + * node count on any real codebase. + */ + private searchNodesFuzzy( + text: string, + options: { kinds?: NodeKind[]; languages?: Language[]; limit: number } + ): SearchResult[] { + const { kinds, languages, limit } = options; + const lowered = text.toLowerCase(); + const maxDist = lowered.length <= 4 ? 1 : 2; + + // Pull the distinct name list once. The set is cached on QueryBuilder + // by getAllNodeNames(); even on a 200k-node project the distinct + // name set is typically O(10k) because most names repeat. The + // candidate-cap below bounds memory regardless. + const allNames = this.getAllNodeNames(); + const candidates: Array<{ name: string; dist: number }> = []; + for (const name of allNames) { + const dist = boundedEditDistance(name.toLowerCase(), lowered, maxDist); + if (dist <= maxDist) candidates.push({ name, dist }); + } + candidates.sort((a, b) => a.dist - b.dist); + + // Cap the per-name follow-up queries. Each survivor triggers a + // separate `SELECT * FROM nodes WHERE name = ?`; without this cap + // a project with many similar names (`getUser1`, `getUser2`...) + // could fan out far beyond `limit` queries before the inner-loop + // limit kicks in. + const FUZZY_FOLLOWUP_CAP = Math.max(limit * 2, 50); + const cappedCandidates = candidates.slice(0, FUZZY_FOLLOWUP_CAP); + + const results: SearchResult[] = []; + const seen = new Set(); + for (const c of cappedCandidates) { + if (results.length >= limit) break; + let sql = 'SELECT * FROM nodes WHERE name = ?'; + const params: (string | number)[] = [c.name]; + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + sql += ' LIMIT 5'; + const rows = this.db.prepare(sql).all(...params) as NodeRow[]; + for (const row of rows) { + if (seen.has(row.id)) continue; + seen.add(row.id); + // Lower the score for each edit step away from the query so + // exact-match fallbacks (dist 0) outrank dist-2 typos. + results.push({ node: rowToNode(row), score: 1 / (1 + c.dist) }); + if (results.length >= limit) break; + } + } + return results; + } + + /** + * FTS5 search with prefix matching + */ + private searchNodesFTS(query: string, options: SearchOptions): SearchResult[] { + const { kinds, languages, limit = 100, offset = 0 } = options; + + // Add prefix wildcard for better matching (e.g., "auth" matches "AuthService", "authenticate") + // Escape special FTS5 characters and add prefix wildcard. + // + // `::` is a qualifier separator in Rust/C++/Ruby, not a token char, + // so treat it as whitespace before the strip step. Otherwise queries + // like `stage_apply::run` collapse to `stage_applyrun` (the colons + // are stripped without splitting) and find nothing. See #173. + const ftsQuery = query + .replace(/::/g, ' ') // Rust/C++/Ruby qualifier separator + .replace(/['"*():^]/g, '') // Remove FTS5 special chars + .split(/\s+/) + .filter(term => term.length > 0) + // Strip FTS5 boolean operators to prevent query manipulation + .filter(term => !/^(AND|OR|NOT|NEAR)$/i.test(term)) + .map(term => `"${term}"*`) // Prefix match each term + .join(' OR '); + + if (!ftsQuery) { + return []; + } + + // BM25 column weights: id=0, name=20, qualified_name=5, docstring=1, signature=2 + // Heavy name weight ensures exact/prefix name matches rank above incidental + // mentions in long docstrings or qualified names of nested symbols. + // Fetch 5x requested limit so post-hoc rescoring (kindBonus, pathRelevance, + // nameMatchBonus) can promote results that BM25 alone undervalues. + const ftsLimit = Math.max(limit * 5, 100); + + let sql = ` + SELECT nodes.*, bm25(nodes_fts, 0, 20, 5, 1, 2) as score + FROM nodes_fts + JOIN nodes ON nodes_fts.id = nodes.id + WHERE nodes_fts MATCH ? + `; + + const params: (string | number)[] = [ftsQuery]; + + if (kinds && kinds.length > 0) { + sql += ` AND nodes.kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + if (languages && languages.length > 0) { + sql += ` AND nodes.language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + + sql += ' ORDER BY score LIMIT ? OFFSET ?'; + params.push(ftsLimit, offset); + + try { + const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; + return rows.map((row) => ({ + node: rowToNode(row), + score: Math.abs(row.score), // bm25 returns negative scores + })); + } catch { + // FTS query failed, return empty + return []; + } + } + + /** + * LIKE-based substring search for cases where FTS doesn't match + * Useful for camelCase matching (e.g., "signIn" finds "signInWithGoogle") + */ + private searchNodesLike(query: string, options: SearchOptions): SearchResult[] { + const { kinds, languages, limit = 100, offset = 0 } = options; + + let sql = ` + SELECT nodes.*, + CASE + WHEN name = ? THEN 1.0 + WHEN name LIKE ? THEN 0.9 + WHEN name LIKE ? THEN 0.8 + WHEN qualified_name LIKE ? THEN 0.7 + ELSE 0.5 + END as score + FROM nodes + WHERE ( + name LIKE ? OR + qualified_name LIKE ? OR + name LIKE ? + ) + `; + + // Pattern variants for better matching + const exactMatch = query; + const startsWith = `${query}%`; + const contains = `%${query}%`; + + const params: (string | number)[] = [ + exactMatch, // Exact match score + startsWith, // Starts with score + contains, // Contains score + contains, // Qualified name score + contains, // WHERE: name contains + contains, // WHERE: qualified_name contains + startsWith, // WHERE: name starts with + ]; + + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + + sql += ' ORDER BY score DESC, length(name) ASC LIMIT ? OFFSET ?'; + params.push(limit, offset); + + const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; + + return rows.map((row) => ({ + node: rowToNode(row), + score: row.score, + })); + } + + /** + * Find nodes by exact name match + * + * Used for hybrid search - looks up symbols by exact name or case-insensitive match. + * Returns high-confidence matches for known symbol names extracted from query. + * + * @param names - Array of symbol names to look up + * @param options - Search options (kinds, languages, limit) + * @returns SearchResult array with exact matches scored at 1.0 + */ + findNodesByExactName(names: string[], options: SearchOptions = {}): SearchResult[] { + if (names.length === 0) return []; + + const { kinds, languages, limit = 50 } = options; + + // Two-pass approach to handle common names (e.g., "run" has 40+ matches): + // Pass 1: Find which files contain distinctive (rare) symbols from the query. + // Pass 2: Query each name, boosting results that co-locate with distinctive symbols. + + // Pass 1: Find files containing each queried name, identify distinctive names + const nameToFiles = new Map>(); + for (const name of names) { + let sql = 'SELECT DISTINCT file_path FROM nodes WHERE name COLLATE NOCASE = ?'; + const params: (string | number)[] = [name]; + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + sql += ' LIMIT 100'; + const rows = this.db.prepare(sql).all(...params) as { file_path: string }[]; + nameToFiles.set(name.toLowerCase(), new Set(rows.map(r => r.file_path))); + } + + // Distinctive names are those with fewer than 10 file matches (e.g., "scrapeLoop" = 1 file) + const distinctiveFiles = new Set(); + for (const [, files] of nameToFiles) { + if (files.size > 0 && files.size < 10) { + for (const f of files) distinctiveFiles.add(f); + } + } + + // Pass 2: Query each name with per-name limit, scoring by co-location + const perNameLimit = Math.max(8, Math.ceil(limit / names.length)); + const allResults: SearchResult[] = []; + const seenIds = new Set(); + + for (const name of names) { + let sql = ` + SELECT nodes.*, 1.0 as score + FROM nodes + WHERE name COLLATE NOCASE = ? + `; + const params: (string | number)[] = [name]; + + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + + // Fetch enough to find co-located results among common names + sql += ' LIMIT ?'; + params.push(Math.max(perNameLimit * 3, 50)); + + const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; + const nameResults: SearchResult[] = []; + for (const row of rows) { + const node = rowToNode(row); + if (seenIds.has(node.id)) continue; + // Boost results in files that also contain distinctive symbols + const coLocationBoost = distinctiveFiles.has(node.filePath) ? 20 : 0; + nameResults.push({ node, score: row.score + coLocationBoost }); + } + + // Sort by score (co-located first), take per-name limit + nameResults.sort((a, b) => b.score - a.score); + for (const r of nameResults.slice(0, perNameLimit)) { + seenIds.add(r.node.id); + allResults.push(r); + } + } + + // Sort all results by score so co-located results bubble up + allResults.sort((a, b) => b.score - a.score); + return allResults.slice(0, limit); + } + + /** + * Find nodes whose name contains a substring (LIKE-based). + * Useful for CamelCase-part matching where FTS fails because + * e.g. "TransportSearchAction" is one FTS token, not matchable by "Search"*. + * + * Results are ordered by name length (shorter = more likely to be the core type). + */ + findNodesByNameSubstring( + substring: string, + options: SearchOptions & { excludePrefix?: boolean } = {} + ): SearchResult[] { + const { kinds, languages, limit = 30, excludePrefix } = options; + + let sql = ` + SELECT nodes.*, 1.0 as score + FROM nodes + WHERE name LIKE ? + `; + const params: (string | number)[] = [`%${substring}%`]; + + // Exclude prefix matches (handled by FTS-based prefix search in Step 2b) + if (excludePrefix) { + sql += ` AND name NOT LIKE ?`; + params.push(`${substring}%`); + } + + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + if (languages && languages.length > 0) { + sql += ` AND language IN (${languages.map(() => '?').join(',')})`; + params.push(...languages); + } + + sql += ' ORDER BY length(name) ASC LIMIT ?'; + params.push(limit); + + const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; + return rows.map((row) => ({ + node: rowToNode(row), + score: row.score, + })); + } + + // =========================================================================== + // Edge Operations + // =========================================================================== + + /** + * Insert a new edge + */ + insertEdge(edge: Edge): void { + if (!this.stmts.insertEdge) { + this.stmts.insertEdge = this.db.prepare(` + INSERT OR IGNORE INTO edges (source, target, kind, metadata, line, col, provenance) + VALUES (@source, @target, @kind, @metadata, @line, @col, @provenance) + `); + } + + this.stmts.insertEdge.run({ + source: edge.source, + target: edge.target, + kind: edge.kind, + metadata: edge.metadata ? JSON.stringify(edge.metadata) : null, + line: edge.line ?? null, + col: edge.column ?? null, + provenance: edge.provenance ?? null, + }); + } + + /** + * Insert multiple edges in a transaction + */ + insertEdges(edges: Edge[]): void { + if (edges.length === 0) return; + + this.db.transaction(() => { + const endpointIds = new Set(); + for (const edge of edges) { + endpointIds.add(edge.source); + endpointIds.add(edge.target); + } + const existingNodeIds = this.getExistingNodeIds([...endpointIds]); + + for (const edge of edges) { + if (!existingNodeIds.has(edge.source) || !existingNodeIds.has(edge.target)) { + continue; + } + this.insertEdge(edge); + } + })(); + } + + /** + * Delete all edges from a source node + */ + deleteEdgesBySource(sourceId: string): void { + if (!this.stmts.deleteEdgesBySource) { + this.stmts.deleteEdgesBySource = this.db.prepare('DELETE FROM edges WHERE source = ?'); + } + this.stmts.deleteEdgesBySource.run(sourceId); + } + + /** + * Get outgoing edges from a node + */ + getOutgoingEdges(sourceId: string, kinds?: EdgeKind[], provenance?: string): Edge[] { + if ((kinds && kinds.length > 0) || provenance) { + let sql = 'SELECT * FROM edges WHERE source = ?'; + const params: (string | number)[] = [sourceId]; + + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + if (provenance) { + sql += ' AND provenance = ?'; + params.push(provenance); + } + + const rows = this.db.prepare(sql).all(...params) as EdgeRow[]; + return rows.map(rowToEdge); + } + + if (!this.stmts.getEdgesBySource) { + this.stmts.getEdgesBySource = this.db.prepare('SELECT * FROM edges WHERE source = ?'); + } + const rows = this.stmts.getEdgesBySource.all(sourceId) as EdgeRow[]; + return rows.map(rowToEdge); + } + + /** + * Get incoming edges to a node + */ + getIncomingEdges(targetId: string, kinds?: EdgeKind[]): Edge[] { + if (kinds && kinds.length > 0) { + const sql = `SELECT * FROM edges WHERE target = ? AND kind IN (${kinds.map(() => '?').join(',')})`; + const rows = this.db.prepare(sql).all(targetId, ...kinds) as EdgeRow[]; + return rows.map(rowToEdge); + } + + if (!this.stmts.getEdgesByTarget) { + this.stmts.getEdgesByTarget = this.db.prepare('SELECT * FROM edges WHERE target = ?'); + } + const rows = this.stmts.getEdgesByTarget.all(targetId) as EdgeRow[]; + return rows.map(rowToEdge); + } + + /** + * Find all edges where both source and target are in the given node set. + * Useful for recovering inter-node connectivity after BFS. + */ + findEdgesBetweenNodes(nodeIds: string[], kinds?: EdgeKind[]): Edge[] { + if (nodeIds.length === 0) return []; + + const idsJson = JSON.stringify(nodeIds); + let sql = `SELECT * FROM edges WHERE source IN (SELECT value FROM json_each(?)) AND target IN (SELECT value FROM json_each(?))`; + const params: string[] = [idsJson, idsJson]; + + if (kinds && kinds.length > 0) { + sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; + params.push(...kinds); + } + + const rows = this.db.prepare(sql).all(...params) as EdgeRow[]; + return rows.map(rowToEdge); + } + + /** + * Distinct file paths that DEPEND ON `filePath`: every file containing a + * symbol with a cross-file edge (any kind except `contains`) into a symbol + * of this file. This is the file-level projection of the symbol dependency + * graph and the basis for blast-radius / `affected` test selection. + * + * It deliberately does NOT restrict to `imports` edges. In this graph an + * `imports` edge connects a file to its own local import declarations + * (it is always same-file), so an imports-only lookup returns zero + * cross-file dependents for every file. The real cross-file dependency + * signal is the resolved call/reference graph — calls, references, + * instantiates, extends, implements, overrides, type_of, returns, + * decorates — exactly what {@link GraphTraverser.getImpactRadius} traverses. + * `contains` is excluded: a parent containing a symbol does not *depend* on + * it. One indexed query (idx_nodes_file_path + idx_edges_target_kind). + */ + getDependentFilePaths(filePath: string): string[] { + const sql = `SELECT DISTINCT src.file_path AS fp + FROM edges e + JOIN nodes tgt ON tgt.id = e.target + JOIN nodes src ON src.id = e.source + WHERE tgt.file_path = ? + AND e.kind != 'contains' + AND src.file_path != ?`; + const rows = this.db.prepare(sql).all(filePath, filePath) as Array<{ fp: string }>; + return rows.map((r) => r.fp); + } + + /** + * Distinct file paths that `filePath` DEPENDS ON — the inverse of + * {@link getDependentFilePaths}: every file containing a symbol that a + * symbol of this file has a cross-file edge into. Same edge-kind rules + * (all kinds except `contains`); same reason imports-only is insufficient. + */ + getDependencyFilePaths(filePath: string): string[] { + const sql = `SELECT DISTINCT tgt.file_path AS fp + FROM edges e + JOIN nodes src ON src.id = e.source + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind != 'contains' + AND tgt.file_path != ?`; + const rows = this.db.prepare(sql).all(filePath, filePath) as Array<{ fp: string }>; + return rows.map((r) => r.fp); + } + + /** + * Cross-file edges whose TARGET is a node in `filePath` and whose SOURCE is a + * node in a *different* file, paired with the target node's (name, kind) so a + * caller can re-resolve the edge to the re-indexed target's new ID (node IDs + * are `sha256(filePath:kind:name:line)`, so any line shift in the callee file + * changes target IDs and a naive re-insert by old ID silently drops them). + * Used by `storeExtractionResult` to preserve incoming edges across a file + * re-index (issue #899). Same edge-kind rules as + * {@link getDependentFilePaths}: all kinds except `contains`. + */ + getCrossFileIncomingEdgesWithTarget( + filePath: string + ): Array { + const sql = `SELECT e.*, tgt.name AS target_name, tgt.kind AS target_kind, + src.file_path AS source_file_path, src.language AS source_language + FROM edges e + JOIN nodes tgt ON tgt.id = e.target + JOIN nodes src ON src.id = e.source + WHERE tgt.file_path = ? + AND e.kind != 'contains' + AND src.file_path != ?`; + const rows = this.db.prepare(sql).all(filePath, filePath) as Array< + EdgeRow & { target_name: string; target_kind: NodeKind; source_file_path: string; source_language: Language } + >; + return rows.map(row => ({ + ...rowToEdge(row), + targetName: row.target_name, + targetKind: row.target_kind, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + })); + } + + // =========================================================================== + // File Operations + // =========================================================================== + + /** + * Insert or update a file record + */ + upsertFile(file: FileRecord): void { + if (!this.stmts.upsertFile) { + this.stmts.upsertFile = this.db.prepare(` + INSERT INTO files (path, content_hash, language, size, modified_at, indexed_at, node_count, errors) + VALUES (@path, @contentHash, @language, @size, @modifiedAt, @indexedAt, @nodeCount, @errors) + ON CONFLICT(path) DO UPDATE SET + content_hash = @contentHash, + language = @language, + size = @size, + modified_at = @modifiedAt, + indexed_at = @indexedAt, + node_count = @nodeCount, + errors = @errors + `); + } + + this.stmts.upsertFile.run({ + path: file.path, + contentHash: file.contentHash, + language: file.language, + size: file.size, + modifiedAt: file.modifiedAt, + indexedAt: file.indexedAt, + nodeCount: file.nodeCount, + errors: file.errors ? JSON.stringify(file.errors) : null, + }); + } + + /** + * Delete a file record and its nodes + */ + deleteFile(filePath: string): void { + this.db.transaction(() => { + this.deleteNodesByFile(filePath); + if (!this.stmts.deleteFile) { + this.stmts.deleteFile = this.db.prepare('DELETE FROM files WHERE path = ?'); + } + this.stmts.deleteFile.run(filePath); + })(); + } + + /** + * Get a file record by path + */ + getFileByPath(filePath: string): FileRecord | null { + if (!this.stmts.getFileByPath) { + this.stmts.getFileByPath = this.db.prepare('SELECT * FROM files WHERE path = ?'); + } + const row = this.stmts.getFileByPath.get(filePath) as FileRow | undefined; + return row ? rowToFileRecord(row) : null; + } + + /** + * Get all tracked files + */ + getAllFiles(): FileRecord[] { + if (!this.stmts.getAllFiles) { + this.stmts.getAllFiles = this.db.prepare('SELECT * FROM files ORDER BY path'); + } + const rows = this.stmts.getAllFiles.all() as FileRow[]; + return rows.map(rowToFileRecord); + } + + /** + * Most recent index timestamp (ms since epoch) across all tracked files, or + * null when nothing is indexed yet. One indexed aggregate, no per-row scan. (#329) + */ + getLastIndexedAt(): number | null { + const row = this.db + .prepare('SELECT MAX(indexed_at) AS last FROM files') + .get() as { last: number | null } | undefined; + return row?.last ?? null; + } + + /** + * Get files that need re-indexing (hash changed) + */ + getStaleFiles(currentHashes: Map): FileRecord[] { + const files = this.getAllFiles(); + return files.filter((f) => { + const currentHash = currentHashes.get(f.path); + return currentHash && currentHash !== f.contentHash; + }); + } + + // =========================================================================== + // Unresolved References + // =========================================================================== + + /** + * Insert an unresolved reference + */ + insertUnresolvedRef(ref: UnresolvedReference): void { + if (!this.stmts.insertUnresolved) { + this.stmts.insertUnresolved = this.db.prepare(` + INSERT INTO unresolved_refs (from_node_id, reference_name, reference_kind, line, col, candidates, file_path, language) + VALUES (@fromNodeId, @referenceName, @referenceKind, @line, @col, @candidates, @filePath, @language) + `); + } + + this.stmts.insertUnresolved.run({ + fromNodeId: ref.fromNodeId, + referenceName: ref.referenceName, + referenceKind: ref.referenceKind, + line: ref.line, + col: ref.column, + candidates: ref.candidates ? JSON.stringify(ref.candidates) : null, + filePath: ref.filePath ?? '', + language: ref.language ?? 'unknown', + }); + } + + /** + * Insert multiple unresolved references in a transaction + */ + insertUnresolvedRefsBatch(refs: UnresolvedReference[]): void { + if (refs.length === 0) return; + const insert = this.db.transaction(() => { + for (const ref of refs) { + this.insertUnresolvedRef(ref); + } + }); + insert(); + } + + /** + * Delete unresolved references from a node + */ + deleteUnresolvedByNode(nodeId: string): void { + if (!this.stmts.deleteUnresolvedByNode) { + this.stmts.deleteUnresolvedByNode = this.db.prepare( + 'DELETE FROM unresolved_refs WHERE from_node_id = ?' + ); + } + this.stmts.deleteUnresolvedByNode.run(nodeId); + } + + /** + * Get unresolved references by name (for resolution) + */ + getUnresolvedByName(name: string): UnresolvedReference[] { + if (!this.stmts.getUnresolvedByName) { + this.stmts.getUnresolvedByName = this.db.prepare( + 'SELECT * FROM unresolved_refs WHERE reference_name = ?' + ); + } + const rows = this.stmts.getUnresolvedByName.all(name) as UnresolvedRefRow[]; + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + })); + } + + /** + * Get all unresolved references + */ + getUnresolvedReferences(): UnresolvedReference[] { + const rows = this.db.prepare('SELECT * FROM unresolved_refs').all() as UnresolvedRefRow[]; + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + })); + } + + /** + * Get the count of PENDING (never-attempted) references without loading + * them into memory. Rows marked status='failed' — attempted by a completed + * pass, no match — are excluded: they are not outstanding work, only retry + * candidates for the #1240 sweep, so they must not trip the #1187 orphan + * sweep or the `status` pending-refs warning. + */ + getUnresolvedReferencesCount(): number { + if (!this.stmts.getUnresolvedCount) { + this.stmts.getUnresolvedCount = this.db.prepare( + "SELECT COUNT(*) as count FROM unresolved_refs WHERE status = 'pending'" + ); + } + const row = this.stmts.getUnresolvedCount.get() as { count: number }; + return row.count; + } + + /** + * Get a batch of PENDING unresolved references using LIMIT/OFFSET + * pagination. Used to process references in bounded memory chunks; failed + * rows are excluded so the batched drain loop terminates once every row + * has been attempted. + */ + getUnresolvedReferencesBatch(offset: number, limit: number): UnresolvedReference[] { + if (!this.stmts.getUnresolvedBatch) { + this.stmts.getUnresolvedBatch = this.db.prepare( + "SELECT * FROM unresolved_refs WHERE status = 'pending' LIMIT ? OFFSET ?" + ); + } + const rows = this.stmts.getUnresolvedBatch.all(limit, offset) as UnresolvedRefRow[]; + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + })); + } + + /** + * Get all tracked file paths (lightweight — no full FileRecord objects) + */ + getAllFilePaths(): string[] { + if (!this.stmts.getAllFilePaths) { + this.stmts.getAllFilePaths = this.db.prepare('SELECT path FROM files ORDER BY path'); + } + const rows = this.stmts.getAllFilePaths.all() as Array<{ path: string }>; + return rows.map((r) => r.path); + } + + /** + * Get all distinct node names (lightweight — just name strings for pre-filtering) + */ + getAllNodeNames(): string[] { + if (!this.stmts.getAllNodeNames) { + this.stmts.getAllNodeNames = this.db.prepare('SELECT DISTINCT name FROM nodes'); + } + const rows = this.stmts.getAllNodeNames.all() as Array<{ name: string }>; + return rows.map((r) => r.name); + } + + /** + * Stream the distinct node names one row at a time — the incremental + * counterpart to {@link getAllNodeNames} for callers that need to yield + * to the event loop mid-scan (resolver cache warm-up on multi-million-node + * indexes). Fresh statement per call: the iterator holds an open cursor. + */ + *iterateNodeNames(): IterableIterator { + const stmt = this.db.prepare('SELECT DISTINCT name FROM nodes'); + for (const row of stmt.iterate()) { + yield (row as { name: string }).name; + } + } + + /** + * Get unresolved references scoped to specific file paths. + * Uses the idx_unresolved_file_path index for efficient lookup. + */ + getUnresolvedReferencesByFiles(filePaths: string[]): UnresolvedReference[] { + if (filePaths.length === 0) return []; + + // Chunk under SQLite's parameter limit: the first sync of a very large repo + // passes every changed file here, which an unbounded `IN (...)` would bind + // as one parameter each — exceeding MAX_VARIABLE_NUMBER and aborting with + // "too many SQL variables". (#540) + const rows: UnresolvedRefRow[] = []; + for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const chunkRows = this.db + .prepare(`SELECT * FROM unresolved_refs WHERE status = 'pending' AND file_path IN (${placeholders})`) + .all(...chunk) as UnresolvedRefRow[]; + rows.push(...chunkRows); + } + + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + })); + } + + /** + * Delete all unresolved references (after resolution) + */ + clearUnresolvedReferences(): void { + this.db.exec('DELETE FROM unresolved_refs'); + } + + /** + * Delete resolved references by their IDs + */ + deleteResolvedReferences(fromNodeIds: string[]): void { + if (fromNodeIds.length === 0) return; + // Chunk under SQLite's parameter limit, matching every other IN-list in + // this file. The internal resolution path uses deleteSpecificResolvedReferences + // instead, but QueryBuilder is part of the public API, so a library consumer + // passing more ids than SQLITE_MAX_VARIABLE_NUMBER (32766 on the bundled + // node:sqlite) would otherwise hit "too many SQL variables". (#540, #1001) + for (let i = 0; i < fromNodeIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = fromNodeIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + this.db.prepare(`DELETE FROM unresolved_refs WHERE from_node_id IN (${placeholders})`).run(...chunk); + } + } + + /** + * Delete specific resolved references by (fromNodeId, referenceName, referenceKind) tuples. + * More precise than deleteResolvedReferences — only removes refs that were actually resolved. + */ + deleteSpecificResolvedReferences(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void { + if (refs.length === 0) return; + const stmt = this.db.prepare( + 'DELETE FROM unresolved_refs WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?' + ); + const deleteMany = this.db.transaction((items: typeof refs) => { + for (const ref of items) { + stmt.run(ref.fromNodeId, ref.referenceName, ref.referenceKind); + } + }); + deleteMany(refs); + } + + /** + * Delete unresolved-ref rows by row id — the precise cleanup for refs a + * resolution pass actually processed. The key-tuple variant above also + * deletes SIBLING rows (same caller calling the same callee at other lines) + * that a later batch hasn't attempted yet, so when a batch boundary split a + * caller's same-named call sites, the later sites' edges were silently never + * created (#1269). + */ + deleteReferencesByRowIds(rowIds: number[]): void { + if (rowIds.length === 0) return; + for (let i = 0; i < rowIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = rowIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + this.db.prepare(`DELETE FROM unresolved_refs WHERE id IN (${placeholders})`).run(...chunk); + } + } + + /** + * Mark refs a completed resolution pass could not resolve as status='failed' + * instead of deleting them (#1240). Failed rows are invisible to the pending + * count/batch readers (so drain loops and the #1187 orphan sweep still + * terminate) but stay queryable by name_tail so a later sync can retry them + * when a changed file introduces a symbol that could satisfy them. name_tail + * is (re)written here so rows inserted before the v8 migration get their + * tail the first time they're attempted. + */ + markReferencesFailed(refs: Array<{ fromNodeId: string; referenceName: string; referenceKind: string }>): void { + if (refs.length === 0) return; + const stmt = this.db.prepare( + "UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE from_node_id = ? AND reference_name = ? AND reference_kind = ?" + ); + const markMany = this.db.transaction((items: typeof refs) => { + for (const ref of items) { + stmt.run(referenceNameTail(ref.referenceName), ref.fromNodeId, ref.referenceName, ref.referenceKind); + } + }); + markMany(refs); + } + + /** + * Park refs as status='failed' by row id — the precise counterpart of + * markReferencesFailed, for the same reason as deleteReferencesByRowIds: + * the key-tuple variant also flips same-key sibling rows in later batches + * to 'failed' before they were ever attempted (#1269). Resolution outcome + * can differ per call site (receiver-type inference reads the ref's line), + * so a sibling must not inherit this row's failure. + */ + markReferencesFailedByRowIds(refs: Array<{ rowId: number; referenceName: string }>): void { + if (refs.length === 0) return; + const stmt = this.db.prepare( + "UPDATE unresolved_refs SET status = 'failed', name_tail = ? WHERE id = ?" + ); + const markMany = this.db.transaction((items: typeof refs) => { + for (const ref of items) { + stmt.run(referenceNameTail(ref.referenceName), ref.rowId); + } + }); + markMany(refs); + } + + /** + * Failed refs whose name tail matches one of the given symbol names — the + * candidates a sync should retry after files carrying those names changed + * (#1240). Names matching more than `perNameCeiling` failed refs are + * skipped entirely: at that population a name is external/builtin noise + * (`get`, `map`, …) that one new definition won't resolve — the same + * rationale as resolution's AMBIGUOUS_NAME_CEILING (#999) — and retrying an + * arbitrary subset would be both wasted work and incoherent coverage. + */ + getRetryableFailedReferences(names: string[], perNameCeiling: number = 500): UnresolvedReference[] { + if (names.length === 0) return []; + + // Pass 1: per-tail counts, chunked under the SQLite parameter limit. + const retryNames: string[] = []; + for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const counts = this.db + .prepare( + `SELECT name_tail, COUNT(*) as count FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders}) GROUP BY name_tail` + ) + .all(...chunk) as Array<{ name_tail: string; count: number }>; + for (const row of counts) { + if (row.count <= perNameCeiling) retryNames.push(row.name_tail); + } + } + if (retryNames.length === 0) return []; + + // Pass 2: load the surviving rows. + const rows: UnresolvedRefRow[] = []; + for (let i = 0; i < retryNames.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = retryNames.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const chunkRows = this.db + .prepare(`SELECT * FROM unresolved_refs WHERE status = 'failed' AND name_tail IN (${placeholders})`) + .all(...chunk) as UnresolvedRefRow[]; + rows.push(...chunkRows); + } + + return rows.map((row) => ({ + fromNodeId: row.from_node_id, + referenceName: row.reference_name, + referenceKind: row.reference_kind as EdgeKind, + line: row.line, + column: row.col, + candidates: row.candidates ? safeJsonParse(row.candidates, undefined) : undefined, + filePath: row.file_path, + language: row.language as Language, + rowId: row.id, + })); + } + + /** + * Distinct node names present in the given files — the symbol names a sync + * pass uses to look up retryable failed refs after those files changed. + */ + getNodeNamesByFiles(filePaths: string[]): string[] { + if (filePaths.length === 0) return []; + const names = new Set(); + for (let i = 0; i < filePaths.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = filePaths.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT DISTINCT name FROM nodes WHERE file_path IN (${placeholders})`) + .all(...chunk) as Array<{ name: string }>; + for (const row of rows) names.add(row.name); + } + return [...names]; + } + + // =========================================================================== + // Statistics + // =========================================================================== + + /** + * Lightweight (nodes, edges) count snapshot. Used around an index/sync + * run to compute true additions across extraction + resolution + + * synthesis — the per-phase counter in the orchestrator only sees + * extraction's contribution, which is why the CLI summary under-reported + * the edge count (resolution + synthesizer edges were invisible). + */ + getNodeAndEdgeCount(): { nodes: number; edges: number } { + return this.db + .prepare('SELECT (SELECT COUNT(*) FROM nodes) AS nodes, (SELECT COUNT(*) FROM edges) AS edges') + .get() as { nodes: number; edges: number }; + } + + /** + * Get graph statistics + */ + getStats(): GraphStats { + // Single query for all three aggregate counts + const counts = this.db.prepare(` + SELECT + (SELECT COUNT(*) FROM nodes) AS node_count, + (SELECT COUNT(*) FROM edges) AS edge_count, + (SELECT COUNT(*) FROM files) AS file_count + `).get() as { node_count: number; edge_count: number; file_count: number }; + + const nodesByKind = {} as Record; + const nodeKindRows = this.db + .prepare('SELECT kind, COUNT(*) as count FROM nodes GROUP BY kind') + .all() as Array<{ kind: string; count: number }>; + for (const row of nodeKindRows) { + nodesByKind[row.kind as NodeKind] = row.count; + } + + const edgesByKind = {} as Record; + const edgeKindRows = this.db + .prepare('SELECT kind, COUNT(*) as count FROM edges GROUP BY kind') + .all() as Array<{ kind: string; count: number }>; + for (const row of edgeKindRows) { + edgesByKind[row.kind as EdgeKind] = row.count; + } + + const filesByLanguage = {} as Record; + const languageRows = this.db + .prepare('SELECT language, COUNT(*) as count FROM files GROUP BY language') + .all() as Array<{ language: string; count: number }>; + for (const row of languageRows) { + filesByLanguage[row.language as Language] = row.count; + } + + return { + nodeCount: counts.node_count, + edgeCount: counts.edge_count, + fileCount: counts.file_count, + nodesByKind, + edgesByKind, + filesByLanguage, + dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize() + lastUpdated: Date.now(), + }; + } + + // =========================================================================== + // Project Metadata + // =========================================================================== + + /** + * Get a metadata value by key + */ + getMetadata(key: string): string | null { + const row = this.db.prepare('SELECT value FROM project_metadata WHERE key = ?').get(key) as { value: string } | undefined; + return row?.value ?? null; + } + + /** + * Set a metadata key-value pair (upsert) + */ + setMetadata(key: string, value: string): void { + this.db.prepare( + 'INSERT INTO project_metadata (key, value, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at' + ).run(key, value, Date.now()); + } + + /** + * Get all metadata as a key-value record + */ + getAllMetadata(): Record { + const rows = this.db.prepare('SELECT key, value FROM project_metadata').all() as { key: string; value: string }[]; + const result: Record = {}; + for (const row of rows) { + result[row.key] = row.value; + } + return result; + } + + /** + * Clear all data from the database + */ + clear(): void { + this.nodeCache.clear(); + this.db.transaction(() => { + this.db.exec('DELETE FROM unresolved_refs'); + this.db.exec('DELETE FROM edges'); + this.db.exec('DELETE FROM nodes'); + this.db.exec('DELETE FROM files'); + })(); + } +} diff --git a/src/db/schema.sql b/src/db/schema.sql new file mode 100644 index 0000000..75df8c0 --- /dev/null +++ b/src/db/schema.sql @@ -0,0 +1,194 @@ +-- CodeGraph SQLite Schema +-- Version 1 + +-- Schema version tracking +CREATE TABLE IF NOT EXISTS schema_versions ( + version INTEGER PRIMARY KEY, + applied_at INTEGER NOT NULL, + description TEXT +); + +-- Insert initial version +INSERT INTO schema_versions (version, applied_at, description) +VALUES (1, strftime('%s', 'now') * 1000, 'Initial schema'); + +-- ============================================================================= +-- Core Tables +-- ============================================================================= + +-- Nodes: Code symbols (functions, classes, variables, etc.) +CREATE TABLE IF NOT EXISTS nodes ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + name TEXT NOT NULL, + qualified_name TEXT NOT NULL, + file_path TEXT NOT NULL, + language TEXT NOT NULL, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + start_column INTEGER NOT NULL, + end_column INTEGER NOT NULL, + docstring TEXT, + signature TEXT, + visibility TEXT, + is_exported INTEGER DEFAULT 0, + is_async INTEGER DEFAULT 0, + is_static INTEGER DEFAULT 0, + is_abstract INTEGER DEFAULT 0, + decorators TEXT, -- JSON array + type_parameters TEXT, -- JSON array + return_type TEXT, -- normalized return/result type name (e.g. C++ method return, for receiver-type inference) + updated_at INTEGER NOT NULL +); + +-- Edges: Relationships between nodes +CREATE TABLE IF NOT EXISTS edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + target TEXT NOT NULL, + kind TEXT NOT NULL, + metadata TEXT, -- JSON object + line INTEGER, + col INTEGER, + provenance TEXT DEFAULT NULL, + FOREIGN KEY (source) REFERENCES nodes(id) ON DELETE CASCADE, + FOREIGN KEY (target) REFERENCES nodes(id) ON DELETE CASCADE +); + +-- Files: Tracked source files +CREATE TABLE IF NOT EXISTS files ( + path TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + language TEXT NOT NULL, + size INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + indexed_at INTEGER NOT NULL, + node_count INTEGER DEFAULT 0, + errors TEXT -- JSON array +); + +-- Unresolved References: References that need resolution after full indexing. +-- status lifecycle: rows are inserted 'pending' by extraction; a completed +-- resolution pass either deletes a row (resolved) or marks it 'failed' +-- (attempted, no match — kept so a later sync can retry it when a changed +-- file introduces a symbol that could satisfy it, #1240). name_tail is the +-- last segment of reference_name ('util.greet' → 'greet'), written when a +-- row is marked failed, so the retry lookup matches new node names against +-- dotted refs too. Rows follow their from_node via ON DELETE CASCADE, so +-- re-extracting or deleting a file clears its stale rows in any status. +CREATE TABLE IF NOT EXISTS unresolved_refs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + from_node_id TEXT NOT NULL, + reference_name TEXT NOT NULL, + reference_kind TEXT NOT NULL, + line INTEGER NOT NULL, + col INTEGER NOT NULL, + candidates TEXT, -- JSON array + file_path TEXT NOT NULL DEFAULT '', + language TEXT NOT NULL DEFAULT 'unknown', + status TEXT NOT NULL DEFAULT 'pending', + name_tail TEXT NOT NULL DEFAULT '', + FOREIGN KEY (from_node_id) REFERENCES nodes(id) ON DELETE CASCADE +); + +-- ============================================================================= +-- Indexes for Query Performance +-- ============================================================================= + +-- Node indexes +CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind); +CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name); +CREATE INDEX IF NOT EXISTS idx_nodes_qualified_name ON nodes(qualified_name); +CREATE INDEX IF NOT EXISTS idx_nodes_file_path ON nodes(file_path); +CREATE INDEX IF NOT EXISTS idx_nodes_language ON nodes(language); +CREATE INDEX IF NOT EXISTS idx_nodes_file_line ON nodes(file_path, start_line); +CREATE INDEX IF NOT EXISTS idx_nodes_lower_name ON nodes(lower(name)); + +-- Full-text search index on node names, docstrings, and signatures +CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5( + id, + name, + qualified_name, + docstring, + signature, + content='nodes', + content_rowid='rowid' +); + +-- Triggers to keep FTS index in sync +CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes BEGIN + INSERT INTO nodes_fts(rowid, id, name, qualified_name, docstring, signature) + VALUES (NEW.rowid, NEW.id, NEW.name, NEW.qualified_name, NEW.docstring, NEW.signature); +END; + +CREATE TRIGGER IF NOT EXISTS nodes_ad AFTER DELETE ON nodes BEGIN + INSERT INTO nodes_fts(nodes_fts, rowid, id, name, qualified_name, docstring, signature) + VALUES ('delete', OLD.rowid, OLD.id, OLD.name, OLD.qualified_name, OLD.docstring, OLD.signature); +END; + +CREATE TRIGGER IF NOT EXISTS nodes_au AFTER UPDATE ON nodes BEGIN + INSERT INTO nodes_fts(nodes_fts, rowid, id, name, qualified_name, docstring, signature) + VALUES ('delete', OLD.rowid, OLD.id, OLD.name, OLD.qualified_name, OLD.docstring, OLD.signature); + INSERT INTO nodes_fts(rowid, id, name, qualified_name, docstring, signature) + VALUES (NEW.rowid, NEW.id, NEW.name, NEW.qualified_name, NEW.docstring, NEW.signature); +END; + +-- Prose-word → symbol-name lookup for the prompt hook's graph-derived gate. +-- One row per (segment, name): segment is a lowercased word of a symbol name +-- ("OrderStateMachine" → order, state, machine — see identifier-segments.ts), +-- which lets natural-language prompt words be verified against the graph in +-- any language whose technical nouns are Latin script. File nodes are +-- excluded — a file's basename duplicates the symbols inside it and skews the +-- singleton-vs-cluster rarity statistics. FTS can't serve this lookup (its +-- tokenizer keeps camelCase names as single tokens), so segments are +-- materialized on the node write path. +-- Deletions leave orphan rows ON PURPOSE: rows are PROPOSALS, always +-- re-verified against nodes before being surfaced (CodeGraph.getSegmentMatches), +-- and a full index clears the table at its start. Populated lazily on old +-- databases (empty until the next index/sync heals it). +CREATE TABLE IF NOT EXISTS name_segment_vocab ( + segment TEXT NOT NULL, + name TEXT NOT NULL, + PRIMARY KEY (segment, name) +) WITHOUT ROWID; + +-- Edge indexes. +-- idx_edges_source / idx_edges_target are intentionally omitted — +-- the (source, kind) and (target, kind) composites below cover the +-- corresponding source-only / target-only lookups via SQLite's +-- left-prefix scan, so the narrow indexes are dead weight on writes. +-- Migration v4 drops them on existing databases. +CREATE INDEX IF NOT EXISTS idx_edges_kind ON edges(kind); +CREATE INDEX IF NOT EXISTS idx_edges_source_kind ON edges(source, kind); +CREATE INDEX IF NOT EXISTS idx_edges_target_kind ON edges(target, kind); + +-- Edge identity uniqueness. An edge IS uniquely (source, target, kind, line, +-- col); insertEdge uses `INSERT OR IGNORE`, but without something UNIQUE to +-- conflict on it behaved like a plain INSERT, so two passes emitting the same +-- edge produced byte-identical duplicate rows that inflated counts and flowed +-- into callers/impact (#1034). IFNULL folds the nullable line/col so +-- coordinate-less edges (synthesized / file-level) dedup too — SQLite treats +-- each NULL as distinct otherwise. Migration v6 dedups existing rows + adds +-- this on older databases. +CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_identity + ON edges(source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); + +-- File indexes +CREATE INDEX IF NOT EXISTS idx_files_language ON files(language); +CREATE INDEX IF NOT EXISTS idx_files_modified_at ON files(modified_at); + +-- Unresolved refs indexes +CREATE INDEX IF NOT EXISTS idx_unresolved_from_node ON unresolved_refs(from_node_id); +CREATE INDEX IF NOT EXISTS idx_unresolved_name ON unresolved_refs(reference_name); +CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path); +CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node_id, reference_name); +CREATE INDEX IF NOT EXISTS idx_unresolved_status ON unresolved_refs(status); +CREATE INDEX IF NOT EXISTS idx_unresolved_failed_tail ON unresolved_refs(name_tail) WHERE status = 'failed'; +CREATE INDEX IF NOT EXISTS idx_edges_provenance ON edges(provenance); + +-- Project metadata for version/provenance tracking +CREATE TABLE IF NOT EXISTS project_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL +); diff --git a/src/db/sqlite-adapter.ts b/src/db/sqlite-adapter.ts new file mode 100644 index 0000000..ab0376c --- /dev/null +++ b/src/db/sqlite-adapter.ts @@ -0,0 +1,149 @@ +/** + * SQLite Adapter + * + * Thin wrapper over Node's built-in `node:sqlite` (`DatabaseSync`), exposed + * through a small better-sqlite3-shaped interface so the rest of the codebase + * is storage-agnostic. + * + * CodeGraph ships with a bundled Node runtime, so `node:sqlite` (real SQLite, + * with WAL + FTS5) is always available — there is no native build step and no + * wasm fallback. When run from source instead, it requires Node >= 22.5. + */ + +export interface SqliteStatement { + run(...params: any[]): { changes: number; lastInsertRowid: number | bigint }; + get(...params: any[]): any; + all(...params: any[]): any[]; + /** + * Lazily yield result rows one at a time instead of materializing the whole + * set with `all()`. Use for unbounded scans (e.g. every function/method node) + * so memory stays O(1) in the row count rather than O(rows) — see #610, where + * `all()`-ing every symbol on a dense project spiked the heap into an OOM. + */ + iterate(...params: any[]): IterableIterator; +} + +export interface SqliteDatabase { + prepare(sql: string): SqliteStatement; + exec(sql: string): void; + pragma(str: string, options?: { simple?: boolean }): any; + transaction(fn: (...args: any[]) => T): (...args: any[]) => T; + close(): void; + readonly open: boolean; +} + +/** + * The active SQLite backend. Only one now (`node:sqlite`); kept as a named type + * so `codegraph status` and the per-instance reporting have a stable shape. + */ +export type SqliteBackend = 'node-sqlite'; + +/** + * Wraps Node's built-in `node:sqlite` (`DatabaseSync`) to match the + * better-sqlite3 interface the rest of the code expects. + * + * node:sqlite is real SQLite compiled into Node, so it supports WAL, FTS5, + * mmap, and `@named` params natively — the only shims needed are the + * better-sqlite3 conveniences node:sqlite omits: a `.pragma()` helper, a + * `.transaction()` helper, and `open` (node:sqlite exposes `isOpen`). + */ +class NodeSqliteAdapter implements SqliteDatabase { + private _db: any; + + constructor(dbPath: string) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require('node:sqlite'); + this._db = new DatabaseSync(dbPath); + } + + get open(): boolean { + return this._db.isOpen; + } + + prepare(sql: string): SqliteStatement { + // node:sqlite matches better-sqlite3's calling convention (variadic + // positional args, or a single object for @named params), so params forward + // through unchanged. + const stmt = this._db.prepare(sql); + return { + run(...params: any[]) { + const r = stmt.run(...params); + return { + changes: Number(r?.changes ?? 0), + lastInsertRowid: r?.lastInsertRowid ?? 0, + }; + }, + get(...params: any[]) { + return stmt.get(...params); + }, + all(...params: any[]) { + return stmt.all(...params); + }, + iterate(...params: any[]) { + return stmt.iterate(...params); + }, + }; + } + + exec(sql: string): void { + this._db.exec(sql); + } + + pragma(str: string, options?: { simple?: boolean }): any { + const trimmed = str.trim(); + // Write pragma ("key = value"): node:sqlite is real SQLite, so every pragma + // (WAL, mmap, synchronous, …) applies as-is. + if (trimmed.includes('=')) { + this._db.exec(`PRAGMA ${trimmed}`); + return; + } + // Read pragma. Default: the row object (e.g. { journal_mode: 'wal' }). + // `{ simple: true }` returns just the single column value, like better-sqlite3. + const row = this._db.prepare(`PRAGMA ${trimmed}`).get(); + if (options?.simple) { + return row && typeof row === 'object' ? Object.values(row)[0] : row; + } + return row; + } + + transaction(fn: (...args: any[]) => T): (...args: any[]) => T { + return (...args: any[]) => { + this._db.exec('BEGIN'); + try { + const result = fn(...args); + this._db.exec('COMMIT'); + return result; + } catch (error) { + this._db.exec('ROLLBACK'); + throw error; + } + }; + } + + close(): void { + // node:sqlite's DatabaseSync.close() throws if already closed; make it + // idempotent to match better-sqlite3 (callers may close more than once). + if (this._db.isOpen) this._db.close(); + } +} + +/** + * Create a database connection backed by `node:sqlite`. + * + * Returns the active backend alongside the db so each `DatabaseConnection` can + * report it per-instance — MCP can open multiple project DBs in one process, so + * a process-global would race. + */ +export function createDatabase(dbPath: string): { db: SqliteDatabase; backend: SqliteBackend } { + try { + return { db: new NodeSqliteAdapter(dbPath), backend: 'node-sqlite' }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + throw new Error( + 'Failed to open SQLite via the built-in node:sqlite module.\n' + + 'CodeGraph requires node:sqlite (Node.js 22.5+). Install the self-contained\n' + + 'CodeGraph release (it bundles a compatible Node), or run on Node 22.5+.\n' + + `Underlying error: ${msg}` + ); + } +} diff --git a/src/db/wal-valve.ts b/src/db/wal-valve.ts new file mode 100644 index 0000000..b509527 --- /dev/null +++ b/src/db/wal-valve.ts @@ -0,0 +1,206 @@ +/** + * WAL checkpoint valve — bounds WAL growth while auto-checkpointing is + * deferred during a bulk index (#1231). + * + * Why deferral: SQLite's default `wal_autocheckpoint` (1000 pages) re-writes + * hot B-tree/FTS pages into the main DB file over and over during a bulk + * index — measured at ~95% of ALL disk I/O, and the difference between 45s + * and 19+ minutes on HDD-class storage (150 random IOPS). Deferring + * checkpoints turns the store into pure sequential WAL appends; each backfill + * pass writes distinct pages once, in page order (≈ sequential). + * + * Why a valve: unbounded deferral is its own failure mode, both measured in + * the #1231 repro. The WAL duplicates hot pages per COMMIT, so it grows far + * faster than the DB (5.9GB WAL for a ~340MB DB on a 3.3k-file index) — + * filling the disk, and poisoning every subsequent read that must page + * through it (the first resolution-phase read blocked the main thread >60s + * and the #850 liveness watchdog killed the healthy index). The valve + * watches WAL growth on a timer and, past a soft threshold, backfills with + * `PRAGMA wal_checkpoint(PASSIVE)` on a worker-thread connection — PASSIVE + * never blocks the writer, and off-thread means the main thread (and the + * watchdog heartbeat) keep turning regardless of how long a backfill takes. + * + * The load-bearing subtlety: a WAL file's SIZE never shrinks. After a full + * backfill, the writer's next commit RESTARTS the WAL from the top and the + * frames recycle inside the same file — so raw size says nothing about the + * un-backfilled backlog, and a size-triggered valve degenerates into firing + * (and pausing the writer) forever once the file passes its threshold + * (measured: guava crawled at ~9min per 160 files). Instead the valve + * tracks `sizeAtLastFullBackfill` — refreshed whenever a checkpoint reports + * `log === checkpointed` (everything backfilled) — and triggers on GROWTH + * beyond that baseline, which only happens when genuinely un-backfilled + * frames push past the file's high-water mark. + * + * Backpressure: if the writer outruns the checkpointer past a hard cap of + * growth (2× soft), {@link backpressure} pauses the writer (at a safe, + * between-transactions boundary) until a FULL backfill lands. One in-flight + * pass is not enough: on a disk saturated by the writer, every concurrent + * PASSIVE pass is already stale by the time it finishes (the writer appended + * past its snapshot), so neither SQLite's WAL wrap nor the baseline ever + * trigger and the WAL grows without bound (measured: 5.9GB on guava at 150 + * IOPS, then a >60s read stall and a watchdog kill). With the writer parked, + * the next pass covers everything, the WAL wraps on the following commit, + * and the pause is the disk's honest catch-up cost — the correct terminal + * mode when hardware genuinely can't keep up with the append rate. + */ + +import type { DatabaseConnection } from './index'; + +/** Soft WAL-growth threshold (MB) that triggers an off-thread passive checkpoint. */ +const DEFAULT_WAL_VALVE_MB = 256; +/** Hard cap = this × soft threshold; past it the writer pauses for a full backfill. */ +const HARD_CAP_MULTIPLIER = 2; +/** Passes attempted per writer pause before giving up (a pinned reader could stall forever). */ +const MAX_PAUSED_BACKFILL_PASSES = 20; +/** How often the timer looks at the WAL file size. */ +const CHECK_INTERVAL_MS = 2000; + +/** + * Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB` + * override; non-numeric / non-positive values fall back to the default. + */ +export function resolveWalValveMb(envVal: string | undefined): number { + if (envVal !== undefined && envVal !== '') { + const n = Number(envVal); + if (Number.isFinite(n) && n > 0) return Math.floor(n); + } + return DEFAULT_WAL_VALVE_MB; +} + +export class WalCheckpointValve { + private timer: ReturnType | null = null; + private inflight: Promise | null = null; + /** Writer pause in progress (hard cap breached): passes loop until a full backfill. */ + private pause: Promise | null = null; + /** + * WAL file size observed when a checkpoint last reported the ENTIRE WAL + * backfilled. Growth is measured against this baseline — see the header + * comment for why absolute size cannot be used. + */ + private sizeAtLastFullBackfill = 0; + private readonly softBytes: number; + private readonly hardBytes: number; + + constructor( + private readonly db: DatabaseConnection, + softMb: number = resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB), + private readonly intervalMs: number = CHECK_INTERVAL_MS, + private readonly log: (msg: string) => void = () => {} + ) { + this.softBytes = softMb * 1024 * 1024; + this.hardBytes = this.softBytes * HARD_CAP_MULTIPLIER; + } + + private mb(n: number): string { + return `${Math.round(n / 1024 / 1024)}MB`; + } + + /** Un-backfilled growth estimate: bytes the WAL has grown past the last full backfill. */ + private growthBytes(): number { + return this.db.getWalSizeBytes() - this.sizeAtLastFullBackfill; + } + + /** Begin watching the WAL. Idempotent; the timer never holds the loop open. */ + start(): void { + if (this.timer) return; + this.timer = setInterval(() => this.check(), this.intervalMs); + this.timer.unref?.(); + } + + /** Stop watching. Any in-flight checkpoint keeps running — await drain(). */ + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** One poll: fire an off-thread passive checkpoint when growth passes the soft threshold. */ + check(): void { + if (!this.pause && !this.inflight && this.growthBytes() > this.softBytes) this.fire(); + } + + /** + * Writer-side backstop, called at a between-transactions boundary. Returns + * null (no wait) while growth is under the hard cap; past it, returns a + * promise that resolves only once a FULL backfill has landed — see the + * header comment for why a single pass is not enough on a saturated disk. + */ + backpressure(): Promise | null { + if (this.pause) return this.pause; + if (this.growthBytes() <= this.hardBytes) return null; + this.log(`backpressure: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)} — pausing writer for full backfill`); + const t0 = Date.now(); + this.pause = this.backfillFully().finally(() => { + this.pause = null; + this.log(`backpressure released after ${Date.now() - t0}ms: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)}`); + }); + return this.pause; + } + + /** Await any in-flight checkpoint and writer pause. */ + async drain(): Promise { + while (this.pause || this.inflight) { + if (this.pause) await this.pause; + if (this.inflight) await this.inflight; + } + } + + /** + * Phase-boundary fold: backfill the ENTIRE WAL now (off-thread, awaited). + * Called between bulk phases — e.g. after parsing, before resolution's + * first reads — so the next phase never pages a bulk-write-sized WAL on + * the main thread (the post-parse read against a multi-GB WAL is what + * blew the #850 watchdog's 60s window in the #1231 repro). The await + * keeps the event loop (and the watchdog heartbeat) turning. + */ + async foldNow(): Promise { + await this.drain(); + if (this.growthBytes() <= 0) return; + this.log(`foldNow: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)}`); + this.pause = this.backfillFully().finally(() => { this.pause = null; }); + await this.pause; + } + + /** + * With the writer parked on the returned promise, loop passive passes until + * one reports the entire WAL backfilled (typically the second: the first + * drains the pass that was already running against a stale snapshot). Gives + * up after a bounded number of passes — e.g. a reader pinning the WAL — + * because unbounded WAL growth degrades; a wedged writer never recovers. + */ + private async backfillFully(): Promise { + for (let i = 0; i < MAX_PAUSED_BACKFILL_PASSES; i++) { + if (this.inflight) await this.inflight; // fold in the stale in-flight pass first + const res = await this.db.checkpointWalPassive(); + if (!res) return; // checkpoint machinery unavailable — don't spin + this.log(`backfill pass ${i + 1}: busy=${res.busy} log=${res.log} checkpointed=${res.checkpointed} wal=${this.mb(this.db.getWalSizeBytes())}`); + if (res.busy === 0 && res.log === res.checkpointed) { + this.sizeAtLastFullBackfill = this.db.getWalSizeBytes(); + return; + } + } + this.log(`backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes — WAL stays unbounded this cycle`); + } + + private fire(): void { + const p = this.db + .checkpointWalPassive() + .then((res) => { + // Full backfill (busy 0, every log frame checkpointed) ⇒ the writer's + // next commit wraps the WAL; the file's current size becomes the new + // growth baseline. A partial pass (writer appended during it, or a + // read transaction pinned frames) leaves the baseline alone, so the + // next tick fires again and copies the remainder. In non-WAL mode + // SQLite reports log = checkpointed = -1, which is harmless here. + if (res && res.busy === 0 && res.log === res.checkpointed) { + this.sizeAtLastFullBackfill = this.db.getWalSizeBytes(); + } + }) + .catch(() => { /* best-effort */ }) + .finally(() => { + if (this.inflight === p) this.inflight = null; + }); + this.inflight = p; + } +} diff --git a/src/directory.ts b/src/directory.ts new file mode 100644 index 0000000..fd4a1aa --- /dev/null +++ b/src/directory.ts @@ -0,0 +1,813 @@ +/** + * Directory Management + * + * Manages the .codegraph/ directory structure for CodeGraph data. + */ + +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +/** The default per-project data directory name. */ +const DEFAULT_CODEGRAPH_DIR = '.codegraph'; + +let warnedBadDirName = false; + +/** + * Resolve the per-project data directory name, honoring the `CODEGRAPH_DIR` + * environment override (default `.codegraph`). The override is a single path + * segment that lives in the project root. + * + * Why this exists: two environments that share one working tree must NOT share + * one `.codegraph/` — most concretely Windows-native and WSL (issue #636). The + * daemon lockfile (`.codegraph/daemon.pid`) records a platform-specific pid and + * socket path (a Windows named pipe vs a WSL Unix socket), and SQLite file + * locking across the WSL2 ↔ Windows filesystem boundary is unreliable, so two + * daemons sharing one index risks corruption. Setting `CODEGRAPH_DIR=.codegraph-win` + * on one side gives each environment its own index in the same tree. + * + * Read live (not captured at load) so it is both process-accurate and testable. + * An override that isn't a plain directory name — empty, containing a path + * separator, `.`, `..`/traversal, or absolute — is ignored (we keep the + * default) rather than risk writing the index outside the project or into the + * project root itself; we warn once to stderr so the misconfiguration is seen. + */ +export function codeGraphDirName(): string { + const raw = process.env.CODEGRAPH_DIR?.trim(); + if (!raw) return DEFAULT_CODEGRAPH_DIR; + const invalid = + raw === '.' || + raw.includes('..') || + raw.includes('/') || + raw.includes('\\') || + path.isAbsolute(raw); + if (invalid) { + if (!warnedBadDirName) { + warnedBadDirName = true; + // stderr only — stdout is the MCP protocol channel. + console.warn( + `[codegraph] Ignoring invalid CODEGRAPH_DIR="${raw}" — it must be a plain ` + + `directory name (no path separators, no "..", not absolute). Using "${DEFAULT_CODEGRAPH_DIR}".` + ); + } + return DEFAULT_CODEGRAPH_DIR; + } + return raw; +} + +/** + * CodeGraph directory name — a load-time snapshot of {@link codeGraphDirName}. + * A running process's environment is fixed, so this equals the live value; + * it's kept as a stable string export for backward compatibility. Internal code + * resolves the name through {@link codeGraphDirName} / {@link getCodeGraphDir} + * so the `CODEGRAPH_DIR` override always applies. + */ +export const CODEGRAPH_DIR = codeGraphDirName(); + +/** + * Is `name` (a single path segment) a CodeGraph data directory? Matches the + * default `.codegraph`, the active `CODEGRAPH_DIR` override, and any + * `.codegraph-*` sibling. File-watching and the indexer skip ALL of these, so + * when two environments share one working tree (Windows + WSL, issue #636) + * neither indexes or watches the other's index directory. + */ +export function isCodeGraphDataDir(name: string): boolean { + return ( + name === DEFAULT_CODEGRAPH_DIR || + name === codeGraphDirName() || + name.startsWith(DEFAULT_CODEGRAPH_DIR + '-') + ); +} + +/** + * Get the .codegraph directory path for a project + */ +export function getCodeGraphDir(projectRoot: string): string { + return path.join(projectRoot, codeGraphDirName()); +} + +/** + * Check if a project has been initialized with CodeGraph + * Requires both .codegraph/ directory AND codegraph.db to exist + */ +export function isInitialized(projectRoot: string): boolean { + const codegraphDir = getCodeGraphDir(projectRoot); + if (!fs.existsSync(codegraphDir) || !fs.statSync(codegraphDir).isDirectory()) { + return false; + } + // Must have codegraph.db, not just .codegraph folder + const dbPath = path.join(codegraphDir, 'codegraph.db'); + return fs.existsSync(dbPath); +} + +/** + * Find the nearest parent directory containing .codegraph/ + * + * Walks up from the given path to find a CodeGraph-initialized project, + * similar to how git finds .git/ directories. + * + * @param startPath - Directory to start searching from + * @returns The project root containing .codegraph/, or null if not found + */ +/** + * Reason a directory is unsafe to use as an index ROOT, or null when it's fine. + * + * Indexing your home directory or a filesystem root drags in caches, `Library`, + * every other project, etc. — a multi-GB index, constant file-watcher churn, and + * (pre-1.0 on macOS) a file-descriptor blowup that exhausted `kern.maxfiles` and + * took unrelated apps / the whole machine down (#845). The classic trigger: + * running the installer or `codegraph init` from `$HOME`, which auto-indexes the + * current directory. These are never intended project roots, so the installer + * and `init`/`index` refuse them (overridable with `--force`). + * + * Pure-ish (reads only `os.homedir()` + realpath) so it's easy to unit-test. + * The returned string is a human phrase that slots into "… looks like {reason}". + */ +export function unsafeIndexRootReason(projectRoot: string): string | null { + const resolve = (p: string): string => { + try { + return fs.realpathSync(path.resolve(p)); + } catch { + return path.resolve(p); + } + }; + const resolved = resolve(projectRoot); + + // Filesystem root: `/` on POSIX, a drive root like `C:\` on Windows. + if (path.parse(resolved).root === resolved) { + return 'the filesystem root'; + } + + const home = resolve(os.homedir()); + // Case-insensitive on macOS/Windows (case-preserving but case-insensitive FS). + const norm = (p: string): string => + process.platform === 'darwin' || process.platform === 'win32' ? p.toLowerCase() : p; + const r = norm(resolved); + const h = norm(home); + + if (r === h) { + return 'your home directory'; + } + // An ancestor of home (e.g. `/Users`, `/home`) — even broader than home. + if (h.startsWith(r + path.sep)) { + return 'a parent of your home directory'; + } + return null; +} + +export function findNearestCodeGraphRoot(startPath: string): string | null { + let current = path.resolve(startPath); + const root = path.parse(current).root; + + while (current !== root) { + if (isInitialized(current)) { + return current; + } + const parent = path.dirname(current); + if (parent === current) break; // Reached filesystem root + current = parent; + } + + // Check root as well + if (isInitialized(current)) { + return current; + } + + return null; +} + +/** Heavy/irrelevant directory names the sub-project scan never descends into. */ +const SUBPROJECT_SCAN_SKIP = new Set([ + 'node_modules', '.git', '.svn', '.hg', 'dist', 'build', 'out', 'target', + 'vendor', 'bin', 'obj', '.next', '.nuxt', '.svelte-kit', '.cache', 'coverage', + '.venv', 'venv', '__pycache__', '.turbo', '.idea', '.vscode', 'tmp', 'temp', +]); + +/** Manifests that mark a directory as a project/workspace root. The down-scan + * is gated on one of these so a non-project cwd (e.g. `$HOME`) is a cheap + * no-op instead of a deep filesystem crawl. */ +const WORKSPACE_ROOT_MANIFESTS = [ + 'package.json', 'pnpm-workspace.yaml', 'lerna.json', 'nx.json', 'turbo.json', + 'go.work', 'go.mod', 'Cargo.toml', 'pom.xml', 'build.gradle', 'build.gradle.kts', + 'settings.gradle', 'pyproject.toml', 'composer.json', 'Gemfile', 'rush.json', + 'WORKSPACE', 'WORKSPACE.bazel', +]; + +function looksLikeProjectRoot(dir: string): boolean { + return WORKSPACE_ROOT_MANIFESTS.some((m) => fs.existsSync(path.join(dir, m))); +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Indexed sub-project roots beneath `root` (bounded breadth-first scan). For + * the monorepo case behind #964: the index lives in a CHILD + * (`packages/x/.codegraph/`), not at the workspace root the agent's cwd points + * at. Descent stops at the first indexed directory on a branch (a project's + * own sub-dirs aren't separate projects) and is bounded by depth + count so it + * never turns into a full-tree crawl on a large repo. + */ +export function findIndexedSubprojectRoots( + root: string, + opts: { maxDepth?: number; max?: number } = {}, +): string[] { + const maxDepth = opts.maxDepth ?? 4; + const max = opts.max ?? 64; + const out: string[] = []; + const walk = (dir: string, depth: number): void => { + if (out.length >= max || depth > maxDepth) return; + let entries: fs.Dirent[]; + try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of entries) { + if (out.length >= max) return; + if (!e.isDirectory()) continue; + if (e.name.startsWith('.') || SUBPROJECT_SCAN_SKIP.has(e.name)) continue; + const child = path.join(dir, e.name); + if (isInitialized(child)) { out.push(child); continue; } // don't descend into an indexed project + walk(child, depth + 1); + } + }; + walk(root, 1); + return out; +} + +/** + * Unicode-aware word-boundary emulation for the keyword lists below. JS's `\b` + * is ASCII-only — it fires only at `[A-Za-z0-9_]` edges — so it can never bound + * a keyword whose first or last character is accented or non-Latin: `/\boù\b/` + * NEVER matches "où est …" (ù isn't an ASCII word char, so no boundary exists + * next to it). That is the #994 CJK mechanism resurfaced for Latin scripts and + * Cyrillic (#1126). A lookaround — "not flanked by a letter, digit, or + * underscore" — is the script-independent equivalent. + */ +const NOT_WORD_BEFORE = /(? 的调用链?" / "where is 定義" prompts no keyword list would. + * + * These are *candidates*, not a verdict: a tech brand like `JavaScript` or + * `GitHub` is identifier-shaped too, so the front-load hook checks each token + * against the actual index ({@link getNodesByName}) and only fires when one is a + * real symbol here — otherwise a brand-name prompt would inject ~16KB of + * low-relevance context (issue #994 follow-up). A doc/data filename ("README.md") + * is excluded from the member-access form since it's a file reference, not a symbol. + */ +export function extractCodeTokens(prompt: string): string[] { + if (!prompt) return []; + const out = new Set(); + // camelCase / PascalCase-with-inner-cap (getUserId, parseToken, UserService) or + // snake_case (article_publish, get_user) — a whole identifier run that has an + // inner lower→upper transition or an underscore flanked by alphanumerics. + for (const m of prompt.matchAll(/[A-Za-z_$][\w$]*/g)) { + const w = m[0]; + if (/[a-z][A-Z]/.test(w) || /[A-Za-z0-9]_[A-Za-z0-9]/.test(w)) out.add(w); + } + // call form: an identifier directly before '(' — parseToken(, render(). No + // whitespace before '(' so prose like "the function (entry point)" doesn't trip it. + for (const m of prompt.matchAll(/([A-Za-z_$][\w$]*)\(/g)) out.add(m[1]!); + // member access on identifiers (user.login) — but not a doc/data filename. + for (const m of prompt.matchAll(/([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)/g)) { + if (!DOC_DATA_EXT.test(m[0])) { out.add(m[1]!); out.add(m[2]!); } + } + return [...out]; +} + +/** + * Cheap, graph-free candidate gate for the front-load hook: could `prompt` be a + * structural / flow / impact / "where-how" question worth front-loading context + * for? True on an explicit keyword in any covered language (#994, #1126) OR an + * identifier-shaped token. A keyword is sufficient to fire on its own; a + * token-only match is only a candidate the hook then verifies against the graph + * (a brand name like `JavaScript` is token-shaped but isn't a symbol). Every + * non-candidate prompt ("fix this typo", in any language) stays a zero-cost no-op. + */ +export function isStructuralPrompt(prompt: string): boolean { + return hasStructuralKeyword(prompt) || extractCodeTokens(prompt).length > 0; +} + +/** + * What the front-load hook should do for a prompt issued from a directory. + */ +export interface FrontloadPlan { + /** Open + explore this project and inject its source as context. `null` when + * there's no single project to front-load (none indexed, or several indexed + * sub-projects with no clear match — see {@link nudgeProjects}). */ + exploreRoot: string | null; + /** Indexed sub-projects to surface in a "pass `projectPath`" nudge: the rest + * of a monorepo's indexed projects alongside `exploreRoot`, or — when no one + * project clearly matches — the full list (with `exploreRoot` null). */ + nudgeProjects: string[]; + /** True when the plan came from scanning DOWN into sub-projects (cwd itself + * is not under any index) — the monorepo case, where a follow-up + * `codegraph_explore` needs an explicit `projectPath`. */ + viaSubScan: boolean; +} + +/** + * Decide what the front-load hook injects for a `prompt` issued from `cwd`, + * shaped by where the `.codegraph/` index(es) actually are: + * 1. **cwd (or an ancestor) is indexed** → front-load that project. The + * normal single-project / nested-file case. + * 2. **cwd isn't indexed but looks like a workspace root** → the indexes live + * in sub-projects (the monorepo case behind #964). One indexed + * sub-project → front-load it; several → front-load the one the prompt + * names (by relative path like `packages/api`, or package directory name) + * and nudge about the rest; several with no match → nudge the full list so + * the agent passes `projectPath`, rather than guessing wrong. + * 3. **nothing indexed reachable** → do nothing (the agent's own tools apply). + */ +export function planFrontload(cwd: string, prompt: string): FrontloadPlan { + const none: FrontloadPlan = { exploreRoot: null, nudgeProjects: [], viaSubScan: false }; + + // 1. up-walk — nearest indexed ancestor (incl. cwd). Cheap; covers the common + // single-project case without a down-scan. + let dir = path.resolve(cwd); + for (let i = 0; i < 6; i++) { + if (isInitialized(dir)) return { exploreRoot: dir, nudgeProjects: [], viaSubScan: false }; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + + // 2. down-scan — only from something that looks like a workspace root, so a + // non-project cwd (e.g. $HOME) is a cheap no-op, not a deep crawl. + const base = path.resolve(cwd); + if (!looksLikeProjectRoot(base)) return none; + const subs = findIndexedSubprojectRoots(base); + if (subs.length === 0) return none; + if (subs.length === 1) return { exploreRoot: subs[0]!, nudgeProjects: [], viaSubScan: true }; + + // Several indexed sub-projects — pick the one the prompt points at, if any. + const p = prompt.toLowerCase(); + let best: { root: string; score: number; relLen: number } | null = null; + for (const s of subs) { + const rel = path.relative(base, s); + const relLc = rel.split(path.sep).join('/').toLowerCase(); + const name = path.basename(s).toLowerCase(); + let score = 0; + if (relLc && p.includes(relLc)) score = 10; // "packages/api" + else if (name.length >= 3 && new RegExp(`\\b${escapeRegExp(name)}\\b`).test(p)) score = 5; // "api" + if (score > 0 && (!best || score > best.score || (score === best.score && rel.length < best.relLen))) { + best = { root: s, score, relLen: rel.length }; + } + } + if (best) { + return { exploreRoot: best.root, nudgeProjects: subs.filter((s) => s !== best!.root), viaSubScan: true }; + } + // No clear match — nudge the full list rather than front-load a guess. + return { exploreRoot: null, nudgeProjects: subs, viaSubScan: true }; +} + +/** + * Contents of `.codegraph/.gitignore`. A single wildcard ignore keeps every + * transient file in the index dir — the database, `daemon.pid`, the socket, + * logs, cache, and anything future versions add — out of git, without having + * to enumerate each name (issues #788, #492, #484). Older versions wrote an + * explicit allowlist that never listed `daemon.pid` or the socket, so those + * runtime files were silently committed. + */ +const GITIGNORE_CONTENT = `# CodeGraph data files — local to each machine, not for committing. +# Ignore everything in .codegraph/ except this file itself, so transient +# files (the database, daemon.pid, sockets, logs) never show up in git. +* +!.gitignore +`; + +/** Header line that prefixes every .gitignore CodeGraph has auto-generated. */ +const GITIGNORE_MARKER = '# CodeGraph data files'; + +/** + * Is `content` a stale CodeGraph-generated `.gitignore` that should be + * regenerated in place? True when it carries our header but predates the + * wildcard ignore (it has no bare `*` line) — i.e. one of the old explicit + * allowlists (`*.db`, `cache/`, `.dirty`, …) that never ignored `daemon.pid` + * or the socket (issue #788). A file WITHOUT our header is user-authored and + * is left untouched; one that already has the wildcard is current. Matching + * on the header (not a byte-exact list of past defaults) heals every old + * variant — v0.7.x through 0.9.9 — and is idempotent once upgraded. + */ +function isStaleDefaultGitignore(content: string): boolean { + if (!content.trimStart().startsWith(GITIGNORE_MARKER)) return false; + return !content.split('\n').some((line) => line.trim() === '*'); +} + +/** + * Write `.codegraph/.gitignore` if it's absent, or upgrade a stale + * CodeGraph-generated default in place; a user-customized file is left alone. + * Best-effort — returns `false` only if a needed write failed. + */ +function ensureGitignore(gitignorePath: string): boolean { + let existing: string | null; + try { + existing = fs.readFileSync(gitignorePath, 'utf-8'); + } catch { + existing = null; // absent (ENOENT) or unreadable — (re)create below + } + // Current default or a user-authored file: nothing to do. + if (existing !== null && !isStaleDefaultGitignore(existing)) return true; + try { + fs.writeFileSync(gitignorePath, GITIGNORE_CONTENT, 'utf-8'); + return true; + } catch { + return false; + } +} + +/** + * Create the .codegraph directory structure + * Note: Only throws if codegraph.db already exists, not just if .codegraph/ exists. + */ +export function createDirectory(projectRoot: string): void { + const codegraphDir = getCodeGraphDir(projectRoot); + const dbPath = path.join(codegraphDir, 'codegraph.db'); + + // Only throw if CodeGraph is actually initialized (db exists) + // .codegraph/ folder alone is fine + if (fs.existsSync(dbPath)) { + throw new Error(`CodeGraph already initialized in ${projectRoot}`); + } + + // Create main directory (if it doesn't exist) + fs.mkdirSync(codegraphDir, { recursive: true }); + + // Write .gitignore inside .codegraph (create if absent, upgrade a stale + // pre-wildcard default left by an older version — issue #788). + ensureGitignore(path.join(codegraphDir, '.gitignore')); +} + +/** + * Remove the .codegraph directory + */ +export function removeDirectory(projectRoot: string): void { + const codegraphDir = getCodeGraphDir(projectRoot); + + if (!fs.existsSync(codegraphDir)) { + return; + } + + // Verify .codegraph is a real directory, not a symlink pointing elsewhere + const lstat = fs.lstatSync(codegraphDir); + if (lstat.isSymbolicLink()) { + // Only remove the symlink itself, never follow it for recursive delete + fs.unlinkSync(codegraphDir); + return; + } + + if (!lstat.isDirectory()) { + // Not a directory - remove the single file + fs.unlinkSync(codegraphDir); + return; + } + + // Recursively remove directory + fs.rmSync(codegraphDir, { recursive: true, force: true }); +} + +/** + * Get all files in the .codegraph directory + */ +export function listDirectoryContents(projectRoot: string): string[] { + const codegraphDir = getCodeGraphDir(projectRoot); + + if (!fs.existsSync(codegraphDir)) { + return []; + } + + const files: string[] = []; + + function walkDir(dir: string, prefix: string = ''): void { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; + + // Skip symlinks to prevent following links outside .codegraph + if (entry.isSymbolicLink()) { + continue; + } + + if (entry.isDirectory()) { + walkDir(path.join(dir, entry.name), relativePath); + } else { + files.push(relativePath); + } + } + } + + walkDir(codegraphDir); + return files; +} + +/** + * Get the total size of the .codegraph directory in bytes + */ +export function getDirectorySize(projectRoot: string): number { + const codegraphDir = getCodeGraphDir(projectRoot); + + if (!fs.existsSync(codegraphDir)) { + return 0; + } + + let totalSize = 0; + + function walkDir(dir: string): void { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + // Skip symlinks to prevent following links outside .codegraph + if (entry.isSymbolicLink()) { + continue; + } + + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + walkDir(fullPath); + } else { + const stats = fs.statSync(fullPath); + totalSize += stats.size; + } + } + } + + walkDir(codegraphDir); + return totalSize; +} + +/** + * Ensure a subdirectory exists within .codegraph + */ +export function ensureSubdirectory(projectRoot: string, subdirName: string): string { + if (subdirName.includes('..') || subdirName.includes(path.sep) || subdirName.includes('/')) { + throw new Error(`Invalid subdirectory name: ${subdirName}`); + } + + const subdirPath = path.join(getCodeGraphDir(projectRoot), subdirName); + + if (!fs.existsSync(subdirPath)) { + fs.mkdirSync(subdirPath, { recursive: true }); + } + + return subdirPath; +} + +/** + * Check if the .codegraph directory has valid structure + */ +export function validateDirectory(projectRoot: string): { + valid: boolean; + errors: string[]; +} { + const errors: string[] = []; + const codegraphDir = getCodeGraphDir(projectRoot); + + if (!fs.existsSync(codegraphDir)) { + errors.push('CodeGraph directory does not exist'); + return { valid: false, errors }; + } + + if (!fs.statSync(codegraphDir).isDirectory()) { + errors.push('.codegraph exists but is not a directory'); + return { valid: false, errors }; + } + + // Auto-repair / upgrade .gitignore (non-critical file). A missing one is + // recreated; a stale pre-wildcard default that never ignored daemon.pid is + // regenerated in place (issue #788); a user-authored file is left alone. + const gitignorePath = path.join(codegraphDir, '.gitignore'); + const existedBefore = fs.existsSync(gitignorePath); + if (!ensureGitignore(gitignorePath) && !existedBefore) { + // Only a missing-and-uncreatable file is surfaced; a failed in-place + // upgrade of an existing file is non-fatal — the index still works. + errors.push('.gitignore missing in .codegraph directory and could not be created'); + } + + return { + valid: errors.length === 0, + errors, + }; +} diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..acd440c --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,240 @@ +/** + * CodeGraph Error Classes + * + * Custom error types for better error handling and debugging. + * + * @module errors + * + * @example + * ```typescript + * import { FileError, ParseError, setLogger, silentLogger } from 'codegraph'; + * + * // Catch specific error types + * try { + * await cg.indexAll(); + * } catch (error) { + * if (error instanceof FileError) { + * console.log(`File error at ${error.filePath}: ${error.message}`); + * } else if (error instanceof ParseError) { + * console.log(`Parse error at ${error.filePath}:${error.line}`); + * } + * } + * + * // Disable logging for tests + * setLogger(silentLogger); + * ``` + */ + +/** + * Base error class for all CodeGraph errors. + * + * All CodeGraph-specific errors extend this class, allowing you to catch + * all CodeGraph errors with a single catch block. + * + * @example + * ```typescript + * try { + * await cg.indexAll(); + * } catch (error) { + * if (error instanceof CodeGraphError) { + * console.log(`CodeGraph error [${error.code}]: ${error.message}`); + * } + * } + * ``` + */ +export class CodeGraphError extends Error { + /** Error code for categorization (e.g., 'FILE_ERROR', 'PARSE_ERROR') */ + readonly code: string; + /** Additional context about the error */ + readonly context?: Record; + + constructor(message: string, code: string, context?: Record) { + super(message); + this.name = 'CodeGraphError'; + this.code = code; + this.context = context; + + // Maintain proper stack trace for V8 + if (Error.captureStackTrace) { + Error.captureStackTrace(this, this.constructor); + } + } +} + +/** + * Error reading or accessing files + */ +export class FileError extends CodeGraphError { + readonly filePath: string; + + constructor(message: string, filePath: string, cause?: Error) { + super(message, 'FILE_ERROR', { filePath, cause: cause?.message }); + this.name = 'FileError'; + this.filePath = filePath; + if (cause) { + this.cause = cause; + } + } +} + +/** + * Error parsing source code + */ +export class ParseError extends CodeGraphError { + readonly filePath: string; + readonly line?: number; + readonly column?: number; + + constructor( + message: string, + filePath: string, + options?: { line?: number; column?: number; cause?: Error } + ) { + super(message, 'PARSE_ERROR', { + filePath, + line: options?.line, + column: options?.column, + cause: options?.cause?.message, + }); + this.name = 'ParseError'; + this.filePath = filePath; + this.line = options?.line; + this.column = options?.column; + if (options?.cause) { + this.cause = options.cause; + } + } +} + +/** + * Error with database operations + */ +export class DatabaseError extends CodeGraphError { + readonly operation: string; + + constructor(message: string, operation: string, cause?: Error) { + super(message, 'DATABASE_ERROR', { operation, cause: cause?.message }); + this.name = 'DatabaseError'; + this.operation = operation; + if (cause) { + this.cause = cause; + } + } +} + +/** + * Error with search operations + */ +export class SearchError extends CodeGraphError { + readonly query: string; + + constructor(message: string, query: string, cause?: Error) { + super(message, 'SEARCH_ERROR', { query, cause: cause?.message }); + this.name = 'SearchError'; + this.query = query; + if (cause) { + this.cause = cause; + } + } +} + +/** + * Error with vector/embedding operations + */ +export class VectorError extends CodeGraphError { + constructor(message: string, operation: string, cause?: Error) { + super(message, 'VECTOR_ERROR', { operation, cause: cause?.message }); + this.name = 'VectorError'; + if (cause) { + this.cause = cause; + } + } +} + +/** + * Error with configuration + */ +export class ConfigError extends CodeGraphError { + constructor(message: string, details?: Record) { + super(message, 'CONFIG_ERROR', details); + this.name = 'ConfigError'; + } +} + +/** + * Simple logger for CodeGraph operations + * + * By default, logs to console.warn for warnings and console.error for errors. + * Can be configured to use custom logging. + */ +export interface Logger { + debug(message: string, context?: Record): void; + warn(message: string, context?: Record): void; + error(message: string, context?: Record): void; +} + +/** + * Default console-based logger + */ +export const defaultLogger: Logger = { + debug(message: string, context?: Record): void { + if (process.env.CODEGRAPH_DEBUG) { + console.debug(`[CodeGraph] ${message}`, context ?? ''); + } + }, + warn(message: string, context?: Record): void { + console.warn(`[CodeGraph] ${message}`, context ?? ''); + }, + error(message: string, context?: Record): void { + console.error(`[CodeGraph] ${message}`, context ?? ''); + }, +}; + +/** + * Silent logger (no output) - useful for tests + */ +export const silentLogger: Logger = { + debug(): void {}, + warn(): void {}, + error(): void {}, +}; + +/** + * Current logger instance (can be replaced) + */ +let currentLogger: Logger = defaultLogger; + +/** + * Set the global logger + */ +export function setLogger(logger: Logger): void { + currentLogger = logger; +} + +/** + * Get the current logger + */ +export function getLogger(): Logger { + return currentLogger; +} + +/** + * Log a debug message + */ +export function logDebug(message: string, context?: Record): void { + currentLogger.debug(message, context); +} + +/** + * Log a warning message + */ +export function logWarn(message: string, context?: Record): void { + currentLogger.warn(message, context); +} + +/** + * Log an error message + */ +export function logError(message: string, context?: Record): void { + currentLogger.error(message, context); +} diff --git a/src/extraction/astro-extractor.ts b/src/extraction/astro-extractor.ts new file mode 100644 index 0000000..e389893 --- /dev/null +++ b/src/extraction/astro-extractor.ts @@ -0,0 +1,365 @@ +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; +import { TreeSitterExtractor } from './tree-sitter'; +import { isLanguageSupported } from './grammars'; + +/** + * Astro built-in components — compiler-provided (``) or shipped by + * `astro:components` (`

`, ``), not user code.
+ */
+const ASTRO_BUILTIN_COMPONENTS = new Set(['Fragment', 'Code', 'Debug']);
+
+/**
+ * AstroExtractor - Extracts code relationships from Astro component files
+ *
+ * Astro files are multi-language: a TypeScript frontmatter block fenced by
+ * `---` lines, a JSX-like HTML template, and optional  and  ranges
+    const tagRegex = /<(script|style)(\s[^>]*)?>[\s\S]*?<\/\1>/g;
+    let tagMatch;
+    while ((tagMatch = tagRegex.exec(this.source)) !== null) {
+      const startLine = (this.source.substring(0, tagMatch.index).match(/\n/g) || []).length;
+      const endLine = startLine + (tagMatch[0].match(/\n/g) || []).length;
+      coveredRanges.push([startLine, endLine]);
+    }
+
+    // Find template expressions: {...} outside of script/style blocks
+    // Matches curly-brace expressions, excluding Svelte block syntax ({#if}, {:else}, {/if}, {@html}, {@render})
+    const lines = this.source.split('\n');
+    const exprRegex = /\{([^}#/:@][^}]*)\}/g;
+
+    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+      // Skip lines inside script/style blocks
+      if (coveredRanges.some(([start, end]) => lineIdx >= start && lineIdx <= end)) continue;
+
+      const line = lines[lineIdx]!;
+      let exprMatch;
+      while ((exprMatch = exprRegex.exec(line)) !== null) {
+        const expr = exprMatch[1]!;
+        // Extract function calls: identifiers followed by (
+        // Matches: cn(...), buttonVariants(...), obj.method(...)
+        const callRegex = /\b([a-zA-Z_$][\w$.]*)\s*\(/g;
+        let callMatch;
+        while ((callMatch = callRegex.exec(expr)) !== null) {
+          const calleeName = callMatch[1]!;
+          // Skip Svelte runes, control flow keywords, and common non-function patterns
+          if (SVELTE_RUNES.has(calleeName)) continue;
+          if (calleeName === 'if' || calleeName === 'else' || calleeName === 'each' || calleeName === 'await') continue;
+
+          this.unresolvedReferences.push({
+            fromNodeId: componentNodeId,
+            referenceName: calleeName,
+            referenceKind: 'calls',
+            line: lineIdx + 1, // 1-indexed
+            column: exprMatch.index + callMatch.index,
+            filePath: this.filePath,
+            language: 'svelte',
+          });
+        }
+      }
+    }
+  }
+
+  /**
+   * Extract component usages from the Svelte template.
+   *
+   * PascalCase tags like ,