Files
wehub-resource-sync a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:33:42 +08:00

7511 lines
287 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package indexer
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pelletier/go-toml/v2"
"go.uber.org/zap"
"golang.org/x/sync/semaphore"
"github.com/zzet/gortex/internal/codegen"
"github.com/zzet/gortex/internal/codeowners"
"github.com/zzet/gortex/internal/config"
"github.com/zzet/gortex/internal/contracts"
"github.com/zzet/gortex/internal/embedding"
"github.com/zzet/gortex/internal/entrypoints"
"github.com/zzet/gortex/internal/excludes"
"github.com/zzet/gortex/internal/fixtures"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/intern"
"github.com/zzet/gortex/internal/licenses"
"github.com/zzet/gortex/internal/modules"
"github.com/zzet/gortex/internal/parser"
"github.com/zzet/gortex/internal/parser/crashpool"
"github.com/zzet/gortex/internal/pathkey"
"github.com/zzet/gortex/internal/progress"
"github.com/zzet/gortex/internal/reach"
"github.com/zzet/gortex/internal/resolver"
"github.com/zzet/gortex/internal/search"
"github.com/zzet/gortex/internal/search/trigram"
"github.com/zzet/gortex/internal/semantic"
gortexsql "github.com/zzet/gortex/internal/sql"
"github.com/zzet/gortex/internal/todos"
)
// IndexResult holds the outcome of an indexing operation.
type IndexResult struct {
NodeCount int `json:"node_count"`
EdgeCount int `json:"edge_count"`
// FileCount is the total number of language files the indexer
// observed for this repo — i.e. how big the repo is on disk, not
// how much work this pass did. Stamped onto RepoMetadata so
// `daemon status` shows a stable file count across both full-track
// and incremental-reconcile paths.
FileCount int `json:"file_count"`
// StaleFileCount is the number of files that were actually
// re-indexed in this pass (only populated by IncrementalReindex
// — full-index passes treat every file as stale and would
// duplicate FileCount). Used by the janitor / reconcile log to
// report "how much work did the snapshot delta require".
StaleFileCount int `json:"stale_file_count,omitempty"`
// FailedFiles lists files an incremental pass could not index even
// after one retry — a parse error, or a file locked or removed
// mid-pass. A caller can replay them explicitly. Whether the next
// incremental pass retries them on its own depends on the failure:
// a file whose bytes could not even be read (locked, permission
// denied, removed mid-walk) never gets an mtime recorded, so it
// stays stale and is retried on every subsequent pass; a file that
// was read but failed to parse (syntax error, crash-isolation
// quarantine, extraction timeout) DOES get its current on-disk
// mtime recorded, so it is retried only once its content changes
// again — this keeps a warm restart from perpetually treating the
// whole repo as changed because of one unparseable file. Empty on
// a clean pass and on full-index passes.
FailedFiles []string `json:"failed_files,omitempty"`
// QuarantinedFiles is the number of files held in the parser
// crash-isolation quarantine after this pass — files that
// SIGSEGV'd / hung / panicked the parser and were skipped with a
// Meta["parse_error"] node. Zero unless crash isolation is on.
QuarantinedFiles int `json:"quarantined_files,omitempty"`
// SkippedFiles is the number of files skipped by the size cap
// (MaxFileSize), the per-file extraction timeout (MaxExtractMillis),
// or the content-admission policy (index.content — oversized documents
// and, by default, binary/vector data assets). Each is recorded in the
// graph as a synthetic file node carrying skipped_due_to_size /
// skipped_due_to_timeout / skipped_due_to_content telemetry. Zero
// unless one of those gates fires.
SkippedFiles int `json:"skipped_files,omitempty"`
// DeletedFileCount is the number of previously-indexed files that
// were evicted this pass because they no longer exist on disk (only
// populated by IncrementalReindex). Together with StaleFileCount it
// lets a batch caller — the daemon warmup loop in particular — decide
// whether a repo actually changed since the last shutdown: when both
// are zero across every repo, the persisted graph already carries
// every resolved / derived edge and the global resolution passes can
// be skipped entirely (the warm-restart fast path).
DeletedFileCount int `json:"deleted_file_count,omitempty"`
// FullRetrack is true when this result came from a whole-repo
// re-track (IndexCtx) rather than an incremental pass — i.e. the
// changed-file set is unknown and StaleFileCount does NOT reflect
// "how many files changed" (it keeps its normal incremental-work
// meaning and is 0 here). Callers that gate global re-resolution on
// "did this repo change" must OR in FullRetrack alongside
// StaleFileCount / DeletedFileCount.
FullRetrack bool `json:"full_retrack,omitempty"`
DurationMs int64 `json:"duration_ms"`
Errors []IndexError `json:"errors,omitempty"`
// RepoPrefix is the prefix the repo was actually registered under.
// It usually equals config.ResolvePrefix(entry), but a git worktree
// tracked as an independent instance gets a derived `<base>@<tag>`
// prefix that the caller cannot recompute from the (by-value) entry
// it passed in — callers that need to attach a watcher or report the
// outcome read it from here. Populated by the multi-repo track and
// reconcile paths; empty in single-repo mode.
RepoPrefix string `json:"repo_prefix,omitempty"`
}
// EdgeSanityViolated reports the post-reindex sanity-check failure: an
// index pass that observed source files and extracted symbol nodes from
// them, yet produced zero edges. A populated graph with no edges still
// looks indexed but answers every "who calls X" / "what imports Y"
// query with nothing — a wholesale edge-extraction failure (a broken
// grammar, an aborted reindex) worth surfacing rather than shipping
// silently. Even a single one-function file yields containment edges,
// so a real repo only trips this when extraction failed across the
// board.
func (r *IndexResult) EdgeSanityViolated() bool {
return r != nil && r.FileCount > 0 && r.NodeCount > 0 && r.EdgeCount == 0
}
// IndexError records a per-file parsing failure.
type IndexError struct {
FilePath string `json:"file_path"`
Error string `json:"error"`
}
// Indexer walks a repository and populates the graph.
type Indexer struct {
graph graph.Store
// indexCount tracks how many IndexCtx calls this Indexer has
// completed. Gates the cold-start shadow-swap: each per-repo
// Indexer in MultiIndexer is fresh (indexCount==0), so all of
// them take the shadow path regardless of what sibling repos
// have already drained into the shared disk store. Per-repo-
// prefixed stub IDs make the concurrent drains conflict-free.
indexCount atomic.Int32
registry *parser.Registry
resolver *resolver.Resolver
search search.Backend
config config.IndexConfig
transforms *transformPipeline
excludes *excludes.Matcher
excludeOnce sync.Once
dirIgnore *excludes.Hierarchical
dirIgnoreOnce sync.Once
rootPath string
// projectName is the repo's own name (go.mod module / package.json /
// dir), computed once per index. Stripped from the BM25-indexed file
// path so a query word matching it doesn't earn a useless uniform
// path-field boost across every document. "" disables the de-weighting.
projectName string
logger *zap.Logger
// Crash-isolation parser pool, lazily created and then reused
// across single-file re-indexes so the watcher hot path never
// forks a worker subprocess per file.
parsePool *crashpool.Pool
parseQuar *crashpool.Quarantine
parsePoolMu sync.Mutex
// Trigram code-search index, lazily built on first GrepText call
// and rebuilt only when indexGen advances past the build it was
// made from. indexGen is bumped by every full or incremental
// index, so a burst of searches between reindexes hits a warm
// index.
indexGen atomic.Uint64
trigramSearcher *trigram.Searcher
trigramGen uint64
trigramMu sync.Mutex
// repoPrefix is set in multi-repo mode to prefix all file paths and node IDs.
// When empty, the indexer operates in single-repo mode (backward compatible).
repoPrefix string
// workspaceID is the hard graph boundary slug for this repo.
// Stamped onto every node emitted by this indexer via applyRepoPrefix
// so query-time scoping doesn't have to look it up by repo prefix.
// Defaults at the MultiIndexer layer to the per-repo `.gortex.yaml`
// `workspace:` slug, falling back to repoPrefix when no slug is
// declared (so legacy configs keep working).
workspaceID string
// projectID is the soft sub-boundary slug. Defaults to the repo
// prefix in single-project repos. Monorepos resolve a per-file
// projectID via the `projects[]` paths-glob mapping in
// `.gortex.yaml`; until that lookup is wired in, every node from
// this indexer carries the repo-default value.
projectID string
// contractRegistry holds detected API contracts (HTTP routes, gRPC, etc.).
contractRegistry *contracts.Registry
// trackedRepoModules maps repo names to Go module paths for cross-repo dependency detection.
// Populated by MultiIndexer from go.mod files of tracked repos.
trackedRepoModules map[string]string
// embedder is the optional embedding provider for semantic search.
embedder embedding.Provider
// skipVectorBuild, when true, makes buildSearchIndex populate only
// the text index and never run the embedding pass — even with an
// embedder set. The daemon flips it on for the warmup re-index loop
// when a snapshot already carries the workspace vector index, so
// the graph is not re-embedded only to have the cached index
// overwrite it. Off by default; a normal index always builds
// vectors when an embedder is present.
skipVectorBuild bool
// bulkVectorSink holds the disk store captured at the bulk-load
// shadow swap, so buildSearchIndex can still persist the vector
// index to the backend while idx.graph points at the in-memory
// shadow (which does not implement graph.VectorSearcher). Without
// it the embedding pass under the bulk loader builds vectors only
// in the in-process HNSW — they never reach the `vectors` table and
// are lost on the next daemon restart, forcing a paid re-embed.
// Set during the shadow swap, cleared when idx.graph is restored.
bulkVectorSink graph.VectorSearcher
// contentSink mirrors bulkVectorSink for the content full-text index:
// the disk store captured at the shadow swap, so the per-file content
// stream reaches content_fts on disk even while idx.graph points at the
// in-memory shadow (which does not implement graph.ContentSearcher).
// Set during the shadow swap, cleared when idx.graph is restored.
contentSink graph.ContentSearcher
// embedChunkOpts tunes the AST sub-chunking buildSearchIndex applies
// to large symbols before embedding. The zero value makes the
// chunker fall back to its package defaults.
embedChunkOpts embedding.ChunkOptions
// embedMaxSymbols overrides the built-in cap on how many texts the
// vector index will hold before buildSearchIndex skips the embed
// pass. Zero keeps the built-in default.
embedMaxSymbols int
// embedAPIConcurrency bounds how many embedding requests run in
// parallel against an API-backed embedder. Zero keeps the built-in
// default. Ignored for in-process embedders, which serialise on an
// inference mutex.
embedAPIConcurrency int
// lastVectorBuildErr records why the most recent buildSearchIndex pass
// shipped text-only instead of a vector index (chunk-embed failure,
// all-vectors-invalid, or the symbol-count guard). Nil after a build that
// produced a vector index. Read via LastVectorBuildError once a build has
// finished — it lets `gortex eval embedders` report the real cause instead
// of a bare "no vector data".
lastVectorBuildErr error
// semanticMgr is the optional semantic enrichment manager.
semanticMgr *semantic.Manager
// resolverLSPHelper, when non-nil, is the resolve-time LSP
// helper installed on idx.resolver. Held here so MultiIndexer
// can mirror it onto the global post-pass resolver in
// RunDeferredPassesAll. See SetResolverLSPHelper.
resolverLSPHelper resolver.LSPHelper
// npmAliasOnce builds npmAlias lazily on the first resolve-time
// import-rewrite request. Lazy because the repo root and prefix
// are set after New(); by the time the resolver runs they are
// final.
npmAliasOnce sync.Once
npmAlias *npmAliasIndex
// workspaceMembersOnce builds workspaceMembers lazily on the first
// resolve-time package-manager-workspace lookup. Lazy for the same
// reason as npmAliasOnce — the repo root and prefix are final only
// after New().
workspaceMembersOnce sync.Once
workspaceMembers *workspaceMembershipIndex
// Mtime tracking and parse error retention for index health diagnostics.
parseErrors []IndexError
fileMtimes map[string]int64
lastIndexTime time.Time
totalDetected int
mtimeMu sync.RWMutex
// contractCache memoizes the contract-extractor output per file.
// Keyed by graph file path (with repo prefix); value is the file's
// disk mtime when last extracted plus the contracts that came out.
// extractContracts replays cache hits to skip the read + 8-extractor
// run for files that haven't changed since the last extraction —
// the dominant cost on repos with tens of thousands of source files.
contractCache map[string]*contractCacheEntry
contractCacheMu sync.RWMutex
// upgradeOnce gates the BM25→Bleve auto-upgrade to exactly one
// goroutine per indexer lifetime. Without this, every post-threshold
// IndexCtx — which fires once per tracked repo during multi-repo
// warmup — would spawn a fresh upgradeSearchToBleve goroutine.
// Each rebuilds ~N-doc Bleve indexes (≈32 KiB/doc), so overlapping
// upgrades peak memory far above steady-state and waste CPU on
// rebuilds that the next Swap immediately discards. Also counts
// scheduled upgrades so tests can observe gating decisions
// without relying on log scrapes or timing.
upgradeOnce sync.Once
upgradeSpawnedMu sync.Mutex
upgradeSpawned int
// deferResolve, when set, makes IndexCtx skip the cross-cutting passes
// (per-repo ResolveAll / semantic enrichment / contract extraction +
// commit) so the multi-repo orchestrator can run them serially after
// the parallel fan-out joins. Without this, two goroutines indexing
// different repos into the shared graph race on Edge.Meta during the
// resolver's mutation phase vs. the contract pass's graph walk via
// AllEdges().
deferResolve bool
pendingContractReg *contracts.Registry
// pendingEnrich is raised by an index pass that did real work — IndexCtx
// that observed files (or a whole-repo re-track) and IncrementalReindex /
// IncrementalReindexPaths that re-indexed or evicted at least one file. It
// is cleared only after runDeferredEnrich completes a fully non-partial
// semantic enrichment for this repo. The daemon warmup enriches every
// indexer it collected, so this gates the (multi-minute LSP hover) pass to
// repos that actually changed: an unchanged repo on a warm restart would
// otherwise re-confirm nothing for 10+ minutes. A partial / abandoned /
// failed enrich leaves it set so a later deferred pass retries.
pendingEnrich atomic.Bool
// fullReindexed is raised by a whole-repo (re-)parse — IndexCtx, reached
// via a full re-track, a cold TrackRepo, or a snapshot-partial forced full
// walk — which evicts and re-creates every node and edge for the repo. That
// drops the LSP hover-enrichment edges a static re-parse cannot reproduce,
// so the deferred-enrichment pass must re-run even when the persisted
// completion marker still records the repo's HEAD on a clean tree. It
// threads Force into RepoEnrichState so enrichMarkerCurrent stops gating the
// pass out. A fresh Indexer per daemon run starts it false, so it only ever
// reflects work this run performed.
fullReindexed atomic.Bool
// reparsedThisRun is the scoped analogue of fullReindexed: it is raised by a
// scoped incremental pass that re-parsed at least one stale file this run —
// IncrementalReindexPaths / IncrementalReindex with a non-zero
// StaleFileCount. A scoped re-parse evicts and re-creates just the changed
// files' nodes, dropping THEIR hover-enrichment edges exactly as a whole-repo
// re-parse drops every node's, so the deferred pass must likewise run past
// the completion marker at an unchanged clean HEAD — otherwise the re-parsed
// files' LSP edges stay durably gone until the repo's HEAD moves or its tree
// goes dirty. runDeferredEnrich ORs it into RepoEnrichState.Force; because the
// hover provider skips already-stamped nodes, the marker bypass re-hovers only
// the freshly-unstamped re-parsed files, not the whole repo, so the cost stays
// bounded. fullReindexed stays clear on the scoped paths — this flag keeps the
// two claims distinct. A fresh Indexer per daemon run starts it false.
reparsedThisRun atomic.Bool
// deferGlobalPasses, when set, makes IndexCtx and IncrementalReindex
// skip the graph-wide derivation passes (InferImplements,
// InferOverrides, markTestSymbolsAndEmitEdges). These passes walk the
// entire shared graph, so running them per-repo inside a batch loop
// (warmup, ReconcileAll) is O(R · global_size) — quadratic for repo
// counts in the hundreds. The batch caller is responsible for invoking
// RunGlobalGraphPasses exactly once at the end. Has no effect on the
// deferResolve path (multi-repo IndexCtx already skips those passes).
deferGlobalPasses bool
// skipResolveInDeferred, when set, makes RunDeferredPasses skip the
// per-repo resolver.ResolveAll() call. ResolveAll walks the entire
// shared graph, so paying it once per indexer across hundreds of
// repos is O(R · E). MultiIndexer.RunDeferredPassesAll sets this
// flag on every indexer and runs a single resolver.New(graph).ResolveAll
// once at the end, which picks up every placeholder edge at once.
// Has no effect on direct (non-batch) callers of RunDeferredPasses.
skipResolveInDeferred bool
// codeownersOnce ensures the repo-level CODEOWNERS file is parsed
// exactly once per indexer lifetime. The rule list is derived
// from .github/CODEOWNERS / CODEOWNERS / docs/CODEOWNERS at
// first use and applied per-file by applyCoverageDomains; an
// absent file produces empty rules and a no-op pass.
codeownersOnce sync.Once
codeownersRules []codeowners.Rule
// cloneIndex maintains the clone-detection CMS + length-stratified
// LSH live across single-file edits, so a steady-state reindex
// updates EdgeSimilarTo edges in O(edited file) instead of the
// whole-graph detectClonesAndEmitEdges recompute. Constructed empty
// (built=false) — a batch/global clone pass calls Rebuild to seed it,
// after which indexFile drives EvictFuncs/UpdateFuncs. While un-built
// indexFile falls back to the whole-graph pass.
cloneIndex *incrementalCloneIndex
// affectedByPasses / affectedByFilesResolved / affectedByDropped
// count the affected-by re-resolution activity (see affected_by.go):
// passes that found a signature delta and ran, referencing files
// re-resolved by them, and files dropped by the fan-out cap.
// Exposed via AffectedByCounts so tests and diagnostics can observe
// that a body-only edit triggered no fan-out.
affectedByPasses atomic.Int64
affectedByFilesResolved atomic.Int64
affectedByDropped atomic.Int64
}
// contractCacheEntry is a cached contract-extraction result for one file.
type contractCacheEntry struct {
mtimeNano int64
contracts []contracts.Contract
}
// New creates an Indexer that writes through the supplied graph.Store.
// Any backend (in-memory, SQLite-on-disk, remote) is acceptable — the
// indexer's mutation paths go through the Store interface methods only,
// so swapping backends is a zero-code-change configuration choice for
// callers.
func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *zap.Logger) *Indexer {
idx := &Indexer{
graph: g,
registry: reg,
resolver: resolver.New(g),
// Wrap in Swappable so the auto-upgrade to Bleve at large
// corpus sizes can happen in a background goroutine without
// racing with concurrent searches. Subsequent reassignments to
// idx.search (Hybrid wrap, etc.) should use swap helpers below.
//
// When the backing store implements graph.SymbolSearcher
// (today only store_sqlite), the initial backend is a thin
// adapter that forwards Search to the store's native FTS.
// The in-process Bleve / BM25 build path is then bypassed
// entirely — saving ~100MB heap on a Vscode-scale repo and
// putting search in the same address space as the rest of
// the graph queries.
search: search.NewSwappable(initialSearchBackend(g)),
config: cfg,
transforms: newTransformPipeline(cfg.Transforms, logger),
logger: logger,
fileMtimes: make(map[string]int64),
contractCache: make(map[string]*contractCacheEntry),
cloneIndex: newIncrementalCloneIndex(),
}
// Resolve JS/TS imports declared through an npm alias against the
// local index. The index is built lazily on first use — the repo
// root and prefix are not final until after New().
idx.resolver.SetNpmAliasResolver(idx.resolveNpmAliasImport)
// Expand JS/TS tsconfig/jsconfig path-alias imports (`@/lib/x`)
// against the local index so cross-directory alias imports resolve
// to their real file. Same lazy-build rationale.
idx.resolver.SetPathAliasResolver(idx.resolvePathAliasImport)
// Break same-named import collisions in favour of the importer's
// own package-manager workspace member. Same lazy-build rationale.
idx.resolver.SetWorkspaceMembership(idx.indexerWorkspaceMembership)
return idx
}
// resolveNpmAliasImport is the resolver.NpmAliasResolver installed on
// this Indexer's resolver. It rewrites a JS/TS import specifier that
// matches an npm-alias dependency key in the importing file's
// nearest-ancestor package.json. Returns "" (no rewrite) when no
// alias applies. The backing npmAliasIndex is built once, lazily.
func (idx *Indexer) resolveNpmAliasImport(callerFile, specifier string) string {
idx.npmAliasOnce.Do(func() {
idx.npmAlias = newNpmAliasIndex(map[string]string{idx.repoPrefix: idx.rootPath})
})
return idx.npmAlias.Resolve(callerFile, specifier)
}
// swappable returns the search backend cast to *search.Swappable. Panics
// if the invariant (idx.search is always a Swappable) is ever broken —
// that would be a programmer error in this file, not a runtime condition.
func (idx *Indexer) swappable() *search.Swappable {
if sw, ok := idx.search.(*search.Swappable); ok {
return sw
}
panic("indexer: search backend is not *search.Swappable — invariant violated")
}
// searchIndexFields returns the text fields fed to the BM25 search
// backend for a node. For an ordinary code symbol that is its
// name, file path, and signature. For a KindDoc prose-section node
// the body is what carries the search signal, so the section text
// (Meta["section_text"]) is indexed alongside the breadcrumb name
// -- a prose query then ranks the section, not just a heading match.
func searchIndexFields(n *graph.Node, projectName string) []string {
// The project-name path segment is stripped from the INDEXED path (not the
// stored FilePath) so it contributes no uniform path-field boost.
indexedPath := search.StripProjectNameFromPath(n.FilePath, projectName)
if n.Kind == graph.KindDoc {
body, _ := n.Meta["section_text"].(string)
return []string{n.Name, indexedPath, body}
}
sig, _ := n.Meta["signature"].(string)
// A symbol's doc comment is the natural-language statement of what it
// does — precisely the vocabulary a task-intent query carries ("union
// the two sequences", "performs matching on the ignore files") when it
// names no identifier verbatim. The identifier and signature alone
// answer name lookups; the doc summary is what lets an intent query
// reach the definition at all. Only the leading summary is indexed
// (docSummary), so a long doc block's examples and edge-case prose
// can't dilute the name/signature tokens under BM25 length
// normalisation. Empty doc → empty field, dropped by the caller, so a
// symbol with no doc indexes exactly as before.
doc, _ := n.Meta["doc"].(string)
return []string{n.Name, indexedPath, sig, docSummary(doc)}
}
// docSummaryMaxRunes bounds how much of a doc comment enters the search
// document. The leading summary is the highest-signal part; the rest of a
// long doc — parameter lists, example code blocks, edge-case notes — is
// lower-signal and, unbounded, would dominate the token bag and dilute the
// identifier under BM25 length normalisation. The bound keeps the doc's
// contribution on the order of a signature's. It is a generic document-size
// budget, independent of any repository or corpus.
const docSummaryMaxRunes = 280
// docSummary returns the leading summary of a doc comment for the search
// index: the first paragraph (up to the first blank line), trimmed and
// bounded to docSummaryMaxRunes. Doc comments across languages put the
// one-line / one-paragraph summary first and defer detail and examples to
// later paragraphs, so the first paragraph is the highest-signal slice.
// Empty in → empty out.
func docSummary(doc string) string {
if doc == "" {
return ""
}
// Normalise CRLF so the blank-line cut is newline-style-agnostic.
if strings.Contains(doc, "\r") {
doc = strings.ReplaceAll(doc, "\r\n", "\n")
doc = strings.ReplaceAll(doc, "\r", "\n")
}
if i := strings.Index(doc, "\n\n"); i >= 0 {
doc = doc[:i]
}
doc = strings.TrimSpace(doc)
if r := []rune(doc); len(r) > docSummaryMaxRunes {
doc = strings.TrimSpace(string(r[:docSummaryMaxRunes]))
}
return doc
}
// vectorSearcherDelegate is the search.VectorDelegate-shaped
// adapter the indexer hands to VectorBackend.SetDelegate when the
// underlying store implements graph.VectorSearcher. SimilarTo just
// forwards — search.VectorDelegate is defined to return
// graph.VectorHit slices directly, so there's no translation work
// here, just a small struct so the in-process search package
// doesn't depend on graph.VectorSearcher's full surface.
type vectorSearcherDelegate struct {
s graph.VectorSearcher
}
func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.VectorHit, error) {
if d == nil || d.s == nil {
return nil, nil
}
return d.s.SimilarTo(vec, limit)
}
// initialSearchBackend picks the search.Backend the indexer wraps
// in its Swappable on construction. When the underlying store
// implements graph.SymbolSearcher (today only store_sqlite), a
// thin adapter routes Search calls through the store's native FTS
// — the in-process BM25 / Bleve build path is bypassed entirely.
// Otherwise falls through to search.NewAuto which picks BM25 for
// small corpora and auto-upgrades to Bleve once the size warrants
// it.
func initialSearchBackend(g graph.Store) search.Backend {
if s, ok := g.(graph.SymbolSearcher); ok {
return search.NewSymbolSearcherBackend(s)
}
return search.NewAuto()
}
// isSymbolSearcherBackend reports whether the swappable's currently
// active backend is the SymbolSearcher adapter. Used to suppress
// the Bleve auto-upgrade goroutine — if the active backend is
// already a native FTS, upgrading to Bleve would re-index the same
// corpus into a parallel in-process Bleve and silently swap it in,
// defeating the FTS path and pinning the ~100MB heap the FTS
// integration was meant to release.
func isSymbolSearcherBackend(b search.Backend) bool {
if b == nil {
return false
}
if sw, ok := b.(*search.Swappable); ok {
b = sw.Inner()
}
_, ok := b.(*search.SymbolSearcherBackend)
return ok
}
// lessEdgeKey orders edges by their logical key (from, to, kind,
// file_path, line) — the same tuple the sqlite edges table's
// UNIQUE(from_id, …) index is built on. Draining edges in this order on
// the cold bulk load keeps that index's inserts local instead of random,
// reducing B-tree page splits.
func lessEdgeKey(a, b *graph.Edge) bool {
if a.From != b.From {
return a.From < b.From
}
if a.To != b.To {
return a.To < b.To
}
if a.Kind != b.Kind {
return a.Kind < b.Kind
}
if a.FilePath != b.FilePath {
return a.FilePath < b.FilePath
}
return a.Line < b.Line
}
// ftsTokensFor produces the pre-tokenised text the backend FTS path
// indexes. Mirrors searchIndexFields' field selection but joins
// every field through search.Tokenize (camelCase / snake_case /
// path-segment splitter) so the resulting token list matches the
// in-process BM25 corpus contract — the same query produces the
// same recall against either backend. Joined with spaces so the
// downstream COPY FROM sees a single STRING column value.
func ftsTokensFor(n *graph.Node, projectName string) string {
fields := searchIndexFields(n, projectName)
if n.QualName != "" {
// QualName carries the dotted form (`pkg.Sub.Type.Method`)
// that adds qualifier-hop recall ("auth" matching
// "auth.ValidateToken"). searchIndexFields omits it for
// the legacy BM25 path (which folds qual into the
// name-token bag separately), so we add it explicitly here.
fields = append(fields, n.QualName)
}
tokens := make([]string, 0, 16)
for _, f := range fields {
if f == "" {
continue
}
tokens = append(tokens, search.Tokenize(f)...)
}
if len(tokens) == 0 {
return ""
}
return strings.Join(tokens, " ")
}
// shouldIndexForSearch reports whether a node should be added to the
// text search index (BM25/Bleve). File and Import nodes are never
// searchable symbols. Beyond that, config.SkipSearch filters out
// (language, kind) pairs that would only add noise — JSON/YAML/TOML
// keys, CSS tokens, Terraform blocks, shell/build variables. All three
// text-index call sites (buildSearchIndex bulk loop, indexFile
// incremental add, upgradeSearchToBleve repopulate) must go through
// this predicate so they can't drift.
func (idx *Indexer) shouldIndexForSearch(n *graph.Node) bool {
// Cross-daemon proxy-edge nodes stand in for remote symbols; they
// are never surfaced in local name search. Inert until
// edge-minting is enabled.
if graph.IsProxyNode(n) {
return false
}
if n.Kind == graph.KindFile || n.Kind == graph.KindImport {
return false
}
// KindLocal nodes are intra-function bindings emitted to satisfy
// rel-table FK constraints on the dataflow edges that target
// locals. They have a real Name (the variable identifier) but
// surfacing them in BM25 would flood every search for common
// names like `err`, `data`, `n`, `i`. Excluded unconditionally.
if n.Kind == graph.KindLocal {
return false
}
// KindBuiltin nodes are language intrinsics (append / len /
// string / int / ...). Surfacing them in name search would
// drown every other hit on common identifiers — agents already
// know `string` / `append`. They remain queryable by kind and
// by ID for the analytics passes that care.
if n.Kind == graph.KindBuiltin {
return false
}
// CONTENT (data_class="content") section nodes live in the dedicated
// content index (content_fts), never the symbol search — keeping the
// symbol corpus code-only and bounded. Markdown prose (KindDoc without
// data_class=content) is unaffected and still honours IndexProse below.
if isContentNode(n) {
return false
}
// Prose-section nodes are searchable only when prose indexing is
// enabled (search.index_prose); the rest of the graph is
// unaffected by the toggle.
if n.Kind == graph.KindDoc && !idx.config.IndexProse {
return false
}
if config.ShouldSkipSearch(idx.config.SkipSearch, n.Language, string(n.Kind)) {
return false
}
return true
}
// upgradeSearchToBleve constructs a Bleve backend from the current graph
// and atomically swaps it in. Designed to run in a background goroutine
// triggered by IndexCtx after the initial index completes. Does nothing
// if Bleve construction fails (caller already hit AutoThreshold but the
// in-memory backend keeps serving correctly, just with worse memory
// characteristics).
// bleveUpgradeEntry is one row of the snapshot the upgrade goroutine
// works from. Snapshotting (id, name, file, signature) up front in
// the foreground — before the goroutine starts reading them — keeps
// the goroutine race-free against subsequent Index calls' Meta-writing
// passes (reach.BuildIndex, ResolveTemporalCalls, ...).
type bleveUpgradeEntry struct {
id string
// fields is the BM25 text payload for the node, as produced by
// searchIndexFields: name + file + signature for a code symbol,
// name + file + section body for a KindDoc prose section.
fields []string
}
// snapshotBleveEntries captures every node currently eligible for the
// search index plus its `signature` Meta string. Called synchronously
// from IndexCtx after every Node.Meta mutating pass has returned, so
// the read of n.Meta happens with no concurrent writer.
func (idx *Indexer) snapshotBleveEntries() []bleveUpgradeEntry {
nodes := idx.graph.AllNodes()
out := make([]bleveUpgradeEntry, 0, len(nodes))
for _, n := range nodes {
if !idx.shouldIndexForSearch(n) {
continue
}
out = append(out, bleveUpgradeEntry{id: n.ID, fields: searchIndexFields(n, idx.projectName)})
}
return out
}
func (idx *Indexer) upgradeSearchToBleve(snapshot []bleveUpgradeEntry) {
// Defensive early-return: if the active text backend is already
// Bleve, there is nothing to upgrade. IndexCtx's sync.Once guard
// prevents re-entry from the auto-upgrade path, but direct
// callers (tests, manual invocation from tooling) could still
// hit this function twice; a second run would pointlessly
// rebuild a full Bleve index and Swap it over an identical one.
inner := idx.swappable().Inner()
if _, ok := inner.(*search.BleveBackend); ok {
return
}
if hyb, ok := inner.(*search.HybridBackend); ok {
if _, ok := hyb.TextBackend().(*search.BleveBackend); ok {
return
}
}
// Opt-in disk backend. Scorch stores the inverted index on disk
// (~10-20× less heap than upsidedown+gtreap) at the cost of file
// I/O during build. Users point GORTEX_BLEVE_DISK_DIR at a
// writable path; we manage the file lifecycle inside it.
diskDir := os.Getenv("GORTEX_BLEVE_DISK_DIR")
var (
blv *search.BleveBackend
err error
)
if diskDir != "" {
blv, err = search.NewBleveDisk(diskDir)
if err != nil {
idx.logger.Warn("search: bleve disk construction failed, falling back to in-memory",
zap.String("dir", diskDir), zap.Error(err))
}
}
if blv == nil {
blv, err = search.NewBleve()
if err != nil {
idx.logger.Warn("search: bleve construction failed, staying on in-memory",
zap.Error(err))
return
}
}
// Use the foreground snapshot the spawner captured rather than
// re-walking idx.graph here: the goroutine outlives the spawning
// IndexCtx call, and subsequent Index calls' Meta-writing passes
// (reach.BuildIndex, ResolveTemporalCalls, ...) mutate Node.Meta
// on the same Node objects. Reading sig from a live n.Meta here
// would race with those writes.
for _, e := range snapshot {
blv.Add(e.id, e.fields...)
}
// Preserve the vector index if one is wired up. The previous inner
// is normally a HybridBackend wrapping text + vector + embedder;
// swapping in raw Bleve would let Swap's old.Close() run on the
// old Hybrid, which closes only its text side (hybrid.Close) but
// leaves the resulting inner — raw *BleveBackend — unwrapped, so
// every downstream hybrid/semantic query silently degrades to
// BM25 until the daemon restarts. Rewrap the fresh Bleve in a new
// Hybrid carrying the old vector + embedder. The vector backend
// itself is never closed by Hybrid.Close, so it stays alive even
// after the old Hybrid is torn down by Swap.
sw := idx.swappable()
var replacement search.Backend = blv
if oldHybrid, ok := sw.Inner().(*search.HybridBackend); ok {
replacement = search.NewHybrid(blv, oldHybrid.VectorIndex(), oldHybrid.Embedder())
}
sw.Swap(replacement)
mode := "memory"
if blv.DiskPath() != "" {
mode = "disk"
}
idx.logger.Info("search: upgraded to Bleve backend (background)",
zap.Int("symbols", blv.Count()),
zap.String("mode", mode),
zap.String("disk_path", blv.DiskPath()))
}
// Graph returns the underlying graph.
func (idx *Indexer) Graph() graph.Store { return idx.graph }
// Search returns the search backend.
func (idx *Indexer) Search() search.Backend { return idx.search }
// Registry returns the parser registry shared across this indexer.
// Exposed for the editor-overlay middleware: the overlay layer-build
// pass parses each pushed buffer through the same per-language
// extractor the indexer uses, ensuring overlay-derived nodes match
// base-derived nodes byte-for-byte for the same input.
func (idx *Indexer) Registry() *parser.Registry { return idx.registry }
// ContractRegistry returns the contract registry populated during indexing.
func (idx *Indexer) ContractRegistry() *contracts.Registry { return idx.contractRegistry }
// SetContractRegistry installs reg as the indexer's contract registry.
// Used by the daemon warmup path to rehydrate the registry from a
// snapshot when IncrementalReindex skipped extraction (no stale files
// → extractContracts never ran → idx.contractRegistry stays nil after
// reconcile, which used to leave multi-repo `contracts` queries silently
// empty). Callers should only install when ContractRegistry() is nil;
// installing over a freshly-extracted registry would roll state backward.
func (idx *Indexer) SetContractRegistry(reg *contracts.Registry) {
idx.contractRegistry = reg
}
// SetTrackedRepoModules sets the map of tracked repo names to Go module paths.
// This enables the GoModExtractor to detect cross-repo dependencies.
func (idx *Indexer) SetTrackedRepoModules(m map[string]string) { idx.trackedRepoModules = m }
// SetDeferResolve toggles whether IndexCtx defers the cross-cutting passes
// to a later RunDeferredPasses call. See the deferResolve field comment.
func (idx *Indexer) SetDeferResolve(v bool) { idx.deferResolve = v }
// SetSkipResolveInDeferred toggles whether RunDeferredPasses calls
// idx.resolver.ResolveAll. The MultiIndexer batch driver sets this so
// the per-repo resolver pass — which walks the entire shared graph —
// runs exactly once globally instead of R times. See the
// skipResolveInDeferred field comment.
func (idx *Indexer) SetSkipResolveInDeferred(v bool) { idx.skipResolveInDeferred = v }
// SetDeferGlobalPasses toggles whether the graph-wide derivation passes
// (InferImplements, InferOverrides, markTestSymbolsAndEmitEdges) run
// inline at the end of IndexCtx / IncrementalReindex. Set true when the
// caller drives a batch (e.g. daemon warmup) and will invoke
// RunGlobalGraphPasses once at the end. See the deferGlobalPasses field
// comment.
func (idx *Indexer) SetDeferGlobalPasses(v bool) { idx.deferGlobalPasses = v }
// RunGlobalGraphPasses runs the graph-wide derivation passes once
// against the indexer's shared graph. Safe to call against a graph that
// already has these edges — InferImplements / InferOverrides skip
// existing parents, and graph.AddEdge dedupes by edgeKey so EdgeTests
// re-emission is a no-op. Logs counts for telemetry. Use when batching
// multiple per-repo TrackRepoCtx / IncrementalReindex calls under
// SetDeferGlobalPasses(true).
func (idx *Indexer) RunGlobalGraphPasses(ctx context.Context) {
if idx.graph == nil {
return
}
reporter := progress.FromContext(ctx)
if added := idx.resolver.InferImplements(); added > 0 {
idx.logger.Info("inferred implements (global)", zap.Int("added", added))
}
if added := idx.resolver.InferOverrides(); added > 0 {
idx.logger.Info("inferred overrides (global)", zap.Int("added", added))
}
marked, emitted := markTestSymbolsAndEmitEdges(idx.graph)
if marked > 0 || emitted > 0 {
idx.logger.Info("test edges emitted (global)",
zap.Int("test_symbols", marked),
zap.Int("edges", emitted),
)
}
if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 {
idx.logger.Info("capability edges emitted (global)",
zap.Int("reads_env", re),
zap.Int("executes_process", ep),
zap.Int("accesses_field", fa),
)
}
reporter.Report("clone detection pass (global)", 0, 0)
if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
idx.logger.Info("clone edges emitted (global)",
zap.Int("items", cs.Items),
zap.Int("clone_pairs", cs.Pairs),
zap.Int("edges", cs.Edges),
zap.Int("skipped_buckets", cs.SkippedBuckets),
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
zap.Int("diffused_pairs", cs.DiffusedPairs),
zap.Int("diffused_edges", cs.DiffusedEdges),
)
}
// Seed the incremental clone index from the freshly-baselined
// signatures + sidecar so steady-state single-file edits after this
// batch go incremental instead of re-running the whole-graph pass.
if idx.cloneIndex != nil {
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
}
// Framework dynamic-dispatch synthesis (gRPC stubs, Temporal
// workflow→activity, in-process / native event channels, native
// bridges). Runs after InferImplements/InferOverrides (the
// interface-satisfaction signals several synthesizers depend on) and
// before DetectCrossRepoEdges so a cross-repo synthesized call gets
// its parallel cross_repo_calls edge.
reporter.Report("framework dispatch synthesis (global)", 0, 0)
if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 {
idx.logger.Info("framework dispatch calls synthesized (global)",
zap.Int("edges", rep.Total),
zap.Any("per_synthesizer", rep.Per),
)
}
// External-call placeholder synthesis (opt-in). Runs after the
// resolver and the gRPC/Temporal stub passes so every edge that
// could land on a real node already has; the leftover external
// terminals are then materialised into synthetic call-chain nodes.
reporter.Report("external-call synthesis (global)", 0, 0)
if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 {
idx.logger.Info("external-call placeholders synthesized (global)",
zap.Int("edges", extCalls),
)
}
// Speculative dynamic-dispatch synthesis (opt-in, default off). Mints
// best-guess hidden-by-default call edges for blind-spot shapes.
if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 {
idx.logger.Info("speculative dispatch edges synthesized (global)",
zap.Int("edges", spec),
)
}
// Content -> code "why" links. Runs before DetectCrossRepoEdges so a
// chunk that motivates a symbol in another repo gets its parallel
// cross_repo_motivates edge minted by the cross-repo pass below.
reporter.Report("content links (global)", 0, 0)
idx.linkContentToCode()
// Cross-repo edge layer. Runs after InferImplements / InferOverrides
// so cross-repo implements / extends edges pick up their parallel
// cross_repo_* edges. No-op on single-repo graphs (no RepoPrefix).
reporter.Report("cross-repo edges (global)", 0, 0)
if crossRepoEdges := resolver.DetectCrossRepoEdges(idx.graph); crossRepoEdges > 0 {
idx.logger.Info("cross-repo edges emitted (global)",
zap.Int("edges", crossRepoEdges),
)
}
// Reachability index — used to be precomputed here for every
// impact seed. The eager pass was retired because the breakeven
// math doesn't work: on a 200 k-seed graph (k8s) the build took
// ~2000 s of cold-index wall time to save ~10 ms per
// AnalyzeImpact call, requiring ~200 k queries to pay off — well
// beyond any realistic agent session. Lookups are now
// compute-on-first-use; we just invalidate the cache so any
// surviving stamps from a previous build don't shadow the fresh
// graph state.
reach.InvalidateIndex()
}
// cloneThreshold returns the configured Jaccard similarity cutoff for
// clone detection (0 = use the clones package default).
func (idx *Indexer) cloneThreshold() float64 {
return idx.config.Coverage.ClonesThreshold()
}
// RunDeferredPasses runs the per-repo cross-cutting passes that IndexCtx
// skipped in deferred mode: per-repo ResolveAll, semantic enrichment, and
// contract extraction + commit. Safe to call only after IndexCtx has
// populated the graph for this repo. Idempotent — second calls are a no-op
// because the pending registry is cleared at the end.
//
// The graph-wide derivation passes (InferImplements, InferOverrides,
// markTestSymbolsAndEmitEdges) intentionally do NOT run here. They walk
// the entire shared graph, so the multi-repo orchestrator must invoke
// MultiIndexer.RunGlobalGraphPasses exactly once after every repo has
// finished its deferred per-repo work.
func (idx *Indexer) RunDeferredPasses(ctx context.Context) {
if idx.pendingContractReg == nil {
return
}
reporter := progress.FromContext(ctx)
tphase := time.Now()
var dGoMod, dResolve, dEnrich, dContract time.Duration
idx.runDeferredGoMod()
dGoMod = time.Since(tphase)
tphase = time.Now()
// Per-repo resolver.ResolveAll walks the entire shared graph; with R
// repos and E edges that's O(R · E). The MultiIndexer batch driver
// sets skipResolveInDeferred so this runs exactly once globally
// (resolver.New(mi.graph).ResolveAll after every per-repo deferred
// pass has committed contracts). Direct (non-batch) callers leave
// the flag false and pay the standard single-repo cost.
if !idx.skipResolveInDeferred {
reporter.Report("resolving references", 0, 0)
idx.populateCppIncludeDirs(false)
idx.resolver.ResolveAll()
}
dResolve = time.Since(tphase)
tphase = time.Now()
reporter.Report("semantic enrichment", 0, 0)
idx.runDeferredEnrich()
dEnrich = time.Since(tphase)
tphase = time.Now()
reporter.Report("extracting contracts", 0, 0)
idx.runDeferredContracts()
dContract = time.Since(tphase)
idx.logger.Info("DEFERRED-TIMING per-repo",
zap.String("repo", idx.repoPrefix),
zap.Duration("gomod", dGoMod),
zap.Duration("resolve", dResolve),
zap.Duration("enrich", dEnrich),
zap.Duration("contract_commit", dContract))
}
// runDeferredGoMod materialises dep::<module> contract nodes from go.mod
// BEFORE ResolveAll so the resolver's import bridge can re-target Go imports
// of declared modules to their dep contract node instead of an external::
// stub. Split out of RunDeferredPasses so the batch driver can run it
// serially across repos ahead of the parallel enrichment phase.
func (idx *Indexer) runDeferredGoMod() {
if idx.pendingContractReg == nil {
return
}
idx.extractGoModContracts(idx.pendingContractReg)
}
// runDeferredEnrich runs semantic enrichment for this repo. Safe to run
// concurrently across repos: the manager fetches a per-repo LSP provider
// instance (keyed by the repo's workspace), the go-types stash is keyed by
// repo root, tstypes is stateless, and every provider serialises its graph
// mutations on the backend resolve mutex.
func (idx *Indexer) runDeferredEnrich() {
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
return
}
// Gate the pass to repos that actually changed. The daemon warmup collects
// every indexer and enriches them all, but an unchanged repo's persisted
// graph already carries its enrichment edges — re-running gopls hover for
// it confirms nothing over many minutes. GORTEX_WARMUP_FORCE_ENRICH=1
// bypasses the gate for a full re-enrich.
forced := os.Getenv("GORTEX_WARMUP_FORCE_ENRICH") == "1"
if !idx.pendingEnrich.Load() {
if !forced {
idx.logger.Info("deferred enrichment skipped",
zap.String("repo", idx.repoPrefix),
zap.String("reason", "unchanged"))
return
}
idx.logger.Info("deferred enrichment forced despite no pending changes",
zap.String("repo", idx.repoPrefix))
}
// Key by the repo prefix so a repo-scoped provider can scope file
// selection to this repo (empty in single-repo mode).
roots := map[string]string{idx.repoPrefix: idx.rootPath}
// Compute the repo's git freshness ONCE and thread it in so the manager's
// per-provider skip gate and the completion-marker write agree on the
// identical (sha, dirty): a provider whose persisted marker still matches
// HEAD on a clean tree is skipped instead of re-running its hover pass.
sha, dirty := repoHeadAndDirty(idx.rootPath)
// A re-parse this run evicted the persisted hover edges of the re-parsed
// files, so force the pass past the completion-marker gate — an unchanged
// clean HEAD would otherwise skip re-enrichment and leave those files' LSP
// edges durably gone. fullReindexed covers a whole-repo re-parse (IndexCtx);
// reparsedThisRun covers a scoped incremental re-parse that dropped only the
// changed files' edges. Either forces the pass; the provider re-hovers only
// the freshly-unstamped nodes, so a scoped force stays bounded to those files.
opts := semantic.EnrichOptions{
RepoState: map[string]semantic.RepoEnrichState{
idx.repoPrefix: {SHA: sha, Dirty: dirty, Force: idx.fullReindexed.Load() || idx.reparsedThisRun.Load()},
},
}
results, partialRepos, err := idx.semanticMgr.EnrichAll(idx.graph, roots, opts)
if err != nil {
idx.logger.Warn("semantic enrichment failed", zap.Error(err))
return
}
for _, r := range results {
idx.logger.Info("semantic enrichment result",
zap.String("provider", r.Provider),
zap.String("language", r.Language),
zap.Int("confirmed", r.EdgesConfirmed),
zap.Int("added", r.EdgesAdded),
zap.Int("refuted", r.EdgesRefuted),
zap.Float64("coverage", r.CoveragePercent),
)
}
// Clear the pending marker only when every provider that ran for this repo
// finished non-partial. A partial / abandoned / failed pass leaves it set
// so a later deferred pass (or the next restart, once the repo changes
// again) retries the enrichment rather than trusting an incomplete graph.
if !partialRepos[idx.repoPrefix] {
idx.pendingEnrich.Store(false)
// Persist a whole-repo completion marker at this HEAD so the next warm
// restart can tell, with one lookup, that this repo's enrichment finished
// and MaybeSeedPendingEnrich need not resume it. A partial / abandoned
// pass takes the other branch and writes no marker, so the absent marker
// re-arms the pass on the next start. No-op on a dirty tree / empty sha.
idx.semanticMgr.RecordRepoEnrichmentComplete(idx.graph, idx.repoPrefix, sha, dirty)
}
}
// MaybeSeedPendingEnrich re-arms the deferred-enrichment gate for a repo whose
// persisted enrichment is known-incomplete at the current clean HEAD, so a warm
// restart resumes a semantic pass a prior process left partial or abandoned.
//
// pendingEnrich otherwise reflects only re-indexing work performed THIS run, so
// an unchanged repo whose first enrichment was cut short by the per-repo
// deadline would short-circuit runDeferredEnrich on every subsequent restart —
// its whole-repo completion marker is absent (a partial pass writes none) yet no
// file changed to raise the flag. The daemon warmup calls this after the parse
// phase, before draining the deferred passes.
//
// Returns whether this repo will enrich (already pending, or newly seeded). A
// no-op — false — for a repo without semantic providers, on a non-git tree (no
// reliable freshness signal), on a backend that does not persist enrichment
// state (it re-indexes from scratch each restart anyway), when the completion
// marker already records the current HEAD, or on a dirty tree (the marker is
// never written or trusted against uncommitted content, and resuming every
// restart while the tree stays dirty would defeat the warm-restart fast path).
func (idx *Indexer) MaybeSeedPendingEnrich() bool {
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
return false
}
if idx.pendingEnrich.Load() {
// This run's re-indexing work already armed the gate.
return true
}
// Cheap probe first: only the sha is needed to tell a repo whose marker is
// already current (the common warm-restart case) from one that must resume,
// so the slower git status shell-out is deferred to the resume path below.
sha := repoHead(idx.rootPath)
if sha == "" {
return false
}
current, persisted := idx.semanticMgr.RepoEnrichmentMarkerState(idx.graph, idx.repoPrefix, sha)
if !persisted || current {
return false
}
if _, dirty := repoHeadAndDirty(idx.rootPath); dirty {
return false
}
idx.logger.Info("deferred enrichment re-armed: persisted enrichment incomplete",
zap.String("repo", idx.repoPrefix),
zap.String("sha", sha))
idx.pendingEnrich.Store(true)
return true
}
// runDeferredContracts extracts and commits this repo's contract nodes and
// clears the pending registration. extractGoModContracts already ran via
// runDeferredGoMod. Mutates the shared graph and walks repo edges, so the
// batch driver runs it serially after the parallel enrichment phase. The
// go-types LookupTypeAtLine it relies on reads the per-repo stash, so it is
// correct even though every repo's enrichment ran before any contracts.
func (idx *Indexer) runDeferredContracts() {
if idx.pendingContractReg == nil {
return
}
idx.extractExternalModules()
idx.extractDIContracts(idx.pendingContractReg)
idx.commitContracts(idx.pendingContractReg)
idx.pendingContractReg = nil
}
// RootPath returns the root path used for relative path computation.
func (idx *Indexer) RootPath() string { return idx.rootPath }
// storeRootPath records the repository root, skipping a redundant
// self-assignment. The watcher goroutine reads idx.rootPath without a
// lock, so re-storing the identical value from a concurrent reindex or
// mtime-census pass would be a data race for no observable change.
func (idx *Indexer) storeRootPath(absRoot string) {
if idx.rootPath != absRoot {
idx.rootPath = absRoot
}
}
// populateCppIncludeDirs reconstructs each C/C++ source file's include search
// path from compile_commands.json and hands it to the resolver, so a quoted
// include binds against the real `-I` dir set (deterministic collision-breaking)
// before the suffix-unique fallback. forceReload drops the cache first, so an
// incremental reindex picks up an edited compile_commands.json without a
// daemon restart. Keys/dirs are prefixed in multi-repo mode to match file IDs.
func (idx *Indexer) populateCppIncludeDirs(forceReload bool) {
if idx.resolver == nil || idx.rootPath == "" {
return
}
if forceReload {
clearCppIncludeDirCache(idx.rootPath)
}
prefix := ""
if idx.repoPrefix != "" {
prefix = idx.repoPrefix + "/"
}
prefixDirs := func(dirs []string) []string {
if prefix == "" || len(dirs) == 0 {
return dirs
}
pd := make([]string, len(dirs))
for i, d := range dirs {
pd[i] = prefix + d
}
return pd
}
tus := loadCompileCommands(idx.rootPath)
if len(tus) == 0 {
// No compile DB: fall back to the conventional include-root heuristic
// so the ordered probe still runs for repos without a compile DB.
idx.resolver.SetCppIncludeDirs(nil)
idx.resolver.SetCppFallbackIncludeDirs(prefixDirs(heuristicIncludeDirs(idx.rootPath)))
return
}
perFile := make(map[string][]string, len(tus))
for f, tu := range tus {
perFile[prefix+f] = prefixDirs(tu.includeDirs)
}
idx.resolver.SetCppIncludeDirs(perFile)
idx.resolver.SetCppFallbackIncludeDirs(nil)
}
// ResolveFilePath maps a graph file path (repo-relative in single-repo mode)
// to an absolute filesystem path. Returns "" when no root is set so callers
// can refuse rather than open against the daemon process CWD. Implements
// analysis.SourceReader.
func (idx *Indexer) ResolveFilePath(graphPath string) string {
if graphPath == "" {
return ""
}
if filepath.IsAbs(graphPath) {
return filepath.Clean(graphPath)
}
if idx.rootPath == "" {
return ""
}
// In multi-repo mode the lone Indexer is wrapped by MultiIndexer
// (which exposes RepoRoot/ResolveFilePath); single-repo callers
// hit this path directly.
rel := graphPath
if idx.repoPrefix != "" {
rel = strings.TrimPrefix(rel, idx.repoPrefix+"/")
}
return filepath.Clean(filepath.Join(idx.rootPath, rel))
}
// relKey reduces an absolute path under the repo root to the canonical
// repo-relative key the graph and the mtime map are indexed by: forward
// slashes, and Unicode NFC.
//
// The NFC fold is load-bearing. A file with a non-ASCII name is handed
// to the indexer in different byte forms depending on the source — the
// filesystem walk (filepath.WalkDir) yields decomposed NFD on macOS,
// while the git watcher decodes `git diff` output that git stored as
// precomposed NFC. Keying the bulk walk under one form and an
// incremental patch under the other would split a single file across
// two graph keys: the watcher's evict would miss the walk's node,
// IncrementalReindex would see the file as both deleted and freshly
// created, and the daemon would carry a stale duplicate. Folding every
// key through one form here removes that whole class of mismatch.
//
// On a path that is not under rootPath (filepath.Rel fails) the input
// is returned slash-normalised and NFC-folded so the result is still a
// stable key, just not repo-relative.
func (idx *Indexer) relKey(absPath string) string {
rel, err := filepath.Rel(idx.rootPath, absPath)
if err != nil {
return pathkey.Normalize(filepath.ToSlash(absPath))
}
return pathkey.Normalize(filepath.ToSlash(rel))
}
// RelKey exposes relKey to in-package collaborators (the watcher) that
// hold an absolute filesystem path and need the canonical repo-relative
// graph key for it — e.g. to look up a file's nodes before and after a
// re-index. Going through one helper keeps the watcher's key in lockstep
// with the keys IndexFile / EvictFile write.
func (idx *Indexer) RelKey(absPath string) string { return idx.relKey(absPath) }
// SetRepoPrefix sets the repository prefix for multi-repo mode.
// When non-empty, all node IDs and file paths are prefixed with "<repoPrefix>/".
func (idx *Indexer) SetRepoPrefix(prefix string) { idx.repoPrefix = prefix }
// RepoPrefix returns the current repository prefix.
func (idx *Indexer) RepoPrefix() string { return idx.repoPrefix }
// SetWorkspaceID sets the workspace slug stamped onto nodes emitted
// by this indexer. Empty means "no workspace declared" — the
// applyRepoPrefix path will fall back to RepoPrefix so multi-repo
// configs without `.gortex.yaml::workspace:` keep working.
func (idx *Indexer) SetWorkspaceID(id string) { idx.workspaceID = id }
// WorkspaceID returns the workspace slug this indexer stamps on nodes.
func (idx *Indexer) WorkspaceID() string { return idx.workspaceID }
// SetProjectID sets the project slug stamped onto nodes emitted by
// this indexer. Single-project repos pass their repo name (the
// MultiIndexer default); monorepos compute a per-file slug from the
// `projects[]` mapping (follow-up work).
func (idx *Indexer) SetProjectID(id string) { idx.projectID = id }
// ProjectID returns the project slug this indexer stamps on nodes.
func (idx *Indexer) ProjectID() string { return idx.projectID }
// SetEmbedder sets the embedding provider for semantic search.
// When set, buildSearchIndex will create a HybridBackend with vector search.
func (idx *Indexer) SetEmbedder(p embedding.Provider) { idx.embedder = p }
// SetSkipVectorBuild toggles the embedding pass in buildSearchIndex.
// When true, buildSearchIndex builds only the text index — used by the
// daemon warmup path when a snapshot already carries the workspace
// vector index, so the graph is not needlessly re-embedded. When false
// (the default) an indexer with an embedder set always builds vectors.
func (idx *Indexer) SetSkipVectorBuild(skip bool) { idx.skipVectorBuild = skip }
// SetEmbeddingChunkOptions tunes the AST sub-chunking applied to large
// symbols before embedding (threshold and window line counts). The
// zero value leaves the chunker on its built-in defaults.
func (idx *Indexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) {
idx.embedChunkOpts = opts
}
// SetEmbeddingMaxSymbols overrides the cap on how many texts the vector
// index will hold before buildSearchIndex skips the embed pass. Zero
// keeps the built-in default.
func (idx *Indexer) SetEmbeddingMaxSymbols(n int) { idx.embedMaxSymbols = n }
// SetEmbeddingAPIConcurrency overrides how many embedding requests run
// in parallel against an API-backed embedder. Zero keeps the built-in
// default. Has no effect on in-process embedders.
func (idx *Indexer) SetEmbeddingAPIConcurrency(n int) { idx.embedAPIConcurrency = n }
// LastVectorBuildError returns why the most recent index build produced no
// vector index (chunk-embed failure, all vectors invalid, or the symbol-count
// guard), or nil when a vector index was built or the embedder was unset. Read
// it after an index build completes; it is not safe to call concurrently with
// one.
func (idx *Indexer) LastVectorBuildError() error { return idx.lastVectorBuildErr }
// SetSemanticManager sets the semantic enrichment manager.
// When set, the indexer runs semantic enrichment after resolution.
func (idx *Indexer) SetSemanticManager(m *semantic.Manager) { idx.semanticMgr = m }
// SemanticManager returns the semantic enrichment manager.
func (idx *Indexer) SemanticManager() *semantic.Manager { return idx.semanticMgr }
// SetResolverLSPHelper installs a resolve-time LSP helper on the
// underlying Resolver. The helper is consulted from inside
// resolveEdge for languages whose extensions the helper claims
// (TS/JS/JSX/TSX today via tsserver); see internal/resolver/
// lsp_helper.go for the contract.
//
// Pass nil to detach. Must be called before ResolveAll / ResolveFile;
// the resolver caches no LSP state across passes, so mid-pass swaps
// are racy and not supported.
func (idx *Indexer) SetResolverLSPHelper(h resolver.LSPHelper) {
if idx.resolver != nil {
idx.resolver.SetLSPHelper(h)
}
idx.resolverLSPHelper = h
}
// ResolverLSPHelper returns the currently installed resolver-time LSP
// helper, or nil. Exported so MultiIndexer can mirror the helper onto
// the global post-pass resolver in RunDeferredPassesAll.
func (idx *Indexer) ResolverLSPHelper() resolver.LSPHelper { return idx.resolverLSPHelper }
// ExportVectorIndex returns the serialized vector index bytes, dims, and count.
// Returns nil, 0, 0 if no vector index is active.
func (idx *Indexer) ExportVectorIndex() ([]byte, int, int) {
hybrid, ok := idx.swappable().Inner().(*search.HybridBackend)
if !ok {
return nil, 0, 0
}
vec := hybrid.VectorIndex()
if vec == nil || vec.Count() == 0 {
return nil, 0, 0
}
var buf bytes.Buffer
if err := vec.Save(&buf); err != nil {
idx.logger.Warn("failed to export vector index", zap.Error(err))
return nil, 0, 0
}
return buf.Bytes(), vec.Dims(), vec.Count()
}
// ImportVectorIndex restores a vector index from serialized data and wraps
// the current text search backend into a HybridBackend.
func (idx *Indexer) ImportVectorIndex(data []byte, dims, count int) error {
if len(data) == 0 || idx.embedder == nil {
return nil
}
// Validate dimensions match the current embedder to avoid mismatches
// when switching providers (e.g., GloVe 50d → ONNX 384d).
embedderDims := idx.embedder.Dimensions()
if embedderDims > 0 && embedderDims != dims {
idx.logger.Info("vector index dims mismatch, will re-embed",
zap.Int("cached_dims", dims), zap.Int("embedder_dims", embedderDims))
return nil // skip import, buildSearchIndex will re-embed
}
vec := search.NewVector(dims)
if err := vec.LoadFrom(bytes.NewReader(data)); err != nil {
return fmt.Errorf("import vector index: %w", err)
}
vec.SetCount(count)
sw := idx.swappable()
sw.Swap(search.NewHybrid(sw.Inner(), vec, idx.embedder))
idx.logger.Info("restored vector index from cache",
zap.Int("vectors", count), zap.Int("dims", dims))
return nil
}
// prefixPath prepends the repoPrefix to a relative path when in multi-repo mode.
// Returns the path unchanged when repoPrefix is empty.
func (idx *Indexer) prefixPath(relPath string) string {
if idx.repoPrefix == "" {
return relPath
}
return idx.repoPrefix + "/" + relPath
}
// graphFilePaths maps reindex file paths (absolute or root-relative, as
// passed to IndexFile) to the canonical graph file-path form
// (prefixPath(relKey)) that GetFileNodes is keyed by — so file-scoped
// passes can look up the just-reindexed files' nodes.
func (idx *Indexer) graphFilePaths(files []string) []string {
out := make([]string, 0, len(files))
for _, f := range files {
abs := f
if !filepath.IsAbs(abs) && idx.rootPath != "" {
abs = filepath.Join(idx.rootPath, f)
}
out = append(out, idx.prefixPath(idx.relKey(abs)))
}
return out
}
// applyRepoPrefix transforms nodes and edges produced by an extractor to include
// the repo prefix in IDs and file paths. Sets Node.RepoPrefix on all nodes.
// This is a no-op when repoPrefix is empty (single-repo mode).
//
// Edge targets beginning with "unresolved::" are a sentinel meaning "the
// resolver will replace this with a real node ID after all files are
// indexed." Prefixing them turns "unresolved::fetchUsers" into
// "web/unresolved::fetchUsers", which the resolver's HasPrefix check on
// "unresolved::" misses — leaving every call edge permanently unresolved
// in multi-repo mode and breaking get_callers / get_call_chain across all
// languages. Skip prefixing on unresolved targets; the resolver will land
// the edge on a real ID that already carries its own correct prefix
// (possibly cross-repo, which the resolver marks explicitly).
// todoTags returns the configured TODO marker set or the default
// (TODO/FIXME/HACK/XXX/NOTE). Reads from the IndexConfig the indexer
// already holds — IsCoverageEnabled gating happens at the call site.
func (idx *Indexer) todoTags() []string {
if tags := idx.config.Coverage.Todos.Tags; len(tags) > 0 {
return tags
}
return []string{"TODO", "FIXME", "HACK", "XXX", "NOTE"}
}
// todoMaxText returns the configured cap on stored TODO text or the
// 200-char default.
func (idx *Indexer) todoMaxText() int {
if n := idx.config.Coverage.Todos.MaxText; n > 0 {
return n
}
return 200
}
// loadCodeownersRules lazily parses the repo's CODEOWNERS file. The
// sync.Once guarantees one parse per indexer; applyCoverageDomains
// is then a pure rule-match per file. Errors silently produce an
// empty rule set — the ownership domain is implicitly gated on
// file presence rather than failing extraction when the file is
// missing or malformed.
func (idx *Indexer) loadCodeownersRules() []codeowners.Rule {
idx.codeownersOnce.Do(func() {
rules, _, ok := codeowners.LoadFromRepo(idx.rootPath)
if !ok {
return
}
idx.codeownersRules = rules
})
return idx.codeownersRules
}
// applyCoverageDomains runs the per-file coverage extractors
// (todos, licenses, ownership) and applies the post-extraction
// strip pass for domains the language extractor always emits but
// the user has gated off (function_shape). Appended/stripped
// nodes/edges flow through the same applyRepoPrefix / graph.AddNode
// pipeline as the language extractor's output. Called from both
// the bulk index worker pool (IndexCtx) and the incremental
// indexFile path.
//
// relPath is the unprefixed file path; lang is the detected
// language; src is the file bytes.
func (idx *Indexer) applyCoverageDomains(relPath, lang string, src []byte, result *parser.ExtractionResult) {
if idx.config.Coverage.IsEnabled("todos") {
findings := todos.Scan(src, idx.todoTags(), idx.todoMaxText())
todoNodes, todoEdges := todos.BuildGraphArtifacts(relPath, findings, lang)
result.Nodes = append(result.Nodes, todoNodes...)
result.Edges = append(result.Edges, todoEdges...)
}
if idx.config.Coverage.IsEnabled("licenses") {
if spdx := licenses.Scan(src); spdx != "" {
licNodes, licEdges := licenses.BuildGraphArtifacts(relPath, spdx, lang)
result.Nodes = append(result.Nodes, licNodes...)
result.Edges = append(result.Edges, licEdges...)
}
}
if idx.config.Coverage.IsEnabled("ownership") {
if rules := idx.loadCodeownersRules(); len(rules) > 0 {
if owners := codeowners.MatchFile(relPath, rules); len(owners) > 0 {
teamNodes, teamEdges := codeowners.BuildGraphArtifacts(relPath, owners, lang)
result.Nodes = append(result.Nodes, teamNodes...)
result.Edges = append(result.Edges, teamEdges...)
}
}
}
if idx.config.Coverage.IsEnabled("codegen") {
if marker := codegen.Scan(src); marker.Generated {
// Stamp the marker on the file node when the language
// extractor produced one. Generated files without a
// file-shaped result node still get the EdgeGeneratedBy
// edge so downstream walks pick them up.
for _, n := range result.Nodes {
if n.Kind == graph.KindFile && n.FilePath == relPath {
if n.Meta == nil {
n.Meta = map[string]any{}
}
codegen.MarkFileNode(n.Meta, marker)
break
}
}
result.Edges = append(result.Edges, codegen.BuildGraphArtifacts(relPath, marker)...)
}
// Annotation-driven codegen: Lombok / MapStruct / Kotlin
// compiler plugins generate members that never appear in
// source. Flag the annotated symbols so they stay visible.
if extra, st := codegen.MarkAnnotatedGenerated(result.Nodes, result.Edges); st.NodesMarked > 0 {
result.Edges = append(result.Edges, extra...)
// Materialize the actual Lombok accessor members (getX/setX/
// builder/log) so `obj.getName()` resolves to a real graph node.
if lnodes, ledges := codegen.MaterializeLombokAccessors(result.Nodes); len(lnodes) > 0 {
result.Nodes = append(result.Nodes, lnodes...)
result.Edges = append(result.Edges, ledges...)
}
}
}
// Framework entry points (Alembic migrations / Next.js pages /
// ASP.NET host files): symbols reachable only from a runtime.
// Stamped so dead-code analysis treats them as live roots.
entrypoints.Detect(relPath, lang, result.Nodes, result.Edges)
if !idx.config.Coverage.IsEnabled("function_shape") {
stripFunctionShape(result)
}
if !idx.config.Coverage.IsEnabled("type_shape") {
stripTypeShape(result)
}
if !idx.config.Coverage.IsEnabled("constants") {
revertConstantsToVariables(result)
}
if !idx.config.Coverage.IsEnabled("concurrency") {
stripConcurrencyEdges(result)
}
if idx.config.Coverage.IsEnabled("fixtures") {
applyFixtureClassification(relPath, lang, result)
}
if !idx.config.Coverage.IsEnabled("observability") {
stripObservabilityArtifacts(result)
}
if !idx.config.Coverage.IsEnabled("pubsub") {
stripPubsubArtifacts(result)
}
if !idx.config.Coverage.IsEnabled("flags") {
stripFlagArtifacts(result)
}
if !idx.config.Coverage.IsEnabled("configs") {
stripConfigArtifacts(result)
}
// SQL migration / DDL extraction is high-signal — CREATE TABLE in a
// dedicated migration file or a `gortex db schema` dump is
// unambiguous, unlike the noisy code-side string-literal SQL the
// `sql` domain gates. Run it always; the strip below removes only the
// gated code-side SQL, preserving migration-origin schema nodes.
applyMigrationExtraction(relPath, src, result)
if !idx.config.Coverage.IsEnabled("sql") {
stripSQLArtifacts(result)
}
if idx.config.Coverage.IsEnabled("clones") {
applyCloneSignatures(src, result)
}
}
// applyMigrationExtraction parses CREATE TABLE DDL out of SQL migration
// files (path under migrate/ or migrations/) and `gortex db schema` dumps
// (recognised by their generated-header marker, wherever they're saved).
// Each declared table becomes a KindTable node with its columns as
// KindColumn nodes (EdgeMemberOf column → table), alongside the synthetic
// KindMigration node for the file, with EdgeProvides edges from migration
// → table so reverse-walk queries answer "which migrations create this
// table". A generated dump carries the real dialect in its header;
// hand-written migrations stay "generic" since the .sql file alone doesn't
// tell us which dialect the application targets. All emitted nodes carry
// Meta["origin"]="migration" so they survive the sql-domain strip — this
// schema DDL is unambiguous and high-signal, unlike the code-side
// string-literal SQL the `sql` coverage gate exists to suppress.
func applyMigrationExtraction(relPath string, src []byte, result *parser.ExtractionResult) {
generated := gortexsql.IsGeneratedSchema(src)
if !gortexsql.IsMigrationPath(relPath) && !generated {
return
}
tables := gortexsql.ExtractCreateTablesWithColumns(string(src))
if len(tables) == 0 {
return
}
dialect := "generic"
if generated {
if d := gortexsql.GeneratedSchemaDialect(src); d != "" {
dialect = d
}
}
migrationID := gortexsql.MigrationNodeID(relPath)
result.Nodes = append(result.Nodes, &graph.Node{
ID: migrationID,
Kind: graph.KindMigration,
Name: filepath.Base(relPath),
FilePath: relPath,
Language: "sql",
Meta: map[string]any{
"dialect": dialect,
"origin": "migration",
},
})
for _, t := range tables {
tableID := gortexsql.TableNodeID(dialect, t.Schema, t.Table)
meta := map[string]any{
"table": t.Table,
"dialect": dialect,
"origin": "migration",
}
if t.Schema != "" {
meta["schema"] = t.Schema
}
result.Nodes = append(result.Nodes, &graph.Node{
ID: tableID,
Kind: graph.KindTable,
Name: t.Table,
FilePath: relPath,
Language: "sql",
Meta: meta,
})
result.Edges = append(result.Edges, &graph.Edge{
From: migrationID,
To: tableID,
Kind: graph.EdgeProvides,
FilePath: relPath,
Origin: graph.OriginASTResolved,
})
for _, c := range t.Columns {
colID := gortexsql.ColumnNodeID(dialect, t.Schema, t.Table, c.Name)
cmeta := map[string]any{
"table": t.Table,
"dialect": dialect,
"origin": "migration",
}
if t.Schema != "" {
cmeta["schema"] = t.Schema
}
if c.Type != "" {
cmeta["type"] = c.Type
}
result.Nodes = append(result.Nodes, &graph.Node{
ID: colID,
Kind: graph.KindColumn,
Name: c.Name,
FilePath: relPath,
Language: "sql",
Meta: cmeta,
})
result.Edges = append(result.Edges, &graph.Edge{
From: colID,
To: tableID,
Kind: graph.EdgeMemberOf,
FilePath: relPath,
Origin: graph.OriginASTResolved,
})
}
}
}
// stripSQLArtifacts drops KindTable + KindMigration nodes plus
// the matching EdgeQueries / EdgeProvides edges when the sql
// coverage domain is gated off. Mirrors the strip passes for
// flags / configs / observability — endpoint-aware so any
// leftover edges to stripped nodes are pruned. SQL extraction
// defaults off because string-literal pattern matching against
// db.Get / db.Query / db.Exec produces false positives when
// domain code shares method names (cache.Get, etc.).
func stripSQLArtifacts(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if (n.Kind == graph.KindTable || n.Kind == graph.KindMigration) && !isMigrationOriginNode(n) {
stripped[n.ID] = struct{}{}
continue
}
// SQL-context KindString registry nodes are the short-circuit
// input for the SQL extractor; gated off alongside the rest
// of the SQL domain so a disabled gate leaves no SQL-related
// residue in the graph.
if n.Kind == graph.KindString {
if ctx, _ := n.Meta["context"].(string); ctx == "sql" {
stripped[n.ID] = struct{}{}
continue
}
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if e.Kind == graph.EdgeQueries {
continue
}
if _, ok := stripped[e.From]; ok {
continue
}
if _, ok := stripped[e.To]; ok {
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// stripConfigArtifacts drops KindConfigKey nodes plus
// EdgeReadsConfig / EdgeWritesConfig edges when the configs
// coverage domain is gated off. Endpoint-aware so any leftover
// edges to stripped key nodes are pruned.
//
// Infrastructure-origin config keys (Meta["origin"] in {"k8s",
// "dockerfile"}) are preserved because they are emitted by the K8s
// manifest, Kustomize, and Dockerfile extractors, which have no
// dedicated coverage flag and always run. Stripping them would
// also strip the EdgeUsesEnv edges those extractors produce (which
// target the same node IDs), defeating the cross-ref between
// container env declarations and code-side `os.Getenv` reads.
func stripConfigArtifacts(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if n.Kind == graph.KindConfigKey && !isInfraOriginConfigKey(n) && !isDocFrontmatterConfigKey(n) {
stripped[n.ID] = struct{}{}
continue
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if e.Kind == graph.EdgeReadsConfig || e.Kind == graph.EdgeWritesConfig {
continue
}
if _, ok := stripped[e.To]; ok {
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// isInfraOriginConfigKey reports whether a KindConfigKey node was
// emitted by the K8s / Kustomize / Dockerfile extractors. These
// nodes carry Meta["origin"] = "k8s" or "dockerfile" by convention.
// The code-side extractors (Go os.Getenv, Python os.environ, viper,
// struct-tag, …) leave Meta["origin"] empty.
// isMigrationOriginNode reports whether a node was emitted by migration /
// live-DB DDL extraction (Meta["origin"]=="migration"). These schema nodes
// are high-signal and survive the sql-domain strip — the gate only removes
// the noisy code-side string-literal SQL.
func isMigrationOriginNode(n *graph.Node) bool {
if n == nil || n.Meta == nil {
return false
}
o, _ := n.Meta["origin"].(string)
return o == "migration"
}
func isInfraOriginConfigKey(n *graph.Node) bool {
if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil {
return false
}
origin, _ := n.Meta["origin"].(string)
return origin == "k8s" || origin == "dockerfile"
}
// isDocFrontmatterConfigKey reports whether a KindConfigKey node is a
// document's frontmatter metadata (Quarto .qmd, …) rather than code /
// infra configuration. These keys ride with the document / prose ingest,
// which is independent of the `configs` coverage domain, so they survive
// the strip the same way infra-origin keys do — keeping a .qmd's declared
// title / format / params searchable by default. Add new frontmatter
// sources to the switch as prose extractors gain frontmatter support.
func isDocFrontmatterConfigKey(n *graph.Node) bool {
if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil {
return false
}
switch src, _ := n.Meta["source"].(string); src {
case "quarto_frontmatter":
return true
}
return false
}
// stripFlagArtifacts drops KindFlag nodes and EdgeTogglesFlag
// edges when the flags coverage domain is gated off. Mirrors the
// observability strip — endpoint-aware so any leftover edges that
// pointed to a removed flag node are also dropped.
func stripFlagArtifacts(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if n.Kind == graph.KindFlag {
stripped[n.ID] = struct{}{}
continue
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if e.Kind == graph.EdgeTogglesFlag {
continue
}
if _, ok := stripped[e.To]; ok {
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// stripObservabilityArtifacts drops the log/metric/trace KindEvent
// nodes and their EdgeEmits edges when the observability coverage
// domain is gated off. Used for the same reason as the function-shape
// and type-shape strips: the language extractor always emits, and the
// indexer prunes per-file before applyRepoPrefix so the gate stays a
// pure-config dial without parser plumbing.
//
// Pub/sub KindEvent nodes (Meta["event_kind"]="pubsub") are a
// separately-gated domain — they share the KindEvent kind and the
// EdgeEmits edge (publish side) but belong to the `pubsub` coverage
// domain, so this pass leaves them and any EdgeEmits/EdgeListensOn
// edge targeting them untouched. stripPubsubArtifacts owns those.
func stripObservabilityArtifacts(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
pubsubNodes := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if isPubsubEventNode(n) {
pubsubNodes[n.ID] = struct{}{}
keptNodes = append(keptNodes, n)
continue
}
if n.Kind == graph.KindEvent {
stripped[n.ID] = struct{}{}
continue
}
// log_message-context KindString registry nodes are the
// string-side shadow of log KindEvent emissions. They gate
// alongside the rest of the observability domain so a
// disabled gate leaves no log residue in the graph.
if n.Kind == graph.KindString {
if ctx, _ := n.Meta["context"].(string); ctx == "log_message" {
stripped[n.ID] = struct{}{}
continue
}
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if _, ok := stripped[e.To]; ok {
continue
}
// The observability gate strips every EdgeEmits — the publish
// side of the log/metric/trace layer. The one exception is an
// EdgeEmits whose target is a pub/sub topic node: that's the
// publish side of the separately-gated pubsub domain, so it
// survives here and is owned by stripPubsubArtifacts.
if e.Kind == graph.EdgeEmits {
if _, ok := pubsubNodes[e.To]; !ok {
continue
}
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// stripPubsubArtifacts drops the pub/sub KindEvent topic nodes
// (Meta["event_kind"]="pubsub"), every EdgeListensOn edge (a
// pubsub-only edge kind), and any EdgeEmits edge whose target is a
// pub/sub topic node, when the pubsub coverage domain is gated off.
// Endpoint-aware so a publish edge into a stripped topic node doesn't
// dangle. Mirrors stripObservabilityArtifacts — the two domains share
// KindEvent + EdgeEmits but are toggled independently.
func stripPubsubArtifacts(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if isPubsubEventNode(n) {
stripped[n.ID] = struct{}{}
continue
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if e.Kind == graph.EdgeListensOn {
continue
}
if _, ok := stripped[e.To]; ok {
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// isPubsubEventNode reports whether a node is a pub/sub topic node — a
// KindEvent carrying Meta["event_kind"]="pubsub". Distinguishes the
// pub/sub domain from observability (log/metric/trace) events, which
// share KindEvent but are gated separately.
func isPubsubEventNode(n *graph.Node) bool {
if n == nil || n.Kind != graph.KindEvent || n.Meta == nil {
return false
}
kind, _ := n.Meta["event_kind"].(string)
return kind == "pubsub"
}
// applyFixtureClassification reclassifies the language extractor's
// emitted file node from KindFile to KindFixture when the file
// lives under a testdata/ directory. When the language extractor
// produced no file node (file types without a registered
// extractor), a standalone KindFixture node is emitted instead.
//
// Reference edges from test functions to fixtures are out of scope
// for v1 — agents can already filter by kind to enumerate fixtures.
func applyFixtureClassification(relPath, lang string, result *parser.ExtractionResult) {
for _, n := range result.Nodes {
if n.Kind == graph.KindFile && n.FilePath == relPath {
if fixtures.ReclassifyFileToFixture(n) {
return
}
break
}
}
result.Nodes = append(result.Nodes, fixtures.BuildGraphArtifacts(relPath, lang)...)
}
// stripConcurrencyEdges removes the EdgeSpawns / EdgeSends /
// EdgeRecvs edges introduced by the concurrency coverage domain.
// EdgeCalls is left in place — spawns are emitted in addition to
// the corresponding call edge, so dropping just the spawn edge
// reverts to the pre-coverage call graph without losing reachability.
func stripConcurrencyEdges(result *parser.ExtractionResult) {
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
switch e.Kind {
case graph.EdgeSpawns, graph.EdgeSends, graph.EdgeRecvs:
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// revertConstantsToVariables re-classifies KindConstant /
// KindEnumMember nodes back to KindVariable when the constants
// coverage domain is gated off. Unlike stripFunctionShape /
// stripTypeShape this is a re-classification, not a removal —
// users who disable the domain still want their `const` and `iota`
// declarations in the graph, just under the original kind that
// pre-coverage code expected.
func revertConstantsToVariables(result *parser.ExtractionResult) {
for _, n := range result.Nodes {
switch n.Kind {
case graph.KindConstant, graph.KindEnumMember:
n.Kind = graph.KindVariable
}
}
}
// stripTypeShape removes the alias / composition edges introduced
// by the type_shape coverage domain (EdgeAliases, EdgeComposes).
// EdgeExtends is left in place — it's an existing edge kind whose
// newtype-derived emissions fall under the spec's "EdgeExtends
// continues to mean newtype / inheritance / interface extension"
// guarantee, not a new domain signal.
func stripTypeShape(result *parser.ExtractionResult) {
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
switch e.Kind {
case graph.EdgeAliases, graph.EdgeComposes:
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
// stripFunctionShape removes the param/closure/generic_param nodes
// and their associated edges from a per-file extraction result.
// Used when the function_shape coverage domain is gated off — the
// language extractor always emits these for resolution-internal
// reasons, and we drop them after the extractor returns rather than
// wire a config dependency through the parser layer.
func stripFunctionShape(result *parser.ExtractionResult) {
stripped := make(map[string]struct{})
keptNodes := result.Nodes[:0]
for _, n := range result.Nodes {
if isFunctionShapeNode(n.Kind) {
stripped[n.ID] = struct{}{}
continue
}
keptNodes = append(keptNodes, n)
}
result.Nodes = keptNodes
keptEdges := result.Edges[:0]
for _, e := range result.Edges {
if isFunctionShapeEdge(e.Kind) {
continue
}
if _, ok := stripped[e.From]; ok {
continue
}
if _, ok := stripped[e.To]; ok {
continue
}
keptEdges = append(keptEdges, e)
}
result.Edges = keptEdges
}
func isFunctionShapeNode(kind graph.NodeKind) bool {
switch kind {
case graph.KindParam, graph.KindClosure, graph.KindGenericParam:
return true
}
return false
}
func isFunctionShapeEdge(kind graph.EdgeKind) bool {
switch kind {
case graph.EdgeParamOf, graph.EdgeReturns, graph.EdgeTypedAs, graph.EdgeCaptures:
return true
}
return false
}
func (idx *Indexer) applyRepoPrefix(nodes []*graph.Node, edges []*graph.Edge) {
// Stamp WorkspaceID / ProjectID on every node emitted by this
// indexer regardless of mode — single-repo and multi-repo both
// need the boundary slugs for query scoping and contract
// matching. Single-repo callers can leave them empty; the
// MultiIndexer path always sets them via SetWorkspaceID /
// SetProjectID before calling Index.
if idx.workspaceID != "" || idx.projectID != "" {
for _, n := range nodes {
if idx.workspaceID != "" && n.WorkspaceID == "" {
n.WorkspaceID = idx.workspaceID
}
if idx.projectID != "" && n.ProjectID == "" {
n.ProjectID = idx.projectID
}
}
}
if idx.repoPrefix == "" {
return
}
prefix := idx.repoPrefix + "/"
const unresolvedMarker = "unresolved::"
// Intern every minted string. A node ID is referenced once on the
// node and again on every edge endpoint that points at it; a file
// path recurs on every node and edge in that file. Without
// interning each reference is a distinct `prefix + s` allocation —
// interning collapses them to one shared backing array, and edge
// endpoints end up sharing storage with the node ID they name.
//
// The file path is identical for every node and edge extracted from
// the same file — thousands of them. Concatenating `prefix + path`
// per reference would mint thousands of throwaway strings before
// interning collapses their storage. This per-call cache computes
// the interned prefixed path once per distinct raw path, so a file
// with N symbols pays one concatenation instead of N.
prefixedPath := make(map[string]string)
internFilePath := func(raw string) string {
if c, ok := prefixedPath[raw]; ok {
return c
}
c := intern.String(prefix + raw)
prefixedPath[raw] = c
return c
}
for _, n := range nodes {
n.ID = intern.String(prefix + n.ID)
n.FilePath = internFilePath(n.FilePath)
n.RepoPrefix = idx.repoPrefix
// Name, Language and Kind are low-cardinality and recur across
// thousands of nodes — method/function names like String, New,
// Get…, the ~20 distinct languages, and the fixed set of node
// kinds. Interning collapses each to a single backing array; it
// also shrinks the byName secondary index, whose keys are these
// same strings.
n.Name = intern.String(n.Name)
n.Language = intern.String(n.Language)
n.Kind = graph.NodeKind(intern.String(string(n.Kind)))
}
for _, e := range edges {
e.From = intern.String(prefix + e.From)
if strings.HasPrefix(e.To, unresolvedMarker) {
// Unresolved targets carry no prefix, but many edges name
// the same unresolved symbol — still worth interning.
e.To = intern.String(e.To)
} else {
e.To = intern.String(prefix + e.To)
}
e.FilePath = internFilePath(e.FilePath)
}
}
// Index walks root and populates the graph using a concurrent worker pool.
//
// This is the backwards-compatible entry point; it delegates to IndexCtx with
// a background context. Callers wanting progress notifications or cancellation
// should use IndexCtx directly.
func (idx *Indexer) Index(root string) (*IndexResult, error) {
return idx.IndexCtx(context.Background(), root)
}
// IndexCtx is Index with a context, enabling progress reporting. The reporter
// is pulled from ctx via progress.FromContext — attach one with
// progress.WithReporter to receive stage updates. If no reporter is attached,
// stage calls are silently dropped.
// clampParseWeight maps a file size to a parse-admission weight bounded to
// [1, budget]. A file larger than the whole budget is admitted alone
// (weight == budget) so the bytes-in-flight semaphore can never deadlock.
func clampParseWeight(size, budget int64) int64 {
if size < 1 {
return 1
}
if size > budget {
return budget
}
return size
}
func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexResult, retErr error) {
start := time.Now()
reporter := progress.FromContext(ctx)
// Cold/full-index GC tuning: raise the GC percent window and install a
// cgroup-aware soft memory limit for the duration of the index, then
// restore the prior settings on every exit path. The knobs affect only GC
// timing and peak RSS during the allocation burst of a full index — the
// graph content is identical with them on or off. Disable for A/B runs
// with GORTEX_INDEX_GC_TUNE=0; see gc_tune.go.
restoreGCTuning := applyIndexGCTuning(idx.logger)
defer restoreGCTuning()
absRoot, err := filepath.Abs(root)
if err != nil {
return nil, err
}
idx.rootPath = absRoot
idx.projectName = search.DetectProjectName(absRoot)
reporter.Report("walking files", 0, 0)
// Collect files. Files over IndexConfig.MaxFileSize are skipped
// during the walk — they're nearly always generated/minified code
// that dominates parse time without contributing useful signal.
// A single summary warning reports how many were skipped so the
// user knows when the cap is biting.
//
// Each surviving file is captured with its walk-time ModTime so
// the worker (contract-cache mtime stamp) and the post-parse
// fileMtimes loop don't have to os.Stat again. d.Info() is one
// syscall per file regardless; trading one walk-time stat for two
// later stats is the net win.
maxSize := idx.config.MaxFileSize
// Corpus-admission gate: drops oversized document assets and (by
// default) binary/vector data artifacts at the walk, before they are
// read and extracted, so a content-heavy repo can't pull gigabytes of
// non-source files into the parse pipeline and OOM (#120). Inert for
// all-code repos.
contentGate := idx.newContentAdmissionGate()
// Git-aware admission (opt-in): when index.skip_untracked_assets is on,
// drop asset-class files git does not track — uncommitted RAG corpora /
// datasets / build outputs that .gitignore can't catch (#120). Inert
// when off, on a non-git repo, or when git is unavailable.
untrackedGate := idx.newUntrackedAssetGate(ctx, absRoot)
var files []walkedFile
var skippedLarge int
var skippedBytes int64
var skippedBySize []skippedFile
var skippedByContent []skippedFile
var skippedContentBytes int64
var parseFailedFiles []skippedFile
err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
if d.IsDir() {
if idx.shouldPruneDir(path, absRoot) {
return filepath.SkipDir
}
return nil
}
lang, ok := idx.effectiveLanguage(path, nil)
if !ok {
return nil
}
if idx.shouldExclude(path, absRoot, false) {
return nil
}
info, statErr := d.Info()
if statErr != nil {
// Couldn't read FileInfo (race with deletion, broken
// symlink, …). Skip — the worker would fail too.
return nil
}
if maxSize > 0 && info.Size() > maxSize {
skippedLarge++
skippedBytes += info.Size()
rel, _ := filepath.Rel(absRoot, path)
skippedBySize = append(skippedBySize, skippedFile{
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(),
})
return nil
}
if reason, skip := untrackedGate.skip(lang, path); skip {
skippedContentBytes += info.Size()
rel, _ := filepath.Rel(absRoot, path)
skippedByContent = append(skippedByContent, skippedFile{
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason,
})
return nil
}
if reason, skip := contentGate.skip(lang, info.Size()); skip {
skippedContentBytes += info.Size()
rel, _ := filepath.Rel(absRoot, path)
skippedByContent = append(skippedByContent, skippedFile{
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason,
})
return nil
}
files = append(files, walkedFile{
path: path,
lang: lang,
size: info.Size(),
mtimeNano: info.ModTime().UnixNano(),
})
return nil
})
if err != nil {
return nil, err
}
if skippedLarge > 0 {
idx.logger.Info("indexer: skipped large files over MaxFileSize",
zap.Int("count", skippedLarge),
zap.Int64("total_bytes", skippedBytes),
zap.Int64("limit_bytes", maxSize))
}
if len(skippedByContent) > 0 {
idx.logger.Info("indexer: skipped content/data assets by admission policy",
zap.Int("count", len(skippedByContent)),
zap.Int64("total_bytes", skippedContentBytes))
}
reporter.Report("walking files", len(files), len(files))
// In-memory shadow for cold-start indexing on disk-backed stores.
// Disk backends pay ms-level per-call cost on every read; running
// the resolver against the disk store turns its ~100k+ point
// lookups into many minutes of wall time. Instead, swap idx.graph
// to an in-memory *Graph for the whole IndexCtx pipeline — parse,
// resolve, all subpasses, every per-edge MERGE/MATCH stays in
// memory at nanosecond latency. At the end, dump the final state
// to the disk backend via one BulkLoad cycle, so the disk has the
// post-resolve graph and the bench's query workload runs against
// the persisted state.
//
// Guards:
// - Backend must implement graph.BulkLoader (the on-disk backend opts in).
// - Store must be empty (NodeCount == 0 && EdgeCount == 0). The
// final dump is BulkLoad's INSERT-only fast path — running it
// against a non-empty store would corrupt or duplicate.
// Incremental / re-index flows fall through to the per-call
// AddBatch path against the disk store directly.
// - File count is below the shadow-max threshold (see
// shadowMaxFileCount). Above the threshold the shadow's RAM
// footprint would exceed available memory — Linux / Firefox
// at full scale (~10M+ edges) would push the shadow past
// 20GB. Override with GORTEX_SHADOW_MAX_FILES.
// - The swap happens before the parse worker pool starts and is
// committed before IndexCtx returns. retErr from the named
// return suppresses the commit when the pipeline errored —
// the disk store stays empty rather than capturing partial
// state.
var diskTarget graph.Store
var inMemShadow *graph.Graph
bl, blOK := idx.graph.(graph.BulkLoader)
// Per-Indexer sentinel: each *Indexer is constructed fresh
// (per-repo in MultiIndexer, once in single-repo daemons), so
// "this Indexer has indexed before" is the right question to
// gate the shadow-swap on. The legacy gate looked at the
// disk store's NodeCount, but in MultiIndexer the disk store
// holds data from sibling repos that already drained — the
// gate would mis-fire and force the big repo onto the per-row
// path. With per-repo-prefixed stub IDs (internal/graph/stub.go)
// concurrent shadow drains no longer conflict on PRIMARY KEY,
// so disk-non-empty is safe.
firstIndex := idx.indexCount.Load() == 0
belowShadowMax := len(files) <= shadowMaxFileCount()
// The file-count ceiling is blind to the few-huge-files shape: a
// content repo of a few hundred PDFs / text dumps / spreadsheets is
// far under shadowMaxFileCount yet holds multiple GB that explode
// into hundreds of thousands of section nodes. Gate the in-memory
// shadow on total input bytes too, so such a repo falls through to
// the bounded per-call disk path instead of pinning the whole
// post-parse graph in RAM and OOMing (see #120).
var totalFileBytes int64
for i := range files {
totalFileBytes += files[i].size
}
maxShadowBytes := shadowMaxBytes()
belowShadowBytes := totalFileBytes <= maxShadowBytes
shadowTaken := blOK && firstIndex && belowShadowMax && belowShadowBytes
preNodes := idx.graph.NodeCount()
preEdges := idx.graph.EdgeCount()
idx.logger.Info("indexer: shadow-swap decision",
zap.String("repo", idx.RepoPrefix()),
zap.Bool("bulk_loader", blOK),
zap.Bool("first_index", firstIndex),
zap.Int("pre_nodes", preNodes),
zap.Int("pre_edges", preEdges),
zap.Int("files", len(files)),
zap.Int("shadow_max_files", shadowMaxFileCount()),
zap.Bool("below_shadow_max", belowShadowMax),
zap.Int64("total_file_bytes", totalFileBytes),
zap.Int64("shadow_max_bytes", maxShadowBytes),
zap.Bool("below_shadow_bytes", belowShadowBytes),
zap.Bool("shadow_taken", shadowTaken),
)
if shadowTaken {
// Warm-restart safety. `firstIndex` is a PER-INDEXER sentinel, and
// a fresh per-repo Indexer is constructed on every daemon restart,
// so firstIndex is true on every restart — even when the
// persistent disk store already holds this repo's nodes from a
// prior run. The shadow drain below ends in BulkLoad's INSERT-only
// COPY, which (per this function's own contract) "running against a
// non-empty store would corrupt or duplicate". A duplicate-primary-
// key bulk load against the persisted rows would fail warmup, and
// because the repo's mtimes never get persisted when warmup dies
// first, the failure re-fires on the next restart: a crash loop.
// Evicting the repo's existing rows first makes the bulk load land
// on a clean slate. EvictRepo self-guards with a count query, so this is a
// cheap no-op for the genuine first-index cases (true cold start,
// a newly-tracked repo) where the disk store has no rows for this
// prefix. preNodes>0 short-circuits the call entirely on the
// first repo of a cold start (empty store).
if preNodes > 0 {
if n, e := idx.graph.EvictRepo(idx.RepoPrefix()); n > 0 || e > 0 {
idx.logger.Info("indexer: evicted stale repo rows before bulk reload (warm restart)",
zap.String("repo", idx.RepoPrefix()),
zap.Int("nodes", n), zap.Int("edges", e))
}
}
idx.indexCount.Add(1)
diskTarget = idx.graph
inMemShadow = graph.New()
idx.graph = inMemShadow
// Capture the disk store as the vector sink: buildSearchIndex runs
// while idx.graph is the shadow (no VectorSearcher), so without this
// the embedded vectors never land on disk. The `vectors` table has no
// FK to `nodes`, so upserting before FlushBulk persists the nodes is
// safe. Cleared when idx.graph is restored below.
idx.bulkVectorSink, _ = diskTarget.(graph.VectorSearcher)
// Same capture for the content index: the per-file content stream
// must reach content_fts on disk while idx.graph is the shadow.
idx.contentSink, _ = diskTarget.(graph.ContentSearcher)
// The resolver was constructed at indexer.New with the disk
// Store. Redirect it at the shadow too, otherwise ResolveAll
// reads from the empty disk Store, finds no pending edges,
// and short-circuits — silently disabling every resolver pass
// (module attribution, relative imports, edge in-place
// resolution, …) for any backend that takes the shadow path.
if idx.resolver != nil {
idx.resolver.SetGraph(inMemShadow)
}
defer func() {
if retErr != nil {
idx.graph = diskTarget
idx.bulkVectorSink = nil
idx.contentSink = nil
if idx.resolver != nil {
idx.resolver.SetGraph(diskTarget)
}
return
}
reporter.Report("persisting bulk graph", 0, 0)
drainStart := time.Now()
shadowNodeCount := inMemShadow.NodeCount()
shadowEdgeCount := inMemShadow.EdgeCount()
idx.logger.Info("indexer: drain start (shadow → disk)",
zap.String("repo", idx.RepoPrefix()),
zap.Int("shadow_nodes", shadowNodeCount),
zap.Int("shadow_edges", shadowEdgeCount),
)
bl.BeginBulkLoad()
// Drain the shadow shard-by-shard so the indexer's hold on
// the 11-GB Linux-scale graph is released progressively
// instead of pinned until persist returns. The drain
// iterators free each shard's node/edge maps as they
// advance, so peak RAM during the persist window is
// roughly the chunk buffer + the backend's working set,
// not full shadow + the disk backend's bulk-COPY buffer.
//
// Collect (id, tokens) for every search-eligible node as
// the drain yields them — feeds the backend's native FTS
// at FlushBulk time when the store implements
// graph.SymbolSearcher. Nodes that fail
// shouldIndexForSearch (KindFile / KindImport /
// KindLocal / KindBuiltin / skip-search lang+kind pairs)
// are excluded so the FTS corpus matches the in-process
// BM25 corpus exactly.
searcher, hasFTS := diskTarget.(graph.SymbolSearcher)
var ftsItems []graph.SymbolFTSItem
if hasFTS {
// Pre-size to the shadow's node count to avoid grow
// churn on a 600k-node Vscode-shape repo.
ftsItems = make([]graph.SymbolFTSItem, 0, inMemShadow.NodeCount())
}
// Key-ordered bulk drain. The nodes table is WITHOUT ROWID
// (its primary key IS the B-tree), so inserting in ascending
// id order makes each insert an append to the rightmost leaf
// instead of a random split mid-tree — far fewer page splits
// on the cold load. The edges table's UNIQUE(from_id, …) index
// benefits the same way from from-id-ordered inserts. So the
// whole drain is collected, sorted once, then chunk-written.
//
// This block only runs on the first/empty cold index (the
// shadow-swap branch above), and the shadow node/edge sets
// already fit in RAM (the branch is gated on the shadow byte /
// file budget). DrainNodes / DrainEdges free each shard as they
// yield, so holding the drained set in one slice to sort costs
// no more peak RAM than the shadow already did.
const persistChunk = 100000
allNodes := make([]*graph.Node, 0, shadowNodeCount)
for n := range inMemShadow.DrainNodes() {
if hasFTS && idx.shouldIndexForSearch(n) {
ftsItems = append(ftsItems, graph.SymbolFTSItem{
NodeID: n.ID,
Tokens: ftsTokensFor(n, idx.projectName),
})
}
allNodes = append(allNodes, n)
}
sort.Slice(allNodes, func(i, j int) bool {
return allNodes[i].ID < allNodes[j].ID
})
for start := 0; start < len(allNodes); start += persistChunk {
end := min(start+persistChunk, len(allNodes))
diskTarget.AddBatch(allNodes[start:end], nil)
}
allNodes = nil
allEdges := make([]*graph.Edge, 0, shadowEdgeCount)
for e := range inMemShadow.DrainEdges() {
allEdges = append(allEdges, e)
}
sort.Slice(allEdges, func(i, j int) bool {
return lessEdgeKey(allEdges[i], allEdges[j])
})
for start := 0; start < len(allEdges); start += persistChunk {
end := min(start+persistChunk, len(allEdges))
diskTarget.AddBatch(nil, allEdges[start:end])
}
allEdges = nil
flushStart := time.Now()
idx.logger.Info("indexer: FlushBulk start",
zap.String("repo", idx.RepoPrefix()),
zap.Duration("drain_elapsed", flushStart.Sub(drainStart)),
)
if ferr := bl.FlushBulk(); ferr != nil {
retErr = fmt.Errorf("indexer: persist bulk graph: %w", ferr)
}
idx.logger.Info("indexer: FlushBulk complete",
zap.String("repo", idx.RepoPrefix()),
zap.Duration("flush_elapsed", time.Since(flushStart)),
zap.Duration("total_drain", time.Since(drainStart)),
zap.Int("nodes", shadowNodeCount),
zap.Int("edges", shadowEdgeCount),
)
// Build the backend FTS after the bulk load completes so
// CREATE_FTS_INDEX has the full corpus to scan in one
// pass. BulkUpsertSymbolFTS does its own
// extension-install dance, so this is the only place the
// indexer needs to know about SymbolSearcher.
if hasFTS && len(ftsItems) > 0 {
reporter.Report("building symbol fts", 0, 0)
if ferr := searcher.BulkUpsertSymbolFTS(idx.RepoPrefix(), ftsItems); ferr != nil {
idx.logger.Warn("indexer: bulk symbol FTS upsert failed",
zap.Error(ferr))
} else if ferr := searcher.BuildSymbolIndex(); ferr != nil {
idx.logger.Warn("indexer: backend FTS build failed",
zap.Error(ferr))
}
reporter.Report("building symbol fts", 1, 1)
}
reporter.Report("persisting bulk graph", 1, 1)
idx.graph = diskTarget
idx.bulkVectorSink = nil
idx.contentSink = nil
// Mirror of the SetGraph(inMemShadow) above: the resolver
// must follow the graph pointer back to the disk store, or
// every post-index per-file resolve (the watcher save path,
// incremental reindex) reads the drained — now empty —
// shadow and silently resolves nothing.
if idx.resolver != nil {
idx.resolver.SetGraph(diskTarget)
}
}()
} else if diskTarget == nil && idx.graph.NodeCount() == 0 && idx.graph.EdgeCount() == 0 {
if _, isBulk := idx.graph.(graph.BulkLoader); isBulk && firstIndex && (!belowShadowMax || !belowShadowBytes) {
idx.logger.Info("indexer: skipping in-memory shadow; building against disk store (bounded RAM)",
zap.Int("files", len(files)),
zap.Int("file_threshold", shadowMaxFileCount()),
zap.Bool("over_file_count", !belowShadowMax),
zap.Int64("total_file_bytes", totalFileBytes),
zap.Int64("byte_threshold", maxShadowBytes),
zap.Bool("over_byte_budget", !belowShadowBytes))
}
}
// Content-index rebuild strategy. The crash-safe path (on-disk store)
// deletes each file's prior content rows as that file re-streams
// (contentWipeFile, invoked at the per-file AddBatch sites below) and
// sweeps every OTHER content row after the authoritative mtime replace —
// so a mid-parse kill leaves a mix of old+new content per file instead of
// the empty table a repo-wide pre-wipe would leave. contentStreamedFiles
// records which files actually streamed content this walk (the wipe's own
// argument, i.e. the node FilePath content_fts carries); the end sweep
// keeps exactly that set, so files that vanished from disk AND files that
// still exist but no longer yield content sections (doc emptied,
// classification changed) are both reaped in one scan — the transitions
// the old repo-wide pre-wipe used to cover. Backends without the per-file
// capability keep that old behaviour: one repo-wide wipe up front (a
// cold-store no-op; a cheap per-repo DELETE on a warm one).
var (
contentWipeFile func(filePath string)
contentStreamedMu sync.Mutex
contentStreamedFiles map[string]struct{}
)
if cs := idx.contentSearcher(); cs != nil {
if w, ok := cs.(interface {
WipeContentFileInRepo(repoPrefix, filePath string) error
}); ok {
repoPrefix := idx.RepoPrefix()
contentStreamedFiles = make(map[string]struct{})
contentWipeFile = func(filePath string) {
if filePath == "" {
return
}
contentStreamedMu.Lock()
contentStreamedFiles[filePath] = struct{}{}
contentStreamedMu.Unlock()
if err := w.WipeContentFileInRepo(repoPrefix, filePath); err != nil {
idx.logger.Warn("indexer: per-file content wipe failed", zap.Error(err))
}
}
} else if err := cs.WipeContent(idx.RepoPrefix()); err != nil {
idx.logger.Warn("indexer: content index wipe failed", zap.Error(err))
}
}
// Worker pool.
workers := idx.config.Workers
if workers <= 0 {
workers = 1
}
// idx.config.Workers defaults to the host's runtime.NumCPU(); inside a
// CPU-limited container that exceeds the cgroup CPU quota and the pool
// over-subscribes, so CFS throttling drags throughput down. Clamp the
// effective pool size to the quota when one is present (lowers only, floor
// of 1, host-identical when unquotaed). GORTEX_INDEX_CPU_CLAMP=0 opts out.
if cpuClampEnabled() {
workers = clampWorkersToCPUQuota(workers, cgroupCPUQuota())
}
// Optional crash isolation: run tree-sitter extraction in worker
// subprocesses so a grammar SIGSEGV / OOM / hang on one
// pathological file is contained — the bad file is quarantined and
// the pass still completes. Off unless index.crash_isolation /
// GORTEX_PARSER_ISOLATION is set.
var parsePool *crashpool.Pool
var quarantine *crashpool.Quarantine
if idx.crashIsolationEnabled() {
quarantine = crashpool.LoadQuarantine(filepath.Join(absRoot, ".gortex", "parser-quarantine.json"))
if p, perr := idx.newParsePool(workers); perr != nil {
idx.logger.Warn("indexer: crash isolation requested but parser pool unavailable; parsing in-process",
zap.Error(perr))
} else {
parsePool = p
defer parsePool.Close()
idx.logger.Info("indexer: parser crash isolation enabled", zap.Int("workers", workers))
}
}
// Workers parse files, write the resulting nodes/edges to the
// sharded graph, and run per-file contract extractors on the same
// src bytes — all in one pass. Reusing src avoids the 10k+ disk
// re-reads the old "parse then extractContracts" flow did; running
// the contract extractors per-worker parallelises what used to be
// a serial post-pass; language-filtered dispatch skips extractors
// that can't match (HTTP on .css, OpenAPI on .ts, etc.).
const parseReportEvery = 50
totalFiles := len(files)
_, contractExtractorsByLang := idx.buildPerFileContractExtractors()
contractReg := contracts.NewRegistry()
var contractMu sync.Mutex
var errMu sync.Mutex
var errors []IndexError
var processed int64
var fileCount int64
var skippedByTimeout int64
var skippedByMinified int64
// Bound peak parse memory: a weighted, bytes-in-flight semaphore
// admits each worker by its file size before it reads + extracts, so
// a cluster of large files (PDFs / office docs in a content repo)
// serialises instead of all workers materialising whole files and
// their parse trees at once. Code files are tiny and flow freely;
// only genuinely large inputs queue. budget <= 0 disables the cap.
parseBudget := idx.config.MaxParseBytesInFlight
var parseSem *semaphore.Weighted
if parseBudget > 0 {
parseSem = semaphore.NewWeighted(parseBudget)
}
// In addition to the bytes-in-flight budget above, cap how many
// genuinely large files are *read* concurrently: a few huge PDFs /
// spreadsheets / vector artifacts can dominate RSS before extraction
// even starts. Small source files bypass the gate and keep full
// throughput.
largeReadGate := make(chan struct{}, largeFileReadParallelism(workers))
readFile := func(wf walkedFile) ([]byte, error) {
if wf.size < largeFileReadThresholdBytes {
return os.ReadFile(wf.path)
}
largeReadGate <- struct{}{}
defer func() { <-largeReadGate }()
return os.ReadFile(wf.path)
}
// recordStreamedMtime persists a file's mtime incrementally, in batches,
// as its nodes land on disk during a full track — so a first-ever index
// killed mid-parse RESUMES on the next boot (the flushed files reconcile
// clean, the remainder re-parses) instead of re-tracking a large repo
// from scratch every time it dies under memory pressure.
//
// INVARIANT: an mtime row must land only AFTER its file's nodes are
// durably in the store, or a fresh mtime would falsely imply indexed
// nodes. The `idx.graph.(graph.FileMtimeWriter)` assertion is what
// enforces it: it succeeds ONLY on the direct-to-disk path, where
// idx.graph is the on-disk store and AddBatch has already made the
// file's nodes durable. On the whole-repo in-memory shadow and the
// streaming-flush per-chunk shadow, idx.graph is a plain graph.Graph
// (no FileMtimeWriter) whose nodes reach disk only at a later drain, so
// this self-skips and those paths persist at their own drain points (the
// final ReplaceFileMtimes / the streaming-flush per-chunk persist). The
// final ReplaceFileMtimes still runs and is authoritative — it also
// prunes deleted files — so these batches only bound the work a crash
// can waste; the leftover under-threshold tail rides that final replace.
var (
streamMtimeMu sync.Mutex
streamMtimes = make(map[string]int64, mtimeStreamPersistEvery)
)
recordStreamedMtime := func(absPath string, mtimeNano int64) {
if mtimeNano <= 0 {
return
}
w, ok := idx.graph.(graph.FileMtimeWriter)
if !ok {
return
}
var flush map[string]int64
streamMtimeMu.Lock()
streamMtimes[idx.relKey(absPath)] = mtimeNano
if len(streamMtimes) >= mtimeStreamPersistEvery {
flush = streamMtimes
streamMtimes = make(map[string]int64, mtimeStreamPersistEvery)
}
streamMtimeMu.Unlock()
if flush != nil {
if err := w.BulkSetFileMtimes(idx.repoPrefix, flush); err != nil {
idx.logger.Warn("indexer: incremental mtime batch persist failed",
zap.String("repo", idx.repoPrefix), zap.Error(err))
}
}
}
// parseChunk runs the per-file worker pool over the supplied
// slice. Closure over outer state (errors, counters, contract
// registry, parsePool, quarantine) so it can be called multiple
// times — once for the non-streaming path, repeatedly for the
// streaming-flush large-repo path where each call processes a
// bounded slice into a per-chunk in-memory shadow.
parseChunk := func(chunkFiles []walkedFile) {
fileCh := make(chan walkedFile, workers*4)
var wg sync.WaitGroup
for range workers {
wg.Add(1)
go func() {
defer wg.Done()
var localContracts []contracts.Contract
for wf := range fileCh {
path := wf.path
p := atomic.AddInt64(&processed, 1)
if p == 1 || p%parseReportEvery == 0 {
reporter.Report("parsing", int(p), totalFiles)
}
// Admit this file into extraction under the
// bytes-in-flight budget before reading it, so large
// content files serialise instead of all workers
// materialising whole files at once.
var weight int64
if parseSem != nil {
weight = clampParseWeight(wf.size, parseBudget)
if aerr := parseSem.Acquire(ctx, weight); aerr != nil {
return
}
}
relPath, _ := filepath.Rel(absRoot, path)
// Streaming content extractors (PDF / office docs) read the
// file themselves — one page/slide/sheet at a time — instead
// of materialising the whole file. Only the in-process route
// streams; the crash-isolation subprocess route keeps bytes.
if walkExt, found := idx.registry.GetByLanguage(wf.lang); found && parsePool == nil {
if se, ok := walkExt.(parser.StreamingExtractor); ok {
result, serr := idx.extractStreaming(se, path, relPath)
if parseSem != nil {
parseSem.Release(weight)
}
if serr != nil {
errMu.Lock()
errors = append(errors, IndexError{FilePath: path, Error: serr.Error()})
if result == nil {
parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: wf.lang, cause: serr.Error()})
}
errMu.Unlock()
}
if result == nil {
continue
}
idx.applyRepoPrefix(result.Nodes, result.Edges)
if contentWipeFile != nil {
contentWipeFile(firstContentFilePath(result.Nodes))
}
idx.streamContentSections(result.Nodes)
idx.graph.AddBatch(result.Nodes, result.Edges)
idx.persistConstValues(result)
recordStreamedMtime(path, wf.mtimeNano)
continue
}
}
src, err := readFile(wf)
if err != nil {
errMu.Lock()
errors = append(errors, IndexError{FilePath: path, Error: err.Error()})
errMu.Unlock()
if parseSem != nil {
parseSem.Release(weight)
}
continue
}
// Reuse the walk-time language. The walk's
// effectiveLanguage call already consulted shebang
// bytes via readSniffPrefix (512-byte probe), so a
// re-detect against the full src would change the
// answer only on the vanishingly rare case where a
// language marker lives past byte 512 — and any such
// case is content-sniffing-by-luck rather than spec'd
// behaviour. The fallback below covers the truly
// pathological case where the walk-time language has
// no extractor registered (effectively dead code).
lang := wf.lang
ext, _ := idx.registry.GetByLanguage(lang)
if ext == nil {
if relang, ok := idx.effectiveLanguage(path, src); ok {
lang = relang
ext, _ = idx.registry.GetByLanguage(lang)
}
}
if ext == nil {
if parseSem != nil {
parseSem.Release(weight)
}
continue
}
// Pre-ingestion transforms: rewrite the bytes before
// extraction (BOM strip, minified-bundle expansion, a
// PDF→markdown command, …).
src = idx.transforms.run(relPath, src)
result, skipped, err := idx.extractFile(parsePool, quarantine, path, relPath, lang, ext, src)
if parseSem != nil {
parseSem.Release(weight)
}
if err != nil {
errMu.Lock()
errors = append(errors, IndexError{FilePath: path, Error: err.Error()})
errMu.Unlock()
}
if result == nil {
// A full-index parse failure that produced no nodes:
// record it for a skip-node post-pass so the file
// stays visible instead of vanishing. (The live-modify
// path never reaches here — it keeps a file's prior
// nodes through a transient parse failure.)
if err != nil {
errMu.Lock()
parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: lang, cause: err.Error()})
errMu.Unlock()
}
continue
}
if skipped && len(result.Nodes) > 0 {
if _, ok := result.Nodes[0].Meta["skipped_due_to_timeout"]; ok {
atomic.AddInt64(&skippedByTimeout, 1)
}
if _, ok := result.Nodes[0].Meta["skipped_due_to_minified"]; ok {
atomic.AddInt64(&skippedByMinified, 1)
}
}
// Append coverage artifacts (todos / licenses /
// ownership) before applyRepoPrefix so they get the
// same multi-repo namespacing treatment as
// language-extractor output. Skipped for quarantined /
// timed-out files — the coverage scanners would re-read
// a source the parser could not survive.
if !skipped {
idx.applyCoverageDomains(relPath, lang, src, result)
}
idx.applyRepoPrefix(result.Nodes, result.Edges)
// Find the file node (if the extractor produced one)
// and collect its outgoing edges — contract extractors
// take the file-scope edge set (imports, etc.), not
// every intra-file edge.
var fileNodeID, fileGraphPath string
for _, n := range result.Nodes {
if n.Kind == graph.KindFile {
fileNodeID = n.ID
fileGraphPath = n.FilePath
break
}
}
var fileScopeEdges []*graph.Edge
if fileNodeID != "" {
for _, e := range result.Edges {
if e.From == fileNodeID {
fileScopeEdges = append(fileScopeEdges, e)
}
}
}
// Stream this file's content (data_class=content) section
// bodies into the dedicated content index and lean the
// nodes to a snippet BEFORE AddBatch, so the bulk text
// never enters the graph, the symbol search, or the
// materialising code passes.
if contentWipeFile != nil {
contentWipeFile(firstContentFilePath(result.Nodes))
}
idx.streamContentSections(result.Nodes)
// Batch the per-file insert into one shard-grouped pass
// so each shard's lock is acquired at most once per
// file instead of N + 2·E times. Profiling showed 69
// of 102 workers blocked on lockTwoWrite under the
// per-edge path during cold-start warmup.
idx.graph.AddBatch(result.Nodes, result.Edges)
idx.persistConstValues(result)
idx.persistFileMeta(relPath, src, result)
recordStreamedMtime(path, wf.mtimeNano)
if !skipped && fileGraphPath != "" {
exts := contractExtractorsByLang[lang]
if len(exts) > 0 {
c := idx.runContractExtractorsForFile(
fileGraphPath, src, result.Nodes, fileScopeEdges, exts, result.Tree)
localContracts = append(localContracts, c...)
// Populate the per-file contract cache so a
// later IncrementalReindex can skip this file
// on a cache hit. Mtime comes from the walk-
// time d.Info() — no extra stat here.
if wf.mtimeNano > 0 {
idx.contractCacheMu.Lock()
idx.contractCache[fileGraphPath] = &contractCacheEntry{
mtimeNano: wf.mtimeNano,
contracts: c,
}
idx.contractCacheMu.Unlock()
}
}
}
// Release the parse tree now that the per-file
// contract pass is done. Post-passes that need a
// tree for this file (cross-file handler resolution)
// re-parse on demand. Nil-safe.
result.Tree.Release()
atomic.AddInt64(&fileCount, 1)
}
if len(localContracts) > 0 {
contractMu.Lock()
for _, c := range localContracts {
contractReg.Add(c)
}
contractMu.Unlock()
}
}()
}
for _, f := range chunkFiles {
fileCh <- f
}
close(fileCh)
wg.Wait()
}
// Dispatch the largest files first. Both dispatch paths below
// consume this slice in order: the plain path feeds it straight to
// the worker pool, and the streaming-flush path carves it into
// chunks from the front. Feeding the biggest file to the workers
// first lets its long parse overlap with the tail of small files,
// instead of the whole index waiting on a large file that would
// otherwise be dispatched last. The sort is stable and a pure
// permutation, so the set of indexed files and the resulting graph
// are identical — only the dispatch order changes.
sortBySizeDesc(files)
// Streaming-flush path: above shadowMaxFileCount with a
// BulkLoader-capable backend, we can't fit the whole shadow in
// RAM but we can still amortise the per-file disk-write cost by
// chunking. Each chunk runs against its own throwaway shadow,
// then flushes via BulkLoad to disk. Resolve runs against the
// disk store afterwards (per-call, slower than the shadow path
// but bounded RAM). Activated by GORTEX_STREAMING_FLUSH=1; off
// by default since it requires the disk-only resolver path
// (~tens of minutes on huge repos) that we haven't yet
// optimised end-to-end.
if diskTarget == nil && streamingFlushActive(idx.graph, len(files)) {
bl, _ := idx.graph.(graph.BulkLoader)
streamingDisk := idx.graph
chunkSize := streamingChunkSize()
idx.logger.Info("indexer: streaming-flush parse",
zap.Int("files", len(files)),
zap.Int("chunk_size", chunkSize))
for chunkStart := 0; chunkStart < len(files); chunkStart += chunkSize {
chunkEnd := min(chunkStart+chunkSize, len(files))
chunkShadow := graph.New()
idx.graph = chunkShadow
parseChunk(files[chunkStart:chunkEnd])
// Flush chunk to disk.
bl.BeginBulkLoad()
streamingDisk.AddBatch(chunkShadow.AllNodes(), chunkShadow.AllEdges())
if err := bl.FlushBulk(); err != nil {
return nil, fmt.Errorf("indexer: streaming-flush chunk %d..%d: %w", chunkStart, chunkEnd, err)
}
// This chunk's nodes are durable on disk now, so persist its
// files' mtimes — a kill mid-track then resumes from this chunk
// boundary instead of re-tracking the whole large first index
// from scratch. The in-worker recordStreamedMtime is a no-op on
// this path (idx.graph was the throwaway per-chunk shadow, not a
// FileMtimeWriter); the persist happens here, after the drain,
// keeping the "fresh mtime implies durable nodes" invariant.
if w, ok := streamingDisk.(graph.FileMtimeWriter); ok {
batch := make(map[string]int64, chunkEnd-chunkStart)
for _, f := range files[chunkStart:chunkEnd] {
if f.mtimeNano > 0 {
batch[idx.relKey(f.path)] = f.mtimeNano
}
}
if len(batch) > 0 {
if err := w.BulkSetFileMtimes(idx.repoPrefix, batch); err != nil {
idx.logger.Warn("indexer: streaming-flush chunk mtime persist failed",
zap.String("repo", idx.repoPrefix), zap.Error(err))
}
}
}
}
// After all chunks, idx.graph points at the disk store so
// the resolver and subpasses read/mutate the merged state.
idx.graph = streamingDisk
} else {
parseChunk(files)
}
// Finalise the content index after the per-file streaming appends so
// its FTS5 segments are merged before the first content query.
if cs := idx.contentSearcher(); cs != nil {
if err := cs.BuildContentIndex(); err != nil {
idx.logger.Warn("indexer: content index build failed", zap.Error(err))
}
}
if processed > 0 {
reporter.Report("parsing", int(processed), totalFiles)
}
// Emit synthetic file nodes for files dropped by the size cap so
// they stay visible in the graph with skip telemetry attached
// instead of vanishing silently.
idx.emitSizeSkipNodes(skippedBySize)
idx.emitContentSkipNodes(skippedByContent)
idx.emitParseFailedSkipNodes(parseFailedFiles)
// Populate fileMtimes for all detected files. Keyed through
// relKey so the mtime map agrees with the graph's file-node keys
// (and with the incremental / git-watcher paths) on the NFC form
// of every non-ASCII filename. Mtimes are the walk-time values
// captured via d.Info(); no per-file os.Stat round-trip here.
idx.mtimeMu.Lock()
idx.fileMtimes = make(map[string]int64, len(files))
for _, f := range files {
if f.mtimeNano > 0 {
idx.fileMtimes[idx.relKey(f.path)] = f.mtimeNano
}
}
mtimeSnapshot := make(map[string]int64, len(idx.fileMtimes))
for k, v := range idx.fileMtimes {
mtimeSnapshot[k] = v
}
idx.mtimeMu.Unlock()
// Persist the per-file mtimes through the store's optional
// FileMtime sidecar table. On the on-disk backend this lets warm
// restarts seed ReconcileRepoCtx without having to read them back
// out of the gob+gzip metadata snapshot; on the in-memory
// backend the capability isn't implemented and the assertion
// short-circuits.
//
// Multi-repo bug: when the shadow-swap path is active, idx.graph
// is the in-memory shadow graph at this point — graph.Graph does
// NOT implement FileMtimeWriter, so the type assertion fails and
// persistence is silently skipped. The actual disk store is
// the local diskTarget variable; checking it first ensures warm-
// restart-skip-reindex actually works. The defer that swaps
// idx.graph back to diskTarget runs LATER, when IndexCtx returns,
// so we can't rely on it here. Falls through to idx.graph for the
// non-shadow path.
mtimeTarget := graph.Store(idx.graph)
if diskTarget != nil {
mtimeTarget = diskTarget
}
// Full-index persist is AUTHORITATIVE: replace the repo's entire mtime
// set so files deleted since the last index are pruned. An upsert-only
// write (BulkSetFileMtimes) leaves deleted-file rows behind, and warm-
// restart reconcile then detects them as phantom deletions on every
// restart — forcing a full re-track that never converges. Prefer the
// replace capability; fall back to upsert for backends without it.
if len(mtimeSnapshot) > 0 {
var perr error
persisted := false
if r, ok := mtimeTarget.(graph.FileMtimeReplacer); ok {
perr, persisted = r.ReplaceFileMtimes(idx.repoPrefix, mtimeSnapshot), true
} else if w, ok := mtimeTarget.(graph.FileMtimeWriter); ok {
perr, persisted = w.BulkSetFileMtimes(idx.repoPrefix, mtimeSnapshot), true
}
if persisted {
if perr != nil {
idx.logger.Warn("persist file mtimes failed",
zap.String("repo", idx.repoPrefix), zap.Error(perr))
} else {
idx.logger.Info("persisted file mtimes",
zap.String("repo", idx.repoPrefix),
zap.Int("count", len(mtimeSnapshot)))
}
}
// Crash-safe content path: reap every content row this walk did NOT
// re-stream. keep is contentStreamedFiles — the files that actually
// produced content sections this run — NOT the surviving-file mtime
// set: a file can survive on disk yet stop yielding content (doc
// emptied, classification changed), and keying keep off mtimes would
// protect its stale rows forever. Recorded keys are the wipe's own
// argument (the node FilePath content_fts carries), so the comparison
// matches the stored rows in single- and multi-repo form alike. A walk
// that streamed NO content falls back to the repo-wide wipe: the repo
// has zero content files now, and the sweep's empty-keep guard (a
// never-wipe-from-empty safety net) would otherwise no-op and leave
// every stale row behind. Only when the per-file wipe path is active;
// on the repo-wide-wipe fallback the up-front pre-wipe already cleared
// both transitions. Runs only under the completed-walk guard above
// (len(mtimeSnapshot) > 0), so a killed parse never triggers it.
if contentWipeFile != nil {
contentStreamedMu.Lock()
keep := contentStreamedFiles
contentStreamedMu.Unlock()
if len(keep) == 0 {
if cs := idx.contentSearcher(); cs != nil {
if err := cs.WipeContent(idx.RepoPrefix()); err != nil {
idx.logger.Warn("indexer: content wipe of contentless repo failed", zap.Error(err))
}
}
} else if sw, ok := idx.contentSearcher().(interface {
DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error
}); ok {
if err := sw.DeleteContentFilesForRepoNotIn(idx.repoPrefix, keep); err != nil {
idx.logger.Warn("indexer: content sweep of stale files failed", zap.Error(err))
}
}
}
}
// Retain parse errors and record index metadata.
idx.parseErrors = errors
idx.totalDetected = len(files)
idx.lastIndexTime = time.Now()
if idx.deferResolve {
// Multi-repo orchestrator runs these serially after wg.Wait()
// to avoid races on the shared graph between this goroutine's
// ResolveAll mutation phase and a sibling goroutine's contract
// pass walking AllEdges. See SetDeferResolve.
idx.pendingContractReg = contractReg
} else {
// Materialise dep::<module> contract nodes from go.mod BEFORE
// ResolveAll so the resolver's import bridge can re-target Go
// imports of declared modules to their dep contract node.
idx.extractGoModContracts(contractReg)
reporter.Report("resolving references", 0, 0)
// Resolve cross-file references.
idx.populateCppIncludeDirs(true)
idx.resolver.ResolveAll()
// Infer structural interface satisfaction + method-level
// overrides. Skipped under deferGlobalPasses so a batch caller
// (warmup, ReconcileAll) can run them once at the end against
// the final shared graph instead of paying the O(global) walk
// per repo. InferOverrides depends on InferImplements running
// first.
if !idx.deferGlobalPasses {
reporter.Report("inferring interfaces", 0, 0)
idx.resolver.InferImplements()
idx.resolver.InferOverrides()
}
// Semantic enrichment (SCIP, go/types, LSP).
if idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders() {
reporter.Report("semantic enrichment", 0, 0)
// Key by the repo prefix so a repo-scoped provider can scope
// file selection to this repo (empty in single-repo mode).
roots := map[string]string{idx.repoPrefix: absRoot}
// The inline full-index path does not gate or persist enrichment
// markers (a zero EnrichOptions): the deferred warmup path owns the
// skip-on-restart optimisation.
results, _, err := idx.semanticMgr.EnrichAll(idx.graph, roots, semantic.EnrichOptions{})
if err != nil {
idx.logger.Warn("semantic enrichment failed", zap.Error(err))
} else if len(results) > 0 {
for _, r := range results {
idx.logger.Info("semantic enrichment result",
zap.String("provider", r.Provider),
zap.String("language", r.Language),
zap.Int("confirmed", r.EdgesConfirmed),
zap.Int("added", r.EdgesAdded),
zap.Int("refuted", r.EdgesRefuted),
zap.Float64("coverage", r.CoveragePercent),
)
}
}
}
}
reporter.Report("building search index", 0, 0)
// Build search index.
idx.buildSearchIndex()
if !idx.deferResolve {
// Contracts were already extracted inline during parse (per file,
// per worker). Here we just finish up. extractGoModContracts
// already ran (see the !deferResolve branch above) so dep
// nodes were available during ResolveAll's import-bridge pass;
// commitContracts is idempotent for those.
reporter.Report("extracting contracts", 0, 0)
idx.extractExternalModules()
idx.extractDIContracts(contractReg)
idx.commitContracts(contractReg)
// Test-edge pass — runs once the call graph is final. Skipped
// under deferGlobalPasses so a batch caller can fold this into
// one global pass after the per-repo loop.
if !idx.deferGlobalPasses {
reporter.Report("test edge pass", 0, 0)
marked, emitted := markTestSymbolsAndEmitEdges(idx.graph)
if marked > 0 || emitted > 0 {
idx.logger.Info("test edges emitted",
zap.Int("test_symbols", marked),
zap.Int("edges", emitted),
)
}
if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 {
idx.logger.Info("capability edges emitted",
zap.Int("reads_env", re),
zap.Int("executes_process", ep),
zap.Int("accesses_field", fa),
)
}
reporter.Report("clone detection pass", 0, 0)
if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
idx.logger.Info("clone edges emitted",
zap.Int("items", cs.Items),
zap.Int("clone_pairs", cs.Pairs),
zap.Int("edges", cs.Edges),
zap.Int("skipped_buckets", cs.SkippedBuckets),
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
zap.Int("diffused_pairs", cs.DiffusedPairs),
zap.Int("diffused_edges", cs.DiffusedEdges),
)
}
// Seed the incremental clone index from the freshly-baselined
// signatures + sidecar so steady-state single-file edits go
// incremental (EvictFuncs/UpdateFuncs) instead of re-running
// this whole-graph pass per file. The batch pass remains the
// re-baseline (corrects CMS drift) and owns diffusion.
if idx.cloneIndex != nil {
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
}
// Framework dynamic-dispatch synthesis — runs once the call
// graph and interface inference are final. Skipped under
// deferGlobalPasses; the batch caller folds it into
// RunGlobalGraphPasses.
reporter.Report("framework dispatch synthesis", 0, 0)
if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 {
idx.logger.Info("framework dispatch calls synthesized",
zap.Int("edges", rep.Total),
zap.Any("per_synthesizer", rep.Per),
)
}
// External-call placeholder synthesis (opt-in) — runs after
// the resolver and stub passes so only genuinely un-indexed
// external targets are left to materialise.
reporter.Report("external-call synthesis", 0, 0)
if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 {
idx.logger.Info("external-call placeholders synthesized",
zap.Int("edges", extCalls),
)
}
if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 {
idx.logger.Info("speculative dispatch edges synthesized",
zap.Int("edges", spec),
)
}
// Reachability index — used to be precomputed for every
// impact seed here. The eager pass was retired because the
// breakeven was untenable on monorepo graphs (k8s:
// ~2000 s build to save ~10 ms per query, ~200 k-query
// breakeven). reach.Lookup now computes the BFS on first
// access per seed and caches the result. The
// InvalidateIndex call bumps the build counter so any
// stale stamps from a prior build (e.g. snapshot reload
// before a partial mutation) no longer shadow the live
// graph state.
reach.InvalidateIndex()
}
}
// Auto-upgrade to Bleve if above threshold. Run in the background
// so the foreground IndexCtx returns immediately — populating
// Bleve with 50k+ symbols takes 30-60s and adding that to the
// initial-index latency was the dominant tail. Searches against
// idx.search keep hitting the in-memory backend until the swap
// completes; nothing observes a half-built Bleve.
//
// upgradeOnce gates the spawn so multi-repo warmup, which calls
// IndexCtx once per tracked repo, doesn't launch one upgrade
// goroutine per post-threshold repo. One per indexer lifetime.
//
// Skip the upgrade when the active search backend is the
// SymbolSearcher adapter: the disk store's native FTS is
// already serving search at engine-native latency, and
// spawning a parallel Bleve build would (a) waste ~100MB heap
// re-indexing the same corpus and (b) silently swap the
// adapter out for Bleve on completion — defeating the whole
// FTS path. The Swappable's current backend tells us which
// branch we're on.
if !isSymbolSearcherBackend(idx.search) && idx.search.Count() >= search.AutoThreshold {
idx.upgradeOnce.Do(func() {
reporter.Report("scheduling search backend upgrade", 0, 0)
idx.upgradeSpawnedMu.Lock()
idx.upgradeSpawned++
idx.upgradeSpawnedMu.Unlock()
// Snapshot upfront so the background goroutine doesn't
// read Node.Meta concurrently with subsequent Index
// calls' Meta-writing passes (reach.BuildIndex,
// ResolveTemporalCalls, ...).
snapshot := idx.snapshotBleveEntries()
go idx.upgradeSearchToBleve(snapshot)
})
}
// Persist the parser quarantine so a file that crashed the parser
// stays skipped across daemon restarts until its content changes.
if quarantine != nil {
if err := quarantine.Save(); err != nil {
idx.logger.Warn("indexer: failed to persist parser quarantine", zap.Error(err))
} else if n := quarantine.Len(); n > 0 {
idx.logger.Info("indexer: parser quarantine", zap.Int("files", n))
}
}
reporter.Report("indexing complete", int(fileCount), len(files))
// Persist the Merkle baseline so the next incremental pass diffs
// against content hashes rather than re-indexing the whole repo.
workspaceFP := ""
if idx.merkleEnabled() {
paths := make([]string, len(files))
for i, wf := range files {
paths[i] = wf.path
}
workspaceFP = idx.saveMerkleBaseline(absRoot, paths)
}
idx.indexGen.Add(1) // invalidate the trigram search cache
nodes, edges := idx.repoNodeEdgeCount()
idx.persistRepoIndexState(diskTarget, absRoot, workspaceFP, nodes, edges)
result = &IndexResult{
NodeCount: nodes,
EdgeCount: edges,
FileCount: int(fileCount),
QuarantinedFiles: quarantine.Len(),
SkippedFiles: len(skippedBySize) + len(skippedByContent) + int(skippedByTimeout) + int(skippedByMinified),
DurationMs: time.Since(start).Milliseconds(),
Errors: errors,
}
idx.warnIfEdgeSanityViolated(result)
// A whole-repo (re-)track that observed files did work: mark the repo for
// deferred semantic enrichment. FullRetrack is stamped by the multi-repo
// caller after this returns, so FileCount carries the signal here.
if result.FileCount > 0 || result.FullRetrack {
idx.pendingEnrich.Store(true)
// IndexCtx re-parsed every file, dropping this repo's hover-enrichment
// edges — force the deferred pass past the completion-marker gate so
// they are restored even at an unchanged clean HEAD.
idx.fullReindexed.Store(true)
}
return result, nil
}
// repoNodeEdgeCount returns this indexer's contribution to the graph,
// scoped to its repoPrefix in multi-repo mode. In single-repo mode
// (empty prefix) every node carries an empty RepoPrefix anyway, so the
// graph totals equal the repo's contribution and we use the cheap
// global accessors. The multi-repo path uses RepoMemoryEstimate which
// walks only this repo's byRepo bucket — O(repo size), not O(graph) —
// so callers that stamp RepoMetadata.NodeCount/EdgeCount no longer
// freeze the workspace-wide total at TrackRepo time.
func (idx *Indexer) repoNodeEdgeCount() (int, int) {
if idx.repoPrefix == "" {
return idx.graph.NodeCount(), idx.graph.EdgeCount()
}
est := idx.graph.RepoMemoryEstimate(idx.repoPrefix)
return est.NodeCount, est.EdgeCount
}
// warnIfEdgeSanityViolated logs a loud warning when an index pass
// produced files and symbol nodes but no edges — see
// IndexResult.EdgeSanityViolated.
func (idx *Indexer) warnIfEdgeSanityViolated(r *IndexResult) {
if r.EdgeSanityViolated() {
idx.logger.Warn("indexer: edge-sanity check failed — index has files and nodes but zero edges; edge extraction likely failed wholesale",
zap.Int("files", r.FileCount),
zap.Int("nodes", r.NodeCount),
zap.Int("edges", r.EdgeCount))
}
}
// IndexFile parses a single file and patches the graph (evict then
// add), including per-file resolver work for cross-file references.
// Use in the single-event fsnotify path where each edit is isolated.
func (idx *Indexer) IndexFile(filePath string) error {
return idx.indexFile(filePath, true)
}
// IndexFileNoResolve is IndexFile minus the per-file resolver call.
// Callers in batch paths (storm mode, branch-switch reconcile, git
// diff dispatch) use this when they will run resolver.ResolveAll()
// once at the end of the batch; otherwise a 500-file checkout pays
// the per-file resolver cost 500 times instead of once.
func (idx *Indexer) IndexFileNoResolve(filePath string) error {
return idx.indexFile(filePath, false)
}
func (idx *Indexer) indexFile(filePath string, resolve bool) error {
absPath, err := filepath.Abs(filePath)
if err != nil {
return err
}
// relKey gives the canonical key (slash form, Unicode NFC). Using
// it here keeps an incremental re-index — which may be driven by a
// git-watcher path in NFC or an FSEvents path in NFD — landing on
// the exact graph key the bulk walk created, so the evict below
// finds the file's existing nodes instead of leaking a duplicate.
relPath := idx.relKey(absPath)
// In multi-repo mode, the graph stores prefixed file paths.
graphPath := idx.prefixPath(relPath)
// Parse-then-swap: we must NOT evict the file's existing nodes/edges
// and search entries until we hold a usable parse result. Evicting
// first leaves the file at zero nodes whenever the on-disk bytes are
// transiently unparseable (a save mid-edit) — a failed extraction
// then returns early and the symbols stay nuked. Capturing the old
// state up front and deferring the actual eviction to evictExisting()
// keeps the file stale-but-present on failure (stale beats empty) and
// shrinks the no-nodes window to the gap between evict and AddBatch.
//
// oldFuncIDs holds this file's function/method node IDs so the
// incremental clone index can drop their CMS/LSH contributions —
// EvictFile removes the nodes (and their clone_sig) from the graph,
// so it must be captured before evictExisting runs.
var oldFuncIDs []string
evictExisting := func() {
for _, n := range idx.graph.GetFileNodes(graphPath) {
if n.Kind != graph.KindFile && n.Kind != graph.KindImport {
idx.search.Remove(n.ID)
}
if n.Kind == graph.KindFunction || n.Kind == graph.KindMethod {
oldFuncIDs = append(oldFuncIDs, n.ID)
}
}
idx.restubIncomingRefs(graphPath)
idx.graph.EvictFile(graphPath)
}
src, err := os.ReadFile(absPath)
if err != nil {
return err
}
lang, ok := idx.effectiveLanguage(absPath, src)
if !ok {
return nil
}
ext, _ := idx.registry.GetByLanguage(lang)
if ext == nil {
return nil
}
// Honour the size cap on the incremental path too: an over-cap
// file gets a synthetic skip node, not a parse — matching the
// bulk IndexCtx walk. This IS a successful result, so it evicts the
// prior state and installs the synthetic node, same as before.
if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize {
n := sizeSkipNode(skippedFile{
relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)),
}, maxSize)
idx.applyRepoPrefix([]*graph.Node{n}, nil)
evictExisting()
idx.graph.AddBatch([]*graph.Node{n}, nil)
return nil
}
// Honour the content-admission gate on the incremental path too, so a
// document over its cap (or a data asset, by default) the cold walk
// would skip doesn't get parsed back in when the watcher re-indexes it
// — same synthetic-skip-node treatment as the size cap above.
if reason, skip := idx.newContentAdmissionGate().skip(lang, int64(len(src))); skip {
n := contentSkipNode(skippedFile{
relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)), reason: reason,
})
idx.applyRepoPrefix([]*graph.Node{n}, nil)
evictExisting()
idx.graph.AddBatch([]*graph.Node{n}, nil)
return nil
}
// Pre-ingestion transforms — same pipeline as the bulk path.
src = idx.transforms.run(relPath, src)
// Crash isolation for the incremental path: a file the user just
// saved that SIGSEGVs the parser is quarantined instead of taking
// the daemon down with it. The pool is long-lived and shared, so
// the watcher hot path never forks a worker subprocess per file.
var pool *crashpool.Pool
var quarantine *crashpool.Quarantine
if idx.crashIsolationEnabled() {
pool, quarantine = idx.sharedParsePool()
}
result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src)
if quarantine != nil && quarantine.Len() > 0 {
_ = quarantine.Save()
}
if result == nil {
// No usable parse result (transient parse failure, quarantine,
// timeout). Do NOT evict — the file's prior nodes/edges/search
// entries stay intact. A stale-but-present file beats an empty
// one, and the next successful re-index swaps cleanly.
//
// The bytes were read successfully (src above), so this is a
// stable fact about the file's current on-disk content, not a
// transient "couldn't even open it" failure (that case returns
// earlier, before relPath/mtime bookkeeping, and deliberately
// leaves the mtime unrecorded so it keeps retrying). Recording
// the mtime here keeps a warm restart's HasChangesSinceMtimes
// from perpetually seeing this one unparseable file as "the
// repo changed" and routing the whole repo through the
// expensive shadow re-track path on every restart.
idx.recordFileMtime(relPath, absPath)
return err
}
// Affected-by snapshot: the symbol shapes and reverse-reference
// sources the post-resolve signature-delta pass compares against,
// captured BEFORE eviction — EvictFile drops in-edges from
// unchanged files and replaces this file's nodes, so neither is
// recoverable afterwards. Skipped on the no-resolve and
// deferred-batch paths, whose callers run a full resolve (and
// persistAllRefFacts) once at the end of the batch.
//
// Also skipped for a quarantined / timed-out / minified-skipped file:
// its synthetic result carries zero symbols, so the delta would read
// every prior symbol as removed and fan out to re-resolve the whole
// reverse graph on a transient parse failure. A failure that yields no
// symbols is not the same as a symbol genuinely deleted from source.
var abSnap *affectedBySnapshot
var reuseIdx map[reuseKey]*reuseVal
var priorUnresolved []*graph.Edge
if resolve && !idx.deferGlobalPasses && !skipped {
abSnap = idx.snapshotAffectedBy(graphPath)
// Snapshot the file's outgoing edges before eviction: resolved ones so
// the re-parse recovers unchanged resolutions (reuseIdx), and the ones
// still unresolved so the forward pass can skip re-trying them
// (priorUnresolved). Together this makes a save re-resolve only the
// references it actually changed instead of the whole file.
reuseIdx, priorUnresolved = captureIncrementalState(idx.graph, graphPath)
}
// We hold a usable result: evict the old state now, then add the
// new — the window where the file has no nodes is just this gap.
evictExisting()
// Coverage extractors (todos, licenses, ownership). Same call
// site exists in the bulk IndexCtx worker pool — see
// applyCoverageDomains. Skipped for a quarantined / timed-out file.
if !skipped {
idx.applyCoverageDomains(relPath, lang, src, result)
}
idx.applyRepoPrefix(result.Nodes, result.Edges)
// Reuse prior resolutions for edges whose source-side shape is unchanged
// (the common case on a small edit), so the resolver below only handles
// genuinely-new references instead of re-resolving the whole file. The
// about-to-be-added node IDs let the reuse recover same-file targets that
// eviction removed and this AddBatch re-adds under identical IDs — without
// them a same-file call's resolution + tier is lost to a full re-resolve on
// every structural save.
newNodeIDs := make(map[string]struct{}, len(result.Nodes))
for _, n := range result.Nodes {
if n != nil {
newNodeIDs[n.ID] = struct{}{}
}
}
if reused := applyResolvedOutEdges(idx.graph, result.Edges, reuseIdx, newNodeIDs); reused > 0 {
idx.logger.Debug("indexer: reused prior resolutions",
zap.Int("edges", reused), zap.String("file", graphPath))
}
// Content (incremental): clear this file's prior content rows, then
// re-stream + lean — mirrors the full-index per-file path so an edited
// content file leaves no stale rows and doesn't revert to full text on
// the node.
if cs := idx.contentSearcher(); cs != nil {
if fp := firstContentFilePath(result.Nodes); fp != "" {
if err := cs.WipeContentFile(fp); err != nil {
idx.logger.Warn("indexer: content index file wipe failed", zap.Error(err))
}
}
}
idx.streamContentSections(result.Nodes)
idx.graph.AddBatch(result.Nodes, result.Edges)
idx.persistConstValues(result)
idx.persistFileMeta(relPath, src, result)
// Add new symbols to search index. shouldIndexForSearch enforces
// the same SkipSearch filter used by the bulk and upgrade paths.
// When the backing store implements graph.SymbolSearcher we
// also mirror each upsert into its native FTS, so an
// incremental reindex doesn't fall out of sync with the
// bulk-built corpus.
searcher, _ := idx.graph.(graph.SymbolSearcher)
for _, n := range result.Nodes {
if !idx.shouldIndexForSearch(n) {
continue
}
idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...)
if searcher != nil {
if err := searcher.UpsertSymbolFTS(n.ID, ftsTokensFor(n, idx.projectName)); err != nil {
idx.logger.Debug("indexer: backend FTS upsert failed",
zap.String("id", n.ID),
zap.Error(err))
}
}
}
if resolve {
// Forward pass (this file's outgoing references) plus the
// reverse pass binding callers in OTHER files that reference a
// symbol (re)defined here — a symbol newly defined or changed
// here leaves callers elsewhere pointing at the unresolved stub
// restubIncomingRefs left when the prior concrete node was
// evicted. Scoped to this file's names — not a whole-graph
// ResolveAll — and run as one combined pass so the resolver's
// per-pass indexes are built once per save, not twice.
//
// Skip re-resolving references that were already unresolved before the
// edit and are unchanged: they stay parked on their stubs for the
// incoming pass, so a small edit to a reference-heavy file no longer
// re-runs the candidate cascade on thousands of stdlib/external calls.
idx.resolver.SetIncrementalSkip(priorUnresolved)
idx.resolver.ResolveFileAndIncoming(graphPath)
idx.resolver.SetIncrementalSkip(nil)
// CPG-lite dataflow placeholders for this file: inter-
// procedural callees may have just been lifted by
// ResolveFile, so re-run the dataflow materialisation pass
// to keep arg_of / returns_to edges in sync with the
// freshly resolved EdgeCalls graph. Scoped to this file's
// out-edges — not a whole-graph AllEdges scan — so an
// incremental edit stays O(file), not O(all edges).
idx.materializeDataflowParamsForFile(graphPath, result.Edges)
// Clone detection. EvictFile above removed this file's
// EdgeSimilarTo edges in both directions. When the incremental
// clone index is built, re-bank just this file's bodies
// (EvictFuncs the old ids, UpdateFuncs the fresh nodes) — an
// O(edited file) update that restores the same edge set the
// whole-graph pass would. Until a batch/global pass has seeded
// the index (built=false) we fall back to the full recompute.
// Skipped under deferGlobalPasses — a batch caller (ReconcileAll,
// warmup) runs the global pass once at the end.
if !idx.deferGlobalPasses {
if idx.cloneIndex != nil {
// Seed the incremental clone index on first use — e.g. after a
// warm restart that loaded the graph from disk without a full
// re-index, `built` is false. Without this we'd fall to the
// whole-graph recompute on EVERY save; instead pay it once and
// then run the O(edited file) update.
if !idx.cloneIndex.built {
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
}
idx.cloneIndex.EvictFuncs(idx.graph, oldFuncIDs)
idx.cloneIndex.UpdateFuncs(idx.graph, idx.repoPrefix, cloneFuncNodes(result.Nodes), idx.cloneThreshold())
} else {
detectClonesAndEmitEdges(idx.graph, idx.repoPrefix, idx.cloneThreshold())
}
}
// in-memory backend. Skipped for a quarantined / timed-out /
// minified file: its synthetic result yields no facts, so a
// delete-then-set would durably drop the file's real facts on a
// transient parse failure and leave them gone until a clean
// reparse — abSnap is nil here too, so the affected-by pass that
// would also fan out is already a no-op.
if !skipped {
idx.persistRefFactsForFiles([]string{graphPath})
// Affected-by re-resolution: if this save changed a symbol's
// signature or kind, or removed a symbol, re-resolve the files
// that referenced it — bounded, synchronous, and gated on the
// signature delta so a body-only edit fans out to nothing.
idx.reresolveAffectedBy(graphPath, abSnap, result.Nodes)
// Incremental semantic enrichment for this single file. Mirrors the
// full-index EnrichAll call but scoped to the saved file, so a
// watcher save re-runs the type resolvers (and any watch-enabled
// LSP / compiler provider) instead of leaving the file's edges at
// their pre-enrichment tier until the next full reindex. Gated
// internally on Config.EnrichOnWatch; a no-op when disabled.
providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders()
reEnriched := false
if providersPresent {
if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil {
idx.logger.Debug("indexer: incremental semantic enrichment failed",
zap.String("file", graphPath),
zap.Error(err))
} else {
reEnriched = idx.semanticMgr.EnrichesOnWatch()
}
}
// Record whether this live re-parse left the file below the
// enrichment tier: providers exist (so there IS an lsp/ast tier to
// fall short of) but the save did not re-run enrichment. When set,
// find_usages / get_callers flag their default text_matched
// suppression as re-verification-pending so a hidden-but-real usage
// is diagnosable rather than silently dropped. Cleared when
// enrichment did re-run for the file.
idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched)
}
}
// Update mtime for this file. relPath is already the canonical
// key (relKey applied slash + NFC), so the mtime entry lines up
// with the graph file-node key and with the bulk-walk mtimes.
idx.recordFileMtime(relPath, absPath)
return nil
}
// recordFileMtime restamps the recorded mtime for relPath (a canonical
// relKey, not repo-prefixed) from absPath's current on-disk mtime, both
// in the in-memory map and — when the backend supports it — the store's
// FileMtime sidecar. Per-file write is ~1ms on the on-disk backend;
// trivial under steady-state file-watcher load. A missing/unstatable
// file is a no-op; a persist error is logged but non-fatal, since the
// in-memory map (which the current process trusts) is already correct.
func (idx *Indexer) recordFileMtime(relPath, absPath string) {
info, err := os.Stat(absPath)
if err != nil {
return
}
mtime := info.ModTime().UnixNano()
idx.mtimeMu.Lock()
idx.fileMtimes[relPath] = mtime
idx.mtimeMu.Unlock()
if w, ok := idx.graph.(graph.FileMtimeWriter); ok {
if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{relPath: mtime}); err != nil {
idx.logger.Warn("persist file mtime failed",
zap.String("repo", idx.repoPrefix), zap.String("file", relPath), zap.Error(err))
}
}
}
// StructuralSymbols parses a file from its current on-disk content and
// returns the structural symbols it defines — functions, methods,
// types, interfaces, constants, variables, fields, enum members — and
// nothing else. It is a read-only probe: the graph and the search
// index are left completely untouched, no mtime is stamped, and no
// resolver runs. The watcher uses it to decide whether a save is
// structurally inert (a comment / whitespace / config-value edit that
// changes no symbol) and can skip the destructive evict + reindex.
//
// The second return reports whether the file was parseable at all: a
// file with no detectable language, an over-cap file, or a read error
// yields (nil, false). A genuinely empty source file yields
// (empty-slice, true).
func (idx *Indexer) StructuralSymbols(filePath string) ([]*graph.Node, bool) {
absPath, err := filepath.Abs(filePath)
if err != nil {
return nil, false
}
relPath, err := filepath.Rel(idx.rootPath, absPath)
if err != nil {
relPath = filePath
}
src, err := os.ReadFile(absPath)
if err != nil {
return nil, false
}
lang, ok := idx.effectiveLanguage(absPath, src)
if !ok {
return nil, false
}
ext, _ := idx.registry.GetByLanguage(lang)
if ext == nil {
return nil, false
}
// An over-cap file is never structurally parsed on the indexing
// path either (it gets a synthetic skip node), so the watcher
// cannot prove inertness for it — fall through to a real reindex.
if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize {
return nil, false
}
// Same pre-ingestion transforms as indexFile so the probe parses
// exactly the bytes the real index pass would.
src = idx.transforms.run(relPath, src)
var pool *crashpool.Pool
var quarantine *crashpool.Quarantine
if idx.crashIsolationEnabled() {
pool, quarantine = idx.sharedParsePool()
}
result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src)
if quarantine != nil && quarantine.Len() > 0 {
_ = quarantine.Save()
}
// A skipped (quarantined / timed-out) file produces only a
// synthetic node — not the real symbol set — so inertness cannot
// be proven and the caller must reindex normally.
if result == nil || skipped || err != nil {
return nil, false
}
out := make([]*graph.Node, 0, len(result.Nodes))
for _, n := range result.Nodes {
if isStructuralKind(n.Kind) {
out = append(out, n)
}
}
return out, true
}
// isStructuralKind reports whether a node kind represents a structural
// code symbol — the kinds whose presence, name, or signature define a
// file's graph shape. File and import nodes (graph bookkeeping),
// params, closures, and the coverage-domain kinds (todos, licenses,
// strings, …) are deliberately excluded: a change confined to those
// does not alter the structural graph the watcher cares about.
func isStructuralKind(k graph.NodeKind) bool {
switch k {
case graph.KindFunction, graph.KindMethod, graph.KindType,
graph.KindInterface, graph.KindVariable, graph.KindConstant,
graph.KindField, graph.KindEnumMember:
return true
default:
return false
}
}
// ResolveAll re-runs the global cross-file reference resolver and
// interface-implementation inference. Exposed for batch paths that
// defer per-file resolver work until the end of a batch.
func (idx *Indexer) ResolveAll() {
idx.populateCppIncludeDirs(true)
idx.resolver.ResolveAll()
idx.resolver.InferImplements()
idx.resolver.InferOverrides()
// Framework dynamic-dispatch synthesis (gRPC / Temporal / event
// channels / native bridges) depends on InferImplements (the
// interface-satisfaction signals) having run first.
resolver.RunFrameworkSynthesizers(idx.graph)
// External-call placeholder synthesis (opt-in) — runs after the
// resolver and stub passes so only genuinely un-indexed external
// targets remain to materialise.
resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled())
resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled())
// CPG-lite dataflow rewriting must run after the call resolver
// has lifted unresolved:: targets; arg_of edges then point at
// real function/method nodes whose param nodes can be found,
// and returns_to placeholders join cleanly against the
// now-resolved EdgeCalls edge at the same caller+line.
idx.materializeDataflowParams()
// Seed the durable reference-facts sidecar from the fully-resolved graph
// (no-op on the in-memory backend).
idx.persistAllRefFacts()
}
// EvictFile removes all nodes and edges belonging to filePath.
//
// filePath may arrive in any Unicode form — the git watcher derives it
// from `git diff` output (NFC), while an FSEvents-driven evict carries
// the filesystem's form (NFD on macOS). relKey folds both to the
// canonical NFC key the graph indexed the file under, so the eviction
// actually finds the file's nodes rather than silently no-opping and
// leaving a stale subtree behind.
func (idx *Indexer) EvictFile(filePath string) (int, int) {
absPath := filePath
if !filepath.IsAbs(absPath) {
absPath = filepath.Join(idx.rootPath, filePath)
}
relPath := idx.relKey(absPath)
// In multi-repo mode, the graph stores prefixed file paths.
graphPath := idx.prefixPath(relPath)
// Remove from search index.
for _, n := range idx.graph.GetFileNodes(graphPath) {
if n.Kind != graph.KindFile && n.Kind != graph.KindImport {
idx.search.Remove(n.ID)
}
}
idx.restubIncomingRefs(graphPath)
idx.evictEnrichment(graphPath)
idx.deleteRefFactsForFiles(idx.repoPrefix, []string{graphPath})
return idx.graph.EvictFile(graphPath)
}
// ReresolveFileScoped forces the scoped re-resolution + LSP re-verify a normal
// IndexFile performs, WITHOUT re-parsing and WITHOUT the IsStale gate — used by
// the watcher's shape-degradation self-heal, where the file's mtime is already
// current (IndexFile just ran) yet its resolved edges came out degraded, so a
// plain IncrementalReindexPaths would stale-gate it out. Re-runs only this
// file's forward + incoming resolve and, when a watch-enabled provider is
// wired, its incremental enrichment. O(file), no whole-graph pass. No-op when
// the file has no nodes (evicted since it was enqueued).
func (idx *Indexer) ReresolveFileScoped(filePath string) error {
absPath := filePath
if !filepath.IsAbs(absPath) {
absPath = filepath.Join(idx.rootPath, filePath)
}
graphPath := idx.prefixPath(idx.relKey(absPath))
if len(idx.graph.GetFileNodes(graphPath)) == 0 {
return nil // file gone / evicted; nothing to re-resolve
}
idx.resolver.ResolveFileAndIncoming(graphPath)
providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders()
reEnriched := false
if providersPresent {
if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil {
idx.logger.Debug("indexer: forced scoped enrichment failed",
zap.String("file", graphPath), zap.Error(err))
} else {
reEnriched = idx.semanticMgr.EnrichesOnWatch()
}
}
// Keep the find_usages staleness marker consistent with the enrichment
// that actually ran during this forced re-resolve (mirrors indexFile).
idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched)
return nil
}
// restubIncomingRefs rewrites every resolved reference edge that points
// INTO a symbol of graphPath from a surviving (other-file) source back
// to an `unresolved::<Name>` stub, in place, BEFORE the file's nodes are
// evicted. Graph eviction otherwise drops those incoming caller edges
// wholesale (it removes the edge from the surviving source's out-edge
// bucket) and nothing recreates them until a cold reindex — so editing
// or deleting a definition silently strips its callers' edges and
// find_usages / get_callers go blank. Re-stubbing detaches the edges
// from the soon-to-be-evicted nodes so they survive as pending stubs;
// ResolveIncomingForFile (after a re-index) rebinds them to the file's
// fresh symbols, or they stay unresolved — the correct state once the
// symbol is gone. Only name-resolvable reference kinds are re-stubbed;
// structural and enrichment edges are left to be dropped. Backend-
// agnostic: GetInEdges + ReindexEdges are the same Store primitives the
// resolver uses, so this behaves identically on the in-memory and disk
// stores.
// evictEnrichment drops the per-node enrichment sidecar rows (churn,
// coverage, release, blame — change A) for a file's nodes on the
// delete/rename paths only, so a removed file leaves no orphan
// enrichment. Capability-gated. A modify re-indexes the same node IDs
// (enrichment stays valid) so it is NOT cascaded there.
func (idx *Indexer) evictEnrichment(graphPath string) {
nodes := idx.graph.GetFileNodes(graphPath)
if len(nodes) == 0 {
return
}
ids := make([]string, 0, len(nodes))
for _, n := range nodes {
ids = append(ids, n.ID)
}
if w, ok := idx.graph.(graph.ChurnEnrichmentWriter); ok {
_ = w.DeleteChurn(ids)
}
if w, ok := idx.graph.(graph.CoverageEnrichmentWriter); ok {
_ = w.DeleteCoverage(ids)
}
if w, ok := idx.graph.(graph.ReleaseEnrichmentWriter); ok {
_ = w.DeleteReleases(ids)
}
if w, ok := idx.graph.(graph.BlameEnrichmentWriter); ok {
_ = w.DeleteBlame(ids)
}
}
func (idx *Indexer) restubIncomingRefs(graphPath string) {
nodes := idx.graph.GetFileNodes(graphPath)
if len(nodes) == 0 {
return
}
evicted := make(map[string]struct{}, len(nodes))
for _, n := range nodes {
evicted[n.ID] = struct{}{}
}
var batch []graph.EdgeReindex
for _, n := range nodes {
if n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) {
continue
}
stub := graph.UnresolvedMarker + n.Name
for _, e := range idx.graph.GetInEdges(n.ID) {
if e == nil || !graph.IsResolvableRefEdge(e.Kind) {
continue
}
if _, fromEvicted := evicted[e.From]; fromEvicted {
continue // intra-file edge: the source is evicted too
}
if graph.IsUnresolvedTarget(e.To) {
continue // already a pending stub
}
oldTo := e.To
// Stash + clear the edge's resolved provenance before restubbing:
// an `unresolved::` stub must not keep advertising a resolved
// tier. The incoming-resolve pass restores it verbatim if the stub
// rebinds to the same target (idempotent re-parse); a deleted or
// moved target leaves the stub honestly unresolved.
graph.StashRestubProvenance(e)
e.To = stub
batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo})
}
}
if len(batch) > 0 {
idx.graph.ReindexEdges(batch)
}
}
// embeddingDimsOrDefault returns the embedder's reported vector width,
// falling back to a neutral placeholder only when the provider cannot
// state its width yet (Dimensions() == 0, the APIProvider-before-first-
// call case). The fallback is never persisted: buildSearchIndex and
// ImportVectorIndex both overwrite it with the true width taken from a
// real vector / the cached header. Kept as a named helper so the
// vector-dimension default has one definition instead of a scattered
// magic number.
func embeddingDimsOrDefault(p embedding.Provider) int {
if p == nil {
return 0
}
if d := p.Dimensions(); d > 0 {
return d
}
// Provider has not committed to a width. 384 matches the default
// transformer backend (MiniLM-L6-v2); a static GloVe provider
// always reports its real 50 and never reaches this branch.
return 384
}
// collectEmbedTexts walks the nodes and produces the parallel texts /
// ids slices the embedding pass consumes, plus a chunkMap recording
// which synthetic IDs are chunks of which symbol.
//
// A symbol whose source span exceeds the configured chunk threshold is
// read from disk and split into AST windows by embedding.ChunkSymbol;
// each window contributes one text (its body, prefixed with the
// symbol's kind + name for a little lexical grounding) under a
// synthetic ID "<symbolID>#chunkK", and chunkMap[syntheticID] = symbolID.
// A symbol below the threshold — or one whose file can't be read —
// contributes a single metadata text under its own ID, exactly as the
// pre-chunking pipeline did. The returned skipped count is the number
// of nodes dropped by the SkipEmbed rules.
func (idx *Indexer) collectEmbedTexts(nodes []*graph.Node) (texts []string, ids []string, chunkMap map[string]string, skipped int) {
chunkMap = make(map[string]string)
opts := idx.embedChunkOpts
threshold := opts.ThresholdLines
if threshold <= 0 {
threshold = embedding.DefaultChunkThresholdLines
}
// fileCache memoizes one os.ReadFile per source file — many symbols
// share a file, and the chunker only needs the bytes once.
fileCache := make(map[string][]byte)
readFile := func(graphPath string) []byte {
if cached, ok := fileCache[graphPath]; ok {
return cached
}
var data []byte
if abs := idx.ResolveFilePath(graphPath); abs != "" {
if b, err := os.ReadFile(abs); err == nil {
data = b
}
}
fileCache[graphPath] = data // cache misses too (nil) — don't re-stat
return data
}
for _, n := range nodes {
if n.Kind == graph.KindFile || n.Kind == graph.KindImport {
continue
}
// CONTENT section bodies are served by the content index, not the
// vector store — excluding them keeps the embed-text count (and the
// 100k auto-disable check) code-only, so a content-heavy repo no
// longer drowns the embedding pass in hundreds of thousands of
// section texts.
if isContentNode(n) {
skipped++
continue
}
if config.ShouldSkipEmbed(idx.config.SkipEmbed, n.Language, string(n.Kind)) {
skipped++
continue
}
sig, _ := n.Meta["signature"].(string)
metaText := fmt.Sprintf("%s %s %s %s", n.Kind, n.Name, sig, n.FilePath)
// Decide whether to sub-chunk: the symbol must declare a
// multi-line span past the threshold and its file must be
// readable. Anything else falls back to the metadata vector.
span := n.EndLine - n.StartLine + 1
body := extractSymbolBody(n, readFile, threshold)
if span <= threshold || len(body) == 0 {
texts = append(texts, metaText)
ids = append(ids, n.ID)
continue
}
windows := embedding.ChunkSymbol(body, n.Language, n.ID, opts)
if len(windows) <= 1 {
// The chunker decided one window was enough (short body,
// no splitter, parse failure) — embed it as a single
// metadata + body vector under the symbol's own ID.
texts = append(texts, metaText+" "+windows[0].Text)
ids = append(ids, n.ID)
continue
}
for _, w := range windows {
chunkID := fmt.Sprintf("%s#chunk%d", n.ID, w.WindowIndex)
texts = append(texts, fmt.Sprintf("%s %s %s", n.Kind, n.Name, w.Text))
ids = append(ids, chunkID)
chunkMap[chunkID] = n.ID
}
}
return texts, ids, chunkMap, skipped
}
// extractSymbolBody returns the source text of a symbol's span, read
// from its file via readFile and sliced by the node's 1-based
// StartLine..EndLine. Returns nil when the file is unreadable, the
// line range is unusable, or the symbol is at or below the threshold
// (small symbols never need their body — the caller embeds metadata).
func extractSymbolBody(n *graph.Node, readFile func(string) []byte, threshold int) []byte {
if n.StartLine <= 0 || n.EndLine < n.StartLine {
return nil
}
if n.EndLine-n.StartLine+1 <= threshold {
return nil
}
data := readFile(n.FilePath)
if len(data) == 0 {
return nil
}
return sliceLines(data, n.StartLine, n.EndLine)
}
// sliceLines returns the bytes of the 1-based inclusive line range
// [start,end] of src. An out-of-range request is clamped; an empty
// result is returned for a range that lands entirely past EOF.
func sliceLines(src []byte, start, end int) []byte {
if start < 1 {
start = 1
}
line := 1
startByte := -1
endByte := len(src)
for i := 0; i < len(src); i++ {
if line == start && startByte < 0 {
startByte = i
}
if src[i] == '\n' {
line++
if line == end+1 {
endByte = i + 1 // include the trailing newline
break
}
}
}
if startByte < 0 {
return nil
}
if endByte < startByte {
endByte = len(src)
}
return src[startByte:endByte]
}
// defaultEmbedAPIConcurrency bounds parallel embedding requests
// against an API-backed embedder when embedding.api_concurrency is
// unset. Four is a conservative default that overlaps round-trips
// without tripping typical hosted-API rate limits.
const defaultEmbedAPIConcurrency = 4
// embedChunkBatch is one unit of work for the embedding pool: the
// texts of one chunk plus the index that fixes where its vectors land
// in the result slice. Carrying the index makes completion order
// irrelevant — workers write by index, never append.
type embedChunkBatch struct {
index int
texts []string
}
// embedAllChunks embeds every text, returning the vectors in the same
// order as texts. The work is split into batches of batchSize texts.
//
// For an API-backed embedder the batches run through a bounded worker
// pool — a hosted embedding round-trip dominates index time, so
// overlapping requests is a real speedup. In-process embedders
// (Hugot / ONNX / GoMLX / static) serialise on an inference mutex, so
// concurrency buys them nothing and they keep the simple serial path.
//
// The abort-on-any-error contract is preserved in both modes: the
// first batch failure cancels the group and embedAllChunks returns the
// error with no partial result, exactly as the old serial loop did.
// embedFn already layers the deadline-halving retry on top of each
// batch.
func (idx *Indexer) embedAllChunks(
texts []string,
batchSize int,
embedFn func(ctx context.Context, items []string) ([][]float32, error),
) ([][]float32, error) {
if len(texts) == 0 {
return nil, nil
}
// Split into batches up front so both the serial and parallel
// paths iterate the same units.
var batches []embedChunkBatch
for start := 0; start < len(texts); start += batchSize {
end := start + batchSize
if end > len(texts) {
end = len(texts)
}
batches = append(batches, embedChunkBatch{index: len(batches), texts: texts[start:end]})
}
// Per-batch result slots, pre-sized so workers write by index and
// completion order never matters.
results := make([][][]float32, len(batches))
// Only run the pool for an embedder that declares itself safe and
// worthwhile to call concurrently — the API-backed provider, where
// overlapped HTTP round-trips are a real win. In-process backends
// (Hugot / ONNX / GoMLX) hold an inference mutex, so a pool would
// only add scheduling overhead; they keep the serial path.
apiBacked := false
if c, ok := idx.embedder.(interface{ Concurrent() bool }); ok {
apiBacked = c.Concurrent()
}
concurrency := idx.embedAPIConcurrency
if concurrency <= 0 {
concurrency = defaultEmbedAPIConcurrency
}
if concurrency > len(batches) {
concurrency = len(batches)
}
if !apiBacked || concurrency <= 1 {
// Serial path — unchanged behaviour for in-process embedders.
ctx := context.Background()
for _, b := range batches {
vecs, err := embedFn(ctx, b.texts)
if err != nil {
return nil, err
}
results[b.index] = vecs
}
return flattenEmbedResults(results), nil
}
// Parallel path — bounded worker pool for an API-backed embedder.
// A cancellable group context means the first failure stops every
// in-flight worker; the indexer's existing per-batch retry still
// runs underneath embedFn.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
jobs := make(chan embedChunkBatch)
var (
wg sync.WaitGroup
errOnce sync.Once
firstErr error
)
fail := func(err error) {
errOnce.Do(func() {
firstErr = err
cancel() // abort siblings on the first error
})
}
for w := 0; w < concurrency; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for b := range jobs {
if ctx.Err() != nil {
return // group already aborted
}
vecs, err := embedFn(ctx, b.texts)
if err != nil {
fail(err)
return
}
// Write into the pre-sized slot — no shared append, so
// no lock and order is fixed by b.index.
results[b.index] = vecs
}
}()
}
idx.logger.Info("embedding vector index with a concurrent API pool",
zap.Int("workers", concurrency),
zap.Int("batches", len(batches)))
for _, b := range batches {
if ctx.Err() != nil {
break // stop feeding once aborted
}
select {
case jobs <- b:
case <-ctx.Done():
}
}
close(jobs)
wg.Wait()
if firstErr != nil {
return nil, firstErr
}
return flattenEmbedResults(results), nil
}
// flattenEmbedResults concatenates per-batch vector slices back into a
// single slice aligned with the original texts order.
func flattenEmbedResults(results [][][]float32) [][]float32 {
total := 0
for _, r := range results {
total += len(r)
}
out := make([][]float32, 0, total)
for _, r := range results {
out = append(out, r...)
}
return out
}
// buildSearchIndex populates the search backend from the current graph.
// When an embedder is set, also builds a vector index and wraps both
// in a HybridBackend with RRF fusion.
//
// In multi-repo mode the search backend is shared across every repo
// (Indexer.search is wired to MultiIndexer.search at construction).
// Walking g.AllNodes() and re-Add()ing every node would mean each
// freshly-tracked repo pays an O(workspace) re-index pass over all
// previously-tracked repos' nodes — quadratic in repo count and the
// dominant cost of warming up a 260-repo workspace. So when this
// indexer carries a non-empty repoPrefix we walk only that repo's
// byRepo bucket; the other repos' entries are already in the shared
// backend from when they were tracked. Single-repo mode keeps the
// AllNodes() path because nodes there carry an empty RepoPrefix and
// GetRepoNodes("") would miss them.
func (idx *Indexer) buildSearchIndex() {
// Start every build from a clean vector-build error: the degraded vector
// paths below set it, and a successful build (or a benign skip / no embedder)
// leaves it nil, so LastVectorBuildError always reflects the current pass.
idx.lastVectorBuildErr = nil
// Code-only enumeration: content (data_class=content) sections live in
// the content index, never the symbol search or the vector store, so the
// FTS loop below and collectEmbedTexts both skip them anyway. Fetching
// the non-content set up front means a content-heavy repo's hundreds of
// thousands of sections never enter memory here (the disk backend filters
// them in SQL), instead of being materialised only to be skipped.
nodes := graph.RepoCodeNodes(idx.graph, idx.repoPrefix)
// Install the learned sub-word boundary table on the BM25 layer
// before populating postings, so the optional sparse-ngram
// tokenizer's split points are data-driven and — crucially —
// identical on the index path here and the query path later. The
// table must be in place before the first Add for postings and
// queries to agree; it is a no-op for non-BM25 backends and a no-op
// for search overall unless GORTEX_SPARSE_NGRAM is set.
search.InstallNgramBoundaries(idx.search, search.BuildNgramBoundaries(idx.graph))
// Build text index. The SkipSearch filter (wired through
// idx.shouldIndexForSearch) drops config-key-style variable nodes
// that would only pad the index — see docs on IndexConfig.SkipSearch.
for _, n := range nodes {
if !idx.shouldIndexForSearch(n) {
continue
}
idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...)
}
// Build vector index if embedder is available.
if idx.embedder == nil {
return
}
// skipVectorBuild short-circuits the embedding pass: the text index
// above is fully populated, but a caller (the daemon warmup loop
// after a snapshot restore) has signalled that the workspace vector
// index will be supplied separately, so re-embedding here would be
// wasted work immediately overwritten by the cached index.
if idx.skipVectorBuild {
return
}
// Provisional dimensionality: trust the embedder's own report.
// A provider that can't state its width yet (an APIProvider before
// its first call returns 0) gets a neutral placeholder — the value
// is overwritten below from the first real vector, so it never
// reaches the persisted index. The old hard-coded 300 was wrong for
// the default static GloVe provider (50d) and misrepresented the
// index width in the interim; deriving from Dimensions() keeps it
// honest for every provider.
dims := embeddingDimsOrDefault(idx.embedder)
// Collect texts and IDs for batch embedding. Nodes matching
// Semantic.SkipEmbed (e.g. CSS custom properties, terraform blocks,
// YAML/TOML/shell config vars) are kept in the text index but
// excluded from the vector index — embedding them is pure cost
// with no semantic payoff and on big monorepos dominates RAM.
//
// A symbol whose source span exceeds the chunk threshold is split
// into AST windows: each window is embedded as its own vector under
// a synthetic ID ("<symbolID>#chunkK"), and chunkMap records the
// chunk → parent mapping so query-time de-chunking maps a chunk hit
// back to the symbol. A small symbol stays a single metadata-only
// vector under its own ID. chunkMap is empty when nothing was split.
texts, ids, chunkMap, skipped := idx.collectEmbedTexts(nodes)
if skipped > 0 {
idx.logger.Info("skipped embedding for low-value nodes",
zap.Int("count", skipped),
zap.Int("embedded", len(texts)))
}
if len(texts) == 0 {
return
}
// Embedding scaling guards. Hard-cap the vector index for repos
// big enough that the cost no longer pays off — BM25 alone is a
// fine fallback and an OOM during initial index is much worse than
// missing the semantic boost. Chunk the EmbedBatch calls so any
// single API request stays small (matters for hosted embedders
// with per-request token limits).
//
// embedChunkTimeout is generous because ONNX inference (Hugot) has
// long tail latency: a 60s budget made one in ~30 chunks miss its
// deadline, which under the old fail-fast policy threw away every
// already-embedded chunk and silently degraded to BM25 with no
// signal to the user. 5 minutes covers observed worst-case spikes
// without changing steady-state behaviour. On a true hang the
// caller can still cancel the parent indexing call.
const (
defaultEmbedMaxSymbols = 100_000
embedChunkSize = 500
embedChunkTimeout = 5 * time.Minute
)
// The cap is over the embeddable-text count, which with AST
// sub-chunking can exceed the symbol count. embedding.max_symbols
// overrides the built-in default for users with memory headroom.
embedMaxSymbols := defaultEmbedMaxSymbols
if idx.embedMaxSymbols > 0 {
embedMaxSymbols = idx.embedMaxSymbols
}
// Env override wins over both the default and the config-wired cap —
// a reliable knob independent of the config plumbing. Lets an operator
// lift the vector-index size guard for a large repo without editing
// (or debugging) the layered config.
if env := os.Getenv("GORTEX_EMBEDDINGS_MAX_SYMBOLS"); env != "" {
if n, err := strconv.Atoi(strings.TrimSpace(env)); err == nil && n > 0 {
embedMaxSymbols = n
}
}
if len(texts) > embedMaxSymbols {
idx.logger.Warn("vector index disabled — embedding text count exceeds threshold",
zap.Int("texts", len(texts)),
zap.Int("threshold", embedMaxSymbols),
zap.String("hint", "BM25 text search remains active; raise embedding.max_symbols if you have memory headroom"))
idx.lastVectorBuildErr = fmt.Errorf("embedding text count %d exceeds threshold %d (raise embedding.max_symbols)", len(texts), embedMaxSymbols)
return
}
// embedWithRetry runs one chunk under ctx; on a context-deadline
// failure it splits the chunk in half and retries each half once.
// A single slow batch shouldn't throw away every already-embedded
// chunk and silently demote the backend to BM25. ctx is the group
// context, so once one chunk fails everywhere the in-flight retries
// here see the cancellation and stop too.
var embedWithRetry func(ctx context.Context, items []string) ([][]float32, error)
embedWithRetry = func(ctx context.Context, items []string) ([][]float32, error) {
chunkCtx, cancel := context.WithTimeout(ctx, embedChunkTimeout)
out, err := idx.embedder.EmbedBatch(chunkCtx, items)
cancel()
if err == nil {
return out, nil
}
// Only retry on deadline-style failures; auth/protocol errors
// won't get better with smaller batches. A cancellation from
// the group context (a sibling chunk already failed) is not a
// retry case either.
if ctx.Err() != nil || !errors.Is(err, context.DeadlineExceeded) || len(items) <= 1 {
return nil, err
}
idx.logger.Warn("embed chunk timed out, retrying with halved batch",
zap.Int("size", len(items)),
zap.Error(err))
mid := len(items) / 2
left, lerr := embedWithRetry(ctx, items[:mid])
if lerr != nil {
return nil, lerr
}
right, rerr := embedWithRetry(ctx, items[mid:])
if rerr != nil {
return nil, rerr
}
return append(left, right...), nil
}
// Embed every chunk. For an API-backed embedder the chunks are run
// through a bounded worker pool (a hosted round-trip dominates
// indexing time, so overlapping requests is a real win); local
// in-process backends serialise on an inference mutex, so they keep
// the simple serial path. Either way the abort-on-any-error
// contract holds — one chunk failure means no vector index ships.
vectors, err := idx.embedAllChunks(texts, embedChunkSize, embedWithRetry)
if err != nil {
// A partial vector index would mis-score later queries (some
// symbols semantically findable, others not) — bail to
// text-only search rather than ship an inconsistent hybrid
// backend.
idx.logger.Warn("vector index aborted on chunk failure", zap.Error(err))
idx.lastVectorBuildErr = fmt.Errorf("chunk embedding failed: %w", err)
return
}
// Detect actual dimensions from first vector.
if len(vectors) > 0 && len(vectors[0]) > 0 {
dims = len(vectors[0])
}
vecBackend := search.NewVector(dims)
// VectorSearcher capability bridging: if the underlying store
// has a native HNSW, install it as the in-process backend's
// delegate — Add becomes a no-op, Search forwards to the
// engine, and we don't allocate `dim × 4 × N` bytes of heap
// for a parallel in-process HNSW. The indexer still drives
// the writes (BulkUpsertEmbeddings below) so the engine
// index lands with the same corpus the in-process one would
// have built.
vecSearcher, _ := idx.graph.(graph.VectorSearcher)
if vecSearcher != nil {
vecBackend.SetDelegate(&vectorSearcherDelegate{s: vecSearcher})
}
// persistSink is where the embedded vectors are written to the backend
// so they survive a restart. Normally it is the active graph (a disk
// store). Under the bulk loader idx.graph is the in-memory shadow, which
// does not implement VectorSearcher; fall back to the disk store captured
// at the shadow swap (bulkVectorSink) so the vector index still reaches
// the `vectors` table instead of living only in the in-process HNSW and
// vanishing on the next restart (which would force a paid re-embed).
persistSink := vecSearcher
if persistSink == nil {
persistSink = idx.bulkVectorSink
}
var backendItems []graph.VectorItem
if persistSink != nil {
backendItems = make([]graph.VectorItem, 0, len(vectors))
}
// Add only well-formed vectors. A nil or wrong-width vector would poison
// the index — a mis-scored or panicking query — so drop it, count it, and
// keep a sample of offending node IDs for the log.
var droppedVectors int
var droppedSample []string
for i, vec := range vectors {
if len(vec) != dims {
droppedVectors++
if len(droppedSample) < 5 {
droppedSample = append(droppedSample, ids[i])
}
continue
}
vecBackend.Add(ids[i], vec)
if persistSink != nil {
backendItems = append(backendItems, graph.VectorItem{
NodeID: ids[i],
Vec: vec,
})
}
}
// If every vector was invalid there is nothing to search on. Ship text-only
// rather than a silently empty vector index that mis-scores every query —
// the same all-or-nothing contract as the chunk-failure abort above.
if len(vectors) > 0 && vecBackend.Count() == 0 {
idx.logger.Warn("vector index aborted — all embeddings invalid",
zap.Int("dropped", droppedVectors),
zap.Int("dimensions", dims),
zap.Strings("sample_ids", droppedSample))
idx.lastVectorBuildErr = fmt.Errorf("all %d embedding vectors were invalid (want width %d)", droppedVectors, dims)
return
}
if persistSink != nil && len(backendItems) > 0 {
if err := persistSink.BulkUpsertEmbeddings(backendItems); err != nil {
idx.logger.Warn("indexer: backend vector bulk upsert failed",
zap.Error(err))
} else if err := persistSink.BuildVectorIndex(dims); err != nil {
idx.logger.Warn("indexer: backend vector index build failed",
zap.Error(err))
}
}
// Install the chunk → parent-symbol mapping so HybridBackend can
// de-chunk vector hits back to symbols at query time. Empty when no
// symbol was large enough to split.
if len(chunkMap) > 0 {
vecBackend.SetChunkMap(chunkMap)
}
// Wrap text + vector into hybrid backend, swapping it in atomically
// so any concurrent searches keep seeing a coherent backend.
//
// Unwrap any existing HybridBackend to its text side before
// re-wrapping. Without this, buildSearchIndex called again (e.g.
// once per tracked repo during daemon warmup) would stack a fresh
// Hybrid on top of the previous one — nested Hybrids retain all
// their stale vector indexes, ballooning live memory by an order
// of magnitude. The text backend (BM25 or Bleve) has already been
// updated with every node via idx.search.Add above; a single
// Hybrid wrapping it + the latest vecBackend is all we need.
sw := idx.swappable()
inner := sw.Inner()
if hyb, ok := inner.(*search.HybridBackend); ok {
inner = hyb.TextBackend()
}
sw.Swap(search.NewHybrid(inner, vecBackend, idx.embedder))
if droppedVectors > 0 {
idx.logger.Warn("indexer: dropped invalid embedding vectors",
zap.Int("dropped", droppedVectors),
zap.Int("dimensions", dims),
zap.Strings("sample_ids", droppedSample))
}
fields := []zap.Field{
zap.Int("vectors", vecBackend.Count()),
zap.Int("chunk_vectors", len(chunkMap)),
zap.Int("dimensions", dims),
zap.Int("dropped", droppedVectors),
}
// Surface the actual token spend of a paid embedding pass when the
// backend reports usage (API providers do; in-process ones don't).
// Without this the cost of an embedding run is invisible after the fact.
if acc, ok := idx.embedder.(interface{ TokensUsed() int64 }); ok {
if tokens := acc.TokensUsed(); tokens > 0 {
fields = append(fields, zap.Int64("embed_tokens", tokens))
}
}
idx.logger.Info("vector index built", fields...)
}
// dirIgnoreFiles are the per-directory ignore-file basenames honored by
// the index walk, siblings to .gitignore: Gortex's own .gortexignore
// plus ripgrep's .ignore and .rgignore. Patterns in each file are
// scoped to the directory that contains it; later filenames win, so a
// directory's .rgignore overrides its .ignore on a conflicting path.
var dirIgnoreFiles = []string{".gortexignore", ".ignore", ".rgignore"}
// shouldExclude reports whether a path is excluded by the effective
// ignore list. The flat matcher is built lazily from idx.config.Exclude,
// which is populated by ConfigManager.GetRepoConfig with the full
// layered list (builtin + global + RepoEntry + workspace). A path is
// also excluded by any per-directory ignore file (dirIgnoreFiles)
// present in one of its ancestor directories. isDir lets a trailing-
// slash pattern prune a directory subtree instead of only its files.
func (idx *Indexer) shouldExclude(path, root string, isDir bool) bool {
// .claude/ and .kiro/ are Builtin-excluded wholesale, but may hold an
// MCP server config the MCP-config-as-graph feature targets (the
// extractor's own docs name .kiro/mcp.json). Descend those subtrees
// and index only the MCP config files within them — everything else
// stays excluded, so the agent-state noise never reaches the graph.
if rel, err := filepath.Rel(root, path); err == nil && excludes.InAgentConfigDir(rel) {
if isDir {
return false
}
return !excludes.IsMCPConfigFile(rel)
}
if m := idx.excludeMatcher(); m != nil && m.MatchAbsDir(path, root, isDir) {
return true
}
return idx.dirIgnoreMatcher(root).Match(path, isDir)
}
// shouldPruneDir reports whether the index walk may skip a directory
// subtree wholesale (filepath.SkipDir) instead of descending it. A
// directory is prunable only when it is excluded AND no re-include ("!")
// pattern targets anything beneath it. go-gitignore's "*" matches across
// "/", so a blanket like "wp-content/plugins/*" reports the parent
// directory "wp-content/plugins" itself as excluded; pruning it would
// skip a later "!wp-content/plugins/foo/" re-include before the walk ever
// reaches the child. Mirroring git, we keep descending such a directory
// and let the per-file shouldExclude check filter its contents.
func (idx *Indexer) shouldPruneDir(path, root string) bool {
if !idx.shouldExclude(path, root, true) {
return false
}
if m := idx.excludeMatcher(); m != nil {
if rel, err := filepath.Rel(root, path); err == nil && m.HasNegatedDescendant(filepath.ToSlash(rel)) {
return false
}
}
if idx.dirIgnoreMatcher(root).HasNegatedDescendant(path) {
return false
}
return true
}
// dirIgnoreMatcher returns the per-directory ignore matcher, built lazily
// against the repo root the index walk is anchored at.
func (idx *Indexer) dirIgnoreMatcher(root string) *excludes.Hierarchical {
idx.dirIgnoreOnce.Do(func() {
idx.dirIgnore = excludes.NewHierarchical(root, dirIgnoreFiles...)
})
return idx.dirIgnore
}
func (idx *Indexer) excludeMatcher() *excludes.Matcher {
idx.excludeOnce.Do(func() {
patterns := idx.config.Exclude
// A nil/empty list from upstream means "no layering was applied"
// (e.g. a direct caller of indexer.New without ConfigManager).
// Fall back to the builtin baseline so the walk still skips the
// obvious non-source dirs.
if len(patterns) == 0 {
patterns = excludes.Builtin
}
idx.excludes = excludes.New(patterns)
})
return idx.excludes
}
// ParseErrors returns the parse errors from the last full index.
func (idx *Indexer) ParseErrors() []IndexError {
return idx.parseErrors
}
// FileMtimes returns a copy of the file modification time map.
func (idx *Indexer) FileMtimes() map[string]int64 {
idx.mtimeMu.RLock()
defer idx.mtimeMu.RUnlock()
out := make(map[string]int64, len(idx.fileMtimes))
for k, v := range idx.fileMtimes {
out[k] = v
}
return out
}
// RefreshFileMtime restamps the recorded modification time for a file
// from its current on-disk mtime, without re-indexing it. The watcher
// calls this when a save turned out to be structurally inert and the
// reindex was skipped: the graph is already correct, but the recorded
// mtime must advance past the save so the poller's mtime sweep does
// not keep re-flagging the same untouched file. A file absent from
// disk or never indexed is a no-op.
//
// The in-memory map is what the current process's poller and
// IsStale checks trust, but a warm restart trusts the persisted
// FileMtime sidecar instead — without also writing through to it here,
// a single inert save during a session left the persisted row at its
// pre-save value, so the next restart's HasChangesSinceMtimes saw this
// file as changed and re-tracked the whole repo. Mirrors the per-file
// indexFile persist (recordFileMtime); a no-op on the in-memory backend.
func (idx *Indexer) RefreshFileMtime(filePath string) {
absPath, err := filepath.Abs(filePath)
if err != nil {
return
}
info, err := os.Stat(absPath)
if err != nil {
return
}
// relKey (slash + NFC) so the lookup hits the same fileMtimes
// entry the index walk created for a non-ASCII filename.
key := idx.relKey(absPath)
mtime := info.ModTime().UnixNano()
idx.mtimeMu.Lock()
_, tracked := idx.fileMtimes[key]
if tracked {
idx.fileMtimes[key] = mtime
}
idx.mtimeMu.Unlock()
if !tracked {
return
}
if w, ok := idx.graph.(graph.FileMtimeWriter); ok {
if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{key: mtime}); err != nil {
idx.logger.Warn("persist file mtime failed",
zap.String("repo", idx.repoPrefix), zap.String("file", key), zap.Error(err))
}
}
}
// pruneDeletedFileMtimes drops the persisted mtime rows for files the
// incremental reindex just confirmed deleted. The in-memory map is already
// pruned by the caller; this keeps the store's FileMtime sidecar in step so
// a later warm restart does not re-discover them as phantom deletions and
// force a full re-track. A no-op when the backend lacks the capability
// (the in-memory backend) or the list is empty.
func (idx *Indexer) pruneDeletedFileMtimes(deleted []string) {
if len(deleted) == 0 {
return
}
if d, ok := idx.graph.(graph.FileMtimeDeleter); ok {
if err := d.DeleteFileMtimes(idx.repoPrefix, deleted); err != nil {
idx.logger.Warn("prune deleted file mtimes failed",
zap.String("repo", idx.repoPrefix), zap.Error(err))
}
}
}
// SetFileMtimes restores the file modification time map from a persisted snapshot.
func (idx *Indexer) SetFileMtimes(mtimes map[string]int64) {
idx.mtimeMu.Lock()
defer idx.mtimeMu.Unlock()
idx.fileMtimes = make(map[string]int64, len(mtimes))
for k, v := range mtimes {
idx.fileMtimes[k] = v
}
}
// SetRootPath sets the root path for relative path computation.
func (idx *Indexer) SetRootPath(root string) {
abs, err := filepath.Abs(root)
if err != nil {
abs = root
}
idx.storeRootPath(abs)
}
// IncrementalReindexPaths re-indexes only the files reachable from the
// supplied paths, instead of walking the whole repository root.
//
// Each path may be absolute or relative to root, and may be a file or a
// directory; directories are walked recursively with the same
// exclude / language filters as a full pass. Within that scoped file
// set the behaviour matches IncrementalReindex: only files that are
// stale (mtime or, in Merkle mode, content) are re-indexed, and a file
// previously tracked under one of the scoped paths but now absent from
// disk is evicted.
//
// When paths is empty the call degrades to IncrementalReindex(root) —
// callers can therefore pass an optional path list unconditionally.
func (idx *Indexer) IncrementalReindexPaths(root string, paths []string) (*IndexResult, error) {
if len(paths) == 0 {
return idx.IncrementalReindex(root)
}
start := time.Now()
absRoot, err := filepath.Abs(root)
if err != nil {
return nil, err
}
idx.storeRootPath(absRoot)
// scopeRels holds the repo-relative slash-paths the caller asked to
// reindex — used both to drive the discovery walk and to bound
// deletion detection to the scoped subtree.
scopeRels := make(map[string]bool)
// diskFiles is the set of in-scope language files currently on
// disk; staleFiles is the subset that changed since the last pass.
diskFiles := make(map[string]bool)
var staleFiles []string
merkleMode := idx.merkleEnabled()
for _, p := range paths {
absPath := p
if !filepath.IsAbs(absPath) {
absPath = filepath.Join(absRoot, filepath.FromSlash(p))
}
absPath = filepath.Clean(absPath)
// A path outside the repo root is rejected: scoping is a
// narrowing operation, never an escape hatch to index files
// the repo doesn't own.
rel, relErr := filepath.Rel(absRoot, absPath)
if relErr != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return nil, fmt.Errorf("incremental reindex: path %q is outside repository root %q", p, absRoot)
}
// Canonical key (slash + NFC) so scopeRels matches fileMtimes
// keys when deletion detection intersects the two below.
scopeRels[idx.relKey(absPath)] = true
info, statErr := os.Stat(absPath)
if statErr != nil {
// A path that no longer exists is not an error: it may be
// a deleted file the caller still wants evicted. Deletion
// detection below handles it via scopeRels.
if errors.Is(statErr, os.ErrNotExist) {
continue
}
return nil, fmt.Errorf("incremental reindex: stat %q: %w", p, statErr)
}
if info.IsDir() {
walkErr := filepath.WalkDir(absPath, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
if d.IsDir() {
if idx.shouldPruneDir(path, absRoot) {
return filepath.SkipDir
}
return nil
}
if _, ok := idx.effectiveLanguage(path, nil); !ok {
return nil
}
if idx.shouldExclude(path, absRoot, false) {
return nil
}
// relKey (slash + NFC) keeps the disk set keyed
// consistently with fileMtimes for non-ASCII names.
relPath := idx.relKey(path)
diskFiles[relPath] = true
if !merkleMode && idx.IsStale(relPath) {
staleFiles = append(staleFiles, path)
}
return nil
})
if walkErr != nil {
return nil, walkErr
}
continue
}
// Single file. Apply the same language / exclude gate so a
// caller can't force a non-source or excluded file in.
if _, ok := idx.effectiveLanguage(absPath, nil); !ok {
continue
}
if idx.shouldExclude(absPath, absRoot, false) {
continue
}
// relKey (slash + NFC) — same canonical key the graph and
// fileMtimes use, so a non-ASCII path passed in here matches
// regardless of the Unicode form the caller supplied.
relPath := idx.relKey(absPath)
diskFiles[relPath] = true
if !merkleMode && idx.IsStale(relPath) {
staleFiles = append(staleFiles, absPath)
}
}
// In Merkle mode the per-file mtime check is skipped; the stale set
// comes from a content-addressed tree diff over the whole repo,
// then intersected back down to the requested scope.
if merkleMode {
for _, abs := range idx.merkleStaleFiles(absRoot, diskFiles) {
rel, relErr := filepath.Rel(absRoot, abs)
if relErr != nil {
continue
}
if diskFiles[filepath.ToSlash(rel)] {
staleFiles = append(staleFiles, abs)
}
}
}
// Deletion detection, bounded to the scoped subtree. A file tracked
// in fileMtimes that sits under one of the requested paths but is
// absent from this scoped discovery walk is a deletion candidate;
// the same stat-before-evict guard as IncrementalReindex applies so
// a newly-excluded or transiently-unreachable file is preserved.
idx.mtimeMu.RLock()
var candidates []string
for relPath := range idx.fileMtimes {
if diskFiles[relPath] {
continue
}
if relPathInScope(relPath, scopeRels) {
candidates = append(candidates, relPath)
}
}
idx.mtimeMu.RUnlock()
var deletedFiles []string
for _, relPath := range candidates {
absPath := filepath.Join(absRoot, filepath.FromSlash(relPath))
_, statErr := os.Stat(absPath)
if statErr == nil {
continue
}
if errors.Is(statErr, os.ErrNotExist) {
deletedFiles = append(deletedFiles, relPath)
continue
}
idx.logger.Warn("incremental reindex: stat failed during scoped deletion detection, preserving",
zap.String("rel", relPath), zap.Error(statErr))
}
for _, relPath := range deletedFiles {
graphPath := idx.prefixPath(relPath)
idx.restubIncomingRefs(graphPath)
idx.evictEnrichment(graphPath)
idx.graph.EvictFile(graphPath)
idx.mtimeMu.Lock()
delete(idx.fileMtimes, relPath)
idx.mtimeMu.Unlock()
}
// Prune the persisted mtime rows for deleted files too, so the next
// warm restart does not see them as phantom deletions (the in-memory
// delete above does not reach the store's sidecar table).
idx.pruneDeletedFileMtimes(deletedFiles)
// Re-index stale files with the same one-shot retry as the
// whole-root path — a file locked or mid-write when the walk caught
// it gets a second chance before landing on FailedFiles.
var failedFiles []string
for _, f := range staleFiles {
if err := idx.IndexFile(f); err != nil {
idx.logger.Debug("incremental reindex: failed to index file",
zap.String("file", f), zap.Error(err))
failedFiles = append(failedFiles, f)
}
}
if len(failedFiles) > 0 {
retry := failedFiles
failedFiles = nil
for _, f := range retry {
if err := idx.IndexFile(f); err != nil {
idx.logger.Warn("incremental reindex: file failed after retry",
zap.String("file", f), zap.Error(err))
failedFiles = append(failedFiles, f)
}
}
}
// Re-infer interface implementations and re-run stub-call passes —
// eviction may have dropped edges. Skipped under deferGlobalPasses
// so a batch caller runs one global pass at the end.
if !idx.deferGlobalPasses && (len(staleFiles) > 0 || len(deletedFiles) > 0) {
// Scoped inference passes re-derive only the affected types/interfaces
// (add-parity with the full pass); fall back to whole-graph when
// scoping is disabled.
if !idx.runScopedInferencePasses(staleFiles) {
idx.resolver.InferImplements()
idx.resolver.InferOverrides()
}
// Capability (reads_env / executes_process / accesses_field) and
// framework-dispatch synthesis derive from code structure; skip them
// when the reconcile touched only non-code files (docs/config) and
// removed nothing — they cannot change any edge in that case, and
// eviction already handled any deletion. Same idempotent re-derive
// RunGlobalGraphPasses runs at full index.
if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) {
synthesizeCapabilityEdges(idx.graph)
resolver.RunFrameworkSynthesizers(idx.graph)
}
// Incremental: synthesize external calls only for the reindexed
// files (O(edited files)), not a full-graph recompute.
resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles))
}
// Skip the search-index rebuild on a zero-change reconcile when the
// backend already persists its search structures (the on-disk
// backend keeps its FTS index and vector embeddings on disk).
// buildSearchIndex re-reads every node (GetRepoNodes) and re-embeds
// them, then BulkUpsertEmbeddings re-writes the embedding rows. On a
// warm restart that work is pure recompute of already-persisted data.
// When nothing changed there is nothing to re-embed, so skip it
// entirely — the persisted index is authoritative. The in-memory
// backends (BM25 / Bleve) must still rebuild from the replayed
// snapshot, so they keep the unconditional path.
if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) {
idx.buildSearchIndex()
}
if len(staleFiles) > 0 || len(deletedFiles) > 0 {
idx.extractContracts()
idx.indexGen.Add(1) // files changed — invalidate the trigram cache
}
nodes, edges := idx.repoNodeEdgeCount()
result := &IndexResult{
NodeCount: nodes,
EdgeCount: edges,
FileCount: len(diskFiles),
StaleFileCount: len(staleFiles),
DeletedFileCount: len(deletedFiles),
FailedFiles: failedFiles,
DurationMs: time.Since(start).Milliseconds(),
}
idx.warnIfEdgeSanityViolated(result)
// An incremental pass that re-indexed or evicted at least one file did
// work — mark the repo for deferred semantic enrichment. A zero-change
// reconcile leaves the marker untouched so an unchanged repo is skipped.
if result.StaleFileCount > 0 || result.DeletedFileCount > 0 {
idx.pendingEnrich.Store(true)
}
// A re-parsed stale file's hover-enrichment edges were evicted with its old
// nodes, so force the deferred pass past the completion marker (a deletion
// evicts nodes but creates no fresh unstamped ones, so it alone does not).
if result.StaleFileCount > 0 {
idx.reparsedThisRun.Store(true)
}
return result, nil
}
// relPathInScope reports whether a repo-relative slash-path falls under
// any of the scoped paths — either an exact file match or anywhere
// inside a scoped directory.
func relPathInScope(relPath string, scope map[string]bool) bool {
if scope[relPath] {
return true
}
for s := range scope {
if s == "." {
return true
}
if strings.HasPrefix(relPath, s+"/") {
return true
}
}
return false
}
// IncrementalReindex walks the file tree and re-indexes only files that changed
// since the last snapshot. It also evicts nodes for deleted files.
func (idx *Indexer) IncrementalReindex(root string) (*IndexResult, error) {
start := time.Now()
absRoot, err := filepath.Abs(root)
if err != nil {
return nil, err
}
idx.storeRootPath(absRoot)
// Collect files currently on disk.
diskFiles := make(map[string]bool)
var staleFiles []string
// Merkle mode replaces per-file mtime staleness with a
// content-addressed Merkle-tree diff computed after the walk.
merkleMode := idx.merkleEnabled()
err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
return nil
}
if d.IsDir() {
if idx.shouldPruneDir(path, absRoot) {
return filepath.SkipDir
}
return nil
}
if _, ok := idx.effectiveLanguage(path, nil); !ok {
return nil
}
if idx.shouldExclude(path, absRoot, false) {
return nil
}
// relKey (slash + NFC) so the disk set is keyed identically
// to fileMtimes — otherwise a non-ASCII file the snapshot
// stored under one Unicode form and this walk observes under
// another would be seen as both deleted and newly created.
relPath := idx.relKey(path)
diskFiles[relPath] = true
if !merkleMode && idx.IsStale(relPath) {
staleFiles = append(staleFiles, path)
}
return nil
})
if err != nil {
return nil, err
}
// In Merkle mode the per-file mtime check above is skipped; the
// stale set comes from a content-addressed tree diff instead.
if merkleMode {
staleFiles = idx.merkleStaleFiles(absRoot, diskFiles)
}
// Detect deleted files. A file that's tracked in fileMtimes but
// absent from the current discovery walk is a candidate, but
// "absent from discovery" is not the same as "absent from disk":
//
// - The exclude list (.gortex.yaml, builtin, workspace) may have
// grown since the last index — every newly-excluded file would
// be classified as deleted.
// - A language extractor's Extensions() may have changed across
// versions — files whose ext is no longer detected would be
// classified as deleted.
// - WalkDir swallowed a transient error (EACCES, EIO, NFS hiccup,
// ELOOP) — the file is unreachable this pass but still on disk.
//
// All three would purge legitimate graph state on every daemon
// restart. Stat the candidate first: only treat ENOENT/ENOTDIR as
// deletion; preserve on success (file exists, just not discovered)
// and on transient errors. The cost is one extra stat per
// previously-indexed-but-not-discovered file, which is bounded by
// the size of the exclusion delta.
idx.mtimeMu.RLock()
var candidates []string
for relPath := range idx.fileMtimes {
if !diskFiles[relPath] {
candidates = append(candidates, relPath)
}
}
idx.mtimeMu.RUnlock()
var deletedFiles []string
for _, relPath := range candidates {
absPath := filepath.Join(absRoot, relPath)
_, err := os.Stat(absPath)
if err == nil {
// File exists on disk; it was excluded or its extension is
// no longer detected. Preserve.
continue
}
if errors.Is(err, os.ErrNotExist) {
deletedFiles = append(deletedFiles, relPath)
continue
}
// Transient error — preserve to be safe.
idx.logger.Warn("incremental reindex: stat failed during deletion detection, preserving",
zap.String("rel", relPath), zap.Error(err))
}
// Evict only files that are truly absent from disk.
for _, relPath := range deletedFiles {
graphPath := idx.prefixPath(relPath)
idx.restubIncomingRefs(graphPath)
idx.evictEnrichment(graphPath)
idx.graph.EvictFile(graphPath)
idx.mtimeMu.Lock()
delete(idx.fileMtimes, relPath)
idx.mtimeMu.Unlock()
}
// Prune the persisted mtime rows for deleted files too, so the next
// warm restart does not see them as phantom deletions (the in-memory
// delete above does not reach the store's sidecar table).
idx.pruneDeletedFileMtimes(deletedFiles)
// Re-index stale files. A file that fails — most often because it
// was locked or mid-write when the walk caught it — is collected
// and retried once below. A failure that survives the retry is
// surfaced on IndexResult.FailedFiles so the caller can replay it.
// A file whose bytes couldn't be read at all keeps no mtime
// recorded, so it stays stale for the next incremental pass; a
// file that read but failed to parse gets its mtime recorded (see
// indexFile's result==nil branch), so it stops being retried until
// its content changes again — see IndexResult.FailedFiles.
var failedFiles []string
for _, f := range staleFiles {
if err := idx.IndexFile(f); err != nil {
idx.logger.Debug("incremental reindex: failed to index file",
zap.String("file", f), zap.Error(err))
failedFiles = append(failedFiles, f)
}
}
if len(failedFiles) > 0 {
retry := failedFiles
failedFiles = nil
for _, f := range retry {
if err := idx.IndexFile(f); err != nil {
idx.logger.Warn("incremental reindex: file failed after retry",
zap.String("file", f), zap.Error(err))
failedFiles = append(failedFiles, f)
}
}
}
// Re-infer interface implementations (edges may have been lost
// during eviction). Skipped under deferGlobalPasses so a batch
// caller (ReconcileAll, warmup) can run a single global pass at
// the end instead of paying O(global) per repo.
if !idx.deferGlobalPasses {
// Scoped inference passes re-derive only the affected types/interfaces
// (add-parity with the full pass); fall back to whole-graph when off.
if !idx.runScopedInferencePasses(staleFiles) {
idx.resolver.InferImplements()
idx.resolver.InferOverrides()
}
// Capability edges (reads_env / executes_process / accesses_field)
// and framework dynamic-dispatch synthesis are whole-graph recomputes
// that derive edges only from structural nodes in the changed files.
// When nothing structural changed — no stale code file (a doc/config
// edit or a true zero-change reconcile) and no deletion — they can
// produce no new edge, so re-running them is pure cost: the dominant
// waste in a no-op IncrementalReindex, which the periodic janitor runs
// per repo per tick. Gate them on the same predicate the path-scoped
// IncrementalReindexPaths already uses. Deletions still trigger a
// re-run because an evicted file may have been a dispatch endpoint.
if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) {
synthesizeCapabilityEdges(idx.graph)
resolver.RunFrameworkSynthesizers(idx.graph)
}
// External-call synthesis (opt-in) — file-scoped to the reindexed
// files (O(edited files)), not a full-graph recompute. Eviction
// already dropped a removed file's synthetic edges; a re-indexed
// file's fresh external terminals are re-materialised here.
resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles))
// Clone detection is not re-run here: each stale file was
// re-indexed through IndexFile above, whose resolve pass
// already recomputed EdgeSimilarTo against the fresh graph,
// and deleted files self-clean via EvictFile's bidirectional
// edge removal. Under deferGlobalPasses the batch caller runs
// the global clone pass once at the end.
}
// Rebuild search index to ensure consistency — but skip it on a
// zero-change reconcile against a backend that persists its search
// structures natively (the on-disk backend). See the matching guard
// in the other incremental path: re-embedding is wasted work and
// there is nothing to rebuild when no file changed.
if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) {
idx.buildSearchIndex()
}
// Update totalDetected so index_health reports correctly after cache restore.
if idx.totalDetected == 0 {
idx.totalDetected = len(diskFiles)
}
// Re-extract contracts only if stale files were re-indexed.
if len(staleFiles) > 0 || len(deletedFiles) > 0 {
idx.extractContracts()
idx.indexGen.Add(1) // files changed — invalidate the trigram cache
}
nodes, edges := idx.repoNodeEdgeCount()
result := &IndexResult{
NodeCount: nodes,
EdgeCount: edges,
FileCount: len(diskFiles),
StaleFileCount: len(staleFiles),
DeletedFileCount: len(deletedFiles),
FailedFiles: failedFiles,
DurationMs: time.Since(start).Milliseconds(),
}
idx.warnIfEdgeSanityViolated(result)
// An incremental pass that re-indexed or evicted at least one file did
// work — mark the repo for deferred semantic enrichment. A zero-change
// reconcile leaves the marker untouched so an unchanged repo is skipped.
if result.StaleFileCount > 0 || result.DeletedFileCount > 0 {
idx.pendingEnrich.Store(true)
}
// A re-parsed stale file's hover-enrichment edges were evicted with its old
// nodes, so force the deferred pass past the completion marker (a deletion
// evicts nodes but creates no fresh unstamped ones, so it alone does not).
if result.StaleFileCount > 0 {
idx.reparsedThisRun.Store(true)
}
return result, nil
}
// LastIndexTime returns the timestamp of the last full index.
func (idx *Indexer) LastIndexTime() time.Time {
return idx.lastIndexTime
}
// TotalDetected returns the total number of files detected during the last full index.
func (idx *Indexer) TotalDetected() int {
return idx.totalDetected
}
// buildPerFileContractExtractors returns the set of extractors that
// operate on a single source file (everything except GoModExtractor,
// which runs once against go.mod at the repo root) plus a language →
// [extractors] map so callers can skip extractors whose
// SupportedLanguages() doesn't include a given file's language.
// Building the language map once avoids doing the string-membership
// check per file.
func (idx *Indexer) buildPerFileContractExtractors() ([]contracts.Extractor, map[string][]contracts.Extractor) {
extractors := []contracts.Extractor{
&contracts.HTTPExtractor{ClientAliases: idx.config.HTTPClientAliases},
&contracts.GRPCExtractor{},
&contracts.ThriftExtractor{},
&contracts.GraphQLExtractor{},
&contracts.TRPCExtractor{},
&contracts.OpenAPIExtractor{},
&contracts.TopicExtractor{},
&contracts.WebSocketExtractor{},
&contracts.NestMicroserviceExtractor{},
&contracts.EnvVarExtractor{},
}
// Config-driven event bus: only registered when the user declared
// boundaries (index.event_bus / CODEGRAPH_EVENT_CONFIG), so the default
// extractor set is unchanged.
if b := idx.eventBusBoundaries(); len(b) > 0 {
extractors = append(extractors, &contracts.EventBusExtractor{Boundaries: b})
}
byLang := make(map[string][]contracts.Extractor)
for _, ex := range extractors {
for _, lang := range ex.SupportedLanguages() {
byLang[lang] = append(byLang[lang], ex)
}
}
return extractors, byLang
}
// runContractExtractorsForFile applies the given extractors to a single
// file and returns the raw contracts (with RepoPrefix already set).
// Called both inline from parse workers and from the full-walk
// extractContracts path — they share the same per-file work.
func (idx *Indexer) runContractExtractorsForFile(
graphPath string,
src []byte,
fileNodes []*graph.Node,
fileEdges []*graph.Edge,
exts []contracts.Extractor,
tree *parser.ParseTree,
) []contracts.Contract {
if len(exts) == 0 {
return nil
}
// Contracts from synthetic test/bench fixtures are kept (so drift
// checks can flag a stale test pinned to an obsolete production
// contract) but tagged with is_test=true and a test_source
// category so the dashboard can filter them out by default.
testSource := fixtures.TestContractSource(graphPath)
var out []contracts.Contract
// The graph store backs graph-wide constant resolution for endpoint
// arguments (a route path / queue / topic referenced by a const). Resolved
// once per call; nil when the backend can't satisfy the reader (const
// dereference is then disabled and store-aware extractors degrade to their
// tree-aware behaviour).
var endpointStore contracts.EndpointConstStore
if es, ok := idx.graph.(contracts.EndpointConstStore); ok {
endpointStore = es
}
for _, ex := range exts {
var found []contracts.Contract
if sae, ok := ex.(contracts.StoreAwareExtractor); ok {
found = sae.ExtractWithStore(graphPath, src, fileNodes, fileEdges, tree, endpointStore, idx.repoPrefix)
} else if tae, ok := ex.(contracts.TreeAwareExtractor); ok && tree != nil {
found = tae.ExtractWithTree(graphPath, src, fileNodes, fileEdges, tree)
} else {
found = ex.Extract(graphPath, src, fileNodes, fileEdges)
}
for i := range found {
found[i].RepoPrefix = idx.repoPrefix
// Stamp the workspace / project slugs alongside the repo
// prefix so the matcher's boundary check has the data it
// needs without a second registry walk. Empty slugs
// default to RepoPrefix at Match time via
// Contract.EffectiveWorkspace / EffectiveProject.
if idx.workspaceID != "" {
found[i].WorkspaceID = idx.workspaceID
}
if idx.projectID != "" {
found[i].ProjectID = idx.projectID
}
if testSource != "" {
if found[i].Meta == nil {
found[i].Meta = map[string]any{}
}
found[i].Meta["is_test"] = true
found[i].Meta["test_source"] = testSource
}
}
out = append(out, found...)
}
return out
}
// commitContracts writes contract nodes + provides/consumes edges for
// every contract in reg, and sets idx.contractRegistry to reg. Called
// once per index pass after all per-file contracts have been collected
// (inline from parse workers) plus go.mod has been processed.
func (idx *Indexer) commitContracts(reg *contracts.Registry) {
// Upgrade bare type names in contract Meta (e.g. "UserResp") to
// full symbol IDs (e.g. "pkg/resp.go::UserResp") now that the
// graph is complete. During extraction the enricher only saw
// the handler's file-scoped node list, so types declared in a
// sibling file stayed as bare names.
reg.UpgradeBareTypeRefs(func(name, repoHint string) []string {
matches := idx.graph.FindNodesByName(name)
var same, others []string
for _, n := range matches {
if n.Kind != graph.KindType {
continue
}
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
same = append(same, n.ID)
continue
}
others = append(others, n.ID)
}
if len(same) > 0 {
return same
}
return others
})
// Cross-file handler resolution. When a route is registered with
// a handler identifier that the file-scoped extractor couldn't
// resolve (`h.ServeArchive` in router.go wiring a method defined
// in archive_handler.go), the contract's SymbolID fell back to
// the enclosing router function and schema extraction ran
// against the router's body — which has every route's bindings
// piled on top of each other. Re-run enrichment with the
// correct per-handler scope now that the graph is complete.
idx.resolveProviderHandlers(reg)
// Cross-file route-prefix joining. A FastAPI APIRouter declared in
// one file (`router = APIRouter(prefix="/users")`) is mounted under
// a second prefix elsewhere (`app.include_router(router,
// prefix="/api")`), so a route declared `@router.get("/{id}")`
// belongs at /api/users/{id}, not /{id}. Rewrite the affected
// provider contract IDs to the joined path before the matcher pairs
// them with consumers. Also handles Express app.use mounts and
// NestJS @Controller class prefixes. Reads source straight off disk
// (cached per file) — the same access pattern resolveProviderHandlers
// uses for cross-file handler bodies.
//
// Mount sites (a main.py that only calls include_router) often carry
// no route contracts, so the scan-file set comes from the graph's
// py/ts/js file nodes, not the registry — but only when at least one
// prefix-eligible route contract exists, so non-FastAPI/Express/Nest
// repos pay nothing.
if scanFiles := idx.routerPrefixScanFiles(reg); len(scanFiles) > 0 {
srcCache := make(map[string][]byte)
contracts.JoinRouterPrefixes(reg, scanFiles, func(filePath string) []byte {
if data, ok := srcCache[filePath]; ok {
return data
}
data := idx.contractFileSrc(filePath)
srcCache[filePath] = data
return data
})
}
// Spring application(-profile)?.{yml,properties} config-key graph: emit the
// value-redacted config-key nodes and reads_config edges from the @Value /
// @ConfigurationProperties beans the Java extractor stamped. Cheap to skip
// on non-Spring repos (no config files + no stamped beans = no work).
contracts.BindSpringConfig(idx.graph, idx.contractFileSrc)
// Trace response variables back to their call-site return types.
// Handles `source, err := h.svc.Get(...)` → response_type is
// whatever `h.svc.Get` returns. The enricher can't do this
// without graph access; this pass reads each method's signature
// directly off the graph node, parses the first non-error
// return type, and resolves it to a symbol ID.
idx.resolveCallReturnTypes(reg)
// Snapshot field-level shapes for every type that's referenced as
// a contract's request / response body. This is Stage 2 — without
// per-field data Stage 3 (validation, breaking-change detection)
// has nothing to diff. We de-duplicate by symbol ID so heavy
// fan-in types (a User DTO used by 40 routes) only get parsed
// once per index pass.
idx.snapshotContractShapes(reg)
// Fold each type's snapshotted Shape into the envelope rows that
// reference it. The dashboard renders these rows as the response
// JSON shape (e.g. `{ workspace: { id: string }, repos: [{ name: string }] }`)
// instead of the bare type-symbol-ID, which answers nothing about
// the wire format.
idx.inlineEnvelopeShapes(reg)
all := reg.All()
nodes := make([]*graph.Node, 0, len(all))
edges := make([]*graph.Edge, 0, len(all))
for _, c := range all {
// dep::<module> nodes were materialised by extractGoModContracts
// before ResolveAll (so the import bridge could find them);
// re-emitting them here would PK-collide on backends whose bulk
// load is INSERT-only (the on-disk backend). The pre-pass is the single
// writer for that contract type.
if c.Type == contracts.ContractDependency {
continue
}
nodes = append(nodes, &graph.Node{
ID: c.ID,
Kind: graph.KindContract,
Name: c.ID,
FilePath: c.FilePath,
Language: "contract",
RepoPrefix: c.RepoPrefix,
WorkspaceID: c.EffectiveWorkspace(),
ProjectID: c.EffectiveProject(),
Meta: map[string]any{
"type": string(c.Type),
"role": string(c.Role),
"symbol_id": c.SymbolID,
"line": c.Line,
"confidence": c.Confidence,
"contract_meta": c.Meta,
},
})
if c.SymbolID == "" {
continue
}
edgeKind := graph.EdgeProvides
if c.Role == contracts.RoleConsumer {
edgeKind = graph.EdgeConsumes
}
edges = append(edges, &graph.Edge{
From: c.SymbolID,
To: c.ID,
Kind: edgeKind,
FilePath: c.FilePath,
Line: c.Line,
})
// Framework-layer EdgeHandlesRoute. Emitted alongside
// EdgeProvides for HTTP / gRPC / WS / GraphQL / topic
// providers so `analyze kind=routes` and other
// framework-aware tools walk one targeted edge instead
// of filtering EdgeProvides by contract type. Consumers
// (callers of routes) and non-route contract types (env,
// OpenAPI specs, DI tokens) intentionally skip this
// edge — they aren't route handlers.
if c.Role == contracts.RoleProvider && isRouteContractType(c.Type) {
edges = append(edges, &graph.Edge{
From: c.SymbolID,
To: c.ID,
Kind: graph.EdgeHandlesRoute,
FilePath: c.FilePath,
Line: c.Line,
Meta: map[string]any{
"contract_type": string(c.Type),
},
})
}
}
bulkStart := time.Now()
idx.bulkCommit(nodes, edges)
bulkElapsed := time.Since(bulkStart)
idx.contractRegistry = reg
repo := idx.rootPath
if idx.repoPrefix != "" {
repo = idx.repoPrefix
}
idx.logger.Info("contracts extracted",
zap.String("repo", repo),
zap.Int("count", len(all)),
zap.Duration("commit_bulk_elapsed", bulkElapsed))
}
// bulkCommit writes nodes + edges in one AddBatch call. The bulk
// load path is intentionally NOT used here: contract IDs often
// coincide with existing source-symbol IDs (a route handler shows
// up as both a Go function and an HTTP-contract anchor), and the
// on-disk backend's bulk load is INSERT-only on the node table so
// any collision fails the whole batch. AddBatch's non-bulk path
// upserts every row so duplicates are absorbed in place; the
// per-call cost is amortised by the chunked write path the backend
// uses internally.
func (idx *Indexer) bulkCommit(nodes []*graph.Node, edges []*graph.Edge) {
if len(nodes) == 0 && len(edges) == 0 {
return
}
idx.graph.AddBatch(nodes, edges)
}
// routerPrefixScanFiles returns the set of source files
// JoinRouterPrefixes must scan for router definitions and mount sites
// (APIRouter / include_router / app.use). Returns nil when no HTTP
// contract uses a prefix-joining framework (FastAPI / Express / NestJS),
// so unrelated repos skip the file enumeration entirely. When eligible,
// it enumerates py / ts / js / tsx / jsx file nodes from the graph — the
// mount file (a FastAPI main.py) frequently has no route contract of its
// own and so can't be discovered from the registry alone.
func (idx *Indexer) routerPrefixScanFiles(reg *contracts.Registry) []string {
eligible := false
for _, c := range reg.All() {
if c.Type != contracts.ContractHTTP || c.Meta == nil {
continue
}
switch fw, _ := c.Meta["framework"].(string); fw {
case "fastapi/flask", "express", "nestjs":
eligible = true
}
if eligible {
break
}
}
if !eligible {
return nil
}
var out []string
for _, n := range idx.graph.AllNodes() {
if n.Kind != graph.KindFile {
continue
}
path := n.FilePath
if path == "" {
path = n.ID
}
switch {
case strings.HasSuffix(path, ".py"),
strings.HasSuffix(path, ".ts"),
strings.HasSuffix(path, ".tsx"),
strings.HasSuffix(path, ".js"),
strings.HasSuffix(path, ".jsx"):
out = append(out, path)
}
}
return out
}
// contractFileSrc reads the on-disk source for a contract FilePath
// (which is repo-prefixed when the indexer uses a repo prefix). Returns
// nil when the file can't be read. Mirrors the disk-resolution logic in
// resolveProviderHandlers so cross-file passes share one access pattern.
func (idx *Indexer) contractFileSrc(filePath string) []byte {
diskPath := filePath
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
}
diskPath = filepath.Join(idx.rootPath, diskPath)
data, err := os.ReadFile(diskPath)
if err != nil {
return nil
}
return data
}
// isRouteContractType reports whether a ContractType corresponds to a
// real network-route handler (HTTP / gRPC / WebSocket / GraphQL /
// topic). Used to gate EdgeHandlesRoute emission so the framework-layer
// edge stays focused on actual handlers and excludes env / OpenAPI /
// dependency / DI-token contracts that share the EdgeProvides edge but
// aren't routes in the agent-asks-which-handler-serves-X sense.
func isRouteContractType(t contracts.ContractType) bool {
switch t {
case contracts.ContractHTTP,
contracts.ContractGRPC,
contracts.ContractThrift,
contracts.ContractGraphQL,
contracts.ContractTopic,
contracts.ContractWS:
return true
}
return false
}
// resolveProviderHandlers finds the actual handler for every HTTP
// provider contract whose per-file extraction couldn't resolve the
// handler identifier (typically routers in one file wiring handlers
// defined in sibling files). For each such contract:
//
// - Take Meta["handler_trail"] — the full expression between the
// HandleFunc parens, which carries every handler candidate
// (wrappers + inner handler). Fall back to "handler_ident"
// when no trail was captured (older contracts, simple consumer
// patterns).
// - Enumerate candidates in source order and look each up in the
// graph; take the innermost (last) one that resolves. That
// picks h.ServeArchive out of WithAuth(h.ServeArchive) instead
// of the WithAuth wrapper.
// - Re-run EnrichHTTPContract against the handler's file with the
// handler's line range so the enricher sees its actual body
// instead of the router's.
// - Drop `handler_ident` / `handler_trail` from meta afterwards —
// they were internal resolution hints.
func (idx *Indexer) resolveProviderHandlers(reg *contracts.Registry) {
type pending struct {
contractID string
trail string
fallback string
repoHint string
// srcDir is the directory of the contract's registration site
// (the file with the HandleFunc call). Used by lookupHandler
// as a tie-breaker when two same-repo functions share a name
// across packages — e.g. `Handler.handleContracts` in the
// `server` pkg vs `Server.handleContracts` in `mcp`. A
// `recv.method` call from inside `server/handler.go` resolves
// to the same-package method, not the cross-package one.
srcDir string
}
var todo []pending
for _, c := range reg.All() {
if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP {
continue
}
trail, _ := c.Meta["handler_trail"].(string)
fallback, _ := c.Meta["handler_ident"].(string)
if trail == "" && fallback == "" {
continue
}
// Skip contracts where schema is already populated — the
// initial file-scoped pass worked.
if src, _ := c.Meta["schema_source"].(string); src == "extracted" || src == "partial" {
continue
}
todo = append(todo, pending{
contractID: c.ID,
trail: trail,
fallback: fallback,
repoHint: c.RepoPrefix,
srcDir: filepath.Dir(c.FilePath),
})
}
// Always strip the internal handler hints from Meta at the end of
// this pass — successful or not. They were only ever intended as
// per-pass resolution scratchpad: when the cross-file lookup
// succeeds we delete them in the patched-contract loop below; when
// it fails (no candidate, ambiguous, etc.) they used to leak to
// the dashboard as values like `handler_trail: "/users", listUsers`
// — useless to a reader. This cleanup runs unconditionally so
// downstream consumers never see internal extractor state.
defer func() {
for _, c := range reg.All() {
if c.Meta == nil {
continue
}
if _, hasIdent := c.Meta["handler_ident"]; !hasIdent {
if _, hasTrail := c.Meta["handler_trail"]; !hasTrail {
continue
}
}
items := reg.ByID(c.ID)
for i := range items {
if items[i].Meta == nil {
continue
}
delete(items[i].Meta, "handler_ident")
delete(items[i].Meta, "handler_trail")
}
reg.ReplaceByID(c.ID, items)
}
}()
if len(todo) == 0 {
return
}
// Cache file source + node list per file path — a single router
// often refers to dozens of handlers in the same sibling file.
fileSrc := make(map[string][]byte)
fileNodes := make(map[string][]*graph.Node)
// fileTrees caches per-file ParseTree handles parsed lazily below
// so a router referencing many handlers in the same sibling file
// only parses that file once.
fileTrees := make(map[string]*parser.ParseTree)
defer func() {
for _, t := range fileTrees {
t.Release()
}
}()
resolved := 0
for _, p := range todo {
handlerNode := idx.resolveInnermostHandler(p.trail, p.fallback, p.repoHint, p.srcDir)
if handlerNode == nil {
continue
}
src, ok := fileSrc[handlerNode.FilePath]
if !ok {
diskPath := handlerNode.FilePath
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
}
diskPath = filepath.Join(idx.rootPath, diskPath)
data, err := os.ReadFile(diskPath)
if err != nil {
fileSrc[handlerNode.FilePath] = nil
continue
}
fileSrc[handlerNode.FilePath] = data
src = data
}
if src == nil {
continue
}
nodes, ok := fileNodes[handlerNode.FilePath]
if !ok {
nodes = idx.graph.GetFileNodes(handlerNode.FilePath)
fileNodes[handlerNode.FilePath] = nodes
}
lang := detectLangFromPath(handlerNode.FilePath)
tree, treeReady := fileTrees[handlerNode.FilePath]
if !treeReady {
tree = contracts.ParseTreeForLang(lang, src)
fileTrees[handlerNode.FilePath] = tree
}
// Re-run enrichment. EnrichHTTPContractWithTree reads the
// contract's SymbolID to locate the handler body range — swap
// it in temporarily to the resolved handler so the lookup
// works. With a tree the AST overlay runs after the regex
// pass and overrides Meta keys it can confidently produce.
matches := reg.ByID(p.contractID)
if len(matches) == 0 {
continue
}
for i, c := range matches {
if c.Role != contracts.RoleProvider {
continue
}
// Operate on a copy; Registry entries are values.
patched := c
patched.SymbolID = handlerNode.ID
patched.FilePath = handlerNode.FilePath
if patched.Meta == nil {
patched.Meta = map[string]any{}
}
// Drop prior path_params so the enricher's fresh pass
// repopulates consistently (path hasn't changed, but we
// want the call-path to be identical to Stage 1).
lines := splitLines(src)
contracts.EnrichHTTPContractWithTree(&patched, lines, nodes, lang, tree)
delete(patched.Meta, "handler_ident")
delete(patched.Meta, "handler_trail")
matches[i] = patched
resolved++
}
// Write back the mutated set. The registry doesn't have an
// "update" API; we use AddAll semantics via Set-like
// operations. Simpler: clear then re-add all roles to this ID.
reg.ReplaceByID(p.contractID, matches)
}
if resolved > 0 {
idx.logger.Info("resolved cross-file provider handlers",
zap.Int("count", resolved),
zap.Int("considered", len(todo)))
}
}
// resolveInnermostHandler picks the innermost handler candidate from
// the call trail that resolves to a real function or method in the
// graph. Walks candidates in source order and keeps the LAST
// successful lookup — for `WithAuth(h.ServeArchive)` that's
// `h.ServeArchive`, not the `WithAuth` wrapper. Falls back to the
// single identifier when no trail is available (e.g. simple bare
// `r.GET("/x", listUsers)` patterns).
func (idx *Indexer) resolveInnermostHandler(trail, fallback, repoHint, srcDir string) *graph.Node {
candidates := contracts.HandlerCandidatesInTrail(trail)
if len(candidates) == 0 && fallback != "" {
candidates = []string{fallback}
}
var best *graph.Node
for _, c := range candidates {
if n := idx.lookupHandler(c, repoHint, srcDir); n != nil {
best = n
}
}
return best
}
// lookupHandler maps a raw identifier from a route pattern to the
// graph node for the handler function / method.
//
// - "h.ServeArchive" → method named "ServeArchive", prefer same repo.
// - "ServeArchive" → function or method of that name.
// - "pkg.Foo" → same as first form, package-qualified call.
//
// Returns nil when no candidate resolves unambiguously.
func (idx *Indexer) lookupHandler(ident, repoHint, srcDir string) *graph.Node {
// Strip a leading receiver / package qualifier — "h.ServeArchive"
// → "ServeArchive".
name := ident
if i := strings.LastIndex(name, "."); i >= 0 {
name = name[i+1:]
}
if name == "" {
return nil
}
candidates := idx.graph.FindNodesByName(name)
if len(candidates) == 0 {
return nil
}
var sameRepo, other []*graph.Node
for _, n := range candidates {
if n.Kind != graph.KindFunction && n.Kind != graph.KindMethod {
continue
}
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
sameRepo = append(sameRepo, n)
continue
}
other = append(other, n)
}
if len(sameRepo) == 1 {
return sameRepo[0]
}
if len(sameRepo) == 0 && len(other) == 1 {
return other[0]
}
// Multiple candidates — try same-package tie-break before giving up.
// A `recv.method` call inside `pkg/foo.go` resolves to a method
// declared in the same package; cross-package lookalikes (e.g.
// `Server.handleContracts` in `mcp` vs `Handler.handleContracts`
// in `server`) are filtered out. Without this, both routers and
// MCP-side handlers compete for the same name and the resolver
// falls back to the enclosing function (`registerRoutes`).
if srcDir != "" {
pool := sameRepo
if len(pool) == 0 {
pool = other
}
var samePkg []*graph.Node
for _, n := range pool {
if filepath.Dir(n.FilePath) == srcDir {
samePkg = append(samePkg, n)
}
}
if len(samePkg) == 1 {
return samePkg[0]
}
}
return nil // ambiguous
}
func splitLines(src []byte) []string {
return strings.Split(string(src), "\n")
}
// detectLangFromPath mirrors internal/contracts.detectLanguage so the
// enricher's language-gate fires correctly for the handler's own file.
func detectLangFromPath(path string) string {
switch {
case strings.HasSuffix(path, ".go"):
return "go"
case strings.HasSuffix(path, ".ts"), strings.HasSuffix(path, ".tsx"):
return "typescript"
case strings.HasSuffix(path, ".js"), strings.HasSuffix(path, ".jsx"):
return "javascript"
case strings.HasSuffix(path, ".py"):
return "python"
case strings.HasSuffix(path, ".java"):
return "java"
case strings.HasSuffix(path, ".kt"), strings.HasSuffix(path, ".kts"):
return "kotlin"
case strings.HasSuffix(path, ".dart"):
return "dart"
}
return ""
}
// responseHelperCallRe pulls the third argument out of a JSON-response
// helper call, e.g. `respondJSON(w, http.StatusOK, source)` → "source",
// `WriteJSON(w, 200, &result)` → "result". Matches every helper name
// the Go enricher knows about so the two pipes stay in sync.
var responseHelperCallRe = regexp.MustCompile(
`(?:[A-Za-z_]\w*\.)?(?:[Rr]espond|[Ww]rite|[Ss]end|[Rr]ender)(?:JSON|Json)\(\s*\w+\s*,\s*[^,]+?\s*,\s*&?([A-Za-z_]\w*)\s*\)`,
)
// receiverMatchesHint decides whether a method node could plausibly
// be the target of a call whose receiver chain includes `hint` as
// its penultimate segment. For `h.tucks.Update`, the hint is
// "tucks" and we accept receivers whose name (stripped of pointer
// marker) contains "tucks" case-insensitively:
//
// *TucksStore.Update ✓ (receiver "TucksStore" contains "tucks")
// *PostgresTuckStore.Update ✓ (contains "tuck")
// *EmailSources.Update ✗ (no "tucks")
//
// The hint may itself be the receiver variable (`h` in `h.Update(...)`)
// when the call has only two segments; in that case any same-repo
// method named `Update` passes — but the upstream `len(matches) != 1`
// check still demands uniqueness, which is the real guard.
func receiverMatchesHint(n *graph.Node, hint string) bool {
if hint == "" {
return true
}
// Method ID looks like "<repo>/<file>::Receiver.Method". Extract
// "Receiver" by splitting once on `::` then taking the type part
// before the last `.`.
idParts := strings.Split(n.ID, "::")
if len(idParts) < 2 {
return true // conservative: no receiver info available → don't filter out
}
last := idParts[len(idParts)-1]
dot := strings.LastIndex(last, ".")
if dot < 0 {
return true // plain function, no receiver
}
recv := strings.TrimPrefix(last[:dot], "*")
// Handle both singular and plural forms: "tucks" in hint matches
// "TuckStore" by containing "tuck", and "tuck" hint matches
// "Tucks" too. Strip a trailing `s` from the longer side to let
// singular/plural pairs match.
return strings.Contains(strings.ToLower(recv), strings.ToLower(hint)) ||
strings.Contains(strings.ToLower(recv), strings.ToLower(strings.TrimSuffix(hint, "s"))) ||
strings.Contains(strings.ToLower(recv), strings.ToLower(hint+"s"))
}
// parseFirstNonErrorReturnType walks a Go function signature and
// returns the first return type that isn't `error`. Signatures as
// stored in the graph have the form:
//
// func ((s *Store)) Get(args) (*EmailSource, error)
// func list() []*User
// func save(x Foo) error
//
// Regex-based extraction struggles with the receiver's `((...))`
// nesting and with multi-paren return groups — we parse with an
// explicit bracket-depth counter so every shape above is handled
// the same way.
// resolveCallReturnTypes is the graph-aware companion to the
// regex-based schema enricher. For every HTTP provider contract whose
// response couldn't be pinned syntactically (response_expr is set,
// response_type is empty), we:
//
// - Pull the bound variable's name out of the helper-call expression.
// - Read the handler's body from disk.
// - Find the variable's declaration line and parse the RHS call.
// - Look up the called method by name in the graph (preferring the
// same file / same repo).
// - Parse the method's signature meta for the first non-error
// return type, strip `*` / `[]`, resolve to a type node's ID.
// - Patch the contract's meta in place.
//
// This is the proper tracing the name-based heuristic only
// approximated — it follows the variable to its definition instead
// of guessing from its name.
func (idx *Indexer) resolveCallReturnTypes(reg *contracts.Registry) {
resolved := 0
bfCache := newBodyFactsCache(idx)
defer bfCache.Close()
for _, c := range reg.All() {
if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP {
continue
}
handler := idx.graph.GetNode(c.SymbolID)
bf := bfCache.For(handler)
// Path 1: bare-variable response (`return WriteJSON(w, code, resp)`).
// Trace the variable to its binding call's return type — and,
// failing that, the literal/builtin shape of its declaration —
// then stamp response_type / response_repeated. Accepts
// response_expr in two forms:
// - Bare identifier ("result") — emitted by the
// AST overlay (or the post-fix Go enricher) when the value
// is a plain var.
// - Full helper call ("WriteJSON(w, …)") — older
// extraction output, kept compatible by extracting the
// third arg via responseHelperCallRe.
if rt, _ := c.Meta["response_type"].(string); rt == "" {
respExpr, _ := c.Meta["response_expr"].(string)
varName := ""
switch {
case respExpr == "":
// nothing to work with
case isLikelyIdentifier(respExpr):
varName = respExpr
default:
if m := responseHelperCallRe.FindStringSubmatch(respExpr); len(m) >= 2 {
varName = m[1]
}
}
if varName != "" {
typeID, repeated := idx.lookupVarTypeForContract(c, bf, varName)
if typeID != "" {
items := reg.ByID(c.ID)
changed := false
for i := range items {
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
continue
}
if items[i].Meta == nil {
items[i].Meta = map[string]any{}
}
items[i].Meta["response_type"] = typeID
if repeated {
items[i].Meta["response_repeated"] = true
}
items[i].Meta["schema_source"] = "extracted"
delete(items[i].Meta, "response_expr")
changed = true
}
if changed {
reg.ReplaceByID(c.ID, items)
resolved++
}
}
}
}
// Path 2: envelope response (`map[string]any{"workspace": ws,
// "repos": repos}`). For each row that didn't resolve a type
// syntactically, trace its expression to a binding call's
// return type and patch the row in place. Pulled out as a
// separate pass so a contract whose top-level response_type
// stays unresolvable can still get per-field signal — which is
// the whole point of the envelope view.
envRaw, ok := c.Meta["response_envelope"].([]map[string]any)
if !ok {
continue
}
if !envelopeNeedsResolution(envRaw) {
continue
}
envChanged := false
for ri := range envRaw {
if t, _ := envRaw[ri]["type"].(string); t != "" {
continue
}
expr, _ := envRaw[ri]["expr"].(string)
if expr == "" {
continue
}
// Strip a leading `&` / `*` so the binding lookup sees
// the underlying identifier.
ident := strings.TrimLeft(expr, "&*")
if !isLikelyIdentifier(ident) {
continue
}
typeID, repeated := idx.lookupVarTypeForContract(c, bf, ident)
if typeID != "" {
envRaw[ri]["type"] = typeID
if repeated {
envRaw[ri]["repeated"] = true
}
envChanged = true
}
}
if !envChanged {
continue
}
// Promote schema_source to "extracted" if every row now has a
// type (or this is a single-key envelope whose lone field
// resolved). Otherwise leave it as "partial" — we have more
// info than before but it's not exhaustive.
items := reg.ByID(c.ID)
patched := false
for i := range items {
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
continue
}
if items[i].Meta == nil {
items[i].Meta = map[string]any{}
}
items[i].Meta["response_envelope"] = envRaw
if envelopeFullyTyped(envRaw) {
items[i].Meta["schema_source"] = "extracted"
}
patched = true
}
if patched {
reg.ReplaceByID(c.ID, items)
resolved++
}
}
if resolved > 0 {
idx.logger.Info("resolved response types from call signatures",
zap.Int("count", resolved))
}
}
func envelopeNeedsResolution(env []map[string]any) bool {
for _, row := range env {
if t, _ := row["type"].(string); t == "" {
return true
}
}
return false
}
func envelopeFullyTyped(env []map[string]any) bool {
if len(env) == 0 {
return false
}
for _, row := range env {
if t, _ := row["type"].(string); t == "" {
return false
}
}
return true
}
// upgradeBareTypeName looks up `name` in the graph and returns the
// matching type node's ID, preferring same-repo matches. Falls back
// to the input string when no graph type is found, so callers can
// still surface primitives ("string", "int", "bool") and external
// types ("map[string]int") that have no graph node. The shape-
// inlining pass will leave those as-is since lookupShape requires a
// `::` separator.
func (idx *Indexer) upgradeBareTypeName(name, repoHint string) string {
if name == "" {
return name
}
if strings.Contains(name, "::") {
return name // already a graph ID
}
candidates := idx.graph.FindNodesByName(name)
var fallback *graph.Node
for _, n := range candidates {
if n.Kind != graph.KindType {
continue
}
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
return n.ID
}
if fallback == nil {
fallback = n
}
}
if fallback != nil {
return fallback.ID
}
return name
}
// isLikelyIdentifier accepts the bare-identifier and dotted-path
// forms that traceVarTypeFromBody can match against a binding line.
// Compound expressions ("len(repos)", "&Foo{}") are out of scope —
// they'd need a more thorough RHS parser than the regex chain here.
func isLikelyIdentifier(s string) bool {
if s == "" {
return false
}
for i, r := range s {
switch {
case r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z'):
continue
case i > 0 && r >= '0' && r <= '9':
continue
case r == '.' && i > 0:
continue
default:
return false
}
}
return true
}
// lookupVarTypeForContract resolves a variable to its return type
// using BodyFacts (AST-driven, structurally correct). Returns
// (typeID, repeated) or ("", false) when the binding can't be
// resolved.
//
// AST-only: phase 1b deleted the body-text regex fallback. Languages
// without a BodyFactsFactory get nopBodyFacts (which returns empty
// Bindings), so this function is a no-op for non-Go contracts.
// Their per-file regex enricher in schema_enrich_<lang>.go still runs
// and populates request_type / response_type via the framework
// detectors — only the post-pass cross-handler trace is AST-only.
func (idx *Indexer) lookupVarTypeForContract(
c contracts.Contract,
bf contracts.BodyFacts,
varName string,
) (string, bool) {
if bf == nil {
return "", false
}
b := bf.VarBinding(varName)
// Highest tier: BindingResolver (go/types via goanalysis.Provider)
// returns compiler-resolved types. When --semantic is enabled and
// the provider has run, this is authoritative for any binding
// whose source line we tracked.
if br := contracts.CurrentBindingResolver(); br != nil && b.Line > 0 {
if typeName, ok := br.LookupTypeAtLine(c.FilePath, b.Line); ok && typeName != "" {
return idx.upgradeBareTypeName(typeName, c.RepoPrefix), b.Repeated
}
}
switch b.Kind {
case contracts.BindingMethodCall, contracts.BindingFuncCall:
if b.CallExpr != "" {
if typeID, repeated, _ := idx.resolveCallExprToType(b.CallExpr, c.RepoPrefix); typeID != "" {
return typeID, repeated
}
}
default:
// Composite / slice / map / literal / path-value /
// header-value / form-value / query-get — already typed.
if b.TypeID != "" {
return idx.upgradeBareTypeName(b.TypeID, c.RepoPrefix), b.Repeated
}
}
return "", false
}
// resolveCallExprToType walks the graph from a call expression like
// `h.svc.GetRepos` to the called method/function's first non-error
// return type, returning (typeID, repeated, pointer). Empty string
// when the call is ambiguous (multiple candidates with disagreeing
// signatures) or the receiver doesn't match any graph node.
//
// Extracted from traceVarTypeFromBodyWithShape so BodyFacts-driven
// callers can reuse the graph walk without going through the regex
// path. The regex-driven traceVarTypeFromBodyWithShape is the
// fallback for non-Go languages until they ship a BodyFacts
// implementation.
func (idx *Indexer) resolveCallExprToType(callExpr, repoHint string) (string, bool, bool) {
if callExpr == "" {
return "", false, false
}
// Split the call path. `h.tucks.Update` → ["h", "tucks", "Update"].
// The last segment is the method name; the penultimate is the
// receiver field / package, which we use to disambiguate when
// multiple methods share the name.
parts := strings.Split(callExpr, ".")
methodName := parts[len(parts)-1]
if methodName == "" {
return "", false, false
}
var receiverHint string
if len(parts) >= 2 {
receiverHint = parts[len(parts)-2]
}
candidates := idx.graph.FindNodesByName(methodName)
var matches []*graph.Node
for _, n := range candidates {
if n.Kind != graph.KindMethod && n.Kind != graph.KindFunction {
continue
}
if repoHint != "" && !strings.HasPrefix(n.ID, repoHint+"/") {
continue
}
if receiverHint != "" && !receiverMatchesHint(n, receiverHint) {
continue
}
matches = append(matches, n)
}
if len(matches) == 0 {
return "", false, false
}
// Interface + implementation stacks often produce multiple
// receivers that share the same method signature — a production
// postgres store and a mock test store both implement
// `emailSources.Update(...) (*EmailSource, error)`. Parse every
// candidate's signature; if they all agree on the first
// non-error return type, use that. Otherwise bail so we don't
// attribute a wrong type silently.
var retType string
for _, m := range matches {
if m.Meta == nil {
continue
}
sig, _ := m.Meta["signature"].(string)
t := parseFirstNonErrorReturnType(sig)
if t == "" {
continue
}
// De-prioritise mock receivers: if a non-mock candidate
// later disagrees we want that to be the authoritative one.
if retType == "" {
retType = t
continue
}
if t != retType {
// Candidates disagree — can't tell which wins. The
// caller sees the raw expression and can drill in.
return "", false, false
}
}
if retType == "" {
return "", false, false
}
// Capture slice/pointer flags before stripping so the caller can
// render `[Foo]` / `*Foo` correctly. Order matters: a return type
// like `[]*Foo` is reported as repeated AND pointer.
repeated := strings.HasPrefix(retType, "[]")
pointer := strings.HasPrefix(retType, "*") ||
(repeated && strings.HasPrefix(retType[2:], "*"))
// Strip `*` / `[]` / package qualifier so resolveTypeByName can
// match the plain type-node name.
retType = strings.TrimLeft(retType, "*[]")
if dot := strings.LastIndex(retType, "."); dot >= 0 {
retType = retType[dot+1:]
}
// Look up the type node, preferring same-repo matches.
typeCandidates := idx.graph.FindNodesByName(retType)
var bestType *graph.Node
for _, n := range typeCandidates {
if n.Kind != graph.KindType {
continue
}
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
bestType = n
break
}
if bestType == nil {
bestType = n
}
}
if bestType != nil {
return bestType.ID, repeated, pointer
}
// Bare name — downstream UpgradeBareTypeRefs can still upgrade
// it later, but we return it as-is so the consumer sees something
// real.
return retType, repeated, pointer
}
func parseFirstNonErrorReturnType(sig string) string {
sig = strings.TrimSpace(sig)
if !strings.HasPrefix(sig, "func") {
return ""
}
sig = strings.TrimSpace(strings.TrimPrefix(sig, "func"))
// Optional receiver. Two forms to recognise:
// `((*Recv)) Name(params)` — gortex's stored double-paren form
// `(r *Recv) Name(params)` — standard Go source form
// In both cases a function name follows the receiver parens.
// Anonymous function types (`func(a, b) (c, d)`) have no
// receiver — the first `(` opens the parameter list and is
// followed by another `(` for the return group or end-of-string.
// Disambiguate by peeking past the first balanced `(...)` group
// for an identifier letter.
if strings.HasPrefix(sig, "(") {
end := findBalancedParenEnd(sig)
if end < 0 {
return ""
}
afterFirstGroup := strings.TrimSpace(sig[end+1:])
if len(afterFirstGroup) > 0 {
r := afterFirstGroup[0]
isIdent := (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_'
if isIdent {
sig = afterFirstGroup
}
}
}
// Skip the function name — everything up to the parameter list's
// opening `(`.
if i := strings.Index(sig, "("); i >= 0 {
sig = sig[i:]
} else {
return ""
}
// Skip parameter list.
end := findBalancedParenEnd(sig)
if end < 0 {
return ""
}
sig = strings.TrimSpace(sig[end+1:])
if sig == "" {
return ""
}
// Return clause — either `(T1, T2, ...)` or a bare single type.
var inner string
if strings.HasPrefix(sig, "(") {
end := findBalancedParenEnd(sig)
if end < 0 {
return ""
}
inner = sig[1:end]
} else {
inner = sig
}
return firstNonErrorReturnField(inner)
}
// findBalancedParenEnd returns the index of the `)` that closes the
// `(` at s[0]. Returns -1 when the parens don't balance.
func findBalancedParenEnd(s string) int {
if len(s) == 0 || s[0] != '(' {
return -1
}
depth := 0
for i := 0; i < len(s); i++ {
switch s[i] {
case '(':
depth++
case ')':
depth--
if depth == 0 {
return i
}
}
}
return -1
}
// firstNonErrorReturnField splits a return-clause body by top-level
// commas and returns the first type expression that isn't `error`.
// Named return parameters (`result *User`) are handled by taking the
// last whitespace-separated token as the type.
func firstNonErrorReturnField(inner string) string {
var fields []string
depth := 0
start := 0
for i := 0; i < len(inner); i++ {
switch inner[i] {
case '(', '[':
depth++
case ')', ']':
depth--
case ',':
if depth == 0 {
fields = append(fields, strings.TrimSpace(inner[start:i]))
start = i + 1
}
}
}
if last := strings.TrimSpace(inner[start:]); last != "" {
fields = append(fields, last)
}
for _, f := range fields {
t := f
// Named return: `ctx context.Context`, `result *User`. Grab
// the last whitespace-separated token — that's the type.
if parts := strings.Fields(f); len(parts) > 1 {
t = parts[len(parts)-1]
}
if t == "error" || strings.HasSuffix(t, ".error") {
continue
}
return t
}
return ""
}
// snapshotContractShapes walks every request_type / response_type
// reference in the registry, loads each referenced type node's source,
// and attaches the extracted Shape to the node's Meta["shape"].
//
// We:
// - Collect the unique set of symbol IDs — a popular DTO might be a
// request/response on dozens of routes and we want to parse its
// source once.
// - Read each file once (cached in the source map).
// - Skip nodes whose ID doesn't look like a symbol (bare names that
// couldn't be upgraded) — those have nothing to dereference.
// - Skip type nodes that already have a shape attached from a prior
// pass on the same session (ETag-style short-circuit).
func (idx *Indexer) snapshotContractShapes(reg *contracts.Registry) {
symbols := make(map[string]struct{})
for _, c := range reg.All() {
for _, key := range []string{"request_type", "response_type"} {
v, _ := c.Meta[key].(string)
if v == "" || !strings.Contains(v, "::") {
continue
}
symbols[v] = struct{}{}
}
// Envelope rows reference types the dashboard wants expanded
// just as much as the top-level response_type does — without
// snapshotting them here, the inlineEnvelopeShapes pass below
// finds no shape to fold into the row.
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok {
for _, row := range env {
v, _ := row["type"].(string)
if v == "" || !strings.Contains(v, "::") {
continue
}
symbols[v] = struct{}{}
}
}
}
if len(symbols) == 0 {
return
}
srcCache := make(map[string][]byte)
attached := 0
for id := range symbols {
node := idx.graph.GetNode(id)
if node == nil {
continue
}
// Accept both KindType and KindInterface — TypeScript /
// Java / Kotlin model their type defs as interfaces, and
// the dashboard wants their fields expanded just like Go
// struct types. Limiting to KindType silently dropped every
// TS interface shape extraction.
if node.Kind != graph.KindType && node.Kind != graph.KindInterface {
continue
}
if _, done := node.Meta["shape"]; done {
continue
}
src, ok := srcCache[node.FilePath]
if !ok {
// File paths in the graph are repo-prefixed; trim the
// prefix for disk I/O.
diskPath := node.FilePath
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
}
diskPath = filepath.Join(idx.rootPath, diskPath)
data, err := os.ReadFile(diskPath)
if err != nil {
srcCache[node.FilePath] = nil
continue
}
srcCache[node.FilePath] = data
src = data
}
if src == nil {
continue
}
shape := contracts.ExtractShape(node.FilePath, src, node.StartLine, node.EndLine)
if shape == nil {
continue
}
if node.Meta == nil {
node.Meta = map[string]any{}
}
node.Meta["shape"] = shape
attached++
}
if attached > 0 {
idx.logger.Info("contract shapes snapshotted",
zap.Int("types", attached),
zap.Int("examined", len(symbols)))
}
}
// inlineEnvelopeShapes folds each type node's snapshotted shape onto
// every response_envelope row that references it. After this pass an
// envelope row carries the full JSON-rendering data:
//
// {
// "name": "repos",
// "expr": "repos",
// "type": "<repo>/service.go::Repo",
// "shape": { "kind": "struct", "fields": [...] }
// }
//
// so the dashboard can render the actual response shape instead of a
// bare type-symbol-ID. Idempotent: rows that already carry "shape"
// are skipped, which lets cross-pass calls (re-extraction, snapshot
// restore) re-run cheaply.
//
// Also handles the top-level response_type / request_type: the
// referenced type's shape is mirrored onto Meta["response_shape"] /
// Meta["request_shape"] so plain-typed responses (no map envelope)
// also expose their JSON object shape on the dashboard.
func (idx *Indexer) inlineEnvelopeShapes(reg *contracts.Registry) {
inlined := 0
for _, c := range reg.All() {
changed := false
// Envelope rows.
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
for ri, row := range env {
if _, has := row["shape"]; has {
continue
}
if shape := idx.lookupShape(row["type"]); shape != nil {
env[ri]["shape"] = shape
changed = true
}
}
if changed {
items := reg.ByID(c.ID)
for i := range items {
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
continue
}
if items[i].Meta == nil {
items[i].Meta = map[string]any{}
}
items[i].Meta["response_envelope"] = env
}
reg.ReplaceByID(c.ID, items)
}
}
// Top-level request/response shapes — same idea, applied to a
// plain `response_type: "<id>::Foo"` so the schema view can
// render the JSON object even when there's no envelope wrapper.
topChanged := false
for metaKey, shapeKey := range map[string]string{
"response_type": "response_shape",
"request_type": "request_shape",
} {
if _, has := c.Meta[shapeKey]; has {
continue
}
if shape := idx.lookupShape(c.Meta[metaKey]); shape != nil {
items := reg.ByID(c.ID)
for i := range items {
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
continue
}
if items[i].Meta == nil {
items[i].Meta = map[string]any{}
}
items[i].Meta[shapeKey] = shape
}
reg.ReplaceByID(c.ID, items)
topChanged = true
}
}
if changed || topChanged {
inlined++
}
}
if inlined > 0 {
idx.logger.Info("response envelopes hydrated with shapes",
zap.Int("contracts", inlined))
}
}
// lookupShape resolves a Meta type reference to the snapshotted shape
// stored on its graph node. Accepts string IDs (the only form used in
// today's pipeline); other shapes pass through as nil so callers can
// chain without type-asserting upstream.
func (idx *Indexer) lookupShape(raw any) any {
id, ok := raw.(string)
if !ok || id == "" || !strings.Contains(id, "::") {
return nil
}
node := idx.graph.GetNode(id)
if node == nil || node.Meta == nil {
return nil
}
shape, ok := node.Meta["shape"]
if !ok || shape == nil {
return nil
}
return shape
}
// extractExternalModules parses the repo's go.mod once and writes
// KindModule nodes plus EdgeDependsOnModule edges into the graph.
// A synthetic KindFile node is emitted for go.mod itself so the
// edges have a real source endpoint that survives applyRepoPrefix.
// Safe to call when no go.mod exists. Other manifest formats
// (package.json, pnpm-lock, requirements.txt, Cargo.toml, …) are
// future additions — each lands as another switch case here.
//
// Import-node → module-node edges (per the broader coverage spec)
// are deferred to v2; the v1 file-level edge is already enough for
// agents asking "what does this repo depend on".
func (idx *Indexer) extractExternalModules() {
if !idx.config.Coverage.IsEnabled("modules") {
return
}
// Walk known manifest formats at the repo root. Each manifest
// produces an independent Spec list and gets its own synthetic
// file node — the file→module edge stays scoped to the
// originating manifest so cross-ecosystem queries (e.g. "what
// does package.json declare") don't bleed into go.mod's
// answer.
manifests := []struct {
path string
parse func([]byte) []modules.Spec
ownPathFromSrc func([]byte) string
}{
{
path: "go.mod",
parse: modules.ParseGoMod,
ownPathFromSrc: readGoModModulePath,
},
{
path: "package.json",
parse: modules.ParsePackageJSON,
ownPathFromSrc: readPackageJSONOwnName,
},
{
path: "package-lock.json",
parse: modules.ParsePackageLockJSON,
ownPathFromSrc: nil, // package-lock has no own-name notion separate from package.json
},
{
path: "yarn.lock",
parse: modules.ParseYarnLock,
ownPathFromSrc: nil,
},
{
path: "pnpm-lock.yaml",
parse: modules.ParsePnpmLock,
ownPathFromSrc: nil,
},
{
path: "pyproject.toml",
parse: modules.ParsePyProject,
ownPathFromSrc: readPyProjectOwnName,
},
{
path: "requirements.txt",
parse: modules.ParseRequirementsTxt,
ownPathFromSrc: nil, // requirements.txt has no own-name notion
},
{
path: "Cargo.toml",
parse: modules.ParseCargoToml,
ownPathFromSrc: readCargoTomlOwnName,
},
{
path: "pom.xml",
parse: modules.ParsePomXML,
ownPathFromSrc: readPomXMLOwnName,
},
}
for _, m := range manifests {
idx.extractOneModuleManifest(m.path, m.parse, m.ownPathFromSrc)
}
// After per-manifest module extraction, detect whether this repo's
// root is a package-manager workspace and materialise its
// root→member edges.
idx.extractPackageWorkspace()
}
// extractOneModuleManifest reads a single manifest file from the
// repo root, parses it via the supplied parser, and writes the
// resulting nodes/edges + import-link edges into the graph. Used
// from extractExternalModules's per-manifest dispatch.
func (idx *Indexer) extractOneModuleManifest(relPath string, parse func([]byte) []modules.Spec, ownPathFromSrc func([]byte) string) {
manifestAbs := filepath.Join(idx.rootPath, relPath)
src, err := os.ReadFile(manifestAbs)
if err != nil {
return
}
specs := parse(src)
nodes, edges := modules.BuildGraphArtifacts(relPath, specs)
if len(nodes) == 0 && len(edges) == 0 {
return
}
// Synthetic file node for the manifest — it isn't represented
// through the language-extractor pipeline (no extractor is
// registered for the .mod extension; package.json may have one
// but the JSON walker doesn't emit a synthetic file node we
// can reuse), so the EdgeDependsOnModule edges would otherwise
// dangle from a missing source endpoint after applyRepoPrefix
// runs in multi-repo mode.
manifestNode := &graph.Node{
ID: relPath,
Kind: graph.KindFile,
Name: filepath.Base(relPath),
FilePath: relPath,
Language: manifestLanguage(relPath),
}
allNodes := append([]*graph.Node{manifestNode}, nodes...)
idx.applyRepoPrefix(allNodes, edges)
idx.graph.AddBatch(allNodes, edges)
// Connect each KindImport node to its matching module via
// longest-prefix path resolution. Repo-internal imports (the
// own-module path) are filtered inside LinkImports — when the
// manifest doesn't expose an own-name, the filter is a no-op
// which is safe (no own-module imports to match against).
var ownModulePath string
if ownPathFromSrc != nil {
ownModulePath = ownPathFromSrc(src)
}
// Scope the walk to this repo's own import nodes. The unscoped
// LinkImports walks g.AllNodes(); under a warmup loop across
// hundreds of repos that's O(R · N). The per-repo byRepo bucket
// keeps this O(repo size).
repoNodes := idx.graph.GetRepoNodes(idx.repoPrefix)
importNodes := make([]*graph.Node, 0, len(repoNodes))
for _, n := range repoNodes {
if n.Kind == graph.KindImport {
importNodes = append(importNodes, n)
}
}
modules.LinkImportsIn(idx.graph, importNodes, specs, ownModulePath)
}
// manifestLanguage returns the language tag stamped on a manifest's
// synthetic file node. Used purely for Brief listings — the kind
// field carries the structural type.
func manifestLanguage(relPath string) string {
switch filepath.Base(relPath) {
case "go.mod":
return "go"
case "package.json", "package-lock.json":
return "json"
case "yarn.lock":
return "yarn"
case "pnpm-lock.yaml":
return "yaml"
case "pyproject.toml", "Cargo.toml":
return "toml"
case "requirements.txt":
return "text"
case "pom.xml":
return "xml"
}
return ""
}
// readPomXMLOwnName builds the project's own Maven coordinate
// (groupId:artifactId) so LinkImports can filter self-references.
// Java workspace setups where a sibling module imports the parent
// project shouldn't accidentally resolve to an external dep with
// the same coordinate. Returns "" when either field is missing —
// the LinkImports filter treats "" as no own-module filter, which
// is safe.
func readPomXMLOwnName(src []byte) string {
var manifest struct {
GroupID string `xml:"groupId"`
ArtifactID string `xml:"artifactId"`
}
if err := xml.Unmarshal(src, &manifest); err != nil {
return ""
}
if manifest.GroupID == "" || manifest.ArtifactID == "" {
return ""
}
return manifest.GroupID + ":" + manifest.ArtifactID
}
// readCargoTomlOwnName reads the crate's own name from
// `[package] name`. Used for LinkImports's own-module filter so
// workspace-internal crate references don't accidentally match
// external crates with the same name.
func readCargoTomlOwnName(src []byte) string {
var manifest struct {
Package struct {
Name string `toml:"name"`
} `toml:"package"`
}
if err := toml.Unmarshal(src, &manifest); err != nil {
return ""
}
return manifest.Package.Name
}
// readPyProjectOwnName returns the package's own name from the
// pyproject.toml `[project] name` field. Used by LinkImports's
// own-module filter so a workspace package's internal imports
// don't accidentally collide with external pypi names. Returns ""
// on parse error or when the field is absent.
func readPyProjectOwnName(src []byte) string {
var manifest struct {
Project struct {
Name string `toml:"name"`
} `toml:"project"`
}
if err := toml.Unmarshal(src, &manifest); err != nil {
return ""
}
return manifest.Project.Name
}
// readPackageJSONOwnName extracts the manifest's `name` field — the
// npm equivalent of go.mod's `module` directive. Returns "" on
// missing or malformed JSON; LinkImports treats "" as "no own-
// module filter", which is safe because internal-package matches
// (e.g. workspaces) won't accidentally collide with external deps
// at the longest-prefix scan.
func readPackageJSONOwnName(src []byte) string {
var manifest struct {
Name string `json:"name"`
}
if err := json.Unmarshal(src, &manifest); err != nil {
return ""
}
return manifest.Name
}
// readGoModModulePath extracts the `module ` directive value from
// go.mod source. Mirrors the inline parse in coverage.ReadModulePath
// — we keep both copies tiny rather than introducing a one-import
// shared helper that would force a layering compromise (coverage
// shouldn't depend on indexer; indexer shouldn't depend on coverage).
func readGoModModulePath(src []byte) string {
for _, line := range strings.Split(string(src), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "module ") {
return strings.TrimSpace(strings.TrimPrefix(line, "module "))
}
}
return ""
}
// extractGoModContracts runs the go.mod-specific extractor once against
// the repo root (go.mod isn't represented as a file node in the graph).
// Results are added to reg. Safe to call when no go.mod exists.
//
// Also materialises the dep::<module> contracts as graph nodes
// immediately, so the resolver's import-bridge (Resolver.lookupDepModule)
// can find them during ResolveAll. commitContracts later AddNode is
// idempotent — it skips nodes that already exist — so this doesn't
// double-emit. We only do this for type=dependency; everything else
// goes through the normal commit path which depends on a resolved
// graph (UpgradeBareTypeRefs, resolveProviderHandlers).
func (idx *Indexer) extractGoModContracts(reg *contracts.Registry) {
goModPath := filepath.Join(idx.rootPath, "go.mod")
goModSrc, err := os.ReadFile(goModPath)
if err != nil {
return
}
goModExtractor := &contracts.GoModExtractor{TrackedRepos: idx.trackedRepoModules}
goModFilePath := "go.mod"
if idx.repoPrefix != "" {
goModFilePath = idx.repoPrefix + "/go.mod"
}
found := goModExtractor.Extract(goModFilePath, goModSrc, nil, nil)
reg.AddAllScoped(found, idx.repoPrefix, idx.workspaceID, idx.projectID)
var nodes []*graph.Node
for i := range found {
c := found[i]
if c.Type != contracts.ContractDependency {
continue
}
if idx.graph.GetNode(c.ID) != nil {
continue
}
nodes = append(nodes, &graph.Node{
ID: c.ID,
Kind: graph.KindContract,
Name: c.ID,
FilePath: c.FilePath,
Language: "contract",
RepoPrefix: idx.repoPrefix,
Meta: map[string]any{"type": string(c.Type), "role": string(c.Role)},
})
}
if len(nodes) > 0 {
idx.graph.AddBatch(nodes, nil)
}
}
// extractContracts scans all file nodes in the graph and runs contract
// extractors to detect API contracts (HTTP routes, gRPC services,
// GraphQL, topics, etc.). Detected contracts are added as graph nodes
// with provides/consumes edges.
//
// This full-walk path is used by IncrementalReindex (where many files
// are already cached). IndexCtx instead runs the per-file work inline
// with parsing — see the worker loop — and skips this function.
func (idx *Indexer) extractContracts() {
reg := contracts.NewRegistry()
_, byLang := idx.buildPerFileContractExtractors()
// Track which file nodes we saw this pass so we can prune stale
// cache entries afterwards (files that left the graph).
seenFiles := make(map[string]struct{})
// Multi-repo mode: walk only this repo's nodes. The previous
// AllNodes()-then-filter pass paid an O(global) walk per repo,
// which compounded with hundreds of tracked siblings.
var nodes []*graph.Node
if idx.repoPrefix != "" {
nodes = idx.graph.GetRepoNodes(idx.repoPrefix)
} else {
nodes = idx.graph.AllNodes()
}
// Pre-bucket the already-fetched node slice by FilePath so the
// per-file body can look up its co-located nodes in O(1) instead
// of firing a fresh GetFileNodes query per file. Likewise pre-
// fetch every out-edge whose source is in this repo as ONE backend
// call and bucket by From so the per-file body can replace
// GetOutEdges(fileNode.ID) — on disk backends the per-file query
// path was the second-largest source of round-trips in
// deferred_passes (after the DI walk).
nodesByFile := make(map[string][]*graph.Node, len(nodes))
for _, n := range nodes {
if n == nil {
continue
}
nodesByFile[n.FilePath] = append(nodesByFile[n.FilePath], n)
}
var edgesByFrom map[string][]*graph.Edge
if idx.repoPrefix != "" {
repoEdges := idx.graph.GetRepoEdges(idx.repoPrefix)
edgesByFrom = make(map[string][]*graph.Edge, len(nodes))
for _, e := range repoEdges {
if e == nil {
continue
}
edgesByFrom[e.From] = append(edgesByFrom[e.From], e)
}
}
for _, fileNode := range nodes {
if fileNode.Kind != graph.KindFile {
continue
}
// In multi-repo mode the byRepo bucket is already scoped, but
// the path-prefix strip below still needs to run.
relPath := fileNode.FilePath
if idx.repoPrefix != "" {
if !strings.HasPrefix(relPath, idx.repoPrefix+"/") {
continue
}
relPath = strings.TrimPrefix(relPath, idx.repoPrefix+"/")
}
absPath := filepath.Join(idx.rootPath, relPath)
info, statErr := os.Stat(absPath)
if statErr != nil {
continue
}
seenFiles[fileNode.FilePath] = struct{}{}
currentMtime := info.ModTime().UnixNano()
// Cache hit: replay the previously-extracted contracts without
// re-reading the file or re-running the 8 extractors. This is
// the dominant savings path on repos with many files where most
// haven't changed since the last extraction (e.g. live re-index
// after a single-file edit).
idx.contractCacheMu.RLock()
cached, ok := idx.contractCache[fileNode.FilePath]
idx.contractCacheMu.RUnlock()
if ok && cached.mtimeNano == currentMtime {
for _, c := range cached.contracts {
reg.Add(c)
}
continue
}
src, err := os.ReadFile(absPath)
if err != nil {
continue
}
var fileNodes []*graph.Node
var fileEdges []*graph.Edge
if idx.repoPrefix != "" {
fileNodes = nodesByFile[fileNode.FilePath]
fileEdges = edgesByFrom[fileNode.ID]
} else {
fileNodes = idx.graph.GetFileNodes(fileNode.FilePath)
fileEdges = idx.graph.GetOutEdges(fileNode.ID)
}
// Language-filtered dispatch: skip extractors that don't list
// this file's language in SupportedLanguages(). On big repos
// with lots of .css / .svg / .json etc. this cuts a lot of
// no-op extractor calls.
exts := byLang[fileNode.Language]
// Re-parse for AST overlay — the language extractor's tree
// from the original index pass was closed when the file was
// first added. Cheap (~milliseconds per file) and cleanly
// scoped to this contract-pass invocation.
tree := contracts.ParseTreeForLang(fileNode.Language, src)
fileContracts := idx.runContractExtractorsForFile(
fileNode.FilePath, src, fileNodes, fileEdges, exts, tree)
tree.Release()
for _, c := range fileContracts {
reg.Add(c)
}
idx.contractCacheMu.Lock()
idx.contractCache[fileNode.FilePath] = &contractCacheEntry{
mtimeNano: currentMtime,
contracts: fileContracts,
}
idx.contractCacheMu.Unlock()
}
// Prune cache entries for files that are no longer in the graph
// (deleted, or repo untracked). Otherwise the cache grows unboundedly
// across the lifetime of the daemon.
idx.contractCacheMu.Lock()
for path := range idx.contractCache {
if _, ok := seenFiles[path]; !ok {
delete(idx.contractCache, path)
}
}
idx.contractCacheMu.Unlock()
idx.extractGoModContracts(reg)
idx.extractDIContracts(reg)
idx.commitContracts(reg)
}
// IsStale returns true if the file at relPath has been modified on disk since
// it was last indexed, based on comparing stored mtime against current disk mtime.
//
// relPath is folded to the canonical key (slash form, Unicode NFC)
// before lookup so a caller passing a non-ASCII path in a different
// Unicode form than fileMtimes was keyed with still resolves — without
// the fold the lookup would miss and the file be reported permanently
// stale, re-indexing it under a second key on every pass.
// HasChangesSinceMtimes reports whether any indexable file under root
// changed (mtime differs or is new) or was deleted, relative to the
// indexer's currently-loaded fileMtimes. It runs the SAME walk +
// staleness + deletion logic as IncrementalReindex but writes nothing.
//
// The daemon warmup uses it to choose a reconcile strategy for a
// reopened repo: a repo with zero changes takes the fast no-op
// IncrementalReindex path, while a repo that changed while the daemon
// was down is routed through the shadow/bulk re-track path instead.
// That routing matters because IncrementalReindex re-resolves changed
// files through per-edge graph.ReindexEdges, and the per-edge write
// path against a freshly reopened disk store is slow and unreliable.
// The shadow path resolves entirely in an in-memory graph and commits
// the result in one bulk load, so it never issues a per-edge write to
// the reopened store. It re-indexes the whole repo (more work than a
// true incremental pass), but it is reliable, and only repos that
// actually changed during downtime pay the cost.
//
// Conservative on error: anything it can't determine (bad root, walk
// error) returns true so the caller re-indexes rather than silently
// serving a stale graph.
func (idx *Indexer) HasChangesSinceMtimes(root string) bool {
absRoot, err := filepath.Abs(root)
if err != nil {
return true
}
idx.storeRootPath(absRoot)
diskFiles := make(map[string]bool)
errStop := errors.New("stop-walk")
walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error {
if werr != nil {
return nil
}
if d.IsDir() {
if idx.shouldPruneDir(path, absRoot) {
return filepath.SkipDir
}
return nil
}
if _, ok := idx.effectiveLanguage(path, nil); !ok {
return nil
}
if idx.shouldExclude(path, absRoot, false) {
return nil
}
rel := idx.relKey(path)
diskFiles[rel] = true
if idx.IsStale(rel) {
return errStop // a single changed/new file is enough
}
return nil
})
if errors.Is(walkErr, errStop) {
return true
}
if walkErr != nil {
return true
}
// Deletion check: a previously-indexed file absent from the walk and
// confirmed gone from disk counts as a change (its edges must drop).
idx.mtimeMu.RLock()
var candidates []string
for rel := range idx.fileMtimes {
if !diskFiles[rel] {
candidates = append(candidates, rel)
}
}
idx.mtimeMu.RUnlock()
for _, rel := range candidates {
if _, err := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(err, os.ErrNotExist) {
return true
}
}
return false
}
// ChangedSinceMtimes is the accumulating variant of HasChangesSinceMtimes:
// it runs the SAME walk + IsStale staleness predicate + deletion-confirm
// logic but, instead of short-circuiting on the first stale file, returns
// the full changed and deleted sets as in-repo relative slash-paths.
//
// The warm-restart reconcile router uses the census to size the reconcile:
// an empty result takes the fast no-op path, a small delta is scoped to
// exactly those files (re-index the changed, evict the deleted), and only a
// large-fraction churn falls back to a whole-repo re-track.
//
// changed holds files that are new or whose mtime drifted since the last
// pass; deleted holds previously-indexed files confirmed gone from disk. A
// walk (or abs-path) error returns a non-nil err with nil slices — the
// caller treats that as "unknown, do a full re-track", exactly as
// HasChangesSinceMtimes conservatively returns true on the same condition.
func (idx *Indexer) ChangedSinceMtimes(root string) (changed []string, deleted []string, err error) {
absRoot, absErr := filepath.Abs(root)
if absErr != nil {
return nil, nil, absErr
}
idx.storeRootPath(absRoot)
diskFiles := make(map[string]bool)
walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error {
if werr != nil {
return nil
}
if d.IsDir() {
if idx.shouldPruneDir(path, absRoot) {
return filepath.SkipDir
}
return nil
}
if _, ok := idx.effectiveLanguage(path, nil); !ok {
return nil
}
if idx.shouldExclude(path, absRoot, false) {
return nil
}
rel := idx.relKey(path)
diskFiles[rel] = true
if idx.IsStale(rel) {
changed = append(changed, rel)
}
return nil
})
if walkErr != nil {
return nil, nil, walkErr
}
// Deletion check: a previously-indexed file absent from the walk and
// confirmed gone from disk counts as a change (its edges must drop).
idx.mtimeMu.RLock()
var candidates []string
for rel := range idx.fileMtimes {
if !diskFiles[rel] {
candidates = append(candidates, rel)
}
}
idx.mtimeMu.RUnlock()
for _, rel := range candidates {
if _, statErr := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(statErr, os.ErrNotExist) {
deleted = append(deleted, rel)
}
}
return changed, deleted, nil
}
func (idx *Indexer) IsStale(relPath string) bool {
relPath = pathkey.Normalize(filepath.ToSlash(relPath))
idx.mtimeMu.RLock()
storedMtime, ok := idx.fileMtimes[relPath]
idx.mtimeMu.RUnlock()
if !ok {
// Unknown file — treat as stale.
return true
}
absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath))
info, err := os.Stat(absPath)
if err != nil {
// Can't stat — treat as stale.
return true
}
return info.ModTime().UnixNano() != storedMtime
}
// IsTrackedStale reports whether a file that IS in the index has changed
// on disk since it was indexed. Unlike IsStale it returns false for an
// untracked path (a new file, a non-source path, or a path-form
// mismatch), so a freshness signal never false-positives on a file the
// index legitimately does not cover.
func (idx *Indexer) IsTrackedStale(relPath string) bool {
return idx.TrackedFileState(relPath) == FileStale
}
// FileFreshness is the verdict TrackedFileState returns for a tracked file:
// its indexed view is current, has drifted, or the file is gone from disk.
type FileFreshness string
const (
// FileFresh means the file is not tracked (so no claim is made) or its
// on-disk mtime still matches what was indexed.
FileFresh FileFreshness = "fresh"
// FileStale means the file is tracked and its on-disk mtime differs from
// the indexed mtime — the graph view lags the working tree.
FileStale FileFreshness = "stale"
// FileMissing means the file is tracked in the graph but no longer exists
// on disk — it was deleted or moved since indexing. A plain staleness
// check folds this into "not stale"; the distinct verdict lets a list
// result flag hits that point at vanished files.
FileMissing FileFreshness = "missing"
)
// TrackedFileState classifies one repo-relative file against the indexer's
// recorded mtimes: fresh (untracked or unchanged), stale (mtime drift), or
// missing (tracked but absent on disk). Splitting the stat-failure branch out
// of IsTrackedStale is what lets the freshness rider distinguish a stale hit
// from one whose underlying file has been deleted.
func (idx *Indexer) TrackedFileState(relPath string) FileFreshness {
relPath = pathkey.Normalize(filepath.ToSlash(relPath))
idx.mtimeMu.RLock()
storedMtime, ok := idx.fileMtimes[relPath]
idx.mtimeMu.RUnlock()
if !ok {
return FileFresh
}
absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath))
info, err := os.Stat(absPath)
if err != nil {
if os.IsNotExist(err) {
return FileMissing
}
// A transient / permission stat error is not evidence of drift —
// don't cry wolf for a file we simply could not read.
return FileFresh
}
if info.ModTime().UnixNano() != storedMtime {
return FileStale
}
return FileFresh
}