a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
7511 lines
287 KiB
Go
7511 lines
287 KiB
Go
package indexer
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"encoding/xml"
|
||
"errors"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/pelletier/go-toml/v2"
|
||
"go.uber.org/zap"
|
||
"golang.org/x/sync/semaphore"
|
||
|
||
"github.com/zzet/gortex/internal/codegen"
|
||
"github.com/zzet/gortex/internal/codeowners"
|
||
"github.com/zzet/gortex/internal/config"
|
||
"github.com/zzet/gortex/internal/contracts"
|
||
"github.com/zzet/gortex/internal/embedding"
|
||
"github.com/zzet/gortex/internal/entrypoints"
|
||
"github.com/zzet/gortex/internal/excludes"
|
||
"github.com/zzet/gortex/internal/fixtures"
|
||
"github.com/zzet/gortex/internal/graph"
|
||
"github.com/zzet/gortex/internal/intern"
|
||
"github.com/zzet/gortex/internal/licenses"
|
||
"github.com/zzet/gortex/internal/modules"
|
||
"github.com/zzet/gortex/internal/parser"
|
||
"github.com/zzet/gortex/internal/parser/crashpool"
|
||
"github.com/zzet/gortex/internal/pathkey"
|
||
"github.com/zzet/gortex/internal/progress"
|
||
"github.com/zzet/gortex/internal/reach"
|
||
"github.com/zzet/gortex/internal/resolver"
|
||
"github.com/zzet/gortex/internal/search"
|
||
"github.com/zzet/gortex/internal/search/trigram"
|
||
"github.com/zzet/gortex/internal/semantic"
|
||
gortexsql "github.com/zzet/gortex/internal/sql"
|
||
"github.com/zzet/gortex/internal/todos"
|
||
)
|
||
|
||
// IndexResult holds the outcome of an indexing operation.
|
||
type IndexResult struct {
|
||
NodeCount int `json:"node_count"`
|
||
EdgeCount int `json:"edge_count"`
|
||
// FileCount is the total number of language files the indexer
|
||
// observed for this repo — i.e. how big the repo is on disk, not
|
||
// how much work this pass did. Stamped onto RepoMetadata so
|
||
// `daemon status` shows a stable file count across both full-track
|
||
// and incremental-reconcile paths.
|
||
FileCount int `json:"file_count"`
|
||
// StaleFileCount is the number of files that were actually
|
||
// re-indexed in this pass (only populated by IncrementalReindex
|
||
// — full-index passes treat every file as stale and would
|
||
// duplicate FileCount). Used by the janitor / reconcile log to
|
||
// report "how much work did the snapshot delta require".
|
||
StaleFileCount int `json:"stale_file_count,omitempty"`
|
||
// FailedFiles lists files an incremental pass could not index even
|
||
// after one retry — a parse error, or a file locked or removed
|
||
// mid-pass. A caller can replay them explicitly. Whether the next
|
||
// incremental pass retries them on its own depends on the failure:
|
||
// a file whose bytes could not even be read (locked, permission
|
||
// denied, removed mid-walk) never gets an mtime recorded, so it
|
||
// stays stale and is retried on every subsequent pass; a file that
|
||
// was read but failed to parse (syntax error, crash-isolation
|
||
// quarantine, extraction timeout) DOES get its current on-disk
|
||
// mtime recorded, so it is retried only once its content changes
|
||
// again — this keeps a warm restart from perpetually treating the
|
||
// whole repo as changed because of one unparseable file. Empty on
|
||
// a clean pass and on full-index passes.
|
||
FailedFiles []string `json:"failed_files,omitempty"`
|
||
// QuarantinedFiles is the number of files held in the parser
|
||
// crash-isolation quarantine after this pass — files that
|
||
// SIGSEGV'd / hung / panicked the parser and were skipped with a
|
||
// Meta["parse_error"] node. Zero unless crash isolation is on.
|
||
QuarantinedFiles int `json:"quarantined_files,omitempty"`
|
||
// SkippedFiles is the number of files skipped by the size cap
|
||
// (MaxFileSize), the per-file extraction timeout (MaxExtractMillis),
|
||
// or the content-admission policy (index.content — oversized documents
|
||
// and, by default, binary/vector data assets). Each is recorded in the
|
||
// graph as a synthetic file node carrying skipped_due_to_size /
|
||
// skipped_due_to_timeout / skipped_due_to_content telemetry. Zero
|
||
// unless one of those gates fires.
|
||
SkippedFiles int `json:"skipped_files,omitempty"`
|
||
// DeletedFileCount is the number of previously-indexed files that
|
||
// were evicted this pass because they no longer exist on disk (only
|
||
// populated by IncrementalReindex). Together with StaleFileCount it
|
||
// lets a batch caller — the daemon warmup loop in particular — decide
|
||
// whether a repo actually changed since the last shutdown: when both
|
||
// are zero across every repo, the persisted graph already carries
|
||
// every resolved / derived edge and the global resolution passes can
|
||
// be skipped entirely (the warm-restart fast path).
|
||
DeletedFileCount int `json:"deleted_file_count,omitempty"`
|
||
// FullRetrack is true when this result came from a whole-repo
|
||
// re-track (IndexCtx) rather than an incremental pass — i.e. the
|
||
// changed-file set is unknown and StaleFileCount does NOT reflect
|
||
// "how many files changed" (it keeps its normal incremental-work
|
||
// meaning and is 0 here). Callers that gate global re-resolution on
|
||
// "did this repo change" must OR in FullRetrack alongside
|
||
// StaleFileCount / DeletedFileCount.
|
||
FullRetrack bool `json:"full_retrack,omitempty"`
|
||
DurationMs int64 `json:"duration_ms"`
|
||
Errors []IndexError `json:"errors,omitempty"`
|
||
// RepoPrefix is the prefix the repo was actually registered under.
|
||
// It usually equals config.ResolvePrefix(entry), but a git worktree
|
||
// tracked as an independent instance gets a derived `<base>@<tag>`
|
||
// prefix that the caller cannot recompute from the (by-value) entry
|
||
// it passed in — callers that need to attach a watcher or report the
|
||
// outcome read it from here. Populated by the multi-repo track and
|
||
// reconcile paths; empty in single-repo mode.
|
||
RepoPrefix string `json:"repo_prefix,omitempty"`
|
||
}
|
||
|
||
// EdgeSanityViolated reports the post-reindex sanity-check failure: an
|
||
// index pass that observed source files and extracted symbol nodes from
|
||
// them, yet produced zero edges. A populated graph with no edges still
|
||
// looks indexed but answers every "who calls X" / "what imports Y"
|
||
// query with nothing — a wholesale edge-extraction failure (a broken
|
||
// grammar, an aborted reindex) worth surfacing rather than shipping
|
||
// silently. Even a single one-function file yields containment edges,
|
||
// so a real repo only trips this when extraction failed across the
|
||
// board.
|
||
func (r *IndexResult) EdgeSanityViolated() bool {
|
||
return r != nil && r.FileCount > 0 && r.NodeCount > 0 && r.EdgeCount == 0
|
||
}
|
||
|
||
// IndexError records a per-file parsing failure.
|
||
type IndexError struct {
|
||
FilePath string `json:"file_path"`
|
||
Error string `json:"error"`
|
||
}
|
||
|
||
// Indexer walks a repository and populates the graph.
|
||
type Indexer struct {
|
||
graph graph.Store
|
||
// indexCount tracks how many IndexCtx calls this Indexer has
|
||
// completed. Gates the cold-start shadow-swap: each per-repo
|
||
// Indexer in MultiIndexer is fresh (indexCount==0), so all of
|
||
// them take the shadow path regardless of what sibling repos
|
||
// have already drained into the shared disk store. Per-repo-
|
||
// prefixed stub IDs make the concurrent drains conflict-free.
|
||
indexCount atomic.Int32
|
||
registry *parser.Registry
|
||
resolver *resolver.Resolver
|
||
search search.Backend
|
||
config config.IndexConfig
|
||
transforms *transformPipeline
|
||
excludes *excludes.Matcher
|
||
excludeOnce sync.Once
|
||
dirIgnore *excludes.Hierarchical
|
||
dirIgnoreOnce sync.Once
|
||
rootPath string
|
||
// projectName is the repo's own name (go.mod module / package.json /
|
||
// dir), computed once per index. Stripped from the BM25-indexed file
|
||
// path so a query word matching it doesn't earn a useless uniform
|
||
// path-field boost across every document. "" disables the de-weighting.
|
||
projectName string
|
||
logger *zap.Logger
|
||
|
||
// Crash-isolation parser pool, lazily created and then reused
|
||
// across single-file re-indexes so the watcher hot path never
|
||
// forks a worker subprocess per file.
|
||
parsePool *crashpool.Pool
|
||
parseQuar *crashpool.Quarantine
|
||
parsePoolMu sync.Mutex
|
||
|
||
// Trigram code-search index, lazily built on first GrepText call
|
||
// and rebuilt only when indexGen advances past the build it was
|
||
// made from. indexGen is bumped by every full or incremental
|
||
// index, so a burst of searches between reindexes hits a warm
|
||
// index.
|
||
indexGen atomic.Uint64
|
||
trigramSearcher *trigram.Searcher
|
||
trigramGen uint64
|
||
trigramMu sync.Mutex
|
||
|
||
// repoPrefix is set in multi-repo mode to prefix all file paths and node IDs.
|
||
// When empty, the indexer operates in single-repo mode (backward compatible).
|
||
repoPrefix string
|
||
|
||
// workspaceID is the hard graph boundary slug for this repo.
|
||
// Stamped onto every node emitted by this indexer via applyRepoPrefix
|
||
// so query-time scoping doesn't have to look it up by repo prefix.
|
||
// Defaults at the MultiIndexer layer to the per-repo `.gortex.yaml`
|
||
// `workspace:` slug, falling back to repoPrefix when no slug is
|
||
// declared (so legacy configs keep working).
|
||
workspaceID string
|
||
|
||
// projectID is the soft sub-boundary slug. Defaults to the repo
|
||
// prefix in single-project repos. Monorepos resolve a per-file
|
||
// projectID via the `projects[]` paths-glob mapping in
|
||
// `.gortex.yaml`; until that lookup is wired in, every node from
|
||
// this indexer carries the repo-default value.
|
||
projectID string
|
||
|
||
// contractRegistry holds detected API contracts (HTTP routes, gRPC, etc.).
|
||
contractRegistry *contracts.Registry
|
||
|
||
// trackedRepoModules maps repo names to Go module paths for cross-repo dependency detection.
|
||
// Populated by MultiIndexer from go.mod files of tracked repos.
|
||
trackedRepoModules map[string]string
|
||
|
||
// embedder is the optional embedding provider for semantic search.
|
||
embedder embedding.Provider
|
||
|
||
// skipVectorBuild, when true, makes buildSearchIndex populate only
|
||
// the text index and never run the embedding pass — even with an
|
||
// embedder set. The daemon flips it on for the warmup re-index loop
|
||
// when a snapshot already carries the workspace vector index, so
|
||
// the graph is not re-embedded only to have the cached index
|
||
// overwrite it. Off by default; a normal index always builds
|
||
// vectors when an embedder is present.
|
||
skipVectorBuild bool
|
||
|
||
// bulkVectorSink holds the disk store captured at the bulk-load
|
||
// shadow swap, so buildSearchIndex can still persist the vector
|
||
// index to the backend while idx.graph points at the in-memory
|
||
// shadow (which does not implement graph.VectorSearcher). Without
|
||
// it the embedding pass under the bulk loader builds vectors only
|
||
// in the in-process HNSW — they never reach the `vectors` table and
|
||
// are lost on the next daemon restart, forcing a paid re-embed.
|
||
// Set during the shadow swap, cleared when idx.graph is restored.
|
||
bulkVectorSink graph.VectorSearcher
|
||
|
||
// contentSink mirrors bulkVectorSink for the content full-text index:
|
||
// the disk store captured at the shadow swap, so the per-file content
|
||
// stream reaches content_fts on disk even while idx.graph points at the
|
||
// in-memory shadow (which does not implement graph.ContentSearcher).
|
||
// Set during the shadow swap, cleared when idx.graph is restored.
|
||
contentSink graph.ContentSearcher
|
||
|
||
// embedChunkOpts tunes the AST sub-chunking buildSearchIndex applies
|
||
// to large symbols before embedding. The zero value makes the
|
||
// chunker fall back to its package defaults.
|
||
embedChunkOpts embedding.ChunkOptions
|
||
|
||
// embedMaxSymbols overrides the built-in cap on how many texts the
|
||
// vector index will hold before buildSearchIndex skips the embed
|
||
// pass. Zero keeps the built-in default.
|
||
embedMaxSymbols int
|
||
|
||
// embedAPIConcurrency bounds how many embedding requests run in
|
||
// parallel against an API-backed embedder. Zero keeps the built-in
|
||
// default. Ignored for in-process embedders, which serialise on an
|
||
// inference mutex.
|
||
embedAPIConcurrency int
|
||
|
||
// lastVectorBuildErr records why the most recent buildSearchIndex pass
|
||
// shipped text-only instead of a vector index (chunk-embed failure,
|
||
// all-vectors-invalid, or the symbol-count guard). Nil after a build that
|
||
// produced a vector index. Read via LastVectorBuildError once a build has
|
||
// finished — it lets `gortex eval embedders` report the real cause instead
|
||
// of a bare "no vector data".
|
||
lastVectorBuildErr error
|
||
|
||
// semanticMgr is the optional semantic enrichment manager.
|
||
semanticMgr *semantic.Manager
|
||
|
||
// resolverLSPHelper, when non-nil, is the resolve-time LSP
|
||
// helper installed on idx.resolver. Held here so MultiIndexer
|
||
// can mirror it onto the global post-pass resolver in
|
||
// RunDeferredPassesAll. See SetResolverLSPHelper.
|
||
resolverLSPHelper resolver.LSPHelper
|
||
|
||
// npmAliasOnce builds npmAlias lazily on the first resolve-time
|
||
// import-rewrite request. Lazy because the repo root and prefix
|
||
// are set after New(); by the time the resolver runs they are
|
||
// final.
|
||
npmAliasOnce sync.Once
|
||
npmAlias *npmAliasIndex
|
||
|
||
// workspaceMembersOnce builds workspaceMembers lazily on the first
|
||
// resolve-time package-manager-workspace lookup. Lazy for the same
|
||
// reason as npmAliasOnce — the repo root and prefix are final only
|
||
// after New().
|
||
workspaceMembersOnce sync.Once
|
||
workspaceMembers *workspaceMembershipIndex
|
||
|
||
// Mtime tracking and parse error retention for index health diagnostics.
|
||
parseErrors []IndexError
|
||
fileMtimes map[string]int64
|
||
lastIndexTime time.Time
|
||
totalDetected int
|
||
mtimeMu sync.RWMutex
|
||
|
||
// contractCache memoizes the contract-extractor output per file.
|
||
// Keyed by graph file path (with repo prefix); value is the file's
|
||
// disk mtime when last extracted plus the contracts that came out.
|
||
// extractContracts replays cache hits to skip the read + 8-extractor
|
||
// run for files that haven't changed since the last extraction —
|
||
// the dominant cost on repos with tens of thousands of source files.
|
||
contractCache map[string]*contractCacheEntry
|
||
contractCacheMu sync.RWMutex
|
||
|
||
// upgradeOnce gates the BM25→Bleve auto-upgrade to exactly one
|
||
// goroutine per indexer lifetime. Without this, every post-threshold
|
||
// IndexCtx — which fires once per tracked repo during multi-repo
|
||
// warmup — would spawn a fresh upgradeSearchToBleve goroutine.
|
||
// Each rebuilds ~N-doc Bleve indexes (≈32 KiB/doc), so overlapping
|
||
// upgrades peak memory far above steady-state and waste CPU on
|
||
// rebuilds that the next Swap immediately discards. Also counts
|
||
// scheduled upgrades so tests can observe gating decisions
|
||
// without relying on log scrapes or timing.
|
||
upgradeOnce sync.Once
|
||
upgradeSpawnedMu sync.Mutex
|
||
upgradeSpawned int
|
||
|
||
// deferResolve, when set, makes IndexCtx skip the cross-cutting passes
|
||
// (per-repo ResolveAll / semantic enrichment / contract extraction +
|
||
// commit) so the multi-repo orchestrator can run them serially after
|
||
// the parallel fan-out joins. Without this, two goroutines indexing
|
||
// different repos into the shared graph race on Edge.Meta during the
|
||
// resolver's mutation phase vs. the contract pass's graph walk via
|
||
// AllEdges().
|
||
deferResolve bool
|
||
pendingContractReg *contracts.Registry
|
||
|
||
// pendingEnrich is raised by an index pass that did real work — IndexCtx
|
||
// that observed files (or a whole-repo re-track) and IncrementalReindex /
|
||
// IncrementalReindexPaths that re-indexed or evicted at least one file. It
|
||
// is cleared only after runDeferredEnrich completes a fully non-partial
|
||
// semantic enrichment for this repo. The daemon warmup enriches every
|
||
// indexer it collected, so this gates the (multi-minute LSP hover) pass to
|
||
// repos that actually changed: an unchanged repo on a warm restart would
|
||
// otherwise re-confirm nothing for 10+ minutes. A partial / abandoned /
|
||
// failed enrich leaves it set so a later deferred pass retries.
|
||
pendingEnrich atomic.Bool
|
||
|
||
// fullReindexed is raised by a whole-repo (re-)parse — IndexCtx, reached
|
||
// via a full re-track, a cold TrackRepo, or a snapshot-partial forced full
|
||
// walk — which evicts and re-creates every node and edge for the repo. That
|
||
// drops the LSP hover-enrichment edges a static re-parse cannot reproduce,
|
||
// so the deferred-enrichment pass must re-run even when the persisted
|
||
// completion marker still records the repo's HEAD on a clean tree. It
|
||
// threads Force into RepoEnrichState so enrichMarkerCurrent stops gating the
|
||
// pass out. A fresh Indexer per daemon run starts it false, so it only ever
|
||
// reflects work this run performed.
|
||
fullReindexed atomic.Bool
|
||
|
||
// reparsedThisRun is the scoped analogue of fullReindexed: it is raised by a
|
||
// scoped incremental pass that re-parsed at least one stale file this run —
|
||
// IncrementalReindexPaths / IncrementalReindex with a non-zero
|
||
// StaleFileCount. A scoped re-parse evicts and re-creates just the changed
|
||
// files' nodes, dropping THEIR hover-enrichment edges exactly as a whole-repo
|
||
// re-parse drops every node's, so the deferred pass must likewise run past
|
||
// the completion marker at an unchanged clean HEAD — otherwise the re-parsed
|
||
// files' LSP edges stay durably gone until the repo's HEAD moves or its tree
|
||
// goes dirty. runDeferredEnrich ORs it into RepoEnrichState.Force; because the
|
||
// hover provider skips already-stamped nodes, the marker bypass re-hovers only
|
||
// the freshly-unstamped re-parsed files, not the whole repo, so the cost stays
|
||
// bounded. fullReindexed stays clear on the scoped paths — this flag keeps the
|
||
// two claims distinct. A fresh Indexer per daemon run starts it false.
|
||
reparsedThisRun atomic.Bool
|
||
|
||
// deferGlobalPasses, when set, makes IndexCtx and IncrementalReindex
|
||
// skip the graph-wide derivation passes (InferImplements,
|
||
// InferOverrides, markTestSymbolsAndEmitEdges). These passes walk the
|
||
// entire shared graph, so running them per-repo inside a batch loop
|
||
// (warmup, ReconcileAll) is O(R · global_size) — quadratic for repo
|
||
// counts in the hundreds. The batch caller is responsible for invoking
|
||
// RunGlobalGraphPasses exactly once at the end. Has no effect on the
|
||
// deferResolve path (multi-repo IndexCtx already skips those passes).
|
||
deferGlobalPasses bool
|
||
|
||
// skipResolveInDeferred, when set, makes RunDeferredPasses skip the
|
||
// per-repo resolver.ResolveAll() call. ResolveAll walks the entire
|
||
// shared graph, so paying it once per indexer across hundreds of
|
||
// repos is O(R · E). MultiIndexer.RunDeferredPassesAll sets this
|
||
// flag on every indexer and runs a single resolver.New(graph).ResolveAll
|
||
// once at the end, which picks up every placeholder edge at once.
|
||
// Has no effect on direct (non-batch) callers of RunDeferredPasses.
|
||
skipResolveInDeferred bool
|
||
|
||
// codeownersOnce ensures the repo-level CODEOWNERS file is parsed
|
||
// exactly once per indexer lifetime. The rule list is derived
|
||
// from .github/CODEOWNERS / CODEOWNERS / docs/CODEOWNERS at
|
||
// first use and applied per-file by applyCoverageDomains; an
|
||
// absent file produces empty rules and a no-op pass.
|
||
codeownersOnce sync.Once
|
||
codeownersRules []codeowners.Rule
|
||
|
||
// cloneIndex maintains the clone-detection CMS + length-stratified
|
||
// LSH live across single-file edits, so a steady-state reindex
|
||
// updates EdgeSimilarTo edges in O(edited file) instead of the
|
||
// whole-graph detectClonesAndEmitEdges recompute. Constructed empty
|
||
// (built=false) — a batch/global clone pass calls Rebuild to seed it,
|
||
// after which indexFile drives EvictFuncs/UpdateFuncs. While un-built
|
||
// indexFile falls back to the whole-graph pass.
|
||
cloneIndex *incrementalCloneIndex
|
||
|
||
// affectedByPasses / affectedByFilesResolved / affectedByDropped
|
||
// count the affected-by re-resolution activity (see affected_by.go):
|
||
// passes that found a signature delta and ran, referencing files
|
||
// re-resolved by them, and files dropped by the fan-out cap.
|
||
// Exposed via AffectedByCounts so tests and diagnostics can observe
|
||
// that a body-only edit triggered no fan-out.
|
||
affectedByPasses atomic.Int64
|
||
affectedByFilesResolved atomic.Int64
|
||
affectedByDropped atomic.Int64
|
||
}
|
||
|
||
// contractCacheEntry is a cached contract-extraction result for one file.
|
||
type contractCacheEntry struct {
|
||
mtimeNano int64
|
||
contracts []contracts.Contract
|
||
}
|
||
|
||
// New creates an Indexer that writes through the supplied graph.Store.
|
||
// Any backend (in-memory, SQLite-on-disk, remote) is acceptable — the
|
||
// indexer's mutation paths go through the Store interface methods only,
|
||
// so swapping backends is a zero-code-change configuration choice for
|
||
// callers.
|
||
func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *zap.Logger) *Indexer {
|
||
idx := &Indexer{
|
||
graph: g,
|
||
registry: reg,
|
||
resolver: resolver.New(g),
|
||
// Wrap in Swappable so the auto-upgrade to Bleve at large
|
||
// corpus sizes can happen in a background goroutine without
|
||
// racing with concurrent searches. Subsequent reassignments to
|
||
// idx.search (Hybrid wrap, etc.) should use swap helpers below.
|
||
//
|
||
// When the backing store implements graph.SymbolSearcher
|
||
// (today only store_sqlite), the initial backend is a thin
|
||
// adapter that forwards Search to the store's native FTS.
|
||
// The in-process Bleve / BM25 build path is then bypassed
|
||
// entirely — saving ~100MB heap on a Vscode-scale repo and
|
||
// putting search in the same address space as the rest of
|
||
// the graph queries.
|
||
search: search.NewSwappable(initialSearchBackend(g)),
|
||
config: cfg,
|
||
transforms: newTransformPipeline(cfg.Transforms, logger),
|
||
logger: logger,
|
||
fileMtimes: make(map[string]int64),
|
||
contractCache: make(map[string]*contractCacheEntry),
|
||
cloneIndex: newIncrementalCloneIndex(),
|
||
}
|
||
// Resolve JS/TS imports declared through an npm alias against the
|
||
// local index. The index is built lazily on first use — the repo
|
||
// root and prefix are not final until after New().
|
||
idx.resolver.SetNpmAliasResolver(idx.resolveNpmAliasImport)
|
||
// Expand JS/TS tsconfig/jsconfig path-alias imports (`@/lib/x`)
|
||
// against the local index so cross-directory alias imports resolve
|
||
// to their real file. Same lazy-build rationale.
|
||
idx.resolver.SetPathAliasResolver(idx.resolvePathAliasImport)
|
||
// Break same-named import collisions in favour of the importer's
|
||
// own package-manager workspace member. Same lazy-build rationale.
|
||
idx.resolver.SetWorkspaceMembership(idx.indexerWorkspaceMembership)
|
||
return idx
|
||
}
|
||
|
||
// resolveNpmAliasImport is the resolver.NpmAliasResolver installed on
|
||
// this Indexer's resolver. It rewrites a JS/TS import specifier that
|
||
// matches an npm-alias dependency key in the importing file's
|
||
// nearest-ancestor package.json. Returns "" (no rewrite) when no
|
||
// alias applies. The backing npmAliasIndex is built once, lazily.
|
||
func (idx *Indexer) resolveNpmAliasImport(callerFile, specifier string) string {
|
||
idx.npmAliasOnce.Do(func() {
|
||
idx.npmAlias = newNpmAliasIndex(map[string]string{idx.repoPrefix: idx.rootPath})
|
||
})
|
||
return idx.npmAlias.Resolve(callerFile, specifier)
|
||
}
|
||
|
||
// swappable returns the search backend cast to *search.Swappable. Panics
|
||
// if the invariant (idx.search is always a Swappable) is ever broken —
|
||
// that would be a programmer error in this file, not a runtime condition.
|
||
func (idx *Indexer) swappable() *search.Swappable {
|
||
if sw, ok := idx.search.(*search.Swappable); ok {
|
||
return sw
|
||
}
|
||
panic("indexer: search backend is not *search.Swappable — invariant violated")
|
||
}
|
||
|
||
// searchIndexFields returns the text fields fed to the BM25 search
|
||
// backend for a node. For an ordinary code symbol that is its
|
||
// name, file path, and signature. For a KindDoc prose-section node
|
||
// the body is what carries the search signal, so the section text
|
||
// (Meta["section_text"]) is indexed alongside the breadcrumb name
|
||
// -- a prose query then ranks the section, not just a heading match.
|
||
func searchIndexFields(n *graph.Node, projectName string) []string {
|
||
// The project-name path segment is stripped from the INDEXED path (not the
|
||
// stored FilePath) so it contributes no uniform path-field boost.
|
||
indexedPath := search.StripProjectNameFromPath(n.FilePath, projectName)
|
||
if n.Kind == graph.KindDoc {
|
||
body, _ := n.Meta["section_text"].(string)
|
||
return []string{n.Name, indexedPath, body}
|
||
}
|
||
sig, _ := n.Meta["signature"].(string)
|
||
// A symbol's doc comment is the natural-language statement of what it
|
||
// does — precisely the vocabulary a task-intent query carries ("union
|
||
// the two sequences", "performs matching on the ignore files") when it
|
||
// names no identifier verbatim. The identifier and signature alone
|
||
// answer name lookups; the doc summary is what lets an intent query
|
||
// reach the definition at all. Only the leading summary is indexed
|
||
// (docSummary), so a long doc block's examples and edge-case prose
|
||
// can't dilute the name/signature tokens under BM25 length
|
||
// normalisation. Empty doc → empty field, dropped by the caller, so a
|
||
// symbol with no doc indexes exactly as before.
|
||
doc, _ := n.Meta["doc"].(string)
|
||
return []string{n.Name, indexedPath, sig, docSummary(doc)}
|
||
}
|
||
|
||
// docSummaryMaxRunes bounds how much of a doc comment enters the search
|
||
// document. The leading summary is the highest-signal part; the rest of a
|
||
// long doc — parameter lists, example code blocks, edge-case notes — is
|
||
// lower-signal and, unbounded, would dominate the token bag and dilute the
|
||
// identifier under BM25 length normalisation. The bound keeps the doc's
|
||
// contribution on the order of a signature's. It is a generic document-size
|
||
// budget, independent of any repository or corpus.
|
||
const docSummaryMaxRunes = 280
|
||
|
||
// docSummary returns the leading summary of a doc comment for the search
|
||
// index: the first paragraph (up to the first blank line), trimmed and
|
||
// bounded to docSummaryMaxRunes. Doc comments across languages put the
|
||
// one-line / one-paragraph summary first and defer detail and examples to
|
||
// later paragraphs, so the first paragraph is the highest-signal slice.
|
||
// Empty in → empty out.
|
||
func docSummary(doc string) string {
|
||
if doc == "" {
|
||
return ""
|
||
}
|
||
// Normalise CRLF so the blank-line cut is newline-style-agnostic.
|
||
if strings.Contains(doc, "\r") {
|
||
doc = strings.ReplaceAll(doc, "\r\n", "\n")
|
||
doc = strings.ReplaceAll(doc, "\r", "\n")
|
||
}
|
||
if i := strings.Index(doc, "\n\n"); i >= 0 {
|
||
doc = doc[:i]
|
||
}
|
||
doc = strings.TrimSpace(doc)
|
||
if r := []rune(doc); len(r) > docSummaryMaxRunes {
|
||
doc = strings.TrimSpace(string(r[:docSummaryMaxRunes]))
|
||
}
|
||
return doc
|
||
}
|
||
|
||
// vectorSearcherDelegate is the search.VectorDelegate-shaped
|
||
// adapter the indexer hands to VectorBackend.SetDelegate when the
|
||
// underlying store implements graph.VectorSearcher. SimilarTo just
|
||
// forwards — search.VectorDelegate is defined to return
|
||
// graph.VectorHit slices directly, so there's no translation work
|
||
// here, just a small struct so the in-process search package
|
||
// doesn't depend on graph.VectorSearcher's full surface.
|
||
type vectorSearcherDelegate struct {
|
||
s graph.VectorSearcher
|
||
}
|
||
|
||
func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.VectorHit, error) {
|
||
if d == nil || d.s == nil {
|
||
return nil, nil
|
||
}
|
||
return d.s.SimilarTo(vec, limit)
|
||
}
|
||
|
||
// initialSearchBackend picks the search.Backend the indexer wraps
|
||
// in its Swappable on construction. When the underlying store
|
||
// implements graph.SymbolSearcher (today only store_sqlite), a
|
||
// thin adapter routes Search calls through the store's native FTS
|
||
// — the in-process BM25 / Bleve build path is bypassed entirely.
|
||
// Otherwise falls through to search.NewAuto which picks BM25 for
|
||
// small corpora and auto-upgrades to Bleve once the size warrants
|
||
// it.
|
||
func initialSearchBackend(g graph.Store) search.Backend {
|
||
if s, ok := g.(graph.SymbolSearcher); ok {
|
||
return search.NewSymbolSearcherBackend(s)
|
||
}
|
||
return search.NewAuto()
|
||
}
|
||
|
||
// isSymbolSearcherBackend reports whether the swappable's currently
|
||
// active backend is the SymbolSearcher adapter. Used to suppress
|
||
// the Bleve auto-upgrade goroutine — if the active backend is
|
||
// already a native FTS, upgrading to Bleve would re-index the same
|
||
// corpus into a parallel in-process Bleve and silently swap it in,
|
||
// defeating the FTS path and pinning the ~100MB heap the FTS
|
||
// integration was meant to release.
|
||
func isSymbolSearcherBackend(b search.Backend) bool {
|
||
if b == nil {
|
||
return false
|
||
}
|
||
if sw, ok := b.(*search.Swappable); ok {
|
||
b = sw.Inner()
|
||
}
|
||
_, ok := b.(*search.SymbolSearcherBackend)
|
||
return ok
|
||
}
|
||
|
||
// lessEdgeKey orders edges by their logical key (from, to, kind,
|
||
// file_path, line) — the same tuple the sqlite edges table's
|
||
// UNIQUE(from_id, …) index is built on. Draining edges in this order on
|
||
// the cold bulk load keeps that index's inserts local instead of random,
|
||
// reducing B-tree page splits.
|
||
func lessEdgeKey(a, b *graph.Edge) bool {
|
||
if a.From != b.From {
|
||
return a.From < b.From
|
||
}
|
||
if a.To != b.To {
|
||
return a.To < b.To
|
||
}
|
||
if a.Kind != b.Kind {
|
||
return a.Kind < b.Kind
|
||
}
|
||
if a.FilePath != b.FilePath {
|
||
return a.FilePath < b.FilePath
|
||
}
|
||
return a.Line < b.Line
|
||
}
|
||
|
||
// ftsTokensFor produces the pre-tokenised text the backend FTS path
|
||
// indexes. Mirrors searchIndexFields' field selection but joins
|
||
// every field through search.Tokenize (camelCase / snake_case /
|
||
// path-segment splitter) so the resulting token list matches the
|
||
// in-process BM25 corpus contract — the same query produces the
|
||
// same recall against either backend. Joined with spaces so the
|
||
// downstream COPY FROM sees a single STRING column value.
|
||
func ftsTokensFor(n *graph.Node, projectName string) string {
|
||
fields := searchIndexFields(n, projectName)
|
||
if n.QualName != "" {
|
||
// QualName carries the dotted form (`pkg.Sub.Type.Method`)
|
||
// that adds qualifier-hop recall ("auth" matching
|
||
// "auth.ValidateToken"). searchIndexFields omits it for
|
||
// the legacy BM25 path (which folds qual into the
|
||
// name-token bag separately), so we add it explicitly here.
|
||
fields = append(fields, n.QualName)
|
||
}
|
||
tokens := make([]string, 0, 16)
|
||
for _, f := range fields {
|
||
if f == "" {
|
||
continue
|
||
}
|
||
tokens = append(tokens, search.Tokenize(f)...)
|
||
}
|
||
if len(tokens) == 0 {
|
||
return ""
|
||
}
|
||
return strings.Join(tokens, " ")
|
||
}
|
||
|
||
// shouldIndexForSearch reports whether a node should be added to the
|
||
// text search index (BM25/Bleve). File and Import nodes are never
|
||
// searchable symbols. Beyond that, config.SkipSearch filters out
|
||
// (language, kind) pairs that would only add noise — JSON/YAML/TOML
|
||
// keys, CSS tokens, Terraform blocks, shell/build variables. All three
|
||
// text-index call sites (buildSearchIndex bulk loop, indexFile
|
||
// incremental add, upgradeSearchToBleve repopulate) must go through
|
||
// this predicate so they can't drift.
|
||
func (idx *Indexer) shouldIndexForSearch(n *graph.Node) bool {
|
||
// Cross-daemon proxy-edge nodes stand in for remote symbols; they
|
||
// are never surfaced in local name search. Inert until
|
||
// edge-minting is enabled.
|
||
if graph.IsProxyNode(n) {
|
||
return false
|
||
}
|
||
if n.Kind == graph.KindFile || n.Kind == graph.KindImport {
|
||
return false
|
||
}
|
||
// KindLocal nodes are intra-function bindings emitted to satisfy
|
||
// rel-table FK constraints on the dataflow edges that target
|
||
// locals. They have a real Name (the variable identifier) but
|
||
// surfacing them in BM25 would flood every search for common
|
||
// names like `err`, `data`, `n`, `i`. Excluded unconditionally.
|
||
if n.Kind == graph.KindLocal {
|
||
return false
|
||
}
|
||
// KindBuiltin nodes are language intrinsics (append / len /
|
||
// string / int / ...). Surfacing them in name search would
|
||
// drown every other hit on common identifiers — agents already
|
||
// know `string` / `append`. They remain queryable by kind and
|
||
// by ID for the analytics passes that care.
|
||
if n.Kind == graph.KindBuiltin {
|
||
return false
|
||
}
|
||
// CONTENT (data_class="content") section nodes live in the dedicated
|
||
// content index (content_fts), never the symbol search — keeping the
|
||
// symbol corpus code-only and bounded. Markdown prose (KindDoc without
|
||
// data_class=content) is unaffected and still honours IndexProse below.
|
||
if isContentNode(n) {
|
||
return false
|
||
}
|
||
// Prose-section nodes are searchable only when prose indexing is
|
||
// enabled (search.index_prose); the rest of the graph is
|
||
// unaffected by the toggle.
|
||
if n.Kind == graph.KindDoc && !idx.config.IndexProse {
|
||
return false
|
||
}
|
||
if config.ShouldSkipSearch(idx.config.SkipSearch, n.Language, string(n.Kind)) {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// upgradeSearchToBleve constructs a Bleve backend from the current graph
|
||
// and atomically swaps it in. Designed to run in a background goroutine
|
||
// triggered by IndexCtx after the initial index completes. Does nothing
|
||
// if Bleve construction fails (caller already hit AutoThreshold but the
|
||
// in-memory backend keeps serving correctly, just with worse memory
|
||
// characteristics).
|
||
// bleveUpgradeEntry is one row of the snapshot the upgrade goroutine
|
||
// works from. Snapshotting (id, name, file, signature) up front in
|
||
// the foreground — before the goroutine starts reading them — keeps
|
||
// the goroutine race-free against subsequent Index calls' Meta-writing
|
||
// passes (reach.BuildIndex, ResolveTemporalCalls, ...).
|
||
type bleveUpgradeEntry struct {
|
||
id string
|
||
// fields is the BM25 text payload for the node, as produced by
|
||
// searchIndexFields: name + file + signature for a code symbol,
|
||
// name + file + section body for a KindDoc prose section.
|
||
fields []string
|
||
}
|
||
|
||
// snapshotBleveEntries captures every node currently eligible for the
|
||
// search index plus its `signature` Meta string. Called synchronously
|
||
// from IndexCtx after every Node.Meta mutating pass has returned, so
|
||
// the read of n.Meta happens with no concurrent writer.
|
||
func (idx *Indexer) snapshotBleveEntries() []bleveUpgradeEntry {
|
||
nodes := idx.graph.AllNodes()
|
||
out := make([]bleveUpgradeEntry, 0, len(nodes))
|
||
for _, n := range nodes {
|
||
if !idx.shouldIndexForSearch(n) {
|
||
continue
|
||
}
|
||
out = append(out, bleveUpgradeEntry{id: n.ID, fields: searchIndexFields(n, idx.projectName)})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (idx *Indexer) upgradeSearchToBleve(snapshot []bleveUpgradeEntry) {
|
||
// Defensive early-return: if the active text backend is already
|
||
// Bleve, there is nothing to upgrade. IndexCtx's sync.Once guard
|
||
// prevents re-entry from the auto-upgrade path, but direct
|
||
// callers (tests, manual invocation from tooling) could still
|
||
// hit this function twice; a second run would pointlessly
|
||
// rebuild a full Bleve index and Swap it over an identical one.
|
||
inner := idx.swappable().Inner()
|
||
if _, ok := inner.(*search.BleveBackend); ok {
|
||
return
|
||
}
|
||
if hyb, ok := inner.(*search.HybridBackend); ok {
|
||
if _, ok := hyb.TextBackend().(*search.BleveBackend); ok {
|
||
return
|
||
}
|
||
}
|
||
|
||
// Opt-in disk backend. Scorch stores the inverted index on disk
|
||
// (~10-20× less heap than upsidedown+gtreap) at the cost of file
|
||
// I/O during build. Users point GORTEX_BLEVE_DISK_DIR at a
|
||
// writable path; we manage the file lifecycle inside it.
|
||
diskDir := os.Getenv("GORTEX_BLEVE_DISK_DIR")
|
||
|
||
var (
|
||
blv *search.BleveBackend
|
||
err error
|
||
)
|
||
if diskDir != "" {
|
||
blv, err = search.NewBleveDisk(diskDir)
|
||
if err != nil {
|
||
idx.logger.Warn("search: bleve disk construction failed, falling back to in-memory",
|
||
zap.String("dir", diskDir), zap.Error(err))
|
||
}
|
||
}
|
||
if blv == nil {
|
||
blv, err = search.NewBleve()
|
||
if err != nil {
|
||
idx.logger.Warn("search: bleve construction failed, staying on in-memory",
|
||
zap.Error(err))
|
||
return
|
||
}
|
||
}
|
||
|
||
// Use the foreground snapshot the spawner captured rather than
|
||
// re-walking idx.graph here: the goroutine outlives the spawning
|
||
// IndexCtx call, and subsequent Index calls' Meta-writing passes
|
||
// (reach.BuildIndex, ResolveTemporalCalls, ...) mutate Node.Meta
|
||
// on the same Node objects. Reading sig from a live n.Meta here
|
||
// would race with those writes.
|
||
for _, e := range snapshot {
|
||
blv.Add(e.id, e.fields...)
|
||
}
|
||
|
||
// Preserve the vector index if one is wired up. The previous inner
|
||
// is normally a HybridBackend wrapping text + vector + embedder;
|
||
// swapping in raw Bleve would let Swap's old.Close() run on the
|
||
// old Hybrid, which closes only its text side (hybrid.Close) but
|
||
// leaves the resulting inner — raw *BleveBackend — unwrapped, so
|
||
// every downstream hybrid/semantic query silently degrades to
|
||
// BM25 until the daemon restarts. Rewrap the fresh Bleve in a new
|
||
// Hybrid carrying the old vector + embedder. The vector backend
|
||
// itself is never closed by Hybrid.Close, so it stays alive even
|
||
// after the old Hybrid is torn down by Swap.
|
||
sw := idx.swappable()
|
||
var replacement search.Backend = blv
|
||
if oldHybrid, ok := sw.Inner().(*search.HybridBackend); ok {
|
||
replacement = search.NewHybrid(blv, oldHybrid.VectorIndex(), oldHybrid.Embedder())
|
||
}
|
||
sw.Swap(replacement)
|
||
|
||
mode := "memory"
|
||
if blv.DiskPath() != "" {
|
||
mode = "disk"
|
||
}
|
||
idx.logger.Info("search: upgraded to Bleve backend (background)",
|
||
zap.Int("symbols", blv.Count()),
|
||
zap.String("mode", mode),
|
||
zap.String("disk_path", blv.DiskPath()))
|
||
}
|
||
|
||
// Graph returns the underlying graph.
|
||
func (idx *Indexer) Graph() graph.Store { return idx.graph }
|
||
|
||
// Search returns the search backend.
|
||
func (idx *Indexer) Search() search.Backend { return idx.search }
|
||
|
||
// Registry returns the parser registry shared across this indexer.
|
||
// Exposed for the editor-overlay middleware: the overlay layer-build
|
||
// pass parses each pushed buffer through the same per-language
|
||
// extractor the indexer uses, ensuring overlay-derived nodes match
|
||
// base-derived nodes byte-for-byte for the same input.
|
||
func (idx *Indexer) Registry() *parser.Registry { return idx.registry }
|
||
|
||
// ContractRegistry returns the contract registry populated during indexing.
|
||
func (idx *Indexer) ContractRegistry() *contracts.Registry { return idx.contractRegistry }
|
||
|
||
// SetContractRegistry installs reg as the indexer's contract registry.
|
||
// Used by the daemon warmup path to rehydrate the registry from a
|
||
// snapshot when IncrementalReindex skipped extraction (no stale files
|
||
// → extractContracts never ran → idx.contractRegistry stays nil after
|
||
// reconcile, which used to leave multi-repo `contracts` queries silently
|
||
// empty). Callers should only install when ContractRegistry() is nil;
|
||
// installing over a freshly-extracted registry would roll state backward.
|
||
func (idx *Indexer) SetContractRegistry(reg *contracts.Registry) {
|
||
idx.contractRegistry = reg
|
||
}
|
||
|
||
// SetTrackedRepoModules sets the map of tracked repo names to Go module paths.
|
||
// This enables the GoModExtractor to detect cross-repo dependencies.
|
||
func (idx *Indexer) SetTrackedRepoModules(m map[string]string) { idx.trackedRepoModules = m }
|
||
|
||
// SetDeferResolve toggles whether IndexCtx defers the cross-cutting passes
|
||
// to a later RunDeferredPasses call. See the deferResolve field comment.
|
||
func (idx *Indexer) SetDeferResolve(v bool) { idx.deferResolve = v }
|
||
|
||
// SetSkipResolveInDeferred toggles whether RunDeferredPasses calls
|
||
// idx.resolver.ResolveAll. The MultiIndexer batch driver sets this so
|
||
// the per-repo resolver pass — which walks the entire shared graph —
|
||
// runs exactly once globally instead of R times. See the
|
||
// skipResolveInDeferred field comment.
|
||
func (idx *Indexer) SetSkipResolveInDeferred(v bool) { idx.skipResolveInDeferred = v }
|
||
|
||
// SetDeferGlobalPasses toggles whether the graph-wide derivation passes
|
||
// (InferImplements, InferOverrides, markTestSymbolsAndEmitEdges) run
|
||
// inline at the end of IndexCtx / IncrementalReindex. Set true when the
|
||
// caller drives a batch (e.g. daemon warmup) and will invoke
|
||
// RunGlobalGraphPasses once at the end. See the deferGlobalPasses field
|
||
// comment.
|
||
func (idx *Indexer) SetDeferGlobalPasses(v bool) { idx.deferGlobalPasses = v }
|
||
|
||
// RunGlobalGraphPasses runs the graph-wide derivation passes once
|
||
// against the indexer's shared graph. Safe to call against a graph that
|
||
// already has these edges — InferImplements / InferOverrides skip
|
||
// existing parents, and graph.AddEdge dedupes by edgeKey so EdgeTests
|
||
// re-emission is a no-op. Logs counts for telemetry. Use when batching
|
||
// multiple per-repo TrackRepoCtx / IncrementalReindex calls under
|
||
// SetDeferGlobalPasses(true).
|
||
func (idx *Indexer) RunGlobalGraphPasses(ctx context.Context) {
|
||
if idx.graph == nil {
|
||
return
|
||
}
|
||
reporter := progress.FromContext(ctx)
|
||
if added := idx.resolver.InferImplements(); added > 0 {
|
||
idx.logger.Info("inferred implements (global)", zap.Int("added", added))
|
||
}
|
||
if added := idx.resolver.InferOverrides(); added > 0 {
|
||
idx.logger.Info("inferred overrides (global)", zap.Int("added", added))
|
||
}
|
||
marked, emitted := markTestSymbolsAndEmitEdges(idx.graph)
|
||
if marked > 0 || emitted > 0 {
|
||
idx.logger.Info("test edges emitted (global)",
|
||
zap.Int("test_symbols", marked),
|
||
zap.Int("edges", emitted),
|
||
)
|
||
}
|
||
if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 {
|
||
idx.logger.Info("capability edges emitted (global)",
|
||
zap.Int("reads_env", re),
|
||
zap.Int("executes_process", ep),
|
||
zap.Int("accesses_field", fa),
|
||
)
|
||
}
|
||
reporter.Report("clone detection pass (global)", 0, 0)
|
||
if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
|
||
idx.logger.Info("clone edges emitted (global)",
|
||
zap.Int("items", cs.Items),
|
||
zap.Int("clone_pairs", cs.Pairs),
|
||
zap.Int("edges", cs.Edges),
|
||
zap.Int("skipped_buckets", cs.SkippedBuckets),
|
||
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
|
||
zap.Int("diffused_pairs", cs.DiffusedPairs),
|
||
zap.Int("diffused_edges", cs.DiffusedEdges),
|
||
)
|
||
}
|
||
// Seed the incremental clone index from the freshly-baselined
|
||
// signatures + sidecar so steady-state single-file edits after this
|
||
// batch go incremental instead of re-running the whole-graph pass.
|
||
if idx.cloneIndex != nil {
|
||
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
|
||
}
|
||
// Framework dynamic-dispatch synthesis (gRPC stubs, Temporal
|
||
// workflow→activity, in-process / native event channels, native
|
||
// bridges). Runs after InferImplements/InferOverrides (the
|
||
// interface-satisfaction signals several synthesizers depend on) and
|
||
// before DetectCrossRepoEdges so a cross-repo synthesized call gets
|
||
// its parallel cross_repo_calls edge.
|
||
reporter.Report("framework dispatch synthesis (global)", 0, 0)
|
||
if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 {
|
||
idx.logger.Info("framework dispatch calls synthesized (global)",
|
||
zap.Int("edges", rep.Total),
|
||
zap.Any("per_synthesizer", rep.Per),
|
||
)
|
||
}
|
||
// External-call placeholder synthesis (opt-in). Runs after the
|
||
// resolver and the gRPC/Temporal stub passes so every edge that
|
||
// could land on a real node already has; the leftover external
|
||
// terminals are then materialised into synthetic call-chain nodes.
|
||
reporter.Report("external-call synthesis (global)", 0, 0)
|
||
if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 {
|
||
idx.logger.Info("external-call placeholders synthesized (global)",
|
||
zap.Int("edges", extCalls),
|
||
)
|
||
}
|
||
// Speculative dynamic-dispatch synthesis (opt-in, default off). Mints
|
||
// best-guess hidden-by-default call edges for blind-spot shapes.
|
||
if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 {
|
||
idx.logger.Info("speculative dispatch edges synthesized (global)",
|
||
zap.Int("edges", spec),
|
||
)
|
||
}
|
||
// Content -> code "why" links. Runs before DetectCrossRepoEdges so a
|
||
// chunk that motivates a symbol in another repo gets its parallel
|
||
// cross_repo_motivates edge minted by the cross-repo pass below.
|
||
reporter.Report("content links (global)", 0, 0)
|
||
idx.linkContentToCode()
|
||
// Cross-repo edge layer. Runs after InferImplements / InferOverrides
|
||
// so cross-repo implements / extends edges pick up their parallel
|
||
// cross_repo_* edges. No-op on single-repo graphs (no RepoPrefix).
|
||
reporter.Report("cross-repo edges (global)", 0, 0)
|
||
if crossRepoEdges := resolver.DetectCrossRepoEdges(idx.graph); crossRepoEdges > 0 {
|
||
idx.logger.Info("cross-repo edges emitted (global)",
|
||
zap.Int("edges", crossRepoEdges),
|
||
)
|
||
}
|
||
// Reachability index — used to be precomputed here for every
|
||
// impact seed. The eager pass was retired because the breakeven
|
||
// math doesn't work: on a 200 k-seed graph (k8s) the build took
|
||
// ~2000 s of cold-index wall time to save ~10 ms per
|
||
// AnalyzeImpact call, requiring ~200 k queries to pay off — well
|
||
// beyond any realistic agent session. Lookups are now
|
||
// compute-on-first-use; we just invalidate the cache so any
|
||
// surviving stamps from a previous build don't shadow the fresh
|
||
// graph state.
|
||
reach.InvalidateIndex()
|
||
}
|
||
|
||
// cloneThreshold returns the configured Jaccard similarity cutoff for
|
||
// clone detection (0 = use the clones package default).
|
||
func (idx *Indexer) cloneThreshold() float64 {
|
||
return idx.config.Coverage.ClonesThreshold()
|
||
}
|
||
|
||
// RunDeferredPasses runs the per-repo cross-cutting passes that IndexCtx
|
||
// skipped in deferred mode: per-repo ResolveAll, semantic enrichment, and
|
||
// contract extraction + commit. Safe to call only after IndexCtx has
|
||
// populated the graph for this repo. Idempotent — second calls are a no-op
|
||
// because the pending registry is cleared at the end.
|
||
//
|
||
// The graph-wide derivation passes (InferImplements, InferOverrides,
|
||
// markTestSymbolsAndEmitEdges) intentionally do NOT run here. They walk
|
||
// the entire shared graph, so the multi-repo orchestrator must invoke
|
||
// MultiIndexer.RunGlobalGraphPasses exactly once after every repo has
|
||
// finished its deferred per-repo work.
|
||
func (idx *Indexer) RunDeferredPasses(ctx context.Context) {
|
||
if idx.pendingContractReg == nil {
|
||
return
|
||
}
|
||
reporter := progress.FromContext(ctx)
|
||
tphase := time.Now()
|
||
var dGoMod, dResolve, dEnrich, dContract time.Duration
|
||
|
||
idx.runDeferredGoMod()
|
||
dGoMod = time.Since(tphase)
|
||
tphase = time.Now()
|
||
|
||
// Per-repo resolver.ResolveAll walks the entire shared graph; with R
|
||
// repos and E edges that's O(R · E). The MultiIndexer batch driver
|
||
// sets skipResolveInDeferred so this runs exactly once globally
|
||
// (resolver.New(mi.graph).ResolveAll after every per-repo deferred
|
||
// pass has committed contracts). Direct (non-batch) callers leave
|
||
// the flag false and pay the standard single-repo cost.
|
||
if !idx.skipResolveInDeferred {
|
||
reporter.Report("resolving references", 0, 0)
|
||
idx.populateCppIncludeDirs(false)
|
||
idx.resolver.ResolveAll()
|
||
}
|
||
dResolve = time.Since(tphase)
|
||
tphase = time.Now()
|
||
|
||
reporter.Report("semantic enrichment", 0, 0)
|
||
idx.runDeferredEnrich()
|
||
dEnrich = time.Since(tphase)
|
||
tphase = time.Now()
|
||
|
||
reporter.Report("extracting contracts", 0, 0)
|
||
idx.runDeferredContracts()
|
||
dContract = time.Since(tphase)
|
||
idx.logger.Info("DEFERRED-TIMING per-repo",
|
||
zap.String("repo", idx.repoPrefix),
|
||
zap.Duration("gomod", dGoMod),
|
||
zap.Duration("resolve", dResolve),
|
||
zap.Duration("enrich", dEnrich),
|
||
zap.Duration("contract_commit", dContract))
|
||
}
|
||
|
||
// runDeferredGoMod materialises dep::<module> contract nodes from go.mod
|
||
// BEFORE ResolveAll so the resolver's import bridge can re-target Go imports
|
||
// of declared modules to their dep contract node instead of an external::
|
||
// stub. Split out of RunDeferredPasses so the batch driver can run it
|
||
// serially across repos ahead of the parallel enrichment phase.
|
||
func (idx *Indexer) runDeferredGoMod() {
|
||
if idx.pendingContractReg == nil {
|
||
return
|
||
}
|
||
idx.extractGoModContracts(idx.pendingContractReg)
|
||
}
|
||
|
||
// runDeferredEnrich runs semantic enrichment for this repo. Safe to run
|
||
// concurrently across repos: the manager fetches a per-repo LSP provider
|
||
// instance (keyed by the repo's workspace), the go-types stash is keyed by
|
||
// repo root, tstypes is stateless, and every provider serialises its graph
|
||
// mutations on the backend resolve mutex.
|
||
func (idx *Indexer) runDeferredEnrich() {
|
||
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
|
||
return
|
||
}
|
||
// Gate the pass to repos that actually changed. The daemon warmup collects
|
||
// every indexer and enriches them all, but an unchanged repo's persisted
|
||
// graph already carries its enrichment edges — re-running gopls hover for
|
||
// it confirms nothing over many minutes. GORTEX_WARMUP_FORCE_ENRICH=1
|
||
// bypasses the gate for a full re-enrich.
|
||
forced := os.Getenv("GORTEX_WARMUP_FORCE_ENRICH") == "1"
|
||
if !idx.pendingEnrich.Load() {
|
||
if !forced {
|
||
idx.logger.Info("deferred enrichment skipped",
|
||
zap.String("repo", idx.repoPrefix),
|
||
zap.String("reason", "unchanged"))
|
||
return
|
||
}
|
||
idx.logger.Info("deferred enrichment forced despite no pending changes",
|
||
zap.String("repo", idx.repoPrefix))
|
||
}
|
||
// Key by the repo prefix so a repo-scoped provider can scope file
|
||
// selection to this repo (empty in single-repo mode).
|
||
roots := map[string]string{idx.repoPrefix: idx.rootPath}
|
||
// Compute the repo's git freshness ONCE and thread it in so the manager's
|
||
// per-provider skip gate and the completion-marker write agree on the
|
||
// identical (sha, dirty): a provider whose persisted marker still matches
|
||
// HEAD on a clean tree is skipped instead of re-running its hover pass.
|
||
sha, dirty := repoHeadAndDirty(idx.rootPath)
|
||
// A re-parse this run evicted the persisted hover edges of the re-parsed
|
||
// files, so force the pass past the completion-marker gate — an unchanged
|
||
// clean HEAD would otherwise skip re-enrichment and leave those files' LSP
|
||
// edges durably gone. fullReindexed covers a whole-repo re-parse (IndexCtx);
|
||
// reparsedThisRun covers a scoped incremental re-parse that dropped only the
|
||
// changed files' edges. Either forces the pass; the provider re-hovers only
|
||
// the freshly-unstamped nodes, so a scoped force stays bounded to those files.
|
||
opts := semantic.EnrichOptions{
|
||
RepoState: map[string]semantic.RepoEnrichState{
|
||
idx.repoPrefix: {SHA: sha, Dirty: dirty, Force: idx.fullReindexed.Load() || idx.reparsedThisRun.Load()},
|
||
},
|
||
}
|
||
results, partialRepos, err := idx.semanticMgr.EnrichAll(idx.graph, roots, opts)
|
||
if err != nil {
|
||
idx.logger.Warn("semantic enrichment failed", zap.Error(err))
|
||
return
|
||
}
|
||
for _, r := range results {
|
||
idx.logger.Info("semantic enrichment result",
|
||
zap.String("provider", r.Provider),
|
||
zap.String("language", r.Language),
|
||
zap.Int("confirmed", r.EdgesConfirmed),
|
||
zap.Int("added", r.EdgesAdded),
|
||
zap.Int("refuted", r.EdgesRefuted),
|
||
zap.Float64("coverage", r.CoveragePercent),
|
||
)
|
||
}
|
||
// Clear the pending marker only when every provider that ran for this repo
|
||
// finished non-partial. A partial / abandoned / failed pass leaves it set
|
||
// so a later deferred pass (or the next restart, once the repo changes
|
||
// again) retries the enrichment rather than trusting an incomplete graph.
|
||
if !partialRepos[idx.repoPrefix] {
|
||
idx.pendingEnrich.Store(false)
|
||
// Persist a whole-repo completion marker at this HEAD so the next warm
|
||
// restart can tell, with one lookup, that this repo's enrichment finished
|
||
// and MaybeSeedPendingEnrich need not resume it. A partial / abandoned
|
||
// pass takes the other branch and writes no marker, so the absent marker
|
||
// re-arms the pass on the next start. No-op on a dirty tree / empty sha.
|
||
idx.semanticMgr.RecordRepoEnrichmentComplete(idx.graph, idx.repoPrefix, sha, dirty)
|
||
}
|
||
}
|
||
|
||
// MaybeSeedPendingEnrich re-arms the deferred-enrichment gate for a repo whose
|
||
// persisted enrichment is known-incomplete at the current clean HEAD, so a warm
|
||
// restart resumes a semantic pass a prior process left partial or abandoned.
|
||
//
|
||
// pendingEnrich otherwise reflects only re-indexing work performed THIS run, so
|
||
// an unchanged repo whose first enrichment was cut short by the per-repo
|
||
// deadline would short-circuit runDeferredEnrich on every subsequent restart —
|
||
// its whole-repo completion marker is absent (a partial pass writes none) yet no
|
||
// file changed to raise the flag. The daemon warmup calls this after the parse
|
||
// phase, before draining the deferred passes.
|
||
//
|
||
// Returns whether this repo will enrich (already pending, or newly seeded). A
|
||
// no-op — false — for a repo without semantic providers, on a non-git tree (no
|
||
// reliable freshness signal), on a backend that does not persist enrichment
|
||
// state (it re-indexes from scratch each restart anyway), when the completion
|
||
// marker already records the current HEAD, or on a dirty tree (the marker is
|
||
// never written or trusted against uncommitted content, and resuming every
|
||
// restart while the tree stays dirty would defeat the warm-restart fast path).
|
||
func (idx *Indexer) MaybeSeedPendingEnrich() bool {
|
||
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
|
||
return false
|
||
}
|
||
if idx.pendingEnrich.Load() {
|
||
// This run's re-indexing work already armed the gate.
|
||
return true
|
||
}
|
||
// Cheap probe first: only the sha is needed to tell a repo whose marker is
|
||
// already current (the common warm-restart case) from one that must resume,
|
||
// so the slower git status shell-out is deferred to the resume path below.
|
||
sha := repoHead(idx.rootPath)
|
||
if sha == "" {
|
||
return false
|
||
}
|
||
current, persisted := idx.semanticMgr.RepoEnrichmentMarkerState(idx.graph, idx.repoPrefix, sha)
|
||
if !persisted || current {
|
||
return false
|
||
}
|
||
if _, dirty := repoHeadAndDirty(idx.rootPath); dirty {
|
||
return false
|
||
}
|
||
idx.logger.Info("deferred enrichment re-armed: persisted enrichment incomplete",
|
||
zap.String("repo", idx.repoPrefix),
|
||
zap.String("sha", sha))
|
||
idx.pendingEnrich.Store(true)
|
||
return true
|
||
}
|
||
|
||
// runDeferredContracts extracts and commits this repo's contract nodes and
|
||
// clears the pending registration. extractGoModContracts already ran via
|
||
// runDeferredGoMod. Mutates the shared graph and walks repo edges, so the
|
||
// batch driver runs it serially after the parallel enrichment phase. The
|
||
// go-types LookupTypeAtLine it relies on reads the per-repo stash, so it is
|
||
// correct even though every repo's enrichment ran before any contracts.
|
||
func (idx *Indexer) runDeferredContracts() {
|
||
if idx.pendingContractReg == nil {
|
||
return
|
||
}
|
||
idx.extractExternalModules()
|
||
idx.extractDIContracts(idx.pendingContractReg)
|
||
idx.commitContracts(idx.pendingContractReg)
|
||
idx.pendingContractReg = nil
|
||
}
|
||
|
||
// RootPath returns the root path used for relative path computation.
|
||
func (idx *Indexer) RootPath() string { return idx.rootPath }
|
||
|
||
// storeRootPath records the repository root, skipping a redundant
|
||
// self-assignment. The watcher goroutine reads idx.rootPath without a
|
||
// lock, so re-storing the identical value from a concurrent reindex or
|
||
// mtime-census pass would be a data race for no observable change.
|
||
func (idx *Indexer) storeRootPath(absRoot string) {
|
||
if idx.rootPath != absRoot {
|
||
idx.rootPath = absRoot
|
||
}
|
||
}
|
||
|
||
// populateCppIncludeDirs reconstructs each C/C++ source file's include search
|
||
// path from compile_commands.json and hands it to the resolver, so a quoted
|
||
// include binds against the real `-I` dir set (deterministic collision-breaking)
|
||
// before the suffix-unique fallback. forceReload drops the cache first, so an
|
||
// incremental reindex picks up an edited compile_commands.json without a
|
||
// daemon restart. Keys/dirs are prefixed in multi-repo mode to match file IDs.
|
||
func (idx *Indexer) populateCppIncludeDirs(forceReload bool) {
|
||
if idx.resolver == nil || idx.rootPath == "" {
|
||
return
|
||
}
|
||
if forceReload {
|
||
clearCppIncludeDirCache(idx.rootPath)
|
||
}
|
||
prefix := ""
|
||
if idx.repoPrefix != "" {
|
||
prefix = idx.repoPrefix + "/"
|
||
}
|
||
prefixDirs := func(dirs []string) []string {
|
||
if prefix == "" || len(dirs) == 0 {
|
||
return dirs
|
||
}
|
||
pd := make([]string, len(dirs))
|
||
for i, d := range dirs {
|
||
pd[i] = prefix + d
|
||
}
|
||
return pd
|
||
}
|
||
tus := loadCompileCommands(idx.rootPath)
|
||
if len(tus) == 0 {
|
||
// No compile DB: fall back to the conventional include-root heuristic
|
||
// so the ordered probe still runs for repos without a compile DB.
|
||
idx.resolver.SetCppIncludeDirs(nil)
|
||
idx.resolver.SetCppFallbackIncludeDirs(prefixDirs(heuristicIncludeDirs(idx.rootPath)))
|
||
return
|
||
}
|
||
perFile := make(map[string][]string, len(tus))
|
||
for f, tu := range tus {
|
||
perFile[prefix+f] = prefixDirs(tu.includeDirs)
|
||
}
|
||
idx.resolver.SetCppIncludeDirs(perFile)
|
||
idx.resolver.SetCppFallbackIncludeDirs(nil)
|
||
}
|
||
|
||
// ResolveFilePath maps a graph file path (repo-relative in single-repo mode)
|
||
// to an absolute filesystem path. Returns "" when no root is set so callers
|
||
// can refuse rather than open against the daemon process CWD. Implements
|
||
// analysis.SourceReader.
|
||
func (idx *Indexer) ResolveFilePath(graphPath string) string {
|
||
if graphPath == "" {
|
||
return ""
|
||
}
|
||
if filepath.IsAbs(graphPath) {
|
||
return filepath.Clean(graphPath)
|
||
}
|
||
if idx.rootPath == "" {
|
||
return ""
|
||
}
|
||
// In multi-repo mode the lone Indexer is wrapped by MultiIndexer
|
||
// (which exposes RepoRoot/ResolveFilePath); single-repo callers
|
||
// hit this path directly.
|
||
rel := graphPath
|
||
if idx.repoPrefix != "" {
|
||
rel = strings.TrimPrefix(rel, idx.repoPrefix+"/")
|
||
}
|
||
return filepath.Clean(filepath.Join(idx.rootPath, rel))
|
||
}
|
||
|
||
// relKey reduces an absolute path under the repo root to the canonical
|
||
// repo-relative key the graph and the mtime map are indexed by: forward
|
||
// slashes, and Unicode NFC.
|
||
//
|
||
// The NFC fold is load-bearing. A file with a non-ASCII name is handed
|
||
// to the indexer in different byte forms depending on the source — the
|
||
// filesystem walk (filepath.WalkDir) yields decomposed NFD on macOS,
|
||
// while the git watcher decodes `git diff` output that git stored as
|
||
// precomposed NFC. Keying the bulk walk under one form and an
|
||
// incremental patch under the other would split a single file across
|
||
// two graph keys: the watcher's evict would miss the walk's node,
|
||
// IncrementalReindex would see the file as both deleted and freshly
|
||
// created, and the daemon would carry a stale duplicate. Folding every
|
||
// key through one form here removes that whole class of mismatch.
|
||
//
|
||
// On a path that is not under rootPath (filepath.Rel fails) the input
|
||
// is returned slash-normalised and NFC-folded so the result is still a
|
||
// stable key, just not repo-relative.
|
||
func (idx *Indexer) relKey(absPath string) string {
|
||
rel, err := filepath.Rel(idx.rootPath, absPath)
|
||
if err != nil {
|
||
return pathkey.Normalize(filepath.ToSlash(absPath))
|
||
}
|
||
return pathkey.Normalize(filepath.ToSlash(rel))
|
||
}
|
||
|
||
// RelKey exposes relKey to in-package collaborators (the watcher) that
|
||
// hold an absolute filesystem path and need the canonical repo-relative
|
||
// graph key for it — e.g. to look up a file's nodes before and after a
|
||
// re-index. Going through one helper keeps the watcher's key in lockstep
|
||
// with the keys IndexFile / EvictFile write.
|
||
func (idx *Indexer) RelKey(absPath string) string { return idx.relKey(absPath) }
|
||
|
||
// SetRepoPrefix sets the repository prefix for multi-repo mode.
|
||
// When non-empty, all node IDs and file paths are prefixed with "<repoPrefix>/".
|
||
func (idx *Indexer) SetRepoPrefix(prefix string) { idx.repoPrefix = prefix }
|
||
|
||
// RepoPrefix returns the current repository prefix.
|
||
func (idx *Indexer) RepoPrefix() string { return idx.repoPrefix }
|
||
|
||
// SetWorkspaceID sets the workspace slug stamped onto nodes emitted
|
||
// by this indexer. Empty means "no workspace declared" — the
|
||
// applyRepoPrefix path will fall back to RepoPrefix so multi-repo
|
||
// configs without `.gortex.yaml::workspace:` keep working.
|
||
func (idx *Indexer) SetWorkspaceID(id string) { idx.workspaceID = id }
|
||
|
||
// WorkspaceID returns the workspace slug this indexer stamps on nodes.
|
||
func (idx *Indexer) WorkspaceID() string { return idx.workspaceID }
|
||
|
||
// SetProjectID sets the project slug stamped onto nodes emitted by
|
||
// this indexer. Single-project repos pass their repo name (the
|
||
// MultiIndexer default); monorepos compute a per-file slug from the
|
||
// `projects[]` mapping (follow-up work).
|
||
func (idx *Indexer) SetProjectID(id string) { idx.projectID = id }
|
||
|
||
// ProjectID returns the project slug this indexer stamps on nodes.
|
||
func (idx *Indexer) ProjectID() string { return idx.projectID }
|
||
|
||
// SetEmbedder sets the embedding provider for semantic search.
|
||
// When set, buildSearchIndex will create a HybridBackend with vector search.
|
||
func (idx *Indexer) SetEmbedder(p embedding.Provider) { idx.embedder = p }
|
||
|
||
// SetSkipVectorBuild toggles the embedding pass in buildSearchIndex.
|
||
// When true, buildSearchIndex builds only the text index — used by the
|
||
// daemon warmup path when a snapshot already carries the workspace
|
||
// vector index, so the graph is not needlessly re-embedded. When false
|
||
// (the default) an indexer with an embedder set always builds vectors.
|
||
func (idx *Indexer) SetSkipVectorBuild(skip bool) { idx.skipVectorBuild = skip }
|
||
|
||
// SetEmbeddingChunkOptions tunes the AST sub-chunking applied to large
|
||
// symbols before embedding (threshold and window line counts). The
|
||
// zero value leaves the chunker on its built-in defaults.
|
||
func (idx *Indexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) {
|
||
idx.embedChunkOpts = opts
|
||
}
|
||
|
||
// SetEmbeddingMaxSymbols overrides the cap on how many texts the vector
|
||
// index will hold before buildSearchIndex skips the embed pass. Zero
|
||
// keeps the built-in default.
|
||
func (idx *Indexer) SetEmbeddingMaxSymbols(n int) { idx.embedMaxSymbols = n }
|
||
|
||
// SetEmbeddingAPIConcurrency overrides how many embedding requests run
|
||
// in parallel against an API-backed embedder. Zero keeps the built-in
|
||
// default. Has no effect on in-process embedders.
|
||
func (idx *Indexer) SetEmbeddingAPIConcurrency(n int) { idx.embedAPIConcurrency = n }
|
||
|
||
// LastVectorBuildError returns why the most recent index build produced no
|
||
// vector index (chunk-embed failure, all vectors invalid, or the symbol-count
|
||
// guard), or nil when a vector index was built or the embedder was unset. Read
|
||
// it after an index build completes; it is not safe to call concurrently with
|
||
// one.
|
||
func (idx *Indexer) LastVectorBuildError() error { return idx.lastVectorBuildErr }
|
||
|
||
// SetSemanticManager sets the semantic enrichment manager.
|
||
// When set, the indexer runs semantic enrichment after resolution.
|
||
func (idx *Indexer) SetSemanticManager(m *semantic.Manager) { idx.semanticMgr = m }
|
||
|
||
// SemanticManager returns the semantic enrichment manager.
|
||
func (idx *Indexer) SemanticManager() *semantic.Manager { return idx.semanticMgr }
|
||
|
||
// SetResolverLSPHelper installs a resolve-time LSP helper on the
|
||
// underlying Resolver. The helper is consulted from inside
|
||
// resolveEdge for languages whose extensions the helper claims
|
||
// (TS/JS/JSX/TSX today via tsserver); see internal/resolver/
|
||
// lsp_helper.go for the contract.
|
||
//
|
||
// Pass nil to detach. Must be called before ResolveAll / ResolveFile;
|
||
// the resolver caches no LSP state across passes, so mid-pass swaps
|
||
// are racy and not supported.
|
||
func (idx *Indexer) SetResolverLSPHelper(h resolver.LSPHelper) {
|
||
if idx.resolver != nil {
|
||
idx.resolver.SetLSPHelper(h)
|
||
}
|
||
idx.resolverLSPHelper = h
|
||
}
|
||
|
||
// ResolverLSPHelper returns the currently installed resolver-time LSP
|
||
// helper, or nil. Exported so MultiIndexer can mirror the helper onto
|
||
// the global post-pass resolver in RunDeferredPassesAll.
|
||
func (idx *Indexer) ResolverLSPHelper() resolver.LSPHelper { return idx.resolverLSPHelper }
|
||
|
||
// ExportVectorIndex returns the serialized vector index bytes, dims, and count.
|
||
// Returns nil, 0, 0 if no vector index is active.
|
||
func (idx *Indexer) ExportVectorIndex() ([]byte, int, int) {
|
||
hybrid, ok := idx.swappable().Inner().(*search.HybridBackend)
|
||
if !ok {
|
||
return nil, 0, 0
|
||
}
|
||
vec := hybrid.VectorIndex()
|
||
if vec == nil || vec.Count() == 0 {
|
||
return nil, 0, 0
|
||
}
|
||
|
||
var buf bytes.Buffer
|
||
if err := vec.Save(&buf); err != nil {
|
||
idx.logger.Warn("failed to export vector index", zap.Error(err))
|
||
return nil, 0, 0
|
||
}
|
||
return buf.Bytes(), vec.Dims(), vec.Count()
|
||
}
|
||
|
||
// ImportVectorIndex restores a vector index from serialized data and wraps
|
||
// the current text search backend into a HybridBackend.
|
||
func (idx *Indexer) ImportVectorIndex(data []byte, dims, count int) error {
|
||
if len(data) == 0 || idx.embedder == nil {
|
||
return nil
|
||
}
|
||
|
||
// Validate dimensions match the current embedder to avoid mismatches
|
||
// when switching providers (e.g., GloVe 50d → ONNX 384d).
|
||
embedderDims := idx.embedder.Dimensions()
|
||
if embedderDims > 0 && embedderDims != dims {
|
||
idx.logger.Info("vector index dims mismatch, will re-embed",
|
||
zap.Int("cached_dims", dims), zap.Int("embedder_dims", embedderDims))
|
||
return nil // skip import, buildSearchIndex will re-embed
|
||
}
|
||
|
||
vec := search.NewVector(dims)
|
||
if err := vec.LoadFrom(bytes.NewReader(data)); err != nil {
|
||
return fmt.Errorf("import vector index: %w", err)
|
||
}
|
||
vec.SetCount(count)
|
||
|
||
sw := idx.swappable()
|
||
sw.Swap(search.NewHybrid(sw.Inner(), vec, idx.embedder))
|
||
idx.logger.Info("restored vector index from cache",
|
||
zap.Int("vectors", count), zap.Int("dims", dims))
|
||
return nil
|
||
}
|
||
|
||
// prefixPath prepends the repoPrefix to a relative path when in multi-repo mode.
|
||
// Returns the path unchanged when repoPrefix is empty.
|
||
func (idx *Indexer) prefixPath(relPath string) string {
|
||
if idx.repoPrefix == "" {
|
||
return relPath
|
||
}
|
||
return idx.repoPrefix + "/" + relPath
|
||
}
|
||
|
||
// graphFilePaths maps reindex file paths (absolute or root-relative, as
|
||
// passed to IndexFile) to the canonical graph file-path form
|
||
// (prefixPath(relKey)) that GetFileNodes is keyed by — so file-scoped
|
||
// passes can look up the just-reindexed files' nodes.
|
||
func (idx *Indexer) graphFilePaths(files []string) []string {
|
||
out := make([]string, 0, len(files))
|
||
for _, f := range files {
|
||
abs := f
|
||
if !filepath.IsAbs(abs) && idx.rootPath != "" {
|
||
abs = filepath.Join(idx.rootPath, f)
|
||
}
|
||
out = append(out, idx.prefixPath(idx.relKey(abs)))
|
||
}
|
||
return out
|
||
}
|
||
|
||
// applyRepoPrefix transforms nodes and edges produced by an extractor to include
|
||
// the repo prefix in IDs and file paths. Sets Node.RepoPrefix on all nodes.
|
||
// This is a no-op when repoPrefix is empty (single-repo mode).
|
||
//
|
||
// Edge targets beginning with "unresolved::" are a sentinel meaning "the
|
||
// resolver will replace this with a real node ID after all files are
|
||
// indexed." Prefixing them turns "unresolved::fetchUsers" into
|
||
// "web/unresolved::fetchUsers", which the resolver's HasPrefix check on
|
||
// "unresolved::" misses — leaving every call edge permanently unresolved
|
||
// in multi-repo mode and breaking get_callers / get_call_chain across all
|
||
// languages. Skip prefixing on unresolved targets; the resolver will land
|
||
// the edge on a real ID that already carries its own correct prefix
|
||
// (possibly cross-repo, which the resolver marks explicitly).
|
||
|
||
// todoTags returns the configured TODO marker set or the default
|
||
// (TODO/FIXME/HACK/XXX/NOTE). Reads from the IndexConfig the indexer
|
||
// already holds — IsCoverageEnabled gating happens at the call site.
|
||
func (idx *Indexer) todoTags() []string {
|
||
if tags := idx.config.Coverage.Todos.Tags; len(tags) > 0 {
|
||
return tags
|
||
}
|
||
return []string{"TODO", "FIXME", "HACK", "XXX", "NOTE"}
|
||
}
|
||
|
||
// todoMaxText returns the configured cap on stored TODO text or the
|
||
// 200-char default.
|
||
func (idx *Indexer) todoMaxText() int {
|
||
if n := idx.config.Coverage.Todos.MaxText; n > 0 {
|
||
return n
|
||
}
|
||
return 200
|
||
}
|
||
|
||
// loadCodeownersRules lazily parses the repo's CODEOWNERS file. The
|
||
// sync.Once guarantees one parse per indexer; applyCoverageDomains
|
||
// is then a pure rule-match per file. Errors silently produce an
|
||
// empty rule set — the ownership domain is implicitly gated on
|
||
// file presence rather than failing extraction when the file is
|
||
// missing or malformed.
|
||
func (idx *Indexer) loadCodeownersRules() []codeowners.Rule {
|
||
idx.codeownersOnce.Do(func() {
|
||
rules, _, ok := codeowners.LoadFromRepo(idx.rootPath)
|
||
if !ok {
|
||
return
|
||
}
|
||
idx.codeownersRules = rules
|
||
})
|
||
return idx.codeownersRules
|
||
}
|
||
|
||
// applyCoverageDomains runs the per-file coverage extractors
|
||
// (todos, licenses, ownership) and applies the post-extraction
|
||
// strip pass for domains the language extractor always emits but
|
||
// the user has gated off (function_shape). Appended/stripped
|
||
// nodes/edges flow through the same applyRepoPrefix / graph.AddNode
|
||
// pipeline as the language extractor's output. Called from both
|
||
// the bulk index worker pool (IndexCtx) and the incremental
|
||
// indexFile path.
|
||
//
|
||
// relPath is the unprefixed file path; lang is the detected
|
||
// language; src is the file bytes.
|
||
func (idx *Indexer) applyCoverageDomains(relPath, lang string, src []byte, result *parser.ExtractionResult) {
|
||
if idx.config.Coverage.IsEnabled("todos") {
|
||
findings := todos.Scan(src, idx.todoTags(), idx.todoMaxText())
|
||
todoNodes, todoEdges := todos.BuildGraphArtifacts(relPath, findings, lang)
|
||
result.Nodes = append(result.Nodes, todoNodes...)
|
||
result.Edges = append(result.Edges, todoEdges...)
|
||
}
|
||
if idx.config.Coverage.IsEnabled("licenses") {
|
||
if spdx := licenses.Scan(src); spdx != "" {
|
||
licNodes, licEdges := licenses.BuildGraphArtifacts(relPath, spdx, lang)
|
||
result.Nodes = append(result.Nodes, licNodes...)
|
||
result.Edges = append(result.Edges, licEdges...)
|
||
}
|
||
}
|
||
if idx.config.Coverage.IsEnabled("ownership") {
|
||
if rules := idx.loadCodeownersRules(); len(rules) > 0 {
|
||
if owners := codeowners.MatchFile(relPath, rules); len(owners) > 0 {
|
||
teamNodes, teamEdges := codeowners.BuildGraphArtifacts(relPath, owners, lang)
|
||
result.Nodes = append(result.Nodes, teamNodes...)
|
||
result.Edges = append(result.Edges, teamEdges...)
|
||
}
|
||
}
|
||
}
|
||
if idx.config.Coverage.IsEnabled("codegen") {
|
||
if marker := codegen.Scan(src); marker.Generated {
|
||
// Stamp the marker on the file node when the language
|
||
// extractor produced one. Generated files without a
|
||
// file-shaped result node still get the EdgeGeneratedBy
|
||
// edge so downstream walks pick them up.
|
||
for _, n := range result.Nodes {
|
||
if n.Kind == graph.KindFile && n.FilePath == relPath {
|
||
if n.Meta == nil {
|
||
n.Meta = map[string]any{}
|
||
}
|
||
codegen.MarkFileNode(n.Meta, marker)
|
||
break
|
||
}
|
||
}
|
||
result.Edges = append(result.Edges, codegen.BuildGraphArtifacts(relPath, marker)...)
|
||
}
|
||
// Annotation-driven codegen: Lombok / MapStruct / Kotlin
|
||
// compiler plugins generate members that never appear in
|
||
// source. Flag the annotated symbols so they stay visible.
|
||
if extra, st := codegen.MarkAnnotatedGenerated(result.Nodes, result.Edges); st.NodesMarked > 0 {
|
||
result.Edges = append(result.Edges, extra...)
|
||
// Materialize the actual Lombok accessor members (getX/setX/
|
||
// builder/log) so `obj.getName()` resolves to a real graph node.
|
||
if lnodes, ledges := codegen.MaterializeLombokAccessors(result.Nodes); len(lnodes) > 0 {
|
||
result.Nodes = append(result.Nodes, lnodes...)
|
||
result.Edges = append(result.Edges, ledges...)
|
||
}
|
||
}
|
||
}
|
||
// Framework entry points (Alembic migrations / Next.js pages /
|
||
// ASP.NET host files): symbols reachable only from a runtime.
|
||
// Stamped so dead-code analysis treats them as live roots.
|
||
entrypoints.Detect(relPath, lang, result.Nodes, result.Edges)
|
||
if !idx.config.Coverage.IsEnabled("function_shape") {
|
||
stripFunctionShape(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("type_shape") {
|
||
stripTypeShape(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("constants") {
|
||
revertConstantsToVariables(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("concurrency") {
|
||
stripConcurrencyEdges(result)
|
||
}
|
||
if idx.config.Coverage.IsEnabled("fixtures") {
|
||
applyFixtureClassification(relPath, lang, result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("observability") {
|
||
stripObservabilityArtifacts(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("pubsub") {
|
||
stripPubsubArtifacts(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("flags") {
|
||
stripFlagArtifacts(result)
|
||
}
|
||
if !idx.config.Coverage.IsEnabled("configs") {
|
||
stripConfigArtifacts(result)
|
||
}
|
||
// SQL migration / DDL extraction is high-signal — CREATE TABLE in a
|
||
// dedicated migration file or a `gortex db schema` dump is
|
||
// unambiguous, unlike the noisy code-side string-literal SQL the
|
||
// `sql` domain gates. Run it always; the strip below removes only the
|
||
// gated code-side SQL, preserving migration-origin schema nodes.
|
||
applyMigrationExtraction(relPath, src, result)
|
||
if !idx.config.Coverage.IsEnabled("sql") {
|
||
stripSQLArtifacts(result)
|
||
}
|
||
if idx.config.Coverage.IsEnabled("clones") {
|
||
applyCloneSignatures(src, result)
|
||
}
|
||
}
|
||
|
||
// applyMigrationExtraction parses CREATE TABLE DDL out of SQL migration
|
||
// files (path under migrate/ or migrations/) and `gortex db schema` dumps
|
||
// (recognised by their generated-header marker, wherever they're saved).
|
||
// Each declared table becomes a KindTable node with its columns as
|
||
// KindColumn nodes (EdgeMemberOf column → table), alongside the synthetic
|
||
// KindMigration node for the file, with EdgeProvides edges from migration
|
||
// → table so reverse-walk queries answer "which migrations create this
|
||
// table". A generated dump carries the real dialect in its header;
|
||
// hand-written migrations stay "generic" since the .sql file alone doesn't
|
||
// tell us which dialect the application targets. All emitted nodes carry
|
||
// Meta["origin"]="migration" so they survive the sql-domain strip — this
|
||
// schema DDL is unambiguous and high-signal, unlike the code-side
|
||
// string-literal SQL the `sql` coverage gate exists to suppress.
|
||
func applyMigrationExtraction(relPath string, src []byte, result *parser.ExtractionResult) {
|
||
generated := gortexsql.IsGeneratedSchema(src)
|
||
if !gortexsql.IsMigrationPath(relPath) && !generated {
|
||
return
|
||
}
|
||
tables := gortexsql.ExtractCreateTablesWithColumns(string(src))
|
||
if len(tables) == 0 {
|
||
return
|
||
}
|
||
dialect := "generic"
|
||
if generated {
|
||
if d := gortexsql.GeneratedSchemaDialect(src); d != "" {
|
||
dialect = d
|
||
}
|
||
}
|
||
migrationID := gortexsql.MigrationNodeID(relPath)
|
||
result.Nodes = append(result.Nodes, &graph.Node{
|
||
ID: migrationID,
|
||
Kind: graph.KindMigration,
|
||
Name: filepath.Base(relPath),
|
||
FilePath: relPath,
|
||
Language: "sql",
|
||
Meta: map[string]any{
|
||
"dialect": dialect,
|
||
"origin": "migration",
|
||
},
|
||
})
|
||
for _, t := range tables {
|
||
tableID := gortexsql.TableNodeID(dialect, t.Schema, t.Table)
|
||
meta := map[string]any{
|
||
"table": t.Table,
|
||
"dialect": dialect,
|
||
"origin": "migration",
|
||
}
|
||
if t.Schema != "" {
|
||
meta["schema"] = t.Schema
|
||
}
|
||
result.Nodes = append(result.Nodes, &graph.Node{
|
||
ID: tableID,
|
||
Kind: graph.KindTable,
|
||
Name: t.Table,
|
||
FilePath: relPath,
|
||
Language: "sql",
|
||
Meta: meta,
|
||
})
|
||
result.Edges = append(result.Edges, &graph.Edge{
|
||
From: migrationID,
|
||
To: tableID,
|
||
Kind: graph.EdgeProvides,
|
||
FilePath: relPath,
|
||
Origin: graph.OriginASTResolved,
|
||
})
|
||
for _, c := range t.Columns {
|
||
colID := gortexsql.ColumnNodeID(dialect, t.Schema, t.Table, c.Name)
|
||
cmeta := map[string]any{
|
||
"table": t.Table,
|
||
"dialect": dialect,
|
||
"origin": "migration",
|
||
}
|
||
if t.Schema != "" {
|
||
cmeta["schema"] = t.Schema
|
||
}
|
||
if c.Type != "" {
|
||
cmeta["type"] = c.Type
|
||
}
|
||
result.Nodes = append(result.Nodes, &graph.Node{
|
||
ID: colID,
|
||
Kind: graph.KindColumn,
|
||
Name: c.Name,
|
||
FilePath: relPath,
|
||
Language: "sql",
|
||
Meta: cmeta,
|
||
})
|
||
result.Edges = append(result.Edges, &graph.Edge{
|
||
From: colID,
|
||
To: tableID,
|
||
Kind: graph.EdgeMemberOf,
|
||
FilePath: relPath,
|
||
Origin: graph.OriginASTResolved,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
// stripSQLArtifacts drops KindTable + KindMigration nodes plus
|
||
// the matching EdgeQueries / EdgeProvides edges when the sql
|
||
// coverage domain is gated off. Mirrors the strip passes for
|
||
// flags / configs / observability — endpoint-aware so any
|
||
// leftover edges to stripped nodes are pruned. SQL extraction
|
||
// defaults off because string-literal pattern matching against
|
||
// db.Get / db.Query / db.Exec produces false positives when
|
||
// domain code shares method names (cache.Get, etc.).
|
||
func stripSQLArtifacts(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if (n.Kind == graph.KindTable || n.Kind == graph.KindMigration) && !isMigrationOriginNode(n) {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
// SQL-context KindString registry nodes are the short-circuit
|
||
// input for the SQL extractor; gated off alongside the rest
|
||
// of the SQL domain so a disabled gate leaves no SQL-related
|
||
// residue in the graph.
|
||
if n.Kind == graph.KindString {
|
||
if ctx, _ := n.Meta["context"].(string); ctx == "sql" {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if e.Kind == graph.EdgeQueries {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.From]; ok {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// stripConfigArtifacts drops KindConfigKey nodes plus
|
||
// EdgeReadsConfig / EdgeWritesConfig edges when the configs
|
||
// coverage domain is gated off. Endpoint-aware so any leftover
|
||
// edges to stripped key nodes are pruned.
|
||
//
|
||
// Infrastructure-origin config keys (Meta["origin"] in {"k8s",
|
||
// "dockerfile"}) are preserved because they are emitted by the K8s
|
||
// manifest, Kustomize, and Dockerfile extractors, which have no
|
||
// dedicated coverage flag and always run. Stripping them would
|
||
// also strip the EdgeUsesEnv edges those extractors produce (which
|
||
// target the same node IDs), defeating the cross-ref between
|
||
// container env declarations and code-side `os.Getenv` reads.
|
||
func stripConfigArtifacts(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if n.Kind == graph.KindConfigKey && !isInfraOriginConfigKey(n) && !isDocFrontmatterConfigKey(n) {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if e.Kind == graph.EdgeReadsConfig || e.Kind == graph.EdgeWritesConfig {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// isInfraOriginConfigKey reports whether a KindConfigKey node was
|
||
// emitted by the K8s / Kustomize / Dockerfile extractors. These
|
||
// nodes carry Meta["origin"] = "k8s" or "dockerfile" by convention.
|
||
// The code-side extractors (Go os.Getenv, Python os.environ, viper,
|
||
// struct-tag, …) leave Meta["origin"] empty.
|
||
// isMigrationOriginNode reports whether a node was emitted by migration /
|
||
// live-DB DDL extraction (Meta["origin"]=="migration"). These schema nodes
|
||
// are high-signal and survive the sql-domain strip — the gate only removes
|
||
// the noisy code-side string-literal SQL.
|
||
func isMigrationOriginNode(n *graph.Node) bool {
|
||
if n == nil || n.Meta == nil {
|
||
return false
|
||
}
|
||
o, _ := n.Meta["origin"].(string)
|
||
return o == "migration"
|
||
}
|
||
|
||
func isInfraOriginConfigKey(n *graph.Node) bool {
|
||
if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil {
|
||
return false
|
||
}
|
||
origin, _ := n.Meta["origin"].(string)
|
||
return origin == "k8s" || origin == "dockerfile"
|
||
}
|
||
|
||
// isDocFrontmatterConfigKey reports whether a KindConfigKey node is a
|
||
// document's frontmatter metadata (Quarto .qmd, …) rather than code /
|
||
// infra configuration. These keys ride with the document / prose ingest,
|
||
// which is independent of the `configs` coverage domain, so they survive
|
||
// the strip the same way infra-origin keys do — keeping a .qmd's declared
|
||
// title / format / params searchable by default. Add new frontmatter
|
||
// sources to the switch as prose extractors gain frontmatter support.
|
||
func isDocFrontmatterConfigKey(n *graph.Node) bool {
|
||
if n == nil || n.Kind != graph.KindConfigKey || n.Meta == nil {
|
||
return false
|
||
}
|
||
switch src, _ := n.Meta["source"].(string); src {
|
||
case "quarto_frontmatter":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// stripFlagArtifacts drops KindFlag nodes and EdgeTogglesFlag
|
||
// edges when the flags coverage domain is gated off. Mirrors the
|
||
// observability strip — endpoint-aware so any leftover edges that
|
||
// pointed to a removed flag node are also dropped.
|
||
func stripFlagArtifacts(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if n.Kind == graph.KindFlag {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if e.Kind == graph.EdgeTogglesFlag {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// stripObservabilityArtifacts drops the log/metric/trace KindEvent
|
||
// nodes and their EdgeEmits edges when the observability coverage
|
||
// domain is gated off. Used for the same reason as the function-shape
|
||
// and type-shape strips: the language extractor always emits, and the
|
||
// indexer prunes per-file before applyRepoPrefix so the gate stays a
|
||
// pure-config dial without parser plumbing.
|
||
//
|
||
// Pub/sub KindEvent nodes (Meta["event_kind"]="pubsub") are a
|
||
// separately-gated domain — they share the KindEvent kind and the
|
||
// EdgeEmits edge (publish side) but belong to the `pubsub` coverage
|
||
// domain, so this pass leaves them and any EdgeEmits/EdgeListensOn
|
||
// edge targeting them untouched. stripPubsubArtifacts owns those.
|
||
func stripObservabilityArtifacts(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
pubsubNodes := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if isPubsubEventNode(n) {
|
||
pubsubNodes[n.ID] = struct{}{}
|
||
keptNodes = append(keptNodes, n)
|
||
continue
|
||
}
|
||
if n.Kind == graph.KindEvent {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
// log_message-context KindString registry nodes are the
|
||
// string-side shadow of log KindEvent emissions. They gate
|
||
// alongside the rest of the observability domain so a
|
||
// disabled gate leaves no log residue in the graph.
|
||
if n.Kind == graph.KindString {
|
||
if ctx, _ := n.Meta["context"].(string); ctx == "log_message" {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
// The observability gate strips every EdgeEmits — the publish
|
||
// side of the log/metric/trace layer. The one exception is an
|
||
// EdgeEmits whose target is a pub/sub topic node: that's the
|
||
// publish side of the separately-gated pubsub domain, so it
|
||
// survives here and is owned by stripPubsubArtifacts.
|
||
if e.Kind == graph.EdgeEmits {
|
||
if _, ok := pubsubNodes[e.To]; !ok {
|
||
continue
|
||
}
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// stripPubsubArtifacts drops the pub/sub KindEvent topic nodes
|
||
// (Meta["event_kind"]="pubsub"), every EdgeListensOn edge (a
|
||
// pubsub-only edge kind), and any EdgeEmits edge whose target is a
|
||
// pub/sub topic node, when the pubsub coverage domain is gated off.
|
||
// Endpoint-aware so a publish edge into a stripped topic node doesn't
|
||
// dangle. Mirrors stripObservabilityArtifacts — the two domains share
|
||
// KindEvent + EdgeEmits but are toggled independently.
|
||
func stripPubsubArtifacts(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if isPubsubEventNode(n) {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if e.Kind == graph.EdgeListensOn {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// isPubsubEventNode reports whether a node is a pub/sub topic node — a
|
||
// KindEvent carrying Meta["event_kind"]="pubsub". Distinguishes the
|
||
// pub/sub domain from observability (log/metric/trace) events, which
|
||
// share KindEvent but are gated separately.
|
||
func isPubsubEventNode(n *graph.Node) bool {
|
||
if n == nil || n.Kind != graph.KindEvent || n.Meta == nil {
|
||
return false
|
||
}
|
||
kind, _ := n.Meta["event_kind"].(string)
|
||
return kind == "pubsub"
|
||
}
|
||
|
||
// applyFixtureClassification reclassifies the language extractor's
|
||
// emitted file node from KindFile to KindFixture when the file
|
||
// lives under a testdata/ directory. When the language extractor
|
||
// produced no file node (file types without a registered
|
||
// extractor), a standalone KindFixture node is emitted instead.
|
||
//
|
||
// Reference edges from test functions to fixtures are out of scope
|
||
// for v1 — agents can already filter by kind to enumerate fixtures.
|
||
func applyFixtureClassification(relPath, lang string, result *parser.ExtractionResult) {
|
||
for _, n := range result.Nodes {
|
||
if n.Kind == graph.KindFile && n.FilePath == relPath {
|
||
if fixtures.ReclassifyFileToFixture(n) {
|
||
return
|
||
}
|
||
break
|
||
}
|
||
}
|
||
result.Nodes = append(result.Nodes, fixtures.BuildGraphArtifacts(relPath, lang)...)
|
||
}
|
||
|
||
// stripConcurrencyEdges removes the EdgeSpawns / EdgeSends /
|
||
// EdgeRecvs edges introduced by the concurrency coverage domain.
|
||
// EdgeCalls is left in place — spawns are emitted in addition to
|
||
// the corresponding call edge, so dropping just the spawn edge
|
||
// reverts to the pre-coverage call graph without losing reachability.
|
||
func stripConcurrencyEdges(result *parser.ExtractionResult) {
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
switch e.Kind {
|
||
case graph.EdgeSpawns, graph.EdgeSends, graph.EdgeRecvs:
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// revertConstantsToVariables re-classifies KindConstant /
|
||
// KindEnumMember nodes back to KindVariable when the constants
|
||
// coverage domain is gated off. Unlike stripFunctionShape /
|
||
// stripTypeShape this is a re-classification, not a removal —
|
||
// users who disable the domain still want their `const` and `iota`
|
||
// declarations in the graph, just under the original kind that
|
||
// pre-coverage code expected.
|
||
func revertConstantsToVariables(result *parser.ExtractionResult) {
|
||
for _, n := range result.Nodes {
|
||
switch n.Kind {
|
||
case graph.KindConstant, graph.KindEnumMember:
|
||
n.Kind = graph.KindVariable
|
||
}
|
||
}
|
||
}
|
||
|
||
// stripTypeShape removes the alias / composition edges introduced
|
||
// by the type_shape coverage domain (EdgeAliases, EdgeComposes).
|
||
// EdgeExtends is left in place — it's an existing edge kind whose
|
||
// newtype-derived emissions fall under the spec's "EdgeExtends
|
||
// continues to mean newtype / inheritance / interface extension"
|
||
// guarantee, not a new domain signal.
|
||
func stripTypeShape(result *parser.ExtractionResult) {
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
switch e.Kind {
|
||
case graph.EdgeAliases, graph.EdgeComposes:
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
// stripFunctionShape removes the param/closure/generic_param nodes
|
||
// and their associated edges from a per-file extraction result.
|
||
// Used when the function_shape coverage domain is gated off — the
|
||
// language extractor always emits these for resolution-internal
|
||
// reasons, and we drop them after the extractor returns rather than
|
||
// wire a config dependency through the parser layer.
|
||
func stripFunctionShape(result *parser.ExtractionResult) {
|
||
stripped := make(map[string]struct{})
|
||
keptNodes := result.Nodes[:0]
|
||
for _, n := range result.Nodes {
|
||
if isFunctionShapeNode(n.Kind) {
|
||
stripped[n.ID] = struct{}{}
|
||
continue
|
||
}
|
||
keptNodes = append(keptNodes, n)
|
||
}
|
||
result.Nodes = keptNodes
|
||
keptEdges := result.Edges[:0]
|
||
for _, e := range result.Edges {
|
||
if isFunctionShapeEdge(e.Kind) {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.From]; ok {
|
||
continue
|
||
}
|
||
if _, ok := stripped[e.To]; ok {
|
||
continue
|
||
}
|
||
keptEdges = append(keptEdges, e)
|
||
}
|
||
result.Edges = keptEdges
|
||
}
|
||
|
||
func isFunctionShapeNode(kind graph.NodeKind) bool {
|
||
switch kind {
|
||
case graph.KindParam, graph.KindClosure, graph.KindGenericParam:
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isFunctionShapeEdge(kind graph.EdgeKind) bool {
|
||
switch kind {
|
||
case graph.EdgeParamOf, graph.EdgeReturns, graph.EdgeTypedAs, graph.EdgeCaptures:
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func (idx *Indexer) applyRepoPrefix(nodes []*graph.Node, edges []*graph.Edge) {
|
||
// Stamp WorkspaceID / ProjectID on every node emitted by this
|
||
// indexer regardless of mode — single-repo and multi-repo both
|
||
// need the boundary slugs for query scoping and contract
|
||
// matching. Single-repo callers can leave them empty; the
|
||
// MultiIndexer path always sets them via SetWorkspaceID /
|
||
// SetProjectID before calling Index.
|
||
if idx.workspaceID != "" || idx.projectID != "" {
|
||
for _, n := range nodes {
|
||
if idx.workspaceID != "" && n.WorkspaceID == "" {
|
||
n.WorkspaceID = idx.workspaceID
|
||
}
|
||
if idx.projectID != "" && n.ProjectID == "" {
|
||
n.ProjectID = idx.projectID
|
||
}
|
||
}
|
||
}
|
||
if idx.repoPrefix == "" {
|
||
return
|
||
}
|
||
prefix := idx.repoPrefix + "/"
|
||
const unresolvedMarker = "unresolved::"
|
||
// Intern every minted string. A node ID is referenced once on the
|
||
// node and again on every edge endpoint that points at it; a file
|
||
// path recurs on every node and edge in that file. Without
|
||
// interning each reference is a distinct `prefix + s` allocation —
|
||
// interning collapses them to one shared backing array, and edge
|
||
// endpoints end up sharing storage with the node ID they name.
|
||
//
|
||
// The file path is identical for every node and edge extracted from
|
||
// the same file — thousands of them. Concatenating `prefix + path`
|
||
// per reference would mint thousands of throwaway strings before
|
||
// interning collapses their storage. This per-call cache computes
|
||
// the interned prefixed path once per distinct raw path, so a file
|
||
// with N symbols pays one concatenation instead of N.
|
||
prefixedPath := make(map[string]string)
|
||
internFilePath := func(raw string) string {
|
||
if c, ok := prefixedPath[raw]; ok {
|
||
return c
|
||
}
|
||
c := intern.String(prefix + raw)
|
||
prefixedPath[raw] = c
|
||
return c
|
||
}
|
||
for _, n := range nodes {
|
||
n.ID = intern.String(prefix + n.ID)
|
||
n.FilePath = internFilePath(n.FilePath)
|
||
n.RepoPrefix = idx.repoPrefix
|
||
// Name, Language and Kind are low-cardinality and recur across
|
||
// thousands of nodes — method/function names like String, New,
|
||
// Get…, the ~20 distinct languages, and the fixed set of node
|
||
// kinds. Interning collapses each to a single backing array; it
|
||
// also shrinks the byName secondary index, whose keys are these
|
||
// same strings.
|
||
n.Name = intern.String(n.Name)
|
||
n.Language = intern.String(n.Language)
|
||
n.Kind = graph.NodeKind(intern.String(string(n.Kind)))
|
||
}
|
||
for _, e := range edges {
|
||
e.From = intern.String(prefix + e.From)
|
||
if strings.HasPrefix(e.To, unresolvedMarker) {
|
||
// Unresolved targets carry no prefix, but many edges name
|
||
// the same unresolved symbol — still worth interning.
|
||
e.To = intern.String(e.To)
|
||
} else {
|
||
e.To = intern.String(prefix + e.To)
|
||
}
|
||
e.FilePath = internFilePath(e.FilePath)
|
||
}
|
||
}
|
||
|
||
// Index walks root and populates the graph using a concurrent worker pool.
|
||
//
|
||
// This is the backwards-compatible entry point; it delegates to IndexCtx with
|
||
// a background context. Callers wanting progress notifications or cancellation
|
||
// should use IndexCtx directly.
|
||
func (idx *Indexer) Index(root string) (*IndexResult, error) {
|
||
return idx.IndexCtx(context.Background(), root)
|
||
}
|
||
|
||
// IndexCtx is Index with a context, enabling progress reporting. The reporter
|
||
// is pulled from ctx via progress.FromContext — attach one with
|
||
// progress.WithReporter to receive stage updates. If no reporter is attached,
|
||
// stage calls are silently dropped.
|
||
// clampParseWeight maps a file size to a parse-admission weight bounded to
|
||
// [1, budget]. A file larger than the whole budget is admitted alone
|
||
// (weight == budget) so the bytes-in-flight semaphore can never deadlock.
|
||
func clampParseWeight(size, budget int64) int64 {
|
||
if size < 1 {
|
||
return 1
|
||
}
|
||
if size > budget {
|
||
return budget
|
||
}
|
||
return size
|
||
}
|
||
|
||
func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexResult, retErr error) {
|
||
start := time.Now()
|
||
reporter := progress.FromContext(ctx)
|
||
|
||
// Cold/full-index GC tuning: raise the GC percent window and install a
|
||
// cgroup-aware soft memory limit for the duration of the index, then
|
||
// restore the prior settings on every exit path. The knobs affect only GC
|
||
// timing and peak RSS during the allocation burst of a full index — the
|
||
// graph content is identical with them on or off. Disable for A/B runs
|
||
// with GORTEX_INDEX_GC_TUNE=0; see gc_tune.go.
|
||
restoreGCTuning := applyIndexGCTuning(idx.logger)
|
||
defer restoreGCTuning()
|
||
|
||
absRoot, err := filepath.Abs(root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
idx.rootPath = absRoot
|
||
idx.projectName = search.DetectProjectName(absRoot)
|
||
|
||
reporter.Report("walking files", 0, 0)
|
||
|
||
// Collect files. Files over IndexConfig.MaxFileSize are skipped
|
||
// during the walk — they're nearly always generated/minified code
|
||
// that dominates parse time without contributing useful signal.
|
||
// A single summary warning reports how many were skipped so the
|
||
// user knows when the cap is biting.
|
||
//
|
||
// Each surviving file is captured with its walk-time ModTime so
|
||
// the worker (contract-cache mtime stamp) and the post-parse
|
||
// fileMtimes loop don't have to os.Stat again. d.Info() is one
|
||
// syscall per file regardless; trading one walk-time stat for two
|
||
// later stats is the net win.
|
||
maxSize := idx.config.MaxFileSize
|
||
// Corpus-admission gate: drops oversized document assets and (by
|
||
// default) binary/vector data artifacts at the walk, before they are
|
||
// read and extracted, so a content-heavy repo can't pull gigabytes of
|
||
// non-source files into the parse pipeline and OOM (#120). Inert for
|
||
// all-code repos.
|
||
contentGate := idx.newContentAdmissionGate()
|
||
// Git-aware admission (opt-in): when index.skip_untracked_assets is on,
|
||
// drop asset-class files git does not track — uncommitted RAG corpora /
|
||
// datasets / build outputs that .gitignore can't catch (#120). Inert
|
||
// when off, on a non-git repo, or when git is unavailable.
|
||
untrackedGate := idx.newUntrackedAssetGate(ctx, absRoot)
|
||
var files []walkedFile
|
||
var skippedLarge int
|
||
var skippedBytes int64
|
||
var skippedBySize []skippedFile
|
||
var skippedByContent []skippedFile
|
||
var skippedContentBytes int64
|
||
var parseFailedFiles []skippedFile
|
||
err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error {
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
if idx.shouldPruneDir(path, absRoot) {
|
||
return filepath.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
lang, ok := idx.effectiveLanguage(path, nil)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
if idx.shouldExclude(path, absRoot, false) {
|
||
return nil
|
||
}
|
||
info, statErr := d.Info()
|
||
if statErr != nil {
|
||
// Couldn't read FileInfo (race with deletion, broken
|
||
// symlink, …). Skip — the worker would fail too.
|
||
return nil
|
||
}
|
||
if maxSize > 0 && info.Size() > maxSize {
|
||
skippedLarge++
|
||
skippedBytes += info.Size()
|
||
rel, _ := filepath.Rel(absRoot, path)
|
||
skippedBySize = append(skippedBySize, skippedFile{
|
||
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(),
|
||
})
|
||
return nil
|
||
}
|
||
if reason, skip := untrackedGate.skip(lang, path); skip {
|
||
skippedContentBytes += info.Size()
|
||
rel, _ := filepath.Rel(absRoot, path)
|
||
skippedByContent = append(skippedByContent, skippedFile{
|
||
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason,
|
||
})
|
||
return nil
|
||
}
|
||
if reason, skip := contentGate.skip(lang, info.Size()); skip {
|
||
skippedContentBytes += info.Size()
|
||
rel, _ := filepath.Rel(absRoot, path)
|
||
skippedByContent = append(skippedByContent, skippedFile{
|
||
relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason,
|
||
})
|
||
return nil
|
||
}
|
||
files = append(files, walkedFile{
|
||
path: path,
|
||
lang: lang,
|
||
size: info.Size(),
|
||
mtimeNano: info.ModTime().UnixNano(),
|
||
})
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if skippedLarge > 0 {
|
||
idx.logger.Info("indexer: skipped large files over MaxFileSize",
|
||
zap.Int("count", skippedLarge),
|
||
zap.Int64("total_bytes", skippedBytes),
|
||
zap.Int64("limit_bytes", maxSize))
|
||
}
|
||
if len(skippedByContent) > 0 {
|
||
idx.logger.Info("indexer: skipped content/data assets by admission policy",
|
||
zap.Int("count", len(skippedByContent)),
|
||
zap.Int64("total_bytes", skippedContentBytes))
|
||
}
|
||
reporter.Report("walking files", len(files), len(files))
|
||
|
||
// In-memory shadow for cold-start indexing on disk-backed stores.
|
||
// Disk backends pay ms-level per-call cost on every read; running
|
||
// the resolver against the disk store turns its ~100k+ point
|
||
// lookups into many minutes of wall time. Instead, swap idx.graph
|
||
// to an in-memory *Graph for the whole IndexCtx pipeline — parse,
|
||
// resolve, all subpasses, every per-edge MERGE/MATCH stays in
|
||
// memory at nanosecond latency. At the end, dump the final state
|
||
// to the disk backend via one BulkLoad cycle, so the disk has the
|
||
// post-resolve graph and the bench's query workload runs against
|
||
// the persisted state.
|
||
//
|
||
// Guards:
|
||
// - Backend must implement graph.BulkLoader (the on-disk backend opts in).
|
||
// - Store must be empty (NodeCount == 0 && EdgeCount == 0). The
|
||
// final dump is BulkLoad's INSERT-only fast path — running it
|
||
// against a non-empty store would corrupt or duplicate.
|
||
// Incremental / re-index flows fall through to the per-call
|
||
// AddBatch path against the disk store directly.
|
||
// - File count is below the shadow-max threshold (see
|
||
// shadowMaxFileCount). Above the threshold the shadow's RAM
|
||
// footprint would exceed available memory — Linux / Firefox
|
||
// at full scale (~10M+ edges) would push the shadow past
|
||
// 20GB. Override with GORTEX_SHADOW_MAX_FILES.
|
||
// - The swap happens before the parse worker pool starts and is
|
||
// committed before IndexCtx returns. retErr from the named
|
||
// return suppresses the commit when the pipeline errored —
|
||
// the disk store stays empty rather than capturing partial
|
||
// state.
|
||
var diskTarget graph.Store
|
||
var inMemShadow *graph.Graph
|
||
bl, blOK := idx.graph.(graph.BulkLoader)
|
||
// Per-Indexer sentinel: each *Indexer is constructed fresh
|
||
// (per-repo in MultiIndexer, once in single-repo daemons), so
|
||
// "this Indexer has indexed before" is the right question to
|
||
// gate the shadow-swap on. The legacy gate looked at the
|
||
// disk store's NodeCount, but in MultiIndexer the disk store
|
||
// holds data from sibling repos that already drained — the
|
||
// gate would mis-fire and force the big repo onto the per-row
|
||
// path. With per-repo-prefixed stub IDs (internal/graph/stub.go)
|
||
// concurrent shadow drains no longer conflict on PRIMARY KEY,
|
||
// so disk-non-empty is safe.
|
||
firstIndex := idx.indexCount.Load() == 0
|
||
belowShadowMax := len(files) <= shadowMaxFileCount()
|
||
// The file-count ceiling is blind to the few-huge-files shape: a
|
||
// content repo of a few hundred PDFs / text dumps / spreadsheets is
|
||
// far under shadowMaxFileCount yet holds multiple GB that explode
|
||
// into hundreds of thousands of section nodes. Gate the in-memory
|
||
// shadow on total input bytes too, so such a repo falls through to
|
||
// the bounded per-call disk path instead of pinning the whole
|
||
// post-parse graph in RAM and OOMing (see #120).
|
||
var totalFileBytes int64
|
||
for i := range files {
|
||
totalFileBytes += files[i].size
|
||
}
|
||
maxShadowBytes := shadowMaxBytes()
|
||
belowShadowBytes := totalFileBytes <= maxShadowBytes
|
||
shadowTaken := blOK && firstIndex && belowShadowMax && belowShadowBytes
|
||
preNodes := idx.graph.NodeCount()
|
||
preEdges := idx.graph.EdgeCount()
|
||
idx.logger.Info("indexer: shadow-swap decision",
|
||
zap.String("repo", idx.RepoPrefix()),
|
||
zap.Bool("bulk_loader", blOK),
|
||
zap.Bool("first_index", firstIndex),
|
||
zap.Int("pre_nodes", preNodes),
|
||
zap.Int("pre_edges", preEdges),
|
||
zap.Int("files", len(files)),
|
||
zap.Int("shadow_max_files", shadowMaxFileCount()),
|
||
zap.Bool("below_shadow_max", belowShadowMax),
|
||
zap.Int64("total_file_bytes", totalFileBytes),
|
||
zap.Int64("shadow_max_bytes", maxShadowBytes),
|
||
zap.Bool("below_shadow_bytes", belowShadowBytes),
|
||
zap.Bool("shadow_taken", shadowTaken),
|
||
)
|
||
if shadowTaken {
|
||
// Warm-restart safety. `firstIndex` is a PER-INDEXER sentinel, and
|
||
// a fresh per-repo Indexer is constructed on every daemon restart,
|
||
// so firstIndex is true on every restart — even when the
|
||
// persistent disk store already holds this repo's nodes from a
|
||
// prior run. The shadow drain below ends in BulkLoad's INSERT-only
|
||
// COPY, which (per this function's own contract) "running against a
|
||
// non-empty store would corrupt or duplicate". A duplicate-primary-
|
||
// key bulk load against the persisted rows would fail warmup, and
|
||
// because the repo's mtimes never get persisted when warmup dies
|
||
// first, the failure re-fires on the next restart: a crash loop.
|
||
// Evicting the repo's existing rows first makes the bulk load land
|
||
// on a clean slate. EvictRepo self-guards with a count query, so this is a
|
||
// cheap no-op for the genuine first-index cases (true cold start,
|
||
// a newly-tracked repo) where the disk store has no rows for this
|
||
// prefix. preNodes>0 short-circuits the call entirely on the
|
||
// first repo of a cold start (empty store).
|
||
if preNodes > 0 {
|
||
if n, e := idx.graph.EvictRepo(idx.RepoPrefix()); n > 0 || e > 0 {
|
||
idx.logger.Info("indexer: evicted stale repo rows before bulk reload (warm restart)",
|
||
zap.String("repo", idx.RepoPrefix()),
|
||
zap.Int("nodes", n), zap.Int("edges", e))
|
||
}
|
||
}
|
||
idx.indexCount.Add(1)
|
||
diskTarget = idx.graph
|
||
inMemShadow = graph.New()
|
||
idx.graph = inMemShadow
|
||
// Capture the disk store as the vector sink: buildSearchIndex runs
|
||
// while idx.graph is the shadow (no VectorSearcher), so without this
|
||
// the embedded vectors never land on disk. The `vectors` table has no
|
||
// FK to `nodes`, so upserting before FlushBulk persists the nodes is
|
||
// safe. Cleared when idx.graph is restored below.
|
||
idx.bulkVectorSink, _ = diskTarget.(graph.VectorSearcher)
|
||
// Same capture for the content index: the per-file content stream
|
||
// must reach content_fts on disk while idx.graph is the shadow.
|
||
idx.contentSink, _ = diskTarget.(graph.ContentSearcher)
|
||
// The resolver was constructed at indexer.New with the disk
|
||
// Store. Redirect it at the shadow too, otherwise ResolveAll
|
||
// reads from the empty disk Store, finds no pending edges,
|
||
// and short-circuits — silently disabling every resolver pass
|
||
// (module attribution, relative imports, edge in-place
|
||
// resolution, …) for any backend that takes the shadow path.
|
||
if idx.resolver != nil {
|
||
idx.resolver.SetGraph(inMemShadow)
|
||
}
|
||
defer func() {
|
||
if retErr != nil {
|
||
idx.graph = diskTarget
|
||
idx.bulkVectorSink = nil
|
||
idx.contentSink = nil
|
||
if idx.resolver != nil {
|
||
idx.resolver.SetGraph(diskTarget)
|
||
}
|
||
return
|
||
}
|
||
reporter.Report("persisting bulk graph", 0, 0)
|
||
drainStart := time.Now()
|
||
shadowNodeCount := inMemShadow.NodeCount()
|
||
shadowEdgeCount := inMemShadow.EdgeCount()
|
||
idx.logger.Info("indexer: drain start (shadow → disk)",
|
||
zap.String("repo", idx.RepoPrefix()),
|
||
zap.Int("shadow_nodes", shadowNodeCount),
|
||
zap.Int("shadow_edges", shadowEdgeCount),
|
||
)
|
||
bl.BeginBulkLoad()
|
||
// Drain the shadow shard-by-shard so the indexer's hold on
|
||
// the 11-GB Linux-scale graph is released progressively
|
||
// instead of pinned until persist returns. The drain
|
||
// iterators free each shard's node/edge maps as they
|
||
// advance, so peak RAM during the persist window is
|
||
// roughly the chunk buffer + the backend's working set,
|
||
// not full shadow + the disk backend's bulk-COPY buffer.
|
||
//
|
||
// Collect (id, tokens) for every search-eligible node as
|
||
// the drain yields them — feeds the backend's native FTS
|
||
// at FlushBulk time when the store implements
|
||
// graph.SymbolSearcher. Nodes that fail
|
||
// shouldIndexForSearch (KindFile / KindImport /
|
||
// KindLocal / KindBuiltin / skip-search lang+kind pairs)
|
||
// are excluded so the FTS corpus matches the in-process
|
||
// BM25 corpus exactly.
|
||
searcher, hasFTS := diskTarget.(graph.SymbolSearcher)
|
||
var ftsItems []graph.SymbolFTSItem
|
||
if hasFTS {
|
||
// Pre-size to the shadow's node count to avoid grow
|
||
// churn on a 600k-node Vscode-shape repo.
|
||
ftsItems = make([]graph.SymbolFTSItem, 0, inMemShadow.NodeCount())
|
||
}
|
||
// Key-ordered bulk drain. The nodes table is WITHOUT ROWID
|
||
// (its primary key IS the B-tree), so inserting in ascending
|
||
// id order makes each insert an append to the rightmost leaf
|
||
// instead of a random split mid-tree — far fewer page splits
|
||
// on the cold load. The edges table's UNIQUE(from_id, …) index
|
||
// benefits the same way from from-id-ordered inserts. So the
|
||
// whole drain is collected, sorted once, then chunk-written.
|
||
//
|
||
// This block only runs on the first/empty cold index (the
|
||
// shadow-swap branch above), and the shadow node/edge sets
|
||
// already fit in RAM (the branch is gated on the shadow byte /
|
||
// file budget). DrainNodes / DrainEdges free each shard as they
|
||
// yield, so holding the drained set in one slice to sort costs
|
||
// no more peak RAM than the shadow already did.
|
||
const persistChunk = 100000
|
||
allNodes := make([]*graph.Node, 0, shadowNodeCount)
|
||
for n := range inMemShadow.DrainNodes() {
|
||
if hasFTS && idx.shouldIndexForSearch(n) {
|
||
ftsItems = append(ftsItems, graph.SymbolFTSItem{
|
||
NodeID: n.ID,
|
||
Tokens: ftsTokensFor(n, idx.projectName),
|
||
})
|
||
}
|
||
allNodes = append(allNodes, n)
|
||
}
|
||
sort.Slice(allNodes, func(i, j int) bool {
|
||
return allNodes[i].ID < allNodes[j].ID
|
||
})
|
||
for start := 0; start < len(allNodes); start += persistChunk {
|
||
end := min(start+persistChunk, len(allNodes))
|
||
diskTarget.AddBatch(allNodes[start:end], nil)
|
||
}
|
||
allNodes = nil
|
||
|
||
allEdges := make([]*graph.Edge, 0, shadowEdgeCount)
|
||
for e := range inMemShadow.DrainEdges() {
|
||
allEdges = append(allEdges, e)
|
||
}
|
||
sort.Slice(allEdges, func(i, j int) bool {
|
||
return lessEdgeKey(allEdges[i], allEdges[j])
|
||
})
|
||
for start := 0; start < len(allEdges); start += persistChunk {
|
||
end := min(start+persistChunk, len(allEdges))
|
||
diskTarget.AddBatch(nil, allEdges[start:end])
|
||
}
|
||
allEdges = nil
|
||
flushStart := time.Now()
|
||
idx.logger.Info("indexer: FlushBulk start",
|
||
zap.String("repo", idx.RepoPrefix()),
|
||
zap.Duration("drain_elapsed", flushStart.Sub(drainStart)),
|
||
)
|
||
if ferr := bl.FlushBulk(); ferr != nil {
|
||
retErr = fmt.Errorf("indexer: persist bulk graph: %w", ferr)
|
||
}
|
||
idx.logger.Info("indexer: FlushBulk complete",
|
||
zap.String("repo", idx.RepoPrefix()),
|
||
zap.Duration("flush_elapsed", time.Since(flushStart)),
|
||
zap.Duration("total_drain", time.Since(drainStart)),
|
||
zap.Int("nodes", shadowNodeCount),
|
||
zap.Int("edges", shadowEdgeCount),
|
||
)
|
||
// Build the backend FTS after the bulk load completes so
|
||
// CREATE_FTS_INDEX has the full corpus to scan in one
|
||
// pass. BulkUpsertSymbolFTS does its own
|
||
// extension-install dance, so this is the only place the
|
||
// indexer needs to know about SymbolSearcher.
|
||
if hasFTS && len(ftsItems) > 0 {
|
||
reporter.Report("building symbol fts", 0, 0)
|
||
if ferr := searcher.BulkUpsertSymbolFTS(idx.RepoPrefix(), ftsItems); ferr != nil {
|
||
idx.logger.Warn("indexer: bulk symbol FTS upsert failed",
|
||
zap.Error(ferr))
|
||
} else if ferr := searcher.BuildSymbolIndex(); ferr != nil {
|
||
idx.logger.Warn("indexer: backend FTS build failed",
|
||
zap.Error(ferr))
|
||
}
|
||
reporter.Report("building symbol fts", 1, 1)
|
||
}
|
||
reporter.Report("persisting bulk graph", 1, 1)
|
||
idx.graph = diskTarget
|
||
idx.bulkVectorSink = nil
|
||
idx.contentSink = nil
|
||
// Mirror of the SetGraph(inMemShadow) above: the resolver
|
||
// must follow the graph pointer back to the disk store, or
|
||
// every post-index per-file resolve (the watcher save path,
|
||
// incremental reindex) reads the drained — now empty —
|
||
// shadow and silently resolves nothing.
|
||
if idx.resolver != nil {
|
||
idx.resolver.SetGraph(diskTarget)
|
||
}
|
||
}()
|
||
} else if diskTarget == nil && idx.graph.NodeCount() == 0 && idx.graph.EdgeCount() == 0 {
|
||
if _, isBulk := idx.graph.(graph.BulkLoader); isBulk && firstIndex && (!belowShadowMax || !belowShadowBytes) {
|
||
idx.logger.Info("indexer: skipping in-memory shadow; building against disk store (bounded RAM)",
|
||
zap.Int("files", len(files)),
|
||
zap.Int("file_threshold", shadowMaxFileCount()),
|
||
zap.Bool("over_file_count", !belowShadowMax),
|
||
zap.Int64("total_file_bytes", totalFileBytes),
|
||
zap.Int64("byte_threshold", maxShadowBytes),
|
||
zap.Bool("over_byte_budget", !belowShadowBytes))
|
||
}
|
||
}
|
||
|
||
// Content-index rebuild strategy. The crash-safe path (on-disk store)
|
||
// deletes each file's prior content rows as that file re-streams
|
||
// (contentWipeFile, invoked at the per-file AddBatch sites below) and
|
||
// sweeps every OTHER content row after the authoritative mtime replace —
|
||
// so a mid-parse kill leaves a mix of old+new content per file instead of
|
||
// the empty table a repo-wide pre-wipe would leave. contentStreamedFiles
|
||
// records which files actually streamed content this walk (the wipe's own
|
||
// argument, i.e. the node FilePath content_fts carries); the end sweep
|
||
// keeps exactly that set, so files that vanished from disk AND files that
|
||
// still exist but no longer yield content sections (doc emptied,
|
||
// classification changed) are both reaped in one scan — the transitions
|
||
// the old repo-wide pre-wipe used to cover. Backends without the per-file
|
||
// capability keep that old behaviour: one repo-wide wipe up front (a
|
||
// cold-store no-op; a cheap per-repo DELETE on a warm one).
|
||
var (
|
||
contentWipeFile func(filePath string)
|
||
contentStreamedMu sync.Mutex
|
||
contentStreamedFiles map[string]struct{}
|
||
)
|
||
if cs := idx.contentSearcher(); cs != nil {
|
||
if w, ok := cs.(interface {
|
||
WipeContentFileInRepo(repoPrefix, filePath string) error
|
||
}); ok {
|
||
repoPrefix := idx.RepoPrefix()
|
||
contentStreamedFiles = make(map[string]struct{})
|
||
contentWipeFile = func(filePath string) {
|
||
if filePath == "" {
|
||
return
|
||
}
|
||
contentStreamedMu.Lock()
|
||
contentStreamedFiles[filePath] = struct{}{}
|
||
contentStreamedMu.Unlock()
|
||
if err := w.WipeContentFileInRepo(repoPrefix, filePath); err != nil {
|
||
idx.logger.Warn("indexer: per-file content wipe failed", zap.Error(err))
|
||
}
|
||
}
|
||
} else if err := cs.WipeContent(idx.RepoPrefix()); err != nil {
|
||
idx.logger.Warn("indexer: content index wipe failed", zap.Error(err))
|
||
}
|
||
}
|
||
|
||
// Worker pool.
|
||
workers := idx.config.Workers
|
||
if workers <= 0 {
|
||
workers = 1
|
||
}
|
||
// idx.config.Workers defaults to the host's runtime.NumCPU(); inside a
|
||
// CPU-limited container that exceeds the cgroup CPU quota and the pool
|
||
// over-subscribes, so CFS throttling drags throughput down. Clamp the
|
||
// effective pool size to the quota when one is present (lowers only, floor
|
||
// of 1, host-identical when unquotaed). GORTEX_INDEX_CPU_CLAMP=0 opts out.
|
||
if cpuClampEnabled() {
|
||
workers = clampWorkersToCPUQuota(workers, cgroupCPUQuota())
|
||
}
|
||
|
||
// Optional crash isolation: run tree-sitter extraction in worker
|
||
// subprocesses so a grammar SIGSEGV / OOM / hang on one
|
||
// pathological file is contained — the bad file is quarantined and
|
||
// the pass still completes. Off unless index.crash_isolation /
|
||
// GORTEX_PARSER_ISOLATION is set.
|
||
var parsePool *crashpool.Pool
|
||
var quarantine *crashpool.Quarantine
|
||
if idx.crashIsolationEnabled() {
|
||
quarantine = crashpool.LoadQuarantine(filepath.Join(absRoot, ".gortex", "parser-quarantine.json"))
|
||
if p, perr := idx.newParsePool(workers); perr != nil {
|
||
idx.logger.Warn("indexer: crash isolation requested but parser pool unavailable; parsing in-process",
|
||
zap.Error(perr))
|
||
} else {
|
||
parsePool = p
|
||
defer parsePool.Close()
|
||
idx.logger.Info("indexer: parser crash isolation enabled", zap.Int("workers", workers))
|
||
}
|
||
}
|
||
|
||
// Workers parse files, write the resulting nodes/edges to the
|
||
// sharded graph, and run per-file contract extractors on the same
|
||
// src bytes — all in one pass. Reusing src avoids the 10k+ disk
|
||
// re-reads the old "parse then extractContracts" flow did; running
|
||
// the contract extractors per-worker parallelises what used to be
|
||
// a serial post-pass; language-filtered dispatch skips extractors
|
||
// that can't match (HTTP on .css, OpenAPI on .ts, etc.).
|
||
const parseReportEvery = 50
|
||
totalFiles := len(files)
|
||
|
||
_, contractExtractorsByLang := idx.buildPerFileContractExtractors()
|
||
contractReg := contracts.NewRegistry()
|
||
var contractMu sync.Mutex
|
||
|
||
var errMu sync.Mutex
|
||
var errors []IndexError
|
||
var processed int64
|
||
var fileCount int64
|
||
var skippedByTimeout int64
|
||
var skippedByMinified int64
|
||
|
||
// Bound peak parse memory: a weighted, bytes-in-flight semaphore
|
||
// admits each worker by its file size before it reads + extracts, so
|
||
// a cluster of large files (PDFs / office docs in a content repo)
|
||
// serialises instead of all workers materialising whole files and
|
||
// their parse trees at once. Code files are tiny and flow freely;
|
||
// only genuinely large inputs queue. budget <= 0 disables the cap.
|
||
parseBudget := idx.config.MaxParseBytesInFlight
|
||
var parseSem *semaphore.Weighted
|
||
if parseBudget > 0 {
|
||
parseSem = semaphore.NewWeighted(parseBudget)
|
||
}
|
||
|
||
// In addition to the bytes-in-flight budget above, cap how many
|
||
// genuinely large files are *read* concurrently: a few huge PDFs /
|
||
// spreadsheets / vector artifacts can dominate RSS before extraction
|
||
// even starts. Small source files bypass the gate and keep full
|
||
// throughput.
|
||
largeReadGate := make(chan struct{}, largeFileReadParallelism(workers))
|
||
readFile := func(wf walkedFile) ([]byte, error) {
|
||
if wf.size < largeFileReadThresholdBytes {
|
||
return os.ReadFile(wf.path)
|
||
}
|
||
largeReadGate <- struct{}{}
|
||
defer func() { <-largeReadGate }()
|
||
return os.ReadFile(wf.path)
|
||
}
|
||
|
||
// recordStreamedMtime persists a file's mtime incrementally, in batches,
|
||
// as its nodes land on disk during a full track — so a first-ever index
|
||
// killed mid-parse RESUMES on the next boot (the flushed files reconcile
|
||
// clean, the remainder re-parses) instead of re-tracking a large repo
|
||
// from scratch every time it dies under memory pressure.
|
||
//
|
||
// INVARIANT: an mtime row must land only AFTER its file's nodes are
|
||
// durably in the store, or a fresh mtime would falsely imply indexed
|
||
// nodes. The `idx.graph.(graph.FileMtimeWriter)` assertion is what
|
||
// enforces it: it succeeds ONLY on the direct-to-disk path, where
|
||
// idx.graph is the on-disk store and AddBatch has already made the
|
||
// file's nodes durable. On the whole-repo in-memory shadow and the
|
||
// streaming-flush per-chunk shadow, idx.graph is a plain graph.Graph
|
||
// (no FileMtimeWriter) whose nodes reach disk only at a later drain, so
|
||
// this self-skips and those paths persist at their own drain points (the
|
||
// final ReplaceFileMtimes / the streaming-flush per-chunk persist). The
|
||
// final ReplaceFileMtimes still runs and is authoritative — it also
|
||
// prunes deleted files — so these batches only bound the work a crash
|
||
// can waste; the leftover under-threshold tail rides that final replace.
|
||
var (
|
||
streamMtimeMu sync.Mutex
|
||
streamMtimes = make(map[string]int64, mtimeStreamPersistEvery)
|
||
)
|
||
recordStreamedMtime := func(absPath string, mtimeNano int64) {
|
||
if mtimeNano <= 0 {
|
||
return
|
||
}
|
||
w, ok := idx.graph.(graph.FileMtimeWriter)
|
||
if !ok {
|
||
return
|
||
}
|
||
var flush map[string]int64
|
||
streamMtimeMu.Lock()
|
||
streamMtimes[idx.relKey(absPath)] = mtimeNano
|
||
if len(streamMtimes) >= mtimeStreamPersistEvery {
|
||
flush = streamMtimes
|
||
streamMtimes = make(map[string]int64, mtimeStreamPersistEvery)
|
||
}
|
||
streamMtimeMu.Unlock()
|
||
if flush != nil {
|
||
if err := w.BulkSetFileMtimes(idx.repoPrefix, flush); err != nil {
|
||
idx.logger.Warn("indexer: incremental mtime batch persist failed",
|
||
zap.String("repo", idx.repoPrefix), zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
|
||
// parseChunk runs the per-file worker pool over the supplied
|
||
// slice. Closure over outer state (errors, counters, contract
|
||
// registry, parsePool, quarantine) so it can be called multiple
|
||
// times — once for the non-streaming path, repeatedly for the
|
||
// streaming-flush large-repo path where each call processes a
|
||
// bounded slice into a per-chunk in-memory shadow.
|
||
parseChunk := func(chunkFiles []walkedFile) {
|
||
fileCh := make(chan walkedFile, workers*4)
|
||
var wg sync.WaitGroup
|
||
for range workers {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
var localContracts []contracts.Contract
|
||
for wf := range fileCh {
|
||
path := wf.path
|
||
p := atomic.AddInt64(&processed, 1)
|
||
if p == 1 || p%parseReportEvery == 0 {
|
||
reporter.Report("parsing", int(p), totalFiles)
|
||
}
|
||
|
||
// Admit this file into extraction under the
|
||
// bytes-in-flight budget before reading it, so large
|
||
// content files serialise instead of all workers
|
||
// materialising whole files at once.
|
||
var weight int64
|
||
if parseSem != nil {
|
||
weight = clampParseWeight(wf.size, parseBudget)
|
||
if aerr := parseSem.Acquire(ctx, weight); aerr != nil {
|
||
return
|
||
}
|
||
}
|
||
|
||
relPath, _ := filepath.Rel(absRoot, path)
|
||
// Streaming content extractors (PDF / office docs) read the
|
||
// file themselves — one page/slide/sheet at a time — instead
|
||
// of materialising the whole file. Only the in-process route
|
||
// streams; the crash-isolation subprocess route keeps bytes.
|
||
if walkExt, found := idx.registry.GetByLanguage(wf.lang); found && parsePool == nil {
|
||
if se, ok := walkExt.(parser.StreamingExtractor); ok {
|
||
result, serr := idx.extractStreaming(se, path, relPath)
|
||
if parseSem != nil {
|
||
parseSem.Release(weight)
|
||
}
|
||
if serr != nil {
|
||
errMu.Lock()
|
||
errors = append(errors, IndexError{FilePath: path, Error: serr.Error()})
|
||
if result == nil {
|
||
parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: wf.lang, cause: serr.Error()})
|
||
}
|
||
errMu.Unlock()
|
||
}
|
||
if result == nil {
|
||
continue
|
||
}
|
||
idx.applyRepoPrefix(result.Nodes, result.Edges)
|
||
if contentWipeFile != nil {
|
||
contentWipeFile(firstContentFilePath(result.Nodes))
|
||
}
|
||
idx.streamContentSections(result.Nodes)
|
||
idx.graph.AddBatch(result.Nodes, result.Edges)
|
||
idx.persistConstValues(result)
|
||
recordStreamedMtime(path, wf.mtimeNano)
|
||
continue
|
||
}
|
||
}
|
||
|
||
src, err := readFile(wf)
|
||
if err != nil {
|
||
errMu.Lock()
|
||
errors = append(errors, IndexError{FilePath: path, Error: err.Error()})
|
||
errMu.Unlock()
|
||
if parseSem != nil {
|
||
parseSem.Release(weight)
|
||
}
|
||
continue
|
||
}
|
||
|
||
// Reuse the walk-time language. The walk's
|
||
// effectiveLanguage call already consulted shebang
|
||
// bytes via readSniffPrefix (512-byte probe), so a
|
||
// re-detect against the full src would change the
|
||
// answer only on the vanishingly rare case where a
|
||
// language marker lives past byte 512 — and any such
|
||
// case is content-sniffing-by-luck rather than spec'd
|
||
// behaviour. The fallback below covers the truly
|
||
// pathological case where the walk-time language has
|
||
// no extractor registered (effectively dead code).
|
||
lang := wf.lang
|
||
ext, _ := idx.registry.GetByLanguage(lang)
|
||
if ext == nil {
|
||
if relang, ok := idx.effectiveLanguage(path, src); ok {
|
||
lang = relang
|
||
ext, _ = idx.registry.GetByLanguage(lang)
|
||
}
|
||
}
|
||
if ext == nil {
|
||
if parseSem != nil {
|
||
parseSem.Release(weight)
|
||
}
|
||
continue
|
||
}
|
||
|
||
// Pre-ingestion transforms: rewrite the bytes before
|
||
// extraction (BOM strip, minified-bundle expansion, a
|
||
// PDF→markdown command, …).
|
||
src = idx.transforms.run(relPath, src)
|
||
|
||
result, skipped, err := idx.extractFile(parsePool, quarantine, path, relPath, lang, ext, src)
|
||
if parseSem != nil {
|
||
parseSem.Release(weight)
|
||
}
|
||
if err != nil {
|
||
errMu.Lock()
|
||
errors = append(errors, IndexError{FilePath: path, Error: err.Error()})
|
||
errMu.Unlock()
|
||
}
|
||
if result == nil {
|
||
// A full-index parse failure that produced no nodes:
|
||
// record it for a skip-node post-pass so the file
|
||
// stays visible instead of vanishing. (The live-modify
|
||
// path never reaches here — it keeps a file's prior
|
||
// nodes through a transient parse failure.)
|
||
if err != nil {
|
||
errMu.Lock()
|
||
parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: lang, cause: err.Error()})
|
||
errMu.Unlock()
|
||
}
|
||
continue
|
||
}
|
||
if skipped && len(result.Nodes) > 0 {
|
||
if _, ok := result.Nodes[0].Meta["skipped_due_to_timeout"]; ok {
|
||
atomic.AddInt64(&skippedByTimeout, 1)
|
||
}
|
||
if _, ok := result.Nodes[0].Meta["skipped_due_to_minified"]; ok {
|
||
atomic.AddInt64(&skippedByMinified, 1)
|
||
}
|
||
}
|
||
|
||
// Append coverage artifacts (todos / licenses /
|
||
// ownership) before applyRepoPrefix so they get the
|
||
// same multi-repo namespacing treatment as
|
||
// language-extractor output. Skipped for quarantined /
|
||
// timed-out files — the coverage scanners would re-read
|
||
// a source the parser could not survive.
|
||
if !skipped {
|
||
idx.applyCoverageDomains(relPath, lang, src, result)
|
||
}
|
||
|
||
idx.applyRepoPrefix(result.Nodes, result.Edges)
|
||
|
||
// Find the file node (if the extractor produced one)
|
||
// and collect its outgoing edges — contract extractors
|
||
// take the file-scope edge set (imports, etc.), not
|
||
// every intra-file edge.
|
||
var fileNodeID, fileGraphPath string
|
||
for _, n := range result.Nodes {
|
||
if n.Kind == graph.KindFile {
|
||
fileNodeID = n.ID
|
||
fileGraphPath = n.FilePath
|
||
break
|
||
}
|
||
}
|
||
var fileScopeEdges []*graph.Edge
|
||
if fileNodeID != "" {
|
||
for _, e := range result.Edges {
|
||
if e.From == fileNodeID {
|
||
fileScopeEdges = append(fileScopeEdges, e)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Stream this file's content (data_class=content) section
|
||
// bodies into the dedicated content index and lean the
|
||
// nodes to a snippet BEFORE AddBatch, so the bulk text
|
||
// never enters the graph, the symbol search, or the
|
||
// materialising code passes.
|
||
if contentWipeFile != nil {
|
||
contentWipeFile(firstContentFilePath(result.Nodes))
|
||
}
|
||
idx.streamContentSections(result.Nodes)
|
||
|
||
// Batch the per-file insert into one shard-grouped pass
|
||
// so each shard's lock is acquired at most once per
|
||
// file instead of N + 2·E times. Profiling showed 69
|
||
// of 102 workers blocked on lockTwoWrite under the
|
||
// per-edge path during cold-start warmup.
|
||
idx.graph.AddBatch(result.Nodes, result.Edges)
|
||
idx.persistConstValues(result)
|
||
idx.persistFileMeta(relPath, src, result)
|
||
recordStreamedMtime(path, wf.mtimeNano)
|
||
|
||
if !skipped && fileGraphPath != "" {
|
||
exts := contractExtractorsByLang[lang]
|
||
if len(exts) > 0 {
|
||
c := idx.runContractExtractorsForFile(
|
||
fileGraphPath, src, result.Nodes, fileScopeEdges, exts, result.Tree)
|
||
localContracts = append(localContracts, c...)
|
||
|
||
// Populate the per-file contract cache so a
|
||
// later IncrementalReindex can skip this file
|
||
// on a cache hit. Mtime comes from the walk-
|
||
// time d.Info() — no extra stat here.
|
||
if wf.mtimeNano > 0 {
|
||
idx.contractCacheMu.Lock()
|
||
idx.contractCache[fileGraphPath] = &contractCacheEntry{
|
||
mtimeNano: wf.mtimeNano,
|
||
contracts: c,
|
||
}
|
||
idx.contractCacheMu.Unlock()
|
||
}
|
||
}
|
||
}
|
||
// Release the parse tree now that the per-file
|
||
// contract pass is done. Post-passes that need a
|
||
// tree for this file (cross-file handler resolution)
|
||
// re-parse on demand. Nil-safe.
|
||
result.Tree.Release()
|
||
atomic.AddInt64(&fileCount, 1)
|
||
}
|
||
if len(localContracts) > 0 {
|
||
contractMu.Lock()
|
||
for _, c := range localContracts {
|
||
contractReg.Add(c)
|
||
}
|
||
contractMu.Unlock()
|
||
}
|
||
}()
|
||
}
|
||
|
||
for _, f := range chunkFiles {
|
||
fileCh <- f
|
||
}
|
||
close(fileCh)
|
||
wg.Wait()
|
||
}
|
||
|
||
// Dispatch the largest files first. Both dispatch paths below
|
||
// consume this slice in order: the plain path feeds it straight to
|
||
// the worker pool, and the streaming-flush path carves it into
|
||
// chunks from the front. Feeding the biggest file to the workers
|
||
// first lets its long parse overlap with the tail of small files,
|
||
// instead of the whole index waiting on a large file that would
|
||
// otherwise be dispatched last. The sort is stable and a pure
|
||
// permutation, so the set of indexed files and the resulting graph
|
||
// are identical — only the dispatch order changes.
|
||
sortBySizeDesc(files)
|
||
|
||
// Streaming-flush path: above shadowMaxFileCount with a
|
||
// BulkLoader-capable backend, we can't fit the whole shadow in
|
||
// RAM but we can still amortise the per-file disk-write cost by
|
||
// chunking. Each chunk runs against its own throwaway shadow,
|
||
// then flushes via BulkLoad to disk. Resolve runs against the
|
||
// disk store afterwards (per-call, slower than the shadow path
|
||
// but bounded RAM). Activated by GORTEX_STREAMING_FLUSH=1; off
|
||
// by default since it requires the disk-only resolver path
|
||
// (~tens of minutes on huge repos) that we haven't yet
|
||
// optimised end-to-end.
|
||
if diskTarget == nil && streamingFlushActive(idx.graph, len(files)) {
|
||
bl, _ := idx.graph.(graph.BulkLoader)
|
||
streamingDisk := idx.graph
|
||
chunkSize := streamingChunkSize()
|
||
idx.logger.Info("indexer: streaming-flush parse",
|
||
zap.Int("files", len(files)),
|
||
zap.Int("chunk_size", chunkSize))
|
||
for chunkStart := 0; chunkStart < len(files); chunkStart += chunkSize {
|
||
chunkEnd := min(chunkStart+chunkSize, len(files))
|
||
chunkShadow := graph.New()
|
||
idx.graph = chunkShadow
|
||
parseChunk(files[chunkStart:chunkEnd])
|
||
// Flush chunk to disk.
|
||
bl.BeginBulkLoad()
|
||
streamingDisk.AddBatch(chunkShadow.AllNodes(), chunkShadow.AllEdges())
|
||
if err := bl.FlushBulk(); err != nil {
|
||
return nil, fmt.Errorf("indexer: streaming-flush chunk %d..%d: %w", chunkStart, chunkEnd, err)
|
||
}
|
||
// This chunk's nodes are durable on disk now, so persist its
|
||
// files' mtimes — a kill mid-track then resumes from this chunk
|
||
// boundary instead of re-tracking the whole large first index
|
||
// from scratch. The in-worker recordStreamedMtime is a no-op on
|
||
// this path (idx.graph was the throwaway per-chunk shadow, not a
|
||
// FileMtimeWriter); the persist happens here, after the drain,
|
||
// keeping the "fresh mtime implies durable nodes" invariant.
|
||
if w, ok := streamingDisk.(graph.FileMtimeWriter); ok {
|
||
batch := make(map[string]int64, chunkEnd-chunkStart)
|
||
for _, f := range files[chunkStart:chunkEnd] {
|
||
if f.mtimeNano > 0 {
|
||
batch[idx.relKey(f.path)] = f.mtimeNano
|
||
}
|
||
}
|
||
if len(batch) > 0 {
|
||
if err := w.BulkSetFileMtimes(idx.repoPrefix, batch); err != nil {
|
||
idx.logger.Warn("indexer: streaming-flush chunk mtime persist failed",
|
||
zap.String("repo", idx.repoPrefix), zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// After all chunks, idx.graph points at the disk store so
|
||
// the resolver and subpasses read/mutate the merged state.
|
||
idx.graph = streamingDisk
|
||
} else {
|
||
parseChunk(files)
|
||
}
|
||
|
||
// Finalise the content index after the per-file streaming appends so
|
||
// its FTS5 segments are merged before the first content query.
|
||
if cs := idx.contentSearcher(); cs != nil {
|
||
if err := cs.BuildContentIndex(); err != nil {
|
||
idx.logger.Warn("indexer: content index build failed", zap.Error(err))
|
||
}
|
||
}
|
||
|
||
if processed > 0 {
|
||
reporter.Report("parsing", int(processed), totalFiles)
|
||
}
|
||
|
||
// Emit synthetic file nodes for files dropped by the size cap so
|
||
// they stay visible in the graph with skip telemetry attached
|
||
// instead of vanishing silently.
|
||
idx.emitSizeSkipNodes(skippedBySize)
|
||
idx.emitContentSkipNodes(skippedByContent)
|
||
idx.emitParseFailedSkipNodes(parseFailedFiles)
|
||
|
||
// Populate fileMtimes for all detected files. Keyed through
|
||
// relKey so the mtime map agrees with the graph's file-node keys
|
||
// (and with the incremental / git-watcher paths) on the NFC form
|
||
// of every non-ASCII filename. Mtimes are the walk-time values
|
||
// captured via d.Info(); no per-file os.Stat round-trip here.
|
||
idx.mtimeMu.Lock()
|
||
idx.fileMtimes = make(map[string]int64, len(files))
|
||
for _, f := range files {
|
||
if f.mtimeNano > 0 {
|
||
idx.fileMtimes[idx.relKey(f.path)] = f.mtimeNano
|
||
}
|
||
}
|
||
mtimeSnapshot := make(map[string]int64, len(idx.fileMtimes))
|
||
for k, v := range idx.fileMtimes {
|
||
mtimeSnapshot[k] = v
|
||
}
|
||
idx.mtimeMu.Unlock()
|
||
|
||
// Persist the per-file mtimes through the store's optional
|
||
// FileMtime sidecar table. On the on-disk backend this lets warm
|
||
// restarts seed ReconcileRepoCtx without having to read them back
|
||
// out of the gob+gzip metadata snapshot; on the in-memory
|
||
// backend the capability isn't implemented and the assertion
|
||
// short-circuits.
|
||
//
|
||
// Multi-repo bug: when the shadow-swap path is active, idx.graph
|
||
// is the in-memory shadow graph at this point — graph.Graph does
|
||
// NOT implement FileMtimeWriter, so the type assertion fails and
|
||
// persistence is silently skipped. The actual disk store is
|
||
// the local diskTarget variable; checking it first ensures warm-
|
||
// restart-skip-reindex actually works. The defer that swaps
|
||
// idx.graph back to diskTarget runs LATER, when IndexCtx returns,
|
||
// so we can't rely on it here. Falls through to idx.graph for the
|
||
// non-shadow path.
|
||
mtimeTarget := graph.Store(idx.graph)
|
||
if diskTarget != nil {
|
||
mtimeTarget = diskTarget
|
||
}
|
||
// Full-index persist is AUTHORITATIVE: replace the repo's entire mtime
|
||
// set so files deleted since the last index are pruned. An upsert-only
|
||
// write (BulkSetFileMtimes) leaves deleted-file rows behind, and warm-
|
||
// restart reconcile then detects them as phantom deletions on every
|
||
// restart — forcing a full re-track that never converges. Prefer the
|
||
// replace capability; fall back to upsert for backends without it.
|
||
if len(mtimeSnapshot) > 0 {
|
||
var perr error
|
||
persisted := false
|
||
if r, ok := mtimeTarget.(graph.FileMtimeReplacer); ok {
|
||
perr, persisted = r.ReplaceFileMtimes(idx.repoPrefix, mtimeSnapshot), true
|
||
} else if w, ok := mtimeTarget.(graph.FileMtimeWriter); ok {
|
||
perr, persisted = w.BulkSetFileMtimes(idx.repoPrefix, mtimeSnapshot), true
|
||
}
|
||
if persisted {
|
||
if perr != nil {
|
||
idx.logger.Warn("persist file mtimes failed",
|
||
zap.String("repo", idx.repoPrefix), zap.Error(perr))
|
||
} else {
|
||
idx.logger.Info("persisted file mtimes",
|
||
zap.String("repo", idx.repoPrefix),
|
||
zap.Int("count", len(mtimeSnapshot)))
|
||
}
|
||
}
|
||
|
||
// Crash-safe content path: reap every content row this walk did NOT
|
||
// re-stream. keep is contentStreamedFiles — the files that actually
|
||
// produced content sections this run — NOT the surviving-file mtime
|
||
// set: a file can survive on disk yet stop yielding content (doc
|
||
// emptied, classification changed), and keying keep off mtimes would
|
||
// protect its stale rows forever. Recorded keys are the wipe's own
|
||
// argument (the node FilePath content_fts carries), so the comparison
|
||
// matches the stored rows in single- and multi-repo form alike. A walk
|
||
// that streamed NO content falls back to the repo-wide wipe: the repo
|
||
// has zero content files now, and the sweep's empty-keep guard (a
|
||
// never-wipe-from-empty safety net) would otherwise no-op and leave
|
||
// every stale row behind. Only when the per-file wipe path is active;
|
||
// on the repo-wide-wipe fallback the up-front pre-wipe already cleared
|
||
// both transitions. Runs only under the completed-walk guard above
|
||
// (len(mtimeSnapshot) > 0), so a killed parse never triggers it.
|
||
if contentWipeFile != nil {
|
||
contentStreamedMu.Lock()
|
||
keep := contentStreamedFiles
|
||
contentStreamedMu.Unlock()
|
||
if len(keep) == 0 {
|
||
if cs := idx.contentSearcher(); cs != nil {
|
||
if err := cs.WipeContent(idx.RepoPrefix()); err != nil {
|
||
idx.logger.Warn("indexer: content wipe of contentless repo failed", zap.Error(err))
|
||
}
|
||
}
|
||
} else if sw, ok := idx.contentSearcher().(interface {
|
||
DeleteContentFilesForRepoNotIn(repoPrefix string, keep map[string]struct{}) error
|
||
}); ok {
|
||
if err := sw.DeleteContentFilesForRepoNotIn(idx.repoPrefix, keep); err != nil {
|
||
idx.logger.Warn("indexer: content sweep of stale files failed", zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Retain parse errors and record index metadata.
|
||
idx.parseErrors = errors
|
||
idx.totalDetected = len(files)
|
||
idx.lastIndexTime = time.Now()
|
||
|
||
if idx.deferResolve {
|
||
// Multi-repo orchestrator runs these serially after wg.Wait()
|
||
// to avoid races on the shared graph between this goroutine's
|
||
// ResolveAll mutation phase and a sibling goroutine's contract
|
||
// pass walking AllEdges. See SetDeferResolve.
|
||
idx.pendingContractReg = contractReg
|
||
} else {
|
||
// Materialise dep::<module> contract nodes from go.mod BEFORE
|
||
// ResolveAll so the resolver's import bridge can re-target Go
|
||
// imports of declared modules to their dep contract node.
|
||
idx.extractGoModContracts(contractReg)
|
||
|
||
reporter.Report("resolving references", 0, 0)
|
||
// Resolve cross-file references.
|
||
idx.populateCppIncludeDirs(true)
|
||
idx.resolver.ResolveAll()
|
||
|
||
// Infer structural interface satisfaction + method-level
|
||
// overrides. Skipped under deferGlobalPasses so a batch caller
|
||
// (warmup, ReconcileAll) can run them once at the end against
|
||
// the final shared graph instead of paying the O(global) walk
|
||
// per repo. InferOverrides depends on InferImplements running
|
||
// first.
|
||
if !idx.deferGlobalPasses {
|
||
reporter.Report("inferring interfaces", 0, 0)
|
||
idx.resolver.InferImplements()
|
||
idx.resolver.InferOverrides()
|
||
}
|
||
|
||
// Semantic enrichment (SCIP, go/types, LSP).
|
||
if idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders() {
|
||
reporter.Report("semantic enrichment", 0, 0)
|
||
// Key by the repo prefix so a repo-scoped provider can scope
|
||
// file selection to this repo (empty in single-repo mode).
|
||
roots := map[string]string{idx.repoPrefix: absRoot}
|
||
// The inline full-index path does not gate or persist enrichment
|
||
// markers (a zero EnrichOptions): the deferred warmup path owns the
|
||
// skip-on-restart optimisation.
|
||
results, _, err := idx.semanticMgr.EnrichAll(idx.graph, roots, semantic.EnrichOptions{})
|
||
if err != nil {
|
||
idx.logger.Warn("semantic enrichment failed", zap.Error(err))
|
||
} else if len(results) > 0 {
|
||
for _, r := range results {
|
||
idx.logger.Info("semantic enrichment result",
|
||
zap.String("provider", r.Provider),
|
||
zap.String("language", r.Language),
|
||
zap.Int("confirmed", r.EdgesConfirmed),
|
||
zap.Int("added", r.EdgesAdded),
|
||
zap.Int("refuted", r.EdgesRefuted),
|
||
zap.Float64("coverage", r.CoveragePercent),
|
||
)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
reporter.Report("building search index", 0, 0)
|
||
// Build search index.
|
||
idx.buildSearchIndex()
|
||
|
||
if !idx.deferResolve {
|
||
// Contracts were already extracted inline during parse (per file,
|
||
// per worker). Here we just finish up. extractGoModContracts
|
||
// already ran (see the !deferResolve branch above) so dep
|
||
// nodes were available during ResolveAll's import-bridge pass;
|
||
// commitContracts is idempotent for those.
|
||
reporter.Report("extracting contracts", 0, 0)
|
||
idx.extractExternalModules()
|
||
idx.extractDIContracts(contractReg)
|
||
idx.commitContracts(contractReg)
|
||
|
||
// Test-edge pass — runs once the call graph is final. Skipped
|
||
// under deferGlobalPasses so a batch caller can fold this into
|
||
// one global pass after the per-repo loop.
|
||
if !idx.deferGlobalPasses {
|
||
reporter.Report("test edge pass", 0, 0)
|
||
marked, emitted := markTestSymbolsAndEmitEdges(idx.graph)
|
||
if marked > 0 || emitted > 0 {
|
||
idx.logger.Info("test edges emitted",
|
||
zap.Int("test_symbols", marked),
|
||
zap.Int("edges", emitted),
|
||
)
|
||
}
|
||
if re, ep, fa := synthesizeCapabilityEdges(idx.graph); re > 0 || ep > 0 || fa > 0 {
|
||
idx.logger.Info("capability edges emitted",
|
||
zap.Int("reads_env", re),
|
||
zap.Int("executes_process", ep),
|
||
zap.Int("accesses_field", fa),
|
||
)
|
||
}
|
||
reporter.Report("clone detection pass", 0, 0)
|
||
if cs := detectClonesAndEmitEdgesCtx(ctx, idx.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
|
||
idx.logger.Info("clone edges emitted",
|
||
zap.Int("items", cs.Items),
|
||
zap.Int("clone_pairs", cs.Pairs),
|
||
zap.Int("edges", cs.Edges),
|
||
zap.Int("skipped_buckets", cs.SkippedBuckets),
|
||
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
|
||
zap.Int("diffused_pairs", cs.DiffusedPairs),
|
||
zap.Int("diffused_edges", cs.DiffusedEdges),
|
||
)
|
||
}
|
||
// Seed the incremental clone index from the freshly-baselined
|
||
// signatures + sidecar so steady-state single-file edits go
|
||
// incremental (EvictFuncs/UpdateFuncs) instead of re-running
|
||
// this whole-graph pass per file. The batch pass remains the
|
||
// re-baseline (corrects CMS drift) and owns diffusion.
|
||
if idx.cloneIndex != nil {
|
||
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
|
||
}
|
||
// Framework dynamic-dispatch synthesis — runs once the call
|
||
// graph and interface inference are final. Skipped under
|
||
// deferGlobalPasses; the batch caller folds it into
|
||
// RunGlobalGraphPasses.
|
||
reporter.Report("framework dispatch synthesis", 0, 0)
|
||
if rep := resolver.RunFrameworkSynthesizers(idx.graph); rep.Total > 0 {
|
||
idx.logger.Info("framework dispatch calls synthesized",
|
||
zap.Int("edges", rep.Total),
|
||
zap.Any("per_synthesizer", rep.Per),
|
||
)
|
||
}
|
||
// External-call placeholder synthesis (opt-in) — runs after
|
||
// the resolver and stub passes so only genuinely un-indexed
|
||
// external targets are left to materialise.
|
||
reporter.Report("external-call synthesis", 0, 0)
|
||
if extCalls := resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled()); extCalls > 0 {
|
||
idx.logger.Info("external-call placeholders synthesized",
|
||
zap.Int("edges", extCalls),
|
||
)
|
||
}
|
||
if spec := resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled()); spec > 0 {
|
||
idx.logger.Info("speculative dispatch edges synthesized",
|
||
zap.Int("edges", spec),
|
||
)
|
||
}
|
||
// Reachability index — used to be precomputed for every
|
||
// impact seed here. The eager pass was retired because the
|
||
// breakeven was untenable on monorepo graphs (k8s:
|
||
// ~2000 s build to save ~10 ms per query, ~200 k-query
|
||
// breakeven). reach.Lookup now computes the BFS on first
|
||
// access per seed and caches the result. The
|
||
// InvalidateIndex call bumps the build counter so any
|
||
// stale stamps from a prior build (e.g. snapshot reload
|
||
// before a partial mutation) no longer shadow the live
|
||
// graph state.
|
||
reach.InvalidateIndex()
|
||
}
|
||
}
|
||
|
||
// Auto-upgrade to Bleve if above threshold. Run in the background
|
||
// so the foreground IndexCtx returns immediately — populating
|
||
// Bleve with 50k+ symbols takes 30-60s and adding that to the
|
||
// initial-index latency was the dominant tail. Searches against
|
||
// idx.search keep hitting the in-memory backend until the swap
|
||
// completes; nothing observes a half-built Bleve.
|
||
//
|
||
// upgradeOnce gates the spawn so multi-repo warmup, which calls
|
||
// IndexCtx once per tracked repo, doesn't launch one upgrade
|
||
// goroutine per post-threshold repo. One per indexer lifetime.
|
||
//
|
||
// Skip the upgrade when the active search backend is the
|
||
// SymbolSearcher adapter: the disk store's native FTS is
|
||
// already serving search at engine-native latency, and
|
||
// spawning a parallel Bleve build would (a) waste ~100MB heap
|
||
// re-indexing the same corpus and (b) silently swap the
|
||
// adapter out for Bleve on completion — defeating the whole
|
||
// FTS path. The Swappable's current backend tells us which
|
||
// branch we're on.
|
||
if !isSymbolSearcherBackend(idx.search) && idx.search.Count() >= search.AutoThreshold {
|
||
idx.upgradeOnce.Do(func() {
|
||
reporter.Report("scheduling search backend upgrade", 0, 0)
|
||
idx.upgradeSpawnedMu.Lock()
|
||
idx.upgradeSpawned++
|
||
idx.upgradeSpawnedMu.Unlock()
|
||
// Snapshot upfront so the background goroutine doesn't
|
||
// read Node.Meta concurrently with subsequent Index
|
||
// calls' Meta-writing passes (reach.BuildIndex,
|
||
// ResolveTemporalCalls, ...).
|
||
snapshot := idx.snapshotBleveEntries()
|
||
go idx.upgradeSearchToBleve(snapshot)
|
||
})
|
||
}
|
||
|
||
// Persist the parser quarantine so a file that crashed the parser
|
||
// stays skipped across daemon restarts until its content changes.
|
||
if quarantine != nil {
|
||
if err := quarantine.Save(); err != nil {
|
||
idx.logger.Warn("indexer: failed to persist parser quarantine", zap.Error(err))
|
||
} else if n := quarantine.Len(); n > 0 {
|
||
idx.logger.Info("indexer: parser quarantine", zap.Int("files", n))
|
||
}
|
||
}
|
||
|
||
reporter.Report("indexing complete", int(fileCount), len(files))
|
||
|
||
// Persist the Merkle baseline so the next incremental pass diffs
|
||
// against content hashes rather than re-indexing the whole repo.
|
||
workspaceFP := ""
|
||
if idx.merkleEnabled() {
|
||
paths := make([]string, len(files))
|
||
for i, wf := range files {
|
||
paths[i] = wf.path
|
||
}
|
||
workspaceFP = idx.saveMerkleBaseline(absRoot, paths)
|
||
}
|
||
idx.indexGen.Add(1) // invalidate the trigram search cache
|
||
|
||
nodes, edges := idx.repoNodeEdgeCount()
|
||
idx.persistRepoIndexState(diskTarget, absRoot, workspaceFP, nodes, edges)
|
||
result = &IndexResult{
|
||
NodeCount: nodes,
|
||
EdgeCount: edges,
|
||
FileCount: int(fileCount),
|
||
QuarantinedFiles: quarantine.Len(),
|
||
SkippedFiles: len(skippedBySize) + len(skippedByContent) + int(skippedByTimeout) + int(skippedByMinified),
|
||
DurationMs: time.Since(start).Milliseconds(),
|
||
Errors: errors,
|
||
}
|
||
idx.warnIfEdgeSanityViolated(result)
|
||
// A whole-repo (re-)track that observed files did work: mark the repo for
|
||
// deferred semantic enrichment. FullRetrack is stamped by the multi-repo
|
||
// caller after this returns, so FileCount carries the signal here.
|
||
if result.FileCount > 0 || result.FullRetrack {
|
||
idx.pendingEnrich.Store(true)
|
||
// IndexCtx re-parsed every file, dropping this repo's hover-enrichment
|
||
// edges — force the deferred pass past the completion-marker gate so
|
||
// they are restored even at an unchanged clean HEAD.
|
||
idx.fullReindexed.Store(true)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// repoNodeEdgeCount returns this indexer's contribution to the graph,
|
||
// scoped to its repoPrefix in multi-repo mode. In single-repo mode
|
||
// (empty prefix) every node carries an empty RepoPrefix anyway, so the
|
||
// graph totals equal the repo's contribution and we use the cheap
|
||
// global accessors. The multi-repo path uses RepoMemoryEstimate which
|
||
// walks only this repo's byRepo bucket — O(repo size), not O(graph) —
|
||
// so callers that stamp RepoMetadata.NodeCount/EdgeCount no longer
|
||
// freeze the workspace-wide total at TrackRepo time.
|
||
func (idx *Indexer) repoNodeEdgeCount() (int, int) {
|
||
if idx.repoPrefix == "" {
|
||
return idx.graph.NodeCount(), idx.graph.EdgeCount()
|
||
}
|
||
est := idx.graph.RepoMemoryEstimate(idx.repoPrefix)
|
||
return est.NodeCount, est.EdgeCount
|
||
}
|
||
|
||
// warnIfEdgeSanityViolated logs a loud warning when an index pass
|
||
// produced files and symbol nodes but no edges — see
|
||
// IndexResult.EdgeSanityViolated.
|
||
func (idx *Indexer) warnIfEdgeSanityViolated(r *IndexResult) {
|
||
if r.EdgeSanityViolated() {
|
||
idx.logger.Warn("indexer: edge-sanity check failed — index has files and nodes but zero edges; edge extraction likely failed wholesale",
|
||
zap.Int("files", r.FileCount),
|
||
zap.Int("nodes", r.NodeCount),
|
||
zap.Int("edges", r.EdgeCount))
|
||
}
|
||
}
|
||
|
||
// IndexFile parses a single file and patches the graph (evict then
|
||
// add), including per-file resolver work for cross-file references.
|
||
// Use in the single-event fsnotify path where each edit is isolated.
|
||
func (idx *Indexer) IndexFile(filePath string) error {
|
||
return idx.indexFile(filePath, true)
|
||
}
|
||
|
||
// IndexFileNoResolve is IndexFile minus the per-file resolver call.
|
||
// Callers in batch paths (storm mode, branch-switch reconcile, git
|
||
// diff dispatch) use this when they will run resolver.ResolveAll()
|
||
// once at the end of the batch; otherwise a 500-file checkout pays
|
||
// the per-file resolver cost 500 times instead of once.
|
||
func (idx *Indexer) IndexFileNoResolve(filePath string) error {
|
||
return idx.indexFile(filePath, false)
|
||
}
|
||
|
||
func (idx *Indexer) indexFile(filePath string, resolve bool) error {
|
||
absPath, err := filepath.Abs(filePath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
// relKey gives the canonical key (slash form, Unicode NFC). Using
|
||
// it here keeps an incremental re-index — which may be driven by a
|
||
// git-watcher path in NFC or an FSEvents path in NFD — landing on
|
||
// the exact graph key the bulk walk created, so the evict below
|
||
// finds the file's existing nodes instead of leaking a duplicate.
|
||
relPath := idx.relKey(absPath)
|
||
|
||
// In multi-repo mode, the graph stores prefixed file paths.
|
||
graphPath := idx.prefixPath(relPath)
|
||
|
||
// Parse-then-swap: we must NOT evict the file's existing nodes/edges
|
||
// and search entries until we hold a usable parse result. Evicting
|
||
// first leaves the file at zero nodes whenever the on-disk bytes are
|
||
// transiently unparseable (a save mid-edit) — a failed extraction
|
||
// then returns early and the symbols stay nuked. Capturing the old
|
||
// state up front and deferring the actual eviction to evictExisting()
|
||
// keeps the file stale-but-present on failure (stale beats empty) and
|
||
// shrinks the no-nodes window to the gap between evict and AddBatch.
|
||
//
|
||
// oldFuncIDs holds this file's function/method node IDs so the
|
||
// incremental clone index can drop their CMS/LSH contributions —
|
||
// EvictFile removes the nodes (and their clone_sig) from the graph,
|
||
// so it must be captured before evictExisting runs.
|
||
var oldFuncIDs []string
|
||
evictExisting := func() {
|
||
for _, n := range idx.graph.GetFileNodes(graphPath) {
|
||
if n.Kind != graph.KindFile && n.Kind != graph.KindImport {
|
||
idx.search.Remove(n.ID)
|
||
}
|
||
if n.Kind == graph.KindFunction || n.Kind == graph.KindMethod {
|
||
oldFuncIDs = append(oldFuncIDs, n.ID)
|
||
}
|
||
}
|
||
idx.restubIncomingRefs(graphPath)
|
||
idx.graph.EvictFile(graphPath)
|
||
}
|
||
|
||
src, err := os.ReadFile(absPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
lang, ok := idx.effectiveLanguage(absPath, src)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
ext, _ := idx.registry.GetByLanguage(lang)
|
||
if ext == nil {
|
||
return nil
|
||
}
|
||
|
||
// Honour the size cap on the incremental path too: an over-cap
|
||
// file gets a synthetic skip node, not a parse — matching the
|
||
// bulk IndexCtx walk. This IS a successful result, so it evicts the
|
||
// prior state and installs the synthetic node, same as before.
|
||
if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize {
|
||
n := sizeSkipNode(skippedFile{
|
||
relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)),
|
||
}, maxSize)
|
||
idx.applyRepoPrefix([]*graph.Node{n}, nil)
|
||
evictExisting()
|
||
idx.graph.AddBatch([]*graph.Node{n}, nil)
|
||
return nil
|
||
}
|
||
|
||
// Honour the content-admission gate on the incremental path too, so a
|
||
// document over its cap (or a data asset, by default) the cold walk
|
||
// would skip doesn't get parsed back in when the watcher re-indexes it
|
||
// — same synthetic-skip-node treatment as the size cap above.
|
||
if reason, skip := idx.newContentAdmissionGate().skip(lang, int64(len(src))); skip {
|
||
n := contentSkipNode(skippedFile{
|
||
relPath: filepath.ToSlash(relPath), lang: lang, size: int64(len(src)), reason: reason,
|
||
})
|
||
idx.applyRepoPrefix([]*graph.Node{n}, nil)
|
||
evictExisting()
|
||
idx.graph.AddBatch([]*graph.Node{n}, nil)
|
||
return nil
|
||
}
|
||
|
||
// Pre-ingestion transforms — same pipeline as the bulk path.
|
||
src = idx.transforms.run(relPath, src)
|
||
|
||
// Crash isolation for the incremental path: a file the user just
|
||
// saved that SIGSEGVs the parser is quarantined instead of taking
|
||
// the daemon down with it. The pool is long-lived and shared, so
|
||
// the watcher hot path never forks a worker subprocess per file.
|
||
var pool *crashpool.Pool
|
||
var quarantine *crashpool.Quarantine
|
||
if idx.crashIsolationEnabled() {
|
||
pool, quarantine = idx.sharedParsePool()
|
||
}
|
||
result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src)
|
||
if quarantine != nil && quarantine.Len() > 0 {
|
||
_ = quarantine.Save()
|
||
}
|
||
if result == nil {
|
||
// No usable parse result (transient parse failure, quarantine,
|
||
// timeout). Do NOT evict — the file's prior nodes/edges/search
|
||
// entries stay intact. A stale-but-present file beats an empty
|
||
// one, and the next successful re-index swaps cleanly.
|
||
//
|
||
// The bytes were read successfully (src above), so this is a
|
||
// stable fact about the file's current on-disk content, not a
|
||
// transient "couldn't even open it" failure (that case returns
|
||
// earlier, before relPath/mtime bookkeeping, and deliberately
|
||
// leaves the mtime unrecorded so it keeps retrying). Recording
|
||
// the mtime here keeps a warm restart's HasChangesSinceMtimes
|
||
// from perpetually seeing this one unparseable file as "the
|
||
// repo changed" and routing the whole repo through the
|
||
// expensive shadow re-track path on every restart.
|
||
idx.recordFileMtime(relPath, absPath)
|
||
return err
|
||
}
|
||
|
||
// Affected-by snapshot: the symbol shapes and reverse-reference
|
||
// sources the post-resolve signature-delta pass compares against,
|
||
// captured BEFORE eviction — EvictFile drops in-edges from
|
||
// unchanged files and replaces this file's nodes, so neither is
|
||
// recoverable afterwards. Skipped on the no-resolve and
|
||
// deferred-batch paths, whose callers run a full resolve (and
|
||
// persistAllRefFacts) once at the end of the batch.
|
||
//
|
||
// Also skipped for a quarantined / timed-out / minified-skipped file:
|
||
// its synthetic result carries zero symbols, so the delta would read
|
||
// every prior symbol as removed and fan out to re-resolve the whole
|
||
// reverse graph on a transient parse failure. A failure that yields no
|
||
// symbols is not the same as a symbol genuinely deleted from source.
|
||
var abSnap *affectedBySnapshot
|
||
var reuseIdx map[reuseKey]*reuseVal
|
||
var priorUnresolved []*graph.Edge
|
||
if resolve && !idx.deferGlobalPasses && !skipped {
|
||
abSnap = idx.snapshotAffectedBy(graphPath)
|
||
// Snapshot the file's outgoing edges before eviction: resolved ones so
|
||
// the re-parse recovers unchanged resolutions (reuseIdx), and the ones
|
||
// still unresolved so the forward pass can skip re-trying them
|
||
// (priorUnresolved). Together this makes a save re-resolve only the
|
||
// references it actually changed instead of the whole file.
|
||
reuseIdx, priorUnresolved = captureIncrementalState(idx.graph, graphPath)
|
||
}
|
||
|
||
// We hold a usable result: evict the old state now, then add the
|
||
// new — the window where the file has no nodes is just this gap.
|
||
evictExisting()
|
||
|
||
// Coverage extractors (todos, licenses, ownership). Same call
|
||
// site exists in the bulk IndexCtx worker pool — see
|
||
// applyCoverageDomains. Skipped for a quarantined / timed-out file.
|
||
if !skipped {
|
||
idx.applyCoverageDomains(relPath, lang, src, result)
|
||
}
|
||
|
||
idx.applyRepoPrefix(result.Nodes, result.Edges)
|
||
|
||
// Reuse prior resolutions for edges whose source-side shape is unchanged
|
||
// (the common case on a small edit), so the resolver below only handles
|
||
// genuinely-new references instead of re-resolving the whole file. The
|
||
// about-to-be-added node IDs let the reuse recover same-file targets that
|
||
// eviction removed and this AddBatch re-adds under identical IDs — without
|
||
// them a same-file call's resolution + tier is lost to a full re-resolve on
|
||
// every structural save.
|
||
newNodeIDs := make(map[string]struct{}, len(result.Nodes))
|
||
for _, n := range result.Nodes {
|
||
if n != nil {
|
||
newNodeIDs[n.ID] = struct{}{}
|
||
}
|
||
}
|
||
if reused := applyResolvedOutEdges(idx.graph, result.Edges, reuseIdx, newNodeIDs); reused > 0 {
|
||
idx.logger.Debug("indexer: reused prior resolutions",
|
||
zap.Int("edges", reused), zap.String("file", graphPath))
|
||
}
|
||
|
||
// Content (incremental): clear this file's prior content rows, then
|
||
// re-stream + lean — mirrors the full-index per-file path so an edited
|
||
// content file leaves no stale rows and doesn't revert to full text on
|
||
// the node.
|
||
if cs := idx.contentSearcher(); cs != nil {
|
||
if fp := firstContentFilePath(result.Nodes); fp != "" {
|
||
if err := cs.WipeContentFile(fp); err != nil {
|
||
idx.logger.Warn("indexer: content index file wipe failed", zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
idx.streamContentSections(result.Nodes)
|
||
|
||
idx.graph.AddBatch(result.Nodes, result.Edges)
|
||
idx.persistConstValues(result)
|
||
idx.persistFileMeta(relPath, src, result)
|
||
|
||
// Add new symbols to search index. shouldIndexForSearch enforces
|
||
// the same SkipSearch filter used by the bulk and upgrade paths.
|
||
// When the backing store implements graph.SymbolSearcher we
|
||
// also mirror each upsert into its native FTS, so an
|
||
// incremental reindex doesn't fall out of sync with the
|
||
// bulk-built corpus.
|
||
searcher, _ := idx.graph.(graph.SymbolSearcher)
|
||
for _, n := range result.Nodes {
|
||
if !idx.shouldIndexForSearch(n) {
|
||
continue
|
||
}
|
||
idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...)
|
||
if searcher != nil {
|
||
if err := searcher.UpsertSymbolFTS(n.ID, ftsTokensFor(n, idx.projectName)); err != nil {
|
||
idx.logger.Debug("indexer: backend FTS upsert failed",
|
||
zap.String("id", n.ID),
|
||
zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
|
||
if resolve {
|
||
// Forward pass (this file's outgoing references) plus the
|
||
// reverse pass binding callers in OTHER files that reference a
|
||
// symbol (re)defined here — a symbol newly defined or changed
|
||
// here leaves callers elsewhere pointing at the unresolved stub
|
||
// restubIncomingRefs left when the prior concrete node was
|
||
// evicted. Scoped to this file's names — not a whole-graph
|
||
// ResolveAll — and run as one combined pass so the resolver's
|
||
// per-pass indexes are built once per save, not twice.
|
||
//
|
||
// Skip re-resolving references that were already unresolved before the
|
||
// edit and are unchanged: they stay parked on their stubs for the
|
||
// incoming pass, so a small edit to a reference-heavy file no longer
|
||
// re-runs the candidate cascade on thousands of stdlib/external calls.
|
||
idx.resolver.SetIncrementalSkip(priorUnresolved)
|
||
idx.resolver.ResolveFileAndIncoming(graphPath)
|
||
idx.resolver.SetIncrementalSkip(nil)
|
||
// CPG-lite dataflow placeholders for this file: inter-
|
||
// procedural callees may have just been lifted by
|
||
// ResolveFile, so re-run the dataflow materialisation pass
|
||
// to keep arg_of / returns_to edges in sync with the
|
||
// freshly resolved EdgeCalls graph. Scoped to this file's
|
||
// out-edges — not a whole-graph AllEdges scan — so an
|
||
// incremental edit stays O(file), not O(all edges).
|
||
idx.materializeDataflowParamsForFile(graphPath, result.Edges)
|
||
// Clone detection. EvictFile above removed this file's
|
||
// EdgeSimilarTo edges in both directions. When the incremental
|
||
// clone index is built, re-bank just this file's bodies
|
||
// (EvictFuncs the old ids, UpdateFuncs the fresh nodes) — an
|
||
// O(edited file) update that restores the same edge set the
|
||
// whole-graph pass would. Until a batch/global pass has seeded
|
||
// the index (built=false) we fall back to the full recompute.
|
||
// Skipped under deferGlobalPasses — a batch caller (ReconcileAll,
|
||
// warmup) runs the global pass once at the end.
|
||
if !idx.deferGlobalPasses {
|
||
if idx.cloneIndex != nil {
|
||
// Seed the incremental clone index on first use — e.g. after a
|
||
// warm restart that loaded the graph from disk without a full
|
||
// re-index, `built` is false. Without this we'd fall to the
|
||
// whole-graph recompute on EVERY save; instead pay it once and
|
||
// then run the O(edited file) update.
|
||
if !idx.cloneIndex.built {
|
||
idx.cloneIndex.Rebuild(idx.graph, idx.repoPrefix)
|
||
}
|
||
idx.cloneIndex.EvictFuncs(idx.graph, oldFuncIDs)
|
||
idx.cloneIndex.UpdateFuncs(idx.graph, idx.repoPrefix, cloneFuncNodes(result.Nodes), idx.cloneThreshold())
|
||
} else {
|
||
detectClonesAndEmitEdges(idx.graph, idx.repoPrefix, idx.cloneThreshold())
|
||
}
|
||
}
|
||
// in-memory backend. Skipped for a quarantined / timed-out /
|
||
// minified file: its synthetic result yields no facts, so a
|
||
// delete-then-set would durably drop the file's real facts on a
|
||
// transient parse failure and leave them gone until a clean
|
||
// reparse — abSnap is nil here too, so the affected-by pass that
|
||
// would also fan out is already a no-op.
|
||
if !skipped {
|
||
idx.persistRefFactsForFiles([]string{graphPath})
|
||
// Affected-by re-resolution: if this save changed a symbol's
|
||
// signature or kind, or removed a symbol, re-resolve the files
|
||
// that referenced it — bounded, synchronous, and gated on the
|
||
// signature delta so a body-only edit fans out to nothing.
|
||
idx.reresolveAffectedBy(graphPath, abSnap, result.Nodes)
|
||
|
||
// Incremental semantic enrichment for this single file. Mirrors the
|
||
// full-index EnrichAll call but scoped to the saved file, so a
|
||
// watcher save re-runs the type resolvers (and any watch-enabled
|
||
// LSP / compiler provider) instead of leaving the file's edges at
|
||
// their pre-enrichment tier until the next full reindex. Gated
|
||
// internally on Config.EnrichOnWatch; a no-op when disabled.
|
||
providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders()
|
||
reEnriched := false
|
||
if providersPresent {
|
||
if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil {
|
||
idx.logger.Debug("indexer: incremental semantic enrichment failed",
|
||
zap.String("file", graphPath),
|
||
zap.Error(err))
|
||
} else {
|
||
reEnriched = idx.semanticMgr.EnrichesOnWatch()
|
||
}
|
||
}
|
||
// Record whether this live re-parse left the file below the
|
||
// enrichment tier: providers exist (so there IS an lsp/ast tier to
|
||
// fall short of) but the save did not re-run enrichment. When set,
|
||
// find_usages / get_callers flag their default text_matched
|
||
// suppression as re-verification-pending so a hidden-but-real usage
|
||
// is diagnosable rather than silently dropped. Cleared when
|
||
// enrichment did re-run for the file.
|
||
idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched)
|
||
}
|
||
}
|
||
|
||
// Update mtime for this file. relPath is already the canonical
|
||
// key (relKey applied slash + NFC), so the mtime entry lines up
|
||
// with the graph file-node key and with the bulk-walk mtimes.
|
||
idx.recordFileMtime(relPath, absPath)
|
||
|
||
return nil
|
||
}
|
||
|
||
// recordFileMtime restamps the recorded mtime for relPath (a canonical
|
||
// relKey, not repo-prefixed) from absPath's current on-disk mtime, both
|
||
// in the in-memory map and — when the backend supports it — the store's
|
||
// FileMtime sidecar. Per-file write is ~1ms on the on-disk backend;
|
||
// trivial under steady-state file-watcher load. A missing/unstatable
|
||
// file is a no-op; a persist error is logged but non-fatal, since the
|
||
// in-memory map (which the current process trusts) is already correct.
|
||
func (idx *Indexer) recordFileMtime(relPath, absPath string) {
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
return
|
||
}
|
||
mtime := info.ModTime().UnixNano()
|
||
idx.mtimeMu.Lock()
|
||
idx.fileMtimes[relPath] = mtime
|
||
idx.mtimeMu.Unlock()
|
||
if w, ok := idx.graph.(graph.FileMtimeWriter); ok {
|
||
if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{relPath: mtime}); err != nil {
|
||
idx.logger.Warn("persist file mtime failed",
|
||
zap.String("repo", idx.repoPrefix), zap.String("file", relPath), zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
|
||
// StructuralSymbols parses a file from its current on-disk content and
|
||
// returns the structural symbols it defines — functions, methods,
|
||
// types, interfaces, constants, variables, fields, enum members — and
|
||
// nothing else. It is a read-only probe: the graph and the search
|
||
// index are left completely untouched, no mtime is stamped, and no
|
||
// resolver runs. The watcher uses it to decide whether a save is
|
||
// structurally inert (a comment / whitespace / config-value edit that
|
||
// changes no symbol) and can skip the destructive evict + reindex.
|
||
//
|
||
// The second return reports whether the file was parseable at all: a
|
||
// file with no detectable language, an over-cap file, or a read error
|
||
// yields (nil, false). A genuinely empty source file yields
|
||
// (empty-slice, true).
|
||
func (idx *Indexer) StructuralSymbols(filePath string) ([]*graph.Node, bool) {
|
||
absPath, err := filepath.Abs(filePath)
|
||
if err != nil {
|
||
return nil, false
|
||
}
|
||
relPath, err := filepath.Rel(idx.rootPath, absPath)
|
||
if err != nil {
|
||
relPath = filePath
|
||
}
|
||
|
||
src, err := os.ReadFile(absPath)
|
||
if err != nil {
|
||
return nil, false
|
||
}
|
||
|
||
lang, ok := idx.effectiveLanguage(absPath, src)
|
||
if !ok {
|
||
return nil, false
|
||
}
|
||
ext, _ := idx.registry.GetByLanguage(lang)
|
||
if ext == nil {
|
||
return nil, false
|
||
}
|
||
|
||
// An over-cap file is never structurally parsed on the indexing
|
||
// path either (it gets a synthetic skip node), so the watcher
|
||
// cannot prove inertness for it — fall through to a real reindex.
|
||
if maxSize := idx.config.MaxFileSize; maxSize > 0 && int64(len(src)) > maxSize {
|
||
return nil, false
|
||
}
|
||
|
||
// Same pre-ingestion transforms as indexFile so the probe parses
|
||
// exactly the bytes the real index pass would.
|
||
src = idx.transforms.run(relPath, src)
|
||
|
||
var pool *crashpool.Pool
|
||
var quarantine *crashpool.Quarantine
|
||
if idx.crashIsolationEnabled() {
|
||
pool, quarantine = idx.sharedParsePool()
|
||
}
|
||
result, skipped, err := idx.extractFile(pool, quarantine, absPath, relPath, lang, ext, src)
|
||
if quarantine != nil && quarantine.Len() > 0 {
|
||
_ = quarantine.Save()
|
||
}
|
||
// A skipped (quarantined / timed-out) file produces only a
|
||
// synthetic node — not the real symbol set — so inertness cannot
|
||
// be proven and the caller must reindex normally.
|
||
if result == nil || skipped || err != nil {
|
||
return nil, false
|
||
}
|
||
|
||
out := make([]*graph.Node, 0, len(result.Nodes))
|
||
for _, n := range result.Nodes {
|
||
if isStructuralKind(n.Kind) {
|
||
out = append(out, n)
|
||
}
|
||
}
|
||
return out, true
|
||
}
|
||
|
||
// isStructuralKind reports whether a node kind represents a structural
|
||
// code symbol — the kinds whose presence, name, or signature define a
|
||
// file's graph shape. File and import nodes (graph bookkeeping),
|
||
// params, closures, and the coverage-domain kinds (todos, licenses,
|
||
// strings, …) are deliberately excluded: a change confined to those
|
||
// does not alter the structural graph the watcher cares about.
|
||
func isStructuralKind(k graph.NodeKind) bool {
|
||
switch k {
|
||
case graph.KindFunction, graph.KindMethod, graph.KindType,
|
||
graph.KindInterface, graph.KindVariable, graph.KindConstant,
|
||
graph.KindField, graph.KindEnumMember:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ResolveAll re-runs the global cross-file reference resolver and
|
||
// interface-implementation inference. Exposed for batch paths that
|
||
// defer per-file resolver work until the end of a batch.
|
||
func (idx *Indexer) ResolveAll() {
|
||
idx.populateCppIncludeDirs(true)
|
||
idx.resolver.ResolveAll()
|
||
idx.resolver.InferImplements()
|
||
idx.resolver.InferOverrides()
|
||
// Framework dynamic-dispatch synthesis (gRPC / Temporal / event
|
||
// channels / native bridges) depends on InferImplements (the
|
||
// interface-satisfaction signals) having run first.
|
||
resolver.RunFrameworkSynthesizers(idx.graph)
|
||
// External-call placeholder synthesis (opt-in) — runs after the
|
||
// resolver and stub passes so only genuinely un-indexed external
|
||
// targets remain to materialise.
|
||
resolver.SynthesizeExternalCalls(idx.graph, idx.externalCallSynthesisEnabled())
|
||
resolver.ResolveSpeculativeDispatch(idx.graph, idx.speculativeDispatchEnabled())
|
||
// CPG-lite dataflow rewriting must run after the call resolver
|
||
// has lifted unresolved:: targets; arg_of edges then point at
|
||
// real function/method nodes whose param nodes can be found,
|
||
// and returns_to placeholders join cleanly against the
|
||
// now-resolved EdgeCalls edge at the same caller+line.
|
||
idx.materializeDataflowParams()
|
||
|
||
// Seed the durable reference-facts sidecar from the fully-resolved graph
|
||
// (no-op on the in-memory backend).
|
||
idx.persistAllRefFacts()
|
||
}
|
||
|
||
// EvictFile removes all nodes and edges belonging to filePath.
|
||
//
|
||
// filePath may arrive in any Unicode form — the git watcher derives it
|
||
// from `git diff` output (NFC), while an FSEvents-driven evict carries
|
||
// the filesystem's form (NFD on macOS). relKey folds both to the
|
||
// canonical NFC key the graph indexed the file under, so the eviction
|
||
// actually finds the file's nodes rather than silently no-opping and
|
||
// leaving a stale subtree behind.
|
||
func (idx *Indexer) EvictFile(filePath string) (int, int) {
|
||
absPath := filePath
|
||
if !filepath.IsAbs(absPath) {
|
||
absPath = filepath.Join(idx.rootPath, filePath)
|
||
}
|
||
relPath := idx.relKey(absPath)
|
||
// In multi-repo mode, the graph stores prefixed file paths.
|
||
graphPath := idx.prefixPath(relPath)
|
||
// Remove from search index.
|
||
for _, n := range idx.graph.GetFileNodes(graphPath) {
|
||
if n.Kind != graph.KindFile && n.Kind != graph.KindImport {
|
||
idx.search.Remove(n.ID)
|
||
}
|
||
}
|
||
idx.restubIncomingRefs(graphPath)
|
||
idx.evictEnrichment(graphPath)
|
||
idx.deleteRefFactsForFiles(idx.repoPrefix, []string{graphPath})
|
||
return idx.graph.EvictFile(graphPath)
|
||
}
|
||
|
||
// ReresolveFileScoped forces the scoped re-resolution + LSP re-verify a normal
|
||
// IndexFile performs, WITHOUT re-parsing and WITHOUT the IsStale gate — used by
|
||
// the watcher's shape-degradation self-heal, where the file's mtime is already
|
||
// current (IndexFile just ran) yet its resolved edges came out degraded, so a
|
||
// plain IncrementalReindexPaths would stale-gate it out. Re-runs only this
|
||
// file's forward + incoming resolve and, when a watch-enabled provider is
|
||
// wired, its incremental enrichment. O(file), no whole-graph pass. No-op when
|
||
// the file has no nodes (evicted since it was enqueued).
|
||
func (idx *Indexer) ReresolveFileScoped(filePath string) error {
|
||
absPath := filePath
|
||
if !filepath.IsAbs(absPath) {
|
||
absPath = filepath.Join(idx.rootPath, filePath)
|
||
}
|
||
graphPath := idx.prefixPath(idx.relKey(absPath))
|
||
if len(idx.graph.GetFileNodes(graphPath)) == 0 {
|
||
return nil // file gone / evicted; nothing to re-resolve
|
||
}
|
||
idx.resolver.ResolveFileAndIncoming(graphPath)
|
||
providersPresent := idx.semanticMgr != nil && idx.semanticMgr.Enabled() && idx.semanticMgr.HasProviders()
|
||
reEnriched := false
|
||
if providersPresent {
|
||
if _, err := idx.semanticMgr.EnrichFile(idx.graph, idx.rootPath, graphPath); err != nil {
|
||
idx.logger.Debug("indexer: forced scoped enrichment failed",
|
||
zap.String("file", graphPath), zap.Error(err))
|
||
} else {
|
||
reEnriched = idx.semanticMgr.EnrichesOnWatch()
|
||
}
|
||
}
|
||
// Keep the find_usages staleness marker consistent with the enrichment
|
||
// that actually ran during this forced re-resolve (mirrors indexFile).
|
||
idx.setReparsePendingEnrichment(graphPath, providersPresent && !reEnriched)
|
||
return nil
|
||
}
|
||
|
||
// restubIncomingRefs rewrites every resolved reference edge that points
|
||
// INTO a symbol of graphPath from a surviving (other-file) source back
|
||
// to an `unresolved::<Name>` stub, in place, BEFORE the file's nodes are
|
||
// evicted. Graph eviction otherwise drops those incoming caller edges
|
||
// wholesale (it removes the edge from the surviving source's out-edge
|
||
// bucket) and nothing recreates them until a cold reindex — so editing
|
||
// or deleting a definition silently strips its callers' edges and
|
||
// find_usages / get_callers go blank. Re-stubbing detaches the edges
|
||
// from the soon-to-be-evicted nodes so they survive as pending stubs;
|
||
// ResolveIncomingForFile (after a re-index) rebinds them to the file's
|
||
// fresh symbols, or they stay unresolved — the correct state once the
|
||
// symbol is gone. Only name-resolvable reference kinds are re-stubbed;
|
||
// structural and enrichment edges are left to be dropped. Backend-
|
||
// agnostic: GetInEdges + ReindexEdges are the same Store primitives the
|
||
// resolver uses, so this behaves identically on the in-memory and disk
|
||
// stores.
|
||
// evictEnrichment drops the per-node enrichment sidecar rows (churn,
|
||
// coverage, release, blame — change A) for a file's nodes on the
|
||
// delete/rename paths only, so a removed file leaves no orphan
|
||
// enrichment. Capability-gated. A modify re-indexes the same node IDs
|
||
// (enrichment stays valid) so it is NOT cascaded there.
|
||
func (idx *Indexer) evictEnrichment(graphPath string) {
|
||
nodes := idx.graph.GetFileNodes(graphPath)
|
||
if len(nodes) == 0 {
|
||
return
|
||
}
|
||
ids := make([]string, 0, len(nodes))
|
||
for _, n := range nodes {
|
||
ids = append(ids, n.ID)
|
||
}
|
||
if w, ok := idx.graph.(graph.ChurnEnrichmentWriter); ok {
|
||
_ = w.DeleteChurn(ids)
|
||
}
|
||
if w, ok := idx.graph.(graph.CoverageEnrichmentWriter); ok {
|
||
_ = w.DeleteCoverage(ids)
|
||
}
|
||
if w, ok := idx.graph.(graph.ReleaseEnrichmentWriter); ok {
|
||
_ = w.DeleteReleases(ids)
|
||
}
|
||
if w, ok := idx.graph.(graph.BlameEnrichmentWriter); ok {
|
||
_ = w.DeleteBlame(ids)
|
||
}
|
||
}
|
||
|
||
func (idx *Indexer) restubIncomingRefs(graphPath string) {
|
||
nodes := idx.graph.GetFileNodes(graphPath)
|
||
if len(nodes) == 0 {
|
||
return
|
||
}
|
||
evicted := make(map[string]struct{}, len(nodes))
|
||
for _, n := range nodes {
|
||
evicted[n.ID] = struct{}{}
|
||
}
|
||
var batch []graph.EdgeReindex
|
||
for _, n := range nodes {
|
||
if n.Name == "" || !graph.IsReferenceableSymbol(n.Kind) {
|
||
continue
|
||
}
|
||
stub := graph.UnresolvedMarker + n.Name
|
||
for _, e := range idx.graph.GetInEdges(n.ID) {
|
||
if e == nil || !graph.IsResolvableRefEdge(e.Kind) {
|
||
continue
|
||
}
|
||
if _, fromEvicted := evicted[e.From]; fromEvicted {
|
||
continue // intra-file edge: the source is evicted too
|
||
}
|
||
if graph.IsUnresolvedTarget(e.To) {
|
||
continue // already a pending stub
|
||
}
|
||
oldTo := e.To
|
||
// Stash + clear the edge's resolved provenance before restubbing:
|
||
// an `unresolved::` stub must not keep advertising a resolved
|
||
// tier. The incoming-resolve pass restores it verbatim if the stub
|
||
// rebinds to the same target (idempotent re-parse); a deleted or
|
||
// moved target leaves the stub honestly unresolved.
|
||
graph.StashRestubProvenance(e)
|
||
e.To = stub
|
||
batch = append(batch, graph.EdgeReindex{Edge: e, OldTo: oldTo})
|
||
}
|
||
}
|
||
if len(batch) > 0 {
|
||
idx.graph.ReindexEdges(batch)
|
||
}
|
||
}
|
||
|
||
// embeddingDimsOrDefault returns the embedder's reported vector width,
|
||
// falling back to a neutral placeholder only when the provider cannot
|
||
// state its width yet (Dimensions() == 0, the APIProvider-before-first-
|
||
// call case). The fallback is never persisted: buildSearchIndex and
|
||
// ImportVectorIndex both overwrite it with the true width taken from a
|
||
// real vector / the cached header. Kept as a named helper so the
|
||
// vector-dimension default has one definition instead of a scattered
|
||
// magic number.
|
||
func embeddingDimsOrDefault(p embedding.Provider) int {
|
||
if p == nil {
|
||
return 0
|
||
}
|
||
if d := p.Dimensions(); d > 0 {
|
||
return d
|
||
}
|
||
// Provider has not committed to a width. 384 matches the default
|
||
// transformer backend (MiniLM-L6-v2); a static GloVe provider
|
||
// always reports its real 50 and never reaches this branch.
|
||
return 384
|
||
}
|
||
|
||
// collectEmbedTexts walks the nodes and produces the parallel texts /
|
||
// ids slices the embedding pass consumes, plus a chunkMap recording
|
||
// which synthetic IDs are chunks of which symbol.
|
||
//
|
||
// A symbol whose source span exceeds the configured chunk threshold is
|
||
// read from disk and split into AST windows by embedding.ChunkSymbol;
|
||
// each window contributes one text (its body, prefixed with the
|
||
// symbol's kind + name for a little lexical grounding) under a
|
||
// synthetic ID "<symbolID>#chunkK", and chunkMap[syntheticID] = symbolID.
|
||
// A symbol below the threshold — or one whose file can't be read —
|
||
// contributes a single metadata text under its own ID, exactly as the
|
||
// pre-chunking pipeline did. The returned skipped count is the number
|
||
// of nodes dropped by the SkipEmbed rules.
|
||
func (idx *Indexer) collectEmbedTexts(nodes []*graph.Node) (texts []string, ids []string, chunkMap map[string]string, skipped int) {
|
||
chunkMap = make(map[string]string)
|
||
opts := idx.embedChunkOpts
|
||
threshold := opts.ThresholdLines
|
||
if threshold <= 0 {
|
||
threshold = embedding.DefaultChunkThresholdLines
|
||
}
|
||
// fileCache memoizes one os.ReadFile per source file — many symbols
|
||
// share a file, and the chunker only needs the bytes once.
|
||
fileCache := make(map[string][]byte)
|
||
readFile := func(graphPath string) []byte {
|
||
if cached, ok := fileCache[graphPath]; ok {
|
||
return cached
|
||
}
|
||
var data []byte
|
||
if abs := idx.ResolveFilePath(graphPath); abs != "" {
|
||
if b, err := os.ReadFile(abs); err == nil {
|
||
data = b
|
||
}
|
||
}
|
||
fileCache[graphPath] = data // cache misses too (nil) — don't re-stat
|
||
return data
|
||
}
|
||
|
||
for _, n := range nodes {
|
||
if n.Kind == graph.KindFile || n.Kind == graph.KindImport {
|
||
continue
|
||
}
|
||
// CONTENT section bodies are served by the content index, not the
|
||
// vector store — excluding them keeps the embed-text count (and the
|
||
// 100k auto-disable check) code-only, so a content-heavy repo no
|
||
// longer drowns the embedding pass in hundreds of thousands of
|
||
// section texts.
|
||
if isContentNode(n) {
|
||
skipped++
|
||
continue
|
||
}
|
||
if config.ShouldSkipEmbed(idx.config.SkipEmbed, n.Language, string(n.Kind)) {
|
||
skipped++
|
||
continue
|
||
}
|
||
sig, _ := n.Meta["signature"].(string)
|
||
metaText := fmt.Sprintf("%s %s %s %s", n.Kind, n.Name, sig, n.FilePath)
|
||
|
||
// Decide whether to sub-chunk: the symbol must declare a
|
||
// multi-line span past the threshold and its file must be
|
||
// readable. Anything else falls back to the metadata vector.
|
||
span := n.EndLine - n.StartLine + 1
|
||
body := extractSymbolBody(n, readFile, threshold)
|
||
if span <= threshold || len(body) == 0 {
|
||
texts = append(texts, metaText)
|
||
ids = append(ids, n.ID)
|
||
continue
|
||
}
|
||
|
||
windows := embedding.ChunkSymbol(body, n.Language, n.ID, opts)
|
||
if len(windows) <= 1 {
|
||
// The chunker decided one window was enough (short body,
|
||
// no splitter, parse failure) — embed it as a single
|
||
// metadata + body vector under the symbol's own ID.
|
||
texts = append(texts, metaText+" "+windows[0].Text)
|
||
ids = append(ids, n.ID)
|
||
continue
|
||
}
|
||
for _, w := range windows {
|
||
chunkID := fmt.Sprintf("%s#chunk%d", n.ID, w.WindowIndex)
|
||
texts = append(texts, fmt.Sprintf("%s %s %s", n.Kind, n.Name, w.Text))
|
||
ids = append(ids, chunkID)
|
||
chunkMap[chunkID] = n.ID
|
||
}
|
||
}
|
||
return texts, ids, chunkMap, skipped
|
||
}
|
||
|
||
// extractSymbolBody returns the source text of a symbol's span, read
|
||
// from its file via readFile and sliced by the node's 1-based
|
||
// StartLine..EndLine. Returns nil when the file is unreadable, the
|
||
// line range is unusable, or the symbol is at or below the threshold
|
||
// (small symbols never need their body — the caller embeds metadata).
|
||
func extractSymbolBody(n *graph.Node, readFile func(string) []byte, threshold int) []byte {
|
||
if n.StartLine <= 0 || n.EndLine < n.StartLine {
|
||
return nil
|
||
}
|
||
if n.EndLine-n.StartLine+1 <= threshold {
|
||
return nil
|
||
}
|
||
data := readFile(n.FilePath)
|
||
if len(data) == 0 {
|
||
return nil
|
||
}
|
||
return sliceLines(data, n.StartLine, n.EndLine)
|
||
}
|
||
|
||
// sliceLines returns the bytes of the 1-based inclusive line range
|
||
// [start,end] of src. An out-of-range request is clamped; an empty
|
||
// result is returned for a range that lands entirely past EOF.
|
||
func sliceLines(src []byte, start, end int) []byte {
|
||
if start < 1 {
|
||
start = 1
|
||
}
|
||
line := 1
|
||
startByte := -1
|
||
endByte := len(src)
|
||
for i := 0; i < len(src); i++ {
|
||
if line == start && startByte < 0 {
|
||
startByte = i
|
||
}
|
||
if src[i] == '\n' {
|
||
line++
|
||
if line == end+1 {
|
||
endByte = i + 1 // include the trailing newline
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if startByte < 0 {
|
||
return nil
|
||
}
|
||
if endByte < startByte {
|
||
endByte = len(src)
|
||
}
|
||
return src[startByte:endByte]
|
||
}
|
||
|
||
// defaultEmbedAPIConcurrency bounds parallel embedding requests
|
||
// against an API-backed embedder when embedding.api_concurrency is
|
||
// unset. Four is a conservative default that overlaps round-trips
|
||
// without tripping typical hosted-API rate limits.
|
||
const defaultEmbedAPIConcurrency = 4
|
||
|
||
// embedChunkBatch is one unit of work for the embedding pool: the
|
||
// texts of one chunk plus the index that fixes where its vectors land
|
||
// in the result slice. Carrying the index makes completion order
|
||
// irrelevant — workers write by index, never append.
|
||
type embedChunkBatch struct {
|
||
index int
|
||
texts []string
|
||
}
|
||
|
||
// embedAllChunks embeds every text, returning the vectors in the same
|
||
// order as texts. The work is split into batches of batchSize texts.
|
||
//
|
||
// For an API-backed embedder the batches run through a bounded worker
|
||
// pool — a hosted embedding round-trip dominates index time, so
|
||
// overlapping requests is a real speedup. In-process embedders
|
||
// (Hugot / ONNX / GoMLX / static) serialise on an inference mutex, so
|
||
// concurrency buys them nothing and they keep the simple serial path.
|
||
//
|
||
// The abort-on-any-error contract is preserved in both modes: the
|
||
// first batch failure cancels the group and embedAllChunks returns the
|
||
// error with no partial result, exactly as the old serial loop did.
|
||
// embedFn already layers the deadline-halving retry on top of each
|
||
// batch.
|
||
func (idx *Indexer) embedAllChunks(
|
||
texts []string,
|
||
batchSize int,
|
||
embedFn func(ctx context.Context, items []string) ([][]float32, error),
|
||
) ([][]float32, error) {
|
||
if len(texts) == 0 {
|
||
return nil, nil
|
||
}
|
||
|
||
// Split into batches up front so both the serial and parallel
|
||
// paths iterate the same units.
|
||
var batches []embedChunkBatch
|
||
for start := 0; start < len(texts); start += batchSize {
|
||
end := start + batchSize
|
||
if end > len(texts) {
|
||
end = len(texts)
|
||
}
|
||
batches = append(batches, embedChunkBatch{index: len(batches), texts: texts[start:end]})
|
||
}
|
||
|
||
// Per-batch result slots, pre-sized so workers write by index and
|
||
// completion order never matters.
|
||
results := make([][][]float32, len(batches))
|
||
|
||
// Only run the pool for an embedder that declares itself safe and
|
||
// worthwhile to call concurrently — the API-backed provider, where
|
||
// overlapped HTTP round-trips are a real win. In-process backends
|
||
// (Hugot / ONNX / GoMLX) hold an inference mutex, so a pool would
|
||
// only add scheduling overhead; they keep the serial path.
|
||
apiBacked := false
|
||
if c, ok := idx.embedder.(interface{ Concurrent() bool }); ok {
|
||
apiBacked = c.Concurrent()
|
||
}
|
||
concurrency := idx.embedAPIConcurrency
|
||
if concurrency <= 0 {
|
||
concurrency = defaultEmbedAPIConcurrency
|
||
}
|
||
if concurrency > len(batches) {
|
||
concurrency = len(batches)
|
||
}
|
||
|
||
if !apiBacked || concurrency <= 1 {
|
||
// Serial path — unchanged behaviour for in-process embedders.
|
||
ctx := context.Background()
|
||
for _, b := range batches {
|
||
vecs, err := embedFn(ctx, b.texts)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
results[b.index] = vecs
|
||
}
|
||
return flattenEmbedResults(results), nil
|
||
}
|
||
|
||
// Parallel path — bounded worker pool for an API-backed embedder.
|
||
// A cancellable group context means the first failure stops every
|
||
// in-flight worker; the indexer's existing per-batch retry still
|
||
// runs underneath embedFn.
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
defer cancel()
|
||
|
||
jobs := make(chan embedChunkBatch)
|
||
var (
|
||
wg sync.WaitGroup
|
||
errOnce sync.Once
|
||
firstErr error
|
||
)
|
||
fail := func(err error) {
|
||
errOnce.Do(func() {
|
||
firstErr = err
|
||
cancel() // abort siblings on the first error
|
||
})
|
||
}
|
||
|
||
for w := 0; w < concurrency; w++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for b := range jobs {
|
||
if ctx.Err() != nil {
|
||
return // group already aborted
|
||
}
|
||
vecs, err := embedFn(ctx, b.texts)
|
||
if err != nil {
|
||
fail(err)
|
||
return
|
||
}
|
||
// Write into the pre-sized slot — no shared append, so
|
||
// no lock and order is fixed by b.index.
|
||
results[b.index] = vecs
|
||
}
|
||
}()
|
||
}
|
||
idx.logger.Info("embedding vector index with a concurrent API pool",
|
||
zap.Int("workers", concurrency),
|
||
zap.Int("batches", len(batches)))
|
||
|
||
for _, b := range batches {
|
||
if ctx.Err() != nil {
|
||
break // stop feeding once aborted
|
||
}
|
||
select {
|
||
case jobs <- b:
|
||
case <-ctx.Done():
|
||
}
|
||
}
|
||
close(jobs)
|
||
wg.Wait()
|
||
|
||
if firstErr != nil {
|
||
return nil, firstErr
|
||
}
|
||
return flattenEmbedResults(results), nil
|
||
}
|
||
|
||
// flattenEmbedResults concatenates per-batch vector slices back into a
|
||
// single slice aligned with the original texts order.
|
||
func flattenEmbedResults(results [][][]float32) [][]float32 {
|
||
total := 0
|
||
for _, r := range results {
|
||
total += len(r)
|
||
}
|
||
out := make([][]float32, 0, total)
|
||
for _, r := range results {
|
||
out = append(out, r...)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// buildSearchIndex populates the search backend from the current graph.
|
||
// When an embedder is set, also builds a vector index and wraps both
|
||
// in a HybridBackend with RRF fusion.
|
||
//
|
||
// In multi-repo mode the search backend is shared across every repo
|
||
// (Indexer.search is wired to MultiIndexer.search at construction).
|
||
// Walking g.AllNodes() and re-Add()ing every node would mean each
|
||
// freshly-tracked repo pays an O(workspace) re-index pass over all
|
||
// previously-tracked repos' nodes — quadratic in repo count and the
|
||
// dominant cost of warming up a 260-repo workspace. So when this
|
||
// indexer carries a non-empty repoPrefix we walk only that repo's
|
||
// byRepo bucket; the other repos' entries are already in the shared
|
||
// backend from when they were tracked. Single-repo mode keeps the
|
||
// AllNodes() path because nodes there carry an empty RepoPrefix and
|
||
// GetRepoNodes("") would miss them.
|
||
func (idx *Indexer) buildSearchIndex() {
|
||
// Start every build from a clean vector-build error: the degraded vector
|
||
// paths below set it, and a successful build (or a benign skip / no embedder)
|
||
// leaves it nil, so LastVectorBuildError always reflects the current pass.
|
||
idx.lastVectorBuildErr = nil
|
||
|
||
// Code-only enumeration: content (data_class=content) sections live in
|
||
// the content index, never the symbol search or the vector store, so the
|
||
// FTS loop below and collectEmbedTexts both skip them anyway. Fetching
|
||
// the non-content set up front means a content-heavy repo's hundreds of
|
||
// thousands of sections never enter memory here (the disk backend filters
|
||
// them in SQL), instead of being materialised only to be skipped.
|
||
nodes := graph.RepoCodeNodes(idx.graph, idx.repoPrefix)
|
||
|
||
// Install the learned sub-word boundary table on the BM25 layer
|
||
// before populating postings, so the optional sparse-ngram
|
||
// tokenizer's split points are data-driven and — crucially —
|
||
// identical on the index path here and the query path later. The
|
||
// table must be in place before the first Add for postings and
|
||
// queries to agree; it is a no-op for non-BM25 backends and a no-op
|
||
// for search overall unless GORTEX_SPARSE_NGRAM is set.
|
||
search.InstallNgramBoundaries(idx.search, search.BuildNgramBoundaries(idx.graph))
|
||
|
||
// Build text index. The SkipSearch filter (wired through
|
||
// idx.shouldIndexForSearch) drops config-key-style variable nodes
|
||
// that would only pad the index — see docs on IndexConfig.SkipSearch.
|
||
for _, n := range nodes {
|
||
if !idx.shouldIndexForSearch(n) {
|
||
continue
|
||
}
|
||
idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...)
|
||
}
|
||
|
||
// Build vector index if embedder is available.
|
||
if idx.embedder == nil {
|
||
return
|
||
}
|
||
|
||
// skipVectorBuild short-circuits the embedding pass: the text index
|
||
// above is fully populated, but a caller (the daemon warmup loop
|
||
// after a snapshot restore) has signalled that the workspace vector
|
||
// index will be supplied separately, so re-embedding here would be
|
||
// wasted work immediately overwritten by the cached index.
|
||
if idx.skipVectorBuild {
|
||
return
|
||
}
|
||
|
||
// Provisional dimensionality: trust the embedder's own report.
|
||
// A provider that can't state its width yet (an APIProvider before
|
||
// its first call returns 0) gets a neutral placeholder — the value
|
||
// is overwritten below from the first real vector, so it never
|
||
// reaches the persisted index. The old hard-coded 300 was wrong for
|
||
// the default static GloVe provider (50d) and misrepresented the
|
||
// index width in the interim; deriving from Dimensions() keeps it
|
||
// honest for every provider.
|
||
dims := embeddingDimsOrDefault(idx.embedder)
|
||
|
||
// Collect texts and IDs for batch embedding. Nodes matching
|
||
// Semantic.SkipEmbed (e.g. CSS custom properties, terraform blocks,
|
||
// YAML/TOML/shell config vars) are kept in the text index but
|
||
// excluded from the vector index — embedding them is pure cost
|
||
// with no semantic payoff and on big monorepos dominates RAM.
|
||
//
|
||
// A symbol whose source span exceeds the chunk threshold is split
|
||
// into AST windows: each window is embedded as its own vector under
|
||
// a synthetic ID ("<symbolID>#chunkK"), and chunkMap records the
|
||
// chunk → parent mapping so query-time de-chunking maps a chunk hit
|
||
// back to the symbol. A small symbol stays a single metadata-only
|
||
// vector under its own ID. chunkMap is empty when nothing was split.
|
||
texts, ids, chunkMap, skipped := idx.collectEmbedTexts(nodes)
|
||
if skipped > 0 {
|
||
idx.logger.Info("skipped embedding for low-value nodes",
|
||
zap.Int("count", skipped),
|
||
zap.Int("embedded", len(texts)))
|
||
}
|
||
|
||
if len(texts) == 0 {
|
||
return
|
||
}
|
||
|
||
// Embedding scaling guards. Hard-cap the vector index for repos
|
||
// big enough that the cost no longer pays off — BM25 alone is a
|
||
// fine fallback and an OOM during initial index is much worse than
|
||
// missing the semantic boost. Chunk the EmbedBatch calls so any
|
||
// single API request stays small (matters for hosted embedders
|
||
// with per-request token limits).
|
||
//
|
||
// embedChunkTimeout is generous because ONNX inference (Hugot) has
|
||
// long tail latency: a 60s budget made one in ~30 chunks miss its
|
||
// deadline, which under the old fail-fast policy threw away every
|
||
// already-embedded chunk and silently degraded to BM25 with no
|
||
// signal to the user. 5 minutes covers observed worst-case spikes
|
||
// without changing steady-state behaviour. On a true hang the
|
||
// caller can still cancel the parent indexing call.
|
||
const (
|
||
defaultEmbedMaxSymbols = 100_000
|
||
embedChunkSize = 500
|
||
embedChunkTimeout = 5 * time.Minute
|
||
)
|
||
|
||
// The cap is over the embeddable-text count, which with AST
|
||
// sub-chunking can exceed the symbol count. embedding.max_symbols
|
||
// overrides the built-in default for users with memory headroom.
|
||
embedMaxSymbols := defaultEmbedMaxSymbols
|
||
if idx.embedMaxSymbols > 0 {
|
||
embedMaxSymbols = idx.embedMaxSymbols
|
||
}
|
||
// Env override wins over both the default and the config-wired cap —
|
||
// a reliable knob independent of the config plumbing. Lets an operator
|
||
// lift the vector-index size guard for a large repo without editing
|
||
// (or debugging) the layered config.
|
||
if env := os.Getenv("GORTEX_EMBEDDINGS_MAX_SYMBOLS"); env != "" {
|
||
if n, err := strconv.Atoi(strings.TrimSpace(env)); err == nil && n > 0 {
|
||
embedMaxSymbols = n
|
||
}
|
||
}
|
||
if len(texts) > embedMaxSymbols {
|
||
idx.logger.Warn("vector index disabled — embedding text count exceeds threshold",
|
||
zap.Int("texts", len(texts)),
|
||
zap.Int("threshold", embedMaxSymbols),
|
||
zap.String("hint", "BM25 text search remains active; raise embedding.max_symbols if you have memory headroom"))
|
||
idx.lastVectorBuildErr = fmt.Errorf("embedding text count %d exceeds threshold %d (raise embedding.max_symbols)", len(texts), embedMaxSymbols)
|
||
return
|
||
}
|
||
|
||
// embedWithRetry runs one chunk under ctx; on a context-deadline
|
||
// failure it splits the chunk in half and retries each half once.
|
||
// A single slow batch shouldn't throw away every already-embedded
|
||
// chunk and silently demote the backend to BM25. ctx is the group
|
||
// context, so once one chunk fails everywhere the in-flight retries
|
||
// here see the cancellation and stop too.
|
||
var embedWithRetry func(ctx context.Context, items []string) ([][]float32, error)
|
||
embedWithRetry = func(ctx context.Context, items []string) ([][]float32, error) {
|
||
chunkCtx, cancel := context.WithTimeout(ctx, embedChunkTimeout)
|
||
out, err := idx.embedder.EmbedBatch(chunkCtx, items)
|
||
cancel()
|
||
if err == nil {
|
||
return out, nil
|
||
}
|
||
// Only retry on deadline-style failures; auth/protocol errors
|
||
// won't get better with smaller batches. A cancellation from
|
||
// the group context (a sibling chunk already failed) is not a
|
||
// retry case either.
|
||
if ctx.Err() != nil || !errors.Is(err, context.DeadlineExceeded) || len(items) <= 1 {
|
||
return nil, err
|
||
}
|
||
idx.logger.Warn("embed chunk timed out, retrying with halved batch",
|
||
zap.Int("size", len(items)),
|
||
zap.Error(err))
|
||
mid := len(items) / 2
|
||
left, lerr := embedWithRetry(ctx, items[:mid])
|
||
if lerr != nil {
|
||
return nil, lerr
|
||
}
|
||
right, rerr := embedWithRetry(ctx, items[mid:])
|
||
if rerr != nil {
|
||
return nil, rerr
|
||
}
|
||
return append(left, right...), nil
|
||
}
|
||
|
||
// Embed every chunk. For an API-backed embedder the chunks are run
|
||
// through a bounded worker pool (a hosted round-trip dominates
|
||
// indexing time, so overlapping requests is a real win); local
|
||
// in-process backends serialise on an inference mutex, so they keep
|
||
// the simple serial path. Either way the abort-on-any-error
|
||
// contract holds — one chunk failure means no vector index ships.
|
||
vectors, err := idx.embedAllChunks(texts, embedChunkSize, embedWithRetry)
|
||
if err != nil {
|
||
// A partial vector index would mis-score later queries (some
|
||
// symbols semantically findable, others not) — bail to
|
||
// text-only search rather than ship an inconsistent hybrid
|
||
// backend.
|
||
idx.logger.Warn("vector index aborted on chunk failure", zap.Error(err))
|
||
idx.lastVectorBuildErr = fmt.Errorf("chunk embedding failed: %w", err)
|
||
return
|
||
}
|
||
|
||
// Detect actual dimensions from first vector.
|
||
if len(vectors) > 0 && len(vectors[0]) > 0 {
|
||
dims = len(vectors[0])
|
||
}
|
||
|
||
vecBackend := search.NewVector(dims)
|
||
// VectorSearcher capability bridging: if the underlying store
|
||
// has a native HNSW, install it as the in-process backend's
|
||
// delegate — Add becomes a no-op, Search forwards to the
|
||
// engine, and we don't allocate `dim × 4 × N` bytes of heap
|
||
// for a parallel in-process HNSW. The indexer still drives
|
||
// the writes (BulkUpsertEmbeddings below) so the engine
|
||
// index lands with the same corpus the in-process one would
|
||
// have built.
|
||
vecSearcher, _ := idx.graph.(graph.VectorSearcher)
|
||
if vecSearcher != nil {
|
||
vecBackend.SetDelegate(&vectorSearcherDelegate{s: vecSearcher})
|
||
}
|
||
// persistSink is where the embedded vectors are written to the backend
|
||
// so they survive a restart. Normally it is the active graph (a disk
|
||
// store). Under the bulk loader idx.graph is the in-memory shadow, which
|
||
// does not implement VectorSearcher; fall back to the disk store captured
|
||
// at the shadow swap (bulkVectorSink) so the vector index still reaches
|
||
// the `vectors` table instead of living only in the in-process HNSW and
|
||
// vanishing on the next restart (which would force a paid re-embed).
|
||
persistSink := vecSearcher
|
||
if persistSink == nil {
|
||
persistSink = idx.bulkVectorSink
|
||
}
|
||
var backendItems []graph.VectorItem
|
||
if persistSink != nil {
|
||
backendItems = make([]graph.VectorItem, 0, len(vectors))
|
||
}
|
||
// Add only well-formed vectors. A nil or wrong-width vector would poison
|
||
// the index — a mis-scored or panicking query — so drop it, count it, and
|
||
// keep a sample of offending node IDs for the log.
|
||
var droppedVectors int
|
||
var droppedSample []string
|
||
for i, vec := range vectors {
|
||
if len(vec) != dims {
|
||
droppedVectors++
|
||
if len(droppedSample) < 5 {
|
||
droppedSample = append(droppedSample, ids[i])
|
||
}
|
||
continue
|
||
}
|
||
vecBackend.Add(ids[i], vec)
|
||
if persistSink != nil {
|
||
backendItems = append(backendItems, graph.VectorItem{
|
||
NodeID: ids[i],
|
||
Vec: vec,
|
||
})
|
||
}
|
||
}
|
||
// If every vector was invalid there is nothing to search on. Ship text-only
|
||
// rather than a silently empty vector index that mis-scores every query —
|
||
// the same all-or-nothing contract as the chunk-failure abort above.
|
||
if len(vectors) > 0 && vecBackend.Count() == 0 {
|
||
idx.logger.Warn("vector index aborted — all embeddings invalid",
|
||
zap.Int("dropped", droppedVectors),
|
||
zap.Int("dimensions", dims),
|
||
zap.Strings("sample_ids", droppedSample))
|
||
idx.lastVectorBuildErr = fmt.Errorf("all %d embedding vectors were invalid (want width %d)", droppedVectors, dims)
|
||
return
|
||
}
|
||
if persistSink != nil && len(backendItems) > 0 {
|
||
if err := persistSink.BulkUpsertEmbeddings(backendItems); err != nil {
|
||
idx.logger.Warn("indexer: backend vector bulk upsert failed",
|
||
zap.Error(err))
|
||
} else if err := persistSink.BuildVectorIndex(dims); err != nil {
|
||
idx.logger.Warn("indexer: backend vector index build failed",
|
||
zap.Error(err))
|
||
}
|
||
}
|
||
// Install the chunk → parent-symbol mapping so HybridBackend can
|
||
// de-chunk vector hits back to symbols at query time. Empty when no
|
||
// symbol was large enough to split.
|
||
if len(chunkMap) > 0 {
|
||
vecBackend.SetChunkMap(chunkMap)
|
||
}
|
||
|
||
// Wrap text + vector into hybrid backend, swapping it in atomically
|
||
// so any concurrent searches keep seeing a coherent backend.
|
||
//
|
||
// Unwrap any existing HybridBackend to its text side before
|
||
// re-wrapping. Without this, buildSearchIndex called again (e.g.
|
||
// once per tracked repo during daemon warmup) would stack a fresh
|
||
// Hybrid on top of the previous one — nested Hybrids retain all
|
||
// their stale vector indexes, ballooning live memory by an order
|
||
// of magnitude. The text backend (BM25 or Bleve) has already been
|
||
// updated with every node via idx.search.Add above; a single
|
||
// Hybrid wrapping it + the latest vecBackend is all we need.
|
||
sw := idx.swappable()
|
||
inner := sw.Inner()
|
||
if hyb, ok := inner.(*search.HybridBackend); ok {
|
||
inner = hyb.TextBackend()
|
||
}
|
||
sw.Swap(search.NewHybrid(inner, vecBackend, idx.embedder))
|
||
if droppedVectors > 0 {
|
||
idx.logger.Warn("indexer: dropped invalid embedding vectors",
|
||
zap.Int("dropped", droppedVectors),
|
||
zap.Int("dimensions", dims),
|
||
zap.Strings("sample_ids", droppedSample))
|
||
}
|
||
fields := []zap.Field{
|
||
zap.Int("vectors", vecBackend.Count()),
|
||
zap.Int("chunk_vectors", len(chunkMap)),
|
||
zap.Int("dimensions", dims),
|
||
zap.Int("dropped", droppedVectors),
|
||
}
|
||
// Surface the actual token spend of a paid embedding pass when the
|
||
// backend reports usage (API providers do; in-process ones don't).
|
||
// Without this the cost of an embedding run is invisible after the fact.
|
||
if acc, ok := idx.embedder.(interface{ TokensUsed() int64 }); ok {
|
||
if tokens := acc.TokensUsed(); tokens > 0 {
|
||
fields = append(fields, zap.Int64("embed_tokens", tokens))
|
||
}
|
||
}
|
||
idx.logger.Info("vector index built", fields...)
|
||
}
|
||
|
||
// dirIgnoreFiles are the per-directory ignore-file basenames honored by
|
||
// the index walk, siblings to .gitignore: Gortex's own .gortexignore
|
||
// plus ripgrep's .ignore and .rgignore. Patterns in each file are
|
||
// scoped to the directory that contains it; later filenames win, so a
|
||
// directory's .rgignore overrides its .ignore on a conflicting path.
|
||
var dirIgnoreFiles = []string{".gortexignore", ".ignore", ".rgignore"}
|
||
|
||
// shouldExclude reports whether a path is excluded by the effective
|
||
// ignore list. The flat matcher is built lazily from idx.config.Exclude,
|
||
// which is populated by ConfigManager.GetRepoConfig with the full
|
||
// layered list (builtin + global + RepoEntry + workspace). A path is
|
||
// also excluded by any per-directory ignore file (dirIgnoreFiles)
|
||
// present in one of its ancestor directories. isDir lets a trailing-
|
||
// slash pattern prune a directory subtree instead of only its files.
|
||
func (idx *Indexer) shouldExclude(path, root string, isDir bool) bool {
|
||
// .claude/ and .kiro/ are Builtin-excluded wholesale, but may hold an
|
||
// MCP server config the MCP-config-as-graph feature targets (the
|
||
// extractor's own docs name .kiro/mcp.json). Descend those subtrees
|
||
// and index only the MCP config files within them — everything else
|
||
// stays excluded, so the agent-state noise never reaches the graph.
|
||
if rel, err := filepath.Rel(root, path); err == nil && excludes.InAgentConfigDir(rel) {
|
||
if isDir {
|
||
return false
|
||
}
|
||
return !excludes.IsMCPConfigFile(rel)
|
||
}
|
||
if m := idx.excludeMatcher(); m != nil && m.MatchAbsDir(path, root, isDir) {
|
||
return true
|
||
}
|
||
return idx.dirIgnoreMatcher(root).Match(path, isDir)
|
||
}
|
||
|
||
// shouldPruneDir reports whether the index walk may skip a directory
|
||
// subtree wholesale (filepath.SkipDir) instead of descending it. A
|
||
// directory is prunable only when it is excluded AND no re-include ("!")
|
||
// pattern targets anything beneath it. go-gitignore's "*" matches across
|
||
// "/", so a blanket like "wp-content/plugins/*" reports the parent
|
||
// directory "wp-content/plugins" itself as excluded; pruning it would
|
||
// skip a later "!wp-content/plugins/foo/" re-include before the walk ever
|
||
// reaches the child. Mirroring git, we keep descending such a directory
|
||
// and let the per-file shouldExclude check filter its contents.
|
||
func (idx *Indexer) shouldPruneDir(path, root string) bool {
|
||
if !idx.shouldExclude(path, root, true) {
|
||
return false
|
||
}
|
||
if m := idx.excludeMatcher(); m != nil {
|
||
if rel, err := filepath.Rel(root, path); err == nil && m.HasNegatedDescendant(filepath.ToSlash(rel)) {
|
||
return false
|
||
}
|
||
}
|
||
if idx.dirIgnoreMatcher(root).HasNegatedDescendant(path) {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// dirIgnoreMatcher returns the per-directory ignore matcher, built lazily
|
||
// against the repo root the index walk is anchored at.
|
||
func (idx *Indexer) dirIgnoreMatcher(root string) *excludes.Hierarchical {
|
||
idx.dirIgnoreOnce.Do(func() {
|
||
idx.dirIgnore = excludes.NewHierarchical(root, dirIgnoreFiles...)
|
||
})
|
||
return idx.dirIgnore
|
||
}
|
||
|
||
func (idx *Indexer) excludeMatcher() *excludes.Matcher {
|
||
idx.excludeOnce.Do(func() {
|
||
patterns := idx.config.Exclude
|
||
// A nil/empty list from upstream means "no layering was applied"
|
||
// (e.g. a direct caller of indexer.New without ConfigManager).
|
||
// Fall back to the builtin baseline so the walk still skips the
|
||
// obvious non-source dirs.
|
||
if len(patterns) == 0 {
|
||
patterns = excludes.Builtin
|
||
}
|
||
idx.excludes = excludes.New(patterns)
|
||
})
|
||
return idx.excludes
|
||
}
|
||
|
||
// ParseErrors returns the parse errors from the last full index.
|
||
func (idx *Indexer) ParseErrors() []IndexError {
|
||
return idx.parseErrors
|
||
}
|
||
|
||
// FileMtimes returns a copy of the file modification time map.
|
||
func (idx *Indexer) FileMtimes() map[string]int64 {
|
||
idx.mtimeMu.RLock()
|
||
defer idx.mtimeMu.RUnlock()
|
||
out := make(map[string]int64, len(idx.fileMtimes))
|
||
for k, v := range idx.fileMtimes {
|
||
out[k] = v
|
||
}
|
||
return out
|
||
}
|
||
|
||
// RefreshFileMtime restamps the recorded modification time for a file
|
||
// from its current on-disk mtime, without re-indexing it. The watcher
|
||
// calls this when a save turned out to be structurally inert and the
|
||
// reindex was skipped: the graph is already correct, but the recorded
|
||
// mtime must advance past the save so the poller's mtime sweep does
|
||
// not keep re-flagging the same untouched file. A file absent from
|
||
// disk or never indexed is a no-op.
|
||
//
|
||
// The in-memory map is what the current process's poller and
|
||
// IsStale checks trust, but a warm restart trusts the persisted
|
||
// FileMtime sidecar instead — without also writing through to it here,
|
||
// a single inert save during a session left the persisted row at its
|
||
// pre-save value, so the next restart's HasChangesSinceMtimes saw this
|
||
// file as changed and re-tracked the whole repo. Mirrors the per-file
|
||
// indexFile persist (recordFileMtime); a no-op on the in-memory backend.
|
||
func (idx *Indexer) RefreshFileMtime(filePath string) {
|
||
absPath, err := filepath.Abs(filePath)
|
||
if err != nil {
|
||
return
|
||
}
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
return
|
||
}
|
||
// relKey (slash + NFC) so the lookup hits the same fileMtimes
|
||
// entry the index walk created for a non-ASCII filename.
|
||
key := idx.relKey(absPath)
|
||
mtime := info.ModTime().UnixNano()
|
||
idx.mtimeMu.Lock()
|
||
_, tracked := idx.fileMtimes[key]
|
||
if tracked {
|
||
idx.fileMtimes[key] = mtime
|
||
}
|
||
idx.mtimeMu.Unlock()
|
||
if !tracked {
|
||
return
|
||
}
|
||
if w, ok := idx.graph.(graph.FileMtimeWriter); ok {
|
||
if err := w.BulkSetFileMtimes(idx.repoPrefix, map[string]int64{key: mtime}); err != nil {
|
||
idx.logger.Warn("persist file mtime failed",
|
||
zap.String("repo", idx.repoPrefix), zap.String("file", key), zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
|
||
// pruneDeletedFileMtimes drops the persisted mtime rows for files the
|
||
// incremental reindex just confirmed deleted. The in-memory map is already
|
||
// pruned by the caller; this keeps the store's FileMtime sidecar in step so
|
||
// a later warm restart does not re-discover them as phantom deletions and
|
||
// force a full re-track. A no-op when the backend lacks the capability
|
||
// (the in-memory backend) or the list is empty.
|
||
func (idx *Indexer) pruneDeletedFileMtimes(deleted []string) {
|
||
if len(deleted) == 0 {
|
||
return
|
||
}
|
||
if d, ok := idx.graph.(graph.FileMtimeDeleter); ok {
|
||
if err := d.DeleteFileMtimes(idx.repoPrefix, deleted); err != nil {
|
||
idx.logger.Warn("prune deleted file mtimes failed",
|
||
zap.String("repo", idx.repoPrefix), zap.Error(err))
|
||
}
|
||
}
|
||
}
|
||
|
||
// SetFileMtimes restores the file modification time map from a persisted snapshot.
|
||
func (idx *Indexer) SetFileMtimes(mtimes map[string]int64) {
|
||
idx.mtimeMu.Lock()
|
||
defer idx.mtimeMu.Unlock()
|
||
idx.fileMtimes = make(map[string]int64, len(mtimes))
|
||
for k, v := range mtimes {
|
||
idx.fileMtimes[k] = v
|
||
}
|
||
}
|
||
|
||
// SetRootPath sets the root path for relative path computation.
|
||
func (idx *Indexer) SetRootPath(root string) {
|
||
abs, err := filepath.Abs(root)
|
||
if err != nil {
|
||
abs = root
|
||
}
|
||
idx.storeRootPath(abs)
|
||
}
|
||
|
||
// IncrementalReindexPaths re-indexes only the files reachable from the
|
||
// supplied paths, instead of walking the whole repository root.
|
||
//
|
||
// Each path may be absolute or relative to root, and may be a file or a
|
||
// directory; directories are walked recursively with the same
|
||
// exclude / language filters as a full pass. Within that scoped file
|
||
// set the behaviour matches IncrementalReindex: only files that are
|
||
// stale (mtime or, in Merkle mode, content) are re-indexed, and a file
|
||
// previously tracked under one of the scoped paths but now absent from
|
||
// disk is evicted.
|
||
//
|
||
// When paths is empty the call degrades to IncrementalReindex(root) —
|
||
// callers can therefore pass an optional path list unconditionally.
|
||
func (idx *Indexer) IncrementalReindexPaths(root string, paths []string) (*IndexResult, error) {
|
||
if len(paths) == 0 {
|
||
return idx.IncrementalReindex(root)
|
||
}
|
||
|
||
start := time.Now()
|
||
|
||
absRoot, err := filepath.Abs(root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
idx.storeRootPath(absRoot)
|
||
|
||
// scopeRels holds the repo-relative slash-paths the caller asked to
|
||
// reindex — used both to drive the discovery walk and to bound
|
||
// deletion detection to the scoped subtree.
|
||
scopeRels := make(map[string]bool)
|
||
|
||
// diskFiles is the set of in-scope language files currently on
|
||
// disk; staleFiles is the subset that changed since the last pass.
|
||
diskFiles := make(map[string]bool)
|
||
var staleFiles []string
|
||
|
||
merkleMode := idx.merkleEnabled()
|
||
|
||
for _, p := range paths {
|
||
absPath := p
|
||
if !filepath.IsAbs(absPath) {
|
||
absPath = filepath.Join(absRoot, filepath.FromSlash(p))
|
||
}
|
||
absPath = filepath.Clean(absPath)
|
||
|
||
// A path outside the repo root is rejected: scoping is a
|
||
// narrowing operation, never an escape hatch to index files
|
||
// the repo doesn't own.
|
||
rel, relErr := filepath.Rel(absRoot, absPath)
|
||
if relErr != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||
return nil, fmt.Errorf("incremental reindex: path %q is outside repository root %q", p, absRoot)
|
||
}
|
||
// Canonical key (slash + NFC) so scopeRels matches fileMtimes
|
||
// keys when deletion detection intersects the two below.
|
||
scopeRels[idx.relKey(absPath)] = true
|
||
|
||
info, statErr := os.Stat(absPath)
|
||
if statErr != nil {
|
||
// A path that no longer exists is not an error: it may be
|
||
// a deleted file the caller still wants evicted. Deletion
|
||
// detection below handles it via scopeRels.
|
||
if errors.Is(statErr, os.ErrNotExist) {
|
||
continue
|
||
}
|
||
return nil, fmt.Errorf("incremental reindex: stat %q: %w", p, statErr)
|
||
}
|
||
|
||
if info.IsDir() {
|
||
walkErr := filepath.WalkDir(absPath, func(path string, d os.DirEntry, err error) error {
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
if idx.shouldPruneDir(path, absRoot) {
|
||
return filepath.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
if _, ok := idx.effectiveLanguage(path, nil); !ok {
|
||
return nil
|
||
}
|
||
if idx.shouldExclude(path, absRoot, false) {
|
||
return nil
|
||
}
|
||
// relKey (slash + NFC) keeps the disk set keyed
|
||
// consistently with fileMtimes for non-ASCII names.
|
||
relPath := idx.relKey(path)
|
||
diskFiles[relPath] = true
|
||
if !merkleMode && idx.IsStale(relPath) {
|
||
staleFiles = append(staleFiles, path)
|
||
}
|
||
return nil
|
||
})
|
||
if walkErr != nil {
|
||
return nil, walkErr
|
||
}
|
||
continue
|
||
}
|
||
|
||
// Single file. Apply the same language / exclude gate so a
|
||
// caller can't force a non-source or excluded file in.
|
||
if _, ok := idx.effectiveLanguage(absPath, nil); !ok {
|
||
continue
|
||
}
|
||
if idx.shouldExclude(absPath, absRoot, false) {
|
||
continue
|
||
}
|
||
// relKey (slash + NFC) — same canonical key the graph and
|
||
// fileMtimes use, so a non-ASCII path passed in here matches
|
||
// regardless of the Unicode form the caller supplied.
|
||
relPath := idx.relKey(absPath)
|
||
diskFiles[relPath] = true
|
||
if !merkleMode && idx.IsStale(relPath) {
|
||
staleFiles = append(staleFiles, absPath)
|
||
}
|
||
}
|
||
|
||
// In Merkle mode the per-file mtime check is skipped; the stale set
|
||
// comes from a content-addressed tree diff over the whole repo,
|
||
// then intersected back down to the requested scope.
|
||
if merkleMode {
|
||
for _, abs := range idx.merkleStaleFiles(absRoot, diskFiles) {
|
||
rel, relErr := filepath.Rel(absRoot, abs)
|
||
if relErr != nil {
|
||
continue
|
||
}
|
||
if diskFiles[filepath.ToSlash(rel)] {
|
||
staleFiles = append(staleFiles, abs)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Deletion detection, bounded to the scoped subtree. A file tracked
|
||
// in fileMtimes that sits under one of the requested paths but is
|
||
// absent from this scoped discovery walk is a deletion candidate;
|
||
// the same stat-before-evict guard as IncrementalReindex applies so
|
||
// a newly-excluded or transiently-unreachable file is preserved.
|
||
idx.mtimeMu.RLock()
|
||
var candidates []string
|
||
for relPath := range idx.fileMtimes {
|
||
if diskFiles[relPath] {
|
||
continue
|
||
}
|
||
if relPathInScope(relPath, scopeRels) {
|
||
candidates = append(candidates, relPath)
|
||
}
|
||
}
|
||
idx.mtimeMu.RUnlock()
|
||
|
||
var deletedFiles []string
|
||
for _, relPath := range candidates {
|
||
absPath := filepath.Join(absRoot, filepath.FromSlash(relPath))
|
||
_, statErr := os.Stat(absPath)
|
||
if statErr == nil {
|
||
continue
|
||
}
|
||
if errors.Is(statErr, os.ErrNotExist) {
|
||
deletedFiles = append(deletedFiles, relPath)
|
||
continue
|
||
}
|
||
idx.logger.Warn("incremental reindex: stat failed during scoped deletion detection, preserving",
|
||
zap.String("rel", relPath), zap.Error(statErr))
|
||
}
|
||
|
||
for _, relPath := range deletedFiles {
|
||
graphPath := idx.prefixPath(relPath)
|
||
idx.restubIncomingRefs(graphPath)
|
||
idx.evictEnrichment(graphPath)
|
||
idx.graph.EvictFile(graphPath)
|
||
idx.mtimeMu.Lock()
|
||
delete(idx.fileMtimes, relPath)
|
||
idx.mtimeMu.Unlock()
|
||
}
|
||
// Prune the persisted mtime rows for deleted files too, so the next
|
||
// warm restart does not see them as phantom deletions (the in-memory
|
||
// delete above does not reach the store's sidecar table).
|
||
idx.pruneDeletedFileMtimes(deletedFiles)
|
||
|
||
// Re-index stale files with the same one-shot retry as the
|
||
// whole-root path — a file locked or mid-write when the walk caught
|
||
// it gets a second chance before landing on FailedFiles.
|
||
var failedFiles []string
|
||
for _, f := range staleFiles {
|
||
if err := idx.IndexFile(f); err != nil {
|
||
idx.logger.Debug("incremental reindex: failed to index file",
|
||
zap.String("file", f), zap.Error(err))
|
||
failedFiles = append(failedFiles, f)
|
||
}
|
||
}
|
||
if len(failedFiles) > 0 {
|
||
retry := failedFiles
|
||
failedFiles = nil
|
||
for _, f := range retry {
|
||
if err := idx.IndexFile(f); err != nil {
|
||
idx.logger.Warn("incremental reindex: file failed after retry",
|
||
zap.String("file", f), zap.Error(err))
|
||
failedFiles = append(failedFiles, f)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Re-infer interface implementations and re-run stub-call passes —
|
||
// eviction may have dropped edges. Skipped under deferGlobalPasses
|
||
// so a batch caller runs one global pass at the end.
|
||
if !idx.deferGlobalPasses && (len(staleFiles) > 0 || len(deletedFiles) > 0) {
|
||
// Scoped inference passes re-derive only the affected types/interfaces
|
||
// (add-parity with the full pass); fall back to whole-graph when
|
||
// scoping is disabled.
|
||
if !idx.runScopedInferencePasses(staleFiles) {
|
||
idx.resolver.InferImplements()
|
||
idx.resolver.InferOverrides()
|
||
}
|
||
// Capability (reads_env / executes_process / accesses_field) and
|
||
// framework-dispatch synthesis derive from code structure; skip them
|
||
// when the reconcile touched only non-code files (docs/config) and
|
||
// removed nothing — they cannot change any edge in that case, and
|
||
// eviction already handled any deletion. Same idempotent re-derive
|
||
// RunGlobalGraphPasses runs at full index.
|
||
if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) {
|
||
synthesizeCapabilityEdges(idx.graph)
|
||
resolver.RunFrameworkSynthesizers(idx.graph)
|
||
}
|
||
// Incremental: synthesize external calls only for the reindexed
|
||
// files (O(edited files)), not a full-graph recompute.
|
||
resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles))
|
||
}
|
||
|
||
// Skip the search-index rebuild on a zero-change reconcile when the
|
||
// backend already persists its search structures (the on-disk
|
||
// backend keeps its FTS index and vector embeddings on disk).
|
||
// buildSearchIndex re-reads every node (GetRepoNodes) and re-embeds
|
||
// them, then BulkUpsertEmbeddings re-writes the embedding rows. On a
|
||
// warm restart that work is pure recompute of already-persisted data.
|
||
// When nothing changed there is nothing to re-embed, so skip it
|
||
// entirely — the persisted index is authoritative. The in-memory
|
||
// backends (BM25 / Bleve) must still rebuild from the replayed
|
||
// snapshot, so they keep the unconditional path.
|
||
if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) {
|
||
idx.buildSearchIndex()
|
||
}
|
||
|
||
if len(staleFiles) > 0 || len(deletedFiles) > 0 {
|
||
idx.extractContracts()
|
||
idx.indexGen.Add(1) // files changed — invalidate the trigram cache
|
||
}
|
||
|
||
nodes, edges := idx.repoNodeEdgeCount()
|
||
result := &IndexResult{
|
||
NodeCount: nodes,
|
||
EdgeCount: edges,
|
||
FileCount: len(diskFiles),
|
||
StaleFileCount: len(staleFiles),
|
||
DeletedFileCount: len(deletedFiles),
|
||
FailedFiles: failedFiles,
|
||
DurationMs: time.Since(start).Milliseconds(),
|
||
}
|
||
idx.warnIfEdgeSanityViolated(result)
|
||
// An incremental pass that re-indexed or evicted at least one file did
|
||
// work — mark the repo for deferred semantic enrichment. A zero-change
|
||
// reconcile leaves the marker untouched so an unchanged repo is skipped.
|
||
if result.StaleFileCount > 0 || result.DeletedFileCount > 0 {
|
||
idx.pendingEnrich.Store(true)
|
||
}
|
||
// A re-parsed stale file's hover-enrichment edges were evicted with its old
|
||
// nodes, so force the deferred pass past the completion marker (a deletion
|
||
// evicts nodes but creates no fresh unstamped ones, so it alone does not).
|
||
if result.StaleFileCount > 0 {
|
||
idx.reparsedThisRun.Store(true)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// relPathInScope reports whether a repo-relative slash-path falls under
|
||
// any of the scoped paths — either an exact file match or anywhere
|
||
// inside a scoped directory.
|
||
func relPathInScope(relPath string, scope map[string]bool) bool {
|
||
if scope[relPath] {
|
||
return true
|
||
}
|
||
for s := range scope {
|
||
if s == "." {
|
||
return true
|
||
}
|
||
if strings.HasPrefix(relPath, s+"/") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// IncrementalReindex walks the file tree and re-indexes only files that changed
|
||
// since the last snapshot. It also evicts nodes for deleted files.
|
||
func (idx *Indexer) IncrementalReindex(root string) (*IndexResult, error) {
|
||
start := time.Now()
|
||
|
||
absRoot, err := filepath.Abs(root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
idx.storeRootPath(absRoot)
|
||
|
||
// Collect files currently on disk.
|
||
diskFiles := make(map[string]bool)
|
||
var staleFiles []string
|
||
|
||
// Merkle mode replaces per-file mtime staleness with a
|
||
// content-addressed Merkle-tree diff computed after the walk.
|
||
merkleMode := idx.merkleEnabled()
|
||
|
||
err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error {
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
if idx.shouldPruneDir(path, absRoot) {
|
||
return filepath.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
if _, ok := idx.effectiveLanguage(path, nil); !ok {
|
||
return nil
|
||
}
|
||
if idx.shouldExclude(path, absRoot, false) {
|
||
return nil
|
||
}
|
||
|
||
// relKey (slash + NFC) so the disk set is keyed identically
|
||
// to fileMtimes — otherwise a non-ASCII file the snapshot
|
||
// stored under one Unicode form and this walk observes under
|
||
// another would be seen as both deleted and newly created.
|
||
relPath := idx.relKey(path)
|
||
diskFiles[relPath] = true
|
||
|
||
if !merkleMode && idx.IsStale(relPath) {
|
||
staleFiles = append(staleFiles, path)
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// In Merkle mode the per-file mtime check above is skipped; the
|
||
// stale set comes from a content-addressed tree diff instead.
|
||
if merkleMode {
|
||
staleFiles = idx.merkleStaleFiles(absRoot, diskFiles)
|
||
}
|
||
|
||
// Detect deleted files. A file that's tracked in fileMtimes but
|
||
// absent from the current discovery walk is a candidate, but
|
||
// "absent from discovery" is not the same as "absent from disk":
|
||
//
|
||
// - The exclude list (.gortex.yaml, builtin, workspace) may have
|
||
// grown since the last index — every newly-excluded file would
|
||
// be classified as deleted.
|
||
// - A language extractor's Extensions() may have changed across
|
||
// versions — files whose ext is no longer detected would be
|
||
// classified as deleted.
|
||
// - WalkDir swallowed a transient error (EACCES, EIO, NFS hiccup,
|
||
// ELOOP) — the file is unreachable this pass but still on disk.
|
||
//
|
||
// All three would purge legitimate graph state on every daemon
|
||
// restart. Stat the candidate first: only treat ENOENT/ENOTDIR as
|
||
// deletion; preserve on success (file exists, just not discovered)
|
||
// and on transient errors. The cost is one extra stat per
|
||
// previously-indexed-but-not-discovered file, which is bounded by
|
||
// the size of the exclusion delta.
|
||
idx.mtimeMu.RLock()
|
||
var candidates []string
|
||
for relPath := range idx.fileMtimes {
|
||
if !diskFiles[relPath] {
|
||
candidates = append(candidates, relPath)
|
||
}
|
||
}
|
||
idx.mtimeMu.RUnlock()
|
||
|
||
var deletedFiles []string
|
||
for _, relPath := range candidates {
|
||
absPath := filepath.Join(absRoot, relPath)
|
||
_, err := os.Stat(absPath)
|
||
if err == nil {
|
||
// File exists on disk; it was excluded or its extension is
|
||
// no longer detected. Preserve.
|
||
continue
|
||
}
|
||
if errors.Is(err, os.ErrNotExist) {
|
||
deletedFiles = append(deletedFiles, relPath)
|
||
continue
|
||
}
|
||
// Transient error — preserve to be safe.
|
||
idx.logger.Warn("incremental reindex: stat failed during deletion detection, preserving",
|
||
zap.String("rel", relPath), zap.Error(err))
|
||
}
|
||
|
||
// Evict only files that are truly absent from disk.
|
||
for _, relPath := range deletedFiles {
|
||
graphPath := idx.prefixPath(relPath)
|
||
idx.restubIncomingRefs(graphPath)
|
||
idx.evictEnrichment(graphPath)
|
||
idx.graph.EvictFile(graphPath)
|
||
idx.mtimeMu.Lock()
|
||
delete(idx.fileMtimes, relPath)
|
||
idx.mtimeMu.Unlock()
|
||
}
|
||
// Prune the persisted mtime rows for deleted files too, so the next
|
||
// warm restart does not see them as phantom deletions (the in-memory
|
||
// delete above does not reach the store's sidecar table).
|
||
idx.pruneDeletedFileMtimes(deletedFiles)
|
||
|
||
// Re-index stale files. A file that fails — most often because it
|
||
// was locked or mid-write when the walk caught it — is collected
|
||
// and retried once below. A failure that survives the retry is
|
||
// surfaced on IndexResult.FailedFiles so the caller can replay it.
|
||
// A file whose bytes couldn't be read at all keeps no mtime
|
||
// recorded, so it stays stale for the next incremental pass; a
|
||
// file that read but failed to parse gets its mtime recorded (see
|
||
// indexFile's result==nil branch), so it stops being retried until
|
||
// its content changes again — see IndexResult.FailedFiles.
|
||
var failedFiles []string
|
||
for _, f := range staleFiles {
|
||
if err := idx.IndexFile(f); err != nil {
|
||
idx.logger.Debug("incremental reindex: failed to index file",
|
||
zap.String("file", f), zap.Error(err))
|
||
failedFiles = append(failedFiles, f)
|
||
}
|
||
}
|
||
if len(failedFiles) > 0 {
|
||
retry := failedFiles
|
||
failedFiles = nil
|
||
for _, f := range retry {
|
||
if err := idx.IndexFile(f); err != nil {
|
||
idx.logger.Warn("incremental reindex: file failed after retry",
|
||
zap.String("file", f), zap.Error(err))
|
||
failedFiles = append(failedFiles, f)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Re-infer interface implementations (edges may have been lost
|
||
// during eviction). Skipped under deferGlobalPasses so a batch
|
||
// caller (ReconcileAll, warmup) can run a single global pass at
|
||
// the end instead of paying O(global) per repo.
|
||
if !idx.deferGlobalPasses {
|
||
// Scoped inference passes re-derive only the affected types/interfaces
|
||
// (add-parity with the full pass); fall back to whole-graph when off.
|
||
if !idx.runScopedInferencePasses(staleFiles) {
|
||
idx.resolver.InferImplements()
|
||
idx.resolver.InferOverrides()
|
||
}
|
||
// Capability edges (reads_env / executes_process / accesses_field)
|
||
// and framework dynamic-dispatch synthesis are whole-graph recomputes
|
||
// that derive edges only from structural nodes in the changed files.
|
||
// When nothing structural changed — no stale code file (a doc/config
|
||
// edit or a true zero-change reconcile) and no deletion — they can
|
||
// produce no new edge, so re-running them is pure cost: the dominant
|
||
// waste in a no-op IncrementalReindex, which the periodic janitor runs
|
||
// per repo per tick. Gate them on the same predicate the path-scoped
|
||
// IncrementalReindexPaths already uses. Deletions still trigger a
|
||
// re-run because an evicted file may have been a dispatch endpoint.
|
||
if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) {
|
||
synthesizeCapabilityEdges(idx.graph)
|
||
resolver.RunFrameworkSynthesizers(idx.graph)
|
||
}
|
||
// External-call synthesis (opt-in) — file-scoped to the reindexed
|
||
// files (O(edited files)), not a full-graph recompute. Eviction
|
||
// already dropped a removed file's synthetic edges; a re-indexed
|
||
// file's fresh external terminals are re-materialised here.
|
||
resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles))
|
||
// Clone detection is not re-run here: each stale file was
|
||
// re-indexed through IndexFile above, whose resolve pass
|
||
// already recomputed EdgeSimilarTo against the fresh graph,
|
||
// and deleted files self-clean via EvictFile's bidirectional
|
||
// edge removal. Under deferGlobalPasses the batch caller runs
|
||
// the global clone pass once at the end.
|
||
}
|
||
|
||
// Rebuild search index to ensure consistency — but skip it on a
|
||
// zero-change reconcile against a backend that persists its search
|
||
// structures natively (the on-disk backend). See the matching guard
|
||
// in the other incremental path: re-embedding is wasted work and
|
||
// there is nothing to rebuild when no file changed.
|
||
if len(staleFiles) > 0 || len(deletedFiles) > 0 || !isSymbolSearcherBackend(idx.search) {
|
||
idx.buildSearchIndex()
|
||
}
|
||
|
||
// Update totalDetected so index_health reports correctly after cache restore.
|
||
if idx.totalDetected == 0 {
|
||
idx.totalDetected = len(diskFiles)
|
||
}
|
||
|
||
// Re-extract contracts only if stale files were re-indexed.
|
||
if len(staleFiles) > 0 || len(deletedFiles) > 0 {
|
||
idx.extractContracts()
|
||
idx.indexGen.Add(1) // files changed — invalidate the trigram cache
|
||
}
|
||
|
||
nodes, edges := idx.repoNodeEdgeCount()
|
||
result := &IndexResult{
|
||
NodeCount: nodes,
|
||
EdgeCount: edges,
|
||
FileCount: len(diskFiles),
|
||
StaleFileCount: len(staleFiles),
|
||
DeletedFileCount: len(deletedFiles),
|
||
FailedFiles: failedFiles,
|
||
DurationMs: time.Since(start).Milliseconds(),
|
||
}
|
||
idx.warnIfEdgeSanityViolated(result)
|
||
// An incremental pass that re-indexed or evicted at least one file did
|
||
// work — mark the repo for deferred semantic enrichment. A zero-change
|
||
// reconcile leaves the marker untouched so an unchanged repo is skipped.
|
||
if result.StaleFileCount > 0 || result.DeletedFileCount > 0 {
|
||
idx.pendingEnrich.Store(true)
|
||
}
|
||
// A re-parsed stale file's hover-enrichment edges were evicted with its old
|
||
// nodes, so force the deferred pass past the completion marker (a deletion
|
||
// evicts nodes but creates no fresh unstamped ones, so it alone does not).
|
||
if result.StaleFileCount > 0 {
|
||
idx.reparsedThisRun.Store(true)
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// LastIndexTime returns the timestamp of the last full index.
|
||
func (idx *Indexer) LastIndexTime() time.Time {
|
||
return idx.lastIndexTime
|
||
}
|
||
|
||
// TotalDetected returns the total number of files detected during the last full index.
|
||
func (idx *Indexer) TotalDetected() int {
|
||
return idx.totalDetected
|
||
}
|
||
|
||
// buildPerFileContractExtractors returns the set of extractors that
|
||
// operate on a single source file (everything except GoModExtractor,
|
||
// which runs once against go.mod at the repo root) plus a language →
|
||
// [extractors] map so callers can skip extractors whose
|
||
// SupportedLanguages() doesn't include a given file's language.
|
||
// Building the language map once avoids doing the string-membership
|
||
// check per file.
|
||
func (idx *Indexer) buildPerFileContractExtractors() ([]contracts.Extractor, map[string][]contracts.Extractor) {
|
||
extractors := []contracts.Extractor{
|
||
&contracts.HTTPExtractor{ClientAliases: idx.config.HTTPClientAliases},
|
||
&contracts.GRPCExtractor{},
|
||
&contracts.ThriftExtractor{},
|
||
&contracts.GraphQLExtractor{},
|
||
&contracts.TRPCExtractor{},
|
||
&contracts.OpenAPIExtractor{},
|
||
&contracts.TopicExtractor{},
|
||
&contracts.WebSocketExtractor{},
|
||
&contracts.NestMicroserviceExtractor{},
|
||
&contracts.EnvVarExtractor{},
|
||
}
|
||
// Config-driven event bus: only registered when the user declared
|
||
// boundaries (index.event_bus / CODEGRAPH_EVENT_CONFIG), so the default
|
||
// extractor set is unchanged.
|
||
if b := idx.eventBusBoundaries(); len(b) > 0 {
|
||
extractors = append(extractors, &contracts.EventBusExtractor{Boundaries: b})
|
||
}
|
||
byLang := make(map[string][]contracts.Extractor)
|
||
for _, ex := range extractors {
|
||
for _, lang := range ex.SupportedLanguages() {
|
||
byLang[lang] = append(byLang[lang], ex)
|
||
}
|
||
}
|
||
return extractors, byLang
|
||
}
|
||
|
||
// runContractExtractorsForFile applies the given extractors to a single
|
||
// file and returns the raw contracts (with RepoPrefix already set).
|
||
// Called both inline from parse workers and from the full-walk
|
||
// extractContracts path — they share the same per-file work.
|
||
func (idx *Indexer) runContractExtractorsForFile(
|
||
graphPath string,
|
||
src []byte,
|
||
fileNodes []*graph.Node,
|
||
fileEdges []*graph.Edge,
|
||
exts []contracts.Extractor,
|
||
tree *parser.ParseTree,
|
||
) []contracts.Contract {
|
||
if len(exts) == 0 {
|
||
return nil
|
||
}
|
||
// Contracts from synthetic test/bench fixtures are kept (so drift
|
||
// checks can flag a stale test pinned to an obsolete production
|
||
// contract) but tagged with is_test=true and a test_source
|
||
// category so the dashboard can filter them out by default.
|
||
testSource := fixtures.TestContractSource(graphPath)
|
||
var out []contracts.Contract
|
||
// The graph store backs graph-wide constant resolution for endpoint
|
||
// arguments (a route path / queue / topic referenced by a const). Resolved
|
||
// once per call; nil when the backend can't satisfy the reader (const
|
||
// dereference is then disabled and store-aware extractors degrade to their
|
||
// tree-aware behaviour).
|
||
var endpointStore contracts.EndpointConstStore
|
||
if es, ok := idx.graph.(contracts.EndpointConstStore); ok {
|
||
endpointStore = es
|
||
}
|
||
for _, ex := range exts {
|
||
var found []contracts.Contract
|
||
if sae, ok := ex.(contracts.StoreAwareExtractor); ok {
|
||
found = sae.ExtractWithStore(graphPath, src, fileNodes, fileEdges, tree, endpointStore, idx.repoPrefix)
|
||
} else if tae, ok := ex.(contracts.TreeAwareExtractor); ok && tree != nil {
|
||
found = tae.ExtractWithTree(graphPath, src, fileNodes, fileEdges, tree)
|
||
} else {
|
||
found = ex.Extract(graphPath, src, fileNodes, fileEdges)
|
||
}
|
||
for i := range found {
|
||
found[i].RepoPrefix = idx.repoPrefix
|
||
// Stamp the workspace / project slugs alongside the repo
|
||
// prefix so the matcher's boundary check has the data it
|
||
// needs without a second registry walk. Empty slugs
|
||
// default to RepoPrefix at Match time via
|
||
// Contract.EffectiveWorkspace / EffectiveProject.
|
||
if idx.workspaceID != "" {
|
||
found[i].WorkspaceID = idx.workspaceID
|
||
}
|
||
if idx.projectID != "" {
|
||
found[i].ProjectID = idx.projectID
|
||
}
|
||
if testSource != "" {
|
||
if found[i].Meta == nil {
|
||
found[i].Meta = map[string]any{}
|
||
}
|
||
found[i].Meta["is_test"] = true
|
||
found[i].Meta["test_source"] = testSource
|
||
}
|
||
}
|
||
out = append(out, found...)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// commitContracts writes contract nodes + provides/consumes edges for
|
||
// every contract in reg, and sets idx.contractRegistry to reg. Called
|
||
// once per index pass after all per-file contracts have been collected
|
||
// (inline from parse workers) plus go.mod has been processed.
|
||
func (idx *Indexer) commitContracts(reg *contracts.Registry) {
|
||
// Upgrade bare type names in contract Meta (e.g. "UserResp") to
|
||
// full symbol IDs (e.g. "pkg/resp.go::UserResp") now that the
|
||
// graph is complete. During extraction the enricher only saw
|
||
// the handler's file-scoped node list, so types declared in a
|
||
// sibling file stayed as bare names.
|
||
reg.UpgradeBareTypeRefs(func(name, repoHint string) []string {
|
||
matches := idx.graph.FindNodesByName(name)
|
||
var same, others []string
|
||
for _, n := range matches {
|
||
if n.Kind != graph.KindType {
|
||
continue
|
||
}
|
||
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
|
||
same = append(same, n.ID)
|
||
continue
|
||
}
|
||
others = append(others, n.ID)
|
||
}
|
||
if len(same) > 0 {
|
||
return same
|
||
}
|
||
return others
|
||
})
|
||
|
||
// Cross-file handler resolution. When a route is registered with
|
||
// a handler identifier that the file-scoped extractor couldn't
|
||
// resolve (`h.ServeArchive` in router.go wiring a method defined
|
||
// in archive_handler.go), the contract's SymbolID fell back to
|
||
// the enclosing router function and schema extraction ran
|
||
// against the router's body — which has every route's bindings
|
||
// piled on top of each other. Re-run enrichment with the
|
||
// correct per-handler scope now that the graph is complete.
|
||
idx.resolveProviderHandlers(reg)
|
||
|
||
// Cross-file route-prefix joining. A FastAPI APIRouter declared in
|
||
// one file (`router = APIRouter(prefix="/users")`) is mounted under
|
||
// a second prefix elsewhere (`app.include_router(router,
|
||
// prefix="/api")`), so a route declared `@router.get("/{id}")`
|
||
// belongs at /api/users/{id}, not /{id}. Rewrite the affected
|
||
// provider contract IDs to the joined path before the matcher pairs
|
||
// them with consumers. Also handles Express app.use mounts and
|
||
// NestJS @Controller class prefixes. Reads source straight off disk
|
||
// (cached per file) — the same access pattern resolveProviderHandlers
|
||
// uses for cross-file handler bodies.
|
||
//
|
||
// Mount sites (a main.py that only calls include_router) often carry
|
||
// no route contracts, so the scan-file set comes from the graph's
|
||
// py/ts/js file nodes, not the registry — but only when at least one
|
||
// prefix-eligible route contract exists, so non-FastAPI/Express/Nest
|
||
// repos pay nothing.
|
||
if scanFiles := idx.routerPrefixScanFiles(reg); len(scanFiles) > 0 {
|
||
srcCache := make(map[string][]byte)
|
||
contracts.JoinRouterPrefixes(reg, scanFiles, func(filePath string) []byte {
|
||
if data, ok := srcCache[filePath]; ok {
|
||
return data
|
||
}
|
||
data := idx.contractFileSrc(filePath)
|
||
srcCache[filePath] = data
|
||
return data
|
||
})
|
||
}
|
||
|
||
// Spring application(-profile)?.{yml,properties} config-key graph: emit the
|
||
// value-redacted config-key nodes and reads_config edges from the @Value /
|
||
// @ConfigurationProperties beans the Java extractor stamped. Cheap to skip
|
||
// on non-Spring repos (no config files + no stamped beans = no work).
|
||
contracts.BindSpringConfig(idx.graph, idx.contractFileSrc)
|
||
|
||
// Trace response variables back to their call-site return types.
|
||
// Handles `source, err := h.svc.Get(...)` → response_type is
|
||
// whatever `h.svc.Get` returns. The enricher can't do this
|
||
// without graph access; this pass reads each method's signature
|
||
// directly off the graph node, parses the first non-error
|
||
// return type, and resolves it to a symbol ID.
|
||
idx.resolveCallReturnTypes(reg)
|
||
|
||
// Snapshot field-level shapes for every type that's referenced as
|
||
// a contract's request / response body. This is Stage 2 — without
|
||
// per-field data Stage 3 (validation, breaking-change detection)
|
||
// has nothing to diff. We de-duplicate by symbol ID so heavy
|
||
// fan-in types (a User DTO used by 40 routes) only get parsed
|
||
// once per index pass.
|
||
idx.snapshotContractShapes(reg)
|
||
|
||
// Fold each type's snapshotted Shape into the envelope rows that
|
||
// reference it. The dashboard renders these rows as the response
|
||
// JSON shape (e.g. `{ workspace: { id: string }, repos: [{ name: string }] }`)
|
||
// instead of the bare type-symbol-ID, which answers nothing about
|
||
// the wire format.
|
||
idx.inlineEnvelopeShapes(reg)
|
||
|
||
all := reg.All()
|
||
nodes := make([]*graph.Node, 0, len(all))
|
||
edges := make([]*graph.Edge, 0, len(all))
|
||
for _, c := range all {
|
||
// dep::<module> nodes were materialised by extractGoModContracts
|
||
// before ResolveAll (so the import bridge could find them);
|
||
// re-emitting them here would PK-collide on backends whose bulk
|
||
// load is INSERT-only (the on-disk backend). The pre-pass is the single
|
||
// writer for that contract type.
|
||
if c.Type == contracts.ContractDependency {
|
||
continue
|
||
}
|
||
nodes = append(nodes, &graph.Node{
|
||
ID: c.ID,
|
||
Kind: graph.KindContract,
|
||
Name: c.ID,
|
||
FilePath: c.FilePath,
|
||
Language: "contract",
|
||
RepoPrefix: c.RepoPrefix,
|
||
WorkspaceID: c.EffectiveWorkspace(),
|
||
ProjectID: c.EffectiveProject(),
|
||
Meta: map[string]any{
|
||
"type": string(c.Type),
|
||
"role": string(c.Role),
|
||
"symbol_id": c.SymbolID,
|
||
"line": c.Line,
|
||
"confidence": c.Confidence,
|
||
"contract_meta": c.Meta,
|
||
},
|
||
})
|
||
|
||
if c.SymbolID == "" {
|
||
continue
|
||
}
|
||
edgeKind := graph.EdgeProvides
|
||
if c.Role == contracts.RoleConsumer {
|
||
edgeKind = graph.EdgeConsumes
|
||
}
|
||
edges = append(edges, &graph.Edge{
|
||
From: c.SymbolID,
|
||
To: c.ID,
|
||
Kind: edgeKind,
|
||
FilePath: c.FilePath,
|
||
Line: c.Line,
|
||
})
|
||
// Framework-layer EdgeHandlesRoute. Emitted alongside
|
||
// EdgeProvides for HTTP / gRPC / WS / GraphQL / topic
|
||
// providers so `analyze kind=routes` and other
|
||
// framework-aware tools walk one targeted edge instead
|
||
// of filtering EdgeProvides by contract type. Consumers
|
||
// (callers of routes) and non-route contract types (env,
|
||
// OpenAPI specs, DI tokens) intentionally skip this
|
||
// edge — they aren't route handlers.
|
||
if c.Role == contracts.RoleProvider && isRouteContractType(c.Type) {
|
||
edges = append(edges, &graph.Edge{
|
||
From: c.SymbolID,
|
||
To: c.ID,
|
||
Kind: graph.EdgeHandlesRoute,
|
||
FilePath: c.FilePath,
|
||
Line: c.Line,
|
||
Meta: map[string]any{
|
||
"contract_type": string(c.Type),
|
||
},
|
||
})
|
||
}
|
||
}
|
||
|
||
bulkStart := time.Now()
|
||
idx.bulkCommit(nodes, edges)
|
||
bulkElapsed := time.Since(bulkStart)
|
||
|
||
idx.contractRegistry = reg
|
||
repo := idx.rootPath
|
||
if idx.repoPrefix != "" {
|
||
repo = idx.repoPrefix
|
||
}
|
||
idx.logger.Info("contracts extracted",
|
||
zap.String("repo", repo),
|
||
zap.Int("count", len(all)),
|
||
zap.Duration("commit_bulk_elapsed", bulkElapsed))
|
||
}
|
||
|
||
// bulkCommit writes nodes + edges in one AddBatch call. The bulk
|
||
// load path is intentionally NOT used here: contract IDs often
|
||
// coincide with existing source-symbol IDs (a route handler shows
|
||
// up as both a Go function and an HTTP-contract anchor), and the
|
||
// on-disk backend's bulk load is INSERT-only on the node table so
|
||
// any collision fails the whole batch. AddBatch's non-bulk path
|
||
// upserts every row so duplicates are absorbed in place; the
|
||
// per-call cost is amortised by the chunked write path the backend
|
||
// uses internally.
|
||
func (idx *Indexer) bulkCommit(nodes []*graph.Node, edges []*graph.Edge) {
|
||
if len(nodes) == 0 && len(edges) == 0 {
|
||
return
|
||
}
|
||
idx.graph.AddBatch(nodes, edges)
|
||
}
|
||
|
||
// routerPrefixScanFiles returns the set of source files
|
||
// JoinRouterPrefixes must scan for router definitions and mount sites
|
||
// (APIRouter / include_router / app.use). Returns nil when no HTTP
|
||
// contract uses a prefix-joining framework (FastAPI / Express / NestJS),
|
||
// so unrelated repos skip the file enumeration entirely. When eligible,
|
||
// it enumerates py / ts / js / tsx / jsx file nodes from the graph — the
|
||
// mount file (a FastAPI main.py) frequently has no route contract of its
|
||
// own and so can't be discovered from the registry alone.
|
||
func (idx *Indexer) routerPrefixScanFiles(reg *contracts.Registry) []string {
|
||
eligible := false
|
||
for _, c := range reg.All() {
|
||
if c.Type != contracts.ContractHTTP || c.Meta == nil {
|
||
continue
|
||
}
|
||
switch fw, _ := c.Meta["framework"].(string); fw {
|
||
case "fastapi/flask", "express", "nestjs":
|
||
eligible = true
|
||
}
|
||
if eligible {
|
||
break
|
||
}
|
||
}
|
||
if !eligible {
|
||
return nil
|
||
}
|
||
|
||
var out []string
|
||
for _, n := range idx.graph.AllNodes() {
|
||
if n.Kind != graph.KindFile {
|
||
continue
|
||
}
|
||
path := n.FilePath
|
||
if path == "" {
|
||
path = n.ID
|
||
}
|
||
switch {
|
||
case strings.HasSuffix(path, ".py"),
|
||
strings.HasSuffix(path, ".ts"),
|
||
strings.HasSuffix(path, ".tsx"),
|
||
strings.HasSuffix(path, ".js"),
|
||
strings.HasSuffix(path, ".jsx"):
|
||
out = append(out, path)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// contractFileSrc reads the on-disk source for a contract FilePath
|
||
// (which is repo-prefixed when the indexer uses a repo prefix). Returns
|
||
// nil when the file can't be read. Mirrors the disk-resolution logic in
|
||
// resolveProviderHandlers so cross-file passes share one access pattern.
|
||
func (idx *Indexer) contractFileSrc(filePath string) []byte {
|
||
diskPath := filePath
|
||
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
|
||
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
|
||
}
|
||
diskPath = filepath.Join(idx.rootPath, diskPath)
|
||
data, err := os.ReadFile(diskPath)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
return data
|
||
}
|
||
|
||
// isRouteContractType reports whether a ContractType corresponds to a
|
||
// real network-route handler (HTTP / gRPC / WebSocket / GraphQL /
|
||
// topic). Used to gate EdgeHandlesRoute emission so the framework-layer
|
||
// edge stays focused on actual handlers and excludes env / OpenAPI /
|
||
// dependency / DI-token contracts that share the EdgeProvides edge but
|
||
// aren't routes in the agent-asks-which-handler-serves-X sense.
|
||
func isRouteContractType(t contracts.ContractType) bool {
|
||
switch t {
|
||
case contracts.ContractHTTP,
|
||
contracts.ContractGRPC,
|
||
contracts.ContractThrift,
|
||
contracts.ContractGraphQL,
|
||
contracts.ContractTopic,
|
||
contracts.ContractWS:
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// resolveProviderHandlers finds the actual handler for every HTTP
|
||
// provider contract whose per-file extraction couldn't resolve the
|
||
// handler identifier (typically routers in one file wiring handlers
|
||
// defined in sibling files). For each such contract:
|
||
//
|
||
// - Take Meta["handler_trail"] — the full expression between the
|
||
// HandleFunc parens, which carries every handler candidate
|
||
// (wrappers + inner handler). Fall back to "handler_ident"
|
||
// when no trail was captured (older contracts, simple consumer
|
||
// patterns).
|
||
// - Enumerate candidates in source order and look each up in the
|
||
// graph; take the innermost (last) one that resolves. That
|
||
// picks h.ServeArchive out of WithAuth(h.ServeArchive) instead
|
||
// of the WithAuth wrapper.
|
||
// - Re-run EnrichHTTPContract against the handler's file with the
|
||
// handler's line range so the enricher sees its actual body
|
||
// instead of the router's.
|
||
// - Drop `handler_ident` / `handler_trail` from meta afterwards —
|
||
// they were internal resolution hints.
|
||
func (idx *Indexer) resolveProviderHandlers(reg *contracts.Registry) {
|
||
type pending struct {
|
||
contractID string
|
||
trail string
|
||
fallback string
|
||
repoHint string
|
||
// srcDir is the directory of the contract's registration site
|
||
// (the file with the HandleFunc call). Used by lookupHandler
|
||
// as a tie-breaker when two same-repo functions share a name
|
||
// across packages — e.g. `Handler.handleContracts` in the
|
||
// `server` pkg vs `Server.handleContracts` in `mcp`. A
|
||
// `recv.method` call from inside `server/handler.go` resolves
|
||
// to the same-package method, not the cross-package one.
|
||
srcDir string
|
||
}
|
||
var todo []pending
|
||
for _, c := range reg.All() {
|
||
if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP {
|
||
continue
|
||
}
|
||
trail, _ := c.Meta["handler_trail"].(string)
|
||
fallback, _ := c.Meta["handler_ident"].(string)
|
||
if trail == "" && fallback == "" {
|
||
continue
|
||
}
|
||
// Skip contracts where schema is already populated — the
|
||
// initial file-scoped pass worked.
|
||
if src, _ := c.Meta["schema_source"].(string); src == "extracted" || src == "partial" {
|
||
continue
|
||
}
|
||
todo = append(todo, pending{
|
||
contractID: c.ID,
|
||
trail: trail,
|
||
fallback: fallback,
|
||
repoHint: c.RepoPrefix,
|
||
srcDir: filepath.Dir(c.FilePath),
|
||
})
|
||
}
|
||
// Always strip the internal handler hints from Meta at the end of
|
||
// this pass — successful or not. They were only ever intended as
|
||
// per-pass resolution scratchpad: when the cross-file lookup
|
||
// succeeds we delete them in the patched-contract loop below; when
|
||
// it fails (no candidate, ambiguous, etc.) they used to leak to
|
||
// the dashboard as values like `handler_trail: "/users", listUsers`
|
||
// — useless to a reader. This cleanup runs unconditionally so
|
||
// downstream consumers never see internal extractor state.
|
||
defer func() {
|
||
for _, c := range reg.All() {
|
||
if c.Meta == nil {
|
||
continue
|
||
}
|
||
if _, hasIdent := c.Meta["handler_ident"]; !hasIdent {
|
||
if _, hasTrail := c.Meta["handler_trail"]; !hasTrail {
|
||
continue
|
||
}
|
||
}
|
||
items := reg.ByID(c.ID)
|
||
for i := range items {
|
||
if items[i].Meta == nil {
|
||
continue
|
||
}
|
||
delete(items[i].Meta, "handler_ident")
|
||
delete(items[i].Meta, "handler_trail")
|
||
}
|
||
reg.ReplaceByID(c.ID, items)
|
||
}
|
||
}()
|
||
|
||
if len(todo) == 0 {
|
||
return
|
||
}
|
||
|
||
// Cache file source + node list per file path — a single router
|
||
// often refers to dozens of handlers in the same sibling file.
|
||
fileSrc := make(map[string][]byte)
|
||
fileNodes := make(map[string][]*graph.Node)
|
||
|
||
// fileTrees caches per-file ParseTree handles parsed lazily below
|
||
// so a router referencing many handlers in the same sibling file
|
||
// only parses that file once.
|
||
fileTrees := make(map[string]*parser.ParseTree)
|
||
defer func() {
|
||
for _, t := range fileTrees {
|
||
t.Release()
|
||
}
|
||
}()
|
||
resolved := 0
|
||
for _, p := range todo {
|
||
handlerNode := idx.resolveInnermostHandler(p.trail, p.fallback, p.repoHint, p.srcDir)
|
||
if handlerNode == nil {
|
||
continue
|
||
}
|
||
src, ok := fileSrc[handlerNode.FilePath]
|
||
if !ok {
|
||
diskPath := handlerNode.FilePath
|
||
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
|
||
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
|
||
}
|
||
diskPath = filepath.Join(idx.rootPath, diskPath)
|
||
data, err := os.ReadFile(diskPath)
|
||
if err != nil {
|
||
fileSrc[handlerNode.FilePath] = nil
|
||
continue
|
||
}
|
||
fileSrc[handlerNode.FilePath] = data
|
||
src = data
|
||
}
|
||
if src == nil {
|
||
continue
|
||
}
|
||
nodes, ok := fileNodes[handlerNode.FilePath]
|
||
if !ok {
|
||
nodes = idx.graph.GetFileNodes(handlerNode.FilePath)
|
||
fileNodes[handlerNode.FilePath] = nodes
|
||
}
|
||
|
||
lang := detectLangFromPath(handlerNode.FilePath)
|
||
tree, treeReady := fileTrees[handlerNode.FilePath]
|
||
if !treeReady {
|
||
tree = contracts.ParseTreeForLang(lang, src)
|
||
fileTrees[handlerNode.FilePath] = tree
|
||
}
|
||
|
||
// Re-run enrichment. EnrichHTTPContractWithTree reads the
|
||
// contract's SymbolID to locate the handler body range — swap
|
||
// it in temporarily to the resolved handler so the lookup
|
||
// works. With a tree the AST overlay runs after the regex
|
||
// pass and overrides Meta keys it can confidently produce.
|
||
matches := reg.ByID(p.contractID)
|
||
if len(matches) == 0 {
|
||
continue
|
||
}
|
||
for i, c := range matches {
|
||
if c.Role != contracts.RoleProvider {
|
||
continue
|
||
}
|
||
// Operate on a copy; Registry entries are values.
|
||
patched := c
|
||
patched.SymbolID = handlerNode.ID
|
||
patched.FilePath = handlerNode.FilePath
|
||
if patched.Meta == nil {
|
||
patched.Meta = map[string]any{}
|
||
}
|
||
// Drop prior path_params so the enricher's fresh pass
|
||
// repopulates consistently (path hasn't changed, but we
|
||
// want the call-path to be identical to Stage 1).
|
||
lines := splitLines(src)
|
||
contracts.EnrichHTTPContractWithTree(&patched, lines, nodes, lang, tree)
|
||
delete(patched.Meta, "handler_ident")
|
||
delete(patched.Meta, "handler_trail")
|
||
matches[i] = patched
|
||
resolved++
|
||
}
|
||
// Write back the mutated set. The registry doesn't have an
|
||
// "update" API; we use AddAll semantics via Set-like
|
||
// operations. Simpler: clear then re-add all roles to this ID.
|
||
reg.ReplaceByID(p.contractID, matches)
|
||
}
|
||
if resolved > 0 {
|
||
idx.logger.Info("resolved cross-file provider handlers",
|
||
zap.Int("count", resolved),
|
||
zap.Int("considered", len(todo)))
|
||
}
|
||
}
|
||
|
||
// resolveInnermostHandler picks the innermost handler candidate from
|
||
// the call trail that resolves to a real function or method in the
|
||
// graph. Walks candidates in source order and keeps the LAST
|
||
// successful lookup — for `WithAuth(h.ServeArchive)` that's
|
||
// `h.ServeArchive`, not the `WithAuth` wrapper. Falls back to the
|
||
// single identifier when no trail is available (e.g. simple bare
|
||
// `r.GET("/x", listUsers)` patterns).
|
||
func (idx *Indexer) resolveInnermostHandler(trail, fallback, repoHint, srcDir string) *graph.Node {
|
||
candidates := contracts.HandlerCandidatesInTrail(trail)
|
||
if len(candidates) == 0 && fallback != "" {
|
||
candidates = []string{fallback}
|
||
}
|
||
var best *graph.Node
|
||
for _, c := range candidates {
|
||
if n := idx.lookupHandler(c, repoHint, srcDir); n != nil {
|
||
best = n
|
||
}
|
||
}
|
||
return best
|
||
}
|
||
|
||
// lookupHandler maps a raw identifier from a route pattern to the
|
||
// graph node for the handler function / method.
|
||
//
|
||
// - "h.ServeArchive" → method named "ServeArchive", prefer same repo.
|
||
// - "ServeArchive" → function or method of that name.
|
||
// - "pkg.Foo" → same as first form, package-qualified call.
|
||
//
|
||
// Returns nil when no candidate resolves unambiguously.
|
||
func (idx *Indexer) lookupHandler(ident, repoHint, srcDir string) *graph.Node {
|
||
// Strip a leading receiver / package qualifier — "h.ServeArchive"
|
||
// → "ServeArchive".
|
||
name := ident
|
||
if i := strings.LastIndex(name, "."); i >= 0 {
|
||
name = name[i+1:]
|
||
}
|
||
if name == "" {
|
||
return nil
|
||
}
|
||
candidates := idx.graph.FindNodesByName(name)
|
||
if len(candidates) == 0 {
|
||
return nil
|
||
}
|
||
var sameRepo, other []*graph.Node
|
||
for _, n := range candidates {
|
||
if n.Kind != graph.KindFunction && n.Kind != graph.KindMethod {
|
||
continue
|
||
}
|
||
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
|
||
sameRepo = append(sameRepo, n)
|
||
continue
|
||
}
|
||
other = append(other, n)
|
||
}
|
||
if len(sameRepo) == 1 {
|
||
return sameRepo[0]
|
||
}
|
||
if len(sameRepo) == 0 && len(other) == 1 {
|
||
return other[0]
|
||
}
|
||
// Multiple candidates — try same-package tie-break before giving up.
|
||
// A `recv.method` call inside `pkg/foo.go` resolves to a method
|
||
// declared in the same package; cross-package lookalikes (e.g.
|
||
// `Server.handleContracts` in `mcp` vs `Handler.handleContracts`
|
||
// in `server`) are filtered out. Without this, both routers and
|
||
// MCP-side handlers compete for the same name and the resolver
|
||
// falls back to the enclosing function (`registerRoutes`).
|
||
if srcDir != "" {
|
||
pool := sameRepo
|
||
if len(pool) == 0 {
|
||
pool = other
|
||
}
|
||
var samePkg []*graph.Node
|
||
for _, n := range pool {
|
||
if filepath.Dir(n.FilePath) == srcDir {
|
||
samePkg = append(samePkg, n)
|
||
}
|
||
}
|
||
if len(samePkg) == 1 {
|
||
return samePkg[0]
|
||
}
|
||
}
|
||
return nil // ambiguous
|
||
}
|
||
|
||
func splitLines(src []byte) []string {
|
||
return strings.Split(string(src), "\n")
|
||
}
|
||
|
||
// detectLangFromPath mirrors internal/contracts.detectLanguage so the
|
||
// enricher's language-gate fires correctly for the handler's own file.
|
||
func detectLangFromPath(path string) string {
|
||
switch {
|
||
case strings.HasSuffix(path, ".go"):
|
||
return "go"
|
||
case strings.HasSuffix(path, ".ts"), strings.HasSuffix(path, ".tsx"):
|
||
return "typescript"
|
||
case strings.HasSuffix(path, ".js"), strings.HasSuffix(path, ".jsx"):
|
||
return "javascript"
|
||
case strings.HasSuffix(path, ".py"):
|
||
return "python"
|
||
case strings.HasSuffix(path, ".java"):
|
||
return "java"
|
||
case strings.HasSuffix(path, ".kt"), strings.HasSuffix(path, ".kts"):
|
||
return "kotlin"
|
||
case strings.HasSuffix(path, ".dart"):
|
||
return "dart"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// responseHelperCallRe pulls the third argument out of a JSON-response
|
||
// helper call, e.g. `respondJSON(w, http.StatusOK, source)` → "source",
|
||
// `WriteJSON(w, 200, &result)` → "result". Matches every helper name
|
||
// the Go enricher knows about so the two pipes stay in sync.
|
||
var responseHelperCallRe = regexp.MustCompile(
|
||
`(?:[A-Za-z_]\w*\.)?(?:[Rr]espond|[Ww]rite|[Ss]end|[Rr]ender)(?:JSON|Json)\(\s*\w+\s*,\s*[^,]+?\s*,\s*&?([A-Za-z_]\w*)\s*\)`,
|
||
)
|
||
|
||
// receiverMatchesHint decides whether a method node could plausibly
|
||
// be the target of a call whose receiver chain includes `hint` as
|
||
// its penultimate segment. For `h.tucks.Update`, the hint is
|
||
// "tucks" and we accept receivers whose name (stripped of pointer
|
||
// marker) contains "tucks" case-insensitively:
|
||
//
|
||
// *TucksStore.Update ✓ (receiver "TucksStore" contains "tucks")
|
||
// *PostgresTuckStore.Update ✓ (contains "tuck")
|
||
// *EmailSources.Update ✗ (no "tucks")
|
||
//
|
||
// The hint may itself be the receiver variable (`h` in `h.Update(...)`)
|
||
// when the call has only two segments; in that case any same-repo
|
||
// method named `Update` passes — but the upstream `len(matches) != 1`
|
||
// check still demands uniqueness, which is the real guard.
|
||
func receiverMatchesHint(n *graph.Node, hint string) bool {
|
||
if hint == "" {
|
||
return true
|
||
}
|
||
// Method ID looks like "<repo>/<file>::Receiver.Method". Extract
|
||
// "Receiver" by splitting once on `::` then taking the type part
|
||
// before the last `.`.
|
||
idParts := strings.Split(n.ID, "::")
|
||
if len(idParts) < 2 {
|
||
return true // conservative: no receiver info available → don't filter out
|
||
}
|
||
last := idParts[len(idParts)-1]
|
||
dot := strings.LastIndex(last, ".")
|
||
if dot < 0 {
|
||
return true // plain function, no receiver
|
||
}
|
||
recv := strings.TrimPrefix(last[:dot], "*")
|
||
// Handle both singular and plural forms: "tucks" in hint matches
|
||
// "TuckStore" by containing "tuck", and "tuck" hint matches
|
||
// "Tucks" too. Strip a trailing `s` from the longer side to let
|
||
// singular/plural pairs match.
|
||
return strings.Contains(strings.ToLower(recv), strings.ToLower(hint)) ||
|
||
strings.Contains(strings.ToLower(recv), strings.ToLower(strings.TrimSuffix(hint, "s"))) ||
|
||
strings.Contains(strings.ToLower(recv), strings.ToLower(hint+"s"))
|
||
}
|
||
|
||
// parseFirstNonErrorReturnType walks a Go function signature and
|
||
// returns the first return type that isn't `error`. Signatures as
|
||
// stored in the graph have the form:
|
||
//
|
||
// func ((s *Store)) Get(args) (*EmailSource, error)
|
||
// func list() []*User
|
||
// func save(x Foo) error
|
||
//
|
||
// Regex-based extraction struggles with the receiver's `((...))`
|
||
// nesting and with multi-paren return groups — we parse with an
|
||
// explicit bracket-depth counter so every shape above is handled
|
||
// the same way.
|
||
|
||
// resolveCallReturnTypes is the graph-aware companion to the
|
||
// regex-based schema enricher. For every HTTP provider contract whose
|
||
// response couldn't be pinned syntactically (response_expr is set,
|
||
// response_type is empty), we:
|
||
//
|
||
// - Pull the bound variable's name out of the helper-call expression.
|
||
// - Read the handler's body from disk.
|
||
// - Find the variable's declaration line and parse the RHS call.
|
||
// - Look up the called method by name in the graph (preferring the
|
||
// same file / same repo).
|
||
// - Parse the method's signature meta for the first non-error
|
||
// return type, strip `*` / `[]`, resolve to a type node's ID.
|
||
// - Patch the contract's meta in place.
|
||
//
|
||
// This is the proper tracing the name-based heuristic only
|
||
// approximated — it follows the variable to its definition instead
|
||
// of guessing from its name.
|
||
func (idx *Indexer) resolveCallReturnTypes(reg *contracts.Registry) {
|
||
resolved := 0
|
||
bfCache := newBodyFactsCache(idx)
|
||
defer bfCache.Close()
|
||
|
||
for _, c := range reg.All() {
|
||
if c.Role != contracts.RoleProvider || c.Type != contracts.ContractHTTP {
|
||
continue
|
||
}
|
||
|
||
handler := idx.graph.GetNode(c.SymbolID)
|
||
bf := bfCache.For(handler)
|
||
|
||
// Path 1: bare-variable response (`return WriteJSON(w, code, resp)`).
|
||
// Trace the variable to its binding call's return type — and,
|
||
// failing that, the literal/builtin shape of its declaration —
|
||
// then stamp response_type / response_repeated. Accepts
|
||
// response_expr in two forms:
|
||
// - Bare identifier ("result") — emitted by the
|
||
// AST overlay (or the post-fix Go enricher) when the value
|
||
// is a plain var.
|
||
// - Full helper call ("WriteJSON(w, …)") — older
|
||
// extraction output, kept compatible by extracting the
|
||
// third arg via responseHelperCallRe.
|
||
if rt, _ := c.Meta["response_type"].(string); rt == "" {
|
||
respExpr, _ := c.Meta["response_expr"].(string)
|
||
varName := ""
|
||
switch {
|
||
case respExpr == "":
|
||
// nothing to work with
|
||
case isLikelyIdentifier(respExpr):
|
||
varName = respExpr
|
||
default:
|
||
if m := responseHelperCallRe.FindStringSubmatch(respExpr); len(m) >= 2 {
|
||
varName = m[1]
|
||
}
|
||
}
|
||
if varName != "" {
|
||
typeID, repeated := idx.lookupVarTypeForContract(c, bf, varName)
|
||
if typeID != "" {
|
||
items := reg.ByID(c.ID)
|
||
changed := false
|
||
for i := range items {
|
||
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
|
||
continue
|
||
}
|
||
if items[i].Meta == nil {
|
||
items[i].Meta = map[string]any{}
|
||
}
|
||
items[i].Meta["response_type"] = typeID
|
||
if repeated {
|
||
items[i].Meta["response_repeated"] = true
|
||
}
|
||
items[i].Meta["schema_source"] = "extracted"
|
||
delete(items[i].Meta, "response_expr")
|
||
changed = true
|
||
}
|
||
if changed {
|
||
reg.ReplaceByID(c.ID, items)
|
||
resolved++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Path 2: envelope response (`map[string]any{"workspace": ws,
|
||
// "repos": repos}`). For each row that didn't resolve a type
|
||
// syntactically, trace its expression to a binding call's
|
||
// return type and patch the row in place. Pulled out as a
|
||
// separate pass so a contract whose top-level response_type
|
||
// stays unresolvable can still get per-field signal — which is
|
||
// the whole point of the envelope view.
|
||
envRaw, ok := c.Meta["response_envelope"].([]map[string]any)
|
||
if !ok {
|
||
continue
|
||
}
|
||
if !envelopeNeedsResolution(envRaw) {
|
||
continue
|
||
}
|
||
envChanged := false
|
||
for ri := range envRaw {
|
||
if t, _ := envRaw[ri]["type"].(string); t != "" {
|
||
continue
|
||
}
|
||
expr, _ := envRaw[ri]["expr"].(string)
|
||
if expr == "" {
|
||
continue
|
||
}
|
||
// Strip a leading `&` / `*` so the binding lookup sees
|
||
// the underlying identifier.
|
||
ident := strings.TrimLeft(expr, "&*")
|
||
if !isLikelyIdentifier(ident) {
|
||
continue
|
||
}
|
||
typeID, repeated := idx.lookupVarTypeForContract(c, bf, ident)
|
||
if typeID != "" {
|
||
envRaw[ri]["type"] = typeID
|
||
if repeated {
|
||
envRaw[ri]["repeated"] = true
|
||
}
|
||
envChanged = true
|
||
}
|
||
}
|
||
if !envChanged {
|
||
continue
|
||
}
|
||
// Promote schema_source to "extracted" if every row now has a
|
||
// type (or this is a single-key envelope whose lone field
|
||
// resolved). Otherwise leave it as "partial" — we have more
|
||
// info than before but it's not exhaustive.
|
||
items := reg.ByID(c.ID)
|
||
patched := false
|
||
for i := range items {
|
||
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
|
||
continue
|
||
}
|
||
if items[i].Meta == nil {
|
||
items[i].Meta = map[string]any{}
|
||
}
|
||
items[i].Meta["response_envelope"] = envRaw
|
||
if envelopeFullyTyped(envRaw) {
|
||
items[i].Meta["schema_source"] = "extracted"
|
||
}
|
||
patched = true
|
||
}
|
||
if patched {
|
||
reg.ReplaceByID(c.ID, items)
|
||
resolved++
|
||
}
|
||
}
|
||
if resolved > 0 {
|
||
idx.logger.Info("resolved response types from call signatures",
|
||
zap.Int("count", resolved))
|
||
}
|
||
}
|
||
|
||
func envelopeNeedsResolution(env []map[string]any) bool {
|
||
for _, row := range env {
|
||
if t, _ := row["type"].(string); t == "" {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func envelopeFullyTyped(env []map[string]any) bool {
|
||
if len(env) == 0 {
|
||
return false
|
||
}
|
||
for _, row := range env {
|
||
if t, _ := row["type"].(string); t == "" {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// upgradeBareTypeName looks up `name` in the graph and returns the
|
||
// matching type node's ID, preferring same-repo matches. Falls back
|
||
// to the input string when no graph type is found, so callers can
|
||
// still surface primitives ("string", "int", "bool") and external
|
||
// types ("map[string]int") that have no graph node. The shape-
|
||
// inlining pass will leave those as-is since lookupShape requires a
|
||
// `::` separator.
|
||
func (idx *Indexer) upgradeBareTypeName(name, repoHint string) string {
|
||
if name == "" {
|
||
return name
|
||
}
|
||
if strings.Contains(name, "::") {
|
||
return name // already a graph ID
|
||
}
|
||
candidates := idx.graph.FindNodesByName(name)
|
||
var fallback *graph.Node
|
||
for _, n := range candidates {
|
||
if n.Kind != graph.KindType {
|
||
continue
|
||
}
|
||
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
|
||
return n.ID
|
||
}
|
||
if fallback == nil {
|
||
fallback = n
|
||
}
|
||
}
|
||
if fallback != nil {
|
||
return fallback.ID
|
||
}
|
||
return name
|
||
}
|
||
|
||
// isLikelyIdentifier accepts the bare-identifier and dotted-path
|
||
// forms that traceVarTypeFromBody can match against a binding line.
|
||
// Compound expressions ("len(repos)", "&Foo{}") are out of scope —
|
||
// they'd need a more thorough RHS parser than the regex chain here.
|
||
func isLikelyIdentifier(s string) bool {
|
||
if s == "" {
|
||
return false
|
||
}
|
||
for i, r := range s {
|
||
switch {
|
||
case r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z'):
|
||
continue
|
||
case i > 0 && r >= '0' && r <= '9':
|
||
continue
|
||
case r == '.' && i > 0:
|
||
continue
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// lookupVarTypeForContract resolves a variable to its return type
|
||
// using BodyFacts (AST-driven, structurally correct). Returns
|
||
// (typeID, repeated) or ("", false) when the binding can't be
|
||
// resolved.
|
||
//
|
||
// AST-only: phase 1b deleted the body-text regex fallback. Languages
|
||
// without a BodyFactsFactory get nopBodyFacts (which returns empty
|
||
// Bindings), so this function is a no-op for non-Go contracts.
|
||
// Their per-file regex enricher in schema_enrich_<lang>.go still runs
|
||
// and populates request_type / response_type via the framework
|
||
// detectors — only the post-pass cross-handler trace is AST-only.
|
||
func (idx *Indexer) lookupVarTypeForContract(
|
||
c contracts.Contract,
|
||
bf contracts.BodyFacts,
|
||
varName string,
|
||
) (string, bool) {
|
||
if bf == nil {
|
||
return "", false
|
||
}
|
||
b := bf.VarBinding(varName)
|
||
// Highest tier: BindingResolver (go/types via goanalysis.Provider)
|
||
// returns compiler-resolved types. When --semantic is enabled and
|
||
// the provider has run, this is authoritative for any binding
|
||
// whose source line we tracked.
|
||
if br := contracts.CurrentBindingResolver(); br != nil && b.Line > 0 {
|
||
if typeName, ok := br.LookupTypeAtLine(c.FilePath, b.Line); ok && typeName != "" {
|
||
return idx.upgradeBareTypeName(typeName, c.RepoPrefix), b.Repeated
|
||
}
|
||
}
|
||
switch b.Kind {
|
||
case contracts.BindingMethodCall, contracts.BindingFuncCall:
|
||
if b.CallExpr != "" {
|
||
if typeID, repeated, _ := idx.resolveCallExprToType(b.CallExpr, c.RepoPrefix); typeID != "" {
|
||
return typeID, repeated
|
||
}
|
||
}
|
||
default:
|
||
// Composite / slice / map / literal / path-value /
|
||
// header-value / form-value / query-get — already typed.
|
||
if b.TypeID != "" {
|
||
return idx.upgradeBareTypeName(b.TypeID, c.RepoPrefix), b.Repeated
|
||
}
|
||
}
|
||
return "", false
|
||
}
|
||
|
||
// resolveCallExprToType walks the graph from a call expression like
|
||
// `h.svc.GetRepos` to the called method/function's first non-error
|
||
// return type, returning (typeID, repeated, pointer). Empty string
|
||
// when the call is ambiguous (multiple candidates with disagreeing
|
||
// signatures) or the receiver doesn't match any graph node.
|
||
//
|
||
// Extracted from traceVarTypeFromBodyWithShape so BodyFacts-driven
|
||
// callers can reuse the graph walk without going through the regex
|
||
// path. The regex-driven traceVarTypeFromBodyWithShape is the
|
||
// fallback for non-Go languages until they ship a BodyFacts
|
||
// implementation.
|
||
func (idx *Indexer) resolveCallExprToType(callExpr, repoHint string) (string, bool, bool) {
|
||
if callExpr == "" {
|
||
return "", false, false
|
||
}
|
||
// Split the call path. `h.tucks.Update` → ["h", "tucks", "Update"].
|
||
// The last segment is the method name; the penultimate is the
|
||
// receiver field / package, which we use to disambiguate when
|
||
// multiple methods share the name.
|
||
parts := strings.Split(callExpr, ".")
|
||
methodName := parts[len(parts)-1]
|
||
if methodName == "" {
|
||
return "", false, false
|
||
}
|
||
var receiverHint string
|
||
if len(parts) >= 2 {
|
||
receiverHint = parts[len(parts)-2]
|
||
}
|
||
|
||
candidates := idx.graph.FindNodesByName(methodName)
|
||
var matches []*graph.Node
|
||
for _, n := range candidates {
|
||
if n.Kind != graph.KindMethod && n.Kind != graph.KindFunction {
|
||
continue
|
||
}
|
||
if repoHint != "" && !strings.HasPrefix(n.ID, repoHint+"/") {
|
||
continue
|
||
}
|
||
if receiverHint != "" && !receiverMatchesHint(n, receiverHint) {
|
||
continue
|
||
}
|
||
matches = append(matches, n)
|
||
}
|
||
if len(matches) == 0 {
|
||
return "", false, false
|
||
}
|
||
|
||
// Interface + implementation stacks often produce multiple
|
||
// receivers that share the same method signature — a production
|
||
// postgres store and a mock test store both implement
|
||
// `emailSources.Update(...) (*EmailSource, error)`. Parse every
|
||
// candidate's signature; if they all agree on the first
|
||
// non-error return type, use that. Otherwise bail so we don't
|
||
// attribute a wrong type silently.
|
||
var retType string
|
||
for _, m := range matches {
|
||
if m.Meta == nil {
|
||
continue
|
||
}
|
||
sig, _ := m.Meta["signature"].(string)
|
||
t := parseFirstNonErrorReturnType(sig)
|
||
if t == "" {
|
||
continue
|
||
}
|
||
// De-prioritise mock receivers: if a non-mock candidate
|
||
// later disagrees we want that to be the authoritative one.
|
||
if retType == "" {
|
||
retType = t
|
||
continue
|
||
}
|
||
if t != retType {
|
||
// Candidates disagree — can't tell which wins. The
|
||
// caller sees the raw expression and can drill in.
|
||
return "", false, false
|
||
}
|
||
}
|
||
if retType == "" {
|
||
return "", false, false
|
||
}
|
||
// Capture slice/pointer flags before stripping so the caller can
|
||
// render `[Foo]` / `*Foo` correctly. Order matters: a return type
|
||
// like `[]*Foo` is reported as repeated AND pointer.
|
||
repeated := strings.HasPrefix(retType, "[]")
|
||
pointer := strings.HasPrefix(retType, "*") ||
|
||
(repeated && strings.HasPrefix(retType[2:], "*"))
|
||
// Strip `*` / `[]` / package qualifier so resolveTypeByName can
|
||
// match the plain type-node name.
|
||
retType = strings.TrimLeft(retType, "*[]")
|
||
if dot := strings.LastIndex(retType, "."); dot >= 0 {
|
||
retType = retType[dot+1:]
|
||
}
|
||
// Look up the type node, preferring same-repo matches.
|
||
typeCandidates := idx.graph.FindNodesByName(retType)
|
||
var bestType *graph.Node
|
||
for _, n := range typeCandidates {
|
||
if n.Kind != graph.KindType {
|
||
continue
|
||
}
|
||
if repoHint != "" && strings.HasPrefix(n.ID, repoHint+"/") {
|
||
bestType = n
|
||
break
|
||
}
|
||
if bestType == nil {
|
||
bestType = n
|
||
}
|
||
}
|
||
if bestType != nil {
|
||
return bestType.ID, repeated, pointer
|
||
}
|
||
// Bare name — downstream UpgradeBareTypeRefs can still upgrade
|
||
// it later, but we return it as-is so the consumer sees something
|
||
// real.
|
||
return retType, repeated, pointer
|
||
}
|
||
|
||
func parseFirstNonErrorReturnType(sig string) string {
|
||
sig = strings.TrimSpace(sig)
|
||
if !strings.HasPrefix(sig, "func") {
|
||
return ""
|
||
}
|
||
sig = strings.TrimSpace(strings.TrimPrefix(sig, "func"))
|
||
|
||
// Optional receiver. Two forms to recognise:
|
||
// `((*Recv)) Name(params)` — gortex's stored double-paren form
|
||
// `(r *Recv) Name(params)` — standard Go source form
|
||
// In both cases a function name follows the receiver parens.
|
||
// Anonymous function types (`func(a, b) (c, d)`) have no
|
||
// receiver — the first `(` opens the parameter list and is
|
||
// followed by another `(` for the return group or end-of-string.
|
||
// Disambiguate by peeking past the first balanced `(...)` group
|
||
// for an identifier letter.
|
||
if strings.HasPrefix(sig, "(") {
|
||
end := findBalancedParenEnd(sig)
|
||
if end < 0 {
|
||
return ""
|
||
}
|
||
afterFirstGroup := strings.TrimSpace(sig[end+1:])
|
||
if len(afterFirstGroup) > 0 {
|
||
r := afterFirstGroup[0]
|
||
isIdent := (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_'
|
||
if isIdent {
|
||
sig = afterFirstGroup
|
||
}
|
||
}
|
||
}
|
||
|
||
// Skip the function name — everything up to the parameter list's
|
||
// opening `(`.
|
||
if i := strings.Index(sig, "("); i >= 0 {
|
||
sig = sig[i:]
|
||
} else {
|
||
return ""
|
||
}
|
||
|
||
// Skip parameter list.
|
||
end := findBalancedParenEnd(sig)
|
||
if end < 0 {
|
||
return ""
|
||
}
|
||
sig = strings.TrimSpace(sig[end+1:])
|
||
if sig == "" {
|
||
return ""
|
||
}
|
||
|
||
// Return clause — either `(T1, T2, ...)` or a bare single type.
|
||
var inner string
|
||
if strings.HasPrefix(sig, "(") {
|
||
end := findBalancedParenEnd(sig)
|
||
if end < 0 {
|
||
return ""
|
||
}
|
||
inner = sig[1:end]
|
||
} else {
|
||
inner = sig
|
||
}
|
||
return firstNonErrorReturnField(inner)
|
||
}
|
||
|
||
// findBalancedParenEnd returns the index of the `)` that closes the
|
||
// `(` at s[0]. Returns -1 when the parens don't balance.
|
||
func findBalancedParenEnd(s string) int {
|
||
if len(s) == 0 || s[0] != '(' {
|
||
return -1
|
||
}
|
||
depth := 0
|
||
for i := 0; i < len(s); i++ {
|
||
switch s[i] {
|
||
case '(':
|
||
depth++
|
||
case ')':
|
||
depth--
|
||
if depth == 0 {
|
||
return i
|
||
}
|
||
}
|
||
}
|
||
return -1
|
||
}
|
||
|
||
// firstNonErrorReturnField splits a return-clause body by top-level
|
||
// commas and returns the first type expression that isn't `error`.
|
||
// Named return parameters (`result *User`) are handled by taking the
|
||
// last whitespace-separated token as the type.
|
||
func firstNonErrorReturnField(inner string) string {
|
||
var fields []string
|
||
depth := 0
|
||
start := 0
|
||
for i := 0; i < len(inner); i++ {
|
||
switch inner[i] {
|
||
case '(', '[':
|
||
depth++
|
||
case ')', ']':
|
||
depth--
|
||
case ',':
|
||
if depth == 0 {
|
||
fields = append(fields, strings.TrimSpace(inner[start:i]))
|
||
start = i + 1
|
||
}
|
||
}
|
||
}
|
||
if last := strings.TrimSpace(inner[start:]); last != "" {
|
||
fields = append(fields, last)
|
||
}
|
||
for _, f := range fields {
|
||
t := f
|
||
// Named return: `ctx context.Context`, `result *User`. Grab
|
||
// the last whitespace-separated token — that's the type.
|
||
if parts := strings.Fields(f); len(parts) > 1 {
|
||
t = parts[len(parts)-1]
|
||
}
|
||
if t == "error" || strings.HasSuffix(t, ".error") {
|
||
continue
|
||
}
|
||
return t
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// snapshotContractShapes walks every request_type / response_type
|
||
// reference in the registry, loads each referenced type node's source,
|
||
// and attaches the extracted Shape to the node's Meta["shape"].
|
||
//
|
||
// We:
|
||
// - Collect the unique set of symbol IDs — a popular DTO might be a
|
||
// request/response on dozens of routes and we want to parse its
|
||
// source once.
|
||
// - Read each file once (cached in the source map).
|
||
// - Skip nodes whose ID doesn't look like a symbol (bare names that
|
||
// couldn't be upgraded) — those have nothing to dereference.
|
||
// - Skip type nodes that already have a shape attached from a prior
|
||
// pass on the same session (ETag-style short-circuit).
|
||
func (idx *Indexer) snapshotContractShapes(reg *contracts.Registry) {
|
||
symbols := make(map[string]struct{})
|
||
for _, c := range reg.All() {
|
||
for _, key := range []string{"request_type", "response_type"} {
|
||
v, _ := c.Meta[key].(string)
|
||
if v == "" || !strings.Contains(v, "::") {
|
||
continue
|
||
}
|
||
symbols[v] = struct{}{}
|
||
}
|
||
// Envelope rows reference types the dashboard wants expanded
|
||
// just as much as the top-level response_type does — without
|
||
// snapshotting them here, the inlineEnvelopeShapes pass below
|
||
// finds no shape to fold into the row.
|
||
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok {
|
||
for _, row := range env {
|
||
v, _ := row["type"].(string)
|
||
if v == "" || !strings.Contains(v, "::") {
|
||
continue
|
||
}
|
||
symbols[v] = struct{}{}
|
||
}
|
||
}
|
||
}
|
||
if len(symbols) == 0 {
|
||
return
|
||
}
|
||
srcCache := make(map[string][]byte)
|
||
attached := 0
|
||
for id := range symbols {
|
||
node := idx.graph.GetNode(id)
|
||
if node == nil {
|
||
continue
|
||
}
|
||
// Accept both KindType and KindInterface — TypeScript /
|
||
// Java / Kotlin model their type defs as interfaces, and
|
||
// the dashboard wants their fields expanded just like Go
|
||
// struct types. Limiting to KindType silently dropped every
|
||
// TS interface shape extraction.
|
||
if node.Kind != graph.KindType && node.Kind != graph.KindInterface {
|
||
continue
|
||
}
|
||
if _, done := node.Meta["shape"]; done {
|
||
continue
|
||
}
|
||
src, ok := srcCache[node.FilePath]
|
||
if !ok {
|
||
// File paths in the graph are repo-prefixed; trim the
|
||
// prefix for disk I/O.
|
||
diskPath := node.FilePath
|
||
if idx.repoPrefix != "" && strings.HasPrefix(diskPath, idx.repoPrefix+"/") {
|
||
diskPath = strings.TrimPrefix(diskPath, idx.repoPrefix+"/")
|
||
}
|
||
diskPath = filepath.Join(idx.rootPath, diskPath)
|
||
data, err := os.ReadFile(diskPath)
|
||
if err != nil {
|
||
srcCache[node.FilePath] = nil
|
||
continue
|
||
}
|
||
srcCache[node.FilePath] = data
|
||
src = data
|
||
}
|
||
if src == nil {
|
||
continue
|
||
}
|
||
shape := contracts.ExtractShape(node.FilePath, src, node.StartLine, node.EndLine)
|
||
if shape == nil {
|
||
continue
|
||
}
|
||
if node.Meta == nil {
|
||
node.Meta = map[string]any{}
|
||
}
|
||
node.Meta["shape"] = shape
|
||
attached++
|
||
}
|
||
if attached > 0 {
|
||
idx.logger.Info("contract shapes snapshotted",
|
||
zap.Int("types", attached),
|
||
zap.Int("examined", len(symbols)))
|
||
}
|
||
}
|
||
|
||
// inlineEnvelopeShapes folds each type node's snapshotted shape onto
|
||
// every response_envelope row that references it. After this pass an
|
||
// envelope row carries the full JSON-rendering data:
|
||
//
|
||
// {
|
||
// "name": "repos",
|
||
// "expr": "repos",
|
||
// "type": "<repo>/service.go::Repo",
|
||
// "shape": { "kind": "struct", "fields": [...] }
|
||
// }
|
||
//
|
||
// so the dashboard can render the actual response shape instead of a
|
||
// bare type-symbol-ID. Idempotent: rows that already carry "shape"
|
||
// are skipped, which lets cross-pass calls (re-extraction, snapshot
|
||
// restore) re-run cheaply.
|
||
//
|
||
// Also handles the top-level response_type / request_type: the
|
||
// referenced type's shape is mirrored onto Meta["response_shape"] /
|
||
// Meta["request_shape"] so plain-typed responses (no map envelope)
|
||
// also expose their JSON object shape on the dashboard.
|
||
func (idx *Indexer) inlineEnvelopeShapes(reg *contracts.Registry) {
|
||
inlined := 0
|
||
for _, c := range reg.All() {
|
||
changed := false
|
||
|
||
// Envelope rows.
|
||
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
|
||
for ri, row := range env {
|
||
if _, has := row["shape"]; has {
|
||
continue
|
||
}
|
||
if shape := idx.lookupShape(row["type"]); shape != nil {
|
||
env[ri]["shape"] = shape
|
||
changed = true
|
||
}
|
||
}
|
||
if changed {
|
||
items := reg.ByID(c.ID)
|
||
for i := range items {
|
||
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
|
||
continue
|
||
}
|
||
if items[i].Meta == nil {
|
||
items[i].Meta = map[string]any{}
|
||
}
|
||
items[i].Meta["response_envelope"] = env
|
||
}
|
||
reg.ReplaceByID(c.ID, items)
|
||
}
|
||
}
|
||
|
||
// Top-level request/response shapes — same idea, applied to a
|
||
// plain `response_type: "<id>::Foo"` so the schema view can
|
||
// render the JSON object even when there's no envelope wrapper.
|
||
topChanged := false
|
||
for metaKey, shapeKey := range map[string]string{
|
||
"response_type": "response_shape",
|
||
"request_type": "request_shape",
|
||
} {
|
||
if _, has := c.Meta[shapeKey]; has {
|
||
continue
|
||
}
|
||
if shape := idx.lookupShape(c.Meta[metaKey]); shape != nil {
|
||
items := reg.ByID(c.ID)
|
||
for i := range items {
|
||
if items[i].Role != contracts.RoleProvider || items[i].SymbolID != c.SymbolID {
|
||
continue
|
||
}
|
||
if items[i].Meta == nil {
|
||
items[i].Meta = map[string]any{}
|
||
}
|
||
items[i].Meta[shapeKey] = shape
|
||
}
|
||
reg.ReplaceByID(c.ID, items)
|
||
topChanged = true
|
||
}
|
||
}
|
||
|
||
if changed || topChanged {
|
||
inlined++
|
||
}
|
||
}
|
||
if inlined > 0 {
|
||
idx.logger.Info("response envelopes hydrated with shapes",
|
||
zap.Int("contracts", inlined))
|
||
}
|
||
}
|
||
|
||
// lookupShape resolves a Meta type reference to the snapshotted shape
|
||
// stored on its graph node. Accepts string IDs (the only form used in
|
||
// today's pipeline); other shapes pass through as nil so callers can
|
||
// chain without type-asserting upstream.
|
||
func (idx *Indexer) lookupShape(raw any) any {
|
||
id, ok := raw.(string)
|
||
if !ok || id == "" || !strings.Contains(id, "::") {
|
||
return nil
|
||
}
|
||
node := idx.graph.GetNode(id)
|
||
if node == nil || node.Meta == nil {
|
||
return nil
|
||
}
|
||
shape, ok := node.Meta["shape"]
|
||
if !ok || shape == nil {
|
||
return nil
|
||
}
|
||
return shape
|
||
}
|
||
|
||
// extractExternalModules parses the repo's go.mod once and writes
|
||
// KindModule nodes plus EdgeDependsOnModule edges into the graph.
|
||
// A synthetic KindFile node is emitted for go.mod itself so the
|
||
// edges have a real source endpoint that survives applyRepoPrefix.
|
||
// Safe to call when no go.mod exists. Other manifest formats
|
||
// (package.json, pnpm-lock, requirements.txt, Cargo.toml, …) are
|
||
// future additions — each lands as another switch case here.
|
||
//
|
||
// Import-node → module-node edges (per the broader coverage spec)
|
||
// are deferred to v2; the v1 file-level edge is already enough for
|
||
// agents asking "what does this repo depend on".
|
||
func (idx *Indexer) extractExternalModules() {
|
||
if !idx.config.Coverage.IsEnabled("modules") {
|
||
return
|
||
}
|
||
// Walk known manifest formats at the repo root. Each manifest
|
||
// produces an independent Spec list and gets its own synthetic
|
||
// file node — the file→module edge stays scoped to the
|
||
// originating manifest so cross-ecosystem queries (e.g. "what
|
||
// does package.json declare") don't bleed into go.mod's
|
||
// answer.
|
||
manifests := []struct {
|
||
path string
|
||
parse func([]byte) []modules.Spec
|
||
ownPathFromSrc func([]byte) string
|
||
}{
|
||
{
|
||
path: "go.mod",
|
||
parse: modules.ParseGoMod,
|
||
ownPathFromSrc: readGoModModulePath,
|
||
},
|
||
{
|
||
path: "package.json",
|
||
parse: modules.ParsePackageJSON,
|
||
ownPathFromSrc: readPackageJSONOwnName,
|
||
},
|
||
{
|
||
path: "package-lock.json",
|
||
parse: modules.ParsePackageLockJSON,
|
||
ownPathFromSrc: nil, // package-lock has no own-name notion separate from package.json
|
||
},
|
||
{
|
||
path: "yarn.lock",
|
||
parse: modules.ParseYarnLock,
|
||
ownPathFromSrc: nil,
|
||
},
|
||
{
|
||
path: "pnpm-lock.yaml",
|
||
parse: modules.ParsePnpmLock,
|
||
ownPathFromSrc: nil,
|
||
},
|
||
{
|
||
path: "pyproject.toml",
|
||
parse: modules.ParsePyProject,
|
||
ownPathFromSrc: readPyProjectOwnName,
|
||
},
|
||
{
|
||
path: "requirements.txt",
|
||
parse: modules.ParseRequirementsTxt,
|
||
ownPathFromSrc: nil, // requirements.txt has no own-name notion
|
||
},
|
||
{
|
||
path: "Cargo.toml",
|
||
parse: modules.ParseCargoToml,
|
||
ownPathFromSrc: readCargoTomlOwnName,
|
||
},
|
||
{
|
||
path: "pom.xml",
|
||
parse: modules.ParsePomXML,
|
||
ownPathFromSrc: readPomXMLOwnName,
|
||
},
|
||
}
|
||
|
||
for _, m := range manifests {
|
||
idx.extractOneModuleManifest(m.path, m.parse, m.ownPathFromSrc)
|
||
}
|
||
|
||
// After per-manifest module extraction, detect whether this repo's
|
||
// root is a package-manager workspace and materialise its
|
||
// root→member edges.
|
||
idx.extractPackageWorkspace()
|
||
}
|
||
|
||
// extractOneModuleManifest reads a single manifest file from the
|
||
// repo root, parses it via the supplied parser, and writes the
|
||
// resulting nodes/edges + import-link edges into the graph. Used
|
||
// from extractExternalModules's per-manifest dispatch.
|
||
func (idx *Indexer) extractOneModuleManifest(relPath string, parse func([]byte) []modules.Spec, ownPathFromSrc func([]byte) string) {
|
||
manifestAbs := filepath.Join(idx.rootPath, relPath)
|
||
src, err := os.ReadFile(manifestAbs)
|
||
if err != nil {
|
||
return
|
||
}
|
||
specs := parse(src)
|
||
nodes, edges := modules.BuildGraphArtifacts(relPath, specs)
|
||
if len(nodes) == 0 && len(edges) == 0 {
|
||
return
|
||
}
|
||
// Synthetic file node for the manifest — it isn't represented
|
||
// through the language-extractor pipeline (no extractor is
|
||
// registered for the .mod extension; package.json may have one
|
||
// but the JSON walker doesn't emit a synthetic file node we
|
||
// can reuse), so the EdgeDependsOnModule edges would otherwise
|
||
// dangle from a missing source endpoint after applyRepoPrefix
|
||
// runs in multi-repo mode.
|
||
manifestNode := &graph.Node{
|
||
ID: relPath,
|
||
Kind: graph.KindFile,
|
||
Name: filepath.Base(relPath),
|
||
FilePath: relPath,
|
||
Language: manifestLanguage(relPath),
|
||
}
|
||
allNodes := append([]*graph.Node{manifestNode}, nodes...)
|
||
idx.applyRepoPrefix(allNodes, edges)
|
||
idx.graph.AddBatch(allNodes, edges)
|
||
|
||
// Connect each KindImport node to its matching module via
|
||
// longest-prefix path resolution. Repo-internal imports (the
|
||
// own-module path) are filtered inside LinkImports — when the
|
||
// manifest doesn't expose an own-name, the filter is a no-op
|
||
// which is safe (no own-module imports to match against).
|
||
var ownModulePath string
|
||
if ownPathFromSrc != nil {
|
||
ownModulePath = ownPathFromSrc(src)
|
||
}
|
||
// Scope the walk to this repo's own import nodes. The unscoped
|
||
// LinkImports walks g.AllNodes(); under a warmup loop across
|
||
// hundreds of repos that's O(R · N). The per-repo byRepo bucket
|
||
// keeps this O(repo size).
|
||
repoNodes := idx.graph.GetRepoNodes(idx.repoPrefix)
|
||
importNodes := make([]*graph.Node, 0, len(repoNodes))
|
||
for _, n := range repoNodes {
|
||
if n.Kind == graph.KindImport {
|
||
importNodes = append(importNodes, n)
|
||
}
|
||
}
|
||
modules.LinkImportsIn(idx.graph, importNodes, specs, ownModulePath)
|
||
}
|
||
|
||
// manifestLanguage returns the language tag stamped on a manifest's
|
||
// synthetic file node. Used purely for Brief listings — the kind
|
||
// field carries the structural type.
|
||
func manifestLanguage(relPath string) string {
|
||
switch filepath.Base(relPath) {
|
||
case "go.mod":
|
||
return "go"
|
||
case "package.json", "package-lock.json":
|
||
return "json"
|
||
case "yarn.lock":
|
||
return "yarn"
|
||
case "pnpm-lock.yaml":
|
||
return "yaml"
|
||
case "pyproject.toml", "Cargo.toml":
|
||
return "toml"
|
||
case "requirements.txt":
|
||
return "text"
|
||
case "pom.xml":
|
||
return "xml"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// readPomXMLOwnName builds the project's own Maven coordinate
|
||
// (groupId:artifactId) so LinkImports can filter self-references.
|
||
// Java workspace setups where a sibling module imports the parent
|
||
// project shouldn't accidentally resolve to an external dep with
|
||
// the same coordinate. Returns "" when either field is missing —
|
||
// the LinkImports filter treats "" as no own-module filter, which
|
||
// is safe.
|
||
func readPomXMLOwnName(src []byte) string {
|
||
var manifest struct {
|
||
GroupID string `xml:"groupId"`
|
||
ArtifactID string `xml:"artifactId"`
|
||
}
|
||
if err := xml.Unmarshal(src, &manifest); err != nil {
|
||
return ""
|
||
}
|
||
if manifest.GroupID == "" || manifest.ArtifactID == "" {
|
||
return ""
|
||
}
|
||
return manifest.GroupID + ":" + manifest.ArtifactID
|
||
}
|
||
|
||
// readCargoTomlOwnName reads the crate's own name from
|
||
// `[package] name`. Used for LinkImports's own-module filter so
|
||
// workspace-internal crate references don't accidentally match
|
||
// external crates with the same name.
|
||
func readCargoTomlOwnName(src []byte) string {
|
||
var manifest struct {
|
||
Package struct {
|
||
Name string `toml:"name"`
|
||
} `toml:"package"`
|
||
}
|
||
if err := toml.Unmarshal(src, &manifest); err != nil {
|
||
return ""
|
||
}
|
||
return manifest.Package.Name
|
||
}
|
||
|
||
// readPyProjectOwnName returns the package's own name from the
|
||
// pyproject.toml `[project] name` field. Used by LinkImports's
|
||
// own-module filter so a workspace package's internal imports
|
||
// don't accidentally collide with external pypi names. Returns ""
|
||
// on parse error or when the field is absent.
|
||
func readPyProjectOwnName(src []byte) string {
|
||
var manifest struct {
|
||
Project struct {
|
||
Name string `toml:"name"`
|
||
} `toml:"project"`
|
||
}
|
||
if err := toml.Unmarshal(src, &manifest); err != nil {
|
||
return ""
|
||
}
|
||
return manifest.Project.Name
|
||
}
|
||
|
||
// readPackageJSONOwnName extracts the manifest's `name` field — the
|
||
// npm equivalent of go.mod's `module` directive. Returns "" on
|
||
// missing or malformed JSON; LinkImports treats "" as "no own-
|
||
// module filter", which is safe because internal-package matches
|
||
// (e.g. workspaces) won't accidentally collide with external deps
|
||
// at the longest-prefix scan.
|
||
func readPackageJSONOwnName(src []byte) string {
|
||
var manifest struct {
|
||
Name string `json:"name"`
|
||
}
|
||
if err := json.Unmarshal(src, &manifest); err != nil {
|
||
return ""
|
||
}
|
||
return manifest.Name
|
||
}
|
||
|
||
// readGoModModulePath extracts the `module ` directive value from
|
||
// go.mod source. Mirrors the inline parse in coverage.ReadModulePath
|
||
// — we keep both copies tiny rather than introducing a one-import
|
||
// shared helper that would force a layering compromise (coverage
|
||
// shouldn't depend on indexer; indexer shouldn't depend on coverage).
|
||
func readGoModModulePath(src []byte) string {
|
||
for _, line := range strings.Split(string(src), "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if strings.HasPrefix(line, "module ") {
|
||
return strings.TrimSpace(strings.TrimPrefix(line, "module "))
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// extractGoModContracts runs the go.mod-specific extractor once against
|
||
// the repo root (go.mod isn't represented as a file node in the graph).
|
||
// Results are added to reg. Safe to call when no go.mod exists.
|
||
//
|
||
// Also materialises the dep::<module> contracts as graph nodes
|
||
// immediately, so the resolver's import-bridge (Resolver.lookupDepModule)
|
||
// can find them during ResolveAll. commitContracts later AddNode is
|
||
// idempotent — it skips nodes that already exist — so this doesn't
|
||
// double-emit. We only do this for type=dependency; everything else
|
||
// goes through the normal commit path which depends on a resolved
|
||
// graph (UpgradeBareTypeRefs, resolveProviderHandlers).
|
||
func (idx *Indexer) extractGoModContracts(reg *contracts.Registry) {
|
||
goModPath := filepath.Join(idx.rootPath, "go.mod")
|
||
goModSrc, err := os.ReadFile(goModPath)
|
||
if err != nil {
|
||
return
|
||
}
|
||
goModExtractor := &contracts.GoModExtractor{TrackedRepos: idx.trackedRepoModules}
|
||
goModFilePath := "go.mod"
|
||
if idx.repoPrefix != "" {
|
||
goModFilePath = idx.repoPrefix + "/go.mod"
|
||
}
|
||
found := goModExtractor.Extract(goModFilePath, goModSrc, nil, nil)
|
||
reg.AddAllScoped(found, idx.repoPrefix, idx.workspaceID, idx.projectID)
|
||
|
||
var nodes []*graph.Node
|
||
for i := range found {
|
||
c := found[i]
|
||
if c.Type != contracts.ContractDependency {
|
||
continue
|
||
}
|
||
if idx.graph.GetNode(c.ID) != nil {
|
||
continue
|
||
}
|
||
nodes = append(nodes, &graph.Node{
|
||
ID: c.ID,
|
||
Kind: graph.KindContract,
|
||
Name: c.ID,
|
||
FilePath: c.FilePath,
|
||
Language: "contract",
|
||
RepoPrefix: idx.repoPrefix,
|
||
Meta: map[string]any{"type": string(c.Type), "role": string(c.Role)},
|
||
})
|
||
}
|
||
if len(nodes) > 0 {
|
||
idx.graph.AddBatch(nodes, nil)
|
||
}
|
||
}
|
||
|
||
// extractContracts scans all file nodes in the graph and runs contract
|
||
// extractors to detect API contracts (HTTP routes, gRPC services,
|
||
// GraphQL, topics, etc.). Detected contracts are added as graph nodes
|
||
// with provides/consumes edges.
|
||
//
|
||
// This full-walk path is used by IncrementalReindex (where many files
|
||
// are already cached). IndexCtx instead runs the per-file work inline
|
||
// with parsing — see the worker loop — and skips this function.
|
||
func (idx *Indexer) extractContracts() {
|
||
reg := contracts.NewRegistry()
|
||
_, byLang := idx.buildPerFileContractExtractors()
|
||
|
||
// Track which file nodes we saw this pass so we can prune stale
|
||
// cache entries afterwards (files that left the graph).
|
||
seenFiles := make(map[string]struct{})
|
||
|
||
// Multi-repo mode: walk only this repo's nodes. The previous
|
||
// AllNodes()-then-filter pass paid an O(global) walk per repo,
|
||
// which compounded with hundreds of tracked siblings.
|
||
var nodes []*graph.Node
|
||
if idx.repoPrefix != "" {
|
||
nodes = idx.graph.GetRepoNodes(idx.repoPrefix)
|
||
} else {
|
||
nodes = idx.graph.AllNodes()
|
||
}
|
||
|
||
// Pre-bucket the already-fetched node slice by FilePath so the
|
||
// per-file body can look up its co-located nodes in O(1) instead
|
||
// of firing a fresh GetFileNodes query per file. Likewise pre-
|
||
// fetch every out-edge whose source is in this repo as ONE backend
|
||
// call and bucket by From so the per-file body can replace
|
||
// GetOutEdges(fileNode.ID) — on disk backends the per-file query
|
||
// path was the second-largest source of round-trips in
|
||
// deferred_passes (after the DI walk).
|
||
nodesByFile := make(map[string][]*graph.Node, len(nodes))
|
||
for _, n := range nodes {
|
||
if n == nil {
|
||
continue
|
||
}
|
||
nodesByFile[n.FilePath] = append(nodesByFile[n.FilePath], n)
|
||
}
|
||
var edgesByFrom map[string][]*graph.Edge
|
||
if idx.repoPrefix != "" {
|
||
repoEdges := idx.graph.GetRepoEdges(idx.repoPrefix)
|
||
edgesByFrom = make(map[string][]*graph.Edge, len(nodes))
|
||
for _, e := range repoEdges {
|
||
if e == nil {
|
||
continue
|
||
}
|
||
edgesByFrom[e.From] = append(edgesByFrom[e.From], e)
|
||
}
|
||
}
|
||
|
||
for _, fileNode := range nodes {
|
||
if fileNode.Kind != graph.KindFile {
|
||
continue
|
||
}
|
||
|
||
// In multi-repo mode the byRepo bucket is already scoped, but
|
||
// the path-prefix strip below still needs to run.
|
||
relPath := fileNode.FilePath
|
||
if idx.repoPrefix != "" {
|
||
if !strings.HasPrefix(relPath, idx.repoPrefix+"/") {
|
||
continue
|
||
}
|
||
relPath = strings.TrimPrefix(relPath, idx.repoPrefix+"/")
|
||
}
|
||
|
||
absPath := filepath.Join(idx.rootPath, relPath)
|
||
info, statErr := os.Stat(absPath)
|
||
if statErr != nil {
|
||
continue
|
||
}
|
||
seenFiles[fileNode.FilePath] = struct{}{}
|
||
currentMtime := info.ModTime().UnixNano()
|
||
|
||
// Cache hit: replay the previously-extracted contracts without
|
||
// re-reading the file or re-running the 8 extractors. This is
|
||
// the dominant savings path on repos with many files where most
|
||
// haven't changed since the last extraction (e.g. live re-index
|
||
// after a single-file edit).
|
||
idx.contractCacheMu.RLock()
|
||
cached, ok := idx.contractCache[fileNode.FilePath]
|
||
idx.contractCacheMu.RUnlock()
|
||
if ok && cached.mtimeNano == currentMtime {
|
||
for _, c := range cached.contracts {
|
||
reg.Add(c)
|
||
}
|
||
continue
|
||
}
|
||
|
||
src, err := os.ReadFile(absPath)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
|
||
var fileNodes []*graph.Node
|
||
var fileEdges []*graph.Edge
|
||
if idx.repoPrefix != "" {
|
||
fileNodes = nodesByFile[fileNode.FilePath]
|
||
fileEdges = edgesByFrom[fileNode.ID]
|
||
} else {
|
||
fileNodes = idx.graph.GetFileNodes(fileNode.FilePath)
|
||
fileEdges = idx.graph.GetOutEdges(fileNode.ID)
|
||
}
|
||
|
||
// Language-filtered dispatch: skip extractors that don't list
|
||
// this file's language in SupportedLanguages(). On big repos
|
||
// with lots of .css / .svg / .json etc. this cuts a lot of
|
||
// no-op extractor calls.
|
||
exts := byLang[fileNode.Language]
|
||
// Re-parse for AST overlay — the language extractor's tree
|
||
// from the original index pass was closed when the file was
|
||
// first added. Cheap (~milliseconds per file) and cleanly
|
||
// scoped to this contract-pass invocation.
|
||
tree := contracts.ParseTreeForLang(fileNode.Language, src)
|
||
fileContracts := idx.runContractExtractorsForFile(
|
||
fileNode.FilePath, src, fileNodes, fileEdges, exts, tree)
|
||
tree.Release()
|
||
for _, c := range fileContracts {
|
||
reg.Add(c)
|
||
}
|
||
|
||
idx.contractCacheMu.Lock()
|
||
idx.contractCache[fileNode.FilePath] = &contractCacheEntry{
|
||
mtimeNano: currentMtime,
|
||
contracts: fileContracts,
|
||
}
|
||
idx.contractCacheMu.Unlock()
|
||
}
|
||
|
||
// Prune cache entries for files that are no longer in the graph
|
||
// (deleted, or repo untracked). Otherwise the cache grows unboundedly
|
||
// across the lifetime of the daemon.
|
||
idx.contractCacheMu.Lock()
|
||
for path := range idx.contractCache {
|
||
if _, ok := seenFiles[path]; !ok {
|
||
delete(idx.contractCache, path)
|
||
}
|
||
}
|
||
idx.contractCacheMu.Unlock()
|
||
|
||
idx.extractGoModContracts(reg)
|
||
idx.extractDIContracts(reg)
|
||
idx.commitContracts(reg)
|
||
}
|
||
|
||
// IsStale returns true if the file at relPath has been modified on disk since
|
||
// it was last indexed, based on comparing stored mtime against current disk mtime.
|
||
//
|
||
// relPath is folded to the canonical key (slash form, Unicode NFC)
|
||
// before lookup so a caller passing a non-ASCII path in a different
|
||
// Unicode form than fileMtimes was keyed with still resolves — without
|
||
// the fold the lookup would miss and the file be reported permanently
|
||
// stale, re-indexing it under a second key on every pass.
|
||
// HasChangesSinceMtimes reports whether any indexable file under root
|
||
// changed (mtime differs or is new) or was deleted, relative to the
|
||
// indexer's currently-loaded fileMtimes. It runs the SAME walk +
|
||
// staleness + deletion logic as IncrementalReindex but writes nothing.
|
||
//
|
||
// The daemon warmup uses it to choose a reconcile strategy for a
|
||
// reopened repo: a repo with zero changes takes the fast no-op
|
||
// IncrementalReindex path, while a repo that changed while the daemon
|
||
// was down is routed through the shadow/bulk re-track path instead.
|
||
// That routing matters because IncrementalReindex re-resolves changed
|
||
// files through per-edge graph.ReindexEdges, and the per-edge write
|
||
// path against a freshly reopened disk store is slow and unreliable.
|
||
// The shadow path resolves entirely in an in-memory graph and commits
|
||
// the result in one bulk load, so it never issues a per-edge write to
|
||
// the reopened store. It re-indexes the whole repo (more work than a
|
||
// true incremental pass), but it is reliable, and only repos that
|
||
// actually changed during downtime pay the cost.
|
||
//
|
||
// Conservative on error: anything it can't determine (bad root, walk
|
||
// error) returns true so the caller re-indexes rather than silently
|
||
// serving a stale graph.
|
||
func (idx *Indexer) HasChangesSinceMtimes(root string) bool {
|
||
absRoot, err := filepath.Abs(root)
|
||
if err != nil {
|
||
return true
|
||
}
|
||
idx.storeRootPath(absRoot)
|
||
|
||
diskFiles := make(map[string]bool)
|
||
errStop := errors.New("stop-walk")
|
||
walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error {
|
||
if werr != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
if idx.shouldPruneDir(path, absRoot) {
|
||
return filepath.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
if _, ok := idx.effectiveLanguage(path, nil); !ok {
|
||
return nil
|
||
}
|
||
if idx.shouldExclude(path, absRoot, false) {
|
||
return nil
|
||
}
|
||
rel := idx.relKey(path)
|
||
diskFiles[rel] = true
|
||
if idx.IsStale(rel) {
|
||
return errStop // a single changed/new file is enough
|
||
}
|
||
return nil
|
||
})
|
||
if errors.Is(walkErr, errStop) {
|
||
return true
|
||
}
|
||
if walkErr != nil {
|
||
return true
|
||
}
|
||
|
||
// Deletion check: a previously-indexed file absent from the walk and
|
||
// confirmed gone from disk counts as a change (its edges must drop).
|
||
idx.mtimeMu.RLock()
|
||
var candidates []string
|
||
for rel := range idx.fileMtimes {
|
||
if !diskFiles[rel] {
|
||
candidates = append(candidates, rel)
|
||
}
|
||
}
|
||
idx.mtimeMu.RUnlock()
|
||
for _, rel := range candidates {
|
||
if _, err := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(err, os.ErrNotExist) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ChangedSinceMtimes is the accumulating variant of HasChangesSinceMtimes:
|
||
// it runs the SAME walk + IsStale staleness predicate + deletion-confirm
|
||
// logic but, instead of short-circuiting on the first stale file, returns
|
||
// the full changed and deleted sets as in-repo relative slash-paths.
|
||
//
|
||
// The warm-restart reconcile router uses the census to size the reconcile:
|
||
// an empty result takes the fast no-op path, a small delta is scoped to
|
||
// exactly those files (re-index the changed, evict the deleted), and only a
|
||
// large-fraction churn falls back to a whole-repo re-track.
|
||
//
|
||
// changed holds files that are new or whose mtime drifted since the last
|
||
// pass; deleted holds previously-indexed files confirmed gone from disk. A
|
||
// walk (or abs-path) error returns a non-nil err with nil slices — the
|
||
// caller treats that as "unknown, do a full re-track", exactly as
|
||
// HasChangesSinceMtimes conservatively returns true on the same condition.
|
||
func (idx *Indexer) ChangedSinceMtimes(root string) (changed []string, deleted []string, err error) {
|
||
absRoot, absErr := filepath.Abs(root)
|
||
if absErr != nil {
|
||
return nil, nil, absErr
|
||
}
|
||
idx.storeRootPath(absRoot)
|
||
|
||
diskFiles := make(map[string]bool)
|
||
walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, werr error) error {
|
||
if werr != nil {
|
||
return nil
|
||
}
|
||
if d.IsDir() {
|
||
if idx.shouldPruneDir(path, absRoot) {
|
||
return filepath.SkipDir
|
||
}
|
||
return nil
|
||
}
|
||
if _, ok := idx.effectiveLanguage(path, nil); !ok {
|
||
return nil
|
||
}
|
||
if idx.shouldExclude(path, absRoot, false) {
|
||
return nil
|
||
}
|
||
rel := idx.relKey(path)
|
||
diskFiles[rel] = true
|
||
if idx.IsStale(rel) {
|
||
changed = append(changed, rel)
|
||
}
|
||
return nil
|
||
})
|
||
if walkErr != nil {
|
||
return nil, nil, walkErr
|
||
}
|
||
|
||
// Deletion check: a previously-indexed file absent from the walk and
|
||
// confirmed gone from disk counts as a change (its edges must drop).
|
||
idx.mtimeMu.RLock()
|
||
var candidates []string
|
||
for rel := range idx.fileMtimes {
|
||
if !diskFiles[rel] {
|
||
candidates = append(candidates, rel)
|
||
}
|
||
}
|
||
idx.mtimeMu.RUnlock()
|
||
for _, rel := range candidates {
|
||
if _, statErr := os.Stat(filepath.Join(absRoot, filepath.FromSlash(rel))); errors.Is(statErr, os.ErrNotExist) {
|
||
deleted = append(deleted, rel)
|
||
}
|
||
}
|
||
return changed, deleted, nil
|
||
}
|
||
|
||
func (idx *Indexer) IsStale(relPath string) bool {
|
||
relPath = pathkey.Normalize(filepath.ToSlash(relPath))
|
||
|
||
idx.mtimeMu.RLock()
|
||
storedMtime, ok := idx.fileMtimes[relPath]
|
||
idx.mtimeMu.RUnlock()
|
||
if !ok {
|
||
// Unknown file — treat as stale.
|
||
return true
|
||
}
|
||
|
||
absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath))
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
// Can't stat — treat as stale.
|
||
return true
|
||
}
|
||
|
||
return info.ModTime().UnixNano() != storedMtime
|
||
}
|
||
|
||
// IsTrackedStale reports whether a file that IS in the index has changed
|
||
// on disk since it was indexed. Unlike IsStale it returns false for an
|
||
// untracked path (a new file, a non-source path, or a path-form
|
||
// mismatch), so a freshness signal never false-positives on a file the
|
||
// index legitimately does not cover.
|
||
func (idx *Indexer) IsTrackedStale(relPath string) bool {
|
||
return idx.TrackedFileState(relPath) == FileStale
|
||
}
|
||
|
||
// FileFreshness is the verdict TrackedFileState returns for a tracked file:
|
||
// its indexed view is current, has drifted, or the file is gone from disk.
|
||
type FileFreshness string
|
||
|
||
const (
|
||
// FileFresh means the file is not tracked (so no claim is made) or its
|
||
// on-disk mtime still matches what was indexed.
|
||
FileFresh FileFreshness = "fresh"
|
||
// FileStale means the file is tracked and its on-disk mtime differs from
|
||
// the indexed mtime — the graph view lags the working tree.
|
||
FileStale FileFreshness = "stale"
|
||
// FileMissing means the file is tracked in the graph but no longer exists
|
||
// on disk — it was deleted or moved since indexing. A plain staleness
|
||
// check folds this into "not stale"; the distinct verdict lets a list
|
||
// result flag hits that point at vanished files.
|
||
FileMissing FileFreshness = "missing"
|
||
)
|
||
|
||
// TrackedFileState classifies one repo-relative file against the indexer's
|
||
// recorded mtimes: fresh (untracked or unchanged), stale (mtime drift), or
|
||
// missing (tracked but absent on disk). Splitting the stat-failure branch out
|
||
// of IsTrackedStale is what lets the freshness rider distinguish a stale hit
|
||
// from one whose underlying file has been deleted.
|
||
func (idx *Indexer) TrackedFileState(relPath string) FileFreshness {
|
||
relPath = pathkey.Normalize(filepath.ToSlash(relPath))
|
||
|
||
idx.mtimeMu.RLock()
|
||
storedMtime, ok := idx.fileMtimes[relPath]
|
||
idx.mtimeMu.RUnlock()
|
||
if !ok {
|
||
return FileFresh
|
||
}
|
||
|
||
absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath))
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return FileMissing
|
||
}
|
||
// A transient / permission stat error is not evidence of drift —
|
||
// don't cry wolf for a file we simply could not read.
|
||
return FileFresh
|
||
}
|
||
if info.ModTime().UnixNano() != storedMtime {
|
||
return FileStale
|
||
}
|
||
return FileFresh
|
||
}
|