a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
3427 lines
125 KiB
Go
3427 lines
125 KiB
Go
package indexer
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"runtime"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"go.uber.org/zap"
|
||
|
||
"github.com/zzet/gortex/internal/config"
|
||
"github.com/zzet/gortex/internal/contracts"
|
||
"github.com/zzet/gortex/internal/embedding"
|
||
"github.com/zzet/gortex/internal/graph"
|
||
"github.com/zzet/gortex/internal/parser"
|
||
"github.com/zzet/gortex/internal/pathkey"
|
||
"github.com/zzet/gortex/internal/progress"
|
||
"github.com/zzet/gortex/internal/resolver"
|
||
"github.com/zzet/gortex/internal/search"
|
||
"github.com/zzet/gortex/internal/search/trigram"
|
||
"github.com/zzet/gortex/internal/semantic"
|
||
)
|
||
|
||
// RepoMetadata holds per-repo indexing state.
|
||
type RepoMetadata struct {
|
||
RepoPrefix string
|
||
RootPath string
|
||
Identity *RepoIdentity
|
||
LastIndexTime time.Time
|
||
FileCount int
|
||
NodeCount int
|
||
EdgeCount int
|
||
ParseErrors []IndexError
|
||
FileMtimes map[string]int64
|
||
// IsWorktree records whether RootPath was a linked git worktree
|
||
// (as opposed to a main checkout) at the time the repo was
|
||
// tracked. Captured here because once the worktree directory is
|
||
// removed from disk it can no longer be classified — the janitor
|
||
// needs this remembered flag to know a vanished root was a
|
||
// worktree and may be garbage-collected.
|
||
IsWorktree bool
|
||
// Unprefixed records that this repo was indexed in single-repo
|
||
// mode: its nodes carry RepoPrefix="" and raw relative file paths.
|
||
// The empty-prefix resolution fallback (RepoRoot, ResolveFilePath)
|
||
// honours only a lone repo that actually minted unprefixed nodes —
|
||
// without the provenance check, stale unprefixed nodes surviving a
|
||
// track/untrack transition would resolve against whatever repo
|
||
// happens to be the lone one. Eviction also branches on it:
|
||
// unprefixed nodes are invisible to the byRepo bucket EvictRepo
|
||
// walks, so UntrackRepo must evict them file-by-file.
|
||
Unprefixed bool
|
||
}
|
||
|
||
// MultiIndexer orchestrates indexing across multiple repositories.
|
||
type MultiIndexer struct {
|
||
graph graph.Store
|
||
registry *parser.Registry
|
||
search search.Backend
|
||
embedder embedding.Provider
|
||
repos map[string]*RepoMetadata // repoPrefix → metadata
|
||
indexers map[string]*Indexer // repoPrefix → per-repo indexer
|
||
configMgr *config.ConfigManager
|
||
logger *zap.Logger
|
||
mu sync.RWMutex
|
||
|
||
// reconcileMu serialises ReconcileContractEdges end-to-end. The pass
|
||
// evicts the prior EdgeMatches / topic / bridge generation and mints
|
||
// a fresh one across many independent graph-store writes — it is NOT
|
||
// atomic. Several goroutines drive it concurrently (the periodic
|
||
// janitor's ReconcileAll, the file-watcher's IncrementalReindex,
|
||
// MCP-triggered track / untrack / index), and mi.mu is only taken in
|
||
// fine-grained spots inside, not across the whole pass. Without this
|
||
// lock two overlapping reconciles can interleave evict and mint and
|
||
// persist a stale generation (a bridge wiped by the other run's
|
||
// EvictFile after it was minted). A dedicated outer mutex keeps the
|
||
// pass self-consistent without widening mi.mu's scope.
|
||
reconcileMu sync.Mutex
|
||
|
||
// stitchProber / proxyBudget wire the cross-daemon proxy-edge feature:
|
||
// when set by the daemon entry point (flag on), every CrossRepoResolver
|
||
// this MultiIndexer builds mints proxy edges to remote-owned symbols
|
||
// on positive evidence. A nil prober keeps the resolvers on read-only
|
||
// federation (the default).
|
||
stitchProber resolver.RemoteDeclarationProber
|
||
proxyBudget int
|
||
|
||
// deferGlobalPasses, when set, propagates SetDeferGlobalPasses(true)
|
||
// to every per-repo Indexer constructed by this MultiIndexer. Batch
|
||
// callers (warmup, ReconcileAll) flip it on around their loop and
|
||
// invoke RunGlobalGraphPasses once at the end so the O(global) walks
|
||
// (InferImplements / InferOverrides / markTestSymbolsAndEmitEdges)
|
||
// don't run R times against an R-repo graph.
|
||
deferGlobalPasses bool
|
||
|
||
// deferResolve, when set, propagates SetDeferResolve(true) to every
|
||
// per-repo Indexer constructed by this MultiIndexer. Used by the
|
||
// parallel warmup path: per-repo ResolveAll / contract extract /
|
||
// semantic enrich mutate the shared graph, so running them in
|
||
// parallel across repos races. With this flag the parallel loop
|
||
// just parses; RunDeferredPassesAll runs the per-repo passes
|
||
// serially after the loop. Independent of deferGlobalPasses — that
|
||
// flag covers a separate (cheaper) set of O(global) walks.
|
||
deferResolve bool
|
||
|
||
// batchChangedPrefixes scopes the per-repo clone-detection and
|
||
// clone-index Rebuild passes in RunGlobalGraphPasses to the repos that
|
||
// actually re-indexed in the current batch. nil — the default, and what
|
||
// every one-off EndBatch caller leaves it as — means "run the clone
|
||
// passes for every tracked repo", the prior whole-workspace behaviour.
|
||
// The daemon warmup arms it (ArmBatchScope) before EndBatch so an
|
||
// N-repo warm restart where only one repo changed stops paying ~N
|
||
// full-graph clone walks for repos whose clone edges are already on
|
||
// disk. Clone edges are per-repo (no cross-repo pair is ever formed),
|
||
// so an unchanged repo's persisted edges stay valid; its in-memory
|
||
// incremental clone index is reseeded lazily on its first later edit.
|
||
// Consumed and cleared by RunGlobalGraphPasses. Guarded by mi.mu.
|
||
batchChangedPrefixes map[string]struct{}
|
||
|
||
// resolverLSPHelper is the resolve-time LSP helper propagated
|
||
// onto every per-repo Indexer and onto the global post-pass
|
||
// resolver in RunDeferredPassesAll. nil means no LSP hot-path
|
||
// (heuristic-only resolution, the pre-N5 behaviour). The
|
||
// daemon installs the helper via SetResolverLSPHelper after
|
||
// constructing the LSP router; a multi-repo composite helper
|
||
// dispatches per-file by repo prefix.
|
||
resolverLSPHelper resolver.LSPHelper
|
||
|
||
// onRepoTracked, when non-nil, is invoked after a fresh
|
||
// TrackRepoCtx call has resolved the repo's prefix and
|
||
// absolute path but before indexing starts. The daemon uses
|
||
// this hook to register a per-repo resolver-time LSP helper
|
||
// against the LSPHelper registry so dynamically-tracked repos
|
||
// participate in the N5 hot path without daemon restart.
|
||
onRepoTracked func(prefix, absPath string)
|
||
|
||
// skipVectorBuild, when set, propagates SetSkipVectorBuild(true) to
|
||
// every per-repo Indexer this MultiIndexer constructs, so their
|
||
// buildSearchIndex passes populate only the text index and never
|
||
// embed. The daemon flips it on for the warmup loop when a snapshot
|
||
// already carries the workspace vector index — re-embedding 30k+
|
||
// symbols only to overwrite them with the cached index is the
|
||
// dominant restart cost. After warmup it restores the cached index
|
||
// once via ImportVectorIndex and clears the flag.
|
||
skipVectorBuild bool
|
||
|
||
// embedChunkOpts is the AST sub-chunking configuration propagated
|
||
// to every per-repo Indexer this MultiIndexer constructs. The zero
|
||
// value leaves the chunker on its built-in defaults.
|
||
embedChunkOpts embedding.ChunkOptions
|
||
|
||
// embedMaxSymbols overrides the vector-index size cap propagated to
|
||
// every per-repo Indexer. Zero keeps the built-in default.
|
||
embedMaxSymbols int
|
||
|
||
// embedAPIConcurrency bounds parallel embedding requests against an
|
||
// API-backed embedder, propagated to every per-repo Indexer. Zero
|
||
// keeps the built-in default.
|
||
embedAPIConcurrency int
|
||
|
||
// semanticMgr is the semantic enrichment manager propagated to
|
||
// every per-repo Indexer. When nil (the default), per-repo
|
||
// deferred passes skip semantic enrichment — this is the root
|
||
// cause of "enrich:0" in daemon mode. Set by the daemon via
|
||
// SetSemanticManager before IndexAll / TrackRepo.
|
||
semanticMgr *semantic.Manager
|
||
}
|
||
|
||
// SetEmbedder installs the embedding provider every per-repo indexer
|
||
// should use. Must be called before IndexAll / TrackRepo for vectors
|
||
// to land in the graph — without this the fresh Indexer created per
|
||
// repo has embedder=nil and buildSearchIndex skips the vector pass.
|
||
// Safe to call zero or one times; subsequent calls silently replace.
|
||
func (mi *MultiIndexer) SetEmbedder(e embedding.Provider) {
|
||
mi.mu.Lock()
|
||
defer mi.mu.Unlock()
|
||
mi.embedder = e
|
||
}
|
||
|
||
// npmAliasResolver builds a resolver.NpmAliasResolver covering every
|
||
// tracked repo's on-disk root. Installed on the global post-pass
|
||
// resolver and the cross-repo resolver so a JS/TS import declared
|
||
// through an npm alias resolves to a locally-vendored real package
|
||
// anywhere in the workspace. Returns nil when no repo has a usable
|
||
// root — callers treat that as "no alias rewriting".
|
||
func (mi *MultiIndexer) npmAliasResolver() resolver.NpmAliasResolver {
|
||
roots := map[string]string{}
|
||
for prefix, meta := range mi.AllMetadata() {
|
||
if meta != nil && meta.RootPath != "" {
|
||
roots[prefix] = meta.RootPath
|
||
}
|
||
}
|
||
idx := newNpmAliasIndex(roots)
|
||
if idx == nil {
|
||
return nil
|
||
}
|
||
return idx.Resolve
|
||
}
|
||
|
||
// workspaceMembershipResolver builds a resolver.WorkspaceMembership
|
||
// covering every tracked repo's on-disk root. Installed on the global
|
||
// post-pass resolver and the cross-repo resolver so a same-named import
|
||
// collision is broken in favour of the importer's own package-manager
|
||
// workspace member. Returns nil when no repo is a workspace root —
|
||
// callers treat that as "no workspace signal".
|
||
func (mi *MultiIndexer) workspaceMembershipResolver() resolver.WorkspaceMembership {
|
||
roots := map[string]string{}
|
||
for prefix, meta := range mi.AllMetadata() {
|
||
if meta != nil && meta.RootPath != "" {
|
||
roots[prefix] = meta.RootPath
|
||
}
|
||
}
|
||
idx := newWorkspaceMembershipIndex(roots)
|
||
if idx == nil {
|
||
return nil
|
||
}
|
||
return idx.Lookup
|
||
}
|
||
|
||
// newPerRepoIndexer constructs a per-repo Indexer with the standard
|
||
// MultiIndexer wiring (shared search backend, embedder if configured,
|
||
// deferred-global-passes flag propagated). Centralised so the flag
|
||
// plumbing stays in one place.
|
||
func (mi *MultiIndexer) newPerRepoIndexer(cfg config.IndexConfig) *Indexer {
|
||
idx := New(mi.graph, mi.registry, cfg, mi.logger)
|
||
idx.search = mi.search
|
||
if mi.embedder != nil {
|
||
idx.SetEmbedder(mi.embedder)
|
||
}
|
||
idx.SetDeferGlobalPasses(mi.deferGlobalPasses)
|
||
idx.SetDeferResolve(mi.deferResolve)
|
||
idx.SetSkipVectorBuild(mi.skipVectorBuild)
|
||
idx.SetEmbeddingChunkOptions(mi.embedChunkOpts)
|
||
idx.SetEmbeddingMaxSymbols(mi.embedMaxSymbols)
|
||
idx.SetEmbeddingAPIConcurrency(mi.embedAPIConcurrency)
|
||
if mi.resolverLSPHelper != nil {
|
||
idx.SetResolverLSPHelper(mi.resolverLSPHelper)
|
||
}
|
||
if mi.semanticMgr != nil {
|
||
idx.SetSemanticManager(mi.semanticMgr)
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// SetEmbeddingChunkOptions installs the AST sub-chunking configuration
|
||
// every per-repo Indexer this MultiIndexer constructs should use, and
|
||
// re-applies it to every per-repo Indexer already built. Call before
|
||
// IndexAll / TrackRepo so the warmup indexers pick it up.
|
||
func (mi *MultiIndexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) {
|
||
mi.mu.Lock()
|
||
mi.embedChunkOpts = opts
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetEmbeddingChunkOptions(opts)
|
||
}
|
||
}
|
||
|
||
// SetEmbeddingMaxSymbols installs the vector-index size cap every
|
||
// per-repo Indexer this MultiIndexer constructs should use, and
|
||
// re-applies it to every per-repo Indexer already built. Zero keeps
|
||
// the built-in default.
|
||
func (mi *MultiIndexer) SetEmbeddingMaxSymbols(n int) {
|
||
mi.mu.Lock()
|
||
mi.embedMaxSymbols = n
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetEmbeddingMaxSymbols(n)
|
||
}
|
||
}
|
||
|
||
// SetEmbeddingAPIConcurrency installs the parallel-embedding-request
|
||
// cap every per-repo Indexer this MultiIndexer constructs should use,
|
||
// and re-applies it to every per-repo Indexer already built. Zero
|
||
// keeps the built-in default; the cap only affects API-backed
|
||
// embedders.
|
||
func (mi *MultiIndexer) SetEmbeddingAPIConcurrency(n int) {
|
||
mi.mu.Lock()
|
||
mi.embedAPIConcurrency = n
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetEmbeddingAPIConcurrency(n)
|
||
}
|
||
}
|
||
|
||
// SetSemanticManager installs the semantic enrichment manager every
|
||
// per-repo Indexer this MultiIndexer constructs should use, and
|
||
// re-applies it to every per-repo Indexer already built. Without
|
||
// this call, daemon-mode enrichment produces zero results because
|
||
// the per-repo Indexers never receive the semantic manager.
|
||
func (mi *MultiIndexer) SetSemanticManager(m *semantic.Manager) {
|
||
mi.mu.Lock()
|
||
mi.semanticMgr = m
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetSemanticManager(m)
|
||
}
|
||
}
|
||
|
||
// SetSkipVectorBuild controls whether per-repo Indexers constructed
|
||
// from now on skip the embedding pass in buildSearchIndex (text index
|
||
// only). The daemon enables it for the warmup loop when a snapshot
|
||
// already carries the workspace vector index, then disables it and
|
||
// restores the cached index once warmup finishes. It also re-applies
|
||
// the flag to every per-repo Indexer already constructed so a flag
|
||
// flip mid-lifecycle takes effect everywhere.
|
||
func (mi *MultiIndexer) SetSkipVectorBuild(skip bool) {
|
||
mi.mu.Lock()
|
||
mi.skipVectorBuild = skip
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetSkipVectorBuild(skip)
|
||
}
|
||
}
|
||
|
||
// SetResolverLSPHelper installs the resolve-time LSP helper used by
|
||
// every per-repo Indexer this MultiIndexer constructs from now on,
|
||
// and by the global post-pass resolver in RunDeferredPassesAll. Pass
|
||
// nil to detach. Safe to call zero or one times; subsequent calls
|
||
// silently replace and propagate to every existing per-repo indexer.
|
||
func (mi *MultiIndexer) SetResolverLSPHelper(h resolver.LSPHelper) {
|
||
mi.mu.Lock()
|
||
mi.resolverLSPHelper = h
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.Unlock()
|
||
for _, idx := range live {
|
||
idx.SetResolverLSPHelper(h)
|
||
}
|
||
}
|
||
|
||
// SetOnRepoTracked installs a callback fired once per TrackRepoCtx
|
||
// after the repo's prefix + absolute path have been resolved but
|
||
// before indexing starts. The daemon registers per-repo resolver-
|
||
// time LSP helpers from this hook so runtime-added repos
|
||
// participate in the N5 hot path. Pass nil to detach.
|
||
func (mi *MultiIndexer) SetOnRepoTracked(fn func(prefix, absPath string)) {
|
||
mi.mu.Lock()
|
||
mi.onRepoTracked = fn
|
||
mi.mu.Unlock()
|
||
}
|
||
|
||
// BeginBatch enables deferred-global-passes mode for every per-repo
|
||
// Indexer that this MultiIndexer constructs after the call AND for
|
||
// every Indexer already in mi.indexers (so ReconcileAll's per-repo
|
||
// IncrementalReindex calls also skip the O(global) walks). Pair with
|
||
// EndBatch.
|
||
func (mi *MultiIndexer) BeginBatch() {
|
||
mi.mu.Lock()
|
||
defer mi.mu.Unlock()
|
||
mi.deferGlobalPasses = true
|
||
for _, idx := range mi.indexers {
|
||
idx.SetDeferGlobalPasses(true)
|
||
}
|
||
}
|
||
|
||
// BeginParallelBatch is BeginBatch plus parallel-safety: it also
|
||
// propagates SetDeferResolve(true) to every per-repo Indexer
|
||
// constructed during the batch. Use this when running the per-repo
|
||
// indexing loop across goroutines (warmup) — the parallel parsers
|
||
// must not race each other inside ResolveAll / contract extract /
|
||
// semantic enrich, which all mutate the shared graph. Pair with
|
||
// EndBatch; call RunDeferredPassesAll between the parallel parse and
|
||
// EndBatch to run the deferred per-repo passes serially.
|
||
func (mi *MultiIndexer) BeginParallelBatch() {
|
||
mi.mu.Lock()
|
||
defer mi.mu.Unlock()
|
||
mi.deferGlobalPasses = true
|
||
mi.deferResolve = true
|
||
for _, idx := range mi.indexers {
|
||
idx.SetDeferGlobalPasses(true)
|
||
}
|
||
}
|
||
|
||
// RunDeferredPassesAll drains the deferred per-repo passes (semantic
|
||
// enrich / contract extract+commit) serially across the indexers the
|
||
// parallel parse populated. Pairs with BeginParallelBatch: the parallel
|
||
// loop parses with deferResolve on; this serial loop runs the passes that
|
||
// would otherwise race on the shared graph. The references-completeness
|
||
// resolve runs ahead of this in RunPreEnrichResolve (so the daemon can mark
|
||
// itself queryable before enrichment); the per-repo resolver pass is
|
||
// suppressed here because resolver.ResolveAll walks the entire shared graph
|
||
// — paying it R times is O(R · E). One master resolver.New(graph).ResolveAll
|
||
// runs at the end to lift the placeholder edges enrichment + contracts added.
|
||
//
|
||
// Returns the number of repos whose deferred semantic enrichment was
|
||
// actually dispatched (pendingEnrich set, or forced via
|
||
// GORTEX_WARMUP_FORCE_ENRICH) rather than skipped as unchanged. Sampled
|
||
// before runDeferredEnrichParallel runs, since a successful non-partial
|
||
// pass clears the flag it reads.
|
||
// SeedPendingEnrichAll re-arms the deferred-enrichment gate for every tracked
|
||
// repo whose persisted enrichment is known-incomplete at its current clean HEAD
|
||
// (see Indexer.MaybeSeedPendingEnrich). The daemon warmup calls it after the
|
||
// parallel parse and before RunDeferredPassesAll so a repo left partial or
|
||
// abandoned by a prior process resumes even when no file changed this run.
|
||
// Returns the number of repos that will enrich (already pending plus newly
|
||
// seeded) — the caller uses a non-zero count to run the deferred passes on a
|
||
// warm restart that changed nothing on disk.
|
||
func (mi *MultiIndexer) SeedPendingEnrichAll() int {
|
||
mi.mu.RLock()
|
||
live := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
live = append(live, idx)
|
||
}
|
||
mi.mu.RUnlock()
|
||
pending := 0
|
||
for _, idx := range live {
|
||
if idx.MaybeSeedPendingEnrich() {
|
||
pending++
|
||
}
|
||
}
|
||
return pending
|
||
}
|
||
|
||
func (mi *MultiIndexer) RunDeferredPassesAll(ctx context.Context) int {
|
||
mi.mu.RLock()
|
||
indexers := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
indexers = append(indexers, idx)
|
||
}
|
||
mi.mu.RUnlock()
|
||
forced := os.Getenv("GORTEX_WARMUP_FORCE_ENRICH") == "1"
|
||
enrichScheduled := 0
|
||
for _, idx := range indexers {
|
||
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
|
||
continue
|
||
}
|
||
if idx.pendingEnrich.Load() || forced {
|
||
enrichScheduled++
|
||
}
|
||
}
|
||
for _, idx := range indexers {
|
||
idx.SetSkipResolveInDeferred(true)
|
||
}
|
||
// Per-repo deferred work in three phases. gomod (materialises dep
|
||
// contract nodes) and contracts (extract + commit, which walk repo edges)
|
||
// mutate the shared graph in ways that race across repos, so they stay
|
||
// serial. Enrichment is the dominant cost — LSP background-indexing and
|
||
// hover I/O — and is safe to overlap across repos: the manager hands each
|
||
// repo its own LSP provider instance, go-types stashes per repo, and every
|
||
// provider serialises its graph mutations on the backend resolve mutex.
|
||
for _, idx := range indexers {
|
||
idx.runDeferredGoMod()
|
||
}
|
||
mi.runDeferredEnrichParallel(indexers)
|
||
for _, idx := range indexers {
|
||
idx.runDeferredContracts()
|
||
}
|
||
for _, idx := range indexers {
|
||
idx.SetSkipResolveInDeferred(false)
|
||
}
|
||
// Re-run the master same-repo resolver to lift the placeholder edges the
|
||
// enrichment + contract passes just added. The references-completeness
|
||
// resolve already ran ahead of enrichment in RunPreEnrichResolve, so this
|
||
// is the idempotent catch-up pass for edges minted during enrichment.
|
||
// Whole-graph (nil scope): enrichment can mint placeholder edges in any
|
||
// repo, and scoping this catch-up is left to a follow-up.
|
||
mi.runMasterResolve(nil)
|
||
return enrichScheduled
|
||
}
|
||
|
||
// runMasterResolve runs one same-repo resolver over the whole shared graph,
|
||
// lifting every placeholder edge to its canonical target. Split out so the
|
||
// pre-enrichment resolve stage (RunPreEnrichResolve) and the post-enrichment
|
||
// catch-up (RunDeferredPassesAll) share one implementation.
|
||
// scope, when non-empty, restricts the pass to the edges that could resolve
|
||
// into one of the named changed repos (see resolver.SetScope). It is honoured
|
||
// only when scoped global passes are enabled; a nil / empty scope or a
|
||
// disabled switch runs the whole-graph resolve, exactly the prior behaviour.
|
||
func (mi *MultiIndexer) runMasterResolve(scope map[string]struct{}) {
|
||
if mi.graph == nil {
|
||
return
|
||
}
|
||
master := resolver.New(mi.graph)
|
||
master.SetLogger(mi.logger)
|
||
// The master resolve is the only pass with whole-graph evidence, so it is
|
||
// the one allowed to durably flag terminally-unresolved edges (permanently
|
||
// external / stdlib / definition-less) that a later scoped warm resolve can
|
||
// skip. A scoped master pass leaves the flag alone (stamping is gated on an
|
||
// empty scope inside ResolveAll); only a full pass stamps and self-heals.
|
||
master.SetStampTerminal(true)
|
||
// Mirror the resolve-time LSP helper onto the master pass so TS/JS-family
|
||
// edges pick up LSP-precision answers just like the per-repo passes do.
|
||
if mi.resolverLSPHelper != nil {
|
||
master.SetLSPHelper(mi.resolverLSPHelper)
|
||
}
|
||
master.SetNpmAliasResolver(mi.npmAliasResolver())
|
||
master.SetPathAliasResolver(mi.pathAliasResolver())
|
||
master.SetWorkspaceMembership(mi.workspaceMembershipResolver())
|
||
scoped := len(scope) > 0 && mi.scopedGlobalPassesEnabled()
|
||
if scoped {
|
||
master.SetScope(scope)
|
||
}
|
||
mt := time.Now()
|
||
stats := master.ResolveAll()
|
||
mi.logger.Info("DEFERRED-TIMING master.ResolveAll",
|
||
zap.Duration("elapsed", time.Since(mt)),
|
||
zap.Bool("scoped", scoped),
|
||
zap.Int("scope_repos", len(scope)),
|
||
zap.Int("pending_before", stats.PendingBefore),
|
||
zap.Int("pending_after", stats.PendingAfter))
|
||
}
|
||
|
||
// RunPreEnrichResolve runs the resolution stage that makes references queryable
|
||
// ahead of the slow semantic-enrichment pass. It materialises go.mod dep
|
||
// contract nodes (so the resolver's import bridge can re-target Go imports of
|
||
// declared modules), runs the same-repo master resolver to lift every
|
||
// parse-time placeholder edge to its canonical target, then runs the cross-repo
|
||
// resolver so references that span repo boundaries resolve too. Contract-bridge
|
||
// reconciliation is intentionally left to RunGlobalResolve, which runs after the
|
||
// contract pass has committed its nodes.
|
||
//
|
||
// The daemon warmup calls this between the parallel parse and the enrichment
|
||
// phase, then marks itself ready — so find_usages / get_callers return complete
|
||
// results as soon as the graph is queryable, independent of enrichment.
|
||
// scope, when non-empty, restricts the same-repo master resolve to the
|
||
// changed repos (see runMasterResolve / resolver.SetScope). The daemon warmup
|
||
// passes the set of repos that re-indexed so a warm restart of one repo out of
|
||
// many skips a whole-graph same-repo resolve; a nil / empty scope keeps the
|
||
// whole-graph behaviour. The cross-repo resolve below stays whole-graph
|
||
// regardless — it is the only pass that binds an unchanged repo's inbound
|
||
// reference into a symbol a changed repo just added, and those inbound edges
|
||
// must be resolved before the daemon reports ready (see runCrossRepoResolve).
|
||
func (mi *MultiIndexer) RunPreEnrichResolve(ctx context.Context, scope map[string]struct{}) {
|
||
mi.mu.RLock()
|
||
indexers := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
indexers = append(indexers, idx)
|
||
}
|
||
mi.mu.RUnlock()
|
||
for _, idx := range indexers {
|
||
idx.runDeferredGoMod()
|
||
}
|
||
mi.runMasterResolve(scope)
|
||
// Cross-repo references resolve here too so a multi-repo workspace is fully
|
||
// queryable at "ready", not just within each repo. Whole-graph so inbound
|
||
// references from unchanged repos into the changed repos bind before ready.
|
||
mi.runCrossRepoResolve(false)
|
||
}
|
||
|
||
// runDeferredEnrichParallel runs each indexer's semantic enrichment in a
|
||
// bounded worker pool. Concurrency is capped so at most a few LSP servers
|
||
// background-index at once (the memory-sensitive part). The manager pins each
|
||
// repo's LSP provider in-use for the duration of its pass, so the router's
|
||
// LRU evictor never closes a provider another repo is still enriching against.
|
||
func (mi *MultiIndexer) runDeferredEnrichParallel(indexers []*Indexer) {
|
||
// Per-repo language sets computed once from a single graph-stats scan,
|
||
// shared by the spec-grouped ordering and the batch pool-raise sizing
|
||
// so the Manager's per-repo enrichment scan is not duplicated here.
|
||
langSets := mi.batchLanguageSets(indexers)
|
||
// Deterministic, spec-grouped order: repos needing the same language
|
||
// servers run contiguously so the router's capped provider pool cycles
|
||
// through far fewer distinct (spec, workspace) keys — a warmed server
|
||
// stays alive across the runs that need it instead of being evicted and
|
||
// respawned per repo.
|
||
indexers = orderIndexersBySpecGroup(indexers, langSets)
|
||
|
||
conc := enrichConcurrency(len(indexers))
|
||
|
||
// Temporarily raise the router's live-provider cap for the batch so the
|
||
// concurrent passes don't evict each other's warmed servers, restoring
|
||
// it (and logging the churn observed) when the batch drains.
|
||
restore := mi.scopeRouterPoolForBatch(langSets, conc)
|
||
defer restore()
|
||
|
||
if conc <= 1 {
|
||
for _, idx := range indexers {
|
||
idx.runDeferredEnrich()
|
||
}
|
||
return
|
||
}
|
||
|
||
sem := make(chan struct{}, conc)
|
||
var wg sync.WaitGroup
|
||
for _, idx := range indexers {
|
||
wg.Add(1)
|
||
sem <- struct{}{}
|
||
go func(idx *Indexer) {
|
||
defer func() { <-sem; wg.Done() }()
|
||
idx.runDeferredEnrich()
|
||
}(idx)
|
||
}
|
||
wg.Wait()
|
||
}
|
||
|
||
// maxBatchProviders caps the temporary live-provider pool raise during
|
||
// batch enrichment. Even a batch touching many distinct language servers
|
||
// at high concurrency is held to this ceiling so warmup cannot spawn an
|
||
// unbounded number of LSP subprocesses at once.
|
||
const maxBatchProviders = 12
|
||
|
||
// batchLanguageSets returns each indexer's sorted set of present languages,
|
||
// computed from a single RepoStats scan (one pass over the shared graph)
|
||
// rather than a per-repo node scan. The sets drive both the spec-grouped
|
||
// enrich ordering and the batch pool-raise sizing.
|
||
func (mi *MultiIndexer) batchLanguageSets(indexers []*Indexer) map[*Indexer][]string {
|
||
out := make(map[*Indexer][]string, len(indexers))
|
||
var stats map[string]graph.GraphStats
|
||
if mi.graph != nil {
|
||
stats = mi.graph.RepoStats()
|
||
}
|
||
for _, idx := range indexers {
|
||
var langs []string
|
||
if s, ok := stats[idx.repoPrefix]; ok {
|
||
for l := range s.ByLanguage {
|
||
langs = append(langs, l)
|
||
}
|
||
}
|
||
sort.Strings(langs)
|
||
out[idx] = langs
|
||
}
|
||
return out
|
||
}
|
||
|
||
// orderIndexersBySpecGroup returns a stable, deterministic ordering of
|
||
// indexers grouped by their language set (so repos needing the same LSP
|
||
// servers run contiguously), breaking ties by repo prefix. It does not
|
||
// mutate the input slice.
|
||
func orderIndexersBySpecGroup(indexers []*Indexer, langSets map[*Indexer][]string) []*Indexer {
|
||
out := make([]*Indexer, len(indexers))
|
||
copy(out, indexers)
|
||
sort.SliceStable(out, func(i, j int) bool {
|
||
ki := strings.Join(langSets[out[i]], ",")
|
||
kj := strings.Join(langSets[out[j]], ",")
|
||
if ki != kj {
|
||
return ki < kj
|
||
}
|
||
return out[i].repoPrefix < out[j].repoPrefix
|
||
})
|
||
return out
|
||
}
|
||
|
||
// distinctBatchSpecs counts the enabled, available LSP specs whose
|
||
// languages intersect any language present in the batch — i.e. how many
|
||
// distinct language servers the batch will actually drive.
|
||
func distinctBatchSpecs(langSets map[*Indexer][]string, router semantic.LSPRouter) int {
|
||
langs := make(map[string]bool)
|
||
for _, ls := range langSets {
|
||
for _, l := range ls {
|
||
langs[l] = true
|
||
}
|
||
}
|
||
if len(langs) == 0 {
|
||
return 0
|
||
}
|
||
seen := make(map[string]bool)
|
||
for _, name := range router.EnabledSpecNames() {
|
||
if !router.SpecAvailable(name) {
|
||
continue
|
||
}
|
||
for _, l := range router.SpecLanguages(name) {
|
||
if langs[l] {
|
||
seen[name] = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return len(seen)
|
||
}
|
||
|
||
// scopeRouterPoolForBatch raises the LSP router's live-provider cap to
|
||
// enrichConcurrency × distinct-provider-specs (ceiling maxBatchProviders)
|
||
// for the duration of a batch enrichment pass, so the concurrent passes
|
||
// don't evict each other's warmed servers. It returns a restore closure
|
||
// that puts the cap back and logs the eviction churn observed during the
|
||
// batch. Safe no-op when no router is installed.
|
||
func (mi *MultiIndexer) scopeRouterPoolForBatch(langSets map[*Indexer][]string, conc int) func() {
|
||
if mi.semanticMgr == nil {
|
||
return func() {}
|
||
}
|
||
router := mi.semanticMgr.LSPRouter()
|
||
if router == nil {
|
||
return func() {}
|
||
}
|
||
before := router.EvictionCount()
|
||
old := router.MaxAlive()
|
||
raised := false
|
||
if specs := distinctBatchSpecs(langSets, router); specs > 0 {
|
||
needed := conc * specs
|
||
if needed > maxBatchProviders {
|
||
needed = maxBatchProviders
|
||
}
|
||
if needed > old {
|
||
router.SetMaxAlive(needed)
|
||
raised = true
|
||
mi.logger.Info("LSP router pool temporarily raised for batch enrichment",
|
||
zap.Int("from", old),
|
||
zap.Int("to", needed),
|
||
zap.Int("distinct_specs", specs),
|
||
zap.Int("concurrency", conc),
|
||
)
|
||
}
|
||
}
|
||
return func() {
|
||
if raised {
|
||
router.SetMaxAlive(old)
|
||
}
|
||
mi.logger.Info("batch enrichment LSP provider churn",
|
||
zap.Uint64("evictions", router.EvictionCount()-before),
|
||
zap.Bool("pool_raised", raised),
|
||
)
|
||
}
|
||
}
|
||
|
||
// enrichConcurrency caps how many repos enrich at once during batch warmup.
|
||
// Half the CPUs, clamped to [1,4] and to the repo count — a few concurrent
|
||
// LSP servers background-index in parallel without a memory blow-up.
|
||
func enrichConcurrency(repos int) int {
|
||
c := runtime.NumCPU() / 2
|
||
if c > 4 {
|
||
c = 4
|
||
}
|
||
if c < 1 {
|
||
c = 1
|
||
}
|
||
if c > repos {
|
||
c = repos
|
||
}
|
||
return c
|
||
}
|
||
|
||
// EndBatch turns off deferred-global-passes mode and runs the graph-
|
||
// wide derivation passes (InferImplements, InferOverrides,
|
||
// markTestSymbolsAndEmitEdges) once against the shared graph. Restores
|
||
// the per-Indexer flag too so a subsequent one-off TrackRepoCtx call
|
||
// runs the passes inline as expected.
|
||
// ArmBatchScope records the prefixes of the repos that re-indexed in the
|
||
// batch about to be ended, so the next RunGlobalGraphPasses runs the
|
||
// per-repo clone-detection + clone-index Rebuild passes only for those
|
||
// repos instead of for every tracked repo. An empty set, or scoped global
|
||
// passes being disabled, leaves the scope nil (run all). Only the daemon
|
||
// warmup arms it; every other EndBatch caller keeps the whole-workspace
|
||
// behaviour.
|
||
func (mi *MultiIndexer) ArmBatchScope(changedPrefixes map[string]struct{}) {
|
||
if mi == nil || len(changedPrefixes) == 0 || !mi.scopedGlobalPassesEnabled() {
|
||
return
|
||
}
|
||
mi.mu.Lock()
|
||
mi.batchChangedPrefixes = changedPrefixes
|
||
mi.mu.Unlock()
|
||
}
|
||
|
||
// takeBatchScope returns the armed clone-pass scope and clears it, so the
|
||
// scope governs exactly one RunGlobalGraphPasses run. A nil result means
|
||
// "no scope — run the clone passes for every repo".
|
||
func (mi *MultiIndexer) takeBatchScope() map[string]struct{} {
|
||
mi.mu.Lock()
|
||
scope := mi.batchChangedPrefixes
|
||
mi.batchChangedPrefixes = nil
|
||
mi.mu.Unlock()
|
||
return scope
|
||
}
|
||
|
||
// scopedGlobalPassesEnabled reports whether per-repo global passes may be
|
||
// scoped to the changed-repo set. GORTEX_INDEX_SCOPED_GLOBAL_PASSES
|
||
// overrides the config key; default ON, mirroring Indexer.scopedGlobalPassesEnabled.
|
||
func (mi *MultiIndexer) scopedGlobalPassesEnabled() bool {
|
||
if v := os.Getenv("GORTEX_INDEX_SCOPED_GLOBAL_PASSES"); v != "" {
|
||
return v == "1" || strings.EqualFold(v, "true")
|
||
}
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
for _, idx := range mi.indexers {
|
||
return idx.config.ScopedGlobalPassesEnabledOrDefault()
|
||
}
|
||
return true
|
||
}
|
||
|
||
func (mi *MultiIndexer) EndBatch() {
|
||
mi.mu.Lock()
|
||
mi.deferGlobalPasses = false
|
||
mi.deferResolve = false
|
||
for _, idx := range mi.indexers {
|
||
idx.SetDeferGlobalPasses(false)
|
||
}
|
||
mi.mu.Unlock()
|
||
mi.RunGlobalGraphPasses(context.Background())
|
||
}
|
||
|
||
// ResetBatch clears deferred-batch mode WITHOUT running the graph-wide
|
||
// derivation passes. It is the warm-restart fast-path counterpart to
|
||
// EndBatch: when the warmup reconcile loop observed zero changed files
|
||
// across every repo, the persistent backend already holds every resolved
|
||
// and derived edge from the prior run, so RunGlobalGraphPasses (plus the
|
||
// RunDeferredPassesAll / RunGlobalResolve the caller also skips) would
|
||
// only recompute what's already on disk — the work that turns a warm
|
||
// restart into a 30s–500s stall. The per-Indexer SetDeferGlobalPasses
|
||
// flag is still restored so a later watch-triggered TrackRepoCtx /
|
||
// IncrementalReindex runs its passes inline as normal.
|
||
func (mi *MultiIndexer) ResetBatch() {
|
||
mi.mu.Lock()
|
||
defer mi.mu.Unlock()
|
||
mi.deferGlobalPasses = false
|
||
mi.deferResolve = false
|
||
for _, idx := range mi.indexers {
|
||
idx.SetDeferGlobalPasses(false)
|
||
}
|
||
}
|
||
|
||
// RunGlobalGraphPasses runs the graph-wide derivation passes once
|
||
// against the shared graph: InferImplements (structural interface
|
||
// satisfaction), InferOverrides (method-level overrides on
|
||
// extends/implements/composes parents), and markTestSymbolsAndEmitEdges
|
||
// (test→subject EdgeTests). Idempotent — graph.AddEdge dedupes by
|
||
// edgeKey and the resolver passes skip already-present parents.
|
||
func (mi *MultiIndexer) RunGlobalGraphPasses(ctx context.Context) {
|
||
if mi.graph == nil {
|
||
return
|
||
}
|
||
reporter := progress.FromContext(ctx)
|
||
r := resolver.New(mi.graph)
|
||
// Unconditional per-sub-pass timing. These global passes run serially and
|
||
// were previously logged only when a pass emitted edges, so a slow but
|
||
// low-yield pass left no breadcrumb — the source of the multi-minute
|
||
// "silent" span after resolve on a cold index.
|
||
globalStart := time.Now()
|
||
|
||
// Acquire the changed-repo scope once for the whole run and derive the two
|
||
// shapes the passes below consume. A nil scope means whole-graph — the
|
||
// fresh-index / one-off behaviour every pass keeps as its fallback — so an
|
||
// unscoped run is byte-identical to before. A non-nil scope (armed by the
|
||
// daemon warmup / hourly janitor via ArmBatchScope) narrows each pass to the
|
||
// repos that re-indexed this batch: an unchanged repo's derived edges are
|
||
// already on disk and were never dropped, so re-deriving them is skipped.
|
||
// - changedPrefixes: the changed-repo prefix set, for the edge-driven passes
|
||
// (capability, test edges, framework synthesis, external calls), each of
|
||
// which owns an edge iff its FROM node is in a changed repo.
|
||
// - scopedTypeIfaceIDs: the changed repos' type/interface node IDs, for the
|
||
// implements/overrides inference. The scoped inference keeps a pair when
|
||
// EITHER endpoint is in this set, so a cross-repo override whose child is
|
||
// in a changed repo and whose parent is in an unchanged one (or vice
|
||
// versa) is still re-derived; structural implements never crosses repos
|
||
// (its same-repo gate), so scoping both its sides here is complete.
|
||
scope := mi.takeBatchScope()
|
||
var changedPrefixes map[string]bool
|
||
var scopedTypeIfaceIDs map[string]bool
|
||
if scope != nil {
|
||
changedPrefixes = make(map[string]bool, len(scope))
|
||
scopedTypeIfaceIDs = map[string]bool{}
|
||
for prefix := range scope {
|
||
changedPrefixes[prefix] = true
|
||
if prefix == "" {
|
||
continue
|
||
}
|
||
for _, n := range repoNodesLightOrFull(mi.graph, prefix) {
|
||
if n == nil {
|
||
continue
|
||
}
|
||
if n.Kind == graph.KindType || n.Kind == graph.KindInterface {
|
||
scopedTypeIfaceIDs[n.ID] = true
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
implStart := time.Now()
|
||
implAdded := 0
|
||
switch {
|
||
case scope == nil:
|
||
implAdded = r.InferImplements()
|
||
case len(scopedTypeIfaceIDs) > 0:
|
||
// Empty set => no type/interface changed in the batch => no inferred
|
||
// implements edge could have been dropped, so the pass is skipped.
|
||
implAdded = r.InferImplementsScoped(scopedTypeIfaceIDs, scopedTypeIfaceIDs)
|
||
}
|
||
mi.logger.Info("global pass: infer implements",
|
||
zap.Int("added", implAdded),
|
||
zap.Bool("scoped", scope != nil),
|
||
zap.Duration("elapsed", time.Since(implStart)))
|
||
overStart := time.Now()
|
||
overAdded := 0
|
||
switch {
|
||
case scope == nil:
|
||
overAdded = r.InferOverrides()
|
||
case len(scopedTypeIfaceIDs) > 0:
|
||
overAdded = r.InferOverridesScoped(scopedTypeIfaceIDs)
|
||
}
|
||
mi.logger.Info("global pass: infer overrides",
|
||
zap.Int("added", overAdded),
|
||
zap.Bool("scoped", scope != nil),
|
||
zap.Duration("elapsed", time.Since(overStart)))
|
||
testStart := time.Now()
|
||
marked, emitted := markTestSymbolsAndEmitEdgesScoped(mi.graph, changedPrefixes)
|
||
mi.logger.Info("global pass: test edges",
|
||
zap.Int("test_symbols", marked),
|
||
zap.Int("edges", emitted),
|
||
zap.Duration("elapsed", time.Since(testStart)))
|
||
capStart := time.Now()
|
||
capRe, capEp, capFa := synthesizeCapabilityEdgesScoped(mi.graph, changedPrefixes)
|
||
mi.logger.Info("global pass: capability edges",
|
||
zap.Int("reads_env", capRe),
|
||
zap.Int("executes_process", capEp),
|
||
zap.Int("accesses_field", capFa),
|
||
zap.Duration("elapsed", time.Since(capStart)))
|
||
// Clone detection is PER-REPOSITORY: each tracked repo gets its own
|
||
// finalise + detect over its own nodes (scoped by RepoPrefix), so no
|
||
// cross-repo candidate pair is ever formed and each repo's boilerplate
|
||
// CMS / threshold is computed from that repo's bodies alone. This
|
||
// matches the per-repo incremental maintainer (cloneIndex.Rebuild /
|
||
// UpdateFuncs) so the batch and incremental edge sets agree.
|
||
reporter.Report("clone detection pass (global)", 0, 0)
|
||
mi.mu.RLock()
|
||
cloneIdx := make([]*Indexer, 0, len(mi.indexers))
|
||
for _, idx := range mi.indexers {
|
||
cloneIdx = append(cloneIdx, idx)
|
||
}
|
||
mi.mu.RUnlock()
|
||
// Scope to the repos that re-indexed this batch when the warmup armed a
|
||
// batch scope. An unchanged repo's clone edges are already on disk and
|
||
// its incremental clone index is reseeded lazily on its first later edit
|
||
// (indexFile: if !built → Rebuild), so skipping its full-graph detect +
|
||
// Rebuild here is sound and cuts an N-repo warm restart from N full-graph
|
||
// clone walks to just the changed repos'. A nil scope runs every repo. The
|
||
// scope is taken once at the top of RunGlobalGraphPasses and shared by every
|
||
// pass; the clone passes reuse it here (takeBatchScope must not be called
|
||
// twice — it clears the armed scope on the first read).
|
||
inCloneScope := func(prefix string) bool {
|
||
if scope == nil {
|
||
return true
|
||
}
|
||
_, ok := scope[prefix]
|
||
return ok
|
||
}
|
||
cloneDetectStart := time.Now()
|
||
clonesDetected := 0
|
||
for _, idx := range cloneIdx {
|
||
if !inCloneScope(idx.repoPrefix) {
|
||
continue
|
||
}
|
||
clonesDetected++
|
||
// Per-repo threshold, NOT a max-over-repos value: the batch must use
|
||
// the same cutoff the per-repo incremental maintainer uses
|
||
// (UpdateFuncs/Rebuild → idx.cloneThreshold()), or the batch and
|
||
// incremental edge sets diverge for any repo whose configured
|
||
// threshold differs from the workspace maximum.
|
||
if cs := detectClonesAndEmitEdgesCtx(ctx, mi.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
|
||
mi.logger.Info("clone edges emitted (global)",
|
||
zap.String("repo", idx.repoPrefix),
|
||
zap.Int("items", cs.Items),
|
||
zap.Int("clone_pairs", cs.Pairs),
|
||
zap.Int("edges", cs.Edges),
|
||
zap.Int("skipped_buckets", cs.SkippedBuckets),
|
||
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
|
||
zap.Int("diffused_pairs", cs.DiffusedPairs),
|
||
zap.Int("diffused_edges", cs.DiffusedEdges),
|
||
)
|
||
}
|
||
}
|
||
// Seed each per-repo indexer's incremental clone index from the
|
||
// freshly-baselined signatures + sidecar (scoped to that repo's
|
||
// prefix) so steady-state single-file edits after this batch go
|
||
// incremental instead of re-running the whole-graph pass per file.
|
||
cloneRebuildStart := time.Now()
|
||
clonesRebuilt := 0
|
||
for _, idx := range cloneIdx {
|
||
if !inCloneScope(idx.repoPrefix) {
|
||
continue
|
||
}
|
||
if idx.cloneIndex != nil {
|
||
idx.cloneIndex.Rebuild(mi.graph, idx.repoPrefix)
|
||
clonesRebuilt++
|
||
}
|
||
}
|
||
// Aggregate timing for the clone passes — previously the most expensive
|
||
// and least observable part of a warm restart (the per-repo logs above
|
||
// only fire when a repo actually has clone pairs, so a long run could
|
||
// pass with no breadcrumbs).
|
||
mi.logger.Info("clone passes done (global)",
|
||
zap.Bool("scoped", scope != nil),
|
||
zap.Int("repos_total", len(cloneIdx)),
|
||
zap.Int("detected", clonesDetected),
|
||
zap.Int("rebuilt", clonesRebuilt),
|
||
zap.Duration("detect_elapsed", cloneRebuildStart.Sub(cloneDetectStart)),
|
||
zap.Duration("rebuild_elapsed", time.Since(cloneRebuildStart)),
|
||
)
|
||
// Framework dynamic-dispatch synthesis (gRPC stubs, Temporal
|
||
// workflow→activity, in-process / native event channels, native
|
||
// bridges). After InferImplements/InferOverrides (the
|
||
// interface-satisfaction signals) and before DetectCrossRepoEdges so
|
||
// a cross-repo synthesized call gets its parallel cross_repo_calls
|
||
// edge.
|
||
reporter.Report("framework dispatch synthesis (global)", 0, 0)
|
||
fwStart := time.Now()
|
||
fwRep := resolver.RunFrameworkSynthesizersScoped(mi.graph, changedPrefixes)
|
||
mi.logger.Info("global pass: framework dispatch synthesis",
|
||
zap.Int("edges", fwRep.Total),
|
||
zap.Any("per_synthesizer", fwRep.Per),
|
||
zap.Int64("gate_ms", fwRep.GateMillis),
|
||
zap.Int64("claim_ms", fwRep.ClaimMillis),
|
||
zap.Int64("demote_ms", fwRep.DemoteMillis),
|
||
zap.Duration("elapsed", time.Since(fwStart)))
|
||
// External-call placeholder synthesis (opt-in). Runs after the
|
||
// stub passes so only genuinely un-indexed external targets are
|
||
// left to materialise into call-chain terminals.
|
||
reporter.Report("external-call synthesis (global)", 0, 0)
|
||
extStart := time.Now()
|
||
extEnabled := mi.externalCallSynthesisEnabled()
|
||
extCalls := 0
|
||
if scope != nil {
|
||
extCalls = resolver.SynthesizeExternalCallsForRepos(mi.graph, extEnabled, changedPrefixes)
|
||
} else {
|
||
extCalls = resolver.SynthesizeExternalCalls(mi.graph, extEnabled)
|
||
}
|
||
mi.logger.Info("global pass: external-call synthesis",
|
||
zap.Int("edges", extCalls),
|
||
zap.Bool("scoped", scope != nil),
|
||
zap.Duration("elapsed", time.Since(extStart)))
|
||
// Cross-repo edge layer. Runs after InferImplements / InferOverrides
|
||
// so the implements / extends edges they materialise across repo
|
||
// boundaries pick up their parallel cross_repo_* edges.
|
||
reporter.Report("cross-repo edges (global)", 0, 0)
|
||
crStart := time.Now()
|
||
crossRepoEdges := resolver.DetectCrossRepoEdges(mi.graph)
|
||
mi.logger.Info("global pass: cross-repo edges",
|
||
zap.Int("edges", crossRepoEdges),
|
||
zap.Duration("elapsed", time.Since(crStart)))
|
||
mi.logger.Info("global passes complete",
|
||
zap.Duration("total", time.Since(globalStart)))
|
||
}
|
||
|
||
// repoNodesLightOrFull returns a repo's nodes for read-only structural
|
||
// inspection (id / kind / repo prefix), preferring the meta-less
|
||
// LightNodeReader fast path when the backend implements it so the enriched meta
|
||
// blob a scope build never reads stays server-side. The returned nodes MUST NOT
|
||
// be written back through AddNode/AddBatch — the light projection drops any
|
||
// non-promoted meta — which holds here because the only caller reads struct
|
||
// fields to build an ID set and discards the nodes. Falls back to the full
|
||
// GetRepoNodes when the backend (e.g. in-memory) has no separate blob to skip.
|
||
func repoNodesLightOrFull(g graph.Store, prefix string) []*graph.Node {
|
||
if lr, ok := g.(graph.LightNodeReader); ok {
|
||
return lr.GetRepoNodesLight(prefix)
|
||
}
|
||
return g.GetRepoNodes(prefix)
|
||
}
|
||
|
||
// externalCallSynthesisEnabled resolves whether external-call placeholder
|
||
// synthesis should run over the shared graph. The pass is graph-wide, so
|
||
// it is enabled when any tracked repo opted in — a repo that wants the
|
||
// external hops in its call chains gets them even when a sibling repo
|
||
// left the option off.
|
||
func (mi *MultiIndexer) externalCallSynthesisEnabled() bool {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
for _, idx := range mi.indexers {
|
||
if idx.externalCallSynthesisEnabled() {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// NewMultiIndexer creates a MultiIndexer.
|
||
func NewMultiIndexer(
|
||
g graph.Store,
|
||
reg *parser.Registry,
|
||
s search.Backend,
|
||
cm *config.ConfigManager,
|
||
logger *zap.Logger,
|
||
) *MultiIndexer {
|
||
return &MultiIndexer{
|
||
graph: g,
|
||
registry: reg,
|
||
search: s,
|
||
repos: make(map[string]*RepoMetadata),
|
||
indexers: make(map[string]*Indexer),
|
||
configMgr: cm,
|
||
logger: logger,
|
||
}
|
||
}
|
||
|
||
// IndexAll indexes all active repos concurrently. Each repo gets its own
|
||
// Indexer instance with repo-specific config. Returns per-repo IndexResults.
|
||
func (mi *MultiIndexer) IndexAll() (map[string]*IndexResult, error) {
|
||
return mi.IndexScoped("", "")
|
||
}
|
||
|
||
// IndexScoped is IndexAll restricted to repos whose workspace and
|
||
// project slugs match. Empty filters disable that axis (so empty/empty
|
||
// is equivalent to IndexAll). Resolution honours the same precedence
|
||
// as resolveWorkspaceID/resolveProjectID — RepoEntry override →
|
||
// `.gortex.yaml::workspace` → repo prefix — so a `gortex server
|
||
// --workspace foo` invocation matches both repos that declare
|
||
// `workspace: foo` in their own `.gortex.yaml` and repos pinned to
|
||
// `foo` from the user's global config.
|
||
//
|
||
// Returns an error when the filters exclude every active repo, so a
|
||
// `--workspace typo` surfaces as a startup failure rather than a
|
||
// silently empty graph.
|
||
func (mi *MultiIndexer) IndexScoped(workspaceSlug, projectSlug string) (map[string]*IndexResult, error) {
|
||
repos := mi.configMgr.ActiveRepos()
|
||
if len(repos) == 0 {
|
||
return nil, nil
|
||
}
|
||
if workspaceSlug != "" || projectSlug != "" {
|
||
filtered := mi.filterReposByScope(repos, workspaceSlug, projectSlug)
|
||
if len(filtered) == 0 {
|
||
return nil, fmt.Errorf("scope filter matched zero of %d active repos (workspace=%q project=%q)", len(repos), workspaceSlug, projectSlug)
|
||
}
|
||
repos = filtered
|
||
}
|
||
|
||
// Single-repo mode: delegate without prefixing.
|
||
if len(repos) == 1 {
|
||
r, err := mi.indexSingleRepo(repos[0])
|
||
if err == nil {
|
||
mi.ReconcileContractEdges()
|
||
}
|
||
return r, err
|
||
}
|
||
|
||
r, err := mi.indexMultiRepo(repos)
|
||
if err == nil {
|
||
mi.ReconcileContractEdges()
|
||
}
|
||
return r, err
|
||
}
|
||
|
||
// filterReposByScope returns the subset of repos whose resolved
|
||
// workspace and project slugs match the supplied filters. Empty
|
||
// filters disable that axis. Loads each repo's `.gortex.yaml` first so
|
||
// resolution sees the workspace/project declared there — matching only
|
||
// against `RepoEntry.Workspace` would miss repos that declare their
|
||
// slug in their own config file (the typical case for first-party
|
||
// repos).
|
||
func (mi *MultiIndexer) filterReposByScope(repos []config.RepoEntry, workspaceSlug, projectSlug string) []config.RepoEntry {
|
||
if workspaceSlug == "" && projectSlug == "" {
|
||
return repos
|
||
}
|
||
out := make([]config.RepoEntry, 0, len(repos))
|
||
for _, e := range repos {
|
||
absPath, err := filepath.Abs(e.Path)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
prefix := config.ResolvePrefix(e)
|
||
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
|
||
cfg := mi.configMgr.GetRepoConfig(prefix)
|
||
entryCopy := e
|
||
if workspaceSlug != "" && resolveWorkspaceID(&entryCopy, cfg, prefix) != workspaceSlug {
|
||
continue
|
||
}
|
||
if projectSlug != "" && resolveProjectID(&entryCopy, cfg, prefix) != projectSlug {
|
||
continue
|
||
}
|
||
out = append(out, e)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// indexSingleRepo indexes a single repo without prefixing for backward compatibility.
|
||
func (mi *MultiIndexer) indexSingleRepo(entry config.RepoEntry) (map[string]*IndexResult, error) {
|
||
absPath, err := filepath.Abs(entry.Path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
|
||
}
|
||
|
||
prefix := config.ResolvePrefix(entry)
|
||
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
|
||
cfg := mi.configMgr.GetRepoConfig(prefix)
|
||
|
||
idx := mi.newPerRepoIndexer(cfg.Index)
|
||
entryCopy := entry
|
||
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
|
||
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
|
||
// No repo prefix in single-repo mode.
|
||
|
||
result, err := idx.Index(absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("indexing %s: %w", absPath, err)
|
||
}
|
||
result.RepoPrefix = prefix
|
||
|
||
identity, _ := DetectIdentity(absPath)
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[prefix] = &RepoMetadata{
|
||
RepoPrefix: prefix,
|
||
RootPath: absPath,
|
||
Identity: identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
IsWorktree: ResolveWorktree(absPath).IsWorktree,
|
||
Unprefixed: true,
|
||
}
|
||
mi.indexers[prefix] = idx
|
||
mi.mu.Unlock()
|
||
|
||
return map[string]*IndexResult{prefix: result}, nil
|
||
}
|
||
|
||
// migrateLoneUnprefixedRepoCtx re-mints the formerly-lone repo's nodes
|
||
// with its real prefix the moment a second repo joins. Without it, the
|
||
// first repo's unprefixed nodes become unreachable (the empty-prefix
|
||
// fallback disarms at two repos) until a cold reindex. Ordering is
|
||
// crash-safe: the prefixed re-index lands first, the stale unprefixed
|
||
// nodes are evicted after — an interruption leaves both ID forms
|
||
// resolvable rather than neither.
|
||
func (mi *MultiIndexer) migrateLoneUnprefixedRepoCtx(ctx context.Context) {
|
||
mi.mu.RLock()
|
||
var oldPrefix string
|
||
var oldMeta *RepoMetadata
|
||
if len(mi.repos) == 1 {
|
||
for p, m := range mi.repos {
|
||
if m != nil && m.Unprefixed && m.RootPath != "" {
|
||
oldPrefix, oldMeta = p, m
|
||
}
|
||
}
|
||
}
|
||
mi.mu.RUnlock()
|
||
if oldMeta == nil {
|
||
return
|
||
}
|
||
|
||
cfg := mi.configMgr.GetRepoConfig(oldPrefix)
|
||
idx := mi.newPerRepoIndexer(cfg.Index)
|
||
idx.SetRepoPrefix(oldPrefix)
|
||
entry := config.RepoEntry{Path: oldMeta.RootPath, Name: oldPrefix}
|
||
if mi.configMgr != nil {
|
||
for _, e := range mi.configMgr.Global().Repos {
|
||
if pathkey.EqualPaths(e.Path, oldMeta.RootPath) {
|
||
entry = e
|
||
break
|
||
}
|
||
}
|
||
}
|
||
idx.SetWorkspaceID(resolveWorkspaceID(&entry, cfg, oldPrefix))
|
||
idx.SetProjectID(resolveProjectID(&entry, cfg, oldPrefix))
|
||
|
||
result, err := idx.IndexCtx(ctx, oldMeta.RootPath)
|
||
if err != nil {
|
||
mi.logger.Warn("re-prefixing lone repo failed; its unprefixed nodes stay until a reindex",
|
||
zap.String("prefix", oldPrefix), zap.Error(err))
|
||
return
|
||
}
|
||
|
||
// The prefixed nodes are live; now drop the unprefixed originals.
|
||
// They are invisible to EvictRepo (no byRepo bucket entry), so evict
|
||
// per file — unprefixed paths cannot collide with prefixed ones.
|
||
for path := range oldMeta.FileMtimes {
|
||
mi.graph.EvictFile(path)
|
||
}
|
||
|
||
// EvictFile clears only nodes+edges; the solo repo's sidecar rows
|
||
// (file_mtimes, repo_index_state, enrichment_state, ...) were written
|
||
// under the empty prefix and would otherwise be orphaned — the very next
|
||
// warm restart would look for mtimes under the new prefix, find zero, and
|
||
// full-re-track a repo that never changed. Re-key the '' sidecar residue
|
||
// onto the new prefix. The re-mint re-index above already wrote fresh
|
||
// new-prefix rows; RekeyRepoPrefix folds the prefix/path-keyed ones and
|
||
// drops the node_id-keyed ones (whose ids changed under the re-mint) —
|
||
// see its per-table rationale. Safe on '': the store is still single-repo
|
||
// here, so '' holds only this repo's data (the synthetic global externals
|
||
// a multi-repo graph parks under '' live in NODES, which the rekey — a
|
||
// sidecar-only operation — never touches).
|
||
if rk, ok := mi.graph.(interface {
|
||
RekeyRepoPrefix(oldPrefix, newPrefix string) error
|
||
}); ok {
|
||
if err := rk.RekeyRepoPrefix("", oldPrefix); err != nil {
|
||
mi.logger.Warn("re-keying lone repo sidecar rows failed; orphaned '' rows remain until purge",
|
||
zap.String("prefix", oldPrefix), zap.Error(err))
|
||
}
|
||
}
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[oldPrefix] = &RepoMetadata{
|
||
RepoPrefix: oldPrefix,
|
||
RootPath: oldMeta.RootPath,
|
||
Identity: oldMeta.Identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
IsWorktree: oldMeta.IsWorktree,
|
||
}
|
||
mi.indexers[oldPrefix] = idx
|
||
mi.mu.Unlock()
|
||
|
||
mi.logger.Info("re-minted lone repo with its prefix for multi-repo mode",
|
||
zap.String("prefix", oldPrefix), zap.Int("nodes", result.NodeCount))
|
||
}
|
||
|
||
// readGoModModule reads the module path from a go.mod file.
|
||
func readGoModModule(repoPath string) string {
|
||
data, err := os.ReadFile(filepath.Join(repoPath, "go.mod"))
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, line := range strings.Split(string(data), "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if strings.HasPrefix(line, "module ") {
|
||
return strings.TrimSpace(strings.TrimPrefix(line, "module"))
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// indexMultiRepo indexes multiple repos concurrently with repo prefixes.
|
||
func (mi *MultiIndexer) indexMultiRepo(repos []config.RepoEntry) (map[string]*IndexResult, error) {
|
||
type repoResult struct {
|
||
prefix string
|
||
result *IndexResult
|
||
idx *Indexer
|
||
meta *RepoMetadata
|
||
err error
|
||
}
|
||
|
||
// Resolve each repo's prefix serially first: git-worktree instancing,
|
||
// the derived-name persistence, and the cross-batch collision guard
|
||
// must be deterministic, not raced across the parallel index
|
||
// goroutines below. The same loop builds the tracked-module map used
|
||
// for cross-repo dependency detection.
|
||
type resolvedRepo struct {
|
||
entry config.RepoEntry
|
||
prefix string
|
||
cfg *config.Config
|
||
absPath string
|
||
identity *RepoIdentity
|
||
}
|
||
trackedModules := make(map[string]string)
|
||
resolved := make([]resolvedRepo, 0, len(repos))
|
||
seenPrefix := make(map[string]string, len(repos)) // prefix → absPath
|
||
var resolveErrors []string
|
||
for _, entry := range repos {
|
||
absPath, err := filepath.Abs(entry.Path)
|
||
if err != nil {
|
||
resolveErrors = append(resolveErrors, fmt.Sprintf("resolving path %s: %v", entry.Path, err))
|
||
continue
|
||
}
|
||
identity, _ := DetectIdentity(absPath)
|
||
e := entry
|
||
prefix, cfg := mi.resolveTrackPrefix(&e, absPath, identity)
|
||
// Two distinct checkouts can still land on the same derived prefix
|
||
// (e.g. two worktrees declaring the same workspace). resolveTrackPrefix
|
||
// only sees already-tracked repos, so guard against collisions within
|
||
// this batch too.
|
||
if prev, ok := seenPrefix[prefix]; ok && prev != absPath {
|
||
prefix += "-" + shortPathHash(absPath)
|
||
e.Name = prefix
|
||
}
|
||
seenPrefix[prefix] = absPath
|
||
resolved = append(resolved, resolvedRepo{entry: e, prefix: prefix, cfg: cfg, absPath: absPath, identity: identity})
|
||
if mod := readGoModModule(absPath); mod != "" {
|
||
trackedModules[prefix] = mod
|
||
mi.logger.Debug("tracked repo module", zap.String("repo", prefix), zap.String("module", mod))
|
||
}
|
||
}
|
||
|
||
resultCh := make(chan repoResult, len(resolved))
|
||
var wg sync.WaitGroup
|
||
|
||
for _, rr := range resolved {
|
||
wg.Add(1)
|
||
go func(r resolvedRepo) {
|
||
defer wg.Done()
|
||
|
||
idx := mi.newPerRepoIndexer(r.cfg.Index)
|
||
idx.SetRepoPrefix(r.prefix)
|
||
entryCopy := r.entry
|
||
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, r.cfg, r.prefix))
|
||
idx.SetProjectID(resolveProjectID(&entryCopy, r.cfg, r.prefix))
|
||
idx.SetTrackedRepoModules(trackedModules)
|
||
// Defer the per-repo cross-cutting passes (ResolveAll,
|
||
// semantic enrich, contract extract+commit) so they don't
|
||
// race against each other across goroutines on the shared
|
||
// graph. They run serially below via RunDeferredPasses after
|
||
// wg.Wait(). The graph-wide derivation passes run once after
|
||
// the loop via mi.RunGlobalGraphPasses().
|
||
idx.SetDeferResolve(true)
|
||
|
||
result, err := idx.Index(r.absPath)
|
||
if err != nil {
|
||
resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s: %w", r.absPath, err)}
|
||
return
|
||
}
|
||
result.RepoPrefix = r.prefix
|
||
|
||
meta := &RepoMetadata{
|
||
RepoPrefix: r.prefix,
|
||
RootPath: r.absPath,
|
||
Identity: r.identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
IsWorktree: ResolveWorktree(r.absPath).IsWorktree,
|
||
}
|
||
|
||
resultCh <- repoResult{prefix: r.prefix, result: result, idx: idx, meta: meta}
|
||
}(rr)
|
||
}
|
||
|
||
go func() {
|
||
wg.Wait()
|
||
close(resultCh)
|
||
}()
|
||
|
||
results := make(map[string]*IndexResult)
|
||
indexErrors := resolveErrors
|
||
|
||
mi.mu.Lock()
|
||
for rr := range resultCh {
|
||
if rr.err != nil {
|
||
mi.logger.Error("failed to index repo", zap.String("prefix", rr.prefix), zap.Error(rr.err))
|
||
indexErrors = append(indexErrors, rr.err.Error())
|
||
continue
|
||
}
|
||
mi.repos[rr.prefix] = rr.meta
|
||
mi.indexers[rr.prefix] = rr.idx
|
||
results[rr.prefix] = rr.result
|
||
}
|
||
mi.mu.Unlock()
|
||
|
||
// Run the per-repo passes that the goroutines above deferred. Serial
|
||
// across repos is the simple correctness fix: ResolveAll mutates
|
||
// Edge.Meta on edges in the shared graph, and the contract pass walks
|
||
// every edge — running them in parallel across repos races. Inside a
|
||
// single repo's ResolveAll the resolver still uses its own worker
|
||
// pool, and parsing (the dominant cost on a fresh index) already ran
|
||
// in parallel above, so the wall-time hit is small.
|
||
deferCtx := context.Background()
|
||
for prefix := range results {
|
||
if idx, ok := mi.indexers[prefix]; ok {
|
||
idx.RunDeferredPasses(deferCtx)
|
||
}
|
||
}
|
||
|
||
// Run a global cross-repo resolution pass once every repo is
|
||
// indexed, with the cross-workspace boundary check wired in.
|
||
// Without this, repos that import each other have unresolved
|
||
// edges that only resolve when an editor touches a file (the
|
||
// watcher path is the only other place this resolver runs). The
|
||
// boundary lookup means cross-workspace candidates only resolve
|
||
// when an explicit `cross_workspace_deps` declaration covers
|
||
// them.
|
||
cr := resolver.NewCrossRepo(mi.graph)
|
||
cr.SetCrossWorkspaceDepLookup(mi.crossWorkspaceLookup())
|
||
cr.SetNpmAliasResolver(mi.npmAliasResolver())
|
||
cr.SetPathAliasResolver(mi.pathAliasResolver())
|
||
cr.SetWorkspaceMembership(mi.workspaceMembershipResolver())
|
||
mi.applyRemoteStitch(cr)
|
||
cr.ResolveAll()
|
||
mi.ReconcileContractEdges()
|
||
|
||
// Graph-wide derivation passes run exactly once after every repo
|
||
// has been parsed, every per-repo and cross-repo resolver has lifted
|
||
// placeholder edges, and contract bridges are in place. RunDeferredPasses
|
||
// intentionally skips these so we don't pay an O(global) walk per
|
||
// repo (was the dominant cost at R≈100+).
|
||
mi.RunGlobalGraphPasses(context.Background())
|
||
|
||
if len(indexErrors) > 0 && len(results) == 0 {
|
||
return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(indexErrors, "; "))
|
||
}
|
||
|
||
return results, nil
|
||
}
|
||
|
||
// IndexRepo re-indexes a single repo by prefix. Evicts existing data first.
|
||
func (mi *MultiIndexer) IndexRepo(repoPrefix string) (*IndexResult, error) {
|
||
mi.mu.RLock()
|
||
meta, ok := mi.repos[repoPrefix]
|
||
mi.mu.RUnlock()
|
||
if !ok {
|
||
return nil, fmt.Errorf("repository not found: %s", repoPrefix)
|
||
}
|
||
|
||
// Evict existing data for this repo before re-indexing. Always — a lone
|
||
// repo is now stored prefixed (see SetRepoPrefix below), so the eviction
|
||
// must clear the prefixed slice regardless of repo count.
|
||
mi.graph.EvictRepo(repoPrefix)
|
||
|
||
mi.configMgr.LoadWorkspaceConfig(repoPrefix, meta.RootPath)
|
||
cfg := mi.configMgr.GetRepoConfig(repoPrefix)
|
||
idx := mi.newPerRepoIndexer(cfg.Index)
|
||
// Always stamp the repo prefix, even when this is the only tracked repo.
|
||
// The multi-repo cold path (indexMultiRepo) already prefixes
|
||
// unconditionally; gating the single-repo re-index on repo count left the
|
||
// two paths producing different id / path shapes for the same repo, so
|
||
// adding a second repo turned the first from bare into prefixed — a lossy
|
||
// whole-graph rewrite. Uniform prefixing removes the single/multi data-shape
|
||
// split: a repo is prefixed from its first index, and adding another is
|
||
// purely additive.
|
||
idx.SetRepoPrefix(repoPrefix)
|
||
entry := mi.configMgr.Global().FindRepoByPrefix(repoPrefix)
|
||
idx.SetWorkspaceID(resolveWorkspaceID(entry, cfg, repoPrefix))
|
||
idx.SetProjectID(resolveProjectID(entry, cfg, repoPrefix))
|
||
|
||
result, err := idx.Index(meta.RootPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("indexing %s: %w", meta.RootPath, err)
|
||
}
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[repoPrefix] = &RepoMetadata{
|
||
RepoPrefix: repoPrefix,
|
||
RootPath: meta.RootPath,
|
||
Identity: meta.Identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
}
|
||
mi.indexers[repoPrefix] = idx
|
||
mi.mu.Unlock()
|
||
|
||
// TODO: After re-indexing, run CrossRepoResolver.ResolveForRepo(repoPrefix)
|
||
// to update cross-repo edges. This will be implemented in Task 7.1.
|
||
|
||
mi.ReconcileContractEdges()
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// IncrementalReindexRepo incrementally re-indexes a single tracked repo
|
||
// by prefix: only the files that changed since the last pass are
|
||
// re-parsed and deleted files are evicted, against the repo's existing
|
||
// per-repo Indexer (so its mtime snapshot is preserved). Unlike
|
||
// IndexRepo it does NOT evict the whole repo first.
|
||
//
|
||
// When paths is non-empty the pass is scoped to those files /
|
||
// directories; otherwise the whole repo root is scanned. Returns an
|
||
// error when the prefix is not a tracked repo.
|
||
func (mi *MultiIndexer) IncrementalReindexRepo(repoPrefix string, paths []string) (*IndexResult, error) {
|
||
mi.mu.RLock()
|
||
meta, ok := mi.repos[repoPrefix]
|
||
idx := mi.indexers[repoPrefix]
|
||
mi.mu.RUnlock()
|
||
if !ok || meta == nil {
|
||
return nil, fmt.Errorf("repository not found: %s", repoPrefix)
|
||
}
|
||
if idx == nil {
|
||
// Tracked but no live indexer (e.g. restored from snapshot
|
||
// without one) — fall back to a full re-index, which rebuilds
|
||
// the per-repo indexer from scratch.
|
||
return mi.IndexRepo(repoPrefix)
|
||
}
|
||
|
||
result, err := idx.IncrementalReindexPaths(meta.RootPath, paths)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("reindexing %s: %w", meta.RootPath, err)
|
||
}
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[repoPrefix] = &RepoMetadata{
|
||
RepoPrefix: repoPrefix,
|
||
RootPath: meta.RootPath,
|
||
Identity: meta.Identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
// Carried over from the prior metadata: this pass doesn't
|
||
// change either property, and dropping them here (both zero
|
||
// value on a fresh struct literal) used to silently flip a
|
||
// worktree or an Unprefixed solo repo back to their false
|
||
// defaults on the very first watcher-triggered incremental
|
||
// update, defeating callers that key behaviour off them (see
|
||
// the Unprefixed branch in cmd/gortex daemon status).
|
||
IsWorktree: meta.IsWorktree,
|
||
Unprefixed: meta.Unprefixed,
|
||
}
|
||
mi.mu.Unlock()
|
||
|
||
mi.ReconcileContractEdges()
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// TrackRepo validates the path, detects identity, indexes, and adds to config.
|
||
func (mi *MultiIndexer) TrackRepo(entry config.RepoEntry) (*IndexResult, error) {
|
||
return mi.TrackRepoCtx(context.Background(), entry)
|
||
}
|
||
|
||
// resolveTrackPrefix determines the repo prefix that absPath should be
|
||
// registered under and loads the per-repo `.gortex.yaml` config keyed to
|
||
// that prefix. It is the single place that decides whether a git
|
||
// worktree of an already-tracked repo becomes an INDEPENDENT instance
|
||
// (see WorktreeInstanceName); the track and reconcile paths both call it
|
||
// before their "already tracked?" check so a worktree that joins a
|
||
// different workspace than the canonical no longer silently coalesces
|
||
// into it.
|
||
//
|
||
// When a separate instance is created the derived `<base>@<tag>` prefix
|
||
// is written back into entry.Name so it is persisted to config and
|
||
// reproduced deterministically on the next daemon warmup. entry is
|
||
// mutated in place; callers pass a pointer to the value they will add to
|
||
// the global config.
|
||
func (mi *MultiIndexer) resolveTrackPrefix(entry *config.RepoEntry, absPath string, identity *RepoIdentity) (string, *config.Config) {
|
||
base := config.ResolvePrefix(*entry)
|
||
if base == "" || base == "." {
|
||
if identity != nil {
|
||
base = identity.RepoPrefix
|
||
}
|
||
}
|
||
if mi.configMgr == nil {
|
||
return base, config.Default()
|
||
}
|
||
|
||
// Load the repo's `.gortex.yaml` under the base prefix first so we can
|
||
// read its declared workspace before deciding the final prefix.
|
||
mi.configMgr.LoadWorkspaceConfig(base, absPath)
|
||
cfg := mi.configMgr.GetRepoConfig(base)
|
||
|
||
// An explicit Name already pins the prefix — honour it verbatim. This
|
||
// is also the warm-restart fast path: once a worktree instance has
|
||
// been persisted with its derived Name, every later load short-circuits
|
||
// here without re-deriving.
|
||
if entry.Name != "" {
|
||
return base, cfg
|
||
}
|
||
|
||
declaredWS := entry.Workspace
|
||
if declaredWS == "" && cfg != nil {
|
||
declaredWS = cfg.Workspace
|
||
}
|
||
|
||
name, separate := WorktreeInstanceName(absPath, base, declaredWS, entry.AsWorktree)
|
||
if !separate {
|
||
return base, cfg
|
||
}
|
||
|
||
// Guard against two different checkouts colliding on the same derived
|
||
// name (e.g. two worktrees that both declare workspace `x`): keep the
|
||
// first, suffix the rest with a path hash.
|
||
prefix := name
|
||
mi.mu.RLock()
|
||
existing, ok := mi.repos[prefix]
|
||
mi.mu.RUnlock()
|
||
if ok && existing != nil && !pathkey.SamePathIdentity(existing.RootPath, absPath) {
|
||
prefix = name + "-" + shortPathHash(absPath)
|
||
}
|
||
|
||
entry.Name = prefix // persist the decision so warmup reproduces it
|
||
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
|
||
return prefix, mi.configMgr.GetRepoConfig(prefix)
|
||
}
|
||
|
||
// EffectiveRepoPrefix returns the prefix a repo entry is tracked under,
|
||
// accounting for git-worktree instancing — the same value
|
||
// resolveTrackPrefix registers, minus the (rare) collision-guard suffix
|
||
// it cannot reproduce without the live registry. Warm-restart keying
|
||
// (the snapshot-store mtime lookup, the resolve-time LSP helper
|
||
// registry) uses this instead of config.ResolvePrefix so a disk-backed
|
||
// reconcile finds the worktree instance's own persisted state rather
|
||
// than the canonical checkout's. For a plain repo it equals
|
||
// config.ResolvePrefix(entry). cm may be nil (then only an explicit
|
||
// RepoEntry.Workspace override can trigger instancing).
|
||
func EffectiveRepoPrefix(cm *config.ConfigManager, entry config.RepoEntry) string {
|
||
base := config.ResolvePrefix(entry)
|
||
if base == "" || base == "." || entry.Name != "" {
|
||
return base
|
||
}
|
||
absPath, err := filepath.Abs(entry.Path)
|
||
if err != nil {
|
||
return base
|
||
}
|
||
declaredWS := entry.Workspace
|
||
if declaredWS == "" && cm != nil {
|
||
cm.LoadWorkspaceConfig(base, absPath)
|
||
if cfg := cm.GetRepoConfig(base); cfg != nil {
|
||
declaredWS = cfg.Workspace
|
||
}
|
||
}
|
||
name, _ := WorktreeInstanceName(absPath, base, declaredWS, entry.AsWorktree)
|
||
return name
|
||
}
|
||
|
||
// foldDistinctRepoCount counts configured repos by folded path identity,
|
||
// so two entries that name the same directory under different spellings
|
||
// (case or Unicode normalisation on a case-insensitive filesystem) count
|
||
// once. It backstops the willBeMultiRepo decision: startup healing already
|
||
// prunes such duplicates, but a not-yet-pruned config must not be allowed
|
||
// to flip the graph into prefixed-ID mode for what is really one repo.
|
||
func foldDistinctRepoCount(repos []config.RepoEntry) int {
|
||
n := 0
|
||
seen := make([]string, 0, len(repos))
|
||
for _, e := range repos {
|
||
abs, err := filepath.Abs(e.Path)
|
||
if err != nil {
|
||
abs = e.Path
|
||
}
|
||
dup := false
|
||
for _, s := range seen {
|
||
if pathkey.EqualPaths(s, abs) {
|
||
dup = true
|
||
break
|
||
}
|
||
}
|
||
if dup {
|
||
continue
|
||
}
|
||
seen = append(seen, abs)
|
||
n++
|
||
}
|
||
return n
|
||
}
|
||
|
||
// TrackRepoCtx is TrackRepo with a context, allowing callers to pipe progress
|
||
// reporters (via progress.WithReporter) through to the underlying Index call.
|
||
func (mi *MultiIndexer) TrackRepoCtx(ctx context.Context, entry config.RepoEntry) (*IndexResult, error) {
|
||
absPath, err := filepath.Abs(entry.Path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
|
||
}
|
||
// Normalise the volume (upper-case a Windows drive letter) so a newly
|
||
// tracked path converges with os.Getwd's convention. The volume is
|
||
// never part of a repo basename, so this cannot rotate a repo prefix;
|
||
// no-op on POSIX.
|
||
absPath = pathkey.NormalizeVolume(absPath)
|
||
|
||
// Validate path exists and is a directory.
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("path does not exist: %s", absPath)
|
||
}
|
||
if !info.IsDir() {
|
||
return nil, fmt.Errorf("path is not a directory: %s", absPath)
|
||
}
|
||
|
||
if reason, blocked := unsafeRootBlocked(absPath, entry.Force); blocked {
|
||
return nil, fmt.Errorf("%s; pass force to track it anyway", reason)
|
||
}
|
||
|
||
identity, err := DetectIdentity(absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("detecting identity for %s: %w", absPath, err)
|
||
}
|
||
|
||
// Resolve the prefix (honouring git-worktree instancing) and load the
|
||
// per-repo `.gortex.yaml` BEFORE the already-tracked check, so a
|
||
// worktree that joins a different workspace than the canonical gets
|
||
// its own `<base>@<tag>` prefix instead of coalescing into it. cfg is
|
||
// keyed to the final prefix; entry.Name is set when a separate
|
||
// instance is created so the decision persists to config. Loading the
|
||
// `.gortex.yaml` here also gives GetRepoConfig the workspace / project
|
||
// slugs declared inside the repo (without it every repo would fall
|
||
// back to `workspace = repoPrefix`, making shared-workspace cross-repo
|
||
// matching impossible to express).
|
||
prefix, cfg := mi.resolveTrackPrefix(&entry, absPath, identity)
|
||
|
||
// Check if already tracked. The prefix-keyed check catches the common
|
||
// case; the path scan additionally catches a case-only or Unicode
|
||
// spelling variant of an already-tracked directory on a
|
||
// case-insensitive filesystem (which resolves to a different derived
|
||
// prefix and would otherwise be minted as a bogus second instance —
|
||
// the very thing that flips the daemon into multi-repo mode, #270).
|
||
mi.mu.RLock()
|
||
if _, exists := mi.repos[prefix]; exists {
|
||
mi.mu.RUnlock()
|
||
return nil, nil // already tracked
|
||
}
|
||
for _, meta := range mi.repos {
|
||
if meta != nil && pathkey.SamePathIdentity(meta.RootPath, absPath) {
|
||
mi.mu.RUnlock()
|
||
return nil, nil // same directory, different path spelling
|
||
}
|
||
}
|
||
hook := mi.onRepoTracked
|
||
mi.mu.RUnlock()
|
||
|
||
if hook != nil {
|
||
hook(prefix, absPath)
|
||
}
|
||
|
||
// Determine if we need to prefix. We must consider both repos already
|
||
// indexed in mi.repos AND the total repos configured — at daemon warmup
|
||
// TrackRepoCtx is called in a loop over all configured repos, so at
|
||
// iteration 0 mi.repos is empty while the config already has N entries.
|
||
// Counting only mi.repos used to leave the first-indexed repo without a
|
||
// prefix while every later repo got one, producing two ID schemes for
|
||
// the same graph and halving cross-file edge density.
|
||
totalConfigured := 1 // ourselves
|
||
if mi.configMgr != nil {
|
||
totalConfigured = foldDistinctRepoCount(mi.configMgr.Global().Repos)
|
||
}
|
||
willBeMultiRepo := len(mi.repos)+1 >= 2 || totalConfigured >= 2
|
||
|
||
// A second repo joining a live single-repo daemon flips the graph
|
||
// into prefixed-ID mode, but the first repo's nodes were minted
|
||
// unprefixed — re-mint them before they become unreachable.
|
||
if willBeMultiRepo {
|
||
mi.migrateLoneUnprefixedRepoCtx(ctx)
|
||
}
|
||
|
||
idx := mi.newPerRepoIndexer(cfg.Index)
|
||
if willBeMultiRepo {
|
||
idx.SetRepoPrefix(prefix)
|
||
}
|
||
// Workspace / project slugs stamped on every node. Resolution
|
||
// order (highest priority first): RepoEntry.Workspace from the
|
||
// global config (lets users pin OSS repos without committing a
|
||
// `.gortex.yaml`) → `.gortex.yaml::workspace` → repoPrefix
|
||
// (default). resolveWorkspaceID encodes the precedence; the
|
||
// WorkspaceID-keyed contract registry and the boundary-enforced
|
||
// matcher both consume the result.
|
||
entryCopy := entry
|
||
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
|
||
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
|
||
|
||
result, err := idx.IndexCtx(ctx, absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("indexing %s: %w", absPath, err)
|
||
}
|
||
result.RepoPrefix = prefix
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[prefix] = &RepoMetadata{
|
||
RepoPrefix: prefix,
|
||
RootPath: absPath,
|
||
Identity: identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
IsWorktree: ResolveWorktree(absPath).IsWorktree,
|
||
Unprefixed: !willBeMultiRepo,
|
||
}
|
||
mi.indexers[prefix] = idx
|
||
mi.mu.Unlock()
|
||
|
||
// Add to global config.
|
||
entry.Path = absPath
|
||
if err := mi.configMgr.Global().AddRepo(entry); err != nil {
|
||
mi.logger.Warn("failed to add repo to config", zap.Error(err))
|
||
}
|
||
|
||
// Skip the per-repo contract reconcile when batching: it walks every
|
||
// edge in the shared graph to evict stale EdgeMatches and rebuilds
|
||
// the matcher across every indexer, so paying it once per repo on a
|
||
// warmup over 100+ repos is O(R · E). The batch caller runs it once
|
||
// after the loop (RunGlobalResolve does, and the janitor's ReconcileAll
|
||
// fires it post-loop too).
|
||
if !mi.deferGlobalPasses {
|
||
mi.ReconcileContractEdges()
|
||
}
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// ReconcileRepoCtx registers a repo that already has nodes in the graph
|
||
// (typically restored from a daemon snapshot) and brings it back into
|
||
// agreement with the filesystem without a full re-index. priorMtimes
|
||
// carries the mtimes recorded at the time the snapshot was taken;
|
||
// IncrementalReindex uses them to detect files that changed offline
|
||
// (re-indexes) and files that were deleted offline (evicts).
|
||
//
|
||
// Falls back to TrackRepoCtx when the repo is not yet tracked AND no
|
||
// prior mtimes are available — in that case there's nothing to
|
||
// reconcile against and a full index is the correct path.
|
||
func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoEntry, priorMtimes map[string]int64) (*IndexResult, error) {
|
||
start := time.Now()
|
||
|
||
absPath, err := filepath.Abs(entry.Path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
|
||
}
|
||
// Normalise the volume (upper-case a Windows drive letter) to converge
|
||
// with os.Getwd's convention. Cosmetic; never touches the basename.
|
||
absPath = pathkey.NormalizeVolume(absPath)
|
||
info, err := os.Stat(absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("path does not exist: %s", absPath)
|
||
}
|
||
if !info.IsDir() {
|
||
return nil, fmt.Errorf("path is not a directory: %s", absPath)
|
||
}
|
||
identity, err := DetectIdentity(absPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("detecting identity for %s: %w", absPath, err)
|
||
}
|
||
|
||
// Resolve the prefix (honouring git-worktree instancing) and load the
|
||
// per-repo `.gortex.yaml` before the already-tracked check. Mirrors
|
||
// TrackRepoCtx so a worktree instance keeps its derived prefix on the
|
||
// reconcile path too; config can change between sessions and warmup-
|
||
// time reconcile runs after a daemon restart.
|
||
prefix, cfg := mi.resolveTrackPrefix(&entry, absPath, identity)
|
||
|
||
// Already tracked — nothing to do.
|
||
mi.mu.RLock()
|
||
_, exists := mi.repos[prefix]
|
||
mi.mu.RUnlock()
|
||
if exists {
|
||
return nil, nil
|
||
}
|
||
|
||
// Fall back to full TrackRepoCtx when we have no prior mtimes:
|
||
// there's nothing meaningful to reconcile against, and
|
||
// IncrementalReindex would treat every file as stale, producing
|
||
// the same duplicate-writes problem we're fixing. entry.Name is
|
||
// already pinned by resolveTrackPrefix, so the fallback reproduces
|
||
// the same prefix.
|
||
if len(priorMtimes) == 0 {
|
||
return mi.TrackRepoCtx(ctx, entry)
|
||
}
|
||
|
||
totalConfigured := 1
|
||
if mi.configMgr != nil {
|
||
totalConfigured = foldDistinctRepoCount(mi.configMgr.Global().Repos)
|
||
}
|
||
willBeMultiRepo := len(mi.repos)+1 >= 2 || totalConfigured >= 2
|
||
|
||
// Same transition guard as TrackRepoCtx: an already-reconciled
|
||
// lone repo with unprefixed nodes must be re-minted before this
|
||
// second repo flips the graph into prefixed-ID mode.
|
||
if willBeMultiRepo {
|
||
mi.migrateLoneUnprefixedRepoCtx(ctx)
|
||
}
|
||
|
||
idx := mi.newPerRepoIndexer(cfg.Index)
|
||
if willBeMultiRepo {
|
||
idx.SetRepoPrefix(prefix)
|
||
}
|
||
entryCopy := entry
|
||
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
|
||
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
|
||
idx.SetRootPath(absPath)
|
||
idx.SetFileMtimes(priorMtimes)
|
||
|
||
// Choose the reconcile strategy from a census of what changed on disk
|
||
// while the daemon was down. Scoped incremental is the default: re-index
|
||
// only the changed files and evict only the deleted ones, leaving the
|
||
// rest of the already-persisted graph untouched. A whole-repo re-track
|
||
// (IndexCtx — which evicts and re-parses every file, then bulk-drains)
|
||
// is reserved for the cases where scoping is unsafe or not worth it: the
|
||
// census could not be taken, the churn is a large fraction of the repo,
|
||
// or an operator forced it via GORTEX_WARMUP_FULL_RETRACK. A repo with
|
||
// zero changes keeps the fast IncrementalReindex no-op (walk + 0 stale →
|
||
// return), which is what makes an unchanged warm restart near-instant.
|
||
//
|
||
// The in-memory backend (*graph.Graph) keeps its exact prior behaviour:
|
||
// IncrementalReindex is the authoritative path there — it evicts
|
||
// offline-deleted files in place, has no reopened disk store, and so no
|
||
// per-edge write to route around. Gate on the store type.
|
||
_, memoryBacked := mi.graph.(*graph.Graph)
|
||
var (
|
||
result *IndexResult
|
||
changed, deleted []string
|
||
route = "incremental"
|
||
)
|
||
// fullRetrack is the whole-repo re-track — the ONE place FullRetrack is
|
||
// stamped. StaleFileCount keeps its honest incremental-work meaning (0
|
||
// here) because the changed-file set is not enumerated on this path, so
|
||
// callers must key "did this repo change" off FullRetrack instead.
|
||
fullRetrack := func() (*IndexResult, error) {
|
||
r, e := idx.IndexCtx(ctx, absPath)
|
||
if e == nil && r != nil {
|
||
r.FullRetrack = true
|
||
}
|
||
return r, e
|
||
}
|
||
switch {
|
||
case memoryBacked:
|
||
result, err = idx.IncrementalReindex(absPath)
|
||
default:
|
||
var censusErr error
|
||
changed, deleted, censusErr = idx.ChangedSinceMtimes(absPath)
|
||
churn := len(changed) + len(deleted)
|
||
priorCount := len(priorMtimes)
|
||
forceFull := os.Getenv("GORTEX_WARMUP_FULL_RETRACK") == "1"
|
||
switch {
|
||
case censusErr != nil || forceFull:
|
||
route = "full_retrack"
|
||
result, err = fullRetrack()
|
||
case churn == 0:
|
||
route = "incremental"
|
||
result, err = idx.IncrementalReindex(absPath)
|
||
case priorCount > 0 && churn*100 > priorCount*40:
|
||
route = "full_retrack"
|
||
result, err = fullRetrack()
|
||
default:
|
||
route = "scoped"
|
||
result, err = idx.IncrementalReindexPaths(absPath, append(changed, deleted...))
|
||
}
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("reconciling %s: %w", absPath, err)
|
||
}
|
||
result.RepoPrefix = prefix
|
||
|
||
mi.mu.Lock()
|
||
mi.repos[prefix] = &RepoMetadata{
|
||
RepoPrefix: prefix,
|
||
RootPath: absPath,
|
||
Identity: identity,
|
||
LastIndexTime: time.Now(),
|
||
FileCount: result.FileCount,
|
||
NodeCount: result.NodeCount,
|
||
EdgeCount: result.EdgeCount,
|
||
ParseErrors: result.Errors,
|
||
FileMtimes: idx.FileMtimes(),
|
||
IsWorktree: ResolveWorktree(absPath).IsWorktree,
|
||
Unprefixed: !willBeMultiRepo,
|
||
}
|
||
mi.indexers[prefix] = idx
|
||
mi.mu.Unlock()
|
||
|
||
entry.Path = absPath
|
||
if err := mi.configMgr.Global().AddRepo(entry); err != nil {
|
||
mi.logger.Warn("failed to add repo to config", zap.Error(err))
|
||
}
|
||
|
||
// See TrackRepoCtx for why this is skipped under deferGlobalPasses.
|
||
if !mi.deferGlobalPasses {
|
||
mi.ReconcileContractEdges()
|
||
}
|
||
|
||
mi.logger.Info("daemon: reconciled repo from snapshot",
|
||
zap.String("prefix", prefix),
|
||
zap.String("route", route),
|
||
zap.Int("changed", len(changed)),
|
||
zap.Int("deleted", len(deleted)),
|
||
zap.Bool("full_retrack", result.FullRetrack),
|
||
zap.Int("stale_files_reindexed", result.StaleFileCount),
|
||
zap.Duration("elapsed", time.Since(start)))
|
||
if len(changed) > 0 || len(deleted) > 0 {
|
||
mi.logger.Debug("daemon: reconcile changed-file census",
|
||
zap.String("prefix", prefix),
|
||
zap.Strings("changed", firstNStrings(changed, 5)),
|
||
zap.Strings("deleted", firstNStrings(deleted, 5)))
|
||
}
|
||
|
||
return result, nil
|
||
}
|
||
|
||
// firstNStrings returns at most the first n elements of s — used to cap the
|
||
// changed/deleted samples in the reconcile debug log so a large-churn repo
|
||
// does not dump thousands of paths into the log line.
|
||
func firstNStrings(s []string, n int) []string {
|
||
if len(s) <= n {
|
||
return s
|
||
}
|
||
return s[:n]
|
||
}
|
||
|
||
// ReconcileAll runs IncrementalReindex on every tracked repo. Used by
|
||
// the daemon's periodic janitor to catch files whose mutations slipped
|
||
// past fsnotify — inotify watch limits, NFS / SMB mounts, kernel event
|
||
// queue overflow, or daemon downtime where edits happened while nobody
|
||
// was listening. Cheap on steady-state repos (one filepath.WalkDir +
|
||
// per-file os.Stat per repo), and correctness is self-healing: whatever
|
||
// was missed gets picked up on the next tick.
|
||
//
|
||
// Returns a map of prefix → IndexResult for logging / metrics. Errors
|
||
// per repo are logged and do not abort the rest of the sweep — a broken
|
||
// permission bit on one repo should not starve reconciliation on the
|
||
// others.
|
||
func (mi *MultiIndexer) ReconcileAll() map[string]*IndexResult {
|
||
mi.mu.RLock()
|
||
prefixes := make([]string, 0, len(mi.indexers))
|
||
for p := range mi.indexers {
|
||
prefixes = append(prefixes, p)
|
||
}
|
||
mi.mu.RUnlock()
|
||
|
||
// Same batch trick as warmup: each per-repo IncrementalReindex
|
||
// triggers an O(global) InferImplements/InferOverrides walk if we
|
||
// don't suppress it. With ~100 repos that's ~100× the work for the
|
||
// hourly janitor.
|
||
mi.BeginBatch()
|
||
// Always restore batch flags on exit (incl. panic) WITHOUT running the
|
||
// graph-wide derivation passes — those are run explicitly below, and
|
||
// only when a repo actually reindexed. The hourly janitor used to run
|
||
// EndBatch unconditionally, walking the full graph (InferImplements /
|
||
// InferOverrides / clone detection over hundreds of thousands of
|
||
// edges) every cycle even when nothing changed — wasted CPU and, on a
|
||
// small resident buffer pool, needless memory churn.
|
||
defer mi.ResetBatch()
|
||
|
||
results := make(map[string]*IndexResult, len(prefixes))
|
||
reindexed := 0
|
||
changedPrefixes := make(map[string]struct{})
|
||
for _, prefix := range prefixes {
|
||
mi.mu.RLock()
|
||
idx, ok := mi.indexers[prefix]
|
||
meta, metaOK := mi.repos[prefix]
|
||
mi.mu.RUnlock()
|
||
if !ok || !metaOK || meta == nil || meta.RootPath == "" {
|
||
continue
|
||
}
|
||
result, err := idx.IncrementalReindex(meta.RootPath)
|
||
if err != nil {
|
||
mi.logger.Warn("janitor: reconcile failed",
|
||
zap.String("prefix", prefix), zap.Error(err))
|
||
continue
|
||
}
|
||
if result != nil && (result.StaleFileCount > 0 || result.DeletedFileCount > 0) {
|
||
mi.logger.Info("janitor: reconciled repo",
|
||
zap.String("prefix", prefix),
|
||
zap.Int("stale_files_reindexed", result.StaleFileCount))
|
||
reindexed++
|
||
changedPrefixes[prefix] = struct{}{}
|
||
}
|
||
results[prefix] = result
|
||
|
||
// Keep RepoMetadata.FileMtimes in sync so the next snapshot
|
||
// picks up the reconciled mtimes.
|
||
mi.mu.Lock()
|
||
if m, ok := mi.repos[prefix]; ok && m != nil {
|
||
m.FileMtimes = idx.FileMtimes()
|
||
m.LastIndexTime = time.Now()
|
||
}
|
||
mi.mu.Unlock()
|
||
}
|
||
|
||
if reindexed > 0 {
|
||
mi.ReconcileContractEdges()
|
||
// Only now — when at least one repo actually reindexed — is it
|
||
// worth the full-graph derivation pass. Nothing changed → skip it
|
||
// (the deferred ResetBatch still clears the batch flags). Scope the
|
||
// per-repo clone detection + Rebuild to just the repos that changed
|
||
// this cycle: an unchanged repo's clone edges are already on disk, so
|
||
// re-detecting all of them holds the resolve mutex for tens of seconds
|
||
// and stalls every concurrent interactive edit. Warmup arms the same
|
||
// scope; without this the janitor re-ran clone detection over every
|
||
// tracked repo on every cycle that touched even one file.
|
||
mi.ArmBatchScope(changedPrefixes)
|
||
mi.RunGlobalGraphPasses(context.Background())
|
||
}
|
||
return results
|
||
}
|
||
|
||
// UntrackRepo evicts a repo from the graph and removes it from config.
|
||
func (mi *MultiIndexer) UntrackRepo(repoPrefix string) (int, int) {
|
||
mi.mu.Lock()
|
||
meta, ok := mi.repos[repoPrefix]
|
||
if !ok {
|
||
mi.mu.Unlock()
|
||
return 0, 0
|
||
}
|
||
delete(mi.repos, repoPrefix)
|
||
delete(mi.indexers, repoPrefix)
|
||
mi.mu.Unlock()
|
||
|
||
var nodesRemoved, edgesRemoved int
|
||
if meta.Unprefixed {
|
||
// Single-repo-mode nodes carry RepoPrefix="" and never enter the
|
||
// byRepo bucket EvictRepo walks — evict them file-by-file off the
|
||
// recorded file set instead, or they linger in the graph and a
|
||
// later lone repo would mis-resolve them.
|
||
for path := range meta.FileMtimes {
|
||
n, e := mi.graph.EvictFile(path)
|
||
nodesRemoved += n
|
||
edgesRemoved += e
|
||
}
|
||
} else if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok {
|
||
// Prefer the full sidecar-aware purge. EvictRepo drops only
|
||
// nodes+edges and leaves fifteen repo_prefix-keyed sidecar tables
|
||
// (file_mtimes, *_enrichment, symbol_fts, content_fts, ...) behind,
|
||
// which accumulate across untrack/retrack cycles until they dominate
|
||
// a long-lived store. PurgeRepo clears them in one transaction. It
|
||
// returns no counts, so report the repo's last-index metadata as the
|
||
// removed estimate; fall back to EvictRepo (real counts) on error.
|
||
if err := purger.PurgeRepo(repoPrefix); err != nil {
|
||
mi.logger.Warn("purge repo failed; falling back to node/edge eviction",
|
||
zap.String("prefix", repoPrefix), zap.Error(err))
|
||
nodesRemoved, edgesRemoved = mi.graph.EvictRepo(repoPrefix)
|
||
} else {
|
||
nodesRemoved, edgesRemoved = meta.NodeCount, meta.EdgeCount
|
||
}
|
||
} else {
|
||
// Backends without the purge capability (the in-memory store has no
|
||
// sidecars, so EvictRepo is already complete there).
|
||
nodesRemoved, edgesRemoved = mi.graph.EvictRepo(repoPrefix)
|
||
}
|
||
|
||
// Remove from global config.
|
||
if meta.RootPath != "" {
|
||
if err := mi.configMgr.Global().RemoveRepo(meta.RootPath); err != nil {
|
||
mi.logger.Warn("failed to remove repo from config",
|
||
zap.String("prefix", repoPrefix), zap.Error(err))
|
||
}
|
||
}
|
||
|
||
mi.ReconcileContractEdges()
|
||
|
||
return nodesRemoved, edgesRemoved
|
||
}
|
||
|
||
// WorktreeGC is the per-repo outcome of GCVanishedWorktrees.
|
||
type WorktreeGC struct {
|
||
RepoPrefix string
|
||
RootPath string
|
||
NodesRemoved int
|
||
EdgesRemoved int
|
||
}
|
||
|
||
// GCVanishedWorktrees garbage-collects the index of any tracked linked
|
||
// git worktree whose root directory has disappeared from disk — the
|
||
// `git worktree remove` (or manual deletion) case. Each vanished
|
||
// worktree's branch-keyed snapshot slot and graph nodes would otherwise
|
||
// leak forever: a removed worktree never fires a per-file fsnotify
|
||
// delete for its whole tree, and the janitor's IncrementalReindex just
|
||
// errors out on the missing root without evicting anything.
|
||
//
|
||
// Only repos recorded as worktrees (RepoMetadata.IsWorktree) are
|
||
// eligible — a vanished *main* checkout is left alone, since that is
|
||
// far more likely a transient mount problem than an intentional
|
||
// removal, and untracking it would also orphan every linked worktree
|
||
// that shares its .git. The directory-existence test uses the same
|
||
// not-exist-only rule as the per-file deletion detector, so a flaky
|
||
// filesystem cannot trigger a destructive eviction.
|
||
//
|
||
// Returns one WorktreeGC record per repo evicted; an empty slice when
|
||
// every tracked worktree is still present.
|
||
func (mi *MultiIndexer) GCVanishedWorktrees() []WorktreeGC {
|
||
// Snapshot the candidate set under the read lock, then evict
|
||
// outside it — UntrackRepo takes the write lock itself.
|
||
type candidate struct {
|
||
prefix string
|
||
root string
|
||
}
|
||
var candidates []candidate
|
||
mi.mu.RLock()
|
||
for prefix, meta := range mi.repos {
|
||
if meta == nil || !meta.IsWorktree || meta.RootPath == "" {
|
||
continue
|
||
}
|
||
if WorktreeRootGone(meta.RootPath) {
|
||
candidates = append(candidates, candidate{prefix: prefix, root: meta.RootPath})
|
||
}
|
||
}
|
||
mi.mu.RUnlock()
|
||
|
||
if len(candidates) == 0 {
|
||
return nil
|
||
}
|
||
|
||
out := make([]WorktreeGC, 0, len(candidates))
|
||
for _, c := range candidates {
|
||
nodes, edges := mi.UntrackRepo(c.prefix)
|
||
mi.logger.Info("janitor: garbage-collected vanished worktree",
|
||
zap.String("prefix", c.prefix),
|
||
zap.String("root", c.root),
|
||
zap.Int("nodes_removed", nodes),
|
||
zap.Int("edges_removed", edges))
|
||
out = append(out, WorktreeGC{
|
||
RepoPrefix: c.prefix,
|
||
RootPath: c.root,
|
||
NodesRemoved: nodes,
|
||
EdgesRemoved: edges,
|
||
})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// GetMetadata returns the metadata for a specific repo, or nil if not found.
|
||
func (mi *MultiIndexer) GetMetadata(repoPrefix string) *RepoMetadata {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
return mi.repos[repoPrefix]
|
||
}
|
||
|
||
// AllMetadata returns a copy of all repo metadata.
|
||
func (mi *MultiIndexer) AllMetadata() map[string]*RepoMetadata {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
out := make(map[string]*RepoMetadata, len(mi.repos))
|
||
for k, v := range mi.repos {
|
||
out[k] = v
|
||
}
|
||
return out
|
||
}
|
||
|
||
// IsMultiRepo returns true when more than one repo is tracked.
|
||
func (mi *MultiIndexer) IsMultiRepo() bool {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
return len(mi.repos) > 1
|
||
}
|
||
|
||
// RepoForFile returns the repo prefix for a given file path by checking
|
||
// which repo root contains it. Returns empty string if no match.
|
||
func (mi *MultiIndexer) RepoForFile(filePath string) string {
|
||
absPath, err := filepath.Abs(filePath)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
|
||
var bestPrefix string
|
||
var bestLen int
|
||
|
||
for prefix, meta := range mi.repos {
|
||
// Fold-aware containment so a case-variant file path still maps
|
||
// to its repo on a case-insensitive filesystem. Longest-root-wins
|
||
// breaks ties by nesting depth; nested roots share a prefix, so
|
||
// the raw RootPath length orders them the same as their folded
|
||
// forms would.
|
||
if pathkey.HasPathPrefix(absPath, meta.RootPath) {
|
||
if len(meta.RootPath) > bestLen {
|
||
bestLen = len(meta.RootPath)
|
||
bestPrefix = prefix
|
||
}
|
||
}
|
||
}
|
||
|
||
return bestPrefix
|
||
}
|
||
|
||
// GetIndexer returns the Indexer for a specific repo prefix, or nil.
|
||
func (mi *MultiIndexer) GetIndexer(repoPrefix string) *Indexer {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
return mi.indexers[repoPrefix]
|
||
}
|
||
|
||
type grepRepoJob struct {
|
||
prefix string
|
||
idx *Indexer
|
||
}
|
||
|
||
func (mi *MultiIndexer) grepRepoJobs(repoAllow map[string]bool) []grepRepoJob {
|
||
if mi == nil {
|
||
return nil
|
||
}
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
capHint := len(mi.indexers)
|
||
if repoAllow != nil {
|
||
capHint = len(repoAllow)
|
||
}
|
||
jobs := make([]grepRepoJob, 0, capHint)
|
||
for prefix, idx := range mi.indexers {
|
||
if idx == nil {
|
||
continue
|
||
}
|
||
if repoAllow != nil && !repoAllow[prefix] {
|
||
continue
|
||
}
|
||
jobs = append(jobs, grepRepoJob{prefix: prefix, idx: idx})
|
||
}
|
||
sort.Slice(jobs, func(i, j int) bool { return jobs[i].prefix < jobs[j].prefix })
|
||
return jobs
|
||
}
|
||
|
||
func singleAllowedRepo(repoAllow map[string]bool) (string, bool) {
|
||
if repoAllow == nil {
|
||
return "", false
|
||
}
|
||
var only string
|
||
count := 0
|
||
for prefix, allowed := range repoAllow {
|
||
if !allowed {
|
||
continue
|
||
}
|
||
only = prefix
|
||
count++
|
||
}
|
||
return only, count == 1
|
||
}
|
||
|
||
func stampGrepMatchPaths(prefix string, hits []trigram.Match) []trigram.Match {
|
||
if prefix == "" {
|
||
return hits
|
||
}
|
||
for i := range hits {
|
||
hits[i].Path = prefix + "/" + hits[i].Path
|
||
}
|
||
return hits
|
||
}
|
||
|
||
func capGrepMatches(matches []trigram.Match, limit int) []trigram.Match {
|
||
if limit > 0 && len(matches) > limit {
|
||
return matches[:limit]
|
||
}
|
||
return matches
|
||
}
|
||
|
||
// GrepText fans out a trigram-accelerated literal search across every
|
||
// tracked per-repo Indexer and returns the union, capped at limit.
|
||
// Match paths are re-stamped from repo-root-relative to repo-prefixed
|
||
// (e.g. "internal/foo.go" → "gortex/internal/foo.go") so callers can
|
||
// route hits back to the right working tree without consulting the
|
||
// MultiIndexer afterwards. A non-positive limit returns every match.
|
||
// Returns nil when no per-repo indexer can serve the query — the
|
||
// single-Indexer path (Indexer.GrepText) is used by callers without a
|
||
// MultiIndexer.
|
||
func (mi *MultiIndexer) GrepText(query string, limit int) []trigram.Match {
|
||
return capGrepMatches(mi.GrepTextForRepos(query, nil, limit), limit)
|
||
}
|
||
|
||
// GrepTextForRepos is the scoped variant of GrepText. When repoAllow is
|
||
// non-nil, only those repo prefixes are searched. perRepoLimit caps each
|
||
// searched repo independently; the returned union is intentionally not
|
||
// globally capped so callers can apply path / graph-scope filters first.
|
||
func (mi *MultiIndexer) GrepTextForRepos(query string, repoAllow map[string]bool, perRepoLimit int) []trigram.Match {
|
||
if mi == nil || query == "" {
|
||
return nil
|
||
}
|
||
if prefix, ok := singleAllowedRepo(repoAllow); ok {
|
||
idx := mi.GetIndexer(prefix)
|
||
if idx == nil {
|
||
return nil
|
||
}
|
||
return stampGrepMatchPaths(prefix, idx.GrepText(query, perRepoLimit))
|
||
}
|
||
|
||
// Per-repo cap mirrors the caller's page size when set. The caller
|
||
// applies the final cap after any path / graph-scope filters, so a
|
||
// repo outside those filters cannot consume the page first. Zero /
|
||
// negative means no per-repo cap (let each searcher return everything).
|
||
jobs := mi.grepRepoJobs(repoAllow)
|
||
out := make([]trigram.Match, 0, len(jobs)*8)
|
||
for _, j := range jobs {
|
||
hits := j.idx.GrepText(query, perRepoLimit)
|
||
if len(hits) == 0 {
|
||
continue
|
||
}
|
||
// Trigram emits forward-slash repo-relative paths. Stamp the repo
|
||
// prefix so downstream tools (resolveGraphPath, path-prefix filters)
|
||
// see the same shape they get from the graph nodes.
|
||
out = append(out, stampGrepMatchPaths(j.prefix, hits)...)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// GrepRegexp fans out a trigram-accelerated regex search across every
|
||
// tracked per-repo Indexer and returns the union, capped at limit.
|
||
// Mirrors GrepText: match paths are re-stamped from repo-root-relative
|
||
// to repo-prefixed so downstream tools route hits back to the right
|
||
// working tree. pathPrefix, when non-empty, restricts the scan to
|
||
// files under that prefix on each indexer. A pattern that does not
|
||
// compile in any indexer is reported once; per-indexer errors after
|
||
// the first compile are otherwise treated as no-match.
|
||
func (mi *MultiIndexer) GrepRegexp(pattern, pathPrefix string, limit int) ([]trigram.Match, error) {
|
||
hits, err := mi.GrepRegexpForRepos(pattern, pathPrefix, nil, limit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return capGrepMatches(hits, limit), nil
|
||
}
|
||
|
||
// GrepRegexpForRepos is the scoped variant of GrepRegexp. repoAllow and
|
||
// perRepoLimit have the same semantics as GrepTextForRepos.
|
||
func (mi *MultiIndexer) GrepRegexpForRepos(pattern, pathPrefix string, repoAllow map[string]bool, perRepoLimit int) ([]trigram.Match, error) {
|
||
if mi == nil || pattern == "" {
|
||
return nil, nil
|
||
}
|
||
if prefix, ok := singleAllowedRepo(repoAllow); ok {
|
||
idx := mi.GetIndexer(prefix)
|
||
if idx == nil {
|
||
return nil, nil
|
||
}
|
||
hits, err := idx.GrepRegexp(pattern, pathPrefix, perRepoLimit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return stampGrepMatchPaths(prefix, hits), nil
|
||
}
|
||
|
||
jobs := mi.grepRepoJobs(repoAllow)
|
||
out := make([]trigram.Match, 0, len(jobs)*8)
|
||
for _, j := range jobs {
|
||
hits, err := j.idx.GrepRegexp(pattern, pathPrefix, perRepoLimit)
|
||
if err != nil {
|
||
// First compile error short-circuits — the pattern is the
|
||
// caller's fault and won't compile in any other indexer
|
||
// either (the trigram searcher uses the same regexp engine).
|
||
return nil, err
|
||
}
|
||
if len(hits) == 0 {
|
||
continue
|
||
}
|
||
out = append(out, stampGrepMatchPaths(j.prefix, hits)...)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// IndexerForFile routes an absolute path to the per-repo Indexer that
|
||
// owns it. Returns (nil, "") when no tracked repo contains the path.
|
||
// Used by the MCP overlay middleware to find the right Indexer for a
|
||
// pushed file when constructing the per-request overlay layer.
|
||
func (mi *MultiIndexer) IndexerForFile(absPath string) (*Indexer, string) {
|
||
prefix := mi.RepoForFile(absPath)
|
||
if prefix == "" {
|
||
return nil, ""
|
||
}
|
||
return mi.GetIndexer(prefix), prefix
|
||
}
|
||
|
||
// ResolveFilePath takes a repo-prefixed relative path (e.g. "ade/internal/foo.go")
|
||
// and returns the absolute filesystem path by looking up the repo's root directory.
|
||
// Returns empty string if the repo prefix is not found.
|
||
func (mi *MultiIndexer) ResolveFilePath(prefixedPath string) string {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
|
||
// Longest matching prefix wins. With worktree instances two prefixes
|
||
// can share a leading segment (`oas-orm` vs `oas-orm@task-ws`); map
|
||
// iteration order is random, so a plain first-match could resolve a
|
||
// `oas-orm@task-ws/...` path against the shorter `oas-orm` root. The
|
||
// "/"-boundary check already keeps the two disjoint, but ranking by
|
||
// length makes that robust regardless of any future prefix shapes.
|
||
var bestPrefix, bestRoot string
|
||
for prefix, meta := range mi.repos {
|
||
if meta == nil {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(prefixedPath, prefix+"/") && len(prefix) > len(bestPrefix) {
|
||
bestPrefix, bestRoot = prefix, meta.RootPath
|
||
}
|
||
}
|
||
if bestPrefix == "" {
|
||
// Single-repo mode mints unprefixed graph paths; resolve them
|
||
// against the lone registered repo instead of failing.
|
||
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
|
||
return filepath.Join(meta.RootPath, prefixedPath)
|
||
}
|
||
return ""
|
||
}
|
||
// Collision guard for the lone unprefixed repo: its graph paths are
|
||
// raw relative paths, so one whose first segment happens to equal
|
||
// the repo's own prefix (repo "api" containing api/handlers.go)
|
||
// would be hijacked by the prefix-strip join. Prefer the raw join
|
||
// when that file actually exists on disk.
|
||
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
|
||
raw := filepath.Join(meta.RootPath, prefixedPath)
|
||
if _, err := os.Stat(raw); err == nil {
|
||
return raw
|
||
}
|
||
}
|
||
return filepath.Join(bestRoot, strings.TrimPrefix(prefixedPath, bestPrefix+"/"))
|
||
}
|
||
|
||
// RepoPrefixes returns the set of registered repo prefixes. The returned
|
||
// slice is a snapshot — safe to retain and iterate concurrently with
|
||
// other multi-indexer operations. Order is unspecified; callers that
|
||
// need stability should sort.
|
||
func (mi *MultiIndexer) RepoPrefixes() []string {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
prefixes := make([]string, 0, len(mi.repos))
|
||
for prefix := range mi.repos {
|
||
prefixes = append(prefixes, prefix)
|
||
}
|
||
return prefixes
|
||
}
|
||
|
||
// RepoRoot returns the local filesystem root for the given repo prefix.
|
||
// ok is true only when the prefix is registered AND meta.RootPath is non-empty.
|
||
// Caller is responsible for joining repo-relative file paths against the root.
|
||
//
|
||
// The empty prefix resolves to the lone registered repo when exactly one is
|
||
// tracked: single-repo mode indexes nodes without a repo prefix (see
|
||
// indexSingleRepo) while registering its metadata under the repo's real
|
||
// prefix, so every node the single-repo indexer mints carries RepoPrefix=""
|
||
// — refusing the empty prefix would orphan all of them. With two or more
|
||
// repos the empty prefix is ambiguous and stays a miss.
|
||
func (mi *MultiIndexer) RepoRoot(repoPrefix string) (string, bool) {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
if repoPrefix == "" {
|
||
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
|
||
return meta.RootPath, true
|
||
}
|
||
return "", false
|
||
}
|
||
meta, ok := mi.repos[repoPrefix]
|
||
if !ok || meta == nil || meta.RootPath == "" {
|
||
return "", false
|
||
}
|
||
return meta.RootPath, true
|
||
}
|
||
|
||
// loneRepoLocked returns the metadata of the only registered repo when
|
||
// exactly one repo is tracked AND that repo was indexed unprefixed
|
||
// (single-repo mode). The provenance check matters: after a 1→2→1
|
||
// track/untrack sequence the lone survivor can be a prefixed repo, and
|
||
// stale unprefixed nodes from the departed repo must keep failing
|
||
// closed instead of resolving against the wrong checkout. Caller must
|
||
// hold mi.mu.
|
||
func (mi *MultiIndexer) loneRepoLocked() *RepoMetadata {
|
||
if len(mi.repos) != 1 {
|
||
return nil
|
||
}
|
||
for _, meta := range mi.repos {
|
||
if meta != nil && meta.Unprefixed {
|
||
return meta
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// LinkedWorktreeRoots returns the on-disk roots of every tracked linked
|
||
// git worktree that shares its .git common directory with the checkout
|
||
// at mainRepoPath — i.e. the worktree siblings of that main repo. The
|
||
// query is keyed on the resolved MainRepoPath so it matches whether the
|
||
// caller passes a main checkout or one of its worktrees.
|
||
//
|
||
// Used by the edit-tool file resolver: because all worktrees of one
|
||
// repo reuse a single index identity, a repo-relative path can resolve
|
||
// against a sibling checkout. When the resolved file belongs to a
|
||
// linked worktree, the resolver re-roots the write there so an edit
|
||
// lands in the worktree's copy rather than the main checkout's.
|
||
func (mi *MultiIndexer) LinkedWorktreeRoots(mainRepoPath string) []string {
|
||
if mainRepoPath == "" {
|
||
return nil
|
||
}
|
||
wantMain := resolvedMainRepo(mainRepoPath)
|
||
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
var out []string
|
||
for _, meta := range mi.repos {
|
||
if meta == nil || meta.RootPath == "" || !meta.IsWorktree {
|
||
continue
|
||
}
|
||
if resolvedMainRepo(meta.RootPath) == wantMain {
|
||
out = append(out, meta.RootPath)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// resolvedMainRepo resolves a checkout path to its repo's main worktree
|
||
// root with symlinks evaluated. ResolveWorktree derives the main path
|
||
// two different ways — filepath.Abs for a main checkout vs git's
|
||
// canonicalized `commondir` for a linked worktree — and on platforms
|
||
// where the temp / home tree is a symlink (macOS /var -> /private/var)
|
||
// those two forms differ for the same repo. Evaluating symlinks on the
|
||
// result gives one stable identity that both inputs agree on.
|
||
func resolvedMainRepo(path string) string {
|
||
main := ResolveWorktree(path).MainRepoPath
|
||
if resolved, err := filepath.EvalSymlinks(main); err == nil {
|
||
return resolved
|
||
}
|
||
return main
|
||
}
|
||
|
||
// MergedContractRegistry combines contract registries from all per-repo
|
||
// indexers into a single registry. In multi-repo mode each repo's indexer
|
||
// runs extractContracts independently; this merges the results.
|
||
func (mi *MultiIndexer) MergedContractRegistry() *contracts.Registry {
|
||
mi.mu.RLock()
|
||
defer mi.mu.RUnlock()
|
||
|
||
merged := contracts.NewRegistry()
|
||
for repoPrefix, idx := range mi.indexers {
|
||
cr := idx.ContractRegistry()
|
||
if cr == nil {
|
||
continue
|
||
}
|
||
// Re-stamp the workspace/project slugs from the indexer
|
||
// alongside the repo prefix on merge. The contracts already
|
||
// carry these slugs from
|
||
// their source registry, but AddAllScoped is idempotent (skips
|
||
// non-empty existing values) so this stays correct even if a
|
||
// future code path forgets the stamp on first insert.
|
||
merged.AddAllScoped(cr.All(), repoPrefix, idx.WorkspaceID(), idx.ProjectID())
|
||
}
|
||
return merged
|
||
}
|
||
|
||
// attachInlinedShapes folds the field shape of each contract's
|
||
// response_type / request_type into the contract's Meta so the
|
||
// dashboard can render the expanded field list. Targets contracts
|
||
// where the type has been resolved to a graph node ID (contains
|
||
// "::") AND the type node has a shape stored in its Meta.
|
||
//
|
||
// Type-shape extraction normally runs in commitContracts via
|
||
// snapshotContractShapes + inlineEnvelopeShapes — but those passes
|
||
// run during the initial extract and miss contracts added later by
|
||
// InlineWrappers. This is the post-inline equivalent: it doesn't
|
||
// re-extract shapes (the type nodes already have them from
|
||
// snapshotContractShapes if they were referenced anywhere), it just
|
||
// attaches them to the new contract entries.
|
||
func (mi *MultiIndexer) attachInlinedShapes(cr *contracts.Registry, g graph.Store) {
|
||
idsToTouch := map[string]bool{}
|
||
for _, c := range cr.All() {
|
||
if c.Meta == nil {
|
||
continue
|
||
}
|
||
for _, key := range []string{"response_type", "request_type"} {
|
||
if v, _ := c.Meta[key].(string); v != "" && strings.Contains(v, "::") {
|
||
idsToTouch[c.ID] = true
|
||
break
|
||
}
|
||
}
|
||
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
|
||
// Touch any contract that has an envelope, even when
|
||
// the rows still carry bare type names — the loop below
|
||
// upgrades them. Otherwise we skip them and lose the
|
||
// shape attachment for sibling-file types.
|
||
idsToTouch[c.ID] = true
|
||
_ = env
|
||
}
|
||
}
|
||
srcCache := map[string][]byte{}
|
||
resolveShape := func(typeID string) any {
|
||
if typeID == "" || !strings.Contains(typeID, "::") {
|
||
return nil
|
||
}
|
||
node := g.GetNode(typeID)
|
||
if node == nil {
|
||
return nil
|
||
}
|
||
if node.Kind != graph.KindType && node.Kind != graph.KindInterface {
|
||
return nil
|
||
}
|
||
if node.Meta == nil {
|
||
node.Meta = map[string]any{}
|
||
}
|
||
if shape, ok := node.Meta["shape"]; ok && shape != nil {
|
||
return shape
|
||
}
|
||
// Lazy-extract: snapshotContractShapes only walks types
|
||
// referenced by the initial bulk extract. Types referenced
|
||
// ONLY by wrapper-inlined contracts need this fallback or
|
||
// their fields stay unread.
|
||
src := srcCache[node.FilePath]
|
||
if src == nil {
|
||
data, ok := mi.readNodeSource(node)
|
||
if !ok {
|
||
srcCache[node.FilePath] = []byte{}
|
||
return nil
|
||
}
|
||
src = data
|
||
srcCache[node.FilePath] = src
|
||
}
|
||
if len(src) == 0 {
|
||
return nil
|
||
}
|
||
extracted := contracts.ExtractShape(node.FilePath, src, node.StartLine, node.EndLine)
|
||
if extracted == nil {
|
||
return nil
|
||
}
|
||
node.Meta["shape"] = extracted
|
||
return extracted
|
||
}
|
||
for id := range idsToTouch {
|
||
items := cr.ByID(id)
|
||
changed := false
|
||
for i := range items {
|
||
if items[i].Meta == nil {
|
||
continue
|
||
}
|
||
// Top-level request/response type shapes.
|
||
for _, pair := range []struct{ typeKey, shapeKey string }{
|
||
{"response_type", "response_shape"},
|
||
{"request_type", "request_shape"},
|
||
} {
|
||
if _, has := items[i].Meta[pair.shapeKey]; has {
|
||
continue
|
||
}
|
||
typeID, _ := items[i].Meta[pair.typeKey].(string)
|
||
if shape := resolveShape(typeID); shape != nil {
|
||
items[i].Meta[pair.shapeKey] = shape
|
||
changed = true
|
||
}
|
||
}
|
||
// Envelope rows — upgrade bare type names to graph IDs
|
||
// (so the shape lookup hits) and attach shapes.
|
||
if env, ok := items[i].Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
|
||
envChanged := false
|
||
for ri, row := range env {
|
||
typeID, _ := row["type"].(string)
|
||
// Upgrade bare type name → graph ID when the
|
||
// in-file resolveTypeInFile pass left it bare
|
||
// (the type lives in a sibling file).
|
||
if typeID != "" && !strings.Contains(typeID, "::") {
|
||
matches := g.FindNodesByName(typeID)
|
||
var resolved string
|
||
for _, n := range matches {
|
||
if n.Kind != graph.KindType && n.Kind != graph.KindInterface {
|
||
continue
|
||
}
|
||
resolved = n.ID
|
||
if items[i].RepoPrefix != "" && strings.HasPrefix(n.ID, items[i].RepoPrefix+"/") {
|
||
break // prefer same-repo
|
||
}
|
||
}
|
||
if resolved != "" {
|
||
env[ri]["type"] = resolved
|
||
typeID = resolved
|
||
envChanged = true
|
||
}
|
||
}
|
||
if _, has := row["shape"]; has {
|
||
continue
|
||
}
|
||
if shape := resolveShape(typeID); shape != nil {
|
||
env[ri]["shape"] = shape
|
||
envChanged = true
|
||
}
|
||
}
|
||
if envChanged {
|
||
items[i].Meta["response_envelope"] = env
|
||
changed = true
|
||
}
|
||
}
|
||
}
|
||
if changed {
|
||
cr.ReplaceByID(id, items)
|
||
}
|
||
}
|
||
}
|
||
|
||
// readNodeSource returns the source bytes of the file the node lives
|
||
// in, resolving the repo prefix to a real disk path via tracked-repo
|
||
// metadata. Mirrors wrapperSourceReader's path-resolution dance.
|
||
func (mi *MultiIndexer) readNodeSource(node *graph.Node) ([]byte, bool) {
|
||
if node == nil || node.FilePath == "" {
|
||
return nil, false
|
||
}
|
||
rel := node.FilePath
|
||
if node.RepoPrefix != "" {
|
||
meta := mi.GetMetadata(node.RepoPrefix)
|
||
if meta == nil || meta.RootPath == "" {
|
||
return nil, false
|
||
}
|
||
rel = strings.TrimPrefix(rel, node.RepoPrefix+"/")
|
||
data, err := os.ReadFile(filepath.Join(meta.RootPath, rel))
|
||
if err != nil {
|
||
return nil, false
|
||
}
|
||
return data, true
|
||
}
|
||
for _, m := range mi.AllMetadata() {
|
||
data, err := os.ReadFile(filepath.Join(m.RootPath, rel))
|
||
if err == nil {
|
||
return data, true
|
||
}
|
||
}
|
||
return nil, false
|
||
}
|
||
|
||
// wrapperSourceReader returns a SourceReader closure that maps a graph
|
||
// node back to its on-disk bytes by joining the node's repo-relative
|
||
// FilePath with the repo's RootPath from MultiIndexer metadata. In
|
||
// single-repo mode (no RepoPrefix on nodes) the indexer's sole root is
|
||
// used. Read results are memoized inside the closure so multi-caller
|
||
// wrappers don't trigger N disk reads per file.
|
||
func (mi *MultiIndexer) wrapperSourceReader() contracts.SourceReader {
|
||
cache := make(map[string][]byte)
|
||
miss := make(map[string]struct{})
|
||
|
||
readFile := func(absPath string) ([]byte, bool) {
|
||
if b, ok := cache[absPath]; ok {
|
||
return b, true
|
||
}
|
||
if _, skipped := miss[absPath]; skipped {
|
||
return nil, false
|
||
}
|
||
b, err := os.ReadFile(absPath)
|
||
if err != nil {
|
||
miss[absPath] = struct{}{}
|
||
return nil, false
|
||
}
|
||
cache[absPath] = b
|
||
return b, true
|
||
}
|
||
|
||
return func(n *graph.Node) ([]byte, bool) {
|
||
if n == nil || n.FilePath == "" {
|
||
return nil, false
|
||
}
|
||
// Multi-repo case: strip "<repo-prefix>/" from the FilePath and
|
||
// join with the repo's recorded root.
|
||
if n.RepoPrefix != "" {
|
||
meta := mi.GetMetadata(n.RepoPrefix)
|
||
if meta == nil || meta.RootPath == "" {
|
||
return nil, false
|
||
}
|
||
rel := strings.TrimPrefix(n.FilePath, n.RepoPrefix+"/")
|
||
return readFile(filepath.Join(meta.RootPath, rel))
|
||
}
|
||
// Single-repo fallback: a node without a RepoPrefix carries its
|
||
// path relative to the sole indexer root. Try each known repo
|
||
// root since the single-repo path wraps through indexSingleRepo.
|
||
for _, meta := range mi.AllMetadata() {
|
||
if b, ok := readFile(filepath.Join(meta.RootPath, n.FilePath)); ok {
|
||
return b, true
|
||
}
|
||
}
|
||
// Last resort: treat FilePath as already absolute (tests).
|
||
return readFile(n.FilePath)
|
||
}
|
||
}
|
||
|
||
// ReconcileContractEdges walks the merged contract registry, runs the
|
||
// consumer↔provider matcher, and writes the results into the graph as
|
||
// EdgeMatches edges pointing from consumer-contract nodes to their matched
|
||
// provider-contract nodes. Every call evicts the prior set of EdgeMatches
|
||
// first so the edges stay in sync with the current contract view; that's
|
||
// correct after track / untrack / re-index / watcher re-scan. Returns the
|
||
// number of match edges added so callers can log or test the effect.
|
||
//
|
||
// This is what makes get_call_chain traverse service boundaries: without a
|
||
// persisted contract→contract edge, the matcher's result is only visible
|
||
// via the `contracts check` tool and traversals stop at each service's
|
||
// boundary.
|
||
func (mi *MultiIndexer) ReconcileContractEdges() int {
|
||
// Serialise the whole pass: the evict-then-mint of EdgeMatches, topic
|
||
// edges, and the bridge subgraph spans many non-atomic store writes,
|
||
// and several goroutines call this concurrently (see reconcileMu).
|
||
mi.reconcileMu.Lock()
|
||
defer mi.reconcileMu.Unlock()
|
||
|
||
g := mi.Graph()
|
||
if g == nil {
|
||
return 0
|
||
}
|
||
|
||
// Evict any prior EdgeMatches so the graph reflects only the current
|
||
// registry. Collect first, remove second — we're mutating the same
|
||
// out-edge lists we'd be iterating otherwise.
|
||
type edgeKey struct{ from, to string }
|
||
var stale []edgeKey
|
||
var staleTopicProduces, staleTopicConsumes []edgeKey
|
||
// Collect only the three reconciled edge kinds via the edges_by_kind
|
||
// index, rather than scanning (and meta-decoding) the whole edge set.
|
||
for e := range g.EdgesByKind(graph.EdgeMatches) {
|
||
stale = append(stale, edgeKey{e.From, e.To})
|
||
}
|
||
for e := range g.EdgesByKind(graph.EdgeProducesTopic) {
|
||
staleTopicProduces = append(staleTopicProduces, edgeKey{e.From, e.To})
|
||
}
|
||
for e := range g.EdgesByKind(graph.EdgeConsumesTopic) {
|
||
staleTopicConsumes = append(staleTopicConsumes, edgeKey{e.From, e.To})
|
||
}
|
||
for _, k := range stale {
|
||
g.RemoveEdge(k.from, k.to, graph.EdgeMatches)
|
||
}
|
||
// Evict prior topic edges so a renamed topic or removed callsite
|
||
// doesn't keep its edges alive. The KindTopic node itself is
|
||
// addressable by ID; when no consumer or producer remains for a
|
||
// given topic node we orphan-collect it after the new edges are
|
||
// emitted (see end of this function).
|
||
for _, k := range staleTopicProduces {
|
||
g.RemoveEdge(k.from, k.to, graph.EdgeProducesTopic)
|
||
}
|
||
for _, k := range staleTopicConsumes {
|
||
g.RemoveEdge(k.from, k.to, graph.EdgeConsumesTopic)
|
||
}
|
||
|
||
merged := mi.MergedContractRegistry()
|
||
if merged == nil {
|
||
return 0
|
||
}
|
||
|
||
// Inline HTTP wrapper callers (T2.4). Codebases that route every
|
||
// endpoint through a helper like `request(path, ...)` produce one
|
||
// parametric consumer contract per wrapper at extraction time —
|
||
// useless for matching. InlineWrappers walks incoming call edges
|
||
// of each wrapper, re-reads the caller's source, and emits a
|
||
// specific consumer contract per literal path. The returned
|
||
// contracts are also persisted into their owning repo's per-repo
|
||
// registry so subsequent `contracts list`/`check` calls see them
|
||
// (MergedContractRegistry rebuilds fresh each call and would
|
||
// otherwise lose them).
|
||
inlined := contracts.InlineWrappers(merged, g, mi.wrapperSourceReader())
|
||
if len(inlined) > 0 {
|
||
mi.mu.RLock()
|
||
for _, c := range inlined {
|
||
if c.RepoPrefix == "" {
|
||
continue
|
||
}
|
||
idx, ok := mi.indexers[c.RepoPrefix]
|
||
if !ok {
|
||
continue
|
||
}
|
||
cr := idx.ContractRegistry()
|
||
if cr == nil {
|
||
continue
|
||
}
|
||
// Skip if the same contract is already persisted —
|
||
// ReconcileContractEdges runs on every repo change, and
|
||
// appending the same inlined contract on every pass would
|
||
// blow up the registry with duplicates. Compare on the
|
||
// Registry.All() dedupe key.
|
||
alreadyPersisted := false
|
||
for _, existing := range cr.ByID(c.ID) {
|
||
if existing.SymbolID == c.SymbolID &&
|
||
existing.FilePath == c.FilePath &&
|
||
existing.Role == c.Role {
|
||
alreadyPersisted = true
|
||
break
|
||
}
|
||
}
|
||
if !alreadyPersisted {
|
||
cr.Add(c)
|
||
}
|
||
}
|
||
mi.mu.RUnlock()
|
||
|
||
// Wrapper-inlined contracts arrive AFTER commitContracts ran
|
||
// its UpgradeBareTypeRefs pass, so their response_type /
|
||
// request_type still carries bare names like "ToolInfo".
|
||
// Re-run the upgrade against the merged graph so downstream
|
||
// snapshotContractShapes finds the type node and the
|
||
// dashboard sees fields instead of a string.
|
||
mi.mu.RLock()
|
||
lookup := func(name, repoHint string) []string {
|
||
matches := mi.graph.FindNodesByName(name)
|
||
if len(matches) == 0 {
|
||
return nil
|
||
}
|
||
ids := make([]string, 0, len(matches))
|
||
for _, n := range matches {
|
||
if n.Kind != graph.KindType && n.Kind != graph.KindInterface {
|
||
continue
|
||
}
|
||
ids = append(ids, n.ID)
|
||
}
|
||
// Prefer same-repo when multiple match.
|
||
if len(ids) > 1 && repoHint != "" {
|
||
var sameRepo []string
|
||
for _, id := range ids {
|
||
if strings.HasPrefix(id, repoHint+"/") {
|
||
sameRepo = append(sameRepo, id)
|
||
}
|
||
}
|
||
if len(sameRepo) > 0 {
|
||
return sameRepo
|
||
}
|
||
}
|
||
return ids
|
||
}
|
||
for _, idx := range mi.indexers {
|
||
cr := idx.ContractRegistry()
|
||
if cr != nil {
|
||
cr.UpgradeBareTypeRefs(lookup)
|
||
}
|
||
}
|
||
// UpgradeBareTypeRefs leaves names with ≥2 candidates alone
|
||
// (e.g. a TS app declaring `DashboardSnapshot` in both
|
||
// `lib/schema.ts` and `lib/types.ts`). disambiguateBareTypesViaImports
|
||
// re-reads the consumer's source, parses its `import` lines,
|
||
// and picks the candidate whose graph FilePath matches an
|
||
// imported module. Runs before attachInlinedShapes so the
|
||
// shape attachment sees fully-qualified IDs.
|
||
for _, idx := range mi.indexers {
|
||
cr := idx.ContractRegistry()
|
||
if cr != nil {
|
||
mi.disambiguateBareTypesViaImports(cr, mi.graph)
|
||
}
|
||
}
|
||
// Now that response_type / request_type point at real graph
|
||
// nodes, fold each referenced type's shape (struct fields)
|
||
// into the contract's Meta so the dashboard renders the
|
||
// expanded field list instead of just the type name. Mirrors
|
||
// what snapshotContractShapes + inlineEnvelopeShapes do for
|
||
// initially-extracted contracts.
|
||
for _, idx := range mi.indexers {
|
||
cr := idx.ContractRegistry()
|
||
if cr == nil {
|
||
continue
|
||
}
|
||
mi.attachInlinedShapes(cr, mi.graph)
|
||
}
|
||
mi.mu.RUnlock()
|
||
}
|
||
|
||
// Bind provider-contract SymbolIDs that came from spec files
|
||
// (.proto for gRPC, OpenAPI YAML/JSON for HTTP). Without this
|
||
// step the matcher finds pairs but the bridge-emission check
|
||
// below skips them because provider SymbolID is empty. Must run
|
||
// before Match so Match sees the updated records.
|
||
contracts.BindProviderSymbols(merged, g)
|
||
|
||
result := contracts.Match(merged)
|
||
added := 0
|
||
// Track which topic nodes we've already materialised so the loop
|
||
// emits one node per (broker, topic) bucket even when a topic has
|
||
// fan-out across many consumers. The dedupe key is the topic
|
||
// node's ID — its repo-prefix is already encoded in Contract.ID.
|
||
topicNodes := make(map[string]struct{})
|
||
for _, m := range result.Matched {
|
||
// Connect the consumer's enclosing symbol directly to the
|
||
// provider's enclosing symbol. Contract nodes in the graph are
|
||
// deduped by Contract.ID, so a provider and a consumer that share
|
||
// the same ID collapse into one node — a contract→contract edge
|
||
// would be a self-loop. Symbol→symbol bypasses that and gives
|
||
// get_call_chain the traversal it needs: saveTuck (extension) →
|
||
// Handler.CreateTuck (core-api).
|
||
if m.Provider.SymbolID == "" || m.Consumer.SymbolID == "" {
|
||
continue
|
||
}
|
||
if m.Provider.SymbolID == m.Consumer.SymbolID {
|
||
continue
|
||
}
|
||
g.AddEdge(&graph.Edge{
|
||
From: m.Consumer.SymbolID,
|
||
To: m.Provider.SymbolID,
|
||
Kind: graph.EdgeMatches,
|
||
FilePath: m.Consumer.FilePath,
|
||
Line: m.Consumer.Line,
|
||
Confidence: 1.0,
|
||
ConfidenceLabel: "EXTRACTED",
|
||
Origin: graph.OriginASTResolved,
|
||
CrossRepo: m.CrossRepo,
|
||
})
|
||
added++
|
||
|
||
// Materialise the broker topic node + producer/consumer
|
||
// edges. Only fires for ContractTopic matches; HTTP / gRPC /
|
||
// WS / GraphQL / env contracts fall through with just the
|
||
// EdgeMatches edge above.
|
||
if m.Provider.Type == contracts.ContractTopic {
|
||
emitTopicEdges(g, m, topicNodes)
|
||
}
|
||
}
|
||
|
||
// Persist the matched contract groups as the bridge subgraph: one
|
||
// KindContractBridge node per group plus EdgeBridges fan-out to
|
||
// the participating contract nodes. The pass evicts the previous
|
||
// bridge generation internally, so it stays idempotent across
|
||
// reconciles and drops bridges whose contracts disappeared.
|
||
MaterializeContractBridges(g, result.Matched)
|
||
|
||
// Topic nodes whose producer and consumer edges all evaporated
|
||
// since the previous reconcile remain in the graph as leaf
|
||
// nodes — Graph has no public RemoveNode and the next reconcile
|
||
// upserts them anyway. A topic that lost every callsite is an
|
||
// invisible-but-harmless cost (a single KindTopic node with no
|
||
// neighbors). Worth revisiting if topic churn ever bloats the
|
||
// graph; in the meantime we lean on EvictRepo / EvictFile to
|
||
// reclaim memory when a whole repo's topic vocabulary changes.
|
||
return added
|
||
}
|
||
|
||
// emitTopicEdges materialises the KindTopic node and the
|
||
// EdgeProducesTopic / EdgeConsumesTopic edges that pair a matched
|
||
// producer/consumer pair across the workspace. The topic ID is
|
||
// reconstructed from the Contract.ID (already `topic::<broker>::
|
||
// <name>`) so the node ID matches the contract ID 1:1 — agents that
|
||
// have the contract ID can also look up the topic node directly.
|
||
// Meta on the node carries the broker family and the raw topic name
|
||
// for filterless queries.
|
||
func emitTopicEdges(g graph.Store, m contracts.CrossLink, topicNodes map[string]struct{}) {
|
||
// Trust the matcher to bucket only same-broker contracts together
|
||
// because Contract.ID already includes the broker token; if the
|
||
// broker isn't on the provider Meta, fall through to the contract
|
||
// ID's middle segment so the node carries something useful.
|
||
broker, _ := m.Provider.Meta["broker"].(string)
|
||
topicName, _ := m.Provider.Meta["topic"].(string)
|
||
if broker == "" || topicName == "" {
|
||
// Defensive fallback — extract from the Contract.ID shape
|
||
// `topic::<broker>::<name>`. If the ID isn't structured we
|
||
// skip rather than emit a node with an unidentifiable broker
|
||
// (such an edge would be misleading in cross-broker queries).
|
||
broker2, name2, ok := parseTopicContractID(m.Provider.ID)
|
||
if !ok {
|
||
return
|
||
}
|
||
if broker == "" {
|
||
broker = broker2
|
||
}
|
||
if topicName == "" {
|
||
topicName = name2
|
||
}
|
||
}
|
||
|
||
topicID := m.Provider.ID // canonical: topic::<broker>::<name>
|
||
if _, ok := topicNodes[topicID]; !ok {
|
||
topicNodes[topicID] = struct{}{}
|
||
// Preserve any existing node (a prior reconcile may have
|
||
// created it) but always refresh Meta so a broker rename
|
||
// isn't sticky across reconciles. AddNode in this codebase
|
||
// is upsert-style — see graph.Graph.AddNode.
|
||
g.AddNode(&graph.Node{
|
||
ID: topicID,
|
||
Kind: graph.KindTopic,
|
||
Name: topicName,
|
||
FilePath: m.Provider.FilePath,
|
||
Language: "contract",
|
||
RepoPrefix: m.Provider.RepoPrefix,
|
||
WorkspaceID: m.Provider.EffectiveWorkspace(),
|
||
ProjectID: m.Provider.EffectiveProject(),
|
||
Meta: map[string]any{
|
||
"broker": broker,
|
||
"name": topicName,
|
||
},
|
||
})
|
||
}
|
||
|
||
g.AddEdge(&graph.Edge{
|
||
From: m.Provider.SymbolID,
|
||
To: topicID,
|
||
Kind: graph.EdgeProducesTopic,
|
||
FilePath: m.Provider.FilePath,
|
||
Line: m.Provider.Line,
|
||
Confidence: 1.0,
|
||
ConfidenceLabel: "EXTRACTED",
|
||
Origin: graph.OriginASTResolved,
|
||
CrossRepo: false,
|
||
Meta: map[string]any{
|
||
"broker": broker,
|
||
},
|
||
})
|
||
g.AddEdge(&graph.Edge{
|
||
From: m.Consumer.SymbolID,
|
||
To: topicID,
|
||
Kind: graph.EdgeConsumesTopic,
|
||
FilePath: m.Consumer.FilePath,
|
||
Line: m.Consumer.Line,
|
||
Confidence: 1.0,
|
||
ConfidenceLabel: "EXTRACTED",
|
||
Origin: graph.OriginASTResolved,
|
||
CrossRepo: m.CrossRepo,
|
||
Meta: map[string]any{
|
||
"broker": broker,
|
||
},
|
||
})
|
||
}
|
||
|
||
// parseTopicContractID splits a Contract.ID of the form
|
||
// `topic::<broker>::<name>` (or its repo-prefixed counterpart
|
||
// `<repo>/topic::<broker>::<name>`) into broker + name. Returns
|
||
// ok==false for any other shape so callers can skip ill-formed
|
||
// topic contracts rather than fabricate a broker label.
|
||
func parseTopicContractID(id string) (broker, name string, ok bool) {
|
||
// Strip any leading repo-prefix segment ("repo/topic::..."). The
|
||
// applyRepoPrefix step prepends `<repo>/` to synthetic IDs of
|
||
// the form `topic::...`, so we look for the inner `topic::`
|
||
// marker rather than splitting on the leading slash.
|
||
idx := strings.Index(id, "topic::")
|
||
if idx < 0 {
|
||
return "", "", false
|
||
}
|
||
rest := id[idx+len("topic::"):]
|
||
sep := strings.Index(rest, "::")
|
||
if sep <= 0 || sep == len(rest)-2 {
|
||
return "", "", false
|
||
}
|
||
return rest[:sep], rest[sep+2:], true
|
||
}
|
||
|
||
// Graph returns the underlying shared graph.
|
||
func (mi *MultiIndexer) Graph() graph.Store {
|
||
return mi.graph
|
||
}
|
||
|
||
// SetRemoteStitch enables cross-daemon proxy-edge minting on every
|
||
// CrossRepoResolver this MultiIndexer constructs. Called once by the
|
||
// daemon entry point when federation.edges.enabled; a nil prober leaves
|
||
// the resolvers on read-only federation.
|
||
func (mi *MultiIndexer) SetRemoteStitch(prober resolver.RemoteDeclarationProber, budget int) {
|
||
mi.mu.Lock()
|
||
defer mi.mu.Unlock()
|
||
mi.stitchProber = prober
|
||
mi.proxyBudget = budget
|
||
}
|
||
|
||
// applyRemoteStitch wires the proxy-edge mint into a freshly built resolver
|
||
// when a prober is installed.
|
||
func (mi *MultiIndexer) applyRemoteStitch(cr *resolver.CrossRepoResolver) {
|
||
mi.mu.RLock()
|
||
prober, budget := mi.stitchProber, mi.proxyBudget
|
||
mi.mu.RUnlock()
|
||
if prober != nil {
|
||
cr.EnableRemoteStitch(prober, budget)
|
||
}
|
||
}
|
||
|
||
// Search returns the shared search backend.
|
||
func (mi *MultiIndexer) Search() search.Backend {
|
||
return mi.search
|
||
}
|
||
|
||
// ExportVectorIndex serializes the workspace-global semantic-search
|
||
// vector index — there is one shared HNSW index across every tracked
|
||
// repo, not one per repo. Returns nil, 0, 0 when no vector index is
|
||
// active (embeddings disabled, or the backend is still text-only).
|
||
// Used by the daemon snapshot path so a default-on daemon does not
|
||
// re-embed the whole graph on every restart.
|
||
func (mi *MultiIndexer) ExportVectorIndex() ([]byte, int, int) {
|
||
sw, ok := mi.search.(*search.Swappable)
|
||
if !ok {
|
||
return nil, 0, 0
|
||
}
|
||
hybrid, ok := sw.Inner().(*search.HybridBackend)
|
||
if !ok {
|
||
return nil, 0, 0
|
||
}
|
||
vec := hybrid.VectorIndex()
|
||
if vec == nil || vec.Count() == 0 {
|
||
return nil, 0, 0
|
||
}
|
||
var buf bytes.Buffer
|
||
if err := vec.Save(&buf); err != nil {
|
||
mi.logger.Warn("failed to export vector index", zap.Error(err))
|
||
return nil, 0, 0
|
||
}
|
||
return buf.Bytes(), vec.Dims(), vec.Count()
|
||
}
|
||
|
||
// ImportVectorIndex restores a previously-exported vector index into
|
||
// the shared search backend, wrapping the current text backend in a
|
||
// HybridBackend. It is a no-op when embeddings are disabled (no
|
||
// configured embedder) or when the cached index's dimensionality does
|
||
// not match the active embedder — a provider switch (GloVe 50d → ONNX
|
||
// 384d) makes the cached vectors meaningless, so the indexer re-embeds
|
||
// instead. Returns an error only on a structurally corrupt index blob.
|
||
func (mi *MultiIndexer) ImportVectorIndex(data []byte, dims, count int) error {
|
||
if len(data) == 0 || mi.embedder == nil {
|
||
return nil
|
||
}
|
||
if embedderDims := mi.embedder.Dimensions(); embedderDims > 0 && embedderDims != dims {
|
||
mi.logger.Info("vector index dims mismatch, will re-embed",
|
||
zap.Int("cached_dims", dims), zap.Int("embedder_dims", embedderDims))
|
||
return nil
|
||
}
|
||
sw, ok := mi.search.(*search.Swappable)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
vec := search.NewVector(dims)
|
||
if err := vec.LoadFrom(bytes.NewReader(data)); err != nil {
|
||
return fmt.Errorf("import vector index: %w", err)
|
||
}
|
||
vec.SetCount(count)
|
||
|
||
// Unwrap an existing HybridBackend to its text side before
|
||
// re-wrapping so we never nest Hybrids (each retains a stale
|
||
// vector index — see buildSearchIndex for the memory rationale).
|
||
inner := sw.Inner()
|
||
if hyb, ok := inner.(*search.HybridBackend); ok {
|
||
inner = hyb.TextBackend()
|
||
}
|
||
sw.Swap(search.NewHybrid(inner, vec, mi.embedder))
|
||
mi.logger.Info("restored vector index from snapshot",
|
||
zap.Int("vectors", count), zap.Int("dims", dims))
|
||
return nil
|
||
}
|
||
|
||
// AutoDetectRepos walks immediate subdirectories of parentPath looking for
|
||
// .git directories. If parentPath itself is a Git repo, it returns a single
|
||
// entry (the caller should index it as single-repo). If zero Git repos are
|
||
// found, it returns nil so the caller can fall back to single-repo mode.
|
||
// This is gated by the workspace.auto_detect config flag.
|
||
func (mi *MultiIndexer) AutoDetectRepos(parentPath string) []config.RepoEntry {
|
||
absPath, err := filepath.Abs(parentPath)
|
||
if err != nil {
|
||
mi.logger.Warn("auto-detect: failed to resolve path", zap.String("path", parentPath), zap.Error(err))
|
||
return nil
|
||
}
|
||
|
||
// If the path itself is a Git repo, return it as a single repo.
|
||
if isGitRepo(absPath) {
|
||
return []config.RepoEntry{{
|
||
Path: absPath,
|
||
Name: filepath.Base(absPath),
|
||
}}
|
||
}
|
||
|
||
// Walk immediate subdirectories (not recursive) for .git dirs.
|
||
entries, err := os.ReadDir(absPath)
|
||
if err != nil {
|
||
mi.logger.Warn("auto-detect: failed to read directory", zap.String("path", absPath), zap.Error(err))
|
||
return nil
|
||
}
|
||
|
||
var repos []config.RepoEntry
|
||
for _, entry := range entries {
|
||
if !entry.IsDir() {
|
||
continue
|
||
}
|
||
subDir := filepath.Join(absPath, entry.Name())
|
||
if isGitRepo(subDir) {
|
||
repos = append(repos, config.RepoEntry{
|
||
Path: subDir,
|
||
Name: entry.Name(), // Derive RepoPrefix from subdirectory name.
|
||
})
|
||
}
|
||
}
|
||
|
||
// If zero Git repos found, return nil — caller falls back to single-repo.
|
||
if len(repos) == 0 {
|
||
return nil
|
||
}
|
||
|
||
return repos
|
||
}
|
||
|
||
// isGitRepo checks whether the given directory contains a .git subdirectory.
|
||
func isGitRepo(dir string) bool {
|
||
info, err := os.Stat(filepath.Join(dir, ".git"))
|
||
if err != nil {
|
||
return false
|
||
}
|
||
return info.IsDir()
|
||
}
|