Files
wehub-resource-sync a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:33:42 +08:00

3427 lines
125 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package indexer
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
"time"
"go.uber.org/zap"
"github.com/zzet/gortex/internal/config"
"github.com/zzet/gortex/internal/contracts"
"github.com/zzet/gortex/internal/embedding"
"github.com/zzet/gortex/internal/graph"
"github.com/zzet/gortex/internal/parser"
"github.com/zzet/gortex/internal/pathkey"
"github.com/zzet/gortex/internal/progress"
"github.com/zzet/gortex/internal/resolver"
"github.com/zzet/gortex/internal/search"
"github.com/zzet/gortex/internal/search/trigram"
"github.com/zzet/gortex/internal/semantic"
)
// RepoMetadata holds per-repo indexing state.
type RepoMetadata struct {
RepoPrefix string
RootPath string
Identity *RepoIdentity
LastIndexTime time.Time
FileCount int
NodeCount int
EdgeCount int
ParseErrors []IndexError
FileMtimes map[string]int64
// IsWorktree records whether RootPath was a linked git worktree
// (as opposed to a main checkout) at the time the repo was
// tracked. Captured here because once the worktree directory is
// removed from disk it can no longer be classified — the janitor
// needs this remembered flag to know a vanished root was a
// worktree and may be garbage-collected.
IsWorktree bool
// Unprefixed records that this repo was indexed in single-repo
// mode: its nodes carry RepoPrefix="" and raw relative file paths.
// The empty-prefix resolution fallback (RepoRoot, ResolveFilePath)
// honours only a lone repo that actually minted unprefixed nodes —
// without the provenance check, stale unprefixed nodes surviving a
// track/untrack transition would resolve against whatever repo
// happens to be the lone one. Eviction also branches on it:
// unprefixed nodes are invisible to the byRepo bucket EvictRepo
// walks, so UntrackRepo must evict them file-by-file.
Unprefixed bool
}
// MultiIndexer orchestrates indexing across multiple repositories.
type MultiIndexer struct {
graph graph.Store
registry *parser.Registry
search search.Backend
embedder embedding.Provider
repos map[string]*RepoMetadata // repoPrefix → metadata
indexers map[string]*Indexer // repoPrefix → per-repo indexer
configMgr *config.ConfigManager
logger *zap.Logger
mu sync.RWMutex
// reconcileMu serialises ReconcileContractEdges end-to-end. The pass
// evicts the prior EdgeMatches / topic / bridge generation and mints
// a fresh one across many independent graph-store writes — it is NOT
// atomic. Several goroutines drive it concurrently (the periodic
// janitor's ReconcileAll, the file-watcher's IncrementalReindex,
// MCP-triggered track / untrack / index), and mi.mu is only taken in
// fine-grained spots inside, not across the whole pass. Without this
// lock two overlapping reconciles can interleave evict and mint and
// persist a stale generation (a bridge wiped by the other run's
// EvictFile after it was minted). A dedicated outer mutex keeps the
// pass self-consistent without widening mi.mu's scope.
reconcileMu sync.Mutex
// stitchProber / proxyBudget wire the cross-daemon proxy-edge feature:
// when set by the daemon entry point (flag on), every CrossRepoResolver
// this MultiIndexer builds mints proxy edges to remote-owned symbols
// on positive evidence. A nil prober keeps the resolvers on read-only
// federation (the default).
stitchProber resolver.RemoteDeclarationProber
proxyBudget int
// deferGlobalPasses, when set, propagates SetDeferGlobalPasses(true)
// to every per-repo Indexer constructed by this MultiIndexer. Batch
// callers (warmup, ReconcileAll) flip it on around their loop and
// invoke RunGlobalGraphPasses once at the end so the O(global) walks
// (InferImplements / InferOverrides / markTestSymbolsAndEmitEdges)
// don't run R times against an R-repo graph.
deferGlobalPasses bool
// deferResolve, when set, propagates SetDeferResolve(true) to every
// per-repo Indexer constructed by this MultiIndexer. Used by the
// parallel warmup path: per-repo ResolveAll / contract extract /
// semantic enrich mutate the shared graph, so running them in
// parallel across repos races. With this flag the parallel loop
// just parses; RunDeferredPassesAll runs the per-repo passes
// serially after the loop. Independent of deferGlobalPasses — that
// flag covers a separate (cheaper) set of O(global) walks.
deferResolve bool
// batchChangedPrefixes scopes the per-repo clone-detection and
// clone-index Rebuild passes in RunGlobalGraphPasses to the repos that
// actually re-indexed in the current batch. nil — the default, and what
// every one-off EndBatch caller leaves it as — means "run the clone
// passes for every tracked repo", the prior whole-workspace behaviour.
// The daemon warmup arms it (ArmBatchScope) before EndBatch so an
// N-repo warm restart where only one repo changed stops paying ~N
// full-graph clone walks for repos whose clone edges are already on
// disk. Clone edges are per-repo (no cross-repo pair is ever formed),
// so an unchanged repo's persisted edges stay valid; its in-memory
// incremental clone index is reseeded lazily on its first later edit.
// Consumed and cleared by RunGlobalGraphPasses. Guarded by mi.mu.
batchChangedPrefixes map[string]struct{}
// resolverLSPHelper is the resolve-time LSP helper propagated
// onto every per-repo Indexer and onto the global post-pass
// resolver in RunDeferredPassesAll. nil means no LSP hot-path
// (heuristic-only resolution, the pre-N5 behaviour). The
// daemon installs the helper via SetResolverLSPHelper after
// constructing the LSP router; a multi-repo composite helper
// dispatches per-file by repo prefix.
resolverLSPHelper resolver.LSPHelper
// onRepoTracked, when non-nil, is invoked after a fresh
// TrackRepoCtx call has resolved the repo's prefix and
// absolute path but before indexing starts. The daemon uses
// this hook to register a per-repo resolver-time LSP helper
// against the LSPHelper registry so dynamically-tracked repos
// participate in the N5 hot path without daemon restart.
onRepoTracked func(prefix, absPath string)
// skipVectorBuild, when set, propagates SetSkipVectorBuild(true) to
// every per-repo Indexer this MultiIndexer constructs, so their
// buildSearchIndex passes populate only the text index and never
// embed. The daemon flips it on for the warmup loop when a snapshot
// already carries the workspace vector index — re-embedding 30k+
// symbols only to overwrite them with the cached index is the
// dominant restart cost. After warmup it restores the cached index
// once via ImportVectorIndex and clears the flag.
skipVectorBuild bool
// embedChunkOpts is the AST sub-chunking configuration propagated
// to every per-repo Indexer this MultiIndexer constructs. The zero
// value leaves the chunker on its built-in defaults.
embedChunkOpts embedding.ChunkOptions
// embedMaxSymbols overrides the vector-index size cap propagated to
// every per-repo Indexer. Zero keeps the built-in default.
embedMaxSymbols int
// embedAPIConcurrency bounds parallel embedding requests against an
// API-backed embedder, propagated to every per-repo Indexer. Zero
// keeps the built-in default.
embedAPIConcurrency int
// semanticMgr is the semantic enrichment manager propagated to
// every per-repo Indexer. When nil (the default), per-repo
// deferred passes skip semantic enrichment — this is the root
// cause of "enrich:0" in daemon mode. Set by the daemon via
// SetSemanticManager before IndexAll / TrackRepo.
semanticMgr *semantic.Manager
}
// SetEmbedder installs the embedding provider every per-repo indexer
// should use. Must be called before IndexAll / TrackRepo for vectors
// to land in the graph — without this the fresh Indexer created per
// repo has embedder=nil and buildSearchIndex skips the vector pass.
// Safe to call zero or one times; subsequent calls silently replace.
func (mi *MultiIndexer) SetEmbedder(e embedding.Provider) {
mi.mu.Lock()
defer mi.mu.Unlock()
mi.embedder = e
}
// npmAliasResolver builds a resolver.NpmAliasResolver covering every
// tracked repo's on-disk root. Installed on the global post-pass
// resolver and the cross-repo resolver so a JS/TS import declared
// through an npm alias resolves to a locally-vendored real package
// anywhere in the workspace. Returns nil when no repo has a usable
// root — callers treat that as "no alias rewriting".
func (mi *MultiIndexer) npmAliasResolver() resolver.NpmAliasResolver {
roots := map[string]string{}
for prefix, meta := range mi.AllMetadata() {
if meta != nil && meta.RootPath != "" {
roots[prefix] = meta.RootPath
}
}
idx := newNpmAliasIndex(roots)
if idx == nil {
return nil
}
return idx.Resolve
}
// workspaceMembershipResolver builds a resolver.WorkspaceMembership
// covering every tracked repo's on-disk root. Installed on the global
// post-pass resolver and the cross-repo resolver so a same-named import
// collision is broken in favour of the importer's own package-manager
// workspace member. Returns nil when no repo is a workspace root —
// callers treat that as "no workspace signal".
func (mi *MultiIndexer) workspaceMembershipResolver() resolver.WorkspaceMembership {
roots := map[string]string{}
for prefix, meta := range mi.AllMetadata() {
if meta != nil && meta.RootPath != "" {
roots[prefix] = meta.RootPath
}
}
idx := newWorkspaceMembershipIndex(roots)
if idx == nil {
return nil
}
return idx.Lookup
}
// newPerRepoIndexer constructs a per-repo Indexer with the standard
// MultiIndexer wiring (shared search backend, embedder if configured,
// deferred-global-passes flag propagated). Centralised so the flag
// plumbing stays in one place.
func (mi *MultiIndexer) newPerRepoIndexer(cfg config.IndexConfig) *Indexer {
idx := New(mi.graph, mi.registry, cfg, mi.logger)
idx.search = mi.search
if mi.embedder != nil {
idx.SetEmbedder(mi.embedder)
}
idx.SetDeferGlobalPasses(mi.deferGlobalPasses)
idx.SetDeferResolve(mi.deferResolve)
idx.SetSkipVectorBuild(mi.skipVectorBuild)
idx.SetEmbeddingChunkOptions(mi.embedChunkOpts)
idx.SetEmbeddingMaxSymbols(mi.embedMaxSymbols)
idx.SetEmbeddingAPIConcurrency(mi.embedAPIConcurrency)
if mi.resolverLSPHelper != nil {
idx.SetResolverLSPHelper(mi.resolverLSPHelper)
}
if mi.semanticMgr != nil {
idx.SetSemanticManager(mi.semanticMgr)
}
return idx
}
// SetEmbeddingChunkOptions installs the AST sub-chunking configuration
// every per-repo Indexer this MultiIndexer constructs should use, and
// re-applies it to every per-repo Indexer already built. Call before
// IndexAll / TrackRepo so the warmup indexers pick it up.
func (mi *MultiIndexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) {
mi.mu.Lock()
mi.embedChunkOpts = opts
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetEmbeddingChunkOptions(opts)
}
}
// SetEmbeddingMaxSymbols installs the vector-index size cap every
// per-repo Indexer this MultiIndexer constructs should use, and
// re-applies it to every per-repo Indexer already built. Zero keeps
// the built-in default.
func (mi *MultiIndexer) SetEmbeddingMaxSymbols(n int) {
mi.mu.Lock()
mi.embedMaxSymbols = n
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetEmbeddingMaxSymbols(n)
}
}
// SetEmbeddingAPIConcurrency installs the parallel-embedding-request
// cap every per-repo Indexer this MultiIndexer constructs should use,
// and re-applies it to every per-repo Indexer already built. Zero
// keeps the built-in default; the cap only affects API-backed
// embedders.
func (mi *MultiIndexer) SetEmbeddingAPIConcurrency(n int) {
mi.mu.Lock()
mi.embedAPIConcurrency = n
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetEmbeddingAPIConcurrency(n)
}
}
// SetSemanticManager installs the semantic enrichment manager every
// per-repo Indexer this MultiIndexer constructs should use, and
// re-applies it to every per-repo Indexer already built. Without
// this call, daemon-mode enrichment produces zero results because
// the per-repo Indexers never receive the semantic manager.
func (mi *MultiIndexer) SetSemanticManager(m *semantic.Manager) {
mi.mu.Lock()
mi.semanticMgr = m
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetSemanticManager(m)
}
}
// SetSkipVectorBuild controls whether per-repo Indexers constructed
// from now on skip the embedding pass in buildSearchIndex (text index
// only). The daemon enables it for the warmup loop when a snapshot
// already carries the workspace vector index, then disables it and
// restores the cached index once warmup finishes. It also re-applies
// the flag to every per-repo Indexer already constructed so a flag
// flip mid-lifecycle takes effect everywhere.
func (mi *MultiIndexer) SetSkipVectorBuild(skip bool) {
mi.mu.Lock()
mi.skipVectorBuild = skip
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetSkipVectorBuild(skip)
}
}
// SetResolverLSPHelper installs the resolve-time LSP helper used by
// every per-repo Indexer this MultiIndexer constructs from now on,
// and by the global post-pass resolver in RunDeferredPassesAll. Pass
// nil to detach. Safe to call zero or one times; subsequent calls
// silently replace and propagate to every existing per-repo indexer.
func (mi *MultiIndexer) SetResolverLSPHelper(h resolver.LSPHelper) {
mi.mu.Lock()
mi.resolverLSPHelper = h
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.Unlock()
for _, idx := range live {
idx.SetResolverLSPHelper(h)
}
}
// SetOnRepoTracked installs a callback fired once per TrackRepoCtx
// after the repo's prefix + absolute path have been resolved but
// before indexing starts. The daemon registers per-repo resolver-
// time LSP helpers from this hook so runtime-added repos
// participate in the N5 hot path. Pass nil to detach.
func (mi *MultiIndexer) SetOnRepoTracked(fn func(prefix, absPath string)) {
mi.mu.Lock()
mi.onRepoTracked = fn
mi.mu.Unlock()
}
// BeginBatch enables deferred-global-passes mode for every per-repo
// Indexer that this MultiIndexer constructs after the call AND for
// every Indexer already in mi.indexers (so ReconcileAll's per-repo
// IncrementalReindex calls also skip the O(global) walks). Pair with
// EndBatch.
func (mi *MultiIndexer) BeginBatch() {
mi.mu.Lock()
defer mi.mu.Unlock()
mi.deferGlobalPasses = true
for _, idx := range mi.indexers {
idx.SetDeferGlobalPasses(true)
}
}
// BeginParallelBatch is BeginBatch plus parallel-safety: it also
// propagates SetDeferResolve(true) to every per-repo Indexer
// constructed during the batch. Use this when running the per-repo
// indexing loop across goroutines (warmup) — the parallel parsers
// must not race each other inside ResolveAll / contract extract /
// semantic enrich, which all mutate the shared graph. Pair with
// EndBatch; call RunDeferredPassesAll between the parallel parse and
// EndBatch to run the deferred per-repo passes serially.
func (mi *MultiIndexer) BeginParallelBatch() {
mi.mu.Lock()
defer mi.mu.Unlock()
mi.deferGlobalPasses = true
mi.deferResolve = true
for _, idx := range mi.indexers {
idx.SetDeferGlobalPasses(true)
}
}
// RunDeferredPassesAll drains the deferred per-repo passes (semantic
// enrich / contract extract+commit) serially across the indexers the
// parallel parse populated. Pairs with BeginParallelBatch: the parallel
// loop parses with deferResolve on; this serial loop runs the passes that
// would otherwise race on the shared graph. The references-completeness
// resolve runs ahead of this in RunPreEnrichResolve (so the daemon can mark
// itself queryable before enrichment); the per-repo resolver pass is
// suppressed here because resolver.ResolveAll walks the entire shared graph
// — paying it R times is O(R · E). One master resolver.New(graph).ResolveAll
// runs at the end to lift the placeholder edges enrichment + contracts added.
//
// Returns the number of repos whose deferred semantic enrichment was
// actually dispatched (pendingEnrich set, or forced via
// GORTEX_WARMUP_FORCE_ENRICH) rather than skipped as unchanged. Sampled
// before runDeferredEnrichParallel runs, since a successful non-partial
// pass clears the flag it reads.
// SeedPendingEnrichAll re-arms the deferred-enrichment gate for every tracked
// repo whose persisted enrichment is known-incomplete at its current clean HEAD
// (see Indexer.MaybeSeedPendingEnrich). The daemon warmup calls it after the
// parallel parse and before RunDeferredPassesAll so a repo left partial or
// abandoned by a prior process resumes even when no file changed this run.
// Returns the number of repos that will enrich (already pending plus newly
// seeded) — the caller uses a non-zero count to run the deferred passes on a
// warm restart that changed nothing on disk.
func (mi *MultiIndexer) SeedPendingEnrichAll() int {
mi.mu.RLock()
live := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
live = append(live, idx)
}
mi.mu.RUnlock()
pending := 0
for _, idx := range live {
if idx.MaybeSeedPendingEnrich() {
pending++
}
}
return pending
}
func (mi *MultiIndexer) RunDeferredPassesAll(ctx context.Context) int {
mi.mu.RLock()
indexers := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
indexers = append(indexers, idx)
}
mi.mu.RUnlock()
forced := os.Getenv("GORTEX_WARMUP_FORCE_ENRICH") == "1"
enrichScheduled := 0
for _, idx := range indexers {
if idx.semanticMgr == nil || !idx.semanticMgr.Enabled() || !idx.semanticMgr.HasProviders() {
continue
}
if idx.pendingEnrich.Load() || forced {
enrichScheduled++
}
}
for _, idx := range indexers {
idx.SetSkipResolveInDeferred(true)
}
// Per-repo deferred work in three phases. gomod (materialises dep
// contract nodes) and contracts (extract + commit, which walk repo edges)
// mutate the shared graph in ways that race across repos, so they stay
// serial. Enrichment is the dominant cost — LSP background-indexing and
// hover I/O — and is safe to overlap across repos: the manager hands each
// repo its own LSP provider instance, go-types stashes per repo, and every
// provider serialises its graph mutations on the backend resolve mutex.
for _, idx := range indexers {
idx.runDeferredGoMod()
}
mi.runDeferredEnrichParallel(indexers)
for _, idx := range indexers {
idx.runDeferredContracts()
}
for _, idx := range indexers {
idx.SetSkipResolveInDeferred(false)
}
// Re-run the master same-repo resolver to lift the placeholder edges the
// enrichment + contract passes just added. The references-completeness
// resolve already ran ahead of enrichment in RunPreEnrichResolve, so this
// is the idempotent catch-up pass for edges minted during enrichment.
// Whole-graph (nil scope): enrichment can mint placeholder edges in any
// repo, and scoping this catch-up is left to a follow-up.
mi.runMasterResolve(nil)
return enrichScheduled
}
// runMasterResolve runs one same-repo resolver over the whole shared graph,
// lifting every placeholder edge to its canonical target. Split out so the
// pre-enrichment resolve stage (RunPreEnrichResolve) and the post-enrichment
// catch-up (RunDeferredPassesAll) share one implementation.
// scope, when non-empty, restricts the pass to the edges that could resolve
// into one of the named changed repos (see resolver.SetScope). It is honoured
// only when scoped global passes are enabled; a nil / empty scope or a
// disabled switch runs the whole-graph resolve, exactly the prior behaviour.
func (mi *MultiIndexer) runMasterResolve(scope map[string]struct{}) {
if mi.graph == nil {
return
}
master := resolver.New(mi.graph)
master.SetLogger(mi.logger)
// The master resolve is the only pass with whole-graph evidence, so it is
// the one allowed to durably flag terminally-unresolved edges (permanently
// external / stdlib / definition-less) that a later scoped warm resolve can
// skip. A scoped master pass leaves the flag alone (stamping is gated on an
// empty scope inside ResolveAll); only a full pass stamps and self-heals.
master.SetStampTerminal(true)
// Mirror the resolve-time LSP helper onto the master pass so TS/JS-family
// edges pick up LSP-precision answers just like the per-repo passes do.
if mi.resolverLSPHelper != nil {
master.SetLSPHelper(mi.resolverLSPHelper)
}
master.SetNpmAliasResolver(mi.npmAliasResolver())
master.SetPathAliasResolver(mi.pathAliasResolver())
master.SetWorkspaceMembership(mi.workspaceMembershipResolver())
scoped := len(scope) > 0 && mi.scopedGlobalPassesEnabled()
if scoped {
master.SetScope(scope)
}
mt := time.Now()
stats := master.ResolveAll()
mi.logger.Info("DEFERRED-TIMING master.ResolveAll",
zap.Duration("elapsed", time.Since(mt)),
zap.Bool("scoped", scoped),
zap.Int("scope_repos", len(scope)),
zap.Int("pending_before", stats.PendingBefore),
zap.Int("pending_after", stats.PendingAfter))
}
// RunPreEnrichResolve runs the resolution stage that makes references queryable
// ahead of the slow semantic-enrichment pass. It materialises go.mod dep
// contract nodes (so the resolver's import bridge can re-target Go imports of
// declared modules), runs the same-repo master resolver to lift every
// parse-time placeholder edge to its canonical target, then runs the cross-repo
// resolver so references that span repo boundaries resolve too. Contract-bridge
// reconciliation is intentionally left to RunGlobalResolve, which runs after the
// contract pass has committed its nodes.
//
// The daemon warmup calls this between the parallel parse and the enrichment
// phase, then marks itself ready — so find_usages / get_callers return complete
// results as soon as the graph is queryable, independent of enrichment.
// scope, when non-empty, restricts the same-repo master resolve to the
// changed repos (see runMasterResolve / resolver.SetScope). The daemon warmup
// passes the set of repos that re-indexed so a warm restart of one repo out of
// many skips a whole-graph same-repo resolve; a nil / empty scope keeps the
// whole-graph behaviour. The cross-repo resolve below stays whole-graph
// regardless — it is the only pass that binds an unchanged repo's inbound
// reference into a symbol a changed repo just added, and those inbound edges
// must be resolved before the daemon reports ready (see runCrossRepoResolve).
func (mi *MultiIndexer) RunPreEnrichResolve(ctx context.Context, scope map[string]struct{}) {
mi.mu.RLock()
indexers := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
indexers = append(indexers, idx)
}
mi.mu.RUnlock()
for _, idx := range indexers {
idx.runDeferredGoMod()
}
mi.runMasterResolve(scope)
// Cross-repo references resolve here too so a multi-repo workspace is fully
// queryable at "ready", not just within each repo. Whole-graph so inbound
// references from unchanged repos into the changed repos bind before ready.
mi.runCrossRepoResolve(false)
}
// runDeferredEnrichParallel runs each indexer's semantic enrichment in a
// bounded worker pool. Concurrency is capped so at most a few LSP servers
// background-index at once (the memory-sensitive part). The manager pins each
// repo's LSP provider in-use for the duration of its pass, so the router's
// LRU evictor never closes a provider another repo is still enriching against.
func (mi *MultiIndexer) runDeferredEnrichParallel(indexers []*Indexer) {
// Per-repo language sets computed once from a single graph-stats scan,
// shared by the spec-grouped ordering and the batch pool-raise sizing
// so the Manager's per-repo enrichment scan is not duplicated here.
langSets := mi.batchLanguageSets(indexers)
// Deterministic, spec-grouped order: repos needing the same language
// servers run contiguously so the router's capped provider pool cycles
// through far fewer distinct (spec, workspace) keys — a warmed server
// stays alive across the runs that need it instead of being evicted and
// respawned per repo.
indexers = orderIndexersBySpecGroup(indexers, langSets)
conc := enrichConcurrency(len(indexers))
// Temporarily raise the router's live-provider cap for the batch so the
// concurrent passes don't evict each other's warmed servers, restoring
// it (and logging the churn observed) when the batch drains.
restore := mi.scopeRouterPoolForBatch(langSets, conc)
defer restore()
if conc <= 1 {
for _, idx := range indexers {
idx.runDeferredEnrich()
}
return
}
sem := make(chan struct{}, conc)
var wg sync.WaitGroup
for _, idx := range indexers {
wg.Add(1)
sem <- struct{}{}
go func(idx *Indexer) {
defer func() { <-sem; wg.Done() }()
idx.runDeferredEnrich()
}(idx)
}
wg.Wait()
}
// maxBatchProviders caps the temporary live-provider pool raise during
// batch enrichment. Even a batch touching many distinct language servers
// at high concurrency is held to this ceiling so warmup cannot spawn an
// unbounded number of LSP subprocesses at once.
const maxBatchProviders = 12
// batchLanguageSets returns each indexer's sorted set of present languages,
// computed from a single RepoStats scan (one pass over the shared graph)
// rather than a per-repo node scan. The sets drive both the spec-grouped
// enrich ordering and the batch pool-raise sizing.
func (mi *MultiIndexer) batchLanguageSets(indexers []*Indexer) map[*Indexer][]string {
out := make(map[*Indexer][]string, len(indexers))
var stats map[string]graph.GraphStats
if mi.graph != nil {
stats = mi.graph.RepoStats()
}
for _, idx := range indexers {
var langs []string
if s, ok := stats[idx.repoPrefix]; ok {
for l := range s.ByLanguage {
langs = append(langs, l)
}
}
sort.Strings(langs)
out[idx] = langs
}
return out
}
// orderIndexersBySpecGroup returns a stable, deterministic ordering of
// indexers grouped by their language set (so repos needing the same LSP
// servers run contiguously), breaking ties by repo prefix. It does not
// mutate the input slice.
func orderIndexersBySpecGroup(indexers []*Indexer, langSets map[*Indexer][]string) []*Indexer {
out := make([]*Indexer, len(indexers))
copy(out, indexers)
sort.SliceStable(out, func(i, j int) bool {
ki := strings.Join(langSets[out[i]], ",")
kj := strings.Join(langSets[out[j]], ",")
if ki != kj {
return ki < kj
}
return out[i].repoPrefix < out[j].repoPrefix
})
return out
}
// distinctBatchSpecs counts the enabled, available LSP specs whose
// languages intersect any language present in the batch — i.e. how many
// distinct language servers the batch will actually drive.
func distinctBatchSpecs(langSets map[*Indexer][]string, router semantic.LSPRouter) int {
langs := make(map[string]bool)
for _, ls := range langSets {
for _, l := range ls {
langs[l] = true
}
}
if len(langs) == 0 {
return 0
}
seen := make(map[string]bool)
for _, name := range router.EnabledSpecNames() {
if !router.SpecAvailable(name) {
continue
}
for _, l := range router.SpecLanguages(name) {
if langs[l] {
seen[name] = true
break
}
}
}
return len(seen)
}
// scopeRouterPoolForBatch raises the LSP router's live-provider cap to
// enrichConcurrency × distinct-provider-specs (ceiling maxBatchProviders)
// for the duration of a batch enrichment pass, so the concurrent passes
// don't evict each other's warmed servers. It returns a restore closure
// that puts the cap back and logs the eviction churn observed during the
// batch. Safe no-op when no router is installed.
func (mi *MultiIndexer) scopeRouterPoolForBatch(langSets map[*Indexer][]string, conc int) func() {
if mi.semanticMgr == nil {
return func() {}
}
router := mi.semanticMgr.LSPRouter()
if router == nil {
return func() {}
}
before := router.EvictionCount()
old := router.MaxAlive()
raised := false
if specs := distinctBatchSpecs(langSets, router); specs > 0 {
needed := conc * specs
if needed > maxBatchProviders {
needed = maxBatchProviders
}
if needed > old {
router.SetMaxAlive(needed)
raised = true
mi.logger.Info("LSP router pool temporarily raised for batch enrichment",
zap.Int("from", old),
zap.Int("to", needed),
zap.Int("distinct_specs", specs),
zap.Int("concurrency", conc),
)
}
}
return func() {
if raised {
router.SetMaxAlive(old)
}
mi.logger.Info("batch enrichment LSP provider churn",
zap.Uint64("evictions", router.EvictionCount()-before),
zap.Bool("pool_raised", raised),
)
}
}
// enrichConcurrency caps how many repos enrich at once during batch warmup.
// Half the CPUs, clamped to [1,4] and to the repo count — a few concurrent
// LSP servers background-index in parallel without a memory blow-up.
func enrichConcurrency(repos int) int {
c := runtime.NumCPU() / 2
if c > 4 {
c = 4
}
if c < 1 {
c = 1
}
if c > repos {
c = repos
}
return c
}
// EndBatch turns off deferred-global-passes mode and runs the graph-
// wide derivation passes (InferImplements, InferOverrides,
// markTestSymbolsAndEmitEdges) once against the shared graph. Restores
// the per-Indexer flag too so a subsequent one-off TrackRepoCtx call
// runs the passes inline as expected.
// ArmBatchScope records the prefixes of the repos that re-indexed in the
// batch about to be ended, so the next RunGlobalGraphPasses runs the
// per-repo clone-detection + clone-index Rebuild passes only for those
// repos instead of for every tracked repo. An empty set, or scoped global
// passes being disabled, leaves the scope nil (run all). Only the daemon
// warmup arms it; every other EndBatch caller keeps the whole-workspace
// behaviour.
func (mi *MultiIndexer) ArmBatchScope(changedPrefixes map[string]struct{}) {
if mi == nil || len(changedPrefixes) == 0 || !mi.scopedGlobalPassesEnabled() {
return
}
mi.mu.Lock()
mi.batchChangedPrefixes = changedPrefixes
mi.mu.Unlock()
}
// takeBatchScope returns the armed clone-pass scope and clears it, so the
// scope governs exactly one RunGlobalGraphPasses run. A nil result means
// "no scope — run the clone passes for every repo".
func (mi *MultiIndexer) takeBatchScope() map[string]struct{} {
mi.mu.Lock()
scope := mi.batchChangedPrefixes
mi.batchChangedPrefixes = nil
mi.mu.Unlock()
return scope
}
// scopedGlobalPassesEnabled reports whether per-repo global passes may be
// scoped to the changed-repo set. GORTEX_INDEX_SCOPED_GLOBAL_PASSES
// overrides the config key; default ON, mirroring Indexer.scopedGlobalPassesEnabled.
func (mi *MultiIndexer) scopedGlobalPassesEnabled() bool {
if v := os.Getenv("GORTEX_INDEX_SCOPED_GLOBAL_PASSES"); v != "" {
return v == "1" || strings.EqualFold(v, "true")
}
mi.mu.RLock()
defer mi.mu.RUnlock()
for _, idx := range mi.indexers {
return idx.config.ScopedGlobalPassesEnabledOrDefault()
}
return true
}
func (mi *MultiIndexer) EndBatch() {
mi.mu.Lock()
mi.deferGlobalPasses = false
mi.deferResolve = false
for _, idx := range mi.indexers {
idx.SetDeferGlobalPasses(false)
}
mi.mu.Unlock()
mi.RunGlobalGraphPasses(context.Background())
}
// ResetBatch clears deferred-batch mode WITHOUT running the graph-wide
// derivation passes. It is the warm-restart fast-path counterpart to
// EndBatch: when the warmup reconcile loop observed zero changed files
// across every repo, the persistent backend already holds every resolved
// and derived edge from the prior run, so RunGlobalGraphPasses (plus the
// RunDeferredPassesAll / RunGlobalResolve the caller also skips) would
// only recompute what's already on disk — the work that turns a warm
// restart into a 30s500s stall. The per-Indexer SetDeferGlobalPasses
// flag is still restored so a later watch-triggered TrackRepoCtx /
// IncrementalReindex runs its passes inline as normal.
func (mi *MultiIndexer) ResetBatch() {
mi.mu.Lock()
defer mi.mu.Unlock()
mi.deferGlobalPasses = false
mi.deferResolve = false
for _, idx := range mi.indexers {
idx.SetDeferGlobalPasses(false)
}
}
// RunGlobalGraphPasses runs the graph-wide derivation passes once
// against the shared graph: InferImplements (structural interface
// satisfaction), InferOverrides (method-level overrides on
// extends/implements/composes parents), and markTestSymbolsAndEmitEdges
// (test→subject EdgeTests). Idempotent — graph.AddEdge dedupes by
// edgeKey and the resolver passes skip already-present parents.
func (mi *MultiIndexer) RunGlobalGraphPasses(ctx context.Context) {
if mi.graph == nil {
return
}
reporter := progress.FromContext(ctx)
r := resolver.New(mi.graph)
// Unconditional per-sub-pass timing. These global passes run serially and
// were previously logged only when a pass emitted edges, so a slow but
// low-yield pass left no breadcrumb — the source of the multi-minute
// "silent" span after resolve on a cold index.
globalStart := time.Now()
// Acquire the changed-repo scope once for the whole run and derive the two
// shapes the passes below consume. A nil scope means whole-graph — the
// fresh-index / one-off behaviour every pass keeps as its fallback — so an
// unscoped run is byte-identical to before. A non-nil scope (armed by the
// daemon warmup / hourly janitor via ArmBatchScope) narrows each pass to the
// repos that re-indexed this batch: an unchanged repo's derived edges are
// already on disk and were never dropped, so re-deriving them is skipped.
// - changedPrefixes: the changed-repo prefix set, for the edge-driven passes
// (capability, test edges, framework synthesis, external calls), each of
// which owns an edge iff its FROM node is in a changed repo.
// - scopedTypeIfaceIDs: the changed repos' type/interface node IDs, for the
// implements/overrides inference. The scoped inference keeps a pair when
// EITHER endpoint is in this set, so a cross-repo override whose child is
// in a changed repo and whose parent is in an unchanged one (or vice
// versa) is still re-derived; structural implements never crosses repos
// (its same-repo gate), so scoping both its sides here is complete.
scope := mi.takeBatchScope()
var changedPrefixes map[string]bool
var scopedTypeIfaceIDs map[string]bool
if scope != nil {
changedPrefixes = make(map[string]bool, len(scope))
scopedTypeIfaceIDs = map[string]bool{}
for prefix := range scope {
changedPrefixes[prefix] = true
if prefix == "" {
continue
}
for _, n := range repoNodesLightOrFull(mi.graph, prefix) {
if n == nil {
continue
}
if n.Kind == graph.KindType || n.Kind == graph.KindInterface {
scopedTypeIfaceIDs[n.ID] = true
}
}
}
}
implStart := time.Now()
implAdded := 0
switch {
case scope == nil:
implAdded = r.InferImplements()
case len(scopedTypeIfaceIDs) > 0:
// Empty set => no type/interface changed in the batch => no inferred
// implements edge could have been dropped, so the pass is skipped.
implAdded = r.InferImplementsScoped(scopedTypeIfaceIDs, scopedTypeIfaceIDs)
}
mi.logger.Info("global pass: infer implements",
zap.Int("added", implAdded),
zap.Bool("scoped", scope != nil),
zap.Duration("elapsed", time.Since(implStart)))
overStart := time.Now()
overAdded := 0
switch {
case scope == nil:
overAdded = r.InferOverrides()
case len(scopedTypeIfaceIDs) > 0:
overAdded = r.InferOverridesScoped(scopedTypeIfaceIDs)
}
mi.logger.Info("global pass: infer overrides",
zap.Int("added", overAdded),
zap.Bool("scoped", scope != nil),
zap.Duration("elapsed", time.Since(overStart)))
testStart := time.Now()
marked, emitted := markTestSymbolsAndEmitEdgesScoped(mi.graph, changedPrefixes)
mi.logger.Info("global pass: test edges",
zap.Int("test_symbols", marked),
zap.Int("edges", emitted),
zap.Duration("elapsed", time.Since(testStart)))
capStart := time.Now()
capRe, capEp, capFa := synthesizeCapabilityEdgesScoped(mi.graph, changedPrefixes)
mi.logger.Info("global pass: capability edges",
zap.Int("reads_env", capRe),
zap.Int("executes_process", capEp),
zap.Int("accesses_field", capFa),
zap.Duration("elapsed", time.Since(capStart)))
// Clone detection is PER-REPOSITORY: each tracked repo gets its own
// finalise + detect over its own nodes (scoped by RepoPrefix), so no
// cross-repo candidate pair is ever formed and each repo's boilerplate
// CMS / threshold is computed from that repo's bodies alone. This
// matches the per-repo incremental maintainer (cloneIndex.Rebuild /
// UpdateFuncs) so the batch and incremental edge sets agree.
reporter.Report("clone detection pass (global)", 0, 0)
mi.mu.RLock()
cloneIdx := make([]*Indexer, 0, len(mi.indexers))
for _, idx := range mi.indexers {
cloneIdx = append(cloneIdx, idx)
}
mi.mu.RUnlock()
// Scope to the repos that re-indexed this batch when the warmup armed a
// batch scope. An unchanged repo's clone edges are already on disk and
// its incremental clone index is reseeded lazily on its first later edit
// (indexFile: if !built → Rebuild), so skipping its full-graph detect +
// Rebuild here is sound and cuts an N-repo warm restart from N full-graph
// clone walks to just the changed repos'. A nil scope runs every repo. The
// scope is taken once at the top of RunGlobalGraphPasses and shared by every
// pass; the clone passes reuse it here (takeBatchScope must not be called
// twice — it clears the armed scope on the first read).
inCloneScope := func(prefix string) bool {
if scope == nil {
return true
}
_, ok := scope[prefix]
return ok
}
cloneDetectStart := time.Now()
clonesDetected := 0
for _, idx := range cloneIdx {
if !inCloneScope(idx.repoPrefix) {
continue
}
clonesDetected++
// Per-repo threshold, NOT a max-over-repos value: the batch must use
// the same cutoff the per-repo incremental maintainer uses
// (UpdateFuncs/Rebuild → idx.cloneThreshold()), or the batch and
// incremental edge sets diverge for any repo whose configured
// threshold differs from the workspace maximum.
if cs := detectClonesAndEmitEdgesCtx(ctx, mi.graph, idx.repoPrefix, idx.cloneThreshold()); cs.Items > 0 {
mi.logger.Info("clone edges emitted (global)",
zap.String("repo", idx.repoPrefix),
zap.Int("items", cs.Items),
zap.Int("clone_pairs", cs.Pairs),
zap.Int("edges", cs.Edges),
zap.Int("skipped_buckets", cs.SkippedBuckets),
zap.Int("skipped_bucket_items", cs.SkippedBucketItems),
zap.Int("diffused_pairs", cs.DiffusedPairs),
zap.Int("diffused_edges", cs.DiffusedEdges),
)
}
}
// Seed each per-repo indexer's incremental clone index from the
// freshly-baselined signatures + sidecar (scoped to that repo's
// prefix) so steady-state single-file edits after this batch go
// incremental instead of re-running the whole-graph pass per file.
cloneRebuildStart := time.Now()
clonesRebuilt := 0
for _, idx := range cloneIdx {
if !inCloneScope(idx.repoPrefix) {
continue
}
if idx.cloneIndex != nil {
idx.cloneIndex.Rebuild(mi.graph, idx.repoPrefix)
clonesRebuilt++
}
}
// Aggregate timing for the clone passes — previously the most expensive
// and least observable part of a warm restart (the per-repo logs above
// only fire when a repo actually has clone pairs, so a long run could
// pass with no breadcrumbs).
mi.logger.Info("clone passes done (global)",
zap.Bool("scoped", scope != nil),
zap.Int("repos_total", len(cloneIdx)),
zap.Int("detected", clonesDetected),
zap.Int("rebuilt", clonesRebuilt),
zap.Duration("detect_elapsed", cloneRebuildStart.Sub(cloneDetectStart)),
zap.Duration("rebuild_elapsed", time.Since(cloneRebuildStart)),
)
// Framework dynamic-dispatch synthesis (gRPC stubs, Temporal
// workflow→activity, in-process / native event channels, native
// bridges). After InferImplements/InferOverrides (the
// interface-satisfaction signals) and before DetectCrossRepoEdges so
// a cross-repo synthesized call gets its parallel cross_repo_calls
// edge.
reporter.Report("framework dispatch synthesis (global)", 0, 0)
fwStart := time.Now()
fwRep := resolver.RunFrameworkSynthesizersScoped(mi.graph, changedPrefixes)
mi.logger.Info("global pass: framework dispatch synthesis",
zap.Int("edges", fwRep.Total),
zap.Any("per_synthesizer", fwRep.Per),
zap.Int64("gate_ms", fwRep.GateMillis),
zap.Int64("claim_ms", fwRep.ClaimMillis),
zap.Int64("demote_ms", fwRep.DemoteMillis),
zap.Duration("elapsed", time.Since(fwStart)))
// External-call placeholder synthesis (opt-in). Runs after the
// stub passes so only genuinely un-indexed external targets are
// left to materialise into call-chain terminals.
reporter.Report("external-call synthesis (global)", 0, 0)
extStart := time.Now()
extEnabled := mi.externalCallSynthesisEnabled()
extCalls := 0
if scope != nil {
extCalls = resolver.SynthesizeExternalCallsForRepos(mi.graph, extEnabled, changedPrefixes)
} else {
extCalls = resolver.SynthesizeExternalCalls(mi.graph, extEnabled)
}
mi.logger.Info("global pass: external-call synthesis",
zap.Int("edges", extCalls),
zap.Bool("scoped", scope != nil),
zap.Duration("elapsed", time.Since(extStart)))
// Cross-repo edge layer. Runs after InferImplements / InferOverrides
// so the implements / extends edges they materialise across repo
// boundaries pick up their parallel cross_repo_* edges.
reporter.Report("cross-repo edges (global)", 0, 0)
crStart := time.Now()
crossRepoEdges := resolver.DetectCrossRepoEdges(mi.graph)
mi.logger.Info("global pass: cross-repo edges",
zap.Int("edges", crossRepoEdges),
zap.Duration("elapsed", time.Since(crStart)))
mi.logger.Info("global passes complete",
zap.Duration("total", time.Since(globalStart)))
}
// repoNodesLightOrFull returns a repo's nodes for read-only structural
// inspection (id / kind / repo prefix), preferring the meta-less
// LightNodeReader fast path when the backend implements it so the enriched meta
// blob a scope build never reads stays server-side. The returned nodes MUST NOT
// be written back through AddNode/AddBatch — the light projection drops any
// non-promoted meta — which holds here because the only caller reads struct
// fields to build an ID set and discards the nodes. Falls back to the full
// GetRepoNodes when the backend (e.g. in-memory) has no separate blob to skip.
func repoNodesLightOrFull(g graph.Store, prefix string) []*graph.Node {
if lr, ok := g.(graph.LightNodeReader); ok {
return lr.GetRepoNodesLight(prefix)
}
return g.GetRepoNodes(prefix)
}
// externalCallSynthesisEnabled resolves whether external-call placeholder
// synthesis should run over the shared graph. The pass is graph-wide, so
// it is enabled when any tracked repo opted in — a repo that wants the
// external hops in its call chains gets them even when a sibling repo
// left the option off.
func (mi *MultiIndexer) externalCallSynthesisEnabled() bool {
mi.mu.RLock()
defer mi.mu.RUnlock()
for _, idx := range mi.indexers {
if idx.externalCallSynthesisEnabled() {
return true
}
}
return false
}
// NewMultiIndexer creates a MultiIndexer.
func NewMultiIndexer(
g graph.Store,
reg *parser.Registry,
s search.Backend,
cm *config.ConfigManager,
logger *zap.Logger,
) *MultiIndexer {
return &MultiIndexer{
graph: g,
registry: reg,
search: s,
repos: make(map[string]*RepoMetadata),
indexers: make(map[string]*Indexer),
configMgr: cm,
logger: logger,
}
}
// IndexAll indexes all active repos concurrently. Each repo gets its own
// Indexer instance with repo-specific config. Returns per-repo IndexResults.
func (mi *MultiIndexer) IndexAll() (map[string]*IndexResult, error) {
return mi.IndexScoped("", "")
}
// IndexScoped is IndexAll restricted to repos whose workspace and
// project slugs match. Empty filters disable that axis (so empty/empty
// is equivalent to IndexAll). Resolution honours the same precedence
// as resolveWorkspaceID/resolveProjectID — RepoEntry override →
// `.gortex.yaml::workspace` → repo prefix — so a `gortex server
// --workspace foo` invocation matches both repos that declare
// `workspace: foo` in their own `.gortex.yaml` and repos pinned to
// `foo` from the user's global config.
//
// Returns an error when the filters exclude every active repo, so a
// `--workspace typo` surfaces as a startup failure rather than a
// silently empty graph.
func (mi *MultiIndexer) IndexScoped(workspaceSlug, projectSlug string) (map[string]*IndexResult, error) {
repos := mi.configMgr.ActiveRepos()
if len(repos) == 0 {
return nil, nil
}
if workspaceSlug != "" || projectSlug != "" {
filtered := mi.filterReposByScope(repos, workspaceSlug, projectSlug)
if len(filtered) == 0 {
return nil, fmt.Errorf("scope filter matched zero of %d active repos (workspace=%q project=%q)", len(repos), workspaceSlug, projectSlug)
}
repos = filtered
}
// Single-repo mode: delegate without prefixing.
if len(repos) == 1 {
r, err := mi.indexSingleRepo(repos[0])
if err == nil {
mi.ReconcileContractEdges()
}
return r, err
}
r, err := mi.indexMultiRepo(repos)
if err == nil {
mi.ReconcileContractEdges()
}
return r, err
}
// filterReposByScope returns the subset of repos whose resolved
// workspace and project slugs match the supplied filters. Empty
// filters disable that axis. Loads each repo's `.gortex.yaml` first so
// resolution sees the workspace/project declared there — matching only
// against `RepoEntry.Workspace` would miss repos that declare their
// slug in their own config file (the typical case for first-party
// repos).
func (mi *MultiIndexer) filterReposByScope(repos []config.RepoEntry, workspaceSlug, projectSlug string) []config.RepoEntry {
if workspaceSlug == "" && projectSlug == "" {
return repos
}
out := make([]config.RepoEntry, 0, len(repos))
for _, e := range repos {
absPath, err := filepath.Abs(e.Path)
if err != nil {
continue
}
prefix := config.ResolvePrefix(e)
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
cfg := mi.configMgr.GetRepoConfig(prefix)
entryCopy := e
if workspaceSlug != "" && resolveWorkspaceID(&entryCopy, cfg, prefix) != workspaceSlug {
continue
}
if projectSlug != "" && resolveProjectID(&entryCopy, cfg, prefix) != projectSlug {
continue
}
out = append(out, e)
}
return out
}
// indexSingleRepo indexes a single repo without prefixing for backward compatibility.
func (mi *MultiIndexer) indexSingleRepo(entry config.RepoEntry) (map[string]*IndexResult, error) {
absPath, err := filepath.Abs(entry.Path)
if err != nil {
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
}
prefix := config.ResolvePrefix(entry)
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
cfg := mi.configMgr.GetRepoConfig(prefix)
idx := mi.newPerRepoIndexer(cfg.Index)
entryCopy := entry
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
// No repo prefix in single-repo mode.
result, err := idx.Index(absPath)
if err != nil {
return nil, fmt.Errorf("indexing %s: %w", absPath, err)
}
result.RepoPrefix = prefix
identity, _ := DetectIdentity(absPath)
mi.mu.Lock()
mi.repos[prefix] = &RepoMetadata{
RepoPrefix: prefix,
RootPath: absPath,
Identity: identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
IsWorktree: ResolveWorktree(absPath).IsWorktree,
Unprefixed: true,
}
mi.indexers[prefix] = idx
mi.mu.Unlock()
return map[string]*IndexResult{prefix: result}, nil
}
// migrateLoneUnprefixedRepoCtx re-mints the formerly-lone repo's nodes
// with its real prefix the moment a second repo joins. Without it, the
// first repo's unprefixed nodes become unreachable (the empty-prefix
// fallback disarms at two repos) until a cold reindex. Ordering is
// crash-safe: the prefixed re-index lands first, the stale unprefixed
// nodes are evicted after — an interruption leaves both ID forms
// resolvable rather than neither.
func (mi *MultiIndexer) migrateLoneUnprefixedRepoCtx(ctx context.Context) {
mi.mu.RLock()
var oldPrefix string
var oldMeta *RepoMetadata
if len(mi.repos) == 1 {
for p, m := range mi.repos {
if m != nil && m.Unprefixed && m.RootPath != "" {
oldPrefix, oldMeta = p, m
}
}
}
mi.mu.RUnlock()
if oldMeta == nil {
return
}
cfg := mi.configMgr.GetRepoConfig(oldPrefix)
idx := mi.newPerRepoIndexer(cfg.Index)
idx.SetRepoPrefix(oldPrefix)
entry := config.RepoEntry{Path: oldMeta.RootPath, Name: oldPrefix}
if mi.configMgr != nil {
for _, e := range mi.configMgr.Global().Repos {
if pathkey.EqualPaths(e.Path, oldMeta.RootPath) {
entry = e
break
}
}
}
idx.SetWorkspaceID(resolveWorkspaceID(&entry, cfg, oldPrefix))
idx.SetProjectID(resolveProjectID(&entry, cfg, oldPrefix))
result, err := idx.IndexCtx(ctx, oldMeta.RootPath)
if err != nil {
mi.logger.Warn("re-prefixing lone repo failed; its unprefixed nodes stay until a reindex",
zap.String("prefix", oldPrefix), zap.Error(err))
return
}
// The prefixed nodes are live; now drop the unprefixed originals.
// They are invisible to EvictRepo (no byRepo bucket entry), so evict
// per file — unprefixed paths cannot collide with prefixed ones.
for path := range oldMeta.FileMtimes {
mi.graph.EvictFile(path)
}
// EvictFile clears only nodes+edges; the solo repo's sidecar rows
// (file_mtimes, repo_index_state, enrichment_state, ...) were written
// under the empty prefix and would otherwise be orphaned — the very next
// warm restart would look for mtimes under the new prefix, find zero, and
// full-re-track a repo that never changed. Re-key the '' sidecar residue
// onto the new prefix. The re-mint re-index above already wrote fresh
// new-prefix rows; RekeyRepoPrefix folds the prefix/path-keyed ones and
// drops the node_id-keyed ones (whose ids changed under the re-mint) —
// see its per-table rationale. Safe on '': the store is still single-repo
// here, so '' holds only this repo's data (the synthetic global externals
// a multi-repo graph parks under '' live in NODES, which the rekey — a
// sidecar-only operation — never touches).
if rk, ok := mi.graph.(interface {
RekeyRepoPrefix(oldPrefix, newPrefix string) error
}); ok {
if err := rk.RekeyRepoPrefix("", oldPrefix); err != nil {
mi.logger.Warn("re-keying lone repo sidecar rows failed; orphaned '' rows remain until purge",
zap.String("prefix", oldPrefix), zap.Error(err))
}
}
mi.mu.Lock()
mi.repos[oldPrefix] = &RepoMetadata{
RepoPrefix: oldPrefix,
RootPath: oldMeta.RootPath,
Identity: oldMeta.Identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
IsWorktree: oldMeta.IsWorktree,
}
mi.indexers[oldPrefix] = idx
mi.mu.Unlock()
mi.logger.Info("re-minted lone repo with its prefix for multi-repo mode",
zap.String("prefix", oldPrefix), zap.Int("nodes", result.NodeCount))
}
// readGoModModule reads the module path from a go.mod file.
func readGoModModule(repoPath string) string {
data, err := os.ReadFile(filepath.Join(repoPath, "go.mod"))
if err != nil {
return ""
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "module ") {
return strings.TrimSpace(strings.TrimPrefix(line, "module"))
}
}
return ""
}
// indexMultiRepo indexes multiple repos concurrently with repo prefixes.
func (mi *MultiIndexer) indexMultiRepo(repos []config.RepoEntry) (map[string]*IndexResult, error) {
type repoResult struct {
prefix string
result *IndexResult
idx *Indexer
meta *RepoMetadata
err error
}
// Resolve each repo's prefix serially first: git-worktree instancing,
// the derived-name persistence, and the cross-batch collision guard
// must be deterministic, not raced across the parallel index
// goroutines below. The same loop builds the tracked-module map used
// for cross-repo dependency detection.
type resolvedRepo struct {
entry config.RepoEntry
prefix string
cfg *config.Config
absPath string
identity *RepoIdentity
}
trackedModules := make(map[string]string)
resolved := make([]resolvedRepo, 0, len(repos))
seenPrefix := make(map[string]string, len(repos)) // prefix → absPath
var resolveErrors []string
for _, entry := range repos {
absPath, err := filepath.Abs(entry.Path)
if err != nil {
resolveErrors = append(resolveErrors, fmt.Sprintf("resolving path %s: %v", entry.Path, err))
continue
}
identity, _ := DetectIdentity(absPath)
e := entry
prefix, cfg := mi.resolveTrackPrefix(&e, absPath, identity)
// Two distinct checkouts can still land on the same derived prefix
// (e.g. two worktrees declaring the same workspace). resolveTrackPrefix
// only sees already-tracked repos, so guard against collisions within
// this batch too.
if prev, ok := seenPrefix[prefix]; ok && prev != absPath {
prefix += "-" + shortPathHash(absPath)
e.Name = prefix
}
seenPrefix[prefix] = absPath
resolved = append(resolved, resolvedRepo{entry: e, prefix: prefix, cfg: cfg, absPath: absPath, identity: identity})
if mod := readGoModModule(absPath); mod != "" {
trackedModules[prefix] = mod
mi.logger.Debug("tracked repo module", zap.String("repo", prefix), zap.String("module", mod))
}
}
resultCh := make(chan repoResult, len(resolved))
var wg sync.WaitGroup
for _, rr := range resolved {
wg.Add(1)
go func(r resolvedRepo) {
defer wg.Done()
idx := mi.newPerRepoIndexer(r.cfg.Index)
idx.SetRepoPrefix(r.prefix)
entryCopy := r.entry
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, r.cfg, r.prefix))
idx.SetProjectID(resolveProjectID(&entryCopy, r.cfg, r.prefix))
idx.SetTrackedRepoModules(trackedModules)
// Defer the per-repo cross-cutting passes (ResolveAll,
// semantic enrich, contract extract+commit) so they don't
// race against each other across goroutines on the shared
// graph. They run serially below via RunDeferredPasses after
// wg.Wait(). The graph-wide derivation passes run once after
// the loop via mi.RunGlobalGraphPasses().
idx.SetDeferResolve(true)
result, err := idx.Index(r.absPath)
if err != nil {
resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s: %w", r.absPath, err)}
return
}
result.RepoPrefix = r.prefix
meta := &RepoMetadata{
RepoPrefix: r.prefix,
RootPath: r.absPath,
Identity: r.identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
IsWorktree: ResolveWorktree(r.absPath).IsWorktree,
}
resultCh <- repoResult{prefix: r.prefix, result: result, idx: idx, meta: meta}
}(rr)
}
go func() {
wg.Wait()
close(resultCh)
}()
results := make(map[string]*IndexResult)
indexErrors := resolveErrors
mi.mu.Lock()
for rr := range resultCh {
if rr.err != nil {
mi.logger.Error("failed to index repo", zap.String("prefix", rr.prefix), zap.Error(rr.err))
indexErrors = append(indexErrors, rr.err.Error())
continue
}
mi.repos[rr.prefix] = rr.meta
mi.indexers[rr.prefix] = rr.idx
results[rr.prefix] = rr.result
}
mi.mu.Unlock()
// Run the per-repo passes that the goroutines above deferred. Serial
// across repos is the simple correctness fix: ResolveAll mutates
// Edge.Meta on edges in the shared graph, and the contract pass walks
// every edge — running them in parallel across repos races. Inside a
// single repo's ResolveAll the resolver still uses its own worker
// pool, and parsing (the dominant cost on a fresh index) already ran
// in parallel above, so the wall-time hit is small.
deferCtx := context.Background()
for prefix := range results {
if idx, ok := mi.indexers[prefix]; ok {
idx.RunDeferredPasses(deferCtx)
}
}
// Run a global cross-repo resolution pass once every repo is
// indexed, with the cross-workspace boundary check wired in.
// Without this, repos that import each other have unresolved
// edges that only resolve when an editor touches a file (the
// watcher path is the only other place this resolver runs). The
// boundary lookup means cross-workspace candidates only resolve
// when an explicit `cross_workspace_deps` declaration covers
// them.
cr := resolver.NewCrossRepo(mi.graph)
cr.SetCrossWorkspaceDepLookup(mi.crossWorkspaceLookup())
cr.SetNpmAliasResolver(mi.npmAliasResolver())
cr.SetPathAliasResolver(mi.pathAliasResolver())
cr.SetWorkspaceMembership(mi.workspaceMembershipResolver())
mi.applyRemoteStitch(cr)
cr.ResolveAll()
mi.ReconcileContractEdges()
// Graph-wide derivation passes run exactly once after every repo
// has been parsed, every per-repo and cross-repo resolver has lifted
// placeholder edges, and contract bridges are in place. RunDeferredPasses
// intentionally skips these so we don't pay an O(global) walk per
// repo (was the dominant cost at R≈100+).
mi.RunGlobalGraphPasses(context.Background())
if len(indexErrors) > 0 && len(results) == 0 {
return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(indexErrors, "; "))
}
return results, nil
}
// IndexRepo re-indexes a single repo by prefix. Evicts existing data first.
func (mi *MultiIndexer) IndexRepo(repoPrefix string) (*IndexResult, error) {
mi.mu.RLock()
meta, ok := mi.repos[repoPrefix]
mi.mu.RUnlock()
if !ok {
return nil, fmt.Errorf("repository not found: %s", repoPrefix)
}
// Evict existing data for this repo before re-indexing. Always — a lone
// repo is now stored prefixed (see SetRepoPrefix below), so the eviction
// must clear the prefixed slice regardless of repo count.
mi.graph.EvictRepo(repoPrefix)
mi.configMgr.LoadWorkspaceConfig(repoPrefix, meta.RootPath)
cfg := mi.configMgr.GetRepoConfig(repoPrefix)
idx := mi.newPerRepoIndexer(cfg.Index)
// Always stamp the repo prefix, even when this is the only tracked repo.
// The multi-repo cold path (indexMultiRepo) already prefixes
// unconditionally; gating the single-repo re-index on repo count left the
// two paths producing different id / path shapes for the same repo, so
// adding a second repo turned the first from bare into prefixed — a lossy
// whole-graph rewrite. Uniform prefixing removes the single/multi data-shape
// split: a repo is prefixed from its first index, and adding another is
// purely additive.
idx.SetRepoPrefix(repoPrefix)
entry := mi.configMgr.Global().FindRepoByPrefix(repoPrefix)
idx.SetWorkspaceID(resolveWorkspaceID(entry, cfg, repoPrefix))
idx.SetProjectID(resolveProjectID(entry, cfg, repoPrefix))
result, err := idx.Index(meta.RootPath)
if err != nil {
return nil, fmt.Errorf("indexing %s: %w", meta.RootPath, err)
}
mi.mu.Lock()
mi.repos[repoPrefix] = &RepoMetadata{
RepoPrefix: repoPrefix,
RootPath: meta.RootPath,
Identity: meta.Identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
}
mi.indexers[repoPrefix] = idx
mi.mu.Unlock()
// TODO: After re-indexing, run CrossRepoResolver.ResolveForRepo(repoPrefix)
// to update cross-repo edges. This will be implemented in Task 7.1.
mi.ReconcileContractEdges()
return result, nil
}
// IncrementalReindexRepo incrementally re-indexes a single tracked repo
// by prefix: only the files that changed since the last pass are
// re-parsed and deleted files are evicted, against the repo's existing
// per-repo Indexer (so its mtime snapshot is preserved). Unlike
// IndexRepo it does NOT evict the whole repo first.
//
// When paths is non-empty the pass is scoped to those files /
// directories; otherwise the whole repo root is scanned. Returns an
// error when the prefix is not a tracked repo.
func (mi *MultiIndexer) IncrementalReindexRepo(repoPrefix string, paths []string) (*IndexResult, error) {
mi.mu.RLock()
meta, ok := mi.repos[repoPrefix]
idx := mi.indexers[repoPrefix]
mi.mu.RUnlock()
if !ok || meta == nil {
return nil, fmt.Errorf("repository not found: %s", repoPrefix)
}
if idx == nil {
// Tracked but no live indexer (e.g. restored from snapshot
// without one) — fall back to a full re-index, which rebuilds
// the per-repo indexer from scratch.
return mi.IndexRepo(repoPrefix)
}
result, err := idx.IncrementalReindexPaths(meta.RootPath, paths)
if err != nil {
return nil, fmt.Errorf("reindexing %s: %w", meta.RootPath, err)
}
mi.mu.Lock()
mi.repos[repoPrefix] = &RepoMetadata{
RepoPrefix: repoPrefix,
RootPath: meta.RootPath,
Identity: meta.Identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
// Carried over from the prior metadata: this pass doesn't
// change either property, and dropping them here (both zero
// value on a fresh struct literal) used to silently flip a
// worktree or an Unprefixed solo repo back to their false
// defaults on the very first watcher-triggered incremental
// update, defeating callers that key behaviour off them (see
// the Unprefixed branch in cmd/gortex daemon status).
IsWorktree: meta.IsWorktree,
Unprefixed: meta.Unprefixed,
}
mi.mu.Unlock()
mi.ReconcileContractEdges()
return result, nil
}
// TrackRepo validates the path, detects identity, indexes, and adds to config.
func (mi *MultiIndexer) TrackRepo(entry config.RepoEntry) (*IndexResult, error) {
return mi.TrackRepoCtx(context.Background(), entry)
}
// resolveTrackPrefix determines the repo prefix that absPath should be
// registered under and loads the per-repo `.gortex.yaml` config keyed to
// that prefix. It is the single place that decides whether a git
// worktree of an already-tracked repo becomes an INDEPENDENT instance
// (see WorktreeInstanceName); the track and reconcile paths both call it
// before their "already tracked?" check so a worktree that joins a
// different workspace than the canonical no longer silently coalesces
// into it.
//
// When a separate instance is created the derived `<base>@<tag>` prefix
// is written back into entry.Name so it is persisted to config and
// reproduced deterministically on the next daemon warmup. entry is
// mutated in place; callers pass a pointer to the value they will add to
// the global config.
func (mi *MultiIndexer) resolveTrackPrefix(entry *config.RepoEntry, absPath string, identity *RepoIdentity) (string, *config.Config) {
base := config.ResolvePrefix(*entry)
if base == "" || base == "." {
if identity != nil {
base = identity.RepoPrefix
}
}
if mi.configMgr == nil {
return base, config.Default()
}
// Load the repo's `.gortex.yaml` under the base prefix first so we can
// read its declared workspace before deciding the final prefix.
mi.configMgr.LoadWorkspaceConfig(base, absPath)
cfg := mi.configMgr.GetRepoConfig(base)
// An explicit Name already pins the prefix — honour it verbatim. This
// is also the warm-restart fast path: once a worktree instance has
// been persisted with its derived Name, every later load short-circuits
// here without re-deriving.
if entry.Name != "" {
return base, cfg
}
declaredWS := entry.Workspace
if declaredWS == "" && cfg != nil {
declaredWS = cfg.Workspace
}
name, separate := WorktreeInstanceName(absPath, base, declaredWS, entry.AsWorktree)
if !separate {
return base, cfg
}
// Guard against two different checkouts colliding on the same derived
// name (e.g. two worktrees that both declare workspace `x`): keep the
// first, suffix the rest with a path hash.
prefix := name
mi.mu.RLock()
existing, ok := mi.repos[prefix]
mi.mu.RUnlock()
if ok && existing != nil && !pathkey.SamePathIdentity(existing.RootPath, absPath) {
prefix = name + "-" + shortPathHash(absPath)
}
entry.Name = prefix // persist the decision so warmup reproduces it
mi.configMgr.LoadWorkspaceConfig(prefix, absPath)
return prefix, mi.configMgr.GetRepoConfig(prefix)
}
// EffectiveRepoPrefix returns the prefix a repo entry is tracked under,
// accounting for git-worktree instancing — the same value
// resolveTrackPrefix registers, minus the (rare) collision-guard suffix
// it cannot reproduce without the live registry. Warm-restart keying
// (the snapshot-store mtime lookup, the resolve-time LSP helper
// registry) uses this instead of config.ResolvePrefix so a disk-backed
// reconcile finds the worktree instance's own persisted state rather
// than the canonical checkout's. For a plain repo it equals
// config.ResolvePrefix(entry). cm may be nil (then only an explicit
// RepoEntry.Workspace override can trigger instancing).
func EffectiveRepoPrefix(cm *config.ConfigManager, entry config.RepoEntry) string {
base := config.ResolvePrefix(entry)
if base == "" || base == "." || entry.Name != "" {
return base
}
absPath, err := filepath.Abs(entry.Path)
if err != nil {
return base
}
declaredWS := entry.Workspace
if declaredWS == "" && cm != nil {
cm.LoadWorkspaceConfig(base, absPath)
if cfg := cm.GetRepoConfig(base); cfg != nil {
declaredWS = cfg.Workspace
}
}
name, _ := WorktreeInstanceName(absPath, base, declaredWS, entry.AsWorktree)
return name
}
// foldDistinctRepoCount counts configured repos by folded path identity,
// so two entries that name the same directory under different spellings
// (case or Unicode normalisation on a case-insensitive filesystem) count
// once. It backstops the willBeMultiRepo decision: startup healing already
// prunes such duplicates, but a not-yet-pruned config must not be allowed
// to flip the graph into prefixed-ID mode for what is really one repo.
func foldDistinctRepoCount(repos []config.RepoEntry) int {
n := 0
seen := make([]string, 0, len(repos))
for _, e := range repos {
abs, err := filepath.Abs(e.Path)
if err != nil {
abs = e.Path
}
dup := false
for _, s := range seen {
if pathkey.EqualPaths(s, abs) {
dup = true
break
}
}
if dup {
continue
}
seen = append(seen, abs)
n++
}
return n
}
// TrackRepoCtx is TrackRepo with a context, allowing callers to pipe progress
// reporters (via progress.WithReporter) through to the underlying Index call.
func (mi *MultiIndexer) TrackRepoCtx(ctx context.Context, entry config.RepoEntry) (*IndexResult, error) {
absPath, err := filepath.Abs(entry.Path)
if err != nil {
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
}
// Normalise the volume (upper-case a Windows drive letter) so a newly
// tracked path converges with os.Getwd's convention. The volume is
// never part of a repo basename, so this cannot rotate a repo prefix;
// no-op on POSIX.
absPath = pathkey.NormalizeVolume(absPath)
// Validate path exists and is a directory.
info, err := os.Stat(absPath)
if err != nil {
return nil, fmt.Errorf("path does not exist: %s", absPath)
}
if !info.IsDir() {
return nil, fmt.Errorf("path is not a directory: %s", absPath)
}
if reason, blocked := unsafeRootBlocked(absPath, entry.Force); blocked {
return nil, fmt.Errorf("%s; pass force to track it anyway", reason)
}
identity, err := DetectIdentity(absPath)
if err != nil {
return nil, fmt.Errorf("detecting identity for %s: %w", absPath, err)
}
// Resolve the prefix (honouring git-worktree instancing) and load the
// per-repo `.gortex.yaml` BEFORE the already-tracked check, so a
// worktree that joins a different workspace than the canonical gets
// its own `<base>@<tag>` prefix instead of coalescing into it. cfg is
// keyed to the final prefix; entry.Name is set when a separate
// instance is created so the decision persists to config. Loading the
// `.gortex.yaml` here also gives GetRepoConfig the workspace / project
// slugs declared inside the repo (without it every repo would fall
// back to `workspace = repoPrefix`, making shared-workspace cross-repo
// matching impossible to express).
prefix, cfg := mi.resolveTrackPrefix(&entry, absPath, identity)
// Check if already tracked. The prefix-keyed check catches the common
// case; the path scan additionally catches a case-only or Unicode
// spelling variant of an already-tracked directory on a
// case-insensitive filesystem (which resolves to a different derived
// prefix and would otherwise be minted as a bogus second instance —
// the very thing that flips the daemon into multi-repo mode, #270).
mi.mu.RLock()
if _, exists := mi.repos[prefix]; exists {
mi.mu.RUnlock()
return nil, nil // already tracked
}
for _, meta := range mi.repos {
if meta != nil && pathkey.SamePathIdentity(meta.RootPath, absPath) {
mi.mu.RUnlock()
return nil, nil // same directory, different path spelling
}
}
hook := mi.onRepoTracked
mi.mu.RUnlock()
if hook != nil {
hook(prefix, absPath)
}
// Determine if we need to prefix. We must consider both repos already
// indexed in mi.repos AND the total repos configured — at daemon warmup
// TrackRepoCtx is called in a loop over all configured repos, so at
// iteration 0 mi.repos is empty while the config already has N entries.
// Counting only mi.repos used to leave the first-indexed repo without a
// prefix while every later repo got one, producing two ID schemes for
// the same graph and halving cross-file edge density.
totalConfigured := 1 // ourselves
if mi.configMgr != nil {
totalConfigured = foldDistinctRepoCount(mi.configMgr.Global().Repos)
}
willBeMultiRepo := len(mi.repos)+1 >= 2 || totalConfigured >= 2
// A second repo joining a live single-repo daemon flips the graph
// into prefixed-ID mode, but the first repo's nodes were minted
// unprefixed — re-mint them before they become unreachable.
if willBeMultiRepo {
mi.migrateLoneUnprefixedRepoCtx(ctx)
}
idx := mi.newPerRepoIndexer(cfg.Index)
if willBeMultiRepo {
idx.SetRepoPrefix(prefix)
}
// Workspace / project slugs stamped on every node. Resolution
// order (highest priority first): RepoEntry.Workspace from the
// global config (lets users pin OSS repos without committing a
// `.gortex.yaml`) → `.gortex.yaml::workspace` → repoPrefix
// (default). resolveWorkspaceID encodes the precedence; the
// WorkspaceID-keyed contract registry and the boundary-enforced
// matcher both consume the result.
entryCopy := entry
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
result, err := idx.IndexCtx(ctx, absPath)
if err != nil {
return nil, fmt.Errorf("indexing %s: %w", absPath, err)
}
result.RepoPrefix = prefix
mi.mu.Lock()
mi.repos[prefix] = &RepoMetadata{
RepoPrefix: prefix,
RootPath: absPath,
Identity: identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
IsWorktree: ResolveWorktree(absPath).IsWorktree,
Unprefixed: !willBeMultiRepo,
}
mi.indexers[prefix] = idx
mi.mu.Unlock()
// Add to global config.
entry.Path = absPath
if err := mi.configMgr.Global().AddRepo(entry); err != nil {
mi.logger.Warn("failed to add repo to config", zap.Error(err))
}
// Skip the per-repo contract reconcile when batching: it walks every
// edge in the shared graph to evict stale EdgeMatches and rebuilds
// the matcher across every indexer, so paying it once per repo on a
// warmup over 100+ repos is O(R · E). The batch caller runs it once
// after the loop (RunGlobalResolve does, and the janitor's ReconcileAll
// fires it post-loop too).
if !mi.deferGlobalPasses {
mi.ReconcileContractEdges()
}
return result, nil
}
// ReconcileRepoCtx registers a repo that already has nodes in the graph
// (typically restored from a daemon snapshot) and brings it back into
// agreement with the filesystem without a full re-index. priorMtimes
// carries the mtimes recorded at the time the snapshot was taken;
// IncrementalReindex uses them to detect files that changed offline
// (re-indexes) and files that were deleted offline (evicts).
//
// Falls back to TrackRepoCtx when the repo is not yet tracked AND no
// prior mtimes are available — in that case there's nothing to
// reconcile against and a full index is the correct path.
func (mi *MultiIndexer) ReconcileRepoCtx(ctx context.Context, entry config.RepoEntry, priorMtimes map[string]int64) (*IndexResult, error) {
start := time.Now()
absPath, err := filepath.Abs(entry.Path)
if err != nil {
return nil, fmt.Errorf("resolving path %s: %w", entry.Path, err)
}
// Normalise the volume (upper-case a Windows drive letter) to converge
// with os.Getwd's convention. Cosmetic; never touches the basename.
absPath = pathkey.NormalizeVolume(absPath)
info, err := os.Stat(absPath)
if err != nil {
return nil, fmt.Errorf("path does not exist: %s", absPath)
}
if !info.IsDir() {
return nil, fmt.Errorf("path is not a directory: %s", absPath)
}
identity, err := DetectIdentity(absPath)
if err != nil {
return nil, fmt.Errorf("detecting identity for %s: %w", absPath, err)
}
// Resolve the prefix (honouring git-worktree instancing) and load the
// per-repo `.gortex.yaml` before the already-tracked check. Mirrors
// TrackRepoCtx so a worktree instance keeps its derived prefix on the
// reconcile path too; config can change between sessions and warmup-
// time reconcile runs after a daemon restart.
prefix, cfg := mi.resolveTrackPrefix(&entry, absPath, identity)
// Already tracked — nothing to do.
mi.mu.RLock()
_, exists := mi.repos[prefix]
mi.mu.RUnlock()
if exists {
return nil, nil
}
// Fall back to full TrackRepoCtx when we have no prior mtimes:
// there's nothing meaningful to reconcile against, and
// IncrementalReindex would treat every file as stale, producing
// the same duplicate-writes problem we're fixing. entry.Name is
// already pinned by resolveTrackPrefix, so the fallback reproduces
// the same prefix.
if len(priorMtimes) == 0 {
return mi.TrackRepoCtx(ctx, entry)
}
totalConfigured := 1
if mi.configMgr != nil {
totalConfigured = foldDistinctRepoCount(mi.configMgr.Global().Repos)
}
willBeMultiRepo := len(mi.repos)+1 >= 2 || totalConfigured >= 2
// Same transition guard as TrackRepoCtx: an already-reconciled
// lone repo with unprefixed nodes must be re-minted before this
// second repo flips the graph into prefixed-ID mode.
if willBeMultiRepo {
mi.migrateLoneUnprefixedRepoCtx(ctx)
}
idx := mi.newPerRepoIndexer(cfg.Index)
if willBeMultiRepo {
idx.SetRepoPrefix(prefix)
}
entryCopy := entry
idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, cfg, prefix))
idx.SetProjectID(resolveProjectID(&entryCopy, cfg, prefix))
idx.SetRootPath(absPath)
idx.SetFileMtimes(priorMtimes)
// Choose the reconcile strategy from a census of what changed on disk
// while the daemon was down. Scoped incremental is the default: re-index
// only the changed files and evict only the deleted ones, leaving the
// rest of the already-persisted graph untouched. A whole-repo re-track
// (IndexCtx — which evicts and re-parses every file, then bulk-drains)
// is reserved for the cases where scoping is unsafe or not worth it: the
// census could not be taken, the churn is a large fraction of the repo,
// or an operator forced it via GORTEX_WARMUP_FULL_RETRACK. A repo with
// zero changes keeps the fast IncrementalReindex no-op (walk + 0 stale →
// return), which is what makes an unchanged warm restart near-instant.
//
// The in-memory backend (*graph.Graph) keeps its exact prior behaviour:
// IncrementalReindex is the authoritative path there — it evicts
// offline-deleted files in place, has no reopened disk store, and so no
// per-edge write to route around. Gate on the store type.
_, memoryBacked := mi.graph.(*graph.Graph)
var (
result *IndexResult
changed, deleted []string
route = "incremental"
)
// fullRetrack is the whole-repo re-track — the ONE place FullRetrack is
// stamped. StaleFileCount keeps its honest incremental-work meaning (0
// here) because the changed-file set is not enumerated on this path, so
// callers must key "did this repo change" off FullRetrack instead.
fullRetrack := func() (*IndexResult, error) {
r, e := idx.IndexCtx(ctx, absPath)
if e == nil && r != nil {
r.FullRetrack = true
}
return r, e
}
switch {
case memoryBacked:
result, err = idx.IncrementalReindex(absPath)
default:
var censusErr error
changed, deleted, censusErr = idx.ChangedSinceMtimes(absPath)
churn := len(changed) + len(deleted)
priorCount := len(priorMtimes)
forceFull := os.Getenv("GORTEX_WARMUP_FULL_RETRACK") == "1"
switch {
case censusErr != nil || forceFull:
route = "full_retrack"
result, err = fullRetrack()
case churn == 0:
route = "incremental"
result, err = idx.IncrementalReindex(absPath)
case priorCount > 0 && churn*100 > priorCount*40:
route = "full_retrack"
result, err = fullRetrack()
default:
route = "scoped"
result, err = idx.IncrementalReindexPaths(absPath, append(changed, deleted...))
}
}
if err != nil {
return nil, fmt.Errorf("reconciling %s: %w", absPath, err)
}
result.RepoPrefix = prefix
mi.mu.Lock()
mi.repos[prefix] = &RepoMetadata{
RepoPrefix: prefix,
RootPath: absPath,
Identity: identity,
LastIndexTime: time.Now(),
FileCount: result.FileCount,
NodeCount: result.NodeCount,
EdgeCount: result.EdgeCount,
ParseErrors: result.Errors,
FileMtimes: idx.FileMtimes(),
IsWorktree: ResolveWorktree(absPath).IsWorktree,
Unprefixed: !willBeMultiRepo,
}
mi.indexers[prefix] = idx
mi.mu.Unlock()
entry.Path = absPath
if err := mi.configMgr.Global().AddRepo(entry); err != nil {
mi.logger.Warn("failed to add repo to config", zap.Error(err))
}
// See TrackRepoCtx for why this is skipped under deferGlobalPasses.
if !mi.deferGlobalPasses {
mi.ReconcileContractEdges()
}
mi.logger.Info("daemon: reconciled repo from snapshot",
zap.String("prefix", prefix),
zap.String("route", route),
zap.Int("changed", len(changed)),
zap.Int("deleted", len(deleted)),
zap.Bool("full_retrack", result.FullRetrack),
zap.Int("stale_files_reindexed", result.StaleFileCount),
zap.Duration("elapsed", time.Since(start)))
if len(changed) > 0 || len(deleted) > 0 {
mi.logger.Debug("daemon: reconcile changed-file census",
zap.String("prefix", prefix),
zap.Strings("changed", firstNStrings(changed, 5)),
zap.Strings("deleted", firstNStrings(deleted, 5)))
}
return result, nil
}
// firstNStrings returns at most the first n elements of s — used to cap the
// changed/deleted samples in the reconcile debug log so a large-churn repo
// does not dump thousands of paths into the log line.
func firstNStrings(s []string, n int) []string {
if len(s) <= n {
return s
}
return s[:n]
}
// ReconcileAll runs IncrementalReindex on every tracked repo. Used by
// the daemon's periodic janitor to catch files whose mutations slipped
// past fsnotify — inotify watch limits, NFS / SMB mounts, kernel event
// queue overflow, or daemon downtime where edits happened while nobody
// was listening. Cheap on steady-state repos (one filepath.WalkDir +
// per-file os.Stat per repo), and correctness is self-healing: whatever
// was missed gets picked up on the next tick.
//
// Returns a map of prefix → IndexResult for logging / metrics. Errors
// per repo are logged and do not abort the rest of the sweep — a broken
// permission bit on one repo should not starve reconciliation on the
// others.
func (mi *MultiIndexer) ReconcileAll() map[string]*IndexResult {
mi.mu.RLock()
prefixes := make([]string, 0, len(mi.indexers))
for p := range mi.indexers {
prefixes = append(prefixes, p)
}
mi.mu.RUnlock()
// Same batch trick as warmup: each per-repo IncrementalReindex
// triggers an O(global) InferImplements/InferOverrides walk if we
// don't suppress it. With ~100 repos that's ~100× the work for the
// hourly janitor.
mi.BeginBatch()
// Always restore batch flags on exit (incl. panic) WITHOUT running the
// graph-wide derivation passes — those are run explicitly below, and
// only when a repo actually reindexed. The hourly janitor used to run
// EndBatch unconditionally, walking the full graph (InferImplements /
// InferOverrides / clone detection over hundreds of thousands of
// edges) every cycle even when nothing changed — wasted CPU and, on a
// small resident buffer pool, needless memory churn.
defer mi.ResetBatch()
results := make(map[string]*IndexResult, len(prefixes))
reindexed := 0
changedPrefixes := make(map[string]struct{})
for _, prefix := range prefixes {
mi.mu.RLock()
idx, ok := mi.indexers[prefix]
meta, metaOK := mi.repos[prefix]
mi.mu.RUnlock()
if !ok || !metaOK || meta == nil || meta.RootPath == "" {
continue
}
result, err := idx.IncrementalReindex(meta.RootPath)
if err != nil {
mi.logger.Warn("janitor: reconcile failed",
zap.String("prefix", prefix), zap.Error(err))
continue
}
if result != nil && (result.StaleFileCount > 0 || result.DeletedFileCount > 0) {
mi.logger.Info("janitor: reconciled repo",
zap.String("prefix", prefix),
zap.Int("stale_files_reindexed", result.StaleFileCount))
reindexed++
changedPrefixes[prefix] = struct{}{}
}
results[prefix] = result
// Keep RepoMetadata.FileMtimes in sync so the next snapshot
// picks up the reconciled mtimes.
mi.mu.Lock()
if m, ok := mi.repos[prefix]; ok && m != nil {
m.FileMtimes = idx.FileMtimes()
m.LastIndexTime = time.Now()
}
mi.mu.Unlock()
}
if reindexed > 0 {
mi.ReconcileContractEdges()
// Only now — when at least one repo actually reindexed — is it
// worth the full-graph derivation pass. Nothing changed → skip it
// (the deferred ResetBatch still clears the batch flags). Scope the
// per-repo clone detection + Rebuild to just the repos that changed
// this cycle: an unchanged repo's clone edges are already on disk, so
// re-detecting all of them holds the resolve mutex for tens of seconds
// and stalls every concurrent interactive edit. Warmup arms the same
// scope; without this the janitor re-ran clone detection over every
// tracked repo on every cycle that touched even one file.
mi.ArmBatchScope(changedPrefixes)
mi.RunGlobalGraphPasses(context.Background())
}
return results
}
// UntrackRepo evicts a repo from the graph and removes it from config.
func (mi *MultiIndexer) UntrackRepo(repoPrefix string) (int, int) {
mi.mu.Lock()
meta, ok := mi.repos[repoPrefix]
if !ok {
mi.mu.Unlock()
return 0, 0
}
delete(mi.repos, repoPrefix)
delete(mi.indexers, repoPrefix)
mi.mu.Unlock()
var nodesRemoved, edgesRemoved int
if meta.Unprefixed {
// Single-repo-mode nodes carry RepoPrefix="" and never enter the
// byRepo bucket EvictRepo walks — evict them file-by-file off the
// recorded file set instead, or they linger in the graph and a
// later lone repo would mis-resolve them.
for path := range meta.FileMtimes {
n, e := mi.graph.EvictFile(path)
nodesRemoved += n
edgesRemoved += e
}
} else if purger, ok := mi.graph.(interface{ PurgeRepo(string) error }); ok {
// Prefer the full sidecar-aware purge. EvictRepo drops only
// nodes+edges and leaves fifteen repo_prefix-keyed sidecar tables
// (file_mtimes, *_enrichment, symbol_fts, content_fts, ...) behind,
// which accumulate across untrack/retrack cycles until they dominate
// a long-lived store. PurgeRepo clears them in one transaction. It
// returns no counts, so report the repo's last-index metadata as the
// removed estimate; fall back to EvictRepo (real counts) on error.
if err := purger.PurgeRepo(repoPrefix); err != nil {
mi.logger.Warn("purge repo failed; falling back to node/edge eviction",
zap.String("prefix", repoPrefix), zap.Error(err))
nodesRemoved, edgesRemoved = mi.graph.EvictRepo(repoPrefix)
} else {
nodesRemoved, edgesRemoved = meta.NodeCount, meta.EdgeCount
}
} else {
// Backends without the purge capability (the in-memory store has no
// sidecars, so EvictRepo is already complete there).
nodesRemoved, edgesRemoved = mi.graph.EvictRepo(repoPrefix)
}
// Remove from global config.
if meta.RootPath != "" {
if err := mi.configMgr.Global().RemoveRepo(meta.RootPath); err != nil {
mi.logger.Warn("failed to remove repo from config",
zap.String("prefix", repoPrefix), zap.Error(err))
}
}
mi.ReconcileContractEdges()
return nodesRemoved, edgesRemoved
}
// WorktreeGC is the per-repo outcome of GCVanishedWorktrees.
type WorktreeGC struct {
RepoPrefix string
RootPath string
NodesRemoved int
EdgesRemoved int
}
// GCVanishedWorktrees garbage-collects the index of any tracked linked
// git worktree whose root directory has disappeared from disk — the
// `git worktree remove` (or manual deletion) case. Each vanished
// worktree's branch-keyed snapshot slot and graph nodes would otherwise
// leak forever: a removed worktree never fires a per-file fsnotify
// delete for its whole tree, and the janitor's IncrementalReindex just
// errors out on the missing root without evicting anything.
//
// Only repos recorded as worktrees (RepoMetadata.IsWorktree) are
// eligible — a vanished *main* checkout is left alone, since that is
// far more likely a transient mount problem than an intentional
// removal, and untracking it would also orphan every linked worktree
// that shares its .git. The directory-existence test uses the same
// not-exist-only rule as the per-file deletion detector, so a flaky
// filesystem cannot trigger a destructive eviction.
//
// Returns one WorktreeGC record per repo evicted; an empty slice when
// every tracked worktree is still present.
func (mi *MultiIndexer) GCVanishedWorktrees() []WorktreeGC {
// Snapshot the candidate set under the read lock, then evict
// outside it — UntrackRepo takes the write lock itself.
type candidate struct {
prefix string
root string
}
var candidates []candidate
mi.mu.RLock()
for prefix, meta := range mi.repos {
if meta == nil || !meta.IsWorktree || meta.RootPath == "" {
continue
}
if WorktreeRootGone(meta.RootPath) {
candidates = append(candidates, candidate{prefix: prefix, root: meta.RootPath})
}
}
mi.mu.RUnlock()
if len(candidates) == 0 {
return nil
}
out := make([]WorktreeGC, 0, len(candidates))
for _, c := range candidates {
nodes, edges := mi.UntrackRepo(c.prefix)
mi.logger.Info("janitor: garbage-collected vanished worktree",
zap.String("prefix", c.prefix),
zap.String("root", c.root),
zap.Int("nodes_removed", nodes),
zap.Int("edges_removed", edges))
out = append(out, WorktreeGC{
RepoPrefix: c.prefix,
RootPath: c.root,
NodesRemoved: nodes,
EdgesRemoved: edges,
})
}
return out
}
// GetMetadata returns the metadata for a specific repo, or nil if not found.
func (mi *MultiIndexer) GetMetadata(repoPrefix string) *RepoMetadata {
mi.mu.RLock()
defer mi.mu.RUnlock()
return mi.repos[repoPrefix]
}
// AllMetadata returns a copy of all repo metadata.
func (mi *MultiIndexer) AllMetadata() map[string]*RepoMetadata {
mi.mu.RLock()
defer mi.mu.RUnlock()
out := make(map[string]*RepoMetadata, len(mi.repos))
for k, v := range mi.repos {
out[k] = v
}
return out
}
// IsMultiRepo returns true when more than one repo is tracked.
func (mi *MultiIndexer) IsMultiRepo() bool {
mi.mu.RLock()
defer mi.mu.RUnlock()
return len(mi.repos) > 1
}
// RepoForFile returns the repo prefix for a given file path by checking
// which repo root contains it. Returns empty string if no match.
func (mi *MultiIndexer) RepoForFile(filePath string) string {
absPath, err := filepath.Abs(filePath)
if err != nil {
return ""
}
mi.mu.RLock()
defer mi.mu.RUnlock()
var bestPrefix string
var bestLen int
for prefix, meta := range mi.repos {
// Fold-aware containment so a case-variant file path still maps
// to its repo on a case-insensitive filesystem. Longest-root-wins
// breaks ties by nesting depth; nested roots share a prefix, so
// the raw RootPath length orders them the same as their folded
// forms would.
if pathkey.HasPathPrefix(absPath, meta.RootPath) {
if len(meta.RootPath) > bestLen {
bestLen = len(meta.RootPath)
bestPrefix = prefix
}
}
}
return bestPrefix
}
// GetIndexer returns the Indexer for a specific repo prefix, or nil.
func (mi *MultiIndexer) GetIndexer(repoPrefix string) *Indexer {
mi.mu.RLock()
defer mi.mu.RUnlock()
return mi.indexers[repoPrefix]
}
type grepRepoJob struct {
prefix string
idx *Indexer
}
func (mi *MultiIndexer) grepRepoJobs(repoAllow map[string]bool) []grepRepoJob {
if mi == nil {
return nil
}
mi.mu.RLock()
defer mi.mu.RUnlock()
capHint := len(mi.indexers)
if repoAllow != nil {
capHint = len(repoAllow)
}
jobs := make([]grepRepoJob, 0, capHint)
for prefix, idx := range mi.indexers {
if idx == nil {
continue
}
if repoAllow != nil && !repoAllow[prefix] {
continue
}
jobs = append(jobs, grepRepoJob{prefix: prefix, idx: idx})
}
sort.Slice(jobs, func(i, j int) bool { return jobs[i].prefix < jobs[j].prefix })
return jobs
}
func singleAllowedRepo(repoAllow map[string]bool) (string, bool) {
if repoAllow == nil {
return "", false
}
var only string
count := 0
for prefix, allowed := range repoAllow {
if !allowed {
continue
}
only = prefix
count++
}
return only, count == 1
}
func stampGrepMatchPaths(prefix string, hits []trigram.Match) []trigram.Match {
if prefix == "" {
return hits
}
for i := range hits {
hits[i].Path = prefix + "/" + hits[i].Path
}
return hits
}
func capGrepMatches(matches []trigram.Match, limit int) []trigram.Match {
if limit > 0 && len(matches) > limit {
return matches[:limit]
}
return matches
}
// GrepText fans out a trigram-accelerated literal search across every
// tracked per-repo Indexer and returns the union, capped at limit.
// Match paths are re-stamped from repo-root-relative to repo-prefixed
// (e.g. "internal/foo.go" → "gortex/internal/foo.go") so callers can
// route hits back to the right working tree without consulting the
// MultiIndexer afterwards. A non-positive limit returns every match.
// Returns nil when no per-repo indexer can serve the query — the
// single-Indexer path (Indexer.GrepText) is used by callers without a
// MultiIndexer.
func (mi *MultiIndexer) GrepText(query string, limit int) []trigram.Match {
return capGrepMatches(mi.GrepTextForRepos(query, nil, limit), limit)
}
// GrepTextForRepos is the scoped variant of GrepText. When repoAllow is
// non-nil, only those repo prefixes are searched. perRepoLimit caps each
// searched repo independently; the returned union is intentionally not
// globally capped so callers can apply path / graph-scope filters first.
func (mi *MultiIndexer) GrepTextForRepos(query string, repoAllow map[string]bool, perRepoLimit int) []trigram.Match {
if mi == nil || query == "" {
return nil
}
if prefix, ok := singleAllowedRepo(repoAllow); ok {
idx := mi.GetIndexer(prefix)
if idx == nil {
return nil
}
return stampGrepMatchPaths(prefix, idx.GrepText(query, perRepoLimit))
}
// Per-repo cap mirrors the caller's page size when set. The caller
// applies the final cap after any path / graph-scope filters, so a
// repo outside those filters cannot consume the page first. Zero /
// negative means no per-repo cap (let each searcher return everything).
jobs := mi.grepRepoJobs(repoAllow)
out := make([]trigram.Match, 0, len(jobs)*8)
for _, j := range jobs {
hits := j.idx.GrepText(query, perRepoLimit)
if len(hits) == 0 {
continue
}
// Trigram emits forward-slash repo-relative paths. Stamp the repo
// prefix so downstream tools (resolveGraphPath, path-prefix filters)
// see the same shape they get from the graph nodes.
out = append(out, stampGrepMatchPaths(j.prefix, hits)...)
}
return out
}
// GrepRegexp fans out a trigram-accelerated regex search across every
// tracked per-repo Indexer and returns the union, capped at limit.
// Mirrors GrepText: match paths are re-stamped from repo-root-relative
// to repo-prefixed so downstream tools route hits back to the right
// working tree. pathPrefix, when non-empty, restricts the scan to
// files under that prefix on each indexer. A pattern that does not
// compile in any indexer is reported once; per-indexer errors after
// the first compile are otherwise treated as no-match.
func (mi *MultiIndexer) GrepRegexp(pattern, pathPrefix string, limit int) ([]trigram.Match, error) {
hits, err := mi.GrepRegexpForRepos(pattern, pathPrefix, nil, limit)
if err != nil {
return nil, err
}
return capGrepMatches(hits, limit), nil
}
// GrepRegexpForRepos is the scoped variant of GrepRegexp. repoAllow and
// perRepoLimit have the same semantics as GrepTextForRepos.
func (mi *MultiIndexer) GrepRegexpForRepos(pattern, pathPrefix string, repoAllow map[string]bool, perRepoLimit int) ([]trigram.Match, error) {
if mi == nil || pattern == "" {
return nil, nil
}
if prefix, ok := singleAllowedRepo(repoAllow); ok {
idx := mi.GetIndexer(prefix)
if idx == nil {
return nil, nil
}
hits, err := idx.GrepRegexp(pattern, pathPrefix, perRepoLimit)
if err != nil {
return nil, err
}
return stampGrepMatchPaths(prefix, hits), nil
}
jobs := mi.grepRepoJobs(repoAllow)
out := make([]trigram.Match, 0, len(jobs)*8)
for _, j := range jobs {
hits, err := j.idx.GrepRegexp(pattern, pathPrefix, perRepoLimit)
if err != nil {
// First compile error short-circuits — the pattern is the
// caller's fault and won't compile in any other indexer
// either (the trigram searcher uses the same regexp engine).
return nil, err
}
if len(hits) == 0 {
continue
}
out = append(out, stampGrepMatchPaths(j.prefix, hits)...)
}
return out, nil
}
// IndexerForFile routes an absolute path to the per-repo Indexer that
// owns it. Returns (nil, "") when no tracked repo contains the path.
// Used by the MCP overlay middleware to find the right Indexer for a
// pushed file when constructing the per-request overlay layer.
func (mi *MultiIndexer) IndexerForFile(absPath string) (*Indexer, string) {
prefix := mi.RepoForFile(absPath)
if prefix == "" {
return nil, ""
}
return mi.GetIndexer(prefix), prefix
}
// ResolveFilePath takes a repo-prefixed relative path (e.g. "ade/internal/foo.go")
// and returns the absolute filesystem path by looking up the repo's root directory.
// Returns empty string if the repo prefix is not found.
func (mi *MultiIndexer) ResolveFilePath(prefixedPath string) string {
mi.mu.RLock()
defer mi.mu.RUnlock()
// Longest matching prefix wins. With worktree instances two prefixes
// can share a leading segment (`oas-orm` vs `oas-orm@task-ws`); map
// iteration order is random, so a plain first-match could resolve a
// `oas-orm@task-ws/...` path against the shorter `oas-orm` root. The
// "/"-boundary check already keeps the two disjoint, but ranking by
// length makes that robust regardless of any future prefix shapes.
var bestPrefix, bestRoot string
for prefix, meta := range mi.repos {
if meta == nil {
continue
}
if strings.HasPrefix(prefixedPath, prefix+"/") && len(prefix) > len(bestPrefix) {
bestPrefix, bestRoot = prefix, meta.RootPath
}
}
if bestPrefix == "" {
// Single-repo mode mints unprefixed graph paths; resolve them
// against the lone registered repo instead of failing.
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
return filepath.Join(meta.RootPath, prefixedPath)
}
return ""
}
// Collision guard for the lone unprefixed repo: its graph paths are
// raw relative paths, so one whose first segment happens to equal
// the repo's own prefix (repo "api" containing api/handlers.go)
// would be hijacked by the prefix-strip join. Prefer the raw join
// when that file actually exists on disk.
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
raw := filepath.Join(meta.RootPath, prefixedPath)
if _, err := os.Stat(raw); err == nil {
return raw
}
}
return filepath.Join(bestRoot, strings.TrimPrefix(prefixedPath, bestPrefix+"/"))
}
// RepoPrefixes returns the set of registered repo prefixes. The returned
// slice is a snapshot — safe to retain and iterate concurrently with
// other multi-indexer operations. Order is unspecified; callers that
// need stability should sort.
func (mi *MultiIndexer) RepoPrefixes() []string {
mi.mu.RLock()
defer mi.mu.RUnlock()
prefixes := make([]string, 0, len(mi.repos))
for prefix := range mi.repos {
prefixes = append(prefixes, prefix)
}
return prefixes
}
// RepoRoot returns the local filesystem root for the given repo prefix.
// ok is true only when the prefix is registered AND meta.RootPath is non-empty.
// Caller is responsible for joining repo-relative file paths against the root.
//
// The empty prefix resolves to the lone registered repo when exactly one is
// tracked: single-repo mode indexes nodes without a repo prefix (see
// indexSingleRepo) while registering its metadata under the repo's real
// prefix, so every node the single-repo indexer mints carries RepoPrefix=""
// — refusing the empty prefix would orphan all of them. With two or more
// repos the empty prefix is ambiguous and stays a miss.
func (mi *MultiIndexer) RepoRoot(repoPrefix string) (string, bool) {
mi.mu.RLock()
defer mi.mu.RUnlock()
if repoPrefix == "" {
if meta := mi.loneRepoLocked(); meta != nil && meta.RootPath != "" {
return meta.RootPath, true
}
return "", false
}
meta, ok := mi.repos[repoPrefix]
if !ok || meta == nil || meta.RootPath == "" {
return "", false
}
return meta.RootPath, true
}
// loneRepoLocked returns the metadata of the only registered repo when
// exactly one repo is tracked AND that repo was indexed unprefixed
// (single-repo mode). The provenance check matters: after a 1→2→1
// track/untrack sequence the lone survivor can be a prefixed repo, and
// stale unprefixed nodes from the departed repo must keep failing
// closed instead of resolving against the wrong checkout. Caller must
// hold mi.mu.
func (mi *MultiIndexer) loneRepoLocked() *RepoMetadata {
if len(mi.repos) != 1 {
return nil
}
for _, meta := range mi.repos {
if meta != nil && meta.Unprefixed {
return meta
}
}
return nil
}
// LinkedWorktreeRoots returns the on-disk roots of every tracked linked
// git worktree that shares its .git common directory with the checkout
// at mainRepoPath — i.e. the worktree siblings of that main repo. The
// query is keyed on the resolved MainRepoPath so it matches whether the
// caller passes a main checkout or one of its worktrees.
//
// Used by the edit-tool file resolver: because all worktrees of one
// repo reuse a single index identity, a repo-relative path can resolve
// against a sibling checkout. When the resolved file belongs to a
// linked worktree, the resolver re-roots the write there so an edit
// lands in the worktree's copy rather than the main checkout's.
func (mi *MultiIndexer) LinkedWorktreeRoots(mainRepoPath string) []string {
if mainRepoPath == "" {
return nil
}
wantMain := resolvedMainRepo(mainRepoPath)
mi.mu.RLock()
defer mi.mu.RUnlock()
var out []string
for _, meta := range mi.repos {
if meta == nil || meta.RootPath == "" || !meta.IsWorktree {
continue
}
if resolvedMainRepo(meta.RootPath) == wantMain {
out = append(out, meta.RootPath)
}
}
return out
}
// resolvedMainRepo resolves a checkout path to its repo's main worktree
// root with symlinks evaluated. ResolveWorktree derives the main path
// two different ways — filepath.Abs for a main checkout vs git's
// canonicalized `commondir` for a linked worktree — and on platforms
// where the temp / home tree is a symlink (macOS /var -> /private/var)
// those two forms differ for the same repo. Evaluating symlinks on the
// result gives one stable identity that both inputs agree on.
func resolvedMainRepo(path string) string {
main := ResolveWorktree(path).MainRepoPath
if resolved, err := filepath.EvalSymlinks(main); err == nil {
return resolved
}
return main
}
// MergedContractRegistry combines contract registries from all per-repo
// indexers into a single registry. In multi-repo mode each repo's indexer
// runs extractContracts independently; this merges the results.
func (mi *MultiIndexer) MergedContractRegistry() *contracts.Registry {
mi.mu.RLock()
defer mi.mu.RUnlock()
merged := contracts.NewRegistry()
for repoPrefix, idx := range mi.indexers {
cr := idx.ContractRegistry()
if cr == nil {
continue
}
// Re-stamp the workspace/project slugs from the indexer
// alongside the repo prefix on merge. The contracts already
// carry these slugs from
// their source registry, but AddAllScoped is idempotent (skips
// non-empty existing values) so this stays correct even if a
// future code path forgets the stamp on first insert.
merged.AddAllScoped(cr.All(), repoPrefix, idx.WorkspaceID(), idx.ProjectID())
}
return merged
}
// attachInlinedShapes folds the field shape of each contract's
// response_type / request_type into the contract's Meta so the
// dashboard can render the expanded field list. Targets contracts
// where the type has been resolved to a graph node ID (contains
// "::") AND the type node has a shape stored in its Meta.
//
// Type-shape extraction normally runs in commitContracts via
// snapshotContractShapes + inlineEnvelopeShapes — but those passes
// run during the initial extract and miss contracts added later by
// InlineWrappers. This is the post-inline equivalent: it doesn't
// re-extract shapes (the type nodes already have them from
// snapshotContractShapes if they were referenced anywhere), it just
// attaches them to the new contract entries.
func (mi *MultiIndexer) attachInlinedShapes(cr *contracts.Registry, g graph.Store) {
idsToTouch := map[string]bool{}
for _, c := range cr.All() {
if c.Meta == nil {
continue
}
for _, key := range []string{"response_type", "request_type"} {
if v, _ := c.Meta[key].(string); v != "" && strings.Contains(v, "::") {
idsToTouch[c.ID] = true
break
}
}
if env, ok := c.Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
// Touch any contract that has an envelope, even when
// the rows still carry bare type names — the loop below
// upgrades them. Otherwise we skip them and lose the
// shape attachment for sibling-file types.
idsToTouch[c.ID] = true
_ = env
}
}
srcCache := map[string][]byte{}
resolveShape := func(typeID string) any {
if typeID == "" || !strings.Contains(typeID, "::") {
return nil
}
node := g.GetNode(typeID)
if node == nil {
return nil
}
if node.Kind != graph.KindType && node.Kind != graph.KindInterface {
return nil
}
if node.Meta == nil {
node.Meta = map[string]any{}
}
if shape, ok := node.Meta["shape"]; ok && shape != nil {
return shape
}
// Lazy-extract: snapshotContractShapes only walks types
// referenced by the initial bulk extract. Types referenced
// ONLY by wrapper-inlined contracts need this fallback or
// their fields stay unread.
src := srcCache[node.FilePath]
if src == nil {
data, ok := mi.readNodeSource(node)
if !ok {
srcCache[node.FilePath] = []byte{}
return nil
}
src = data
srcCache[node.FilePath] = src
}
if len(src) == 0 {
return nil
}
extracted := contracts.ExtractShape(node.FilePath, src, node.StartLine, node.EndLine)
if extracted == nil {
return nil
}
node.Meta["shape"] = extracted
return extracted
}
for id := range idsToTouch {
items := cr.ByID(id)
changed := false
for i := range items {
if items[i].Meta == nil {
continue
}
// Top-level request/response type shapes.
for _, pair := range []struct{ typeKey, shapeKey string }{
{"response_type", "response_shape"},
{"request_type", "request_shape"},
} {
if _, has := items[i].Meta[pair.shapeKey]; has {
continue
}
typeID, _ := items[i].Meta[pair.typeKey].(string)
if shape := resolveShape(typeID); shape != nil {
items[i].Meta[pair.shapeKey] = shape
changed = true
}
}
// Envelope rows — upgrade bare type names to graph IDs
// (so the shape lookup hits) and attach shapes.
if env, ok := items[i].Meta["response_envelope"].([]map[string]any); ok && len(env) > 0 {
envChanged := false
for ri, row := range env {
typeID, _ := row["type"].(string)
// Upgrade bare type name → graph ID when the
// in-file resolveTypeInFile pass left it bare
// (the type lives in a sibling file).
if typeID != "" && !strings.Contains(typeID, "::") {
matches := g.FindNodesByName(typeID)
var resolved string
for _, n := range matches {
if n.Kind != graph.KindType && n.Kind != graph.KindInterface {
continue
}
resolved = n.ID
if items[i].RepoPrefix != "" && strings.HasPrefix(n.ID, items[i].RepoPrefix+"/") {
break // prefer same-repo
}
}
if resolved != "" {
env[ri]["type"] = resolved
typeID = resolved
envChanged = true
}
}
if _, has := row["shape"]; has {
continue
}
if shape := resolveShape(typeID); shape != nil {
env[ri]["shape"] = shape
envChanged = true
}
}
if envChanged {
items[i].Meta["response_envelope"] = env
changed = true
}
}
}
if changed {
cr.ReplaceByID(id, items)
}
}
}
// readNodeSource returns the source bytes of the file the node lives
// in, resolving the repo prefix to a real disk path via tracked-repo
// metadata. Mirrors wrapperSourceReader's path-resolution dance.
func (mi *MultiIndexer) readNodeSource(node *graph.Node) ([]byte, bool) {
if node == nil || node.FilePath == "" {
return nil, false
}
rel := node.FilePath
if node.RepoPrefix != "" {
meta := mi.GetMetadata(node.RepoPrefix)
if meta == nil || meta.RootPath == "" {
return nil, false
}
rel = strings.TrimPrefix(rel, node.RepoPrefix+"/")
data, err := os.ReadFile(filepath.Join(meta.RootPath, rel))
if err != nil {
return nil, false
}
return data, true
}
for _, m := range mi.AllMetadata() {
data, err := os.ReadFile(filepath.Join(m.RootPath, rel))
if err == nil {
return data, true
}
}
return nil, false
}
// wrapperSourceReader returns a SourceReader closure that maps a graph
// node back to its on-disk bytes by joining the node's repo-relative
// FilePath with the repo's RootPath from MultiIndexer metadata. In
// single-repo mode (no RepoPrefix on nodes) the indexer's sole root is
// used. Read results are memoized inside the closure so multi-caller
// wrappers don't trigger N disk reads per file.
func (mi *MultiIndexer) wrapperSourceReader() contracts.SourceReader {
cache := make(map[string][]byte)
miss := make(map[string]struct{})
readFile := func(absPath string) ([]byte, bool) {
if b, ok := cache[absPath]; ok {
return b, true
}
if _, skipped := miss[absPath]; skipped {
return nil, false
}
b, err := os.ReadFile(absPath)
if err != nil {
miss[absPath] = struct{}{}
return nil, false
}
cache[absPath] = b
return b, true
}
return func(n *graph.Node) ([]byte, bool) {
if n == nil || n.FilePath == "" {
return nil, false
}
// Multi-repo case: strip "<repo-prefix>/" from the FilePath and
// join with the repo's recorded root.
if n.RepoPrefix != "" {
meta := mi.GetMetadata(n.RepoPrefix)
if meta == nil || meta.RootPath == "" {
return nil, false
}
rel := strings.TrimPrefix(n.FilePath, n.RepoPrefix+"/")
return readFile(filepath.Join(meta.RootPath, rel))
}
// Single-repo fallback: a node without a RepoPrefix carries its
// path relative to the sole indexer root. Try each known repo
// root since the single-repo path wraps through indexSingleRepo.
for _, meta := range mi.AllMetadata() {
if b, ok := readFile(filepath.Join(meta.RootPath, n.FilePath)); ok {
return b, true
}
}
// Last resort: treat FilePath as already absolute (tests).
return readFile(n.FilePath)
}
}
// ReconcileContractEdges walks the merged contract registry, runs the
// consumer↔provider matcher, and writes the results into the graph as
// EdgeMatches edges pointing from consumer-contract nodes to their matched
// provider-contract nodes. Every call evicts the prior set of EdgeMatches
// first so the edges stay in sync with the current contract view; that's
// correct after track / untrack / re-index / watcher re-scan. Returns the
// number of match edges added so callers can log or test the effect.
//
// This is what makes get_call_chain traverse service boundaries: without a
// persisted contract→contract edge, the matcher's result is only visible
// via the `contracts check` tool and traversals stop at each service's
// boundary.
func (mi *MultiIndexer) ReconcileContractEdges() int {
// Serialise the whole pass: the evict-then-mint of EdgeMatches, topic
// edges, and the bridge subgraph spans many non-atomic store writes,
// and several goroutines call this concurrently (see reconcileMu).
mi.reconcileMu.Lock()
defer mi.reconcileMu.Unlock()
g := mi.Graph()
if g == nil {
return 0
}
// Evict any prior EdgeMatches so the graph reflects only the current
// registry. Collect first, remove second — we're mutating the same
// out-edge lists we'd be iterating otherwise.
type edgeKey struct{ from, to string }
var stale []edgeKey
var staleTopicProduces, staleTopicConsumes []edgeKey
// Collect only the three reconciled edge kinds via the edges_by_kind
// index, rather than scanning (and meta-decoding) the whole edge set.
for e := range g.EdgesByKind(graph.EdgeMatches) {
stale = append(stale, edgeKey{e.From, e.To})
}
for e := range g.EdgesByKind(graph.EdgeProducesTopic) {
staleTopicProduces = append(staleTopicProduces, edgeKey{e.From, e.To})
}
for e := range g.EdgesByKind(graph.EdgeConsumesTopic) {
staleTopicConsumes = append(staleTopicConsumes, edgeKey{e.From, e.To})
}
for _, k := range stale {
g.RemoveEdge(k.from, k.to, graph.EdgeMatches)
}
// Evict prior topic edges so a renamed topic or removed callsite
// doesn't keep its edges alive. The KindTopic node itself is
// addressable by ID; when no consumer or producer remains for a
// given topic node we orphan-collect it after the new edges are
// emitted (see end of this function).
for _, k := range staleTopicProduces {
g.RemoveEdge(k.from, k.to, graph.EdgeProducesTopic)
}
for _, k := range staleTopicConsumes {
g.RemoveEdge(k.from, k.to, graph.EdgeConsumesTopic)
}
merged := mi.MergedContractRegistry()
if merged == nil {
return 0
}
// Inline HTTP wrapper callers (T2.4). Codebases that route every
// endpoint through a helper like `request(path, ...)` produce one
// parametric consumer contract per wrapper at extraction time —
// useless for matching. InlineWrappers walks incoming call edges
// of each wrapper, re-reads the caller's source, and emits a
// specific consumer contract per literal path. The returned
// contracts are also persisted into their owning repo's per-repo
// registry so subsequent `contracts list`/`check` calls see them
// (MergedContractRegistry rebuilds fresh each call and would
// otherwise lose them).
inlined := contracts.InlineWrappers(merged, g, mi.wrapperSourceReader())
if len(inlined) > 0 {
mi.mu.RLock()
for _, c := range inlined {
if c.RepoPrefix == "" {
continue
}
idx, ok := mi.indexers[c.RepoPrefix]
if !ok {
continue
}
cr := idx.ContractRegistry()
if cr == nil {
continue
}
// Skip if the same contract is already persisted —
// ReconcileContractEdges runs on every repo change, and
// appending the same inlined contract on every pass would
// blow up the registry with duplicates. Compare on the
// Registry.All() dedupe key.
alreadyPersisted := false
for _, existing := range cr.ByID(c.ID) {
if existing.SymbolID == c.SymbolID &&
existing.FilePath == c.FilePath &&
existing.Role == c.Role {
alreadyPersisted = true
break
}
}
if !alreadyPersisted {
cr.Add(c)
}
}
mi.mu.RUnlock()
// Wrapper-inlined contracts arrive AFTER commitContracts ran
// its UpgradeBareTypeRefs pass, so their response_type /
// request_type still carries bare names like "ToolInfo".
// Re-run the upgrade against the merged graph so downstream
// snapshotContractShapes finds the type node and the
// dashboard sees fields instead of a string.
mi.mu.RLock()
lookup := func(name, repoHint string) []string {
matches := mi.graph.FindNodesByName(name)
if len(matches) == 0 {
return nil
}
ids := make([]string, 0, len(matches))
for _, n := range matches {
if n.Kind != graph.KindType && n.Kind != graph.KindInterface {
continue
}
ids = append(ids, n.ID)
}
// Prefer same-repo when multiple match.
if len(ids) > 1 && repoHint != "" {
var sameRepo []string
for _, id := range ids {
if strings.HasPrefix(id, repoHint+"/") {
sameRepo = append(sameRepo, id)
}
}
if len(sameRepo) > 0 {
return sameRepo
}
}
return ids
}
for _, idx := range mi.indexers {
cr := idx.ContractRegistry()
if cr != nil {
cr.UpgradeBareTypeRefs(lookup)
}
}
// UpgradeBareTypeRefs leaves names with ≥2 candidates alone
// (e.g. a TS app declaring `DashboardSnapshot` in both
// `lib/schema.ts` and `lib/types.ts`). disambiguateBareTypesViaImports
// re-reads the consumer's source, parses its `import` lines,
// and picks the candidate whose graph FilePath matches an
// imported module. Runs before attachInlinedShapes so the
// shape attachment sees fully-qualified IDs.
for _, idx := range mi.indexers {
cr := idx.ContractRegistry()
if cr != nil {
mi.disambiguateBareTypesViaImports(cr, mi.graph)
}
}
// Now that response_type / request_type point at real graph
// nodes, fold each referenced type's shape (struct fields)
// into the contract's Meta so the dashboard renders the
// expanded field list instead of just the type name. Mirrors
// what snapshotContractShapes + inlineEnvelopeShapes do for
// initially-extracted contracts.
for _, idx := range mi.indexers {
cr := idx.ContractRegistry()
if cr == nil {
continue
}
mi.attachInlinedShapes(cr, mi.graph)
}
mi.mu.RUnlock()
}
// Bind provider-contract SymbolIDs that came from spec files
// (.proto for gRPC, OpenAPI YAML/JSON for HTTP). Without this
// step the matcher finds pairs but the bridge-emission check
// below skips them because provider SymbolID is empty. Must run
// before Match so Match sees the updated records.
contracts.BindProviderSymbols(merged, g)
result := contracts.Match(merged)
added := 0
// Track which topic nodes we've already materialised so the loop
// emits one node per (broker, topic) bucket even when a topic has
// fan-out across many consumers. The dedupe key is the topic
// node's ID — its repo-prefix is already encoded in Contract.ID.
topicNodes := make(map[string]struct{})
for _, m := range result.Matched {
// Connect the consumer's enclosing symbol directly to the
// provider's enclosing symbol. Contract nodes in the graph are
// deduped by Contract.ID, so a provider and a consumer that share
// the same ID collapse into one node — a contract→contract edge
// would be a self-loop. Symbol→symbol bypasses that and gives
// get_call_chain the traversal it needs: saveTuck (extension) →
// Handler.CreateTuck (core-api).
if m.Provider.SymbolID == "" || m.Consumer.SymbolID == "" {
continue
}
if m.Provider.SymbolID == m.Consumer.SymbolID {
continue
}
g.AddEdge(&graph.Edge{
From: m.Consumer.SymbolID,
To: m.Provider.SymbolID,
Kind: graph.EdgeMatches,
FilePath: m.Consumer.FilePath,
Line: m.Consumer.Line,
Confidence: 1.0,
ConfidenceLabel: "EXTRACTED",
Origin: graph.OriginASTResolved,
CrossRepo: m.CrossRepo,
})
added++
// Materialise the broker topic node + producer/consumer
// edges. Only fires for ContractTopic matches; HTTP / gRPC /
// WS / GraphQL / env contracts fall through with just the
// EdgeMatches edge above.
if m.Provider.Type == contracts.ContractTopic {
emitTopicEdges(g, m, topicNodes)
}
}
// Persist the matched contract groups as the bridge subgraph: one
// KindContractBridge node per group plus EdgeBridges fan-out to
// the participating contract nodes. The pass evicts the previous
// bridge generation internally, so it stays idempotent across
// reconciles and drops bridges whose contracts disappeared.
MaterializeContractBridges(g, result.Matched)
// Topic nodes whose producer and consumer edges all evaporated
// since the previous reconcile remain in the graph as leaf
// nodes — Graph has no public RemoveNode and the next reconcile
// upserts them anyway. A topic that lost every callsite is an
// invisible-but-harmless cost (a single KindTopic node with no
// neighbors). Worth revisiting if topic churn ever bloats the
// graph; in the meantime we lean on EvictRepo / EvictFile to
// reclaim memory when a whole repo's topic vocabulary changes.
return added
}
// emitTopicEdges materialises the KindTopic node and the
// EdgeProducesTopic / EdgeConsumesTopic edges that pair a matched
// producer/consumer pair across the workspace. The topic ID is
// reconstructed from the Contract.ID (already `topic::<broker>::
// <name>`) so the node ID matches the contract ID 1:1 — agents that
// have the contract ID can also look up the topic node directly.
// Meta on the node carries the broker family and the raw topic name
// for filterless queries.
func emitTopicEdges(g graph.Store, m contracts.CrossLink, topicNodes map[string]struct{}) {
// Trust the matcher to bucket only same-broker contracts together
// because Contract.ID already includes the broker token; if the
// broker isn't on the provider Meta, fall through to the contract
// ID's middle segment so the node carries something useful.
broker, _ := m.Provider.Meta["broker"].(string)
topicName, _ := m.Provider.Meta["topic"].(string)
if broker == "" || topicName == "" {
// Defensive fallback — extract from the Contract.ID shape
// `topic::<broker>::<name>`. If the ID isn't structured we
// skip rather than emit a node with an unidentifiable broker
// (such an edge would be misleading in cross-broker queries).
broker2, name2, ok := parseTopicContractID(m.Provider.ID)
if !ok {
return
}
if broker == "" {
broker = broker2
}
if topicName == "" {
topicName = name2
}
}
topicID := m.Provider.ID // canonical: topic::<broker>::<name>
if _, ok := topicNodes[topicID]; !ok {
topicNodes[topicID] = struct{}{}
// Preserve any existing node (a prior reconcile may have
// created it) but always refresh Meta so a broker rename
// isn't sticky across reconciles. AddNode in this codebase
// is upsert-style — see graph.Graph.AddNode.
g.AddNode(&graph.Node{
ID: topicID,
Kind: graph.KindTopic,
Name: topicName,
FilePath: m.Provider.FilePath,
Language: "contract",
RepoPrefix: m.Provider.RepoPrefix,
WorkspaceID: m.Provider.EffectiveWorkspace(),
ProjectID: m.Provider.EffectiveProject(),
Meta: map[string]any{
"broker": broker,
"name": topicName,
},
})
}
g.AddEdge(&graph.Edge{
From: m.Provider.SymbolID,
To: topicID,
Kind: graph.EdgeProducesTopic,
FilePath: m.Provider.FilePath,
Line: m.Provider.Line,
Confidence: 1.0,
ConfidenceLabel: "EXTRACTED",
Origin: graph.OriginASTResolved,
CrossRepo: false,
Meta: map[string]any{
"broker": broker,
},
})
g.AddEdge(&graph.Edge{
From: m.Consumer.SymbolID,
To: topicID,
Kind: graph.EdgeConsumesTopic,
FilePath: m.Consumer.FilePath,
Line: m.Consumer.Line,
Confidence: 1.0,
ConfidenceLabel: "EXTRACTED",
Origin: graph.OriginASTResolved,
CrossRepo: m.CrossRepo,
Meta: map[string]any{
"broker": broker,
},
})
}
// parseTopicContractID splits a Contract.ID of the form
// `topic::<broker>::<name>` (or its repo-prefixed counterpart
// `<repo>/topic::<broker>::<name>`) into broker + name. Returns
// ok==false for any other shape so callers can skip ill-formed
// topic contracts rather than fabricate a broker label.
func parseTopicContractID(id string) (broker, name string, ok bool) {
// Strip any leading repo-prefix segment ("repo/topic::..."). The
// applyRepoPrefix step prepends `<repo>/` to synthetic IDs of
// the form `topic::...`, so we look for the inner `topic::`
// marker rather than splitting on the leading slash.
idx := strings.Index(id, "topic::")
if idx < 0 {
return "", "", false
}
rest := id[idx+len("topic::"):]
sep := strings.Index(rest, "::")
if sep <= 0 || sep == len(rest)-2 {
return "", "", false
}
return rest[:sep], rest[sep+2:], true
}
// Graph returns the underlying shared graph.
func (mi *MultiIndexer) Graph() graph.Store {
return mi.graph
}
// SetRemoteStitch enables cross-daemon proxy-edge minting on every
// CrossRepoResolver this MultiIndexer constructs. Called once by the
// daemon entry point when federation.edges.enabled; a nil prober leaves
// the resolvers on read-only federation.
func (mi *MultiIndexer) SetRemoteStitch(prober resolver.RemoteDeclarationProber, budget int) {
mi.mu.Lock()
defer mi.mu.Unlock()
mi.stitchProber = prober
mi.proxyBudget = budget
}
// applyRemoteStitch wires the proxy-edge mint into a freshly built resolver
// when a prober is installed.
func (mi *MultiIndexer) applyRemoteStitch(cr *resolver.CrossRepoResolver) {
mi.mu.RLock()
prober, budget := mi.stitchProber, mi.proxyBudget
mi.mu.RUnlock()
if prober != nil {
cr.EnableRemoteStitch(prober, budget)
}
}
// Search returns the shared search backend.
func (mi *MultiIndexer) Search() search.Backend {
return mi.search
}
// ExportVectorIndex serializes the workspace-global semantic-search
// vector index — there is one shared HNSW index across every tracked
// repo, not one per repo. Returns nil, 0, 0 when no vector index is
// active (embeddings disabled, or the backend is still text-only).
// Used by the daemon snapshot path so a default-on daemon does not
// re-embed the whole graph on every restart.
func (mi *MultiIndexer) ExportVectorIndex() ([]byte, int, int) {
sw, ok := mi.search.(*search.Swappable)
if !ok {
return nil, 0, 0
}
hybrid, ok := sw.Inner().(*search.HybridBackend)
if !ok {
return nil, 0, 0
}
vec := hybrid.VectorIndex()
if vec == nil || vec.Count() == 0 {
return nil, 0, 0
}
var buf bytes.Buffer
if err := vec.Save(&buf); err != nil {
mi.logger.Warn("failed to export vector index", zap.Error(err))
return nil, 0, 0
}
return buf.Bytes(), vec.Dims(), vec.Count()
}
// ImportVectorIndex restores a previously-exported vector index into
// the shared search backend, wrapping the current text backend in a
// HybridBackend. It is a no-op when embeddings are disabled (no
// configured embedder) or when the cached index's dimensionality does
// not match the active embedder — a provider switch (GloVe 50d → ONNX
// 384d) makes the cached vectors meaningless, so the indexer re-embeds
// instead. Returns an error only on a structurally corrupt index blob.
func (mi *MultiIndexer) ImportVectorIndex(data []byte, dims, count int) error {
if len(data) == 0 || mi.embedder == nil {
return nil
}
if embedderDims := mi.embedder.Dimensions(); embedderDims > 0 && embedderDims != dims {
mi.logger.Info("vector index dims mismatch, will re-embed",
zap.Int("cached_dims", dims), zap.Int("embedder_dims", embedderDims))
return nil
}
sw, ok := mi.search.(*search.Swappable)
if !ok {
return nil
}
vec := search.NewVector(dims)
if err := vec.LoadFrom(bytes.NewReader(data)); err != nil {
return fmt.Errorf("import vector index: %w", err)
}
vec.SetCount(count)
// Unwrap an existing HybridBackend to its text side before
// re-wrapping so we never nest Hybrids (each retains a stale
// vector index — see buildSearchIndex for the memory rationale).
inner := sw.Inner()
if hyb, ok := inner.(*search.HybridBackend); ok {
inner = hyb.TextBackend()
}
sw.Swap(search.NewHybrid(inner, vec, mi.embedder))
mi.logger.Info("restored vector index from snapshot",
zap.Int("vectors", count), zap.Int("dims", dims))
return nil
}
// AutoDetectRepos walks immediate subdirectories of parentPath looking for
// .git directories. If parentPath itself is a Git repo, it returns a single
// entry (the caller should index it as single-repo). If zero Git repos are
// found, it returns nil so the caller can fall back to single-repo mode.
// This is gated by the workspace.auto_detect config flag.
func (mi *MultiIndexer) AutoDetectRepos(parentPath string) []config.RepoEntry {
absPath, err := filepath.Abs(parentPath)
if err != nil {
mi.logger.Warn("auto-detect: failed to resolve path", zap.String("path", parentPath), zap.Error(err))
return nil
}
// If the path itself is a Git repo, return it as a single repo.
if isGitRepo(absPath) {
return []config.RepoEntry{{
Path: absPath,
Name: filepath.Base(absPath),
}}
}
// Walk immediate subdirectories (not recursive) for .git dirs.
entries, err := os.ReadDir(absPath)
if err != nil {
mi.logger.Warn("auto-detect: failed to read directory", zap.String("path", absPath), zap.Error(err))
return nil
}
var repos []config.RepoEntry
for _, entry := range entries {
if !entry.IsDir() {
continue
}
subDir := filepath.Join(absPath, entry.Name())
if isGitRepo(subDir) {
repos = append(repos, config.RepoEntry{
Path: subDir,
Name: entry.Name(), // Derive RepoPrefix from subdirectory name.
})
}
}
// If zero Git repos found, return nil — caller falls back to single-repo.
if len(repos) == 0 {
return nil
}
return repos
}
// isGitRepo checks whether the given directory contains a .git subdirectory.
func isGitRepo(dir string) bool {
info, err := os.Stat(filepath.Join(dir, ".git"))
if err != nil {
return false
}
return info.IsDir()
}